From 73266df0f5e2508fb7276b99069a0e858ab8c45e Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 14 Jan 2025 00:01:21 +0100 Subject: [PATCH 01/21] feat(metrics): add cache, process and http metrics --- modules/cache/cache.go | 22 ++++++++++++++++++++++ modules/cache/string_cache.go | 6 ++++++ routers/common/middleware.go | 28 ++++++++++++++++++++++++++++ routers/web/web.go | 2 ++ 4 files changed, 58 insertions(+) diff --git a/modules/cache/cache.go b/modules/cache/cache.go index f7828e3cae..49bb3bff9f 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -9,11 +9,33 @@ import ( "time" "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" _ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache" ) var defaultCache StringCache +var hitCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache hit count", + Subsystem: "cache", + Name: "hit", +}) +var missCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache miss count", + Subsystem: "cache", + Name: "miss", +}) +var latencyHistogram = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "gitea", + Help: "Cache latency", + Subsystem: "cache", + Name: "duration", + }, +) // Init start cache service func Init() error { diff --git a/modules/cache/string_cache.go b/modules/cache/string_cache.go index 4f659616f5..328cb6d213 100644 --- a/modules/cache/string_cache.go +++ b/modules/cache/string_cache.go @@ -6,6 +6,7 @@ package cache import ( "errors" "strings" + "time" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" @@ -63,10 +64,15 @@ func (sc *stringCache) Ping() error { } func (sc *stringCache) Get(key string) (string, bool) { + start := time.Now() v := sc.chiCache.Get(key) + elapsed := time.Since(start).Seconds() + latencyHistogram.Observe(elapsed) if v == nil { + missCounter.Add(1) return "", false } + hitCounter.Add(1) s, ok := v.(string) return s, ok } diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 2ba02de8ed..739c92918a 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "strings" + "time" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/gtprof" @@ -19,8 +20,17 @@ import ( "gitea.com/go-chi/session" "github.com/chi-middleware/proxy" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) +var responseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "request", + Name: "duration_seconds", + Help: "Gitea response time", +}, []string{"route"}) + // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery func ProtocolMiddlewares() (handlers []any) { // the order is important @@ -38,6 +48,9 @@ func ProtocolMiddlewares() (handlers []any) { if setting.IsAccessLogEnabled() { handlers = append(handlers, context.AccessLogger()) } + if setting.Metrics.Enabled { + handlers = append(handlers, RouteMetrics()) + } return handlers } @@ -107,6 +120,21 @@ func ForwardedHeadersHandler(limit int, trustedProxies []string) func(h http.Han return proxy.ForwardedHeaders(opt) } +func RouteMetrics() func(h http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + start := time.Now() + next.ServeHTTP(resp, req) + elapsed := time.Since(start).Seconds() + route := "ui" + if strings.HasPrefix(req.URL.Path, "/api") { + route = "api" + } + responseLatency.WithLabelValues(route).Observe(elapsed) + }) + } +} + func Sessioner() func(next http.Handler) http.Handler { return session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, diff --git a/routers/web/web.go b/routers/web/web.go index ef26dd9a74..02167b60d7 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -52,6 +52,7 @@ import ( "github.com/go-chi/cors" "github.com/klauspost/compress/gzhttp" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" ) var GzipMinSize = 1400 // min size to compress for the body size of response @@ -258,6 +259,7 @@ func Routes() *web.Router { } if setting.Metrics.Enabled { + prometheus.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{Namespace: "gitea"})) prometheus.MustRegister(metrics.NewCollector()) routes.Get("/metrics", append(mid, Metrics)...) } From 1dad933ec580068a3b004ce82a0acc6a49ed6d66 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 14 Jan 2025 00:12:10 +0100 Subject: [PATCH 02/21] refactor(metrics)!: deprecate issues open/closed for labels --- modules/metrics/collector.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 230260ff94..6d42803a3f 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -89,7 +89,7 @@ func NewCollector() Collector { Issues: prometheus.NewDesc( namespace+"issues", "Number of Issues", - nil, nil, + []string{"state"}, nil, ), IssuesByLabel: prometheus.NewDesc( namespace+"issues_by_label", @@ -103,12 +103,12 @@ func NewCollector() Collector { ), IssuesOpen: prometheus.NewDesc( namespace+"issues_open", - "Number of open Issues", + "DEPRECATED: Use Issues with state: open", nil, nil, ), IssuesClosed: prometheus.NewDesc( namespace+"issues_closed", - "Number of closed Issues", + "DEPRECATED: Use Issues with state: closed", nil, nil, ), Labels: prometheus.NewDesc( @@ -272,8 +272,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.Issues, prometheus.GaugeValue, - float64(stats.Counter.Issue), + float64(stats.Counter.IssueOpen), "open", ) + ch <- prometheus.MustNewConstMetric( + c.Issues, + prometheus.GaugeValue, + float64(stats.Counter.IssueClosed), "closed", + ) + for _, il := range stats.Counter.IssueByLabel { ch <- prometheus.MustNewConstMetric( c.IssuesByLabel, From 3e06c39d2f3f421990018cdd9233bfb4c22acd00 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Thu, 16 Jan 2025 21:02:50 +0100 Subject: [PATCH 03/21] feat(metrics): add http metrics --- modules/metrics/collector.go | 5 +++ routers/common/middleware.go | 59 ++++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 6d42803a3f..3701ba9ec7 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -11,10 +11,15 @@ import ( "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" ) const namespace = "gitea_" +func init() { + model.NameValidationScheme = model.UTF8Validation +} + // Collector implements the prometheus.Collector interface and // exposes gitea metrics for prometheus type Collector struct { diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 739c92918a..431cecf6ab 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -6,8 +6,8 @@ package common import ( "fmt" "net/http" + "strconv" "strings" - "time" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/gtprof" @@ -19,17 +19,42 @@ import ( "gitea.com/go-chi/session" "github.com/chi-middleware/proxy" + "github.com/felixge/httpsnoop" "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) -var responseLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "http", - Subsystem: "request", - Name: "duration_seconds", - Help: "Gitea response time", -}, []string{"route"}) +var ( + // reqInflightGauge tracks the amount of currently handled requests + reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "http", + Subsystem: "server", + Name: "active_requests", + Help: "Number of active HTTP server requests.", + }, []string{"http.request.method"}) + // reqDurationHistogram tracks the time taken by http request + reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server", + Name: "request_duration", + Help: "Measures the latency of HTTP requests processed by the server", + }, []string{"http.request.method", "http.response.status_code", "http.route"}) + // reqSizeHistogram tracks the size of request + reqSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server_request", + Name: "body_size", + Help: "Size of HTTP server request bodies.", + }, []string{"http.request.method", "http.response.status_code", "http.route"}) + // respSizeHistogram tracks the size of the response + respSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server_response", + Name: "body_size", + Help: "Size of HTTP server response bodies.", + }, []string{"http.request.method", "http.response.status_code", "http.route"}) +) // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery func ProtocolMiddlewares() (handlers []any) { @@ -120,17 +145,25 @@ func ForwardedHeadersHandler(limit int, trustedProxies []string) func(h http.Han return proxy.ForwardedHeaders(opt) } +// RouteMetrics instruments http requests and responses func RouteMetrics() func(h http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - start := time.Now() + inflight := reqInflightGauge.WithLabelValues(req.Method) + inflight.Inc() + defer inflight.Dec() + + m := httpsnoop.CaptureMetrics(next, resp, req) next.ServeHTTP(resp, req) - elapsed := time.Since(start).Seconds() - route := "ui" - if strings.HasPrefix(req.URL.Path, "/api") { - route = "api" + route := chi.RouteContext(req.Context()).RoutePattern() + code := strconv.Itoa(m.Code) + reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(m.Duration.Seconds()) + respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.Written)) + size := req.ContentLength + if size < 0 { + size = 0 } - responseLatency.WithLabelValues(route).Observe(elapsed) + reqSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(size)) }) } } From c666f4d9cc57eedc65292daaa4c8a104d754fffe Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Thu, 16 Jan 2025 21:33:22 +0100 Subject: [PATCH 04/21] feat(metrics): add use labels in cache response --- modules/cache/cache.go | 47 ++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 49bb3bff9f..e1af57faa7 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -9,32 +9,39 @@ import ( "time" "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" _ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache" ) -var defaultCache StringCache -var hitCounter = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "gitea", - Help: "Cache hit count", - Subsystem: "cache", - Name: "hit", -}) -var missCounter = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "gitea", - Help: "Cache miss count", - Subsystem: "cache", - Name: "miss", -}) -var latencyHistogram = promauto.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "gitea", - Help: "Cache latency", - Subsystem: "cache", - Name: "duration", - }, +var ( + defaultCache StringCache + + // TODO: Combine hit and miss into one + hitCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache count", + Subsystem: "cache", + Name: "response", + ConstLabels: prometheus.Labels{"state": "hit"}, + }) + missCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache count", + Subsystem: "cache", + Name: "response", + ConstLabels: prometheus.Labels{"state": "miss"}, + }) + latencyHistogram = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "gitea", + Help: "Cache latency", + Subsystem: "cache", + Name: "duration", + }, + ) ) // Init start cache service From 91919de7207a17ce5da5a0ea46545e01c006e200 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Thu, 16 Jan 2025 21:33:49 +0100 Subject: [PATCH 05/21] feat(metrics): add migration counters --- services/migrations/migrate.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 51b22d6111..edff7d4a95 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -21,6 +21,14 @@ import ( base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + repoMigrationsInflightGauge = promauto.NewGauge(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "inflight_migrations", Help: "Number of inflight repository migrations"}) + repoMigrationsCounter = promauto.NewGaugeVec(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "migrations", Help: "Total migrations"}, []string{"result"}) ) // MigrateOptions is equal to base.MigrateOptions @@ -124,6 +132,9 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str return nil, err } + repoMigrationsInflightGauge.Inc() + defer repoMigrationsInflightGauge.Dec() + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType @@ -134,8 +145,10 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { log.Error("create respotiry notice failed: ", err2) } + repoMigrationsCounter.WithLabelValues("fail").Inc() return nil, err } + repoMigrationsCounter.WithLabelValues("success").Inc() return uploader.repo, nil } From eddef364358719dc71826e8fc1c7326091dc596b Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Thu, 16 Jan 2025 23:39:23 +0100 Subject: [PATCH 06/21] fix(metrics): fallback to legacy naming --- assets/go-licenses.json | 5 +++++ go.mod | 4 ++-- routers/common/middleware.go | 14 ++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index a20494184b..817c84bc04 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -439,6 +439,11 @@ "path": "github.com/felixge/fgprof/LICENSE.txt", "licenseText": "The MIT License (MIT)\nCopyright © 2020 Felix Geisendörfer \u003cfelix@felixge.de\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "github.com/felixge/httpsnoop", + "path": "github.com/felixge/httpsnoop/LICENSE.txt", + "licenseText": "Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com)\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in\n all copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n THE SOFTWARE.\n" + }, { "name": "github.com/fsnotify/fsnotify", "path": "github.com/fsnotify/fsnotify/LICENSE", diff --git a/go.mod b/go.mod index 0ee4257f13..d4da8f6707 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.5 + github.com/felixge/httpsnoop v1.0.4 github.com/fsnotify/fsnotify v1.7.0 github.com/gliderlabs/ssh v0.3.8 github.com/go-ap/activitypub v0.0.0-20240910141749-b4b8c8aa484c @@ -99,6 +100,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/common v0.60.1 github.com/quasoft/websspi v1.1.2 github.com/redis/go-redis/v9 v9.7.0 github.com/robfig/cron/v3 v3.0.1 @@ -193,7 +195,6 @@ require ( github.com/dlclark/regexp2 v1.11.4 // indirect github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect github.com/fatih/color v1.18.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1 // indirect github.com/go-ap/errors v0.0.0-20240910140019-1e9d33cc1568 // indirect @@ -268,7 +269,6 @@ require ( github.com/pjbgf/sha1cd v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rhysd/actionlint v1.7.3 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 431cecf6ab..7feadf7943 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -25,6 +25,12 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) +const ( + httpRequestMethod = "http_request_method" + httpResponseStatusCode = "http_response_status_code" + httpRoute = "http_route" +) + var ( // reqInflightGauge tracks the amount of currently handled requests reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ @@ -32,28 +38,28 @@ var ( Subsystem: "server", Name: "active_requests", Help: "Number of active HTTP server requests.", - }, []string{"http.request.method"}) + }, []string{httpRequestMethod}) // reqDurationHistogram tracks the time taken by http request reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "http", Subsystem: "server", Name: "request_duration", Help: "Measures the latency of HTTP requests processed by the server", - }, []string{"http.request.method", "http.response.status_code", "http.route"}) + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) // reqSizeHistogram tracks the size of request reqSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "http", Subsystem: "server_request", Name: "body_size", Help: "Size of HTTP server request bodies.", - }, []string{"http.request.method", "http.response.status_code", "http.route"}) + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) // respSizeHistogram tracks the size of the response respSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "http", Subsystem: "server_response", Name: "body_size", Help: "Size of HTTP server response bodies.", - }, []string{"http.request.method", "http.response.status_code", "http.route"}) + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) ) // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery From 0b1e3254dd3ae9c7d0a20cdbdf1e7c6456c1cb2e Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Sat, 18 Jan 2025 00:07:59 +0100 Subject: [PATCH 07/21] refactor(metrics): remove external library --- go.mod | 2 +- routers/common/middleware.go | 11 ++++++----- routers/common/middleware_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 routers/common/middleware_test.go diff --git a/go.mod b/go.mod index d4da8f6707..69f5b2728d 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,6 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.5 - github.com/felixge/httpsnoop v1.0.4 github.com/fsnotify/fsnotify v1.7.0 github.com/gliderlabs/ssh v0.3.8 github.com/go-ap/activitypub v0.0.0-20240910141749-b4b8c8aa484c @@ -195,6 +194,7 @@ require ( github.com/dlclark/regexp2 v1.11.4 // indirect github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect github.com/fatih/color v1.18.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1 // indirect github.com/go-ap/errors v0.0.0-20240910140019-1e9d33cc1568 // indirect diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 7feadf7943..30484583fd 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -8,6 +8,7 @@ import ( "net/http" "strconv" "strings" + "time" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/gtprof" @@ -19,7 +20,6 @@ import ( "gitea.com/go-chi/session" "github.com/chi-middleware/proxy" - "github.com/felixge/httpsnoop" "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -158,13 +158,14 @@ func RouteMetrics() func(h http.Handler) http.Handler { inflight := reqInflightGauge.WithLabelValues(req.Method) inflight.Inc() defer inflight.Dec() + start := time.Now() - m := httpsnoop.CaptureMetrics(next, resp, req) next.ServeHTTP(resp, req) + m := context.WrapResponseWriter(resp) route := chi.RouteContext(req.Context()).RoutePattern() - code := strconv.Itoa(m.Code) - reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(m.Duration.Seconds()) - respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.Written)) + code := strconv.Itoa(m.WrittenStatus()) + reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(time.Since(start).Seconds()) + respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.Size())) size := req.ContentLength if size < 0 { size = 0 diff --git a/routers/common/middleware_test.go b/routers/common/middleware_test.go new file mode 100644 index 0000000000..03612faf22 --- /dev/null +++ b/routers/common/middleware_test.go @@ -0,0 +1,30 @@ +package common_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "code.gitea.io/gitea/routers/common" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/require" +) + +func TestMetricsMiddlewere(t *testing.T) { + + middleware := common.RouteMetrics() + r := chi.NewRouter() + r.Use(middleware) + r.Get("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("test")) + time.Sleep(5 * time.Millisecond) + })) + + testServer := httptest.NewServer(r) + + _, err := http.Get(testServer.URL) + require.NoError(t, err) + +} From 056a308b2a8da4d20d8e8819d6c6b930cc67e3e1 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Thu, 23 Jan 2025 23:59:44 +0100 Subject: [PATCH 08/21] test(metrics): add middleware test --- go.mod | 1 + routers/common/middleware.go | 2 +- routers/common/middleware_test.go | 22 +++++++++++++++------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 69f5b2728d..88e495d9ed 100644 --- a/go.mod +++ b/go.mod @@ -242,6 +242,7 @@ require ( github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/libdns/libdns v0.2.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 30484583fd..5d7a2ee03e 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -165,7 +165,7 @@ func RouteMetrics() func(h http.Handler) http.Handler { route := chi.RouteContext(req.Context()).RoutePattern() code := strconv.Itoa(m.WrittenStatus()) reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(time.Since(start).Seconds()) - respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.Size())) + respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.WrittenSize())) size := req.ContentLength if size < 0 { size = 0 diff --git a/routers/common/middleware_test.go b/routers/common/middleware_test.go index 03612faf22..b2ec73c565 100644 --- a/routers/common/middleware_test.go +++ b/routers/common/middleware_test.go @@ -1,4 +1,4 @@ -package common_test +package common import ( "net/http" @@ -6,25 +6,33 @@ import ( "testing" "time" - "code.gitea.io/gitea/routers/common" - "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMetricsMiddlewere(t *testing.T) { - - middleware := common.RouteMetrics() + middleware := RouteMetrics() r := chi.NewRouter() r.Use(middleware) r.Get("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("test")) time.Sleep(5 * time.Millisecond) })) - testServer := httptest.NewServer(r) + // Check all defined metrics + verify := func(i int) { + assert.Equal(t, testutil.CollectAndCount(reqDurationHistogram, "http_server_request_duration"), i) + assert.Equal(t, testutil.CollectAndCount(reqSizeHistogram, "http_server_request_body_size"), i) + assert.Equal(t, testutil.CollectAndCount(respSizeHistogram, "http_server_response_body_size"), i) + assert.Equal(t, testutil.CollectAndCount(reqInflightGauge, "http_server_active_requests"), i) + } + // Check they don't exist before making a request + verify(0) _, err := http.Get(testServer.URL) require.NoError(t, err) - + // Check they do exist after making the request + verify(1) } From 206eb3e16fc8f4cc401513d1a2e0c10c6fe565dd Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Mon, 7 Jul 2025 00:33:52 +0200 Subject: [PATCH 09/21] test out non default buckets --- go.mod | 12 +----------- routers/common/middleware.go | 9 ++++++++- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index f7bf9f5a33..10841e1aea 100644 --- a/go.mod +++ b/go.mod @@ -97,9 +97,8 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 - github.com/prometheus/client_golang v1.20.5 - github.com/prometheus/common v0.60.1 github.com/prometheus/client_golang v1.22.0 + github.com/prometheus/common v0.63.0 github.com/quasoft/websspi v1.1.2 github.com/redis/go-redis/v9 v9.7.3 github.com/robfig/cron/v3 v3.0.1 @@ -221,12 +220,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/libdns/libdns v0.2.2 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/libdns/libdns v1.0.0-beta.1 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/markbates/going v1.0.3 // indirect @@ -249,11 +243,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/rhysd/actionlint v1.7.3 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.63.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rhysd/actionlint v1.7.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 5d7a2ee03e..aa8ab791b8 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -29,9 +29,13 @@ const ( httpRequestMethod = "http_request_method" httpResponseStatusCode = "http_response_status_code" httpRoute = "http_route" + kb = 1000 + mb = kb * kb ) +// reference: https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#http-server var ( + sizeBuckets = []float64{1 * kb, 2 * kb, 5 * kb, 10 * kb, 100 * kb, 500 * kb, 1 * mb, 2 * mb, 5 * mb, 10 * mb} // reqInflightGauge tracks the amount of currently handled requests reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "http", @@ -43,8 +47,9 @@ var ( reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "http", Subsystem: "server", - Name: "request_duration", + Name: "request_duration_seconds", // diverge from spec to store the unit in metric. Help: "Measures the latency of HTTP requests processed by the server", + Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30, 60, 120, 300}, // based on dotnet buckets https://github.com/open-telemetry/semantic-conventions/issues/336 }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) // reqSizeHistogram tracks the size of request reqSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ @@ -52,6 +57,7 @@ var ( Subsystem: "server_request", Name: "body_size", Help: "Size of HTTP server request bodies.", + Buckets: sizeBuckets, }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) // respSizeHistogram tracks the size of the response respSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ @@ -59,6 +65,7 @@ var ( Subsystem: "server_response", Name: "body_size", Help: "Size of HTTP server response bodies.", + Buckets: sizeBuckets, }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) ) From 734c2da3ac471a51f680a45c17f447e448adf5e2 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Mon, 7 Jul 2025 00:35:03 +0200 Subject: [PATCH 10/21] add inflight and duration to git commands still kind of buggy as it doesn't properly encode commands --- modules/git/command.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/modules/git/command.go b/modules/git/command.go index 22f1d02339..f800e5f3d5 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -22,6 +22,9 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) // TrustedCmdArgs returns the trusted arguments for git command. @@ -29,12 +32,29 @@ import ( // In most cases, it shouldn't be used. Use AddXxx function instead type TrustedCmdArgs []internal.CmdArg +const gitOperation = "command" + var ( // globalCommandArgs global command args for external package setting globalCommandArgs TrustedCmdArgs // defaultCommandExecutionTimeout default command execution timeout duration defaultCommandExecutionTimeout = 360 * time.Second + + reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "gitea", + Subsystem: "git", + Name: "active_commands", + Help: "Number of active git subprocesses.", + }, []string{gitOperation}) + // reqDurationHistogram tracks the time taken by http request + reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "gitea", + Subsystem: "git", + Name: "command_duration_seconds", // diverge from spec to store the unit in metric. + Help: "Measures the time taken by git subprocesses", + Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30, 60, 120, 300}, // based on dotnet buckets https://github.com/open-telemetry/semantic-conventions/issues/336 + }, []string{gitOperation}) ) // DefaultLocale is the default LC_ALL to run git commands in. @@ -315,6 +335,10 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { desc := fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), cmdLogString) log.Debug("git.Command: %s", desc) + inflight := reqInflightGauge.WithLabelValues(c.args[0]) // add command type + inflight.Inc() + defer inflight.Dec() + _, span := gtprof.GetTracer().Start(ctx, gtprof.TraceSpanGitRun) defer span.End() span.SetAttributeString(gtprof.TraceAttrFuncCaller, callerInfo) @@ -364,6 +388,7 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { if elapsed > time.Second { log.Debug("slow git.Command.Run: %s (%s)", c, elapsed) } + reqDurationHistogram.WithLabelValues(c.args[0]).Observe(elapsed.Seconds()) // We need to check if the context is canceled by the program on Windows. // This is because Windows does not have signal checking when terminating the process. From 6e64232f42bec1504ac687facaf82cddfcc61505 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Mon, 7 Jul 2025 00:42:44 +0200 Subject: [PATCH 11/21] drop deprecated validation setting --- modules/metrics/collector.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 7a34fd54ba..ac276cc723 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -11,15 +11,10 @@ import ( "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" ) const namespace = "gitea_" -func init() { - model.NameValidationScheme = model.UTF8Validation -} - // Collector implements the prometheus.Collector interface and // exposes gitea metrics for prometheus type Collector struct { From d310cd08f8eb063374fecc6f33c2ffe22638f1a8 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Mon, 7 Jul 2025 15:38:40 +0200 Subject: [PATCH 12/21] drop label to be reintroduced later, for now it would require figuring out which arg is the actual command --- modules/git/command.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index f800e5f3d5..f9bd1551c1 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -41,20 +41,20 @@ var ( // defaultCommandExecutionTimeout default command execution timeout duration defaultCommandExecutionTimeout = 360 * time.Second - reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + reqInflightGauge = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "gitea", Subsystem: "git", Name: "active_commands", Help: "Number of active git subprocesses.", - }, []string{gitOperation}) - // reqDurationHistogram tracks the time taken by http request - reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + }) + // reqDurationHistogram tracks the time taken by git call + reqDurationHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: "gitea", Subsystem: "git", Name: "command_duration_seconds", // diverge from spec to store the unit in metric. Help: "Measures the time taken by git subprocesses", Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30, 60, 120, 300}, // based on dotnet buckets https://github.com/open-telemetry/semantic-conventions/issues/336 - }, []string{gitOperation}) + }) ) // DefaultLocale is the default LC_ALL to run git commands in. @@ -335,7 +335,7 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { desc := fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), cmdLogString) log.Debug("git.Command: %s", desc) - inflight := reqInflightGauge.WithLabelValues(c.args[0]) // add command type + inflight := reqInflightGauge // add command type inflight.Inc() defer inflight.Dec() @@ -388,7 +388,7 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { if elapsed > time.Second { log.Debug("slow git.Command.Run: %s (%s)", c, elapsed) } - reqDurationHistogram.WithLabelValues(c.args[0]).Observe(elapsed.Seconds()) + reqDurationHistogram.Observe(elapsed.Seconds()) // We need to check if the context is canceled by the program on Windows. // This is because Windows does not have signal checking when terminating the process. From 93116f5d4791ef68a17b0644d2b7bb6a34bd5244 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 8 Jul 2025 11:08:25 +0200 Subject: [PATCH 13/21] add database histogram --- models/db/engine_hook.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/db/engine_hook.go b/models/db/engine_hook.go index 8709a2c2a1..8ac40ec9d2 100644 --- a/models/db/engine_hook.go +++ b/models/db/engine_hook.go @@ -10,6 +10,8 @@ import ( "code.gitea.io/gitea/modules/gtprof" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "xorm.io/xorm/contexts" ) @@ -21,12 +23,22 @@ type EngineHook struct { var _ contexts.Hook = (*EngineHook)(nil) +// follows: https://opentelemetry.io/docs/specs/semconv/database/database-metrics/#metric-dbclientoperationduration +var durationHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "db", + Subsystem: "client", + Name: "operation.duration", + Help: "Duration of database client operations.", + // ConstLabels: prometheus.Labels{"db.system.name": BuilderDialect()}, //TODO: add type of database per spec. +}) + func (*EngineHook) BeforeProcess(c *contexts.ContextHook) (context.Context, error) { ctx, _ := gtprof.GetTracer().Start(c.Ctx, gtprof.TraceSpanDatabase) return ctx, nil } func (h *EngineHook) AfterProcess(c *contexts.ContextHook) error { + durationHistogram.Observe(c.ExecuteTime.Seconds()) span := gtprof.GetContextSpan(c.Ctx) if span != nil { // Do not record SQL parameters here: From fb08fa6909b3e043ffb42aa632931754595d7a12 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 8 Jul 2025 13:52:58 +0200 Subject: [PATCH 14/21] fix formatting --- models/db/engine_hook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/db/engine_hook.go b/models/db/engine_hook.go index 8ac40ec9d2..ce4eb1f736 100644 --- a/models/db/engine_hook.go +++ b/models/db/engine_hook.go @@ -10,9 +10,9 @@ import ( "code.gitea.io/gitea/modules/gtprof" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "xorm.io/xorm/contexts" ) From 0e501fee116c8581540559f98ba6e557e4714598 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 8 Jul 2025 16:56:57 +0200 Subject: [PATCH 15/21] fix linting issues and failing test --- assets/go-licenses.json | 5 ----- go.mod | 2 +- modules/git/command.go | 2 +- routers/common/middleware_test.go | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index b73585330a..d961444239 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -449,11 +449,6 @@ "path": "github.com/felixge/fgprof/LICENSE.txt", "licenseText": "The MIT License (MIT)\nCopyright © 2020 Felix Geisendörfer \u003cfelix@felixge.de\u003e\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, - { - "name": "github.com/felixge/httpsnoop", - "path": "github.com/felixge/httpsnoop/LICENSE.txt", - "licenseText": "Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com)\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in\n all copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n THE SOFTWARE.\n" - }, { "name": "github.com/fsnotify/fsnotify", "path": "github.com/fsnotify/fsnotify/LICENSE", diff --git a/go.mod b/go.mod index 10841e1aea..fc37bd05fb 100644 --- a/go.mod +++ b/go.mod @@ -98,7 +98,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 github.com/prometheus/client_golang v1.22.0 - github.com/prometheus/common v0.63.0 github.com/quasoft/websspi v1.1.2 github.com/redis/go-redis/v9 v9.7.3 github.com/robfig/cron/v3 v3.0.1 @@ -244,6 +243,7 @@ require ( github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.63.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rhysd/actionlint v1.7.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/modules/git/command.go b/modules/git/command.go index f9bd1551c1..a998a2b14a 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -32,7 +32,7 @@ import ( // In most cases, it shouldn't be used. Use AddXxx function instead type TrustedCmdArgs []internal.CmdArg -const gitOperation = "command" +// const gitOperation = "command" var ( // globalCommandArgs global command args for external package setting diff --git a/routers/common/middleware_test.go b/routers/common/middleware_test.go index b2ec73c565..f455a57ad6 100644 --- a/routers/common/middleware_test.go +++ b/routers/common/middleware_test.go @@ -23,7 +23,7 @@ func TestMetricsMiddlewere(t *testing.T) { testServer := httptest.NewServer(r) // Check all defined metrics verify := func(i int) { - assert.Equal(t, testutil.CollectAndCount(reqDurationHistogram, "http_server_request_duration"), i) + assert.Equal(t, testutil.CollectAndCount(reqDurationHistogram, "http_server_request_duration_seconds"), i) assert.Equal(t, testutil.CollectAndCount(reqSizeHistogram, "http_server_request_body_size"), i) assert.Equal(t, testutil.CollectAndCount(respSizeHistogram, "http_server_response_body_size"), i) assert.Equal(t, testutil.CollectAndCount(reqInflightGauge, "http_server_active_requests"), i) From c02ac93968cfd128a7461bd62fcd2fc721173147 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 8 Jul 2025 17:14:28 +0200 Subject: [PATCH 16/21] add copyright info and unit --- models/db/engine_hook.go | 2 +- routers/common/middleware_test.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/models/db/engine_hook.go b/models/db/engine_hook.go index ce4eb1f736..5a2a7187da 100644 --- a/models/db/engine_hook.go +++ b/models/db/engine_hook.go @@ -27,7 +27,7 @@ var _ contexts.Hook = (*EngineHook)(nil) var durationHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: "db", Subsystem: "client", - Name: "operation.duration", + Name: "operation_duration_seconds", Help: "Duration of database client operations.", // ConstLabels: prometheus.Labels{"db.system.name": BuilderDialect()}, //TODO: add type of database per spec. }) diff --git a/routers/common/middleware_test.go b/routers/common/middleware_test.go index f455a57ad6..fd53bf6ac7 100644 --- a/routers/common/middleware_test.go +++ b/routers/common/middleware_test.go @@ -1,3 +1,6 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + package common import ( From 96246b93aad78fc1e42d8971feef745fd3686604 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 8 Jul 2025 17:26:02 +0200 Subject: [PATCH 17/21] modernize --- routers/common/middleware.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/routers/common/middleware.go b/routers/common/middleware.go index aa8ab791b8..6c3e48076d 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -173,10 +173,8 @@ func RouteMetrics() func(h http.Handler) http.Handler { code := strconv.Itoa(m.WrittenStatus()) reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(time.Since(start).Seconds()) respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.WrittenSize())) - size := req.ContentLength - if size < 0 { - size = 0 - } + + size := max(req.ContentLength, 0) reqSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(size)) }) } From 25eee166c4902ab45443b325babc0e3d2d3a8823 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Tue, 8 Jul 2025 23:22:00 +0200 Subject: [PATCH 18/21] add cron inflight metric --- services/cron/tasks.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index f8a7444c49..cea9a5c815 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -19,9 +19,17 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) var ( + cronInflight = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "gitea", + Subsystem: "cron", + Name: "active_tasks", + Help: "Number of running cron tasks", + }) lock = sync.Mutex{} started = false tasks = []*Task{} @@ -86,6 +94,8 @@ func (t *Task) RunWithUser(doer *user_model.User, config Config) { taskStatusTable.Stop(t.Name) }() graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) { + cronInflight.Inc() + defer cronInflight.Dec() defer func() { if err := recover(); err != nil { // Recover a panic within the execution of the task. From c4a81c56f02da282336f40f1a0193e5038dd59bd Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Wed, 9 Jul 2025 21:16:49 +0200 Subject: [PATCH 19/21] fix formatting --- services/cron/tasks.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/cron/tasks.go b/services/cron/tasks.go index cea9a5c815..b2fcaab5fc 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) From 9d7283ae3cac16f88cd0212289e6388b17aed0ca Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Wed, 9 Jul 2025 23:06:12 +0200 Subject: [PATCH 20/21] add system notice counter --- modules/metrics/collector.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index ac276cc723..a53f7a3e89 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -8,6 +8,7 @@ import ( activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" @@ -41,6 +42,7 @@ type Collector struct { Releases *prometheus.Desc Repositories *prometheus.Desc Stars *prometheus.Desc + SystemNotices *prometheus.Desc Teams *prometheus.Desc UpdateTasks *prometheus.Desc Users *prometheus.Desc @@ -171,6 +173,10 @@ func NewCollector() Collector { "Number of Stars", nil, nil, ), + SystemNotices: prometheus.NewDesc( + namespace+"system_notices", + "Number of system notices", + nil, nil), Teams: prometheus.NewDesc( namespace+"teams", "Number of Teams", @@ -234,6 +240,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { // Collect returns the metrics with values func (c Collector) Collect(ch chan<- prometheus.Metric) { stats := activities_model.GetStatistic(db.DefaultContext) + noticeCount := system.CountNotices(db.DefaultContext) ch <- prometheus.MustNewConstMetric( c.Accesses, @@ -366,6 +373,11 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(stats.Counter.Star), ) + ch <- prometheus.MustNewConstMetric( + c.SystemNotices, + prometheus.GaugeValue, + float64(noticeCount), + ) ch <- prometheus.MustNewConstMetric( c.Teams, prometheus.GaugeValue, From f23be74c868f84607a9a1bce6332f1576cc78bf3 Mon Sep 17 00:00:00 2001 From: TheFox0x7 Date: Wed, 9 Jul 2025 23:35:07 +0200 Subject: [PATCH 21/21] add login and webhook counters --- routers/web/auth/auth.go | 6 ++++++ services/webhook/deliver.go | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/routers/web/auth/auth.go b/routers/web/auth/auth.go index 94f75f69ff..eed7862563 100644 --- a/routers/web/auth/auth.go +++ b/routers/web/auth/auth.go @@ -35,6 +35,8 @@ import ( user_service "code.gitea.io/gitea/services/user" "github.com/markbates/goth" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) const ( @@ -180,6 +182,8 @@ func prepareSignInPageData(ctx *context.Context) { } } +var loginCounter = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: "gitea", Subsystem: "auth", Name: "login"}, []string{"status"}) // TODO: Add source/provider in the future? + // SignIn render sign in page func SignIn(ctx *context.Context) { if CheckAutoLogin(ctx) { @@ -217,6 +221,7 @@ func SignInPost(ctx *context.Context) { u, source, err := auth_service.UserSignIn(ctx, form.UserName, form.Password) if err != nil { + loginCounter.WithLabelValues("failure").Inc() if errors.Is(err, util.ErrNotExist) || errors.Is(err, util.ErrInvalidArgument) { ctx.RenderWithErr(ctx.Tr("form.username_password_incorrect"), tplSignIn, &form) log.Warn("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) @@ -237,6 +242,7 @@ func SignInPost(ctx *context.Context) { ctx.HTML(http.StatusOK, "user/auth/prohibit_login") } } else { + loginCounter.WithLabelValues("success").Inc() ctx.ServerError("UserSignIn", err) } return diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index e8e6ed19c1..e66ea06b08 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -32,6 +32,8 @@ import ( webhook_module "code.gitea.io/gitea/modules/webhook" "github.com/gobwas/glob" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (req *http.Request, body []byte, err error) { @@ -148,6 +150,8 @@ func addDefaultHeaders(req *http.Request, secret []byte, w *webhook_model.Webhoo return nil } +var webhookCounter = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: "gitea", Subsystem: "webhook", Name: "deliveries", Help: "Number of webhook deliveries"}, []string{"success"}) + // Deliver creates the [http.Request] (depending on the webhook type), sends it // and records the status and response. func Deliver(ctx context.Context, t *webhook_model.HookTask) error { @@ -265,6 +269,12 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error { t.ResponseInfo.Headers[k] = strings.Join(vals, ",") } + if t.IsSucceed { + webhookCounter.WithLabelValues("success").Inc() + } else { + webhookCounter.WithLabelValues("failure").Inc() + } + p, err := io.ReadAll(resp.Body) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)