diff --git a/go.mod b/go.mod index 8b8d61dc65..79b3e182a5 100644 --- a/go.mod +++ b/go.mod @@ -219,6 +219,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/libdns/libdns v1.0.0-beta.1 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/markbates/going v1.0.3 // indirect diff --git a/models/db/engine_hook.go b/models/db/engine_hook.go index 8709a2c2a1..5a2a7187da 100644 --- a/models/db/engine_hook.go +++ b/models/db/engine_hook.go @@ -11,6 +11,8 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "xorm.io/xorm/contexts" ) @@ -21,12 +23,22 @@ type EngineHook struct { var _ contexts.Hook = (*EngineHook)(nil) +// follows: https://opentelemetry.io/docs/specs/semconv/database/database-metrics/#metric-dbclientoperationduration +var durationHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "db", + Subsystem: "client", + Name: "operation_duration_seconds", + Help: "Duration of database client operations.", + // ConstLabels: prometheus.Labels{"db.system.name": BuilderDialect()}, //TODO: add type of database per spec. +}) + func (*EngineHook) BeforeProcess(c *contexts.ContextHook) (context.Context, error) { ctx, _ := gtprof.GetTracer().Start(c.Ctx, gtprof.TraceSpanDatabase) return ctx, nil } func (h *EngineHook) AfterProcess(c *contexts.ContextHook) error { + durationHistogram.Observe(c.ExecuteTime.Seconds()) span := gtprof.GetContextSpan(c.Ctx) if span != nil { // Do not record SQL parameters here: diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 039caa9fbc..4f0e50e078 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -12,10 +12,39 @@ import ( "code.gitea.io/gitea/modules/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + _ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache" ) -var defaultCache StringCache +var ( + defaultCache StringCache + + // TODO: Combine hit and miss into one + hitCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache count", + Subsystem: "cache", + Name: "response", + ConstLabels: prometheus.Labels{"state": "hit"}, + }) + missCounter = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "gitea", + Help: "Cache count", + Subsystem: "cache", + Name: "response", + ConstLabels: prometheus.Labels{"state": "miss"}, + }) + latencyHistogram = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "gitea", + Help: "Cache latency", + Subsystem: "cache", + Name: "duration", + }, + ) +) // Init start cache service func Init() error { diff --git a/modules/cache/string_cache.go b/modules/cache/string_cache.go index 3562b7a926..d03e98c027 100644 --- a/modules/cache/string_cache.go +++ b/modules/cache/string_cache.go @@ -6,6 +6,7 @@ package cache import ( "errors" "strings" + "time" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" @@ -63,10 +64,15 @@ func (sc *stringCache) Ping() error { } func (sc *stringCache) Get(key string) (string, bool) { + start := time.Now() v := sc.chiCache.Get(key) + elapsed := time.Since(start).Seconds() + latencyHistogram.Observe(elapsed) if v == nil { + missCounter.Add(1) return "", false } + hitCounter.Add(1) s, ok := v.(string) return s, ok } diff --git a/modules/git/command.go b/modules/git/command.go index 22f1d02339..a998a2b14a 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -22,6 +22,9 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) // TrustedCmdArgs returns the trusted arguments for git command. @@ -29,12 +32,29 @@ import ( // In most cases, it shouldn't be used. Use AddXxx function instead type TrustedCmdArgs []internal.CmdArg +// const gitOperation = "command" + var ( // globalCommandArgs global command args for external package setting globalCommandArgs TrustedCmdArgs // defaultCommandExecutionTimeout default command execution timeout duration defaultCommandExecutionTimeout = 360 * time.Second + + reqInflightGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "gitea", + Subsystem: "git", + Name: "active_commands", + Help: "Number of active git subprocesses.", + }) + // reqDurationHistogram tracks the time taken by git call + reqDurationHistogram = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "gitea", + Subsystem: "git", + Name: "command_duration_seconds", // diverge from spec to store the unit in metric. + Help: "Measures the time taken by git subprocesses", + Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30, 60, 120, 300}, // based on dotnet buckets https://github.com/open-telemetry/semantic-conventions/issues/336 + }) ) // DefaultLocale is the default LC_ALL to run git commands in. @@ -315,6 +335,10 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { desc := fmt.Sprintf("git.Run(by:%s, repo:%s): %s", callerInfo, logArgSanitize(opts.Dir), cmdLogString) log.Debug("git.Command: %s", desc) + inflight := reqInflightGauge // add command type + inflight.Inc() + defer inflight.Dec() + _, span := gtprof.GetTracer().Start(ctx, gtprof.TraceSpanGitRun) defer span.End() span.SetAttributeString(gtprof.TraceAttrFuncCaller, callerInfo) @@ -364,6 +388,7 @@ func (c *Command) run(ctx context.Context, skip int, opts *RunOpts) error { if elapsed > time.Second { log.Debug("slow git.Command.Run: %s (%s)", c, elapsed) } + reqDurationHistogram.Observe(elapsed.Seconds()) // We need to check if the context is canceled by the program on Windows. // This is because Windows does not have signal checking when terminating the process. diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 4d2ec287a9..a53f7a3e89 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -8,6 +8,7 @@ import ( activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/modules/setting" "github.com/prometheus/client_golang/prometheus" @@ -41,6 +42,7 @@ type Collector struct { Releases *prometheus.Desc Repositories *prometheus.Desc Stars *prometheus.Desc + SystemNotices *prometheus.Desc Teams *prometheus.Desc UpdateTasks *prometheus.Desc Users *prometheus.Desc @@ -89,7 +91,7 @@ func NewCollector() Collector { Issues: prometheus.NewDesc( namespace+"issues", "Number of Issues", - nil, nil, + []string{"state"}, nil, ), IssuesByLabel: prometheus.NewDesc( namespace+"issues_by_label", @@ -103,12 +105,12 @@ func NewCollector() Collector { ), IssuesOpen: prometheus.NewDesc( namespace+"issues_open", - "Number of open Issues", + "DEPRECATED: Use Issues with state: open", nil, nil, ), IssuesClosed: prometheus.NewDesc( namespace+"issues_closed", - "Number of closed Issues", + "DEPRECATED: Use Issues with state: closed", nil, nil, ), Labels: prometheus.NewDesc( @@ -171,6 +173,10 @@ func NewCollector() Collector { "Number of Stars", nil, nil, ), + SystemNotices: prometheus.NewDesc( + namespace+"system_notices", + "Number of system notices", + nil, nil), Teams: prometheus.NewDesc( namespace+"teams", "Number of Teams", @@ -234,6 +240,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) { // Collect returns the metrics with values func (c Collector) Collect(ch chan<- prometheus.Metric) { stats := activities_model.GetStatistic(db.DefaultContext) + noticeCount := system.CountNotices(db.DefaultContext) ch <- prometheus.MustNewConstMetric( c.Accesses, @@ -272,8 +279,14 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.Issues, prometheus.GaugeValue, - float64(stats.Counter.Issue), + float64(stats.Counter.IssueOpen), "open", ) + ch <- prometheus.MustNewConstMetric( + c.Issues, + prometheus.GaugeValue, + float64(stats.Counter.IssueClosed), "closed", + ) + for _, il := range stats.Counter.IssueByLabel { ch <- prometheus.MustNewConstMetric( c.IssuesByLabel, @@ -360,6 +373,11 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { prometheus.GaugeValue, float64(stats.Counter.Star), ) + ch <- prometheus.MustNewConstMetric( + c.SystemNotices, + prometheus.GaugeValue, + float64(noticeCount), + ) ch <- prometheus.MustNewConstMetric( c.Teams, prometheus.GaugeValue, diff --git a/routers/common/middleware.go b/routers/common/middleware.go index 2ba02de8ed..6c3e48076d 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -6,7 +6,9 @@ package common import ( "fmt" "net/http" + "strconv" "strings" + "time" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/gtprof" @@ -19,6 +21,52 @@ import ( "gitea.com/go-chi/session" "github.com/chi-middleware/proxy" "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + httpRequestMethod = "http_request_method" + httpResponseStatusCode = "http_response_status_code" + httpRoute = "http_route" + kb = 1000 + mb = kb * kb +) + +// reference: https://opentelemetry.io/docs/specs/semconv/http/http-metrics/#http-server +var ( + sizeBuckets = []float64{1 * kb, 2 * kb, 5 * kb, 10 * kb, 100 * kb, 500 * kb, 1 * mb, 2 * mb, 5 * mb, 10 * mb} + // reqInflightGauge tracks the amount of currently handled requests + reqInflightGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "http", + Subsystem: "server", + Name: "active_requests", + Help: "Number of active HTTP server requests.", + }, []string{httpRequestMethod}) + // reqDurationHistogram tracks the time taken by http request + reqDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server", + Name: "request_duration_seconds", // diverge from spec to store the unit in metric. + Help: "Measures the latency of HTTP requests processed by the server", + Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30, 60, 120, 300}, // based on dotnet buckets https://github.com/open-telemetry/semantic-conventions/issues/336 + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) + // reqSizeHistogram tracks the size of request + reqSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server_request", + Name: "body_size", + Help: "Size of HTTP server request bodies.", + Buckets: sizeBuckets, + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) + // respSizeHistogram tracks the size of the response + respSizeHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "http", + Subsystem: "server_response", + Name: "body_size", + Help: "Size of HTTP server response bodies.", + Buckets: sizeBuckets, + }, []string{httpRequestMethod, httpResponseStatusCode, httpRoute}) ) // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery @@ -38,6 +86,9 @@ func ProtocolMiddlewares() (handlers []any) { if setting.IsAccessLogEnabled() { handlers = append(handlers, context.AccessLogger()) } + if setting.Metrics.Enabled { + handlers = append(handlers, RouteMetrics()) + } return handlers } @@ -107,6 +158,28 @@ func ForwardedHeadersHandler(limit int, trustedProxies []string) func(h http.Han return proxy.ForwardedHeaders(opt) } +// RouteMetrics instruments http requests and responses +func RouteMetrics() func(h http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + inflight := reqInflightGauge.WithLabelValues(req.Method) + inflight.Inc() + defer inflight.Dec() + start := time.Now() + + next.ServeHTTP(resp, req) + m := context.WrapResponseWriter(resp) + route := chi.RouteContext(req.Context()).RoutePattern() + code := strconv.Itoa(m.WrittenStatus()) + reqDurationHistogram.WithLabelValues(req.Method, code, route).Observe(time.Since(start).Seconds()) + respSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(m.WrittenSize())) + + size := max(req.ContentLength, 0) + reqSizeHistogram.WithLabelValues(req.Method, code, route).Observe(float64(size)) + }) + } +} + func Sessioner() func(next http.Handler) http.Handler { return session.Sessioner(session.Options{ Provider: setting.SessionConfig.Provider, diff --git a/routers/common/middleware_test.go b/routers/common/middleware_test.go new file mode 100644 index 0000000000..fd53bf6ac7 --- /dev/null +++ b/routers/common/middleware_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetricsMiddlewere(t *testing.T) { + middleware := RouteMetrics() + r := chi.NewRouter() + r.Use(middleware) + r.Get("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("test")) + time.Sleep(5 * time.Millisecond) + })) + testServer := httptest.NewServer(r) + // Check all defined metrics + verify := func(i int) { + assert.Equal(t, testutil.CollectAndCount(reqDurationHistogram, "http_server_request_duration_seconds"), i) + assert.Equal(t, testutil.CollectAndCount(reqSizeHistogram, "http_server_request_body_size"), i) + assert.Equal(t, testutil.CollectAndCount(respSizeHistogram, "http_server_response_body_size"), i) + assert.Equal(t, testutil.CollectAndCount(reqInflightGauge, "http_server_active_requests"), i) + } + + // Check they don't exist before making a request + verify(0) + _, err := http.Get(testServer.URL) + require.NoError(t, err) + // Check they do exist after making the request + verify(1) +} diff --git a/routers/web/auth/auth.go b/routers/web/auth/auth.go index 13cd083771..e2cdac47ef 100644 --- a/routers/web/auth/auth.go +++ b/routers/web/auth/auth.go @@ -35,6 +35,8 @@ import ( user_service "code.gitea.io/gitea/services/user" "github.com/markbates/goth" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) const ( @@ -180,6 +182,8 @@ func prepareSignInPageData(ctx *context.Context) { } } +var loginCounter = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: "gitea", Subsystem: "auth", Name: "login"}, []string{"status"}) // TODO: Add source/provider in the future? + // SignIn render sign in page func SignIn(ctx *context.Context) { if CheckAutoLogin(ctx) { @@ -217,6 +221,7 @@ func SignInPost(ctx *context.Context) { u, source, err := auth_service.UserSignIn(ctx, form.UserName, form.Password) if err != nil { + loginCounter.WithLabelValues("failure").Inc() if errors.Is(err, util.ErrNotExist) || errors.Is(err, util.ErrInvalidArgument) { ctx.RenderWithErr(ctx.Tr("form.username_password_incorrect"), tplSignIn, &form) log.Warn("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) @@ -237,6 +242,7 @@ func SignInPost(ctx *context.Context) { ctx.HTML(http.StatusOK, "user/auth/prohibit_login") } } else { + loginCounter.WithLabelValues("success").Inc() ctx.ServerError("UserSignIn", err) } return diff --git a/routers/web/web.go b/routers/web/web.go index b9c7013f63..04fef252e2 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -53,6 +53,7 @@ import ( "github.com/go-chi/cors" "github.com/klauspost/compress/gzhttp" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" ) var GzipMinSize = 1400 // min size to compress for the body size of response @@ -259,6 +260,7 @@ func Routes() *web.Router { } if setting.Metrics.Enabled { + prometheus.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{Namespace: "gitea"})) prometheus.MustRegister(metrics.NewCollector()) routes.Get("/metrics", append(mid, Metrics)...) } diff --git a/services/cron/tasks.go b/services/cron/tasks.go index f8a7444c49..b2fcaab5fc 100644 --- a/services/cron/tasks.go +++ b/services/cron/tasks.go @@ -19,9 +19,18 @@ import ( "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) var ( + cronInflight = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "gitea", + Subsystem: "cron", + Name: "active_tasks", + Help: "Number of running cron tasks", + }) lock = sync.Mutex{} started = false tasks = []*Task{} @@ -86,6 +95,8 @@ func (t *Task) RunWithUser(doer *user_model.User, config Config) { taskStatusTable.Stop(t.Name) }() graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) { + cronInflight.Inc() + defer cronInflight.Dec() defer func() { if err := recover(); err != nil { // Recover a panic within the execution of the task. diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index eba9c79df5..1615e3e6f1 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -22,6 +22,14 @@ import ( base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + repoMigrationsInflightGauge = promauto.NewGauge(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "inflight_migrations", Help: "Number of inflight repository migrations"}) + repoMigrationsCounter = promauto.NewGaugeVec(prometheus.GaugeOpts{Namespace: "gitea", Subsystem: "repository", Name: "migrations", Help: "Total migrations"}, []string{"result"}) ) // MigrateOptions is equal to base.MigrateOptions @@ -123,6 +131,9 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str return nil, err } + repoMigrationsInflightGauge.Inc() + defer repoMigrationsInflightGauge.Dec() + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType @@ -133,8 +144,10 @@ func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName str if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { log.Error("create respotiry notice failed: ", err2) } + repoMigrationsCounter.WithLabelValues("fail").Inc() return nil, err } + repoMigrationsCounter.WithLabelValues("success").Inc() return uploader.repo, nil } diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index e8e6ed19c1..e66ea06b08 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -32,6 +32,8 @@ import ( webhook_module "code.gitea.io/gitea/modules/webhook" "github.com/gobwas/glob" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) func newDefaultRequest(ctx context.Context, w *webhook_model.Webhook, t *webhook_model.HookTask) (req *http.Request, body []byte, err error) { @@ -148,6 +150,8 @@ func addDefaultHeaders(req *http.Request, secret []byte, w *webhook_model.Webhoo return nil } +var webhookCounter = promauto.NewCounterVec(prometheus.CounterOpts{Namespace: "gitea", Subsystem: "webhook", Name: "deliveries", Help: "Number of webhook deliveries"}, []string{"success"}) + // Deliver creates the [http.Request] (depending on the webhook type), sends it // and records the status and response. func Deliver(ctx context.Context, t *webhook_model.HookTask) error { @@ -265,6 +269,12 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error { t.ResponseInfo.Headers[k] = strings.Join(vals, ",") } + if t.IsSucceed { + webhookCounter.WithLabelValues("success").Inc() + } else { + webhookCounter.WithLabelValues("failure").Inc() + } + p, err := io.ReadAll(resp.Body) if err != nil { t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)