diff --git a/.golangci.yml b/.golangci.yml index 2ad39fbae2..483843bc55 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -153,6 +153,7 @@ linters: text: '(?i)exitAfterDefer:' paths: - node_modules + - .venv - public - web_src - third_party$ @@ -172,6 +173,7 @@ formatters: generated: lax paths: - node_modules + - .venv - public - web_src - third_party$ diff --git a/Dockerfile b/Dockerfile index a8415e8a18..d093aed46f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,11 +27,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \ if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \ && make -# Begin env-to-ini build -RUN --mount=type=cache,target=/go/pkg/mod \ - --mount=type=cache,target="/root/.cache/go-build" \ - go build contrib/environment-to-ini/environment-to-ini.go - FROM docker.io/library/alpine:3.22 AS gitea EXPOSE 22 3000 @@ -63,7 +58,6 @@ RUN addgroup \ COPY docker/root / COPY --chmod=755 --from=build-env /go/src/code.gitea.io/gitea/gitea /app/gitea/gitea -COPY --chmod=755 --from=build-env /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini ENV USER=git ENV GITEA_CUSTOM=/data/gitea diff --git a/Dockerfile.rootless b/Dockerfile.rootless index 2360c0741e..4b806c776c 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -27,10 +27,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \ if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \ && make -# Begin env-to-ini build -RUN --mount=type=cache,target=/go/pkg/mod \ - --mount=type=cache,target="/root/.cache/go-build" \ - go build contrib/environment-to-ini/environment-to-ini.go FROM docker.io/library/alpine:3.22 AS gitea-rootless @@ -62,7 +58,6 @@ RUN chown git:git /var/lib/gitea /etc/gitea COPY docker/rootless / COPY --from=build-env --chmod=755 --chown=root:root /go/src/code.gitea.io/gitea/gitea /app/gitea/gitea -COPY --from=build-env --chmod=755 --chown=root:root /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini # git:git USER 1000:1000 diff --git a/Makefile b/Makefile index fc507367e7..e81dab7f6c 100644 --- a/Makefile +++ b/Makefile @@ -31,11 +31,11 @@ XGO_VERSION := go-1.25.x AIR_PACKAGE ?= github.com/air-verse/air@v1 EDITORCONFIG_CHECKER_PACKAGE ?= github.com/editorconfig-checker/editorconfig-checker/v3/cmd/editorconfig-checker@v3 -GOFUMPT_PACKAGE ?= mvdan.cc/gofumpt@v0.9.1 -GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.4.0 +GOFUMPT_PACKAGE ?= mvdan.cc/gofumpt@v0.9.2 +GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.5.0 GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.15 MISSPELL_PACKAGE ?= github.com/golangci/misspell/cmd/misspell@v0.7.0 -SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@717e3cb29becaaf00e56953556c6d80f8a01b286 +SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@v0.33.1 XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest GO_LICENSES_PACKAGE ?= github.com/google/go-licenses@v1 GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1 @@ -258,7 +258,7 @@ clean: ## delete backend and integration files .PHONY: fmt fmt: ## format the Go and template code - @GOFUMPT_PACKAGE=$(GOFUMPT_PACKAGE) $(GO) run build/code-batch-process.go gitea-fmt -w '{file-list}' + @GOFUMPT_PACKAGE=$(GOFUMPT_PACKAGE) $(GO) run tools/code-batch-process.go gitea-fmt -w '{file-list}' $(eval TEMPLATES := $(shell find templates -type f -name '*.tmpl')) @# strip whitespace after '{{' or '(' and before '}}' or ')' unless there is only @# whitespace before it @@ -472,7 +472,7 @@ test\#%: coverage: grep '^\(mode: .*\)\|\(.*:[0-9]\+\.[0-9]\+,[0-9]\+\.[0-9]\+ [0-9]\+ [0-9]\+\)$$' coverage.out > coverage-bodged.out grep '^\(mode: .*\)\|\(.*:[0-9]\+\.[0-9]\+,[0-9]\+\.[0-9]\+ [0-9]\+ [0-9]\+\)$$' integration.coverage.out > integration.coverage-bodged.out - $(GO) run build/gocovmerge.go integration.coverage-bodged.out coverage-bodged.out > coverage.all + $(GO) run tools/gocovmerge.go integration.coverage-bodged.out coverage-bodged.out > coverage.all .PHONY: unit-test-coverage unit-test-coverage: diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000000..5303b0e1e0 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,156 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli/v3" +) + +func cmdConfig() *cli.Command { + subcmdConfigEditIni := &cli.Command{ + Name: "edit-ini", + Usage: "Load an existing INI file, apply environment variables, keep specified keys, and output to a new INI file.", + Description: ` +Help users to edit the Gitea configuration INI file. + +# Keep Specified Keys + +If you need to re-create the configuration file with only a subset of keys, +you can provide an INI template file for the kept keys and use the "--config-keep-keys" flag. +For example, if a helm chart needs to reset the settings and only keep SECRET_KEY, +it can use a template file (only keys take effect, values are ignored): + + [security] + SECRET_KEY= + +$ ./gitea config edit-ini --config app-old.ini --config-keep-keys app-keys.ini --out app-new.ini + +# Map Environment Variables to INI Configuration + +Environment variables of the form "GITEA__section_name__KEY_NAME" +will be mapped to the ini section "[section_name]" and the key +"KEY_NAME" with the value as provided. + +Environment variables of the form "GITEA__section_name__KEY_NAME__FILE" +will be mapped to the ini section "[section_name]" and the key +"KEY_NAME" with the value loaded from the specified file. + +Environment variable keys can only contain characters "0-9A-Z_", +if a section or key name contains dot ".", it needs to be escaped as _0x2E_. +For example, to apply this config: + + [git.config] + foo.bar=val + +$ export GITEA__git_0x2E_config__foo_0x2E_bar=val + +# Put All Together + +$ ./gitea config edit-ini --config app.ini --config-keep-keys app-keys.ini --apply-env {--in-place|--out app-new.ini} +`, + Flags: []cli.Flag{ + // "--config" flag is provided by global flags, and this flag is also used by "environment-to-ini" script wrapper + // "--in-place" is also used by "environment-to-ini" script wrapper for its old behavior: always overwrite the existing config file + &cli.BoolFlag{ + Name: "in-place", + Usage: "Output to the same config file as input. This flag will be ignored if --out is set.", + }, + &cli.StringFlag{ + Name: "config-keep-keys", + Usage: "An INI template file containing keys for keeping. Only the keys defined in the INI template will be kept from old config. If not set, all keys will be kept.", + }, + &cli.BoolFlag{ + Name: "apply-env", + Usage: "Apply all GITEA__* variables from the environment to the config.", + }, + &cli.StringFlag{ + Name: "out", + Usage: "Destination config file to write to.", + }, + }, + Action: runConfigEditIni, + } + + return &cli.Command{ + Name: "config", + Usage: "Manage Gitea configuration", + Commands: []*cli.Command{ + subcmdConfigEditIni, + }, + } +} + +func runConfigEditIni(_ context.Context, c *cli.Command) error { + // the config system may change the environment variables, so get a copy first, to be used later + env := append([]string{}, os.Environ()...) + + // don't use the guessed setting.CustomConf, instead, require the user to provide --config explicitly + if !c.IsSet("config") { + return errors.New("flag is required but not set: --config") + } + configFileIn := c.String("config") + + cfgIn, err := setting.NewConfigProviderFromFile(configFileIn) + if err != nil { + return fmt.Errorf("failed to load config file %q: %v", configFileIn, err) + } + + // determine output config file: use "--out" flag or use "--in-place" flag to overwrite input file + inPlace := c.Bool("in-place") + configFileOut := c.String("out") + if configFileOut == "" { + if !inPlace { + return errors.New("either --in-place or --out must be specified") + } + configFileOut = configFileIn // in-place edit + } + + needWriteOut := configFileOut != configFileIn + + cfgOut := cfgIn + configKeepKeys := c.String("config-keep-keys") + if configKeepKeys != "" { + needWriteOut = true + cfgOut, err = setting.NewConfigProviderFromFile(configKeepKeys) + if err != nil { + return fmt.Errorf("failed to load config-keep-keys template file %q: %v", configKeepKeys, err) + } + + for _, secOut := range cfgOut.Sections() { + for _, keyOut := range secOut.Keys() { + secIn := cfgIn.Section(secOut.Name()) + keyIn := setting.ConfigSectionKey(secIn, keyOut.Name()) + if keyIn != nil { + keyOut.SetValue(keyIn.String()) + } else { + secOut.DeleteKey(keyOut.Name()) + } + } + if len(secOut.Keys()) == 0 { + cfgOut.DeleteSection(secOut.Name()) + } + } + } + + if c.Bool("apply-env") { + if setting.EnvironmentToConfig(cfgOut, env) { + needWriteOut = true + } + } + + if needWriteOut { + err = cfgOut.SaveTo(configFileOut) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/config_test.go b/cmd/config_test.go new file mode 100644 index 0000000000..d123daa617 --- /dev/null +++ b/cmd/config_test.go @@ -0,0 +1,85 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigEdit(t *testing.T) { + tmpDir := t.TempDir() + configOld := tmpDir + "/app-old.ini" + configTemplate := tmpDir + "/app-template.ini" + _ = os.WriteFile(configOld, []byte(` +[sec] +k1=v1 +k2=v2 +`), os.ModePerm) + + _ = os.WriteFile(configTemplate, []byte(` +[sec] +k1=in-template + +[sec2] +k3=v3 +`), os.ModePerm) + + t.Setenv("GITEA__EnV__KeY", "val") + + t.Run("OutputToNewWithEnv", func(t *testing.T) { + configNew := tmpDir + "/app-new.ini" + err := NewMainApp(AppVersion{}).Run(t.Context(), []string{ + "./gitea", "--config", configOld, + "config", "edit-ini", + "--apply-env", + "--config-keep-keys", configTemplate, + "--out", configNew, + }) + require.NoError(t, err) + + // "k1" old value is kept because its key is in the template + // "k2" is removed because it isn't in the template + // "k3" isn't in new config because it isn't in the old config + // [env] is applied from environment variable + data, _ := os.ReadFile(configNew) + require.Equal(t, `[sec] +k1 = v1 + +[env] +KeY = val +`, string(data)) + }) + + t.Run("OutputToExisting(environment-to-ini)", func(t *testing.T) { + // the legacy "environment-to-ini" (now a wrapper script) behavior: + // if no "--out", then "--in-place" must be used to overwrite the existing "--config" file + err := NewMainApp(AppVersion{}).Run(t.Context(), []string{ + "./gitea", "config", "edit-ini", + "--apply-env", + "--config", configOld, + }) + require.ErrorContains(t, err, "either --in-place or --out must be specified") + + // simulate the "environment-to-ini" behavior with "--in-place" + err = NewMainApp(AppVersion{}).Run(t.Context(), []string{ + "./gitea", "config", "edit-ini", + "--in-place", + "--apply-env", + "--config", configOld, + }) + require.NoError(t, err) + + data, _ := os.ReadFile(configOld) + require.Equal(t, `[sec] +k1 = v1 +k2 = v2 + +[env] +KeY = val +`, string(data)) + }) +} diff --git a/cmd/main.go b/cmd/main.go index 3fdaf48ed9..3a38d675a1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -128,6 +128,7 @@ func NewMainApp(appVer AppVersion) *cli.Command { // these sub-commands do not need the config file, and they do not depend on any path or environment variable. subCmdStandalone := []*cli.Command{ + cmdConfig(), cmdCert(), CmdGenerate, CmdDocs, diff --git a/cmd/serv.go b/cmd/serv.go index 76d8c81544..72ca7c4a00 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -13,7 +13,6 @@ import ( "path/filepath" "strconv" "strings" - "time" "unicode" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -32,7 +31,6 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/lfs" - "github.com/golang-jwt/jwt/v5" "github.com/kballard/go-shellquote" "github.com/urfave/cli/v3" ) @@ -133,27 +131,6 @@ func getAccessMode(verb, lfsVerb string) perm.AccessMode { return perm.AccessModeNone } -func getLFSAuthToken(ctx context.Context, lfsVerb string, results *private.ServCommandResults) (string, error) { - now := time.Now() - claims := lfs.Claims{ - RegisteredClaims: jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)), - NotBefore: jwt.NewNumericDate(now), - }, - RepoID: results.RepoID, - Op: lfsVerb, - UserID: results.UserID, - } - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - - // Sign and get the complete encoded token as a string using the secret - tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes) - if err != nil { - return "", fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err) - } - return "Bearer " + tokenString, nil -} - func runServ(ctx context.Context, c *cli.Command) error { // FIXME: This needs to internationalised setup(ctx, c.Bool("debug")) @@ -283,7 +260,7 @@ func runServ(ctx context.Context, c *cli.Command) error { // LFS SSH protocol if verb == git.CmdVerbLfsTransfer { - token, err := getLFSAuthToken(ctx, lfsVerb, results) + token, err := lfs.GetLFSAuthTokenWithBearer(lfs.AuthTokenOptions{Op: lfsVerb, UserID: results.UserID, RepoID: results.RepoID}) if err != nil { return err } @@ -294,7 +271,7 @@ func runServ(ctx context.Context, c *cli.Command) error { if verb == git.CmdVerbLfsAuthenticate { url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName)) - token, err := getLFSAuthToken(ctx, lfsVerb, results) + token, err := lfs.GetLFSAuthTokenWithBearer(lfs.AuthTokenOptions{Op: lfsVerb, UserID: results.UserID, RepoID: results.RepoID}) if err != nil { return err } diff --git a/cmd/web.go b/cmd/web.go index 4723ddbbdd..6e39db2178 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -156,7 +156,6 @@ func serveInstall(cmd *cli.Command) error { case <-graceful.GetManager().IsShutdown(): <-graceful.GetManager().Done() log.Info("PID: %d Gitea Web Finished", os.Getpid()) - log.GetManager().Close() return err default: } @@ -231,7 +230,6 @@ func serveInstalled(c *cli.Command) error { err := listen(webRoutes, true) <-graceful.GetManager().Done() log.Info("PID: %d Gitea Web Finished", os.Getpid()) - log.GetManager().Close() return err } diff --git a/contrib/environment-to-ini/README b/contrib/environment-to-ini/README deleted file mode 100644 index f1d3f2ae83..0000000000 --- a/contrib/environment-to-ini/README +++ /dev/null @@ -1,47 +0,0 @@ -Environment To Ini -================== - -Multiple docker users have requested that the Gitea docker is changed -to permit arbitrary configuration via environment variables. - -Gitea needs to use an ini file for configuration because the running -environment that starts the docker may not be the same as that used -by the hooks. An ini file also gives a good default and means that -users do not have to completely provide a full environment. - -With those caveats above, this command provides a generic way of -converting suitably structured environment variables into any ini -value. - -To use the command is very simple just run it and the default gitea -app.ini will be rewritten to take account of the variables provided, -however there are various options to give slightly different -behavior and these can be interrogated with the `-h` option. - -The environment variables should be of the form: - - GITEA__SECTION_NAME__KEY_NAME - -Note, SECTION_NAME in the notation above is case-insensitive. - -Environment variables are usually restricted to a reduced character -set "0-9A-Z_" - in order to allow the setting of sections with -characters outside of that set, they should be escaped as following: -"_0X2E_" for "." and "_0X2D_" for "-". The entire section and key names -can be escaped as a UTF8 byte string if necessary. E.g. to configure: - - """ - ... - [log.console] - COLORIZE=false - STDERR=true - ... - """ - -You would set the environment variables: "GITEA__LOG_0x2E_CONSOLE__COLORIZE=false" -and "GITEA__LOG_0x2E_CONSOLE__STDERR=false". Other examples can be found -on the configuration cheat sheet. - -To build locally, run: - - go build contrib/environment-to-ini/environment-to-ini.go diff --git a/contrib/environment-to-ini/environment-to-ini.go b/contrib/environment-to-ini/environment-to-ini.go deleted file mode 100644 index 5eb576c6fe..0000000000 --- a/contrib/environment-to-ini/environment-to-ini.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package main - -import ( - "context" - "os" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" - - "github.com/urfave/cli/v3" -) - -func main() { - app := cli.Command{} - app.Name = "environment-to-ini" - app.Usage = "Use provided environment to update configuration ini" - app.Description = `As a helper to allow docker users to update the gitea configuration - through the environment, this command allows environment variables to - be mapped to values in the ini. - - Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME" - will be mapped to the ini section "[section_name]" and the key - "KEY_NAME" with the value as provided. - - Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME__FILE" - will be mapped to the ini section "[section_name]" and the key - "KEY_NAME" with the value loaded from the specified file. - - Environment variables are usually restricted to a reduced character - set "0-9A-Z_" - in order to allow the setting of sections with - characters outside of that set, they should be escaped as following: - "_0X2E_" for ".". The entire section and key names can be escaped as - a UTF8 byte string if necessary. E.g. to configure: - - """ - ... - [log.console] - COLORIZE=false - STDERR=true - ... - """ - - You would set the environment variables: "GITEA__LOG_0x2E_CONSOLE__COLORIZE=false" - and "GITEA__LOG_0x2E_CONSOLE__STDERR=false". Other examples can be found - on the configuration cheat sheet.` - app.Flags = []cli.Flag{ - &cli.StringFlag{ - Name: "custom-path", - Aliases: []string{"C"}, - Value: setting.CustomPath, - Usage: "Custom path file path", - }, - &cli.StringFlag{ - Name: "config", - Aliases: []string{"c"}, - Value: setting.CustomConf, - Usage: "Custom configuration file path", - }, - &cli.StringFlag{ - Name: "work-path", - Aliases: []string{"w"}, - Value: setting.AppWorkPath, - Usage: "Set the gitea working path", - }, - &cli.StringFlag{ - Name: "out", - Aliases: []string{"o"}, - Value: "", - Usage: "Destination file to write to", - }, - } - app.Action = runEnvironmentToIni - err := app.Run(context.Background(), os.Args) - if err != nil { - log.Fatal("Failed to run app with %s: %v", os.Args, err) - } -} - -func runEnvironmentToIni(_ context.Context, c *cli.Command) error { - // the config system may change the environment variables, so get a copy first, to be used later - env := append([]string{}, os.Environ()...) - setting.InitWorkPathAndCfgProvider(os.Getenv, setting.ArgWorkPathAndCustomConf{ - WorkPath: c.String("work-path"), - CustomPath: c.String("custom-path"), - CustomConf: c.String("config"), - }) - - cfg, err := setting.NewConfigProviderFromFile(setting.CustomConf) - if err != nil { - log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err) - } - - changed := setting.EnvironmentToConfig(cfg, env) - - // try to save the config file - destination := c.String("out") - if len(destination) == 0 { - destination = setting.CustomConf - } - if destination != setting.CustomConf || changed { - log.Info("Settings saved to: %q", destination) - err = cfg.SaveTo(destination) - if err != nil { - return err - } - } - - return nil -} diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index aa2fcee765..5fee78af54 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1343,6 +1343,10 @@ LEVEL = Info ;; Dont mistake it for Reactions. ;CUSTOM_EMOJIS = gitea, codeberg, gitlab, git, github, gogs ;; +;; Comma separated list of enabled emojis, for example: smile, thumbsup, thumbsdown +;; Leave it empty to enable all emojis. +;ENABLED_EMOJIS = +;; ;; Whether the full name of the users should be shown where possible. If the full name isn't set, the username will be used. ;DEFAULT_SHOW_FULL_NAME = false ;; @@ -2536,7 +2540,19 @@ LEVEL = Info ;; * sanitized: Sanitize the content and render it inside current page, default to only allow a few HTML tags and attributes. Customized sanitizer rules can be defined in [markup.sanitizer.*] . ;; * no-sanitizer: Disable the sanitizer and render the content inside current page. It's **insecure** and may lead to XSS attack if the content contains malicious code. ;; * iframe: Render the content in a separate standalone page and embed it into current page by iframe. The iframe is in sandbox mode with same-origin disabled, and the JS code are safely isolated from parent page. -;RENDER_CONTENT_MODE=sanitized +;RENDER_CONTENT_MODE = sanitized +;; The sandbox applied to the iframe and Content-Security-Policy header when RENDER_CONTENT_MODE is `iframe`. +;; It defaults to a safe set of "allow-*" restrictions (space separated). +;; You can also set it by your requirements or use "disabled" to disable the sandbox completely. +;; When set it, make sure there is no security risk: +;; * PDF-only content: generally safe to use "disabled", and it needs to be "disabled" because PDF only renders with no sandbox. +;; * HTML content with JS: if the "RENDER_COMMAND" can guarantee there is no XSS, then it is safe, otherwise, you need to fine tune the "allow-*" restrictions. +;RENDER_CONTENT_SANDBOX = +;; Whether post-process the rendered HTML content, including: +;; resolve relative links and image sources, recognizing issue/commit references, escaping invisible characters, +;; mentioning users, rendering permlink code blocks, replacing emoji shorthands, etc. +;; By default, this is true when RENDER_CONTENT_MODE is `sanitized`, otherwise false. +;NEED_POST_PROCESS = false ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docker/root/usr/local/bin/environment-to-ini b/docker/root/usr/local/bin/environment-to-ini new file mode 100644 index 0000000000..bb0c540685 --- /dev/null +++ b/docker/root/usr/local/bin/environment-to-ini @@ -0,0 +1,2 @@ +#!/bin/bash +exec /app/gitea/gitea config edit-ini --in-place --apply-env "$@" diff --git a/docker/rootless/usr/local/bin/environment-to-ini b/docker/rootless/usr/local/bin/environment-to-ini new file mode 100644 index 0000000000..bb0c540685 --- /dev/null +++ b/docker/rootless/usr/local/bin/environment-to-ini @@ -0,0 +1,2 @@ +#!/bin/bash +exec /app/gitea/gitea config edit-ini --in-place --apply-env "$@" diff --git a/eslint.config.ts b/eslint.config.ts index 02aacefca2..d9c4bcae3a 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -49,6 +49,7 @@ export default defineConfig([ }, linterOptions: { reportUnusedDisableDirectives: 2, + reportUnusedInlineConfigs: 2, }, plugins: { '@eslint-community/eslint-comments': comments, diff --git a/go.mod b/go.mod index cf4774801e..81187804a3 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( github.com/ulikunitz/xz v0.5.15 github.com/urfave/cli-docs/v3 v3.0.0-alpha6 github.com/urfave/cli/v3 v3.4.1 - github.com/wneessen/go-mail v0.7.1 + github.com/wneessen/go-mail v0.7.2 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yohcop/openid-go v1.0.1 github.com/yuin/goldmark v1.7.13 diff --git a/go.sum b/go.sum index 9acef3b977..02a710e7f0 100644 --- a/go.sum +++ b/go.sum @@ -768,8 +768,8 @@ github.com/urfave/cli/v3 v3.4.1/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZ github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/wneessen/go-mail v0.7.1 h1:rvy63sp14N06/kdGqCYwW8Na5gDCXjTQM1E7So4PuKk= -github.com/wneessen/go-mail v0.7.1/go.mod h1:+TkW6QP3EVkgTEqHtVmnAE/1MRhmzb8Y9/W3pweuS+k= +github.com/wneessen/go-mail v0.7.2 h1:xxPnhZ6IZLSgxShebmZ6DPKh1b6OJcoHfzy7UjOkzS8= +github.com/wneessen/go-mail v0.7.2/go.mod h1:+TkW6QP3EVkgTEqHtVmnAE/1MRhmzb8Y9/W3pweuS+k= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= diff --git a/main.go b/main.go index 2c25bac4e3..bc2121b1e7 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,7 @@ func main() { } app := cmd.NewMainApp(cmd.AppVersion{Version: Version, Extra: formatBuiltWith()}) _ = cmd.RunMainApp(app, os.Args...) // all errors should have been handled by the RunMainApp + // flush the queued logs before exiting, it is a MUST, otherwise there will be log loss log.GetManager().Close() } diff --git a/models/actions/runner.go b/models/actions/runner.go index 81d4249ae0..84398b143b 100644 --- a/models/actions/runner.go +++ b/models/actions/runner.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/shared/types" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -173,6 +174,13 @@ func (r *ActionRunner) GenerateToken() (err error) { return err } +// CanMatchLabels checks whether the runner's labels can match a job's "runs-on" +// See https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idruns-on +func (r *ActionRunner) CanMatchLabels(jobRunsOn []string) bool { + runnerLabelSet := container.SetOf(r.AgentLabels...) + return runnerLabelSet.Contains(jobRunsOn...) // match all labels +} + func init() { db.RegisterModel(&ActionRunner{}) } diff --git a/models/actions/task.go b/models/actions/task.go index 7417af8b45..8b4ecf28f7 100644 --- a/models/actions/task.go +++ b/models/actions/task.go @@ -13,7 +13,6 @@ import ( auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unit" - "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -245,7 +244,7 @@ func CreateTaskForRunner(ctx context.Context, runner *ActionRunner) (*ActionTask var job *ActionRunJob log.Trace("runner labels: %v", runner.AgentLabels) for _, v := range jobs { - if isSubset(runner.AgentLabels, v.RunsOn) { + if runner.CanMatchLabels(v.RunsOn) { job = v break } @@ -475,20 +474,6 @@ func FindOldTasksToExpire(ctx context.Context, olderThan timeutil.TimeStamp, lim Find(&tasks) } -func isSubset(set, subset []string) bool { - m := make(container.Set[string], len(set)) - for _, v := range set { - m.Add(v) - } - - for _, v := range subset { - if !m.Contains(v) { - return false - } - } - return true -} - func convertTimestamp(timestamp *timestamppb.Timestamp) timeutil.TimeStamp { if timestamp.GetSeconds() == 0 && timestamp.GetNanos() == 0 { return timeutil.TimeStamp(0) diff --git a/models/admin/task.go b/models/admin/task.go index 0541a8ec78..5d2b9bbff6 100644 --- a/models/admin/task.go +++ b/models/admin/task.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/secret" "code.gitea.io/gitea/modules/setting" @@ -123,17 +124,17 @@ func (task *Task) MigrateConfig() (*migration.MigrateOptions, error) { // decrypt credentials if opts.CloneAddrEncrypted != "" { if opts.CloneAddr, err = secret.DecryptSecret(setting.SecretKey, opts.CloneAddrEncrypted); err != nil { - return nil, err + log.Error("Unable to decrypt CloneAddr, maybe SECRET_KEY is wrong: %v", err) } } if opts.AuthPasswordEncrypted != "" { if opts.AuthPassword, err = secret.DecryptSecret(setting.SecretKey, opts.AuthPasswordEncrypted); err != nil { - return nil, err + log.Error("Unable to decrypt AuthPassword, maybe SECRET_KEY is wrong: %v", err) } } if opts.AuthTokenEncrypted != "" { if opts.AuthToken, err = secret.DecryptSecret(setting.SecretKey, opts.AuthTokenEncrypted); err != nil { - return nil, err + log.Error("Unable to decrypt AuthToken, maybe SECRET_KEY is wrong: %v", err) } } diff --git a/models/asymkey/ssh_key.go b/models/asymkey/ssh_key.go index 87205f0651..d77b5d46a7 100644 --- a/models/asymkey/ssh_key.go +++ b/models/asymkey/ssh_key.go @@ -67,13 +67,6 @@ func (key *PublicKey) OmitEmail() string { return strings.Join(strings.Split(key.Content, " ")[:2], " ") } -// AuthorizedString returns formatted public key string for authorized_keys file. -// -// TODO: Consider dropping this function -func (key *PublicKey) AuthorizedString() string { - return AuthorizedStringForKey(key) -} - func addKey(ctx context.Context, key *PublicKey) (err error) { if len(key.Fingerprint) == 0 { key.Fingerprint, err = CalcFingerprint(key.Content) diff --git a/models/asymkey/ssh_key_authorized_keys.go b/models/asymkey/ssh_key_authorized_keys.go index 2e4cd62e5c..db4730f00a 100644 --- a/models/asymkey/ssh_key_authorized_keys.go +++ b/models/asymkey/ssh_key_authorized_keys.go @@ -17,29 +17,13 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + + "golang.org/x/crypto/ssh" ) -// _____ __ .__ .__ .___ -// / _ \ __ ___/ |_| |__ ___________|__|_______ ____ __| _/ -// / /_\ \| | \ __\ | \ / _ \_ __ \ \___ // __ \ / __ | -// / | \ | /| | | Y ( <_> ) | \/ |/ /\ ___// /_/ | -// \____|__ /____/ |__| |___| /\____/|__| |__/_____ \\___ >____ | -// \/ \/ \/ \/ \/ -// ____ __. -// | |/ _|____ ___.__. ______ -// | <_/ __ < | |/ ___/ -// | | \ ___/\___ |\___ \ -// |____|__ \___ > ____/____ > -// \/ \/\/ \/ -// -// This file contains functions for creating authorized_keys files -// -// There is a dependence on the database within RegeneratePublicKeys however most of these functions probably belong in a module - -const ( - tplCommentPrefix = `# gitea public key` - tplPublicKey = tplCommentPrefix + "\n" + `command=%s,no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,no-user-rc,restrict %s` + "\n" -) +// AuthorizedStringCommentPrefix is a magic tag +// some functions like RegeneratePublicKeys needs this tag to skip the keys generated by Gitea, while keep other keys +const AuthorizedStringCommentPrefix = `# gitea public key` var sshOpLocker sync.Mutex @@ -50,17 +34,45 @@ func WithSSHOpLocker(f func() error) error { } // AuthorizedStringForKey creates the authorized keys string appropriate for the provided key -func AuthorizedStringForKey(key *PublicKey) string { +func AuthorizedStringForKey(key *PublicKey) (string, error) { sb := &strings.Builder{} - _ = setting.SSH.AuthorizedKeysCommandTemplateTemplate.Execute(sb, map[string]any{ + _, err := writeAuthorizedStringForKey(key, sb) + return sb.String(), err +} + +// WriteAuthorizedStringForValidKey writes the authorized key for the provided key. If the key is invalid, it does nothing. +func WriteAuthorizedStringForValidKey(key *PublicKey, w io.Writer) error { + validKey, err := writeAuthorizedStringForKey(key, w) + if !validKey { + log.Debug("WriteAuthorizedStringForValidKey: key %s is not valid: %v", key, err) + return nil + } + return err +} + +func writeAuthorizedStringForKey(key *PublicKey, w io.Writer) (keyValid bool, err error) { + const tpl = AuthorizedStringCommentPrefix + "\n" + `command=%s,no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,no-user-rc,restrict %s %s` + "\n" + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(key.Content)) + if err != nil { + return false, err + } + // now the key is valid, the code below could only return template/IO related errors + sbCmd := &strings.Builder{} + err = setting.SSH.AuthorizedKeysCommandTemplateTemplate.Execute(sbCmd, map[string]any{ "AppPath": util.ShellEscape(setting.AppPath), "AppWorkPath": util.ShellEscape(setting.AppWorkPath), "CustomConf": util.ShellEscape(setting.CustomConf), "CustomPath": util.ShellEscape(setting.CustomPath), "Key": key, }) - - return fmt.Sprintf(tplPublicKey, util.ShellEscape(sb.String()), key.Content) + if err != nil { + return true, err + } + sshCommandEscaped := util.ShellEscape(sbCmd.String()) + sshKeyMarshalled := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(pubKey))) + sshKeyComment := fmt.Sprintf("user-%d", key.OwnerID) + _, err = fmt.Fprintf(w, tpl, sshCommandEscaped, sshKeyMarshalled, sshKeyComment) + return true, err } // appendAuthorizedKeysToFile appends new SSH keys' content to authorized_keys file. @@ -112,7 +124,7 @@ func appendAuthorizedKeysToFile(keys ...*PublicKey) error { if key.Type == KeyTypePrincipal { continue } - if _, err = f.WriteString(key.AuthorizedString()); err != nil { + if err = WriteAuthorizedStringForValidKey(key, f); err != nil { return err } } @@ -120,10 +132,9 @@ func appendAuthorizedKeysToFile(keys ...*PublicKey) error { } // RegeneratePublicKeys regenerates the authorized_keys file -func RegeneratePublicKeys(ctx context.Context, t io.StringWriter) error { +func RegeneratePublicKeys(ctx context.Context, t io.Writer) error { if err := db.GetEngine(ctx).Where("type != ?", KeyTypePrincipal).Iterate(new(PublicKey), func(idx int, bean any) (err error) { - _, err = t.WriteString((bean.(*PublicKey)).AuthorizedString()) - return err + return WriteAuthorizedStringForValidKey(bean.(*PublicKey), t) }); err != nil { return err } @@ -144,11 +155,11 @@ func RegeneratePublicKeys(ctx context.Context, t io.StringWriter) error { scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() - if strings.HasPrefix(line, tplCommentPrefix) { + if strings.HasPrefix(line, AuthorizedStringCommentPrefix) { scanner.Scan() continue } - _, err = t.WriteString(line + "\n") + _, err = io.WriteString(t, line+"\n") if err != nil { return err } diff --git a/models/auth/twofactor.go b/models/auth/twofactor.go index 200ce7c7c0..4263495650 100644 --- a/models/auth/twofactor.go +++ b/models/auth/twofactor.go @@ -111,11 +111,11 @@ func (t *TwoFactor) SetSecret(secretString string) error { func (t *TwoFactor) ValidateTOTP(passcode string) (bool, error) { decodedStoredSecret, err := base64.StdEncoding.DecodeString(t.Secret) if err != nil { - return false, err + return false, fmt.Errorf("ValidateTOTP invalid base64: %w", err) } secretBytes, err := secret.AesDecrypt(t.getEncryptionKey(), decodedStoredSecret) if err != nil { - return false, err + return false, fmt.Errorf("ValidateTOTP unable to decrypt (maybe SECRET_KEY is wrong): %w", err) } secretStr := string(secretBytes) return totp.Validate(passcode, secretStr), nil diff --git a/models/fixtures/action_run.yml b/models/fixtures/action_run.yml index 09dfa6cccb..b9688dd5f5 100644 --- a/models/fixtures/action_run.yml +++ b/models/fixtures/action_run.yml @@ -139,3 +139,23 @@ updated: 1683636626 need_approval: 0 approved_by: 0 +- + id: 804 + title: "use a private action" + repo_id: 60 + owner_id: 40 + workflow_id: "run.yaml" + index: 189 + trigger_user_id: 40 + ref: "refs/heads/master" + commit_sha: "6e64b26de7ba966d01d90ecfaf5c7f14ef203e86" + event: "push" + trigger_event: "push" + is_fork_pull_request: 0 + status: 1 + started: 1683636528 + stopped: 1683636626 + created: 1683636108 + updated: 1683636626 + need_approval: 0 + approved_by: 0 diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml index 6c06d94aa4..337e83605a 100644 --- a/models/fixtures/action_run_job.yml +++ b/models/fixtures/action_run_job.yml @@ -129,3 +129,17 @@ status: 5 started: 1683636528 stopped: 1683636626 +- + id: 205 + run_id: 804 + repo_id: 6 + owner_id: 10 + commit_sha: 6e64b26de7ba966d01d90ecfaf5c7f14ef203e86 + is_fork_pull_request: 0 + name: job_2 + attempt: 1 + job_id: job_2 + task_id: 48 + status: 1 + started: 1683636528 + stopped: 1683636626 diff --git a/models/fixtures/action_task.yml b/models/fixtures/action_task.yml index c79fb07050..e09fd6f2ec 100644 --- a/models/fixtures/action_task.yml +++ b/models/fixtures/action_task.yml @@ -177,3 +177,23 @@ log_length: 0 log_size: 0 log_expired: 0 +- + id: 55 + job_id: 205 + attempt: 1 + runner_id: 1 + status: 6 # 6 is the status code for "running" + started: 1683636528 + stopped: 1683636626 + repo_id: 6 + owner_id: 10 + commit_sha: 6e64b26de7ba966d01d90ecfaf5c7f14ef203e86 + is_fork_pull_request: 0 + token_hash: b8d3962425466b6709b9ac51446f93260c54afe8e7b6d3686e34f991fb8a8953822b0deed86fe41a103f34bc48dbc478422b + token_salt: ERxJGHvg3I + token_last_eight: 182199eb + log_filename: collaborative-owner-test/1a/49.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 diff --git a/models/fixtures/branch.yml b/models/fixtures/branch.yml index 717230149b..a17999091e 100644 --- a/models/fixtures/branch.yml +++ b/models/fixtures/branch.yml @@ -225,3 +225,27 @@ is_deleted: false deleted_by_id: 0 deleted_unix: 0 + +- + id: 27 + repo_id: 1 + name: 'DefaultBranch' + commit_id: '90c1019714259b24fb81711d4416ac0f18667dfa' + commit_message: 'add license' + commit_time: 1709345946 + pusher_id: 1 + is_deleted: false + deleted_by_id: 0 + deleted_unix: 0 + +- + id: 28 + repo_id: 1 + name: 'sub-home-md-img-check' + commit_id: '4649299398e4d39a5c09eb4f534df6f1e1eb87cc' + commit_message: "Test how READMEs render images when found in a subfolder" + commit_time: 1678403550 + pusher_id: 1 + is_deleted: false + deleted_by_id: 0 + deleted_unix: 0 diff --git a/models/fixtures/repo_unit.yml b/models/fixtures/repo_unit.yml index f6b6252da1..f8bb8ef0d3 100644 --- a/models/fixtures/repo_unit.yml +++ b/models/fixtures/repo_unit.yml @@ -733,3 +733,10 @@ type: 3 config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}" created_unix: 946684810 + +- + id: 111 + repo_id: 3 + type: 10 + config: "{}" + created_unix: 946684810 diff --git a/models/git/commit_status.go b/models/git/commit_status.go index e255bca5d0..2ae5937a3d 100644 --- a/models/git/commit_status.go +++ b/models/git/commit_status.go @@ -30,17 +30,21 @@ import ( // CommitStatus holds a single Status of a single Commit type CommitStatus struct { - ID int64 `xorm:"pk autoincr"` - Index int64 `xorm:"INDEX UNIQUE(repo_sha_index)"` - RepoID int64 `xorm:"INDEX UNIQUE(repo_sha_index)"` - Repo *repo_model.Repository `xorm:"-"` - State commitstatus.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"` - SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"` - TargetURL string `xorm:"TEXT"` - Description string `xorm:"TEXT"` - ContextHash string `xorm:"VARCHAR(64) index"` - Context string `xorm:"TEXT"` - Creator *user_model.User `xorm:"-"` + ID int64 `xorm:"pk autoincr"` + Index int64 `xorm:"INDEX UNIQUE(repo_sha_index)"` + RepoID int64 `xorm:"INDEX UNIQUE(repo_sha_index)"` + Repo *repo_model.Repository `xorm:"-"` + State commitstatus.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"` + SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"` + + // TargetURL points to the commit status page reported by a CI system + // If Gitea Actions is used, it is a relative link like "{RepoLink}/actions/runs/{RunID}/jobs{JobID}" + TargetURL string `xorm:"TEXT"` + + Description string `xorm:"TEXT"` + ContextHash string `xorm:"VARCHAR(64) index"` + Context string `xorm:"TEXT"` + Creator *user_model.User `xorm:"-"` CreatorID int64 CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` @@ -211,21 +215,45 @@ func (status *CommitStatus) LocaleString(lang translation.Locale) string { // HideActionsURL set `TargetURL` to an empty string if the status comes from Gitea Actions func (status *CommitStatus) HideActionsURL(ctx context.Context) { + if _, ok := status.cutTargetURLGiteaActionsPrefix(ctx); ok { + status.TargetURL = "" + } +} + +func (status *CommitStatus) cutTargetURLGiteaActionsPrefix(ctx context.Context) (string, bool) { if status.RepoID == 0 { - return + return "", false } if status.Repo == nil { if err := status.loadRepository(ctx); err != nil { log.Error("loadRepository: %v", err) - return + return "", false } } prefix := status.Repo.Link() + "/actions" - if strings.HasPrefix(status.TargetURL, prefix) { - status.TargetURL = "" + return strings.CutPrefix(status.TargetURL, prefix) +} + +// ParseGiteaActionsTargetURL parses the commit status target URL as Gitea Actions link +func (status *CommitStatus) ParseGiteaActionsTargetURL(ctx context.Context) (runID, jobID int64, ok bool) { + s, ok := status.cutTargetURLGiteaActionsPrefix(ctx) + if !ok { + return 0, 0, false } + + parts := strings.Split(s, "/") // expect: /runs/{runID}/jobs/{jobID} + if len(parts) < 5 || parts[1] != "runs" || parts[3] != "jobs" { + return 0, 0, false + } + + runID, err1 := strconv.ParseInt(parts[2], 10, 64) + jobID, err2 := strconv.ParseInt(parts[4], 10, 64) + if err1 != nil || err2 != nil { + return 0, 0, false + } + return runID, jobID, true } // CalcCommitStatus returns commit status state via some status, the commit statues should order by id desc diff --git a/models/git/lfs.go b/models/git/lfs.go index 8bba060ff9..a4ae3e7bee 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -8,7 +8,6 @@ import ( "fmt" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" @@ -42,30 +41,6 @@ func (err ErrLFSLockNotExist) Unwrap() error { return util.ErrNotExist } -// ErrLFSUnauthorizedAction represents a "LFSUnauthorizedAction" kind of error. -type ErrLFSUnauthorizedAction struct { - RepoID int64 - UserName string - Mode perm.AccessMode -} - -// IsErrLFSUnauthorizedAction checks if an error is a ErrLFSUnauthorizedAction. -func IsErrLFSUnauthorizedAction(err error) bool { - _, ok := err.(ErrLFSUnauthorizedAction) - return ok -} - -func (err ErrLFSUnauthorizedAction) Error() string { - if err.Mode == perm.AccessModeWrite { - return fmt.Sprintf("User %s doesn't have write access for lfs lock [rid: %d]", err.UserName, err.RepoID) - } - return fmt.Sprintf("User %s doesn't have read access for lfs lock [rid: %d]", err.UserName, err.RepoID) -} - -func (err ErrLFSUnauthorizedAction) Unwrap() error { - return util.ErrPermissionDenied -} - // ErrLFSLockAlreadyExist represents a "LFSLockAlreadyExist" kind of error. type ErrLFSLockAlreadyExist struct { RepoID int64 @@ -93,12 +68,6 @@ type ErrLFSFileLocked struct { UserName string } -// IsErrLFSFileLocked checks if an error is a ErrLFSFileLocked. -func IsErrLFSFileLocked(err error) bool { - _, ok := err.(ErrLFSFileLocked) - return ok -} - func (err ErrLFSFileLocked) Error() string { return fmt.Sprintf("File is lfs locked [repo: %d, locked by: %s, path: %s]", err.RepoID, err.UserName, err.Path) } diff --git a/models/git/lfs_lock.go b/models/git/lfs_lock.go index c5f9a4e6de..184e616915 100644 --- a/models/git/lfs_lock.go +++ b/models/git/lfs_lock.go @@ -11,10 +11,7 @@ import ( "time" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/perm" - access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -71,10 +68,6 @@ func (l *LFSLock) LoadOwner(ctx context.Context) error { // CreateLFSLock creates a new lock. func CreateLFSLock(ctx context.Context, repo *repo_model.Repository, lock *LFSLock) (*LFSLock, error) { return db.WithTx2(ctx, func(ctx context.Context) (*LFSLock, error) { - if err := CheckLFSAccessForRepo(ctx, lock.OwnerID, repo, perm.AccessModeWrite); err != nil { - return nil, err - } - lock.Path = util.PathJoinRel(lock.Path) lock.RepoID = repo.ID @@ -165,10 +158,6 @@ func DeleteLFSLockByID(ctx context.Context, id int64, repo *repo_model.Repositor return nil, err } - if err := CheckLFSAccessForRepo(ctx, u.ID, repo, perm.AccessModeWrite); err != nil { - return nil, err - } - if !force && u.ID != lock.OwnerID { return nil, errors.New("user doesn't own lock and force flag is not set") } @@ -180,22 +169,3 @@ func DeleteLFSLockByID(ctx context.Context, id int64, repo *repo_model.Repositor return lock, nil }) } - -// CheckLFSAccessForRepo check needed access mode base on action -func CheckLFSAccessForRepo(ctx context.Context, ownerID int64, repo *repo_model.Repository, mode perm.AccessMode) error { - if ownerID == 0 { - return ErrLFSUnauthorizedAction{repo.ID, "undefined", mode} - } - u, err := user_model.GetUserByID(ctx, ownerID) - if err != nil { - return err - } - perm, err := access_model.GetUserRepoPermission(ctx, repo, u) - if err != nil { - return err - } - if !perm.CanAccess(mode, unit.TypeCode) { - return ErrLFSUnauthorizedAction{repo.ID, u.DisplayName(), mode} - } - return nil -} diff --git a/models/git/protected_branch.go b/models/git/protected_branch.go index 511f7563cf..13e1ced0e1 100644 --- a/models/git/protected_branch.go +++ b/models/git/protected_branch.go @@ -5,7 +5,6 @@ package git import ( "context" - "errors" "fmt" "slices" "strings" @@ -25,7 +24,7 @@ import ( "xorm.io/builder" ) -var ErrBranchIsProtected = errors.New("branch is protected") +var ErrBranchIsProtected = util.ErrorWrap(util.ErrPermissionDenied, "branch is protected") // ProtectedBranch struct type ProtectedBranch struct { diff --git a/models/issues/issue_search.go b/models/issues/issue_search.go index 466e788d6c..049dcc7de8 100644 --- a/models/issues/issue_search.go +++ b/models/issues/issue_search.go @@ -476,7 +476,7 @@ func applySubscribedCondition(sess *xorm.Session, subscriberID int64) { ), builder.Eq{"issue.poster_id": subscriberID}, builder.In("issue.repo_id", builder. - Select("id"). + Select("repo_id"). From("watch"). Where(builder.And(builder.Eq{"user_id": subscriberID}, builder.In("mode", repo_model.WatchModeNormal, repo_model.WatchModeAuto))), diff --git a/models/issues/issue_test.go b/models/issues/issue_test.go index 09fd492667..55a90f50a1 100644 --- a/models/issues/issue_test.go +++ b/models/issues/issue_test.go @@ -197,6 +197,12 @@ func TestIssues(t *testing.T) { }, []int64{2}, }, + { + issues_model.IssuesOptions{ + SubscriberID: 11, + }, + []int64{11, 5, 9, 8, 3, 2, 1}, + }, } { issues, err := issues_model.Issues(t.Context(), &test.Opts) assert.NoError(t, err) diff --git a/models/organization/org.go b/models/organization/org.go index 9ece044d6c..b4d28f5405 100644 --- a/models/organization/org.go +++ b/models/organization/org.go @@ -429,6 +429,10 @@ func HasOrgOrUserVisible(ctx context.Context, orgOrUser, user *user_model.User) return true } + if !setting.Service.RequireSignInViewStrict && orgOrUser.Visibility == structs.VisibleTypePublic { + return true + } + if (orgOrUser.Visibility == structs.VisibleTypePrivate || user.IsRestricted) && !OrgFromUser(orgOrUser).hasMemberWithUserID(ctx, user.ID) { return false } diff --git a/models/organization/org_test.go b/models/organization/org_test.go index e7c4d2f9f7..7a74c5f5fc 100644 --- a/models/organization/org_test.go +++ b/models/organization/org_test.go @@ -13,7 +13,9 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -382,6 +384,12 @@ func TestHasOrgVisibleTypePublic(t *testing.T) { assert.True(t, test1) // owner of org assert.True(t, test2) // user not a part of org assert.True(t, test3) // logged out user + + restrictedUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 29, IsRestricted: true}) + require.True(t, restrictedUser.IsRestricted) + assert.True(t, organization.HasOrgOrUserVisible(t.Context(), org.AsUser(), restrictedUser)) + defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)() + assert.False(t, organization.HasOrgOrUserVisible(t.Context(), org.AsUser(), restrictedUser)) } func TestHasOrgVisibleTypeLimited(t *testing.T) { diff --git a/models/perm/access/access.go b/models/perm/access/access.go index 6a0a901f71..6433c4675c 100644 --- a/models/perm/access/access.go +++ b/models/perm/access/access.go @@ -13,6 +13,8 @@ import ( "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "xorm.io/builder" ) @@ -41,7 +43,12 @@ func accessLevel(ctx context.Context, user *user_model.User, repo *repo_model.Re restricted = user.IsRestricted } - if !restricted && !repo.IsPrivate { + if err := repo.LoadOwner(ctx); err != nil { + return mode, err + } + + repoIsFullyPublic := !setting.Service.RequireSignInViewStrict && repo.Owner.Visibility == structs.VisibleTypePublic && !repo.IsPrivate + if (restricted && repoIsFullyPublic) || (!restricted && !repo.IsPrivate) { mode = perm.AccessModeRead } diff --git a/models/perm/access/access_test.go b/models/perm/access/access_test.go index f01993ab4e..15d18b368c 100644 --- a/models/perm/access/access_test.go +++ b/models/perm/access/access_test.go @@ -12,6 +12,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" ) @@ -51,7 +52,14 @@ func TestAccessLevel(t *testing.T) { assert.NoError(t, err) assert.Equal(t, perm_model.AccessModeNone, level) - // restricted user has no access to a public repo + // restricted user has default access to a public repo if no sign-in is required + setting.Service.RequireSignInViewStrict = false + level, err = access_model.AccessLevel(t.Context(), user29, repo1) + assert.NoError(t, err) + assert.Equal(t, perm_model.AccessModeRead, level) + + // restricted user has no access to a public repo if sign-in is required + setting.Service.RequireSignInViewStrict = true level, err = access_model.AccessLevel(t.Context(), user29, repo1) assert.NoError(t, err) assert.Equal(t, perm_model.AccessModeNone, level) diff --git a/models/perm/access/repo_permission.go b/models/perm/access/repo_permission.go index 678b18442e..ba7544f343 100644 --- a/models/perm/access/repo_permission.go +++ b/models/perm/access/repo_permission.go @@ -5,9 +5,11 @@ package access import ( "context" + "errors" "fmt" "slices" + actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/organization" perm_model "code.gitea.io/gitea/models/perm" @@ -253,6 +255,43 @@ func finalProcessRepoUnitPermission(user *user_model.User, perm *Permission) { } } +// GetActionsUserRepoPermission returns the actions user permissions to the repository +func GetActionsUserRepoPermission(ctx context.Context, repo *repo_model.Repository, actionsUser *user_model.User, taskID int64) (perm Permission, err error) { + if actionsUser.ID != user_model.ActionsUserID { + return perm, errors.New("api GetActionsUserRepoPermission can only be called by the actions user") + } + task, err := actions_model.GetTaskByID(ctx, taskID) + if err != nil { + return perm, err + } + + var accessMode perm_model.AccessMode + if task.RepoID != repo.ID { + taskRepo, exist, err := db.GetByID[repo_model.Repository](ctx, task.RepoID) + if err != nil || !exist { + return perm, err + } + actionsCfg := repo.MustGetUnit(ctx, unit.TypeActions).ActionsConfig() + if !actionsCfg.IsCollaborativeOwner(taskRepo.OwnerID) || !taskRepo.IsPrivate { + // The task repo can access the current repo only if the task repo is private and + // the owner of the task repo is a collaborative owner of the current repo. + // FIXME allow public repo read access if tokenless pull is enabled + return perm, nil + } + accessMode = perm_model.AccessModeRead + } else if task.IsForkPullRequest { + accessMode = perm_model.AccessModeRead + } else { + accessMode = perm_model.AccessModeWrite + } + + if err := repo.LoadUnits(ctx); err != nil { + return perm, err + } + perm.SetUnitsWithDefaultAccessMode(repo.Units, accessMode) + return perm, nil +} + // GetUserRepoPermission returns the user permissions to the repository func GetUserRepoPermission(ctx context.Context, repo *repo_model.Repository, user *user_model.User) (perm Permission, err error) { defer func() { diff --git a/models/repo/repo_list.go b/models/repo/repo_list.go index f2cdd2f284..811f83c999 100644 --- a/models/repo/repo_list.go +++ b/models/repo/repo_list.go @@ -642,6 +642,17 @@ func SearchRepositoryIDsByCondition(ctx context.Context, cond builder.Cond) ([]i Find(&repoIDs) } +func userAllPublicRepoCond(cond builder.Cond, orgVisibilityLimit []structs.VisibleType) builder.Cond { + return cond.Or(builder.And( + builder.Eq{"`repository`.is_private": false}, + // Aren't in a private organisation or limited organisation if we're not logged in + builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where( + builder.And( + builder.Eq{"type": user_model.UserTypeOrganization}, + builder.In("visibility", orgVisibilityLimit)), + )))) +} + // AccessibleRepositoryCondition takes a user a returns a condition for checking if a repository is accessible func AccessibleRepositoryCondition(user *user_model.User, unitType unit.Type) builder.Cond { cond := builder.NewCond() @@ -651,15 +662,8 @@ func AccessibleRepositoryCondition(user *user_model.User, unitType unit.Type) bu if user == nil || user.ID <= 0 { orgVisibilityLimit = append(orgVisibilityLimit, structs.VisibleTypeLimited) } - // 1. Be able to see all non-private repositories that either: - cond = cond.Or(builder.And( - builder.Eq{"`repository`.is_private": false}, - // 2. Aren't in an private organisation or limited organisation if we're not logged in - builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where( - builder.And( - builder.Eq{"type": user_model.UserTypeOrganization}, - builder.In("visibility", orgVisibilityLimit)), - )))) + // 1. Be able to see all non-private repositories + cond = userAllPublicRepoCond(cond, orgVisibilityLimit) } if user != nil { @@ -683,6 +687,9 @@ func AccessibleRepositoryCondition(user *user_model.User, unitType unit.Type) bu if !user.IsRestricted { // 5. Be able to see all public repos in private organizations that we are an org_user of cond = cond.Or(userOrgPublicRepoCond(user.ID)) + } else if !setting.Service.RequireSignInViewStrict { + orgVisibilityLimit := []structs.VisibleType{structs.VisibleTypePrivate, structs.VisibleTypeLimited} + cond = userAllPublicRepoCond(cond, orgVisibilityLimit) } } diff --git a/models/repo/repo_list_test.go b/models/repo/repo_list_test.go index 6cc0d3155c..943e0c5025 100644 --- a/models/repo/repo_list_test.go +++ b/models/repo/repo_list_test.go @@ -10,9 +10,14 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func getTestCases() []struct { @@ -182,7 +187,16 @@ func getTestCases() []struct { func TestSearchRepository(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) + t.Run("SearchRepositoryPublic", testSearchRepositoryPublic) + t.Run("SearchRepositoryPublicRestricted", testSearchRepositoryRestricted) + t.Run("SearchRepositoryPrivate", testSearchRepositoryPrivate) + t.Run("SearchRepositoryNonExistingOwner", testSearchRepositoryNonExistingOwner) + t.Run("SearchRepositoryWithInDescription", testSearchRepositoryWithInDescription) + t.Run("SearchRepositoryNotInDescription", testSearchRepositoryNotInDescription) + t.Run("SearchRepositoryCases", testSearchRepositoryCases) +} +func testSearchRepositoryPublic(t *testing.T) { // test search public repository on explore page repos, count, err := repo_model.SearchRepositoryByName(t.Context(), repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ @@ -211,9 +225,54 @@ func TestSearchRepository(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(2), count) assert.Len(t, repos, 2) +} +func testSearchRepositoryRestricted(t *testing.T) { + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + restrictedUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 29, IsRestricted: true}) + + performSearch := func(t *testing.T, user *user_model.User) (publicRepoIDs []int64) { + repos, count, err := repo_model.SearchRepositoryByName(t.Context(), repo_model.SearchRepoOptions{ + ListOptions: db.ListOptions{Page: 1, PageSize: 10000}, + Actor: user, + }) + require.NoError(t, err) + assert.Len(t, repos, int(count)) + for _, repo := range repos { + require.NoError(t, repo.LoadOwner(t.Context())) + if repo.Owner.Visibility == structs.VisibleTypePublic && !repo.IsPrivate { + publicRepoIDs = append(publicRepoIDs, repo.ID) + } + } + return publicRepoIDs + } + + normalPublicRepoIDs := performSearch(t, user2) + require.Greater(t, len(normalPublicRepoIDs), 10) // quite a lot + + t.Run("RestrictedUser-NoSignInRequirement", func(t *testing.T) { + // restricted user can also see public repositories if no "required sign-in" + repoIDs := performSearch(t, restrictedUser) + assert.ElementsMatch(t, normalPublicRepoIDs, repoIDs) + }) + + defer test.MockVariableValue(&setting.Service.RequireSignInViewStrict, true)() + + t.Run("NormalUser-RequiredSignIn", func(t *testing.T) { + // normal user can still see all public repos, not affected by "required sign-in" + repoIDs := performSearch(t, user2) + assert.ElementsMatch(t, normalPublicRepoIDs, repoIDs) + }) + t.Run("RestrictedUser-RequiredSignIn", func(t *testing.T) { + // restricted user can see only their own repo + repoIDs := performSearch(t, restrictedUser) + assert.Equal(t, []int64{4}, repoIDs) + }) +} + +func testSearchRepositoryPrivate(t *testing.T) { // test search private repository on explore page - repos, count, err = repo_model.SearchRepositoryByName(t.Context(), repo_model.SearchRepoOptions{ + repos, count, err := repo_model.SearchRepositoryByName(t.Context(), repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ Page: 1, PageSize: 10, @@ -242,16 +301,18 @@ func TestSearchRepository(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(3), count) assert.Len(t, repos, 3) +} - // Test non existing owner - repos, count, err = repo_model.SearchRepositoryByName(t.Context(), repo_model.SearchRepoOptions{OwnerID: unittest.NonexistentID}) +func testSearchRepositoryNonExistingOwner(t *testing.T) { + repos, count, err := repo_model.SearchRepositoryByName(t.Context(), repo_model.SearchRepoOptions{OwnerID: unittest.NonexistentID}) assert.NoError(t, err) assert.Empty(t, repos) assert.Equal(t, int64(0), count) +} - // Test search within description - repos, count, err = repo_model.SearchRepository(t.Context(), repo_model.SearchRepoOptions{ +func testSearchRepositoryWithInDescription(t *testing.T) { + repos, count, err := repo_model.SearchRepository(t.Context(), repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ Page: 1, PageSize: 10, @@ -266,9 +327,10 @@ func TestSearchRepository(t *testing.T) { assert.Equal(t, "test_repo_14", repos[0].Name) } assert.Equal(t, int64(1), count) +} - // Test NOT search within description - repos, count, err = repo_model.SearchRepository(t.Context(), repo_model.SearchRepoOptions{ +func testSearchRepositoryNotInDescription(t *testing.T) { + repos, count, err := repo_model.SearchRepository(t.Context(), repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ Page: 1, PageSize: 10, @@ -281,7 +343,9 @@ func TestSearchRepository(t *testing.T) { assert.NoError(t, err) assert.Empty(t, repos) assert.Equal(t, int64(0), count) +} +func testSearchRepositoryCases(t *testing.T) { testCases := getTestCases() for _, testCase := range testCases { diff --git a/models/repo/repo_unit.go b/models/repo/repo_unit.go index a5207bc22a..ad0bb9d3f8 100644 --- a/models/repo/repo_unit.go +++ b/models/repo/repo_unit.go @@ -170,6 +170,9 @@ func (cfg *PullRequestsConfig) GetDefaultMergeStyle() MergeStyle { type ActionsConfig struct { DisabledWorkflows []string + // CollaborativeOwnerIDs is a list of owner IDs used to share actions from private repos. + // Only workflows from the private repos whose owners are in CollaborativeOwnerIDs can access the current repo's actions. + CollaborativeOwnerIDs []int64 } func (cfg *ActionsConfig) EnableWorkflow(file string) { @@ -192,6 +195,20 @@ func (cfg *ActionsConfig) DisableWorkflow(file string) { cfg.DisabledWorkflows = append(cfg.DisabledWorkflows, file) } +func (cfg *ActionsConfig) AddCollaborativeOwner(ownerID int64) { + if !slices.Contains(cfg.CollaborativeOwnerIDs, ownerID) { + cfg.CollaborativeOwnerIDs = append(cfg.CollaborativeOwnerIDs, ownerID) + } +} + +func (cfg *ActionsConfig) RemoveCollaborativeOwner(ownerID int64) { + cfg.CollaborativeOwnerIDs = util.SliceRemoveAll(cfg.CollaborativeOwnerIDs, ownerID) +} + +func (cfg *ActionsConfig) IsCollaborativeOwner(ownerID int64) bool { + return slices.Contains(cfg.CollaborativeOwnerIDs, ownerID) +} + // FromDB fills up a ActionsConfig from serialized format. func (cfg *ActionsConfig) FromDB(bs []byte) error { return json.UnmarshalHandleDoubleEncode(bs, &cfg) diff --git a/models/repo/upload.go b/models/repo/upload.go index f7d4749842..b9bda8fdbf 100644 --- a/models/repo/upload.go +++ b/models/repo/upload.go @@ -127,16 +127,9 @@ func DeleteUploads(ctx context.Context, uploads ...*Upload) (err error) { for _, upload := range uploads { localPath := upload.LocalPath() - isFile, err := util.IsFile(localPath) - if err != nil { - log.Error("Unable to check if %s is a file. Error: %v", localPath, err) - } - if !isFile { - continue - } - if err := util.Remove(localPath); err != nil { - return fmt.Errorf("remove upload: %w", err) + // just continue, don't fail the whole operation if a file is missing (removed by others) + log.Error("unable to remove upload file %s: %v", localPath, err) } } diff --git a/models/secret/secret.go b/models/secret/secret.go index 10a0287dfd..a82a924c39 100644 --- a/models/secret/secret.go +++ b/models/secret/secret.go @@ -178,8 +178,8 @@ func GetSecretsOfTask(ctx context.Context, task *actions_model.ActionTask) (map[ for _, secret := range append(ownerSecrets, repoSecrets...) { v, err := secret_module.DecryptSecret(setting.SecretKey, secret.Data) if err != nil { - log.Error("decrypt secret %v %q: %v", secret.ID, secret.Name, err) - return nil, err + log.Error("Unable to decrypt Actions secret %v %q, maybe SECRET_KEY is wrong: %v", secret.ID, secret.Name, err) + continue } secrets[secret.Name] = v } diff --git a/models/user/search.go b/models/user/search.go index cfd0d011bc..db4b07f64a 100644 --- a/models/user/search.go +++ b/models/user/search.go @@ -6,6 +6,7 @@ package user import ( "context" "fmt" + "slices" "strings" "code.gitea.io/gitea/models/db" @@ -22,7 +23,7 @@ type SearchUserOptions struct { db.ListOptions Keyword string - Type UserType + Types []UserType UID int64 LoginName string // this option should be used only for admin user SourceID int64 // this option should be used only for admin user @@ -43,16 +44,16 @@ type SearchUserOptions struct { func (opts *SearchUserOptions) toSearchQueryBase(ctx context.Context) *xorm.Session { var cond builder.Cond - cond = builder.Eq{"type": opts.Type} + cond = builder.In("type", opts.Types) if opts.IncludeReserved { - switch opts.Type { - case UserTypeIndividual: + switch { + case slices.Contains(opts.Types, UserTypeIndividual): cond = cond.Or(builder.Eq{"type": UserTypeUserReserved}).Or( builder.Eq{"type": UserTypeBot}, ).Or( builder.Eq{"type": UserTypeRemoteUser}, ) - case UserTypeOrganization: + case slices.Contains(opts.Types, UserTypeOrganization): cond = cond.Or(builder.Eq{"type": UserTypeOrganizationReserved}) } } diff --git a/models/user/user.go b/models/user/user.go index 80d5eb5ec4..d6e1eec276 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -249,8 +249,13 @@ func (u *User) MaxCreationLimit() int { } // CanCreateRepoIn checks whether the doer(u) can create a repository in the owner -// NOTE: functions calling this assume a failure due to repository count limit; it ONLY checks the repo number LIMIT, if new checks are added, those functions should be revised +// NOTE: functions calling this assume a failure due to repository count limit, or the owner is not a real user. +// It ONLY checks the repo number LIMIT or whether owner user is real. If new checks are added, those functions should be revised. +// TODO: the callers can only return ErrReachLimitOfRepo, need to fine tune to support other error types in the future. func (u *User) CanCreateRepoIn(owner *User) bool { + if u.ID <= 0 || owner.ID <= 0 { + return false // fake user like Ghost or Actions user + } if u.IsAdmin { return true } @@ -1444,3 +1449,15 @@ func DisabledFeaturesWithLoginType(user *User) *container.Set[string] { } return &setting.Admin.UserDisabledFeatures } + +// GetUserOrOrgIDByName returns the id for a user or an org by name +func GetUserOrOrgIDByName(ctx context.Context, name string) (int64, error) { + var id int64 + has, err := db.GetEngine(ctx).Table("user").Where("name = ?", name).Cols("id").Get(&id) + if err != nil { + return 0, err + } else if !has { + return 0, fmt.Errorf("user or org with name %s: %w", name, util.ErrNotExist) + } + return id, nil +} diff --git a/models/user/user_system.go b/models/user/user_system.go index e07274d291..11008c77d4 100644 --- a/models/user/user_system.go +++ b/models/user/user_system.go @@ -48,17 +48,16 @@ func IsGiteaActionsUserName(name string) bool { // NewActionsUser creates and returns a fake user for running the actions. func NewActionsUser() *User { return &User{ - ID: ActionsUserID, - Name: ActionsUserName, - LowerName: ActionsUserName, - IsActive: true, - FullName: "Gitea Actions", - Email: ActionsUserEmail, - KeepEmailPrivate: true, - LoginName: ActionsUserName, - Type: UserTypeBot, - AllowCreateOrganization: true, - Visibility: structs.VisibleTypePublic, + ID: ActionsUserID, + Name: ActionsUserName, + LowerName: ActionsUserName, + IsActive: true, + FullName: "Gitea Actions", + Email: ActionsUserEmail, + KeepEmailPrivate: true, + LoginName: ActionsUserName, + Type: UserTypeBot, + Visibility: structs.VisibleTypePublic, } } diff --git a/models/user/user_test.go b/models/user/user_test.go index 4201ec4816..923f2cd40e 100644 --- a/models/user/user_test.go +++ b/models/user/user_test.go @@ -126,7 +126,7 @@ func TestSearchUsers(t *testing.T) { // test orgs testOrgSuccess := func(opts user_model.SearchUserOptions, expectedOrgIDs []int64) { - opts.Type = user_model.UserTypeOrganization + opts.Types = []user_model.UserType{user_model.UserTypeOrganization} testSuccess(opts, expectedOrgIDs) } @@ -150,7 +150,7 @@ func TestSearchUsers(t *testing.T) { // test users testUserSuccess := func(opts user_model.SearchUserOptions, expectedUserIDs []int64) { - opts.Type = user_model.UserTypeIndividual + opts.Types = []user_model.UserType{user_model.UserTypeIndividual} testSuccess(opts, expectedUserIDs) } @@ -648,33 +648,36 @@ func TestGetInactiveUsers(t *testing.T) { func TestCanCreateRepo(t *testing.T) { defer test.MockVariableValue(&setting.Repository.MaxCreationLimit)() const noLimit = -1 - doerNormal := &user_model.User{} - doerAdmin := &user_model.User{IsAdmin: true} + doerActions := user_model.NewActionsUser() + doerNormal := &user_model.User{ID: 2} + doerAdmin := &user_model.User{ID: 1, IsAdmin: true} t.Run("NoGlobalLimit", func(t *testing.T) { setting.Repository.MaxCreationLimit = noLimit - assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: noLimit})) - assert.False(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 0})) - assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 100})) + assert.False(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 0})) + assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 100})) + assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: noLimit})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: noLimit})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 0})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 100})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 0})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 100})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: noLimit})) + assert.False(t, doerActions.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: noLimit})) + assert.False(t, doerAdmin.CanCreateRepoIn(doerActions)) }) t.Run("GlobalLimit50", func(t *testing.T) { setting.Repository.MaxCreationLimit = 50 - assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: noLimit})) - assert.False(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 60, MaxRepoCreation: noLimit})) // limited by global limit - assert.False(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 0})) - assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 100})) - assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{NumRepos: 60, MaxRepoCreation: 100})) + assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: noLimit})) + assert.False(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 60, MaxRepoCreation: noLimit})) // limited by global limit + assert.False(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 0})) + assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 100})) + assert.True(t, doerNormal.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 60, MaxRepoCreation: 100})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: noLimit})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 60, MaxRepoCreation: noLimit})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 0})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 10, MaxRepoCreation: 100})) - assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{NumRepos: 60, MaxRepoCreation: 100})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: noLimit})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 60, MaxRepoCreation: noLimit})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 0})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 10, MaxRepoCreation: 100})) + assert.True(t, doerAdmin.CanCreateRepoIn(&user_model.User{ID: 2, NumRepos: 60, MaxRepoCreation: 100})) }) } diff --git a/modules/emoji/emoji.go b/modules/emoji/emoji.go index 3d4ef8599b..891a0b9ab3 100644 --- a/modules/emoji/emoji.go +++ b/modules/emoji/emoji.go @@ -8,7 +8,9 @@ import ( "io" "sort" "strings" - "sync" + "sync/atomic" + + "code.gitea.io/gitea/modules/setting" ) // Gemoji is a set of emoji data. @@ -23,74 +25,78 @@ type Emoji struct { SkinTones bool } -var ( - // codeMap provides a map of the emoji unicode code to its emoji data. - codeMap map[string]int +type globalVarsStruct struct { + codeMap map[string]int // emoji unicode code to its emoji data. + aliasMap map[string]int // the alias to its emoji data. + emptyReplacer *strings.Replacer // string replacer for emoji codes, used for finding emoji positions. + codeReplacer *strings.Replacer // string replacer for emoji codes. + aliasReplacer *strings.Replacer // string replacer for emoji aliases. +} - // aliasMap provides a map of the alias to its emoji data. - aliasMap map[string]int +var globalVarsStore atomic.Pointer[globalVarsStruct] - // emptyReplacer is the string replacer for emoji codes. - emptyReplacer *strings.Replacer +func globalVars() *globalVarsStruct { + vars := globalVarsStore.Load() + if vars != nil { + return vars + } + // although there can be concurrent calls, the result should be the same, and there is no performance problem + vars = &globalVarsStruct{} + vars.codeMap = make(map[string]int, len(GemojiData)) + vars.aliasMap = make(map[string]int, len(GemojiData)) - // codeReplacer is the string replacer for emoji codes. - codeReplacer *strings.Replacer + // process emoji codes and aliases + codePairs := make([]string, 0) + emptyPairs := make([]string, 0) + aliasPairs := make([]string, 0) - // aliasReplacer is the string replacer for emoji aliases. - aliasReplacer *strings.Replacer + // sort from largest to small so we match combined emoji first + sort.Slice(GemojiData, func(i, j int) bool { + return len(GemojiData[i].Emoji) > len(GemojiData[j].Emoji) + }) - once sync.Once -) - -func loadMap() { - once.Do(func() { - // initialize - codeMap = make(map[string]int, len(GemojiData)) - aliasMap = make(map[string]int, len(GemojiData)) - - // process emoji codes and aliases - codePairs := make([]string, 0) - emptyPairs := make([]string, 0) - aliasPairs := make([]string, 0) - - // sort from largest to small so we match combined emoji first - sort.Slice(GemojiData, func(i, j int) bool { - return len(GemojiData[i].Emoji) > len(GemojiData[j].Emoji) - }) - - for i, e := range GemojiData { - if e.Emoji == "" || len(e.Aliases) == 0 { - continue - } - - // setup codes - codeMap[e.Emoji] = i - codePairs = append(codePairs, e.Emoji, ":"+e.Aliases[0]+":") - emptyPairs = append(emptyPairs, e.Emoji, e.Emoji) - - // setup aliases - for _, a := range e.Aliases { - if a == "" { - continue - } - - aliasMap[a] = i - aliasPairs = append(aliasPairs, ":"+a+":", e.Emoji) - } + for idx, emoji := range GemojiData { + if emoji.Emoji == "" || len(emoji.Aliases) == 0 { + continue } - // create replacers - emptyReplacer = strings.NewReplacer(emptyPairs...) - codeReplacer = strings.NewReplacer(codePairs...) - aliasReplacer = strings.NewReplacer(aliasPairs...) - }) + // process aliases + firstAlias := "" + for _, alias := range emoji.Aliases { + if alias == "" { + continue + } + enabled := len(setting.UI.EnabledEmojisSet) == 0 || setting.UI.EnabledEmojisSet.Contains(alias) + if !enabled { + continue + } + if firstAlias == "" { + firstAlias = alias + } + vars.aliasMap[alias] = idx + aliasPairs = append(aliasPairs, ":"+alias+":", emoji.Emoji) + } + + // process emoji code + if firstAlias != "" { + vars.codeMap[emoji.Emoji] = idx + codePairs = append(codePairs, emoji.Emoji, ":"+emoji.Aliases[0]+":") + emptyPairs = append(emptyPairs, emoji.Emoji, emoji.Emoji) + } + } + + // create replacers + vars.emptyReplacer = strings.NewReplacer(emptyPairs...) + vars.codeReplacer = strings.NewReplacer(codePairs...) + vars.aliasReplacer = strings.NewReplacer(aliasPairs...) + globalVarsStore.Store(vars) + return vars } // FromCode retrieves the emoji data based on the provided unicode code (ie, // "\u2618" will return the Gemoji data for "shamrock"). func FromCode(code string) *Emoji { - loadMap() - i, ok := codeMap[code] + i, ok := globalVars().codeMap[code] if !ok { return nil } @@ -102,12 +108,11 @@ func FromCode(code string) *Emoji { // "alias" or ":alias:" (ie, "shamrock" or ":shamrock:" will return the Gemoji // data for "shamrock"). func FromAlias(alias string) *Emoji { - loadMap() if strings.HasPrefix(alias, ":") && strings.HasSuffix(alias, ":") { alias = alias[1 : len(alias)-1] } - i, ok := aliasMap[alias] + i, ok := globalVars().aliasMap[alias] if !ok { return nil } @@ -119,15 +124,13 @@ func FromAlias(alias string) *Emoji { // alias (in the form of ":alias:") (ie, "\u2618" will be converted to // ":shamrock:"). func ReplaceCodes(s string) string { - loadMap() - return codeReplacer.Replace(s) + return globalVars().codeReplacer.Replace(s) } // ReplaceAliases replaces all aliases of the form ":alias:" with its // corresponding unicode value. func ReplaceAliases(s string) string { - loadMap() - return aliasReplacer.Replace(s) + return globalVars().aliasReplacer.Replace(s) } type rememberSecondWriteWriter struct { @@ -163,7 +166,6 @@ func (n *rememberSecondWriteWriter) WriteString(s string) (int, error) { // FindEmojiSubmatchIndex returns index pair of longest emoji in a string func FindEmojiSubmatchIndex(s string) []int { - loadMap() secondWriteWriter := rememberSecondWriteWriter{} // A faster and clean implementation would copy the trie tree formation in strings.NewReplacer but @@ -175,7 +177,7 @@ func FindEmojiSubmatchIndex(s string) []int { // Therefore we can simply take the index of the second write as our first emoji // // FIXME: just copy the trie implementation from strings.NewReplacer - _, _ = emptyReplacer.WriteString(&secondWriteWriter, s) + _, _ = globalVars().emptyReplacer.WriteString(&secondWriteWriter, s) // if we wrote less than twice then we never "replaced" if secondWriteWriter.writecount < 2 { diff --git a/modules/emoji/emoji_test.go b/modules/emoji/emoji_test.go index fbf80fe41a..607299cdc1 100644 --- a/modules/emoji/emoji_test.go +++ b/modules/emoji/emoji_test.go @@ -7,14 +7,13 @@ package emoji import ( "testing" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "github.com/stretchr/testify/assert" ) -func TestDumpInfo(t *testing.T) { - t.Logf("codes: %d", len(codeMap)) - t.Logf("aliases: %d", len(aliasMap)) -} - func TestLookup(t *testing.T) { a := FromCode("\U0001f37a") b := FromCode("🍺") @@ -24,7 +23,6 @@ func TestLookup(t *testing.T) { assert.Equal(t, a, b) assert.Equal(t, b, c) assert.Equal(t, c, d) - assert.Equal(t, a, d) m := FromCode("\U0001f44d") n := FromAlias(":thumbsup:") @@ -32,7 +30,20 @@ func TestLookup(t *testing.T) { assert.Equal(t, m, n) assert.Equal(t, m, o) - assert.Equal(t, n, o) + + defer test.MockVariableValue(&setting.UI.EnabledEmojisSet, container.SetOf("thumbsup"))() + defer globalVarsStore.Store(nil) + globalVarsStore.Store(nil) + a = FromCode("\U0001f37a") + c = FromAlias(":beer:") + m = FromCode("\U0001f44d") + n = FromAlias(":thumbsup:") + o = FromAlias("+1") + assert.Nil(t, a) + assert.Nil(t, c) + assert.NotNil(t, m) + assert.NotNil(t, n) + assert.Nil(t, o) } func TestReplacers(t *testing.T) { diff --git a/modules/git/hook.go b/modules/git/hook.go index 361aa53100..0e19387d97 100644 --- a/modules/git/hook.go +++ b/modules/git/hook.go @@ -47,30 +47,16 @@ func GetHook(repoPath, name string) (*Hook, error) { name: name, path: filepath.Join(repoPath, filepath.Join("hooks", name+".d", name)), } - isFile, err := util.IsFile(h.path) - if err != nil { - return nil, err - } - if isFile { - data, err := os.ReadFile(h.path) - if err != nil { - return nil, err - } + if data, err := os.ReadFile(h.path); err == nil { h.IsActive = true h.Content = string(data) return h, nil + } else if !os.IsNotExist(err) { + return nil, err } samplePath := filepath.Join(repoPath, "hooks", name+".sample") - isFile, err = util.IsFile(samplePath) - if err != nil { - return nil, err - } - if isFile { - data, err := os.ReadFile(samplePath) - if err != nil { - return nil, err - } + if data, err := os.ReadFile(samplePath); err == nil { h.Sample = string(data) } return h, nil diff --git a/modules/git/notes_test.go b/modules/git/notes_test.go index 7db2dbc0b9..5abb68b102 100644 --- a/modules/git/notes_test.go +++ b/modules/git/notes_test.go @@ -47,5 +47,5 @@ func TestGetNonExistentNotes(t *testing.T) { note := Note{} err = GetNote(t.Context(), bareRepo1, "non_existent_sha", ¬e) assert.Error(t, err) - assert.IsType(t, ErrNotExist{}, err) + assert.ErrorAs(t, err, &ErrNotExist{}) } diff --git a/modules/git/utils.go b/modules/git/utils.go index b5f188904a..e7d30ce9ee 100644 --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -6,7 +6,6 @@ package git import ( "crypto/sha1" "encoding/hex" - "io" "strconv" "strings" "sync" @@ -68,32 +67,6 @@ func ParseBool(value string) (result, valid bool) { return intValue != 0, true } -// LimitedReaderCloser is a limited reader closer -type LimitedReaderCloser struct { - R io.Reader - C io.Closer - N int64 -} - -// Read implements io.Reader -func (l *LimitedReaderCloser) Read(p []byte) (n int, err error) { - if l.N <= 0 { - _ = l.C.Close() - return 0, io.EOF - } - if int64(len(p)) > l.N { - p = p[0:l.N] - } - n, err = l.R.Read(p) - l.N -= int64(n) - return n, err -} - -// Close implements io.Closer -func (l *LimitedReaderCloser) Close() error { - return l.C.Close() -} - func HashFilePathForWebUI(s string) string { h := sha1.New() _, _ = h.Write([]byte(s)) diff --git a/modules/graceful/server.go b/modules/graceful/server.go index 2525a83e77..b440f68ab5 100644 --- a/modules/graceful/server.go +++ b/modules/graceful/server.go @@ -11,7 +11,6 @@ import ( "os" "strings" "sync" - "sync/atomic" "syscall" "time" @@ -30,12 +29,15 @@ type ServeFunction = func(net.Listener) error // Server represents our graceful server type Server struct { - network string - address string - listener net.Listener - wg sync.WaitGroup - state state - lock *sync.RWMutex + network string + address string + listener net.Listener + + lock sync.RWMutex + state state + connCounter int64 + connEmptyCond *sync.Cond + BeforeBegin func(network, address string) OnShutdown func() PerWriteTimeout time.Duration @@ -50,14 +52,13 @@ func NewServer(network, address, name string) *Server { log.Info("Starting new %s server: %s:%s on PID: %d", name, network, address, os.Getpid()) } srv := &Server{ - wg: sync.WaitGroup{}, state: stateInit, - lock: &sync.RWMutex{}, network: network, address: address, PerWriteTimeout: setting.PerWriteTimeout, PerWritePerKbTimeout: setting.PerWritePerKbTimeout, } + srv.connEmptyCond = sync.NewCond(&srv.lock) srv.BeforeBegin = func(network, addr string) { log.Debug("Starting server on %s:%s (PID: %d)", network, addr, syscall.Getpid()) @@ -154,7 +155,7 @@ func (srv *Server) Serve(serve ServeFunction) error { GetManager().RegisterServer() err := serve(srv.listener) log.Debug("Waiting for connections to finish... (PID: %d)", syscall.Getpid()) - srv.wg.Wait() + srv.waitForActiveConnections() srv.setState(stateTerminate) GetManager().ServerDone() // use of closed means that the listeners are closed - i.e. we should be shutting down - return nil @@ -178,16 +179,62 @@ func (srv *Server) setState(st state) { srv.state = st } +func (srv *Server) waitForActiveConnections() { + srv.lock.Lock() + for srv.connCounter > 0 { + srv.connEmptyCond.Wait() + } + srv.lock.Unlock() +} + +func (srv *Server) wrapConnection(c net.Conn) (net.Conn, error) { + srv.lock.Lock() + defer srv.lock.Unlock() + + if srv.state != stateRunning { + _ = c.Close() + return nil, syscall.EINVAL // same as AcceptTCP + } + + srv.connCounter++ + return &wrappedConn{Conn: c, server: srv}, nil +} + +func (srv *Server) removeConnection(_ *wrappedConn) { + srv.lock.Lock() + defer srv.lock.Unlock() + + srv.connCounter-- + if srv.connCounter <= 0 { + srv.connEmptyCond.Broadcast() + } +} + +// closeAllConnections forcefully closes all active connections +func (srv *Server) closeAllConnections() { + srv.lock.Lock() + if srv.connCounter > 0 { + log.Warn("After graceful shutdown period, %d connections are still active. Forcefully close.", srv.connCounter) + srv.connCounter = 0 // OS will close all the connections after the process exits, so we just assume there is no active connection now + } + srv.lock.Unlock() + srv.connEmptyCond.Broadcast() +} + type filer interface { File() (*os.File, error) } type wrappedListener struct { net.Listener - stopped bool - server *Server + server *Server } +var ( + _ net.Listener = (*wrappedListener)(nil) + _ filer = (*wrappedListener)(nil) +) + func newWrappedListener(l net.Listener, srv *Server) *wrappedListener { return &wrappedListener{ Listener: l, @@ -195,46 +242,24 @@ func newWrappedListener(l net.Listener, srv *Server) *wrappedListener { } } -func (wl *wrappedListener) Accept() (net.Conn, error) { - var c net.Conn - // Set keepalive on TCPListeners connections. +func (wl *wrappedListener) Accept() (c net.Conn, err error) { if tcl, ok := wl.Listener.(*net.TCPListener); ok { + // Set keepalive on TCPListeners connections if possible, see http.tcpKeepAliveListener tc, err := tcl.AcceptTCP() if err != nil { return nil, err } - _ = tc.SetKeepAlive(true) // see http.tcpKeepAliveListener - _ = tc.SetKeepAlivePeriod(3 * time.Minute) // see http.tcpKeepAliveListener + _ = tc.SetKeepAlive(true) + _ = tc.SetKeepAlivePeriod(3 * time.Minute) c = tc } else { - var err error c, err = wl.Listener.Accept() if err != nil { return nil, err } } - closed := int32(0) - - c = &wrappedConn{ - Conn: c, - server: wl.server, - closed: &closed, - perWriteTimeout: wl.server.PerWriteTimeout, - perWritePerKbTimeout: wl.server.PerWritePerKbTimeout, - } - - wl.server.wg.Add(1) - return c, nil -} - -func (wl *wrappedListener) Close() error { - if wl.stopped { - return syscall.EINVAL - } - - wl.stopped = true - return wl.Listener.Close() + return wl.server.wrapConnection(c) } func (wl *wrappedListener) File() (*os.File, error) { @@ -244,17 +269,14 @@ func (wl *wrappedListener) File() (*os.File, error) { type wrappedConn struct { net.Conn - server *Server - closed *int32 - deadline time.Time - perWriteTimeout time.Duration - perWritePerKbTimeout time.Duration + server *Server + deadline time.Time } func (w *wrappedConn) Write(p []byte) (n int, err error) { - if w.perWriteTimeout > 0 { - minTimeout := time.Duration(len(p)/1024) * w.perWritePerKbTimeout - minDeadline := time.Now().Add(minTimeout).Add(w.perWriteTimeout) + if w.server.PerWriteTimeout > 0 { + minTimeout := time.Duration(len(p)/1024) * w.server.PerWritePerKbTimeout + minDeadline := time.Now().Add(minTimeout).Add(w.server.PerWriteTimeout) w.deadline = w.deadline.Add(minTimeout) if minDeadline.After(w.deadline) { @@ -266,19 +288,6 @@ func (w *wrappedConn) Write(p []byte) (n int, err error) { } func (w *wrappedConn) Close() error { - if atomic.CompareAndSwapInt32(w.closed, 0, 1) { - defer func() { - if err := recover(); err != nil { - select { - case <-GetManager().IsHammer(): - // Likely deadlocked request released at hammertime - log.Warn("Panic during connection close! %v. Likely there has been a deadlocked request which has been released by forced shutdown.", err) - default: - log.Error("Panic during connection close! %v", err) - } - } - }() - w.server.wg.Done() - } + w.server.removeConnection(w) return w.Conn.Close() } diff --git a/modules/graceful/server_hooks.go b/modules/graceful/server_hooks.go index 9b67589571..b800c32503 100644 --- a/modules/graceful/server_hooks.go +++ b/modules/graceful/server_hooks.go @@ -5,7 +5,6 @@ package graceful import ( "os" - "runtime" "code.gitea.io/gitea/modules/log" ) @@ -48,26 +47,8 @@ func (srv *Server) doShutdown() { } func (srv *Server) doHammer() { - defer func() { - // We call srv.wg.Done() until it panics. - // This happens if we call Done() when the WaitGroup counter is already at 0 - // So if it panics -> we're done, Serve() will return and the - // parent will goroutine will exit. - if r := recover(); r != nil { - log.Error("WaitGroup at 0: Error: %v", r) - } - }() if srv.getState() != stateShuttingDown { return } - log.Warn("Forcefully shutting down parent") - for { - if srv.getState() == stateTerminate { - break - } - srv.wg.Done() - - // Give other goroutines a chance to finish before we forcibly stop them. - runtime.Gosched() - } + srv.closeAllConnections() } diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index 7c1edf432d..b4c5e7fe1e 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -126,6 +126,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byt // no sandbox attribute for pdf as it breaks rendering in at least safari. this // should generally be safe as scripts inside PDF can not escape the PDF document // see https://bugs.chromium.org/p/chromium/issues/detail?id=413851 for more discussion + // HINT: PDF-RENDER-SANDBOX: PDF won't render in sandboxed context w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'") } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 39861ade12..3cbe14b86a 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -15,6 +15,8 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + + "github.com/kballard/go-shellquote" ) // RegisterRenderers registers all supported third part renderers according settings @@ -56,14 +58,11 @@ func (p *Renderer) SanitizerRules() []setting.MarkupSanitizerRule { return p.MarkupSanitizerRules } -// SanitizerDisabled disabled sanitize if return true -func (p *Renderer) SanitizerDisabled() bool { - return p.RenderContentMode == setting.RenderContentModeNoSanitizer || p.RenderContentMode == setting.RenderContentModeIframe -} - -// DisplayInIFrame represents whether render the content with an iframe -func (p *Renderer) DisplayInIFrame() bool { - return p.RenderContentMode == setting.RenderContentModeIframe +func (p *Renderer) GetExternalRendererOptions() (ret markup.ExternalRendererOptions) { + ret.SanitizerDisabled = p.RenderContentMode == setting.RenderContentModeNoSanitizer || p.RenderContentMode == setting.RenderContentModeIframe + ret.DisplayInIframe = p.RenderContentMode == setting.RenderContentModeIframe + ret.ContentSandbox = p.RenderContentSandbox + return ret } func envMark(envName string) string { @@ -81,7 +80,10 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. envMark("GITEA_PREFIX_SRC"), baseLinkSrc, envMark("GITEA_PREFIX_RAW"), baseLinkRaw, ).Replace(p.Command) - commands := strings.Fields(command) + commands, err := shellquote.Split(command) + if err != nil || len(commands) == 0 { + return fmt.Errorf("%s invalid command %q: %w", p.Name(), p.Command, err) + } args := commands[1:] if p.IsInputFile { diff --git a/modules/markup/html_emoji.go b/modules/markup/html_emoji.go index c638065425..91ba26c676 100644 --- a/modules/markup/html_emoji.go +++ b/modules/markup/html_emoji.go @@ -5,6 +5,7 @@ package markup import ( "strings" + "unicode" "code.gitea.io/gitea/modules/emoji" "code.gitea.io/gitea/modules/setting" @@ -66,26 +67,31 @@ func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) { } m[0] += start m[1] += start - start = m[1] alias := node.Data[m[0]:m[1]] - alias = strings.ReplaceAll(alias, ":", "") - converted := emoji.FromAlias(alias) - if converted == nil { - // check if this is a custom reaction - if _, exist := setting.UI.CustomEmojisMap[alias]; exist { - replaceContent(node, m[0], m[1], createCustomEmoji(ctx, alias)) - node = node.NextSibling.NextSibling - start = 0 - continue - } + + var nextChar byte + if m[1] < len(node.Data) { + nextChar = node.Data[m[1]] + } + if nextChar == ':' || unicode.IsLetter(rune(nextChar)) || unicode.IsDigit(rune(nextChar)) { continue } - replaceContent(node, m[0], m[1], createEmoji(ctx, converted.Emoji, converted.Description)) - node = node.NextSibling.NextSibling - start = 0 + alias = strings.Trim(alias, ":") + converted := emoji.FromAlias(alias) + if converted != nil { + // standard emoji + replaceContent(node, m[0], m[1], createEmoji(ctx, converted.Emoji, converted.Description)) + node = node.NextSibling.NextSibling + start = 0 // restart searching start since node has changed + } else if _, exist := setting.UI.CustomEmojisMap[alias]; exist { + // custom reaction + replaceContent(node, m[0], m[1], createCustomEmoji(ctx, alias)) + node = node.NextSibling.NextSibling + start = 0 // restart searching start since node has changed + } } } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 5fdbf43f7c..08b050baae 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -357,12 +357,9 @@ func TestRender_emoji(t *testing.T) { `

😎🤪🔐🤑❓

`) // should match nothing - test( - "2001:0db8:85a3:0000:0000:8a2e:0370:7334", - `

2001:0db8:85a3:0000:0000:8a2e:0370:7334

`) - test( - ":not exist:", - `

:not exist:

`) + test(":100:200", `

:100:200

`) + test("std::thread::something", `

std::thread::something

`) + test(":not exist:", `

:not exist:

`) } func TestRender_ShortLinks(t *testing.T) { diff --git a/modules/markup/internal/finalprocessor.go b/modules/markup/internal/finalprocessor.go index 14d46a161f..4442afa0c9 100644 --- a/modules/markup/internal/finalprocessor.go +++ b/modules/markup/internal/finalprocessor.go @@ -5,11 +5,13 @@ package internal import ( "bytes" + "html/template" "io" ) type finalProcessor struct { renderInternal *RenderInternal + extraHeadHTML template.HTML output io.Writer buf bytes.Buffer @@ -25,6 +27,32 @@ func (p *finalProcessor) Close() error { // because "postProcess" already does so. In the future we could optimize the code to process data on the fly. buf := p.buf.Bytes() buf = bytes.ReplaceAll(buf, []byte(` data-attr-class="`+p.renderInternal.secureIDPrefix), []byte(` class="`)) - _, err := p.output.Write(buf) + + tmp := bytes.TrimSpace(buf) + isLikelyHTML := len(tmp) != 0 && tmp[0] == '<' && tmp[len(tmp)-1] == '>' && bytes.Index(tmp, []byte(` 0 + if !isLikelyHTML { + // not HTML, write back directly + _, err := p.output.Write(buf) + return err + } + + // add our extra head HTML into output + headBytes := []byte("") + posHead := bytes.Index(buf, headBytes) + var part1, part2 []byte + if posHead >= 0 { + part1, part2 = buf[:posHead+len(headBytes)], buf[posHead+len(headBytes):] + } else { + part1, part2 = nil, buf + } + if len(part1) > 0 { + if _, err := p.output.Write(part1); err != nil { + return err + } + } + if _, err := io.WriteString(p.output, string(p.extraHeadHTML)); err != nil { + return err + } + _, err := p.output.Write(part2) return err } diff --git a/modules/markup/internal/internal_test.go b/modules/markup/internal/internal_test.go index 590bcbb67f..a216d75203 100644 --- a/modules/markup/internal/internal_test.go +++ b/modules/markup/internal/internal_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestRenderInternal(t *testing.T) { +func TestRenderInternalAttrs(t *testing.T) { cases := []struct { input, protected, recovered string }{ @@ -30,7 +30,7 @@ func TestRenderInternal(t *testing.T) { for _, c := range cases { var r RenderInternal out := &bytes.Buffer{} - in := r.init("sec", out) + in := r.init("sec", out, "") protected := r.ProtectSafeAttrs(template.HTML(c.input)) assert.EqualValues(t, c.protected, protected) _, _ = io.WriteString(in, string(protected)) @@ -41,7 +41,7 @@ func TestRenderInternal(t *testing.T) { var r1, r2 RenderInternal protected := r1.ProtectSafeAttrs(`
`) assert.EqualValues(t, `
`, protected, "non-initialized RenderInternal should not protect any attributes") - _ = r1.init("sec", nil) + _ = r1.init("sec", nil, "") protected = r1.ProtectSafeAttrs(`
`) assert.EqualValues(t, `
`, protected) assert.Equal(t, "data-attr-class", r1.SafeAttr("class")) @@ -54,8 +54,37 @@ func TestRenderInternal(t *testing.T) { assert.Empty(t, recovered) out2 := &bytes.Buffer{} - in2 := r2.init("sec-other", out2) + in2 := r2.init("sec-other", out2, "") _, _ = io.WriteString(in2, string(protected)) _ = in2.Close() assert.Equal(t, `
`, out2.String(), "different secureID should not recover the value") } + +func TestRenderInternalExtraHead(t *testing.T) { + t.Run("HeadExists", func(t *testing.T) { + out := &bytes.Buffer{} + var r RenderInternal + in := r.init("sec", out, ``) + _, _ = io.WriteString(in, `any`) + _ = in.Close() + assert.Equal(t, `any`, out.String()) + }) + + t.Run("HeadNotExists", func(t *testing.T) { + out := &bytes.Buffer{} + var r RenderInternal + in := r.init("sec", out, ``) + _, _ = io.WriteString(in, `
`) + _ = in.Close() + assert.Equal(t, `
`, out.String()) + }) + + t.Run("NotHTML", func(t *testing.T) { + out := &bytes.Buffer{} + var r RenderInternal + in := r.init("sec", out, ``) + _, _ = io.WriteString(in, ``) + _ = in.Close() + assert.Equal(t, ``, out.String()) + }) +} diff --git a/modules/markup/internal/renderinternal.go b/modules/markup/internal/renderinternal.go index 7a3e37b120..9fd9a1c0e8 100644 --- a/modules/markup/internal/renderinternal.go +++ b/modules/markup/internal/renderinternal.go @@ -29,19 +29,19 @@ type RenderInternal struct { secureIDPrefix string } -func (r *RenderInternal) Init(output io.Writer) io.WriteCloser { +func (r *RenderInternal) Init(output io.Writer, extraHeadHTML template.HTML) io.WriteCloser { buf := make([]byte, 12) _, err := rand.Read(buf) if err != nil { panic("unable to generate secure id") } - return r.init(base64.URLEncoding.EncodeToString(buf), output) + return r.init(base64.URLEncoding.EncodeToString(buf), output, extraHeadHTML) } -func (r *RenderInternal) init(secID string, output io.Writer) io.WriteCloser { +func (r *RenderInternal) init(secID string, output io.Writer, extraHeadHTML template.HTML) io.WriteCloser { r.secureID = secID r.secureIDPrefix = r.secureID + ":" - return &finalProcessor{renderInternal: r, output: output} + return &finalProcessor{renderInternal: r, output: output, extraHeadHTML: extraHeadHTML} } func (r *RenderInternal) RecoverProtectedValue(v string) (string, bool) { diff --git a/modules/markup/render.go b/modules/markup/render.go index 79f1f473c2..c645749065 100644 --- a/modules/markup/render.go +++ b/modules/markup/render.go @@ -6,12 +6,14 @@ package markup import ( "context" "fmt" + "html/template" "io" "net/url" "strconv" "strings" "time" + "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/markup/internal" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -120,31 +122,38 @@ func (ctx *RenderContext) WithHelper(helper RenderHelper) *RenderContext { return ctx } -// Render renders markup file to HTML with all specific handling stuff. -func Render(ctx *RenderContext, input io.Reader, output io.Writer) error { +// FindRendererByContext finds renderer by RenderContext +// TODO: it should be merged with other similar functions like GetRendererByFileName, DetectMarkupTypeByFileName, etc +func FindRendererByContext(ctx *RenderContext) (Renderer, error) { if ctx.RenderOptions.MarkupType == "" && ctx.RenderOptions.RelativePath != "" { ctx.RenderOptions.MarkupType = DetectMarkupTypeByFileName(ctx.RenderOptions.RelativePath) if ctx.RenderOptions.MarkupType == "" { - return util.NewInvalidArgumentErrorf("unsupported file to render: %q", ctx.RenderOptions.RelativePath) + return nil, util.NewInvalidArgumentErrorf("unsupported file to render: %q", ctx.RenderOptions.RelativePath) } } renderer := renderers[ctx.RenderOptions.MarkupType] if renderer == nil { - return util.NewInvalidArgumentErrorf("unsupported markup type: %q", ctx.RenderOptions.MarkupType) + return nil, util.NewNotExistErrorf("unsupported markup type: %q", ctx.RenderOptions.MarkupType) } - if ctx.RenderOptions.RelativePath != "" { - if externalRender, ok := renderer.(ExternalRenderer); ok && externalRender.DisplayInIFrame() { - if !ctx.RenderOptions.InStandalonePage { - // for an external "DisplayInIFrame" render, it could only output its content in a standalone page - // otherwise, a `, - setting.AppSubURL, +func renderIFrame(ctx *RenderContext, sandbox string, output io.Writer) error { + src := fmt.Sprintf("%s/%s/%s/render/%s/%s", setting.AppSubURL, url.PathEscape(ctx.RenderOptions.Metas["user"]), url.PathEscape(ctx.RenderOptions.Metas["repo"]), - ctx.RenderOptions.Metas["RefTypeNameSubURL"], - url.PathEscape(ctx.RenderOptions.RelativePath), - )) + util.PathEscapeSegments(ctx.RenderOptions.Metas["RefTypeNameSubURL"]), + util.PathEscapeSegments(ctx.RenderOptions.RelativePath), + ) + + var sandboxAttrValue template.HTML + if sandbox != "" { + sandboxAttrValue = htmlutil.HTMLFormat(`sandbox="%s"`, sandbox) + } + iframe := htmlutil.HTMLFormat(``, src, sandboxAttrValue) + _, err := io.WriteString(output, string(iframe)) return err } @@ -185,13 +190,34 @@ func pipes() (io.ReadCloser, io.WriteCloser, func()) { } } -func render(ctx *RenderContext, renderer Renderer, input io.Reader, output io.Writer) error { +func getExternalRendererOptions(renderer Renderer) (ret ExternalRendererOptions, _ bool) { + if externalRender, ok := renderer.(ExternalRenderer); ok { + return externalRender.GetExternalRendererOptions(), true + } + return ret, false +} + +func RenderWithRenderer(ctx *RenderContext, renderer Renderer, input io.Reader, output io.Writer) error { + var extraHeadHTML template.HTML + if extOpts, ok := getExternalRendererOptions(renderer); ok && extOpts.DisplayInIframe { + if !ctx.RenderOptions.InStandalonePage { + // for an external "DisplayInIFrame" render, it could only output its content in a standalone page + // otherwise, a