Merge remote-tracking branch 'upstream/main' into limit-repo-size

.devcontainer/devcontainer.json

@@ -5,7 +5,10 @@
     // installs nodejs into container
     "ghcr.io/devcontainers/features/node:1": {
       "version":"20"
-    }
+    },
+    "ghcr.io/devcontainers/features/git-lfs:1.1.0": {},
+    "ghcr.io/devcontainers-contrib/features/poetry:2": {},
+    "ghcr.io/devcontainers/features/python:1": {}
   },
   "customizations": {
     "vscode": {
@@ -20,7 +23,7 @@
         "Vue.volar",
         "ms-azuretools.vscode-docker",
         "zixuanchen.vitest-explorer",
-        "alexcvzz.vscode-sqlite"
+        "qwtel.sqlite-viewer"
       ]
     }
   },

.eslintrc.yaml

@@ -25,10 +25,11 @@ env:
   es2022: true
   node: true
 
-globals:
-  __webpack_public_path__: true
-
 overrides:
+  - files: ["web_src/**/*"]
+    globals:
+      __webpack_public_path__: true
+      process: false # https://github.com/webpack/webpack/issues/15833
   - files: ["web_src/**/*", "docs/**/*"]
     env:
       browser: true

.github/workflows/files-changed.yml

@@ -20,7 +20,6 @@ jobs:
   detect:
     runs-on: ubuntu-latest
     timeout-minutes: 3
-    # Map a step output to a job output
     outputs:
       backend: ${{ steps.changes.outputs.backend }}
       frontend: ${{ steps.changes.outputs.frontend }}
@@ -39,12 +38,14 @@ jobs:
               - "templates/**/*.tmpl"
               - "go.mod"
               - "go.sum"
+              - "Makefile"
 
             frontend:
               - "**/*.js"
               - "web_src/**"
               - "package.json"
               - "package-lock.json"
+              - "Makefile"
 
             docs:
               - "**/*.md"
@@ -56,7 +57,9 @@ jobs:
             templates:
               - "templates/**/*.tmpl"
               - "poetry.lock"
+
             docker:
               - "Dockerfile"
               - "Dockerfile.rootless"
               - "docker/**"
+              - "Makefile"

.github/workflows/pull-compliance.yml

@@ -96,6 +96,7 @@ jobs:
       - run: make deps-frontend
       - run: make lint-frontend
       - run: make checks-frontend
+      - run: make test-frontend
       - run: make frontend
 
   backend:
@@ -110,7 +111,7 @@ jobs:
           check-latest: true
       # no frontend build here as backend should be able to build
       # even without any frontend files
-      - run: make deps-backend deps-tools
+      - run: make deps-backend
       - run: go build -o gitea_no_gcc # test if build succeeds without the sqlite tag
       - name: build-backend-arm64
         run: make backend # test cross compile

.github/workflows/release-nightly.yml

@@ -55,6 +55,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      # fetch all commits instead of only the last as some branches are long lived and could have many between versions
+      # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567
+      - run: git fetch --unshallow --quiet --tags --force
       - uses: docker/setup-qemu-action@v2
       - uses: docker/setup-buildx-action@v2
       - name: Get cleaned branch name
@@ -75,12 +78,14 @@ jobs:
       - name: build rootful docker image
         uses: docker/build-push-action@v4
         with:
+          context: .
           platforms: linux/amd64,linux/arm64
           push: true
           tags: gitea/gitea:${{ steps.clean_name.outputs.branch }}
       - name: build rootless docker image
         uses: docker/build-push-action@v4
         with:
+          context: .
           platforms: linux/amd64,linux/arm64
           push: true
           file: Dockerfile.rootless

.gitignore

@@ -53,8 +53,6 @@ cpu.out
 /bin
 /dist
 /custom/*
-!/custom/conf
-/custom/conf/*
 !/custom/conf/app.example.ini
 /data
 /indexers

.gitpod.yml

@@ -34,7 +34,7 @@ vscode:
     - Vue.volar
     - ms-azuretools.vscode-docker
     - zixuanchen.vitest-explorer
-    - alexcvzz.vscode-sqlite
+    - qwtel.sqlite-viewer
 
 ports:
   - name: Gitea

.stylelintrc.yaml

@@ -50,7 +50,7 @@ rules:
   declaration-no-important: null
   declaration-property-max-values: null
   declaration-property-unit-allowed-list: null
-  declaration-property-unit-disallowed-list: null
+  declaration-property-unit-disallowed-list: {line-height: [em]}
   declaration-property-value-allowed-list: null
   declaration-property-value-disallowed-list: null
   declaration-property-value-no-unknown: true

CONTRIBUTING.md

@@ -557,7 +557,7 @@ be reviewed by two maintainers and must pass the automatic tests.
 - And then push the tag as `git push origin v$vmaj.$vmin.$`. Drone CI will automatically create a release and upload all the compiled binary. (But currently it doesn't add the release notes automatically. Maybe we should fix that.)
 - If needed send a frontport PR for the changelog to branch `main` and update the version in `docs/config.yaml` to refer to the new version.
 - Send PR to [blog repository](https://gitea.com/gitea/blog) announcing the release.
-- Verify all release assets were correctly published through CI on dl.gitea.io and GitHub releases. Once ACKed:
-  - bump the version of https://dl.gitea.io/gitea/version.json
+- Verify all release assets were correctly published through CI on dl.gitea.com and GitHub releases. Once ACKed:
+  - bump the version of https://dl.gitea.com/gitea/version.json
   - merge the blog post PR
   - announce the release in discord `#announcements`

Makefile

@@ -79,9 +79,12 @@ endif
 STORED_VERSION_FILE := VERSION
 HUGO_VERSION ?= 0.111.3
 
+GITHUB_REF_TYPE ?= branch
+GITHUB_REF_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+
 ifneq ($(GITHUB_REF_TYPE),branch)
 	VERSION ?= $(subst v,,$(GITHUB_REF_NAME))
-	GITEA_VERSION ?= $(GITHUB_REF_NAME)
+	GITEA_VERSION ?= $(VERSION)
 else
 	ifneq ($(GITHUB_REF_NAME),)
 		VERSION ?= $(subst release/v,,$(GITHUB_REF_NAME))
@@ -223,6 +226,8 @@ help:
 	@echo " - test-frontend                    test frontend files"
 	@echo " - test-backend                     test backend files"
 	@echo " - test-e2e[\#TestSpecificName]     test end to end using playwright"
+	@echo " - update-js                        update js dependencies"
+	@echo " - update-py                        update py dependencies"
 	@echo " - webpack                          build webpack files"
 	@echo " - svg                              build svg files"
 	@echo " - fomantic                         build fomantic files"
@@ -921,13 +926,20 @@ node_modules: package-lock.json
 	poetry install
 	@touch .venv
 
-.PHONY: npm-update
-npm-update: node-check | node_modules
-	npx updates -cu
+.PHONY: update-js
+update-js: node-check | node_modules
+	npx updates -u -f package.json
 	rm -rf node_modules package-lock.json
 	npm install --package-lock
 	@touch node_modules
 
+.PHONY: update-py
+update-py: node-check | node_modules
+	npx updates -u -f pyproject.toml
+	rm -rf .venv poetry.lock
+	poetry install
+	@touch .venv
+
 .PHONY: fomantic
 fomantic:
 	rm -rf $(FOMANTIC_WORK_DIR)/build
@@ -1011,9 +1023,5 @@ docker:
 	docker build --disable-content-trust=false -t $(DOCKER_REF) .
 # support also build args docker build --build-arg GITEA_VERSION=v1.2.3 --build-arg TAGS="bindata sqlite sqlite_unlock_notify"  .
 
-.PHONY: docker-build
-docker-build:
-	docker run -ti --rm -v "$(CURDIR):/srv/app/src/code.gitea.io/gitea" -w /srv/app/src/code.gitea.io/gitea -e TAGS="bindata $(TAGS)" LDFLAGS="$(LDFLAGS)" CGO_EXTRA_CFLAGS="$(CGO_EXTRA_CFLAGS)" webhippie/golang:edge make clean build
-
 # This endif closes the if at the top of the file
 endif

README.md

@@ -173,8 +173,8 @@ for the full license text.
 
 Looking for an overview of the interface? Check it out!
 
-|![Dashboard](https://dl.gitea.io/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.io/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.io/screenshots/global_issues.png)|
+|![Dashboard](https://dl.gitea.com/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.com/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.com/screenshots/global_issues.png)|
 |:---:|:---:|:---:|
-|![Branches](https://dl.gitea.io/screenshots/branches.png)|![Web Editor](https://dl.gitea.io/screenshots/web_editor.png)|![Activity](https://dl.gitea.io/screenshots/activity.png)|
-|![New Migration](https://dl.gitea.io/screenshots/migration.png)|![Migrating](https://dl.gitea.io/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
-![Pull Request Dark](https://dl.gitea.io/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.io/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.io/screenshots/diff_dark.png)|
+|![Branches](https://dl.gitea.com/screenshots/branches.png)|![Web Editor](https://dl.gitea.com/screenshots/web_editor.png)|![Activity](https://dl.gitea.com/screenshots/activity.png)|
+|![New Migration](https://dl.gitea.com/screenshots/migration.png)|![Migrating](https://dl.gitea.com/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
+![Pull Request Dark](https://dl.gitea.com/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.com/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.com/screenshots/diff_dark.png)|

README_ZH.md

@@ -91,8 +91,8 @@ Fork -> Patch -> Push -> Pull Request
 
 ## 截图
 
-|![Dashboard](https://dl.gitea.io/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.io/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.io/screenshots/global_issues.png)|
+|![Dashboard](https://dl.gitea.com/screenshots/home_timeline.png)|![User Profile](https://dl.gitea.com/screenshots/user_profile.png)|![Global Issues](https://dl.gitea.com/screenshots/global_issues.png)|
 |:---:|:---:|:---:|
-|![Branches](https://dl.gitea.io/screenshots/branches.png)|![Web Editor](https://dl.gitea.io/screenshots/web_editor.png)|![Activity](https://dl.gitea.io/screenshots/activity.png)|
-|![New Migration](https://dl.gitea.io/screenshots/migration.png)|![Migrating](https://dl.gitea.io/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
-![Pull Request Dark](https://dl.gitea.io/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.io/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.io/screenshots/diff_dark.png)|
+|![Branches](https://dl.gitea.com/screenshots/branches.png)|![Web Editor](https://dl.gitea.com/screenshots/web_editor.png)|![Activity](https://dl.gitea.com/screenshots/activity.png)|
+|![New Migration](https://dl.gitea.com/screenshots/migration.png)|![Migrating](https://dl.gitea.com/screenshots/migration.gif)|![Pull Request View](https://image.ibb.co/e02dSb/6.png)
+![Pull Request Dark](https://dl.gitea.com/screenshots/pull_requests_dark.png)|![Diff Review Dark](https://dl.gitea.com/screenshots/review_dark.png)|![Diff Dark](https://dl.gitea.com/screenshots/diff_dark.png)|

cmd/actions.go

@@ -42,7 +42,7 @@ func runGenerateActionsRunnerToken(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	scope := c.String("scope")
 

cmd/cmd.go

@@ -58,7 +58,7 @@ func confirm() (bool, error) {
 }
 
 func initDB(ctx context.Context) error {
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 	setting.InitSQLLoggersForCli(log.INFO)
 
@@ -106,5 +106,21 @@ func setupConsoleLogger(level log.Level, colorize bool, out io.Writer) {
 		WriterOption: log.WriterConsoleOption{Stderr: out == os.Stderr},
 	}
 	writer := log.NewEventWriterConsole("console-default", writeMode)
-	log.GetManager().GetLogger(log.DEFAULT).RemoveAllWriters().AddWriters(writer)
+	log.GetManager().GetLogger(log.DEFAULT).ReplaceAllWriters(writer)
+}
+
+// PrepareConsoleLoggerLevel by default, use INFO level for console logger, but some sub-commands (for git/ssh protocol) shouldn't output any log to stdout.
+// Any log appears in git stdout pipe will break the git protocol, eg: client can't push and hangs forever.
+func PrepareConsoleLoggerLevel(defaultLevel log.Level) func(*cli.Context) error {
+	return func(c *cli.Context) error {
+		level := defaultLevel
+		if c.Bool("quiet") || c.GlobalBoolT("quiet") {
+			level = log.FATAL
+		}
+		if c.Bool("debug") || c.GlobalBool("debug") || c.Bool("verbose") || c.GlobalBool("verbose") {
+			level = log.TRACE
+		}
+		log.SetConsoleLogger(log.DEFAULT, "console-default", level)
+		return nil
+	}
 }

cmd/doctor.go

@@ -91,7 +91,7 @@ func runRecreateTable(ctx *cli.Context) error {
 	golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info))
 
 	debug := ctx.Bool("debug")
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 
 	if debug {
@@ -151,7 +151,7 @@ func setupDoctorDefaultLogger(ctx *cli.Context, colorize bool) {
 			log.FallbackErrorf("unable to create file log writer: %v", err)
 			return
 		}
-		log.GetManager().GetLogger(log.DEFAULT).RemoveAllWriters().AddWriters(writer)
+		log.GetManager().GetLogger(log.DEFAULT).ReplaceAllWriters(writer)
 	}
 }
 

cmd/dump.go

@@ -182,7 +182,7 @@ func runDump(ctx *cli.Context) error {
 		}
 		fileName += "." + outType
 	}
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	// make sure we are logging to the console no matter what the configuration tells us do to
 	// FIXME: don't use CfgProvider directly

cmd/embedded.go

@@ -22,9 +22,9 @@ import (
 	"github.com/urfave/cli"
 )
 
-// Cmdembedded represents the available extract sub-command.
+// CmdEmbedded represents the available extract sub-command.
 var (
-	Cmdembedded = cli.Command{
+	CmdEmbedded = cli.Command{
 		Name:        "embedded",
 		Usage:       "Extract embedded resources",
 		Description: "A command for extracting embedded resources, like templates and images",
@@ -99,11 +99,6 @@ type assetFile struct {
 func initEmbeddedExtractor(c *cli.Context) error {
 	setupConsoleLogger(log.ERROR, log.CanColorStderr, os.Stderr)
 
-	// Read configuration file
-	setting.Init(&setting.Options{
-		AllowEmpty: true,
-	})
-
 	patterns, err := compileCollectPatterns(c.Args())
 	if err != nil {
 		return err

cmd/hook.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
@@ -32,6 +33,7 @@ var (
 		Name:        "hook",
 		Usage:       "Delegate commands to corresponding Git hooks",
 		Description: "This should only be called by Git",
+		Before:      PrepareConsoleLoggerLevel(log.FATAL),
 		Subcommands: []cli.Command{
 			subcmdHookPreReceive,
 			subcmdHookUpdate,

cmd/keys.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"strings"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
 
 	"github.com/urfave/cli"
@@ -17,6 +18,7 @@ import (
 var CmdKeys = cli.Command{
 	Name:   "keys",
 	Usage:  "This command queries the Gitea database to get the authorized command for a given ssh key fingerprint",
+	Before: PrepareConsoleLoggerLevel(log.FATAL),
 	Action: runKeys,
 	Flags: []cli.Flag{
 		cli.StringFlag{

cmd/mailer.go

@@ -16,7 +16,7 @@ func runSendMail(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	if err := argsSet(c, "title"); err != nil {
 		return err

cmd/restore_repo.go

@@ -51,7 +51,7 @@ func runRestoreRepository(c *cli.Context) error {
 	ctx, cancel := installSignals()
 	defer cancel()
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	var units []string
 	if s := c.String("units"); s != "" {
 		units = strings.Split(s, ",")

cmd/serv.go

@@ -44,6 +44,7 @@ var CmdServ = cli.Command{
 	Name:        "serv",
 	Usage:       "This command should only be called by SSH shell",
 	Description: "Serv provides access auth for repositories",
+	Before:      PrepareConsoleLoggerLevel(log.FATAL),
 	Action:      runServ,
 	Flags: []cli.Flag{
 		cli.BoolFlag{
@@ -61,7 +62,7 @@ func setup(ctx context.Context, debug bool) {
 	} else {
 		setupConsoleLogger(log.FATAL, false, os.Stderr)
 	}
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	if debug {
 		setting.RunMode = "dev"
 	}

cmd/web.go

@@ -35,6 +35,7 @@ var CmdWeb = cli.Command{
 	Usage: "Start Gitea web server",
 	Description: `Gitea web server is the only thing you need to run,
 and it takes care of all the other things for you`,
+	Before: PrepareConsoleLoggerLevel(log.INFO),
 	Action: runWeb,
 	Flags: []cli.Flag{
 		cli.StringFlag{
@@ -101,12 +102,111 @@ func createPIDFile(pidPath string) {
 	}
 }
 
-func runWeb(ctx *cli.Context) error {
-	if ctx.Bool("verbose") {
-		setupConsoleLogger(log.TRACE, log.CanColorStdout, os.Stdout)
-	} else if ctx.Bool("quiet") {
-		setupConsoleLogger(log.FATAL, log.CanColorStdout, os.Stdout)
+func serveInstall(ctx *cli.Context) error {
+	log.Info("Gitea version: %s%s", setting.AppVer, setting.AppBuiltWith)
+	log.Info("App path: %s", setting.AppPath)
+	log.Info("Work path: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Config file: %s", setting.CustomConf)
+	log.Info("Prepare to run install page")
+
+	routers.InitWebInstallPage(graceful.GetManager().HammerContext())
+
+	// Flag for port number in case first time run conflict
+	if ctx.IsSet("port") {
+		if err := setPort(ctx.String("port")); err != nil {
+			return err
+		}
 	}
+	if ctx.IsSet("install-port") {
+		if err := setPort(ctx.String("install-port")); err != nil {
+			return err
+		}
+	}
+	c := install.Routes()
+	err := listen(c, false)
+	if err != nil {
+		log.Critical("Unable to open listener for installer. Is Gitea already running?")
+		graceful.GetManager().DoGracefulShutdown()
+	}
+	select {
+	case <-graceful.GetManager().IsShutdown():
+		<-graceful.GetManager().Done()
+		log.Info("PID: %d Gitea Web Finished", os.Getpid())
+		log.GetManager().Close()
+		return err
+	default:
+	}
+	return nil
+}
+
+func serveInstalled(ctx *cli.Context) error {
+	setting.InitCfgProvider(setting.CustomConf)
+	setting.LoadCommonSettings()
+	setting.MustInstalled()
+
+	log.Info("Gitea version: %s%s", setting.AppVer, setting.AppBuiltWith)
+	log.Info("App path: %s", setting.AppPath)
+	log.Info("Work path: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Config file: %s", setting.CustomConf)
+	log.Info("Run mode: %s", setting.RunMode)
+	log.Info("Prepare to run web server")
+
+	if setting.AppWorkPathMismatch {
+		log.Error("WORK_PATH from config %q doesn't match other paths from environment variables or command arguments. "+
+			"Only WORK_PATH in config should be set and used. Please remove the other outdated work paths from environment variables and command arguments", setting.CustomConf)
+	}
+
+	rootCfg := setting.CfgProvider
+	if rootCfg.Section("").Key("WORK_PATH").String() == "" {
+		saveCfg, err := rootCfg.PrepareSaving()
+		if err != nil {
+			log.Error("Unable to prepare saving WORK_PATH=%s to config %q: %v\nYou must set it manually, otherwise there might be bugs when accessing the git repositories.", setting.AppWorkPath, setting.CustomConf, err)
+		} else {
+			rootCfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath)
+			saveCfg.Section("").Key("WORK_PATH").SetValue(setting.AppWorkPath)
+			if err = saveCfg.Save(); err != nil {
+				log.Error("Unable to update WORK_PATH=%s to config %q: %v\nYou must set it manually, otherwise there might be bugs when accessing the git repositories.", setting.AppWorkPath, setting.CustomConf, err)
+			}
+		}
+	}
+
+	routers.InitWebInstalled(graceful.GetManager().HammerContext())
+
+	// We check that AppDataPath exists here (it should have been created during installation)
+	// We can't check it in `InitWebInstalled`, because some integration tests
+	// use cmd -> InitWebInstalled, but the AppDataPath doesn't exist during those tests.
+	if _, err := os.Stat(setting.AppDataPath); err != nil {
+		log.Fatal("Can not find APP_DATA_PATH %q", setting.AppDataPath)
+	}
+
+	// Override the provided port number within the configuration
+	if ctx.IsSet("port") {
+		if err := setPort(ctx.String("port")); err != nil {
+			return err
+		}
+	}
+
+	// Set up Chi routes
+	c := routers.NormalRoutes()
+	err := listen(c, true)
+	<-graceful.GetManager().Done()
+	log.Info("PID: %d Gitea Web Finished", os.Getpid())
+	log.GetManager().Close()
+	return err
+}
+
+func servePprof() {
+	http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
+	_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
+	// The pprof server is for debug purpose only, it shouldn't be exposed on public network. At the moment it's not worth to introduce a configurable option for it.
+	log.Info("Starting pprof server on localhost:6060")
+	log.Info("Stopped pprof server: %v", http.ListenAndServe("localhost:6060", nil))
+	finished()
+}
+
+func runWeb(ctx *cli.Context) error {
 	defer func() {
 		if panicked := recover(); panicked != nil {
 			log.Fatal("PANIC: %v\n%s", panicked, log.Stack(2))
@@ -128,75 +228,19 @@ func runWeb(ctx *cli.Context) error {
 		createPIDFile(ctx.String("pid"))
 	}
 
-	// Perform pre-initialization
-	needsInstall := install.PreloadSettings(graceful.GetManager().HammerContext())
-	if needsInstall {
-		// Flag for port number in case first time run conflict
-		if ctx.IsSet("port") {
-			if err := setPort(ctx.String("port")); err != nil {
-				return err
-			}
-		}
-		if ctx.IsSet("install-port") {
-			if err := setPort(ctx.String("install-port")); err != nil {
-				return err
-			}
-		}
-		c := install.Routes()
-		err := listen(c, false)
-		if err != nil {
-			log.Critical("Unable to open listener for installer. Is Gitea already running?")
-			graceful.GetManager().DoGracefulShutdown()
-		}
-		select {
-		case <-graceful.GetManager().IsShutdown():
-			<-graceful.GetManager().Done()
-			log.Info("PID: %d Gitea Web Finished", os.Getpid())
-			log.GetManager().Close()
+	if !setting.InstallLock {
+		if err := serveInstall(ctx); err != nil {
 			return err
-		default:
 		}
 	} else {
 		NoInstallListener()
 	}
 
 	if setting.EnablePprof {
-		go func() {
-			http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
-			_, _, finished := process.GetManager().AddTypedContext(context.Background(), "Web: PProf Server", process.SystemProcessType, true)
-			// The pprof server is for debug purpose only, it shouldn't be exposed on public network. At the moment it's not worth to introduce a configurable option for it.
-			log.Info("Starting pprof server on localhost:6060")
-			log.Info("Stopped pprof server: %v", http.ListenAndServe("localhost:6060", nil))
-			finished()
-		}()
+		go servePprof()
 	}
 
-	log.Info("Global init")
-	// Perform global initialization
-	setting.Init(&setting.Options{})
-	routers.GlobalInitInstalled(graceful.GetManager().HammerContext())
-
-	// We check that AppDataPath exists here (it should have been created during installation)
-	// We can't check it in `GlobalInitInstalled`, because some integration tests
-	// use cmd -> GlobalInitInstalled, but the AppDataPath doesn't exist during those tests.
-	if _, err := os.Stat(setting.AppDataPath); err != nil {
-		log.Fatal("Can not find APP_DATA_PATH '%s'", setting.AppDataPath)
-	}
-
-	// Override the provided port number within the configuration
-	if ctx.IsSet("port") {
-		if err := setPort(ctx.String("port")); err != nil {
-			return err
-		}
-	}
-
-	// Set up Chi routes
-	c := routers.NormalRoutes()
-	err := listen(c, true)
-	<-graceful.GetManager().Done()
-	log.Info("PID: %d Gitea Web Finished", os.Getpid())
-	log.GetManager().Close()
-	return err
+	return serveInstalled(ctx)
 }
 
 func setPort(port string) error {
@@ -217,9 +261,15 @@ func setPort(port string) error {
 		defaultLocalURL += ":" + setting.HTTPPort + "/"
 
 		// Save LOCAL_ROOT_URL if port changed
-		setting.CfgProvider.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
-		if err := setting.CfgProvider.Save(); err != nil {
-			return fmt.Errorf("Failed to save config file: %v", err)
+		rootCfg := setting.CfgProvider
+		saveCfg, err := rootCfg.PrepareSaving()
+		if err != nil {
+			return fmt.Errorf("failed to save config file: %v", err)
+		}
+		rootCfg.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
+		saveCfg.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
+		if err = saveCfg.Save(); err != nil {
+			return fmt.Errorf("failed to save config file: %v", err)
 		}
 	}
 	return nil

contrib/environment-to-ini/environment-to-ini.go

@@ -81,8 +81,6 @@ func main() {
 		},
 	}
 	app.Action = runEnvironmentToIni
-	setting.SetCustomPathAndConf("", "", "")
-
 	err := app.Run(os.Args)
 	if err != nil {
 		log.Fatal("Failed to run app with %s: %v", os.Args, err)
@@ -90,12 +88,13 @@ func main() {
 }
 
 func runEnvironmentToIni(c *cli.Context) error {
-	providedCustom := c.String("custom-path")
-	providedConf := c.String("config")
-	providedWorkPath := c.String("work-path")
-	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
+	setting.InitWorkPathAndCfgProvider(os.Getenv, setting.ArgWorkPathAndCustomConf{
+		WorkPath:   c.String("work-path"),
+		CustomPath: c.String("custom-path"),
+		CustomConf: c.String("config"),
+	})
 
-	cfg, err := setting.NewConfigProviderFromFile(&setting.Options{CustomConf: setting.CustomConf, AllowEmpty: true})
+	cfg, err := setting.NewConfigProviderFromFile(setting.CustomConf)
 	if err != nil {
 		log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err)
 	}

custom/conf/app.example.ini

@@ -119,10 +119,13 @@ RUN_USER = ; git
 ;; Permission for unix socket
 ;UNIX_SOCKET_PERMISSION = 666
 ;;
-;; Local (DMZ) URL for Gitea workers (such as SSH update) accessing web service.
-;; In most cases you do not need to change the default value.
-;; Alter it only if your SSH server node is not the same as HTTP node.
-;; Do not set this variable if PROTOCOL is set to 'unix'.
+;; Local (DMZ) URL for Gitea workers (such as SSH update) accessing web service. In
+;; most cases you do not need to change the default value. Alter it only if
+;; your SSH server node is not the same as HTTP node. For different protocol, the default
+;; values are different. If `PROTOCOL` is `http+unix`, the default value is `http://unix/`.
+;; If `PROTOCOL` is `fcgi` or `fcgi+unix`, the default value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
+;; If listen on `0.0.0.0`, the default value is `%(PROTOCOL)s://localhost:%(HTTP_PORT)s/`, Otherwise the default
+;; value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
 ;LOCAL_ROOT_URL = %(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/
 ;;
 ;; When making local connections pass the PROXY protocol header.
@@ -344,9 +347,6 @@ NAME = gitea
 USER = root
 ;PASSWD = ;Use PASSWD = `your password` for quoting if you use special characters in the password.
 ;SSL_MODE = false ; either "false" (default), "true", or "skip-verify"
-;CHARSET = utf8mb4 ;either "utf8" or "utf8mb4", default is "utf8mb4".
-;;
-;; NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
@@ -1346,10 +1346,10 @@ LEVEL = Info
 ;; Issue indexer storage path, available when ISSUE_INDEXER_TYPE is bleve
 ;ISSUE_INDEXER_PATH = indexers/issues.bleve ; Relative paths will be made absolute against _`AppWorkPath`_.
 ;;
-;; Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch or meilisearch
-;ISSUE_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
+;; Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch (e.g. http://elastic:password@localhost:9200) or meilisearch (e.g. http://:apikey@localhost:7700)
+;ISSUE_INDEXER_CONN_STR =
 ;;
-;; Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
+;; Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch or meilisearch.
 ;ISSUE_INDEXER_NAME = gitea_issues
 ;;
 ;; Timeout the indexer if it takes longer than this to start.
@@ -2168,7 +2168,7 @@ LEVEL = Info
 ;RUN_AT_START = false
 ;ENABLE_SUCCESS_NOTICE = false
 ;SCHEDULE = @every 168h
-;HTTP_ENDPOINT = https://dl.gitea.io/gitea/version.json
+;HTTP_ENDPOINT = https://dl.gitea.com/gitea/version.json
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docker/manifest.rootless.tmpl

@@ -1,12 +1,14 @@
 image: gitea/gitea:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{#if (hasPrefix "refs/heads/release/v" build.ref)}}{{trimPrefix "refs/heads/release/v" build.ref}}-{{/if}}nightly{{/if}}-rootless
 {{#if build.tags}}
 {{#unless (contains "-rc" build.tag)}}
+{{#unless (contains "-dev" build.tag)}}
 tags:
 {{#each build.tags}}
   - {{this}}-rootless
 {{/each}}
   - "latest-rootless"
 {{/unless}}
+{{/unless}}
 {{/if}}
 manifests:
   -

docker/manifest.tmpl

@@ -1,12 +1,14 @@
 image: gitea/gitea:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{#if (hasPrefix "refs/heads/release/v" build.ref)}}{{trimPrefix "refs/heads/release/v" build.ref}}-{{/if}}nightly{{/if}}
 {{#if build.tags}}
 {{#unless (contains "-rc" build.tag)}}
+{{#unless (contains "-dev" build.tag)}}
 tags:
 {{#each build.tags}}
   - {{this}}
 {{/each}}
   - "latest"
 {{/unless}}
+{{/unless}}
 {{/if}}
 manifests:
   -

docs/content/doc/administration/command-line.en-us.md

@@ -108,6 +108,14 @@ Admin operations:
         - `--all`, `-A`: Force a password change for all users
         - `--exclude username`, `-e username`: Exclude the given user. Can be set multiple times.
         - `--unset`: Revoke forced password change for the given users
+    - `generate-access-token`:
+      - Options:
+        - `--username value`, `-u value`: Username. Required.
+        - `--token-name value`, `-t value`: Token name. Required.
+        - `--scopes value`: Comma-separated list of scopes. Scopes follow the format `[read|write]:<block>` or `all` where `<block>` is one of the available visual groups you can see when opening the API page showing the available routes (for example `repo`).
+      - Examples:
+        - `gitea admin user generate-access-token --username myname --token-name mytoken`
+        - `gitea admin user generate-access-token --help`
   - `regenerate`
     - Options:
       - `hooks`: Regenerate Git Hooks for all repositories

docs/content/doc/administration/config-cheat-sheet.en-us.md

@@ -317,8 +317,11 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `LOCAL_ROOT_URL`: **%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/**: Local
    (DMZ) URL for Gitea workers (such as SSH update) accessing web service. In
    most cases you do not need to change the default value. Alter it only if
-   your SSH server node is not the same as HTTP node. Do not set this variable
-   if `PROTOCOL` is set to `http+unix`.
+   your SSH server node is not the same as HTTP node. For different protocol, the default
+   values are different. If `PROTOCOL` is `http+unix`, the default value is `http://unix/`.
+   If `PROTOCOL` is `fcgi` or `fcgi+unix`, the default value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
+   If listen on `0.0.0.0`, the default value is `%(PROTOCOL)s://localhost:%(HTTP_PORT)s/`, Otherwise the default
+   value is `%(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/`.
 - `LOCAL_USE_PROXY_PROTOCOL`: **%(USE_PROXY_PROTOCOL)s**: When making local connections pass the PROXY protocol header.
    This should be set to false if the local connection will go through the proxy.
 - `PER_WRITE_TIMEOUT`: **30s**: Timeout for any write to the connection. (Set to -1 to
@@ -446,7 +449,6 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `SQLITE_TIMEOUT`: **500**: Query timeout for SQLite3 only.
 - `SQLITE_JOURNAL_MODE`: **""**: Change journal mode for SQlite3. Can be used to enable [WAL mode](https://www.sqlite.org/wal.html) when high load causes write congestion. See [SQlite3 docs](https://www.sqlite.org/pragma.html#pragma_journal_mode) for possible values. Defaults to the default for the database file, often DELETE.
 - `ITERATE_BUFFER_SIZE`: **50**: Internal buffer size for iterating.
-- `CHARSET`: **utf8mb4**: For MySQL only, either "utf8" or "utf8mb4". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.
 - `LOG_SQL`: **true**: Log the executed SQL.
 - `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed.
@@ -462,15 +464,15 @@ relation to port exhaustion.
 ## Indexer (`indexer`)
 
 - `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently supported: `bleve`, `db`, `elasticsearch` or `meilisearch`.
-- `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch, or meilisearch. i.e. http://elastic:changeme@localhost:9200
-- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
+- `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch (e.g. http://elastic:password@localhost:9200) or meilisearch (e.g. http://:apikey@localhost:7700)
+- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch or meilisearch.
 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_REPO_TYPES`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`.
 - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
-- `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200
+- `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:password@localhost:9200
 - `REPO_INDEXER_NAME`: **gitea_codes**: Code indexer name, available when `REPO_INDEXER_TYPE` is elasticsearch
 
 - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
@@ -1016,7 +1018,7 @@ Default templates for project boards:
 - `RUN_AT_START`: **false**: Run tasks at start up time (if ENABLED).
 - `ENABLE_SUCCESS_NOTICE`: **true**: Set to false to switch off success notices.
 - `SCHEDULE`: **@every 168h**: Cron syntax for scheduling a work, e.g. `@every 168h`.
-- `HTTP_ENDPOINT`: **https://dl.gitea.io/gitea/version.json**: the endpoint that Gitea will check for newer versions
+- `HTTP_ENDPOINT`: **https://dl.gitea.com/gitea/version.json**: the endpoint that Gitea will check for newer versions
 
 #### Cron -  Delete all old system notices from database (`cron.delete_old_system_notices`)
 

docs/content/doc/help/faq.en-us.md

@@ -396,8 +396,6 @@ Please run `gitea convert`, or run `ALTER DATABASE database_name CHARACTER SET u
 for the database_name and run `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`
 for each table in the database.
 
-You will also need to change the app.ini database charset to `CHARSET=utf8mb4`.
-
 ## Why are Emoji displaying only as placeholders or in monochrome
 
 Gitea requires the system or browser to have one of the supported Emoji fonts installed, which are Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji and Twemoji Mozilla. Generally, the operating system should already provide one of these fonts, but especially on Linux, it may be necessary to install them manually.

docs/content/doc/help/faq.zh-cn.md

@@ -31,7 +31,7 @@ menu:
 
 **注意：**此示例也适用于Docker镜像！
 
-在我们的[下载页面](https://dl.gitea.io/gitea/)上，您会看到一个1.7目录，以及1.7.0、1.7.1、1.7.2、1.7.3、1.7.4、1.7.5和1.7.6的目录。
+在我们的[下载页面](https://dl.gitea.com/gitea/)上，您会看到一个1.7目录，以及1.7.0、1.7.1、1.7.2、1.7.3、1.7.4、1.7.5和1.7.6的目录。
 
 1.7目录和1.7.0目录是**不同**的。1.7目录是在每个合并到[`release/v1.7`](https://github.com/go-gitea/gitea/tree/release/v1.7)分支的提交上构建的。
 

docs/content/doc/installation/from-binary.fr-fr.md

@@ -17,10 +17,10 @@ menu:
 
 # Installation avec le binaire pré-compilé
 
-Tous les binaires sont livrés avec le support de SQLite, MySQL et PostgreSQL, et sont construits avec les ressources incorporées. Gardez à l'esprit que cela peut être différent pour les versions antérieures. L'installation basée sur nos binaires est assez simple, il suffit de choisir le fichier correspondant à votre plateforme à partir de la [page de téléchargement](https://dl.gitea.io/gitea). Copiez l'URL et remplacer l'URL dans les commandes suivantes par la nouvelle:
+Tous les binaires sont livrés avec le support de SQLite, MySQL et PostgreSQL, et sont construits avec les ressources incorporées. Gardez à l'esprit que cela peut être différent pour les versions antérieures. L'installation basée sur nos binaires est assez simple, il suffit de choisir le fichier correspondant à votre plateforme à partir de la [page de téléchargement](https://dl.gitea.com/gitea). Copiez l'URL et remplacer l'URL dans les commandes suivantes par la nouvelle:
 
 ```
-wget -O gitea https://dl.gitea.io/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
+wget -O gitea https://dl.gitea.com/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
 chmod +x gitea
 ```
 

docs/content/doc/installation/from-binary.zh-tw.md

@@ -17,10 +17,10 @@ menu:
 
 # 從執行檔安裝
 
-所有的執行檔皆支援 SQLite, MySQL and PostgreSQL，且所有檔案都已經包在執行檔內，這一點跟之前的版本有所不同。關於執行檔的安裝方式非常簡單，只要從[下載頁面](https://dl.gitea.io/gitea)選擇相對應平台，複製下載連結，使用底下指令就可以完成了:
+所有的執行檔皆支援 SQLite, MySQL and PostgreSQL，且所有檔案都已經包在執行檔內，這一點跟之前的版本有所不同。關於執行檔的安裝方式非常簡單，只要從[下載頁面](https://dl.gitea.com/gitea)選擇相對應平台，複製下載連結，使用底下指令就可以完成了:
 
 ```
-wget -O gitea https://dl.gitea.io/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
+wget -O gitea https://dl.gitea.com/gitea/{{< version >}}/gitea-{{< version >}}-linux-amd64
 chmod +x gitea
 ```
 

docs/content/doc/installation/on-kubernetes.en-us.md

@@ -22,7 +22,7 @@ Gitea provides a Helm Chart to allow for installation on kubernetes.
 A non-customized install can be done with:
 
 ```
-helm repo add gitea-charts https://dl.gitea.io/charts/
+helm repo add gitea-charts https://dl.gitea.com/charts/
 helm install gitea gitea-charts/gitea
 ```
 

docs/content/doc/installation/on-kubernetes.zh-cn.md

@@ -22,7 +22,7 @@ Gitea 已经提供了便于在 Kubernetes 云原生环境中安装所需的 Helm
 默认安装指令为：
 
 ```bash
-helm repo add gitea https://dl.gitea.io/charts
+helm repo add gitea https://dl.gitea.com/charts
 helm repo update
 helm install gitea gitea/gitea
 ```

docs/content/doc/installation/on-kubernetes.zh-tw.md

@@ -22,7 +22,7 @@ Gitea 提供 Helm Chart 用來安裝於 kubernetes。
 非自訂安裝可使用下列指令：
 
 ```
-helm repo add gitea-charts https://dl.gitea.io/charts/
+helm repo add gitea-charts https://dl.gitea.com/charts/
 helm install gitea gitea-charts/gitea
 ```
 

docs/content/doc/installation/upgrade-from-gogs.en-us.md

@@ -27,7 +27,7 @@ There are some basic steps to follow. On a Linux system run as the Gogs user:
 
 * Create a Gogs backup with `gogs backup`. This creates `gogs-backup-[timestamp].zip` file
   containing all important Gogs data. You would need it if you wanted to move to the `gogs` back later.
-* Download the file matching the destination platform from the [downloads page](https://dl.gitea.io/gitea/).
+* Download the file matching the destination platform from the [downloads page](https://dl.gitea.com/gitea/).
  It should be `1.0.x` version. Migrating from `gogs` to any other version is impossible.
 * Put the binary at the desired install location.
 * Copy `gogs/custom/conf/app.ini` to `gitea/custom/conf/app.ini`.
@@ -79,11 +79,11 @@ There are some basic steps to follow. On a Linux system run as the Gogs user:
 After successful migration from `gogs` to `gitea 1.0.x`, it is possible to upgrade `gitea` to a modern version
 in a two steps process.
 
-Upgrade to [`gitea 1.6.4`](https://dl.gitea.io/gitea/1.6.4/) first. Download the file matching
-the destination platform from the [downloads page](https://dl.gitea.io/gitea/1.6.4/) and replace the binary.
+Upgrade to [`gitea 1.6.4`](https://dl.gitea.com/gitea/1.6.4/) first. Download the file matching
+the destination platform from the [downloads page](https://dl.gitea.com/gitea/1.6.4/) and replace the binary.
 Run Gitea at least once and check that everything works as expected.
 
-Then repeat the procedure, but this time using the [latest release](https://dl.gitea.io/gitea/{{< version >}}/).
+Then repeat the procedure, but this time using the [latest release](https://dl.gitea.com/gitea/{{< version >}}/).
 
 ## Upgrading from a more recent version of Gogs
 

docs/content/doc/installation/upgrade-from-gogs.fr-fr.md

@@ -22,7 +22,7 @@ menu:
 Veuillez suivre les étapes ci-dessous. Sur Unix, toute les commandes s'exécutent en tant que l'utilisateur utilisé pour votre installation de Gogs :
 
 * Crééer une sauvegarde de Gogs avec la commande `gogs dump`. Le fichier nouvellement créé `gogs-dump-[timestamp].zip` contient toutes les données de votre instance de Gogs.
-* Téléchargez le fichier correspondant à votre plateforme à partir de la [page de téléchargements](https://dl.gitea.io/gitea).
+* Téléchargez le fichier correspondant à votre plateforme à partir de la [page de téléchargements](https://dl.gitea.com/gitea).
 * Mettez la binaire dans le répertoire d'installation souhaité.
 * Copiez le fichier `gogs/custom/conf/app.ini` vers `gitea/custom/conf/app.ini`.
 * Si vous avez personnalisé les répertoires `templates, public` dans `gogs/custom/`, copiez-les vers `gitea/custom/`.

docs/content/doc/installation/upgrade-from-gogs.zh-tw.md

@@ -27,7 +27,7 @@ menu:
 
 - 使用 `gogs backup` 建立 Gogs 的備份。這會建立檔案 `gogs-backup-[timestamp].zip` 包含所有重要的 Gogs 資料。
   如果稍後您要恢復到 `gogs` 時會用到它。
-- 從[下載頁](https://dl.gitea.io/gitea/)下載對應您平臺的檔案。請下載 `1.0.x` 版，從 `gogs` 遷移到其它版本是不可行的。
+- 從[下載頁](https://dl.gitea.com/gitea/)下載對應您平臺的檔案。請下載 `1.0.x` 版，從 `gogs` 遷移到其它版本是不可行的。
 - 將二進位檔放到適當的安裝位置。
 - 複製 `gogs/custom/conf/app.ini` 到 `gitea/custom/conf/app.ini`。
 - 從 `gogs/custom/` 複製自訂 `templates, public` 到 `gitea/custom/`。
@@ -77,10 +77,10 @@ menu:
 
 成功從 `gogs` 升級到 `gitea 1.0.x` 後再用 2 個步驟即可升級到最新版的 `gitea`。
 
-請先升級到 [`gitea 1.6.4`](https://dl.gitea.io/gitea/1.6.4/)，先從[下載頁](https://dl.gitea.io/gitea/1.6.4/)下載
+請先升級到 [`gitea 1.6.4`](https://dl.gitea.com/gitea/1.6.4/)，先從[下載頁](https://dl.gitea.com/gitea/1.6.4/)下載
 您平臺的二進位檔取代既有的。至少執行一次 Gitea 並確認一切符合預期。
 
-接著重複上述步驟，但這次請使用[最新發行版本](https://dl.gitea.io/gitea/{{< version >}}/)。
+接著重複上述步驟，但這次請使用[最新發行版本](https://dl.gitea.com/gitea/{{< version >}}/)。
 
 ## 從更新版本的 Gogs 升級
 

docs/content/doc/usage/template-repositories.en-us.md

@@ -51,6 +51,8 @@ a/b/c/d.json
 
 In any file matched by the above globs, certain variables will be expanded.
 
+Matching filenames and paths can also be expanded, and are conservatively sanitized to support cross-platform filesystems.
+
 All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}`
 
 | Variable             | Expands To                                          | Transformable |

go.mod

@@ -122,7 +122,7 @@ require (
 	mvdan.cc/xurls/v2 v2.4.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.12
-	xorm.io/xorm v1.3.3-0.20230219231735-056cecc97e9e
+	xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75
 )
 
 require (

go.sum

@@ -1923,5 +1923,5 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1:
 xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
 xorm.io/builder v0.3.12 h1:ASZYX7fQmy+o8UJdhlLHSW57JDOkM8DNhcAF5d0LiJM=
 xorm.io/builder v0.3.12/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
-xorm.io/xorm v1.3.3-0.20230219231735-056cecc97e9e h1:d5PY6mwuQK5/7T6VKfFswaKMzLmGTHkJ/ZS7+cUIAjk=
-xorm.io/xorm v1.3.3-0.20230219231735-056cecc97e9e/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw=
+xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75 h1:ReBAlO50dCIXCWF8Gbi0ZRa62AGAwCJNCPaUNUa7JSg=
+xorm.io/xorm v1.3.3-0.20230623150031-18f8e7a86c75/go.mod h1:9NbjqdnjX6eyjRRhh01GHm64r6N9shTb/8Ak3YRt8Nw=

main.go

@@ -33,162 +33,164 @@ var (
 	Tags = ""
 	// MakeVersion holds the current Make version if built with make
 	MakeVersion = ""
-
-	originalAppHelpTemplate        = ""
-	originalCommandHelpTemplate    = ""
-	originalSubcommandHelpTemplate = ""
 )
 
 func init() {
 	setting.AppVer = Version
 	setting.AppBuiltWith = formatBuiltWith()
 	setting.AppStartTime = time.Now().UTC()
+}
 
-	// Grab the original help templates
-	originalAppHelpTemplate = cli.AppHelpTemplate
-	originalCommandHelpTemplate = cli.CommandHelpTemplate
-	originalSubcommandHelpTemplate = cli.SubcommandHelpTemplate
+// cmdHelp is our own help subcommand with more information
+// test cases:
+// ./gitea help
+// ./gitea -h
+// ./gitea web help
+// ./gitea web -h (due to cli lib limitation, this won't call our cmdHelp, so no extra info)
+// ./gitea admin
+// ./gitea admin help
+// ./gitea admin auth help
+// ./gitea -c /tmp/app.ini -h
+// ./gitea -c /tmp/app.ini help
+// ./gitea help -c /tmp/app.ini
+// GITEA_WORK_DIR=/tmp ./gitea help
+// GITEA_WORK_DIR=/tmp ./gitea help --work-path /tmp/other
+// GITEA_WORK_DIR=/tmp ./gitea help --config /tmp/app-other.ini
+var cmdHelp = cli.Command{
+	Name:      "help",
+	Aliases:   []string{"h"},
+	Usage:     "Shows a list of commands or help for one command",
+	ArgsUsage: "[command]",
+	Action: func(c *cli.Context) (err error) {
+		args := c.Args()
+		if args.Present() {
+			err = cli.ShowCommandHelp(c, args.First())
+		} else {
+			err = cli.ShowAppHelp(c)
+		}
+		_, _ = fmt.Fprintf(c.App.Writer, `
+DEFAULT CONFIGURATION:
+   AppPath:    %s
+   WorkPath:   %s
+   CustomPath: %s
+   ConfigFile: %s
+
+`, setting.AppPath, setting.AppWorkPath, setting.CustomPath, setting.CustomConf)
+		return err
+	},
 }
 
 func main() {
 	app := cli.NewApp()
 	app.Name = "Gitea"
 	app.Usage = "A painless self-hosted Git service"
-	app.Description = `By default, gitea will start serving using the webserver with no
-arguments - which can alternatively be run by running the subcommand web.`
+	app.Description = `By default, Gitea will start serving using the web-server with no argument, which can alternatively be run by running the subcommand "web".`
 	app.Version = Version + formatBuiltWith()
-	app.Commands = []cli.Command{
+	app.EnableBashCompletion = true
+
+	// these sub-commands need to use config file
+	subCmdWithIni := []cli.Command{
 		cmd.CmdWeb,
 		cmd.CmdServ,
 		cmd.CmdHook,
 		cmd.CmdDump,
-		cmd.CmdCert,
 		cmd.CmdAdmin,
-		cmd.CmdGenerate,
 		cmd.CmdMigrate,
 		cmd.CmdKeys,
 		cmd.CmdConvert,
 		cmd.CmdDoctor,
 		cmd.CmdManager,
-		cmd.Cmdembedded,
+		cmd.CmdEmbedded,
 		cmd.CmdMigrateStorage,
-		cmd.CmdDocs,
 		cmd.CmdDumpRepository,
 		cmd.CmdRestoreRepository,
 		cmd.CmdActions,
+		cmdHelp, // TODO: the "help" sub-command was used to show the more information for "work path" and "custom config", in the future, it should avoid doing so
+	}
+	// these sub-commands do not need the config file, and they do not depend on any path or environment variable.
+	subCmdStandalone := []cli.Command{
+		cmd.CmdCert,
+		cmd.CmdGenerate,
+		cmd.CmdDocs,
 	}
-	// Now adjust these commands to add our global configuration options
 
-	// First calculate the default paths and set the AppHelpTemplates in this context
-	setting.SetCustomPathAndConf("", "", "")
-	setAppHelpTemplates()
-
-	// default configuration flags
-	defaultFlags := []cli.Flag{
+	// shared configuration flags, they are for global and for each sub-command at the same time
+	// eg: such command is valid: "./gitea --config /tmp/app.ini web --config /tmp/app.ini", while it's discouraged indeed
+	// keep in mind that the short flags like "-C", "-c" and "-w" are globally polluted, they can't be used for sub-commands anymore.
+	globalFlags := []cli.Flag{
+		cli.HelpFlag,
 		cli.StringFlag{
 			Name:  "custom-path, C",
-			Value: setting.CustomPath,
-			Usage: "Custom path file path",
+			Usage: "Set custom path (defaults to '{WorkPath}/custom')",
 		},
 		cli.StringFlag{
 			Name:  "config, c",
 			Value: setting.CustomConf,
-			Usage: "Custom configuration file path",
+			Usage: "Set custom config file (defaults to '{WorkPath}/custom/conf/app.ini')",
 		},
-		cli.VersionFlag,
 		cli.StringFlag{
 			Name:  "work-path, w",
-			Value: setting.AppWorkPath,
-			Usage: "Set the gitea working path",
+			Usage: "Set Gitea's working path (defaults to the Gitea's binary directory)",
 		},
 	}
 
 	// Set the default to be equivalent to cmdWeb and add the default flags
-	app.Flags = append(app.Flags, cmd.CmdWeb.Flags...)
-	app.Flags = append(app.Flags, defaultFlags...)
-	app.Action = cmd.CmdWeb.Action
-
-	// Add functions to set these paths and these flags to the commands
-	app.Before = establishCustomPath
-	for i := range app.Commands {
-		setFlagsAndBeforeOnSubcommands(&app.Commands[i], defaultFlags, establishCustomPath)
+	app.Flags = append(app.Flags, globalFlags...)
+	app.Flags = append(app.Flags, cmd.CmdWeb.Flags...) // TODO: the web flags polluted the global flags, they are not really global flags
+	app.Action = prepareWorkPathAndCustomConf(cmd.CmdWeb.Action)
+	app.HideHelp = true // use our own help action to show helps (with more information like default config)
+	app.Before = cmd.PrepareConsoleLoggerLevel(log.INFO)
+	for i := range subCmdWithIni {
+		prepareSubcommands(&subCmdWithIni[i], globalFlags)
 	}
-
-	app.EnableBashCompletion = true
+	app.Commands = append(app.Commands, subCmdWithIni...)
+	app.Commands = append(app.Commands, subCmdStandalone...)
 
 	err := app.Run(os.Args)
 	if err != nil {
-		log.Fatal("Failed to run app with %s: %v", os.Args, err)
+		_, _ = fmt.Fprintf(app.Writer, "\nFailed to run with %s: %v\n", os.Args, err)
 	}
 
 	log.GetManager().Close()
 }
 
-func setFlagsAndBeforeOnSubcommands(command *cli.Command, defaultFlags []cli.Flag, before cli.BeforeFunc) {
+func prepareSubcommands(command *cli.Command, defaultFlags []cli.Flag) {
 	command.Flags = append(command.Flags, defaultFlags...)
-	command.Before = establishCustomPath
+	command.Action = prepareWorkPathAndCustomConf(command.Action)
+	command.HideHelp = true
+	if command.Name != "help" {
+		command.Subcommands = append(command.Subcommands, cmdHelp)
+	}
 	for i := range command.Subcommands {
-		setFlagsAndBeforeOnSubcommands(&command.Subcommands[i], defaultFlags, before)
+		prepareSubcommands(&command.Subcommands[i], defaultFlags)
 	}
 }
 
-func establishCustomPath(ctx *cli.Context) error {
-	var providedCustom string
-	var providedConf string
-	var providedWorkPath string
-
-	currentCtx := ctx
-	for {
-		if len(providedCustom) != 0 && len(providedConf) != 0 && len(providedWorkPath) != 0 {
-			break
+// prepareWorkPathAndCustomConf wraps the Action to prepare the work path and custom config
+// It can't use "Before", because each level's sub-command's Before will be called one by one, so the "init" would be done multiple times
+func prepareWorkPathAndCustomConf(action any) func(ctx *cli.Context) error {
+	return func(ctx *cli.Context) error {
+		var args setting.ArgWorkPathAndCustomConf
+		curCtx := ctx
+		for curCtx != nil {
+			if curCtx.IsSet("work-path") && args.WorkPath == "" {
+				args.WorkPath = curCtx.String("work-path")
+			}
+			if curCtx.IsSet("custom-path") && args.CustomPath == "" {
+				args.CustomPath = curCtx.String("custom-path")
+			}
+			if curCtx.IsSet("config") && args.CustomConf == "" {
+				args.CustomConf = curCtx.String("config")
+			}
+			curCtx = curCtx.Parent()
 		}
-		if currentCtx == nil {
-			break
+		setting.InitWorkPathAndCommonConfig(os.Getenv, args)
+		if ctx.Bool("help") || action == nil {
+			// the default behavior of "urfave/cli": "nil action" means "show help"
+			return cmdHelp.Action.(func(ctx *cli.Context) error)(ctx)
 		}
-		if currentCtx.IsSet("custom-path") && len(providedCustom) == 0 {
-			providedCustom = currentCtx.String("custom-path")
-		}
-		if currentCtx.IsSet("config") && len(providedConf) == 0 {
-			providedConf = currentCtx.String("config")
-		}
-		if currentCtx.IsSet("work-path") && len(providedWorkPath) == 0 {
-			providedWorkPath = currentCtx.String("work-path")
-		}
-		currentCtx = currentCtx.Parent()
-
+		return action.(func(*cli.Context) error)(ctx)
 	}
-	setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath)
-
-	setAppHelpTemplates()
-
-	if ctx.IsSet("version") {
-		cli.ShowVersion(ctx)
-		os.Exit(0)
-	}
-
-	return nil
-}
-
-func setAppHelpTemplates() {
-	cli.AppHelpTemplate = adjustHelpTemplate(originalAppHelpTemplate)
-	cli.CommandHelpTemplate = adjustHelpTemplate(originalCommandHelpTemplate)
-	cli.SubcommandHelpTemplate = adjustHelpTemplate(originalSubcommandHelpTemplate)
-}
-
-func adjustHelpTemplate(originalTemplate string) string {
-	overridden := ""
-	if _, ok := os.LookupEnv("GITEA_CUSTOM"); ok {
-		overridden = "(GITEA_CUSTOM)"
-	}
-
-	return fmt.Sprintf(`%s
-DEFAULT CONFIGURATION:
-     CustomPath:  %s %s
-     CustomConf:  %s
-     AppPath:     %s
-     AppWorkPath: %s
-
-`, originalTemplate, setting.CustomPath, overridden, setting.CustomConf, setting.AppPath, setting.AppWorkPath)
 }
 
 func formatBuiltWith() string {

models/actions/run.go

@@ -36,12 +36,13 @@ type ActionRun struct {
 	TriggerUser       *user_model.User       `xorm:"-"`
 	Ref               string
 	CommitSHA         string
-	IsForkPullRequest bool  // If this is triggered by a PR from a forked repository or an untrusted user, we need to check if it is approved and limit permissions when running the workflow.
-	NeedApproval      bool  // may need approval if it's a fork pull request
-	ApprovedBy        int64 `xorm:"index"` // who approved
-	Event             webhook_module.HookEventType
-	EventPayload      string `xorm:"LONGTEXT"`
-	Status            Status `xorm:"index"`
+	IsForkPullRequest bool                         // If this is triggered by a PR from a forked repository or an untrusted user, we need to check if it is approved and limit permissions when running the workflow.
+	NeedApproval      bool                         // may need approval if it's a fork pull request
+	ApprovedBy        int64                        `xorm:"index"` // who approved
+	Event             webhook_module.HookEventType // the webhook event that causes the workflow to run
+	EventPayload      string                       `xorm:"LONGTEXT"`
+	TriggerEvent      string                       // the trigger event defined in the `on` configuration of the triggered workflow
+	Status            Status                       `xorm:"index"`
 	Started           timeutil.TimeStamp
 	Stopped           timeutil.TimeStamp
 	Created           timeutil.TimeStamp `xorm:"created"`

models/actions/run_list.go

@@ -71,6 +71,7 @@ type FindRunOptions struct {
 	WorkflowFileName string
 	TriggerUserID    int64
 	Approved         bool // not util.OptionalBool, it works only when it's true
+	Status           Status
 }
 
 func (opts FindRunOptions) toConds() builder.Cond {
@@ -90,6 +91,9 @@ func (opts FindRunOptions) toConds() builder.Cond {
 	if opts.Approved {
 		cond = cond.And(builder.Gt{"approved_by": 0})
 	}
+	if opts.Status > StatusUnknown {
+		cond = cond.And(builder.Eq{"status": opts.Status})
+	}
 	return cond
 }
 
@@ -106,3 +110,34 @@ func FindRuns(ctx context.Context, opts FindRunOptions) (RunList, int64, error)
 func CountRuns(ctx context.Context, opts FindRunOptions) (int64, error) {
 	return db.GetEngine(ctx).Where(opts.toConds()).Count(new(ActionRun))
 }
+
+type StatusInfo struct {
+	Status          int
+	DisplayedStatus string
+}
+
+// GetStatusInfoList returns a slice of StatusInfo
+func GetStatusInfoList(ctx context.Context) []StatusInfo {
+	// same as those in aggregateJobStatus
+	allStatus := []Status{StatusSuccess, StatusFailure, StatusWaiting, StatusRunning}
+	statusInfoList := make([]StatusInfo, 0, 4)
+	for _, s := range allStatus {
+		statusInfoList = append(statusInfoList, StatusInfo{
+			Status:          int(s),
+			DisplayedStatus: s.String(),
+		})
+	}
+	return statusInfoList
+}
+
+// GetActors returns a slice of Actors
+func GetActors(ctx context.Context, repoID int64) ([]*user_model.User, error) {
+	actors := make([]*user_model.User, 0, 10)
+
+	return actors, db.GetEngine(ctx).Where(builder.In("id", builder.Select("`action_run`.trigger_user_id").From("`action_run`").
+		GroupBy("`action_run`.trigger_user_id").
+		Where(builder.Eq{"`action_run`.repo_id": repoID}))).
+		Cols("id", "name", "full_name", "avatar", "avatar_email", "use_custom_avatar").
+		OrderBy(user_model.GetOrderByName()).
+		Find(&actors)
+}

models/actions/task.go

@@ -344,6 +344,9 @@ func UpdateTask(ctx context.Context, task *ActionTask, cols ...string) error {
 	return err
 }
 
+// UpdateTaskByState updates the task by the state.
+// It will always update the task if the state is not final, even there is no change.
+// So it will update ActionTask.Updated to avoid the task being judged as a zombie task.
 func UpdateTaskByState(ctx context.Context, state *runnerv1.TaskState) (*ActionTask, error) {
 	stepStates := map[int64]*runnerv1.StepState{}
 	for _, v := range state.Steps {
@@ -384,6 +387,12 @@ func UpdateTaskByState(ctx context.Context, state *runnerv1.TaskState) (*ActionT
 		}, nil); err != nil {
 			return nil, err
 		}
+	} else {
+		// Force update ActionTask.Updated to avoid the task being judged as a zombie task
+		task.Updated = timeutil.TimeStampNow()
+		if err := UpdateTask(ctx, task, "updated"); err != nil {
+			return nil, err
+		}
 	}
 
 	if err := task.LoadAttributes(ctx); err != nil {

models/actions/variable.go

@@ -0,0 +1,97 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
+
+	"xorm.io/builder"
+)
+
+type ActionVariable struct {
+	ID          int64              `xorm:"pk autoincr"`
+	OwnerID     int64              `xorm:"UNIQUE(owner_repo_name)"`
+	RepoID      int64              `xorm:"INDEX UNIQUE(owner_repo_name)"`
+	Name        string             `xorm:"UNIQUE(owner_repo_name) NOT NULL"`
+	Data        string             `xorm:"LONGTEXT NOT NULL"`
+	CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"`
+	UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
+}
+
+func init() {
+	db.RegisterModel(new(ActionVariable))
+}
+
+func (v *ActionVariable) Validate() error {
+	if v.OwnerID == 0 && v.RepoID == 0 {
+		return errors.New("the variable is not bound to any scope")
+	}
+	return nil
+}
+
+func InsertVariable(ctx context.Context, ownerID, repoID int64, name, data string) (*ActionVariable, error) {
+	variable := &ActionVariable{
+		OwnerID: ownerID,
+		RepoID:  repoID,
+		Name:    strings.ToUpper(name),
+		Data:    data,
+	}
+	if err := variable.Validate(); err != nil {
+		return variable, err
+	}
+	return variable, db.Insert(ctx, variable)
+}
+
+type FindVariablesOpts struct {
+	db.ListOptions
+	OwnerID int64
+	RepoID  int64
+}
+
+func (opts *FindVariablesOpts) toConds() builder.Cond {
+	cond := builder.NewCond()
+	if opts.OwnerID > 0 {
+		cond = cond.And(builder.Eq{"owner_id": opts.OwnerID})
+	}
+	if opts.RepoID > 0 {
+		cond = cond.And(builder.Eq{"repo_id": opts.RepoID})
+	}
+	return cond
+}
+
+func FindVariables(ctx context.Context, opts FindVariablesOpts) ([]*ActionVariable, error) {
+	var variables []*ActionVariable
+	sess := db.GetEngine(ctx)
+	if opts.PageSize != 0 {
+		sess = db.SetSessionPagination(sess, &opts.ListOptions)
+	}
+	return variables, sess.Where(opts.toConds()).Find(&variables)
+}
+
+func GetVariableByID(ctx context.Context, variableID int64) (*ActionVariable, error) {
+	var variable ActionVariable
+	has, err := db.GetEngine(ctx).Where("id=?", variableID).Get(&variable)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, fmt.Errorf("variable with id %d: %w", variableID, util.ErrNotExist)
+	}
+	return &variable, nil
+}
+
+func UpdateVariable(ctx context.Context, variable *ActionVariable) (bool, error) {
+	count, err := db.GetEngine(ctx).ID(variable.ID).Cols("name", "data").
+		Update(&ActionVariable{
+			Name: variable.Name,
+			Data: variable.Data,
+		})
+	return count != 0, err
+}

models/db/engine.go

@@ -123,7 +123,10 @@ func newXORMEngine() (*xorm.Engine, error) {
 
 // SyncAllTables sync the schemas of all tables, is required by unit test code
 func SyncAllTables() error {
-	return x.StoreEngine("InnoDB").Sync2(tables...)
+	_, err := x.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{
+		WarnIfDatabaseColumnMissed: true,
+	}, tables...)
+	return err
 }
 
 // InitEngine initializes the xorm.Engine and sets it as db.DefaultContext

models/db/search.go

@@ -20,6 +20,10 @@ const (
 	SearchOrderByNewest                SearchOrderBy = "created_unix DESC"
 	SearchOrderBySize                  SearchOrderBy = "size ASC"
 	SearchOrderBySizeReverse           SearchOrderBy = "size DESC"
+	SearchOrderByGitSize               SearchOrderBy = "git_size ASC"
+	SearchOrderByGitSizeReverse        SearchOrderBy = "git_size DESC"
+	SearchOrderByLFSSize               SearchOrderBy = "lfs_size ASC"
+	SearchOrderByLFSSizeReverse        SearchOrderBy = "lfs_size DESC"
 	SearchOrderByID                    SearchOrderBy = "id ASC"
 	SearchOrderByIDReverse             SearchOrderBy = "id DESC"
 	SearchOrderByStars                 SearchOrderBy = "num_stars ASC"

models/dbfs/dbfile.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -21,6 +22,7 @@ var defaultFileBlockSize int64 = 32 * 1024
 type File interface {
 	io.ReadWriteCloser
 	io.Seeker
+	fs.File
 }
 
 type file struct {
@@ -193,10 +195,26 @@ func (f *file) Close() error {
 	return nil
 }
 
+func (f *file) Stat() (os.FileInfo, error) {
+	if f.metaID == 0 {
+		return nil, os.ErrInvalid
+	}
+
+	fileMeta, err := findFileMetaByID(f.ctx, f.metaID)
+	if err != nil {
+		return nil, err
+	}
+	return fileMeta, nil
+}
+
 func timeToFileTimestamp(t time.Time) int64 {
 	return t.UnixMicro()
 }
 
+func fileTimestampToTime(timestamp int64) time.Time {
+	return time.UnixMicro(timestamp)
+}
+
 func (f *file) loadMetaByPath() (*dbfsMeta, error) {
 	var fileMeta dbfsMeta
 	if ok, err := db.GetEngine(f.ctx).Where("full_path = ?", f.fullPath).Get(&fileMeta); err != nil {

models/dbfs/dbfs.go

@@ -5,7 +5,10 @@ package dbfs
 
 import (
 	"context"
+	"io/fs"
 	"os"
+	"path"
+	"time"
 
 	"code.gitea.io/gitea/models/db"
 )
@@ -100,3 +103,29 @@ func Remove(ctx context.Context, name string) error {
 	defer f.Close()
 	return f.delete()
 }
+
+var _ fs.FileInfo = (*dbfsMeta)(nil)
+
+func (m *dbfsMeta) Name() string {
+	return path.Base(m.FullPath)
+}
+
+func (m *dbfsMeta) Size() int64 {
+	return m.FileSize
+}
+
+func (m *dbfsMeta) Mode() fs.FileMode {
+	return os.ModePerm
+}
+
+func (m *dbfsMeta) ModTime() time.Time {
+	return fileTimestampToTime(m.ModifyTimestamp)
+}
+
+func (m *dbfsMeta) IsDir() bool {
+	return false
+}
+
+func (m *dbfsMeta) Sys() any {
+	return nil
+}

models/dbfs/dbfs_test.go

@@ -111,6 +111,19 @@ func TestDbfsBasic(t *testing.T) {
 
 	_, err = OpenFile(db.DefaultContext, "test2.txt", os.O_RDONLY)
 	assert.Error(t, err)
+
+	// test stat
+	f, err = OpenFile(db.DefaultContext, "test/test.txt", os.O_RDWR|os.O_CREATE)
+	assert.NoError(t, err)
+	stat, err := f.Stat()
+	assert.NoError(t, err)
+	assert.EqualValues(t, "test.txt", stat.Name())
+	assert.EqualValues(t, 0, stat.Size())
+	_, err = f.Write([]byte("0123456789"))
+	assert.NoError(t, err)
+	stat, err = f.Stat()
+	assert.NoError(t, err)
+	assert.EqualValues(t, 10, stat.Size())
 }
 
 func TestDbfsReadWrite(t *testing.T) {

models/issues/comment.go

@@ -167,7 +167,7 @@ func AsCommentType(typeName string) CommentType {
 
 func (t CommentType) HasContentSupport() bool {
 	switch t {
-	case CommentTypeComment, CommentTypeCode, CommentTypeReview:
+	case CommentTypeComment, CommentTypeCode, CommentTypeReview, CommentTypeDismissReview:
 		return true
 	}
 	return false

models/issues/comment_code.go

@@ -18,11 +18,11 @@ import (
 type CodeComments map[string]map[int64][]*Comment
 
 // FetchCodeComments will return a 2d-map: ["Path"]["Line"] = Comments at line
-func FetchCodeComments(ctx context.Context, issue *Issue, currentUser *user_model.User) (CodeComments, error) {
-	return fetchCodeCommentsByReview(ctx, issue, currentUser, nil)
+func FetchCodeComments(ctx context.Context, issue *Issue, currentUser *user_model.User, showOutdatedComments bool) (CodeComments, error) {
+	return fetchCodeCommentsByReview(ctx, issue, currentUser, nil, showOutdatedComments)
 }
 
-func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *user_model.User, review *Review) (CodeComments, error) {
+func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *user_model.User, review *Review, showOutdatedComments bool) (CodeComments, error) {
 	pathToLineToComment := make(CodeComments)
 	if review == nil {
 		review = &Review{ID: 0}
@@ -33,7 +33,7 @@ func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *u
 		ReviewID: review.ID,
 	}
 
-	comments, err := findCodeComments(ctx, opts, issue, currentUser, review)
+	comments, err := findCodeComments(ctx, opts, issue, currentUser, review, showOutdatedComments)
 	if err != nil {
 		return nil, err
 	}
@@ -47,15 +47,17 @@ func fetchCodeCommentsByReview(ctx context.Context, issue *Issue, currentUser *u
 	return pathToLineToComment, nil
 }
 
-func findCodeComments(ctx context.Context, opts FindCommentsOptions, issue *Issue, currentUser *user_model.User, review *Review) ([]*Comment, error) {
+func findCodeComments(ctx context.Context, opts FindCommentsOptions, issue *Issue, currentUser *user_model.User, review *Review, showOutdatedComments bool) ([]*Comment, error) {
 	var comments CommentList
 	if review == nil {
 		review = &Review{ID: 0}
 	}
 	conds := opts.ToConds()
-	if review.ID == 0 {
+
+	if !showOutdatedComments && review.ID == 0 {
 		conds = conds.And(builder.Eq{"invalidated": false})
 	}
+
 	e := db.GetEngine(ctx)
 	if err := e.Where(conds).
 		Asc("comment.created_unix").
@@ -118,12 +120,12 @@ func findCodeComments(ctx context.Context, opts FindCommentsOptions, issue *Issu
 }
 
 // FetchCodeCommentsByLine fetches the code comments for a given treePath and line number
-func FetchCodeCommentsByLine(ctx context.Context, issue *Issue, currentUser *user_model.User, treePath string, line int64) ([]*Comment, error) {
+func FetchCodeCommentsByLine(ctx context.Context, issue *Issue, currentUser *user_model.User, treePath string, line int64, showOutdatedComments bool) ([]*Comment, error) {
 	opts := FindCommentsOptions{
 		Type:     CommentTypeCode,
 		IssueID:  issue.ID,
 		TreePath: treePath,
 		Line:     line,
 	}
-	return findCodeComments(ctx, opts, issue, currentUser, nil)
+	return findCodeComments(ctx, opts, issue, currentUser, nil, showOutdatedComments)
 }

models/issues/comment_test.go

@@ -50,7 +50,7 @@ func TestFetchCodeComments(t *testing.T) {
 
 	issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2})
 	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
-	res, err := issues_model.FetchCodeComments(db.DefaultContext, issue, user)
+	res, err := issues_model.FetchCodeComments(db.DefaultContext, issue, user, false)
 	assert.NoError(t, err)
 	assert.Contains(t, res, "README.md")
 	assert.Contains(t, res["README.md"], int64(4))
@@ -58,7 +58,7 @@ func TestFetchCodeComments(t *testing.T) {
 	assert.Equal(t, int64(4), res["README.md"][4][0].ID)
 
 	user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
-	res, err = issues_model.FetchCodeComments(db.DefaultContext, issue, user2)
+	res, err = issues_model.FetchCodeComments(db.DefaultContext, issue, user2, false)
 	assert.NoError(t, err)
 	assert.Len(t, res, 1)
 }

models/issues/issue_list.go

@@ -229,39 +229,41 @@ func (issues IssueList) loadMilestones(ctx context.Context) error {
 	return nil
 }
 
-func (issues IssueList) getProjectIDs() []int64 {
-	ids := make(container.Set[int64], len(issues))
-	for _, issue := range issues {
-		ids.Add(issue.ProjectID())
-	}
-	return ids.Values()
-}
+func (issues IssueList) LoadProjects(ctx context.Context) error {
+	issueIDs := issues.getIssueIDs()
+	projectMaps := make(map[int64]*project_model.Project, len(issues))
+	left := len(issueIDs)
 
-func (issues IssueList) loadProjects(ctx context.Context) error {
-	projectIDs := issues.getProjectIDs()
-	if len(projectIDs) == 0 {
-		return nil
+	type projectWithIssueID struct {
+		*project_model.Project `xorm:"extends"`
+		IssueID                int64
 	}
 
-	projectMaps := make(map[int64]*project_model.Project, len(projectIDs))
-	left := len(projectIDs)
 	for left > 0 {
 		limit := db.DefaultMaxInSize
 		if left < limit {
 			limit = left
 		}
+
+		projects := make([]*projectWithIssueID, 0, limit)
 		err := db.GetEngine(ctx).
-			In("id", projectIDs[:limit]).
-			Find(&projectMaps)
+			Table("project").
+			Select("project.*, project_issue.issue_id").
+			Join("INNER", "project_issue", "project.id = project_issue.project_id").
+			In("project_issue.issue_id", issueIDs[:limit]).
+			Find(&projects)
 		if err != nil {
 			return err
 		}
+		for _, project := range projects {
+			projectMaps[project.IssueID] = project.Project
+		}
 		left -= limit
-		projectIDs = projectIDs[limit:]
+		issueIDs = issueIDs[limit:]
 	}
 
 	for _, issue := range issues {
-		issue.Project = projectMaps[issue.ProjectID()]
+		issue.Project = projectMaps[issue.ID]
 	}
 	return nil
 }
@@ -541,7 +543,7 @@ func (issues IssueList) loadAttributes(ctx context.Context) error {
 		return fmt.Errorf("issue.loadAttributes: loadMilestones: %w", err)
 	}
 
-	if err := issues.loadProjects(ctx); err != nil {
+	if err := issues.LoadProjects(ctx); err != nil {
 		return fmt.Errorf("issue.loadAttributes: loadProjects: %w", err)
 	}
 

models/issues/issue_list_test.go

@@ -66,8 +66,12 @@ func TestIssueList_LoadAttributes(t *testing.T) {
 		}
 		if issue.ID == int64(1) {
 			assert.Equal(t, int64(400), issue.TotalTrackedTime)
+			assert.NotNil(t, issue.Project)
 		} else if issue.ID == int64(2) {
 			assert.Equal(t, int64(3682), issue.TotalTrackedTime)
+			assert.Nil(t, issue.Project)
+		} else {
+			assert.Nil(t, issue.Project)
 		}
 	}
 }

models/issues/issue_project.go

@@ -27,11 +27,6 @@ func (issue *Issue) LoadProject(ctx context.Context) (err error) {
 	return err
 }
 
-// ProjectID return project id if issue was assigned to one
-func (issue *Issue) ProjectID() int64 {
-	return issue.projectID(db.DefaultContext)
-}
-
 func (issue *Issue) projectID(ctx context.Context) int64 {
 	var ip project_model.ProjectIssue
 	has, err := db.GetEngine(ctx).Where("issue_id=?", issue.ID).Get(&ip)

models/issues/review.go

@@ -141,7 +141,7 @@ func (r *Review) LoadCodeComments(ctx context.Context) (err error) {
 	if err = r.loadIssue(ctx); err != nil {
 		return
 	}
-	r.CodeComments, err = fetchCodeCommentsByReview(ctx, r.Issue, nil, r)
+	r.CodeComments, err = fetchCodeCommentsByReview(ctx, r.Issue, nil, r, false)
 	return err
 }
 

models/issues/tracked_time.go

@@ -6,6 +6,7 @@ package issues
 import (
 	"context"
 	"errors"
+	"fmt"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
@@ -173,10 +174,12 @@ func AddTime(user *user_model.User, issue *Issue, amount int64, created time.Tim
 	}
 
 	if _, err := CreateComment(ctx, &CreateCommentOptions{
-		Issue:   issue,
-		Repo:    issue.Repo,
-		Doer:    user,
-		Content: util.SecToTime(amount),
+		Issue: issue,
+		Repo:  issue.Repo,
+		Doer:  user,
+		// Content before v1.21 did store the formated string instead of seconds,
+		// so use "|" as delimeter to mark the new format
+		Content: fmt.Sprintf("|%d", amount),
 		Type:    CommentTypeAddTimeManual,
 		TimeID:  t.ID,
 	}); err != nil {
@@ -199,8 +202,8 @@ func addTime(ctx context.Context, user *user_model.User, issue *Issue, amount in
 	return tt, db.Insert(ctx, tt)
 }
 
-// TotalTimes returns the spent time for each user by an issue
-func TotalTimes(options *FindTrackedTimesOptions) (map[*user_model.User]string, error) {
+// TotalTimes returns the spent time in seconds for each user by an issue
+func TotalTimes(options *FindTrackedTimesOptions) (map[*user_model.User]int64, error) {
 	trackedTimes, err := GetTrackedTimes(db.DefaultContext, options)
 	if err != nil {
 		return nil, err
@@ -211,7 +214,7 @@ func TotalTimes(options *FindTrackedTimesOptions) (map[*user_model.User]string,
 		totalTimesByUser[t.UserID] += t.Time
 	}
 
-	totalTimes := make(map[*user_model.User]string)
+	totalTimes := make(map[*user_model.User]int64)
 	// Fetching User and making time human readable
 	for userID, total := range totalTimesByUser {
 		user, err := user_model.GetUserByID(db.DefaultContext, userID)
@@ -221,7 +224,7 @@ func TotalTimes(options *FindTrackedTimesOptions) (map[*user_model.User]string,
 			}
 			return nil, err
 		}
-		totalTimes[user] = util.SecToTime(total)
+		totalTimes[user] = total
 	}
 	return totalTimes, nil
 }
@@ -251,10 +254,12 @@ func DeleteIssueUserTimes(issue *Issue, user *user_model.User) error {
 		return err
 	}
 	if _, err := CreateComment(ctx, &CreateCommentOptions{
-		Issue:   issue,
-		Repo:    issue.Repo,
-		Doer:    user,
-		Content: "- " + util.SecToTime(removedTime),
+		Issue: issue,
+		Repo:  issue.Repo,
+		Doer:  user,
+		// Content before v1.21 did store the formated string instead of seconds,
+		// so use "|" as delimeter to mark the new format
+		Content: fmt.Sprintf("|%d", removedTime),
 		Type:    CommentTypeDeleteTimeManual,
 	}); err != nil {
 		return err
@@ -280,10 +285,12 @@ func DeleteTime(t *TrackedTime) error {
 	}
 
 	if _, err := CreateComment(ctx, &CreateCommentOptions{
-		Issue:   t.Issue,
-		Repo:    t.Issue.Repo,
-		Doer:    t.User,
-		Content: "- " + util.SecToTime(t.Time),
+		Issue: t.Issue,
+		Repo:  t.Issue.Repo,
+		Doer:  t.User,
+		// Content before v1.21 did store the formated string instead of seconds,
+		// so use "|" as delimeter to mark the new format
+		Content: fmt.Sprintf("|%d", t.Time),
 		Type:    CommentTypeDeleteTimeManual,
 	}); err != nil {
 		return err

models/issues/tracked_time_test.go

@@ -35,7 +35,7 @@ func TestAddTime(t *testing.T) {
 	assert.Equal(t, int64(3661), tt.Time)
 
 	comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{Type: issues_model.CommentTypeAddTimeManual, PosterID: 3, IssueID: 1})
-	assert.Equal(t, "1 hour 1 minute", comment.Content)
+	assert.Equal(t, "|3661", comment.Content)
 }
 
 func TestGetTrackedTimes(t *testing.T) {
@@ -86,8 +86,8 @@ func TestTotalTimes(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, total, 1)
 	for user, time := range total {
-		assert.Equal(t, int64(1), user.ID)
-		assert.Equal(t, "6 minutes 40 seconds", time)
+		assert.EqualValues(t, 1, user.ID)
+		assert.EqualValues(t, 400, time)
 	}
 
 	total, err = issues_model.TotalTimes(&issues_model.FindTrackedTimesOptions{IssueID: 2})
@@ -95,9 +95,9 @@ func TestTotalTimes(t *testing.T) {
 	assert.Len(t, total, 2)
 	for user, time := range total {
 		if user.ID == 2 {
-			assert.Equal(t, "1 hour 1 minute", time)
+			assert.EqualValues(t, 3662, time)
 		} else if user.ID == 1 {
-			assert.Equal(t, "20 seconds", time)
+			assert.EqualValues(t, 20, time)
 		} else {
 			assert.Error(t, assert.AnError)
 		}
@@ -107,8 +107,8 @@ func TestTotalTimes(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, total, 1)
 	for user, time := range total {
-		assert.Equal(t, int64(2), user.ID)
-		assert.Equal(t, "1 second", time)
+		assert.EqualValues(t, 2, user.ID)
+		assert.EqualValues(t, 1, time)
 	}
 
 	total, err = issues_model.TotalTimes(&issues_model.FindTrackedTimesOptions{IssueID: 4})

models/migrations/base/tests.go

@@ -147,9 +147,9 @@ func MainTest(m *testing.M) {
 		os.Exit(1)
 	}
 
+	setting.CustomPath = filepath.Join(setting.AppWorkPath, "custom")
 	setting.AppDataPath = tmpDataPath
 
-	setting.SetCustomPathAndConf("", "", "")
 	unittest.InitSettings()
 	if err = git.InitFull(context.Background()); err != nil {
 		fmt.Printf("Unable to InitFull: %v\n", err)

models/migrations/migrations.go

@@ -503,6 +503,12 @@ var migrations = []Migration{
 
 	// v260 -> v261
 	NewMigration("Drop custom_labels column of action_runner table", v1_21.DropCustomLabelsColumnOfActionRunner),
+	// v261 -> v262
+	NewMigration("Add variable table", v1_21.CreateVariableTable),
+	// v262 -> v263
+	NewMigration("Add TriggerEvent to action_run table", v1_21.AddTriggerEventToActionRun),
+	// v263 -> v264
+	NewMigration("Add git_size and lfs_size columns to repository table", v1_21.AddGitSizeAndLFSSizeToRepositoryTable),
 	// to modify later
 	NewMigration("Add size limit on repository", v1_21.AddSizeLimitOnRepo),
 }

models/migrations/v1_21/v261.go

@@ -0,0 +1,24 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_21 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreateVariableTable(x *xorm.Engine) error {
+	type ActionVariable struct {
+		ID          int64              `xorm:"pk autoincr"`
+		OwnerID     int64              `xorm:"UNIQUE(owner_repo_name)"`
+		RepoID      int64              `xorm:"INDEX UNIQUE(owner_repo_name)"`
+		Name        string             `xorm:"UNIQUE(owner_repo_name) NOT NULL"`
+		Data        string             `xorm:"LONGTEXT NOT NULL"`
+		CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"`
+		UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
+	}
+
+	return x.Sync(new(ActionVariable))
+}

models/migrations/v1_21/v262.go

@@ -0,0 +1,16 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_21 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddTriggerEventToActionRun(x *xorm.Engine) error {
+	type ActionRun struct {
+		TriggerEvent string
+	}
+
+	return x.Sync(new(ActionRun))
+}

models/migrations/v1_21/v263.go

@@ -0,0 +1,41 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_21 //nolint
+
+import (
+	"fmt"
+
+	"xorm.io/xorm"
+)
+
+// AddGitSizeAndLFSSizeToRepositoryTable: add GitSize and LFSSize columns to Repository
+func AddGitSizeAndLFSSizeToRepositoryTable(x *xorm.Engine) error {
+	type Repository struct {
+		GitSize int64 `xorm:"NOT NULL DEFAULT 0"`
+		LFSSize int64 `xorm:"NOT NULL DEFAULT 0"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(Repository)); err != nil {
+		return fmt.Errorf("Sync2: %w", err)
+	}
+
+	_, err := sess.Exec(`UPDATE repository SET lfs_size=(SELECT SUM(size) FROM lfs_meta_object WHERE lfs_meta_object.repository_id=repository.ID) WHERE EXISTS (SELECT 1 FROM lfs_meta_object WHERE lfs_meta_object.repository_id=repository.ID)`)
+	if err != nil {
+		return err
+	}
+
+	_, err = sess.Exec(`UPDATE repository SET git_size = size - lfs_size`)
+	if err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/repo/repo.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
@@ -165,6 +166,8 @@ type Repository struct {
 	SizeLimit                       int64              `xorm:"NOT NULL DEFAULT 0"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
 	EnableSizeLimit                 bool               `xorm:"NOT NULL DEFAULT true"`
+	GitSize                         int64              `xorm:"NOT NULL DEFAULT 0"`
+	LFSSize                         int64              `xorm:"NOT NULL DEFAULT 0"`
 	CodeIndexerStatus               *RepoIndexerStatus `xorm:"-"`
 	StatsIndexerStatus              *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
@@ -198,6 +201,42 @@ func (repo *Repository) SanitizedOriginalURL() string {
 	return u.String()
 }
 
+// text representations to be returned in SizeDetail.Name
+const (
+	SizeDetailNameGit = "git"
+	SizeDetailNameLFS = "lfs"
+)
+
+type SizeDetail struct {
+	Name string
+	Size int64
+}
+
+// SizeDetails forms a struct with various size details about repository
+func (repo *Repository) SizeDetails() []SizeDetail {
+	sizeDetails := []SizeDetail{
+		{
+			Name: SizeDetailNameGit,
+			Size: repo.GitSize,
+		},
+		{
+			Name: SizeDetailNameLFS,
+			Size: repo.LFSSize,
+		},
+	}
+	return sizeDetails
+}
+
+// SizeDetailsString returns a concatenation of all repository size details as a string
+func (repo *Repository) SizeDetailsString() string {
+	var str strings.Builder
+	sizeDetails := repo.SizeDetails()
+	for _, detail := range sizeDetails {
+		str.WriteString(fmt.Sprintf("%s: %s, ", detail.Name, base.FileSize(detail.Size)))
+	}
+	return strings.TrimSuffix(str.String(), ", ")
+}
+
 func (repo *Repository) LogString() string {
 	if repo == nil {
 		return "<Repository nil>"

models/repo/update.go

@@ -185,9 +185,11 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s
 }
 
 // UpdateRepoSize updates the repository size, calculating it using getDirectorySize
-func UpdateRepoSize(ctx context.Context, repoID, size int64) error {
-	_, err := db.GetEngine(ctx).ID(repoID).Cols("size").NoAutoTime().Update(&Repository{
-		Size: size,
+func UpdateRepoSize(ctx context.Context, repoID, gitSize, lfsSize int64) error {
+	_, err := db.GetEngine(ctx).ID(repoID).Cols("size", "git_size", "lfs_size").NoAutoTime().Update(&Repository{
+		Size:    gitSize + lfsSize,
+		GitSize: gitSize,
+		LFSSize: lfsSize,
 	})
 	return err
 }

models/secret/secret.go

@@ -5,38 +5,17 @@ package secret
 
 import (
 	"context"
-	"fmt"
-	"regexp"
+	"errors"
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
 	secret_module "code.gitea.io/gitea/modules/secret"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
-	"code.gitea.io/gitea/modules/util"
 
 	"xorm.io/builder"
 )
 
-type ErrSecretInvalidValue struct {
-	Name *string
-	Data *string
-}
-
-func (err ErrSecretInvalidValue) Error() string {
-	if err.Name != nil {
-		return fmt.Sprintf("secret name %q is invalid", *err.Name)
-	}
-	if err.Data != nil {
-		return fmt.Sprintf("secret data %q is invalid", *err.Data)
-	}
-	return util.ErrInvalidArgument.Error()
-}
-
-func (err ErrSecretInvalidValue) Unwrap() error {
-	return util.ErrInvalidArgument
-}
-
 // Secret represents a secret
 type Secret struct {
 	ID          int64
@@ -74,24 +53,11 @@ func init() {
 	db.RegisterModel(new(Secret))
 }
 
-var (
-	secretNameReg            = regexp.MustCompile("^[A-Z_][A-Z0-9_]*$")
-	forbiddenSecretPrefixReg = regexp.MustCompile("^GIT(EA|HUB)_")
-)
-
-// Validate validates the required fields and formats.
 func (s *Secret) Validate() error {
-	switch {
-	case len(s.Name) == 0 || len(s.Name) > 50:
-		return ErrSecretInvalidValue{Name: &s.Name}
-	case len(s.Data) == 0:
-		return ErrSecretInvalidValue{Data: &s.Data}
-	case !secretNameReg.MatchString(s.Name) ||
-		forbiddenSecretPrefixReg.MatchString(s.Name):
-		return ErrSecretInvalidValue{Name: &s.Name}
-	default:
-		return nil
+	if s.OwnerID == 0 && s.RepoID == 0 {
+		return errors.New("the secret is not bound to any scope")
 	}
+	return nil
 }
 
 type FindSecretsOptions struct {

models/unittest/testdb.go

@@ -42,12 +42,14 @@ func fatalTestError(fmtStr string, args ...interface{}) {
 	os.Exit(1)
 }
 
-// InitSettings initializes config provider and load common setttings for tests
+// InitSettings initializes config provider and load common settings for tests
 func InitSettings(extraConfigs ...string) {
-	setting.Init(&setting.Options{
-		AllowEmpty:  true,
-		ExtraConfig: strings.Join(extraConfigs, "\n"),
-	})
+	if setting.CustomConf == "" {
+		setting.CustomConf = filepath.Join(setting.CustomPath, "conf/app-unittest-tmp.ini")
+		_ = os.Remove(setting.CustomConf)
+	}
+	setting.InitCfgProvider(setting.CustomConf, strings.Join(extraConfigs, "\n"))
+	setting.LoadCommonSettings()
 
 	if err := setting.PrepareAppDataPath(); err != nil {
 		log.Fatalf("Can not prepare APP_DATA_PATH: %v", err)
@@ -69,7 +71,7 @@ type TestOptions struct {
 // MainTest a reusable TestMain(..) function for unit tests that need to use a
 // test database. Creates the test database, and sets necessary settings.
 func MainTest(m *testing.M, testOpts *TestOptions) {
-	setting.SetCustomPathAndConf("", "", "")
+	setting.CustomPath = filepath.Join(testOpts.GiteaRootPath, "custom")
 	InitSettings()
 
 	var err error

models/user/setting_keys.go

@@ -8,6 +8,8 @@ const (
 	SettingsKeyHiddenCommentTypes = "issue.hidden_comment_types"
 	// SettingsKeyDiffWhitespaceBehavior is the setting key for whitespace behavior of diff
 	SettingsKeyDiffWhitespaceBehavior = "diff.whitespace_behaviour"
+	// SettingsKeyShowOutdatedComments is the setting key wether or not to show outdated comments in PRs
+	SettingsKeyShowOutdatedComments = "comment_code.show_outdated"
 	// UserActivityPubPrivPem is user's private key
 	UserActivityPubPrivPem = "activitypub.priv_pem"
 	// UserActivityPubPubPem is user's public key

modules/actions/github.go

@@ -8,33 +8,33 @@ import (
 )
 
 const (
-	githubEventPullRequest              = "pull_request"
-	githubEventPullRequestTarget        = "pull_request_target"
-	githubEventPullRequestReviewComment = "pull_request_review_comment"
-	githubEventPullRequestReview        = "pull_request_review"
-	githubEventRegistryPackage          = "registry_package"
-	githubEventCreate                   = "create"
-	githubEventDelete                   = "delete"
-	githubEventFork                     = "fork"
-	githubEventPush                     = "push"
-	githubEventIssues                   = "issues"
-	githubEventIssueComment             = "issue_comment"
-	githubEventRelease                  = "release"
-	githubEventPullRequestComment       = "pull_request_comment"
-	githubEventGollum                   = "gollum"
+	GithubEventPullRequest              = "pull_request"
+	GithubEventPullRequestTarget        = "pull_request_target"
+	GithubEventPullRequestReviewComment = "pull_request_review_comment"
+	GithubEventPullRequestReview        = "pull_request_review"
+	GithubEventRegistryPackage          = "registry_package"
+	GithubEventCreate                   = "create"
+	GithubEventDelete                   = "delete"
+	GithubEventFork                     = "fork"
+	GithubEventPush                     = "push"
+	GithubEventIssues                   = "issues"
+	GithubEventIssueComment             = "issue_comment"
+	GithubEventRelease                  = "release"
+	GithubEventPullRequestComment       = "pull_request_comment"
+	GithubEventGollum                   = "gollum"
 )
 
 // canGithubEventMatch check if the input Github event can match any Gitea event.
 func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEventType) bool {
 	switch eventName {
-	case githubEventRegistryPackage:
+	case GithubEventRegistryPackage:
 		return triggedEvent == webhook_module.HookEventPackage
 
 	// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#gollum
-	case githubEventGollum:
+	case GithubEventGollum:
 		return triggedEvent == webhook_module.HookEventWiki
 
-	case githubEventIssues:
+	case GithubEventIssues:
 		switch triggedEvent {
 		case webhook_module.HookEventIssues,
 			webhook_module.HookEventIssueAssign,
@@ -46,7 +46,7 @@ func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEvent
 			return false
 		}
 
-	case githubEventPullRequest, githubEventPullRequestTarget:
+	case GithubEventPullRequest, GithubEventPullRequestTarget:
 		switch triggedEvent {
 		case webhook_module.HookEventPullRequest,
 			webhook_module.HookEventPullRequestSync,
@@ -58,7 +58,7 @@ func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEvent
 			return false
 		}
 
-	case githubEventPullRequestReview:
+	case GithubEventPullRequestReview:
 		switch triggedEvent {
 		case webhook_module.HookEventPullRequestReviewApproved,
 			webhook_module.HookEventPullRequestReviewComment,

modules/actions/github_test.go

@@ -21,85 +21,85 @@ func TestCanGithubEventMatch(t *testing.T) {
 		// registry_package event
 		{
 			"registry_package matches",
-			githubEventRegistryPackage,
+			GithubEventRegistryPackage,
 			webhook_module.HookEventPackage,
 			true,
 		},
 		{
 			"registry_package cannot match",
-			githubEventRegistryPackage,
+			GithubEventRegistryPackage,
 			webhook_module.HookEventPush,
 			false,
 		},
 		// issues event
 		{
 			"issue matches",
-			githubEventIssues,
+			GithubEventIssues,
 			webhook_module.HookEventIssueLabel,
 			true,
 		},
 		{
 			"issue cannot match",
-			githubEventIssues,
+			GithubEventIssues,
 			webhook_module.HookEventIssueComment,
 			false,
 		},
 		// issue_comment event
 		{
 			"issue_comment matches",
-			githubEventIssueComment,
+			GithubEventIssueComment,
 			webhook_module.HookEventIssueComment,
 			true,
 		},
 		{
 			"issue_comment cannot match",
-			githubEventIssueComment,
+			GithubEventIssueComment,
 			webhook_module.HookEventIssues,
 			false,
 		},
 		// pull_request event
 		{
 			"pull_request matches",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequestSync,
 			true,
 		},
 		{
 			"pull_request cannot match",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequestComment,
 			false,
 		},
 		// pull_request_target event
 		{
 			"pull_request_target matches",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequest,
 			true,
 		},
 		{
 			"pull_request_target cannot match",
-			githubEventPullRequest,
+			GithubEventPullRequest,
 			webhook_module.HookEventPullRequestComment,
 			false,
 		},
 		// pull_request_review event
 		{
 			"pull_request_review matches",
-			githubEventPullRequestReview,
+			GithubEventPullRequestReview,
 			webhook_module.HookEventPullRequestReviewComment,
 			true,
 		},
 		{
 			"pull_request_review cannot match",
-			githubEventPullRequestReview,
+			GithubEventPullRequestReview,
 			webhook_module.HookEventPullRequestComment,
 			false,
 		},
 		// other events
 		{
 			"create event",
-			githubEventCreate,
+			GithubEventCreate,
 			webhook_module.HookEventCreate,
 			true,
 		},

modules/actions/log.go

@@ -29,12 +29,28 @@ const (
 )
 
 func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runnerv1.LogRow) ([]int, error) {
+	flag := os.O_WRONLY
+	if offset == 0 {
+		// Create file only if offset is 0, or it could result in content holes if the file doesn't exist.
+		flag |= os.O_CREATE
+	}
 	name := DBFSPrefix + filename
-	f, err := dbfs.OpenFile(ctx, name, os.O_WRONLY|os.O_CREATE)
+	f, err := dbfs.OpenFile(ctx, name, flag)
 	if err != nil {
 		return nil, fmt.Errorf("dbfs OpenFile %q: %w", name, err)
 	}
 	defer f.Close()
+
+	stat, err := f.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("dbfs Stat %q: %w", name, err)
+	}
+	if stat.Size() < offset {
+		// If the size is less than offset, refuse to write, or it could result in content holes.
+		// However, if the size is greater than offset, we can still write to overwrite the content.
+		return nil, fmt.Errorf("size of %q is less than offset", name)
+	}
+
 	if _, err := f.Seek(offset, io.SeekStart); err != nil {
 		return nil, fmt.Errorf("dbfs Seek %q: %w", name, err)
 	}
@@ -57,7 +73,7 @@ func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runne
 }
 
 func ReadLogs(ctx context.Context, inStorage bool, filename string, offset, limit int64) ([]*runnerv1.LogRow, error) {
-	f, err := openLogs(ctx, inStorage, filename)
+	f, err := OpenLogs(ctx, inStorage, filename)
 	if err != nil {
 		return nil, err
 	}
@@ -125,7 +141,7 @@ func RemoveLogs(ctx context.Context, inStorage bool, filename string) error {
 	return nil
 }
 
-func openLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) {
+func OpenLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) {
 	if !inStorage {
 		name := DBFSPrefix + filename
 		f, err := dbfs.Open(ctx, name)

modules/actions/workflows.go

@@ -20,6 +20,14 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+type DetectedWorkflow struct {
+	EntryName    string
+	TriggerEvent string
+	Commit       *git.Commit
+	Ref          string
+	Content      []byte
+}
+
 func init() {
 	model.OnDecodeNodeError = func(node yaml.Node, out interface{}, err error) {
 		// Log the error instead of panic or fatal.
@@ -89,13 +97,13 @@ func GetEventsFromContent(content []byte) ([]*jobparser.Event, error) {
 	return events, nil
 }
 
-func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) (map[string][]byte, error) {
+func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) ([]*DetectedWorkflow, error) {
 	entries, err := ListWorkflows(commit)
 	if err != nil {
 		return nil, err
 	}
 
-	workflows := make(map[string][]byte, len(entries))
+	workflows := make([]*DetectedWorkflow, 0, len(entries))
 	for _, entry := range entries {
 		content, err := GetContentFromEntry(entry)
 		if err != nil {
@@ -109,7 +117,13 @@ func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventTy
 		for _, evt := range events {
 			log.Trace("detect workflow %q for event %#v matching %q", entry.Name(), evt, triggedEvent)
 			if detectMatched(commit, triggedEvent, payload, evt) {
-				workflows[entry.Name()] = content
+				dwf := &DetectedWorkflow{
+					EntryName:    entry.Name(),
+					TriggerEvent: evt.Name,
+					Commit:       commit,
+					Content:      content,
+				}
+				workflows = append(workflows, dwf)
 			}
 		}
 	}

modules/actions/workflows_test.go

@@ -23,77 +23,77 @@ func TestDetectMatched(t *testing.T) {
 		expected     bool
 	}{
 		{
-			desc:         "HookEventCreate(create) matches githubEventCreate(create)",
+			desc:         "HookEventCreate(create) matches GithubEventCreate(create)",
 			triggedEvent: webhook_module.HookEventCreate,
 			payload:      nil,
 			yamlOn:       "on: create",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventIssues(issues) `opened` action matches githubEventIssues(issues)",
+			desc:         "HookEventIssues(issues) `opened` action matches GithubEventIssues(issues)",
 			triggedEvent: webhook_module.HookEventIssues,
 			payload:      &api.IssuePayload{Action: api.HookIssueOpened},
 			yamlOn:       "on: issues",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventIssues(issues) `milestoned` action matches githubEventIssues(issues)",
+			desc:         "HookEventIssues(issues) `milestoned` action matches GithubEventIssues(issues)",
 			triggedEvent: webhook_module.HookEventIssues,
 			payload:      &api.IssuePayload{Action: api.HookIssueMilestoned},
 			yamlOn:       "on: issues",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequestSync(pull_request_sync) matches githubEventPullRequest(pull_request)",
+			desc:         "HookEventPullRequestSync(pull_request_sync) matches GithubEventPullRequest(pull_request)",
 			triggedEvent: webhook_module.HookEventPullRequestSync,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueSynchronized},
 			yamlOn:       "on: pull_request",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequest(pull_request) `label_updated` action doesn't match githubEventPullRequest(pull_request) with no activity type",
+			desc:         "HookEventPullRequest(pull_request) `label_updated` action doesn't match GithubEventPullRequest(pull_request) with no activity type",
 			triggedEvent: webhook_module.HookEventPullRequest,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
 			yamlOn:       "on: pull_request",
 			expected:     false,
 		},
 		{
-			desc:         "HookEventPullRequest(pull_request) `label_updated` action matches githubEventPullRequest(pull_request) with `label` activity type",
+			desc:         "HookEventPullRequest(pull_request) `label_updated` action matches GithubEventPullRequest(pull_request) with `label` activity type",
 			triggedEvent: webhook_module.HookEventPullRequest,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
 			yamlOn:       "on:\n  pull_request:\n    types: [labeled]",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequestReviewComment(pull_request_review_comment) matches githubEventPullRequestReviewComment(pull_request_review_comment)",
+			desc:         "HookEventPullRequestReviewComment(pull_request_review_comment) matches GithubEventPullRequestReviewComment(pull_request_review_comment)",
 			triggedEvent: webhook_module.HookEventPullRequestReviewComment,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueReviewed},
 			yamlOn:       "on:\n  pull_request_review_comment:\n    types: [created]",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPullRequestReviewRejected(pull_request_review_rejected) doesn't match githubEventPullRequestReview(pull_request_review) with `dismissed` activity type (we don't support `dismissed` at present)",
+			desc:         "HookEventPullRequestReviewRejected(pull_request_review_rejected) doesn't match GithubEventPullRequestReview(pull_request_review) with `dismissed` activity type (we don't support `dismissed` at present)",
 			triggedEvent: webhook_module.HookEventPullRequestReviewRejected,
 			payload:      &api.PullRequestPayload{Action: api.HookIssueReviewed},
 			yamlOn:       "on:\n  pull_request_review:\n    types: [dismissed]",
 			expected:     false,
 		},
 		{
-			desc:         "HookEventRelease(release) `published` action matches githubEventRelease(release) with `published` activity type",
+			desc:         "HookEventRelease(release) `published` action matches GithubEventRelease(release) with `published` activity type",
 			triggedEvent: webhook_module.HookEventRelease,
 			payload:      &api.ReleasePayload{Action: api.HookReleasePublished},
 			yamlOn:       "on:\n  release:\n    types: [published]",
 			expected:     true,
 		},
 		{
-			desc:         "HookEventPackage(package) `created` action doesn't match githubEventRegistryPackage(registry_package) with `updated` activity type",
+			desc:         "HookEventPackage(package) `created` action doesn't match GithubEventRegistryPackage(registry_package) with `updated` activity type",
 			triggedEvent: webhook_module.HookEventPackage,
 			payload:      &api.PackagePayload{Action: api.HookPackageCreated},
 			yamlOn:       "on:\n  registry_package:\n    types: [updated]",
 			expected:     false,
 		},
 		{
-			desc:         "HookEventWiki(wiki) matches githubEventGollum(gollum)",
+			desc:         "HookEventWiki(wiki) matches GithubEventGollum(gollum)",
 			triggedEvent: webhook_module.HookEventWiki,
 			payload:      nil,
 			yamlOn:       "on: gollum",

modules/base/tool.go

@@ -217,7 +217,7 @@ func EntryIcon(entry *git.TreeEntry) string {
 			return "file-symlink-file"
 		}
 		if te.IsDir() {
-			return "file-submodule"
+			return "file-directory-symlink"
 		}
 		return "file-symlink-file"
 	case entry.IsDir():

modules/context/base.go

@@ -140,6 +140,10 @@ func (b *Base) JSONRedirect(redirect string) {
 	b.JSON(http.StatusOK, map[string]any{"redirect": redirect})
 }
 
+func (b *Base) JSONOK() {
+	b.JSON(http.StatusOK, map[string]any{"ok": true}) // this is only a dummy response, frontend seldom uses it
+}
+
 func (b *Base) JSONError(msg string) {
 	b.JSON(http.StatusBadRequest, map[string]any{"errorMessage": msg})
 }

modules/context/repo.go

@@ -593,7 +593,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 
 	ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
 	if setting.Indexer.RepoIndexerEnabled {
-		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
+		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable(ctx)
 	}
 
 	if ctx.IsSigned {

modules/doctor/doctor.go

@@ -28,7 +28,7 @@ type Check struct {
 }
 
 func initDBSkipLogger(ctx context.Context) error {
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 	setting.LoadDBSetting()
 	if err := db.InitEngine(ctx); err != nil {
 		return fmt.Errorf("db.InitEngine: %w", err)

modules/doctor/paths.go

@@ -66,7 +66,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo
 		return err
 	}
 
-	setting.Init(&setting.Options{})
+	setting.MustInstalled()
 
 	configurationFiles := []configurationFile{
 		{"Configuration File Path", setting.CustomConf, false, true, false},

modules/indexer/code/bleve/bleve.go

@@ -1,14 +1,13 @@
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-package code
+package bleve
 
 import (
 	"bufio"
 	"context"
 	"fmt"
 	"io"
-	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -17,12 +16,13 @@ import (
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
-	gitea_bleve "code.gitea.io/gitea/modules/indexer/bleve"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/typesniffer"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/blevesearch/bleve/v2"
 	analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -31,10 +31,8 @@ import (
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
-	"github.com/blevesearch/bleve/v2/index/upsidedown"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/blevesearch/bleve/v2/search/query"
-	"github.com/ethantkoenig/rupture"
 	"github.com/go-enry/go-enry/v2"
 )
 
@@ -59,38 +57,6 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
 	})
 }
 
-// openBleveIndexer open the index at the specified path, checking for metadata
-// updates and bleve version updates.  If index needs to be created (or
-// re-created), returns (nil, nil)
-func openBleveIndexer(path string, latestVersion int) (bleve.Index, error) {
-	_, err := os.Stat(path)
-	if err != nil && os.IsNotExist(err) {
-		return nil, nil
-	} else if err != nil {
-		return nil, err
-	}
-
-	metadata, err := rupture.ReadIndexMetadata(path)
-	if err != nil {
-		return nil, err
-	}
-	if metadata.Version < latestVersion {
-		// the indexer is using a previous version, so we should delete it and
-		// re-populate
-		return nil, util.RemoveAll(path)
-	}
-
-	index, err := bleve.Open(path)
-	if err != nil && err == upsidedown.IncompatibleVersion {
-		// the indexer was built with a previous version of bleve, so we should
-		// delete it and re-populate
-		return nil, util.RemoveAll(path)
-	} else if err != nil {
-		return nil, err
-	}
-	return index, nil
-}
-
 // RepoIndexerData data stored in the repo indexer
 type RepoIndexerData struct {
 	RepoID    int64
@@ -111,8 +77,8 @@ const (
 	repoIndexerLatestVersion = 6
 )
 
-// createBleveIndexer create a bleve repo indexer if one does not already exist
-func createBleveIndexer(path string, latestVersion int) (bleve.Index, error) {
+// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
+func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 	docMapping := bleve.NewDocumentMapping()
 	numericFieldMapping := bleve.NewNumericFieldMapping()
 	numericFieldMapping.IncludeInAll = false
@@ -147,42 +113,28 @@ func createBleveIndexer(path string, latestVersion int) (bleve.Index, error) {
 	mapping.AddDocumentMapping(repoIndexerDocType, docMapping)
 	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
 
-	indexer, err := bleve.New(path, mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
-		Version: latestVersion,
-	}); err != nil {
-		return nil, err
-	}
-	return indexer, nil
+	return mapping, nil
 }
 
-var _ Indexer = &BleveIndexer{}
+var _ internal.Indexer = &Indexer{}
 
-// BleveIndexer represents a bleve indexer implementation
-type BleveIndexer struct {
-	indexDir string
-	indexer  bleve.Index
+// Indexer represents a bleve indexer implementation
+type Indexer struct {
+	inner                    *inner_bleve.Indexer
+	indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much
 }
 
-// NewBleveIndexer creates a new bleve local indexer
-func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) {
-	indexer := &BleveIndexer{
-		indexDir: indexDir,
+// NewIndexer creates a new bleve local indexer
+func NewIndexer(indexDir string) *Indexer {
+	inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping)
+	return &Indexer{
+		Indexer: inner,
+		inner:   inner,
 	}
-	created, err := indexer.init()
-	if err != nil {
-		indexer.Close()
-		return nil, false, err
-	}
-	return indexer, created, err
 }
 
-func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string,
-	update fileUpdate, repo *repo_model.Repository, batch *gitea_bleve.FlushingBatch,
+func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string,
+	update internal.FileUpdate, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch,
 ) error {
 	// Ignore vendored files in code search
 	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
@@ -227,7 +179,7 @@ func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteClose
 	if _, err = batchReader.Discard(1); err != nil {
 		return err
 	}
-	id := filenameIndexerID(repo.ID, update.Filename)
+	id := internal.FilenameIndexerID(repo.ID, update.Filename)
 	return batch.Index(id, &RepoIndexerData{
 		RepoID:    repo.ID,
 		CommitID:  commitSha,
@@ -237,50 +189,14 @@ func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteClose
 	})
 }
 
-func (b *BleveIndexer) addDelete(filename string, repo *repo_model.Repository, batch *gitea_bleve.FlushingBatch) error {
-	id := filenameIndexerID(repo.ID, filename)
+func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch) error {
+	id := internal.FilenameIndexerID(repo.ID, filename)
 	return batch.Delete(id)
 }
 
-// init init the indexer
-func (b *BleveIndexer) init() (bool, error) {
-	var err error
-	b.indexer, err = openBleveIndexer(b.indexDir, repoIndexerLatestVersion)
-	if err != nil {
-		return false, err
-	}
-	if b.indexer != nil {
-		return false, nil
-	}
-
-	b.indexer, err = createBleveIndexer(b.indexDir, repoIndexerLatestVersion)
-	if err != nil {
-		return false, err
-	}
-
-	return true, nil
-}
-
-// Close close the indexer
-func (b *BleveIndexer) Close() {
-	log.Debug("Closing repo indexer")
-	if b.indexer != nil {
-		err := b.indexer.Close()
-		if err != nil {
-			log.Error("Error whilst closing the repository indexer: %v", err)
-		}
-	}
-	log.Info("PID: %d Repository Indexer closed", os.Getpid())
-}
-
-// Ping does nothing
-func (b *BleveIndexer) Ping() bool {
-	return true
-}
-
 // Index indexes the data
-func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
-	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
+func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
 	if len(changes.Updates) > 0 {
 
 		// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
@@ -308,14 +224,14 @@ func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, s
 }
 
 // Delete deletes indexes by ids
-func (b *BleveIndexer) Delete(repoID int64) error {
+func (b *Indexer) Delete(_ context.Context, repoID int64) error {
 	query := numericEqualityQuery(repoID, "RepoID")
 	searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
-	result, err := b.indexer.Search(searchRequest)
+	result, err := b.inner.Indexer.Search(searchRequest)
 	if err != nil {
 		return err
 	}
-	batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
 	for _, hit := range result.Hits {
 		if err = batch.Delete(hit.ID); err != nil {
 			return err
@@ -326,7 +242,7 @@ func (b *BleveIndexer) Delete(repoID int64) error {
 
 // Search searches for files in the specified repo.
 // Returns the matching file-paths
-func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	var (
 		indexerQuery query.Query
 		keywordQuery query.Query
@@ -379,14 +295,14 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 	}
 
-	result, err := b.indexer.SearchInContext(ctx, searchRequest)
+	result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
 	if err != nil {
 		return 0, nil, nil, err
 	}
 
 	total := int64(result.Total)
 
-	searchResults := make([]*SearchResult, len(result.Hits))
+	searchResults := make([]*internal.SearchResult, len(result.Hits))
 	for i, hit := range result.Hits {
 		startIndex, endIndex := -1, -1
 		for _, locations := range hit.Locations["Content"] {
@@ -405,11 +321,11 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		if t, err := time.Parse(time.RFC3339, hit.Fields["UpdatedAt"].(string)); err == nil {
 			updatedUnix = timeutil.TimeStamp(t.Unix())
 		}
-		searchResults[i] = &SearchResult{
+		searchResults[i] = &internal.SearchResult{
 			RepoID:      int64(hit.Fields["RepoID"].(float64)),
 			StartIndex:  startIndex,
 			EndIndex:    endIndex,
-			Filename:    filenameOfIndexerID(hit.ID),
+			Filename:    internal.FilenameOfIndexerID(hit.ID),
 			Content:     hit.Fields["Content"].(string),
 			CommitID:    hit.Fields["CommitID"].(string),
 			UpdatedUnix: updatedUnix,
@@ -418,7 +334,7 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		}
 	}
 
-	searchResultLanguages := make([]*SearchResultLanguages, 0, 10)
+	searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
 	if len(language) > 0 {
 		// Use separate query to go get all language counts
 		facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
@@ -426,7 +342,7 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		facetRequest.IncludeLocations = true
 		facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 
-		if result, err = b.indexer.Search(facetRequest); err != nil {
+		if result, err = b.inner.Indexer.Search(facetRequest); err != nil {
 			return 0, nil, nil, err
 		}
 
@@ -436,7 +352,7 @@ func (b *BleveIndexer) Search(ctx context.Context, repoIDs []int64, language, ke
 		if len(term.Term) == 0 {
 			continue
 		}
-		searchResultLanguages = append(searchResultLanguages, &SearchResultLanguages{
+		searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
 			Language: term.Term,
 			Color:    enry.GetColor(term.Term),
 			Count:    term.Count,

modules/indexer/code/bleve_test.go

@@ -1,30 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package code
-
-import (
-	"testing"
-
-	"code.gitea.io/gitea/models/unittest"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBleveIndexAndSearch(t *testing.T) {
-	unittest.PrepareTestEnv(t)
-
-	dir := t.TempDir()
-
-	idx, _, err := NewBleveIndexer(dir)
-	if err != nil {
-		assert.Fail(t, "Unable to create bleve indexer Error: %v", err)
-		if idx != nil {
-			idx.Close()
-		}
-		return
-	}
-	defer idx.Close()
-
-	testIndexer("beleve", t, idx)
-}

modules/indexer/code/elastic_search_test.go

@@ -1,41 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package code
-
-import (
-	"os"
-	"testing"
-
-	"code.gitea.io/gitea/models/unittest"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestESIndexAndSearch(t *testing.T) {
-	unittest.PrepareTestEnv(t)
-
-	u := os.Getenv("TEST_INDEXER_CODE_ES_URL")
-	if u == "" {
-		t.SkipNow()
-		return
-	}
-
-	indexer, _, err := NewElasticSearchIndexer(u, "gitea_codes")
-	if err != nil {
-		assert.Fail(t, "Unable to create ES indexer Error: %v", err)
-		if indexer != nil {
-			indexer.Close()
-		}
-		return
-	}
-	defer indexer.Close()
-
-	testIndexer("elastic_search", t, indexer)
-}
-
-func TestIndexPos(t *testing.T) {
-	startIdx, endIdx := indexPos("test index start and end", "start", "end")
-	assert.EqualValues(t, 11, startIdx)
-	assert.EqualValues(t, 24, endIdx)
-}

modules/indexer/code/elasticsearch/elasticsearch.go

@@ -1,25 +1,23 @@
 // Copyright 2020 The Gitea Authors. All rights reserved.
 // SPDX-License-Identifier: MIT
 
-package code
+package elasticsearch
 
 import (
 	"bufio"
 	"context"
-	"errors"
 	"fmt"
 	"io"
-	"net"
 	"strconv"
 	"strings"
-	"sync"
-	"time"
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
-	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
+	indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
+	inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -38,63 +36,22 @@ const (
 	esMultiMatchTypePhrasePrefix = "phrase_prefix"
 )
 
-var _ Indexer = &ElasticSearchIndexer{}
+var _ internal.Indexer = &Indexer{}
 
-// ElasticSearchIndexer implements Indexer interface
-type ElasticSearchIndexer struct {
-	client           *elastic.Client
-	indexerAliasName string
-	available        bool
-	stopTimer        chan struct{}
-	lock             sync.RWMutex
+// Indexer implements Indexer interface
+type Indexer struct {
+	inner                    *inner_elasticsearch.Indexer
+	indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
 }
 
-// NewElasticSearchIndexer creates a new elasticsearch indexer
-func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bool, error) {
-	opts := []elastic.ClientOptionFunc{
-		elastic.SetURL(url),
-		elastic.SetSniff(false),
-		elastic.SetHealthcheckInterval(10 * time.Second),
-		elastic.SetGzip(false),
+// NewIndexer creates a new elasticsearch indexer
+func NewIndexer(url, indexerName string) *Indexer {
+	inner := inner_elasticsearch.NewIndexer(url, indexerName, esRepoIndexerLatestVersion, defaultMapping)
+	indexer := &Indexer{
+		inner:   inner,
+		Indexer: inner,
 	}
-
-	logger := log.GetLogger(log.DEFAULT)
-
-	opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
-	opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
-	opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
-
-	client, err := elastic.NewClient(opts...)
-	if err != nil {
-		return nil, false, err
-	}
-
-	indexer := &ElasticSearchIndexer{
-		client:           client,
-		indexerAliasName: indexerName,
-		available:        true,
-		stopTimer:        make(chan struct{}),
-	}
-
-	ticker := time.NewTicker(10 * time.Second)
-	go func() {
-		for {
-			select {
-			case <-ticker.C:
-				indexer.checkAvailability()
-			case <-indexer.stopTimer:
-				ticker.Stop()
-				return
-			}
-		}
-	}()
-
-	exists, err := indexer.init()
-	if err != nil {
-		indexer.Close()
-		return nil, false, err
-	}
-	return indexer, !exists, err
+	return indexer
 }
 
 const (
@@ -127,72 +84,7 @@ const (
 	}`
 )
 
-func (b *ElasticSearchIndexer) realIndexerName() string {
-	return fmt.Sprintf("%s.v%d", b.indexerAliasName, esRepoIndexerLatestVersion)
-}
-
-// Init will initialize the indexer
-func (b *ElasticSearchIndexer) init() (bool, error) {
-	ctx := graceful.GetManager().HammerContext()
-	exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx)
-	if err != nil {
-		return false, b.checkError(err)
-	}
-	if !exists {
-		mapping := defaultMapping
-
-		createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !createIndex.Acknowledged {
-			return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping)
-		}
-	}
-
-	// check version
-	r, err := b.client.Aliases().Do(ctx)
-	if err != nil {
-		return false, b.checkError(err)
-	}
-
-	realIndexerNames := r.IndicesByAlias(b.indexerAliasName)
-	if len(realIndexerNames) < 1 {
-		res, err := b.client.Alias().
-			Add(b.realIndexerName(), b.indexerAliasName).
-			Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !res.Acknowledged {
-			return false, fmt.Errorf("create alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
-		}
-	} else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() {
-		log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.",
-			realIndexerNames[0], b.realIndexerName())
-		res, err := b.client.Alias().
-			Remove(realIndexerNames[0], b.indexerAliasName).
-			Add(b.realIndexerName(), b.indexerAliasName).
-			Do(ctx)
-		if err != nil {
-			return false, b.checkError(err)
-		}
-		if !res.Acknowledged {
-			return false, fmt.Errorf("change alias %s to index %s failed", b.indexerAliasName, b.realIndexerName())
-		}
-	}
-
-	return exists, nil
-}
-
-// Ping checks if elastic is available
-func (b *ElasticSearchIndexer) Ping() bool {
-	b.lock.RLock()
-	defer b.lock.RUnlock()
-	return b.available
-}
-
-func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
+func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
 	// Ignore vendored files in code search
 	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
 		return nil, nil
@@ -235,11 +127,11 @@ func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.Wr
 	if _, err = batchReader.Discard(1); err != nil {
 		return nil, err
 	}
-	id := filenameIndexerID(repo.ID, update.Filename)
+	id := internal.FilenameIndexerID(repo.ID, update.Filename)
 
 	return []elastic.BulkableRequest{
 		elastic.NewBulkIndexRequest().
-			Index(b.indexerAliasName).
+			Index(b.inner.VersionedIndexName()).
 			Id(id).
 			Doc(map[string]interface{}{
 				"repo_id":    repo.ID,
@@ -251,15 +143,15 @@ func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.Wr
 	}, nil
 }
 
-func (b *ElasticSearchIndexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest {
-	id := filenameIndexerID(repo.ID, filename)
+func (b *Indexer) addDelete(filename string, repo *repo_model.Repository) elastic.BulkableRequest {
+	id := internal.FilenameIndexerID(repo.ID, filename)
 	return elastic.NewBulkDeleteRequest().
-		Index(b.indexerAliasName).
+		Index(b.inner.VersionedIndexName()).
 		Id(id)
 }
 
 // Index will save the index data
-func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
+func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
 	reqs := make([]elastic.BulkableRequest, 0)
 	if len(changes.Updates) > 0 {
 		// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
@@ -288,21 +180,21 @@ func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repos
 	}
 
 	if len(reqs) > 0 {
-		_, err := b.client.Bulk().
-			Index(b.indexerAliasName).
+		_, err := b.inner.Client.Bulk().
+			Index(b.inner.VersionedIndexName()).
 			Add(reqs...).
 			Do(ctx)
-		return b.checkError(err)
+		return err
 	}
 	return nil
 }
 
 // Delete deletes indexes by ids
-func (b *ElasticSearchIndexer) Delete(repoID int64) error {
-	_, err := b.client.DeleteByQuery(b.indexerAliasName).
+func (b *Indexer) Delete(ctx context.Context, repoID int64) error {
+	_, err := b.inner.Client.DeleteByQuery(b.inner.VersionedIndexName()).
 		Query(elastic.NewTermsQuery("repo_id", repoID)).
-		Do(graceful.GetManager().HammerContext())
-	return b.checkError(err)
+		Do(ctx)
+	return err
 }
 
 // indexPos find words positions for start and the following end on content. It will
@@ -321,8 +213,8 @@ func indexPos(content, start, end string) (int, int) {
 	return startIdx, startIdx + len(start) + endIdx + len(end)
 }
 
-func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
-	hits := make([]*SearchResult, 0, pageSize)
+func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+	hits := make([]*internal.SearchResult, 0, pageSize)
 	for _, hit := range searchResult.Hits.Hits {
 		// FIXME: There is no way to get the position the keyword on the content currently on the same request.
 		// So we get it from content, this may made the query slower. See
@@ -341,7 +233,7 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
 			panic(fmt.Sprintf("2===%#v", hit.Highlight))
 		}
 
-		repoID, fileName := parseIndexerID(hit.Id)
+		repoID, fileName := internal.ParseIndexerID(hit.Id)
 		res := make(map[string]interface{})
 		if err := json.Unmarshal(hit.Source, &res); err != nil {
 			return 0, nil, nil, err
@@ -349,7 +241,7 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
 
 		language := res["language"].(string)
 
-		hits = append(hits, &SearchResult{
+		hits = append(hits, &internal.SearchResult{
 			RepoID:      repoID,
 			Filename:    fileName,
 			CommitID:    res["commit_id"].(string),
@@ -365,14 +257,14 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int)
 	return searchResult.TotalHits(), hits, extractAggs(searchResult), nil
 }
 
-func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
-	var searchResultLanguages []*SearchResultLanguages
+func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLanguages {
+	var searchResultLanguages []*internal.SearchResultLanguages
 	agg, found := searchResult.Aggregations.Terms("language")
 	if found {
-		searchResultLanguages = make([]*SearchResultLanguages, 0, 10)
+		searchResultLanguages = make([]*internal.SearchResultLanguages, 0, 10)
 
 		for _, bucket := range agg.Buckets {
-			searchResultLanguages = append(searchResultLanguages, &SearchResultLanguages{
+			searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
 				Language: bucket.Key.(string),
 				Color:    enry.GetColor(bucket.Key.(string)),
 				Count:    int(bucket.DocCount),
@@ -383,7 +275,7 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
 }
 
 // Search searches for codes and language stats by given conditions.
-func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	searchType := esMultiMatchTypeBestFields
 	if isMatch {
 		searchType = esMultiMatchTypePhrasePrefix
@@ -412,8 +304,8 @@ func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, lang
 	}
 
 	if len(language) == 0 {
-		searchResult, err := b.client.Search().
-			Index(b.indexerAliasName).
+		searchResult, err := b.inner.Client.Search().
+			Index(b.inner.VersionedIndexName()).
 			Aggregation("language", aggregation).
 			Query(query).
 			Highlight(
@@ -426,26 +318,26 @@ func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, lang
 			From(start).Size(pageSize).
 			Do(ctx)
 		if err != nil {
-			return 0, nil, nil, b.checkError(err)
+			return 0, nil, nil, err
 		}
 
 		return convertResult(searchResult, kw, pageSize)
 	}
 
 	langQuery := elastic.NewMatchQuery("language", language)
-	countResult, err := b.client.Search().
-		Index(b.indexerAliasName).
+	countResult, err := b.inner.Client.Search().
+		Index(b.inner.VersionedIndexName()).
 		Aggregation("language", aggregation).
 		Query(query).
-		Size(0). // We only needs stats information
+		Size(0). // We only need stats information
 		Do(ctx)
 	if err != nil {
-		return 0, nil, nil, b.checkError(err)
+		return 0, nil, nil, err
 	}
 
 	query = query.Must(langQuery)
-	searchResult, err := b.client.Search().
-		Index(b.indexerAliasName).
+	searchResult, err := b.inner.Client.Search().
+		Index(b.inner.VersionedIndexName()).
 		Query(query).
 		Highlight(
 			elastic.NewHighlight().
@@ -457,56 +349,10 @@ func (b *ElasticSearchIndexer) Search(ctx context.Context, repoIDs []int64, lang
 		From(start).Size(pageSize).
 		Do(ctx)
 	if err != nil {
-		return 0, nil, nil, b.checkError(err)
+		return 0, nil, nil, err
 	}
 
 	total, hits, _, err := convertResult(searchResult, kw, pageSize)
 
 	return total, hits, extractAggs(countResult), err
 }
-
-// Close implements indexer
-func (b *ElasticSearchIndexer) Close() {
-	select {
-	case <-b.stopTimer:
-	default:
-		close(b.stopTimer)
-	}
-}
-
-func (b *ElasticSearchIndexer) checkError(err error) error {
-	var opErr *net.OpError
-	if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
-		return err
-	}
-
-	b.setAvailability(false)
-
-	return err
-}
-
-func (b *ElasticSearchIndexer) checkAvailability() {
-	if b.Ping() {
-		return
-	}
-
-	// Request cluster state to check if elastic is available again
-	_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
-	if err != nil {
-		b.setAvailability(false)
-		return
-	}
-
-	b.setAvailability(true)
-}
-
-func (b *ElasticSearchIndexer) setAvailability(available bool) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.available == available {
-		return
-	}
-
-	b.available = available
-}

modules/indexer/code/elasticsearch/elasticsearch_test.go

@@ -0,0 +1,16 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIndexPos(t *testing.T) {
+	startIdx, endIdx := indexPos("test index start and end", "start", "end")
+	assert.EqualValues(t, 11, startIdx)
+	assert.EqualValues(t, 24, endIdx)
+}

modules/indexer/code/git.go

@@ -10,23 +10,11 @@ import (
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 )
 
-type fileUpdate struct {
-	Filename string
-	BlobSha  string
-	Size     int64
-	Sized    bool
-}
-
-// repoChanges changes (file additions/updates/removals) to a repo
-type repoChanges struct {
-	Updates          []fileUpdate
-	RemovedFilenames []string
-}
-
 func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) {
 	stdout, _, err := git.NewCommand(ctx, "show-ref", "-s").AddDynamicArguments(git.BranchPrefix + repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
 	if err != nil {
@@ -36,7 +24,7 @@ func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (stri
 }
 
 // getRepoChanges returns changes to repo since last indexer update
-func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
+func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) {
 	status, err := repo_model.GetIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode)
 	if err != nil {
 		return nil, err
@@ -67,16 +55,16 @@ func isIndexable(entry *git.TreeEntry) bool {
 }
 
 // parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command
-func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
+func parseGitLsTreeOutput(stdout []byte) ([]internal.FileUpdate, error) {
 	entries, err := git.ParseTreeEntries(stdout)
 	if err != nil {
 		return nil, err
 	}
 	idxCount := 0
-	updates := make([]fileUpdate, len(entries))
+	updates := make([]internal.FileUpdate, len(entries))
 	for _, entry := range entries {
 		if isIndexable(entry) {
-			updates[idxCount] = fileUpdate{
+			updates[idxCount] = internal.FileUpdate{
 				Filename: entry.Name(),
 				BlobSha:  entry.ID.String(),
 				Size:     entry.Size(),
@@ -89,8 +77,8 @@ func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
 }
 
 // genesisChanges get changes to add repo to the indexer for the first time
-func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
-	var changes repoChanges
+func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) {
+	var changes internal.RepoChanges
 	stdout, _, runErr := git.NewCommand(ctx, "ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).RunStdBytes(&git.RunOpts{Dir: repo.RepoPath()})
 	if runErr != nil {
 		return nil, runErr
@@ -102,20 +90,20 @@ func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision s
 }
 
 // nonGenesisChanges get changes since the previous indexer update
-func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
+func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*internal.RepoChanges, error) {
 	diffCmd := git.NewCommand(ctx, "diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision)
 	stdout, _, runErr := diffCmd.RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
 	if runErr != nil {
 		// previous commit sha may have been removed by a force push, so
 		// try rebuilding from scratch
 		log.Warn("git diff: %v", runErr)
-		if err := indexer.Delete(repo.ID); err != nil {
+		if err := (*globalIndexer.Load()).Delete(ctx, repo.ID); err != nil {
 			return nil, err
 		}
 		return genesisChanges(ctx, repo, revision)
 	}
 
-	var changes repoChanges
+	var changes internal.RepoChanges
 	var err error
 	updatedFilenames := make([]string, 0, 10)
 	for _, line := range strings.Split(stdout, "\n") {

modules/indexer/code/indexer.go

@@ -7,86 +7,41 @@ import (
 	"context"
 	"os"
 	"runtime/pprof"
-	"strconv"
-	"strings"
+	"sync/atomic"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/indexer/code/bleve"
+	"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 )
 
-// SearchResult result of performing a search in a repo
-type SearchResult struct {
-	RepoID      int64
-	StartIndex  int
-	EndIndex    int
-	Filename    string
-	Content     string
-	CommitID    string
-	UpdatedUnix timeutil.TimeStamp
-	Language    string
-	Color       string
+var (
+	indexerQueue *queue.WorkerPoolQueue[*internal.IndexerData]
+	// globalIndexer is the global indexer, it cannot be nil.
+	// When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready.
+	// So it's always safe use it as *globalIndexer.Load() and call its methods.
+	globalIndexer atomic.Pointer[internal.Indexer]
+	dummyIndexer  *internal.Indexer
+)
+
+func init() {
+	i := internal.NewDummyIndexer()
+	dummyIndexer = &i
+	globalIndexer.Store(dummyIndexer)
 }
 
-// SearchResultLanguages result of top languages count in search results
-type SearchResultLanguages struct {
-	Language string
-	Color    string
-	Count    int
-}
-
-// Indexer defines an interface to index and search code contents
-type Indexer interface {
-	Ping() bool
-	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
-	Delete(repoID int64) error
-	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
-	Close()
-}
-
-func filenameIndexerID(repoID int64, filename string) string {
-	return indexerID(repoID) + "_" + filename
-}
-
-func indexerID(id int64) string {
-	return strconv.FormatInt(id, 36)
-}
-
-func parseIndexerID(indexerID string) (int64, string) {
-	index := strings.IndexByte(indexerID, '_')
-	if index == -1 {
-		log.Error("Unexpected ID in repo indexer: %s", indexerID)
-	}
-	repoID, _ := strconv.ParseInt(indexerID[:index], 36, 64)
-	return repoID, indexerID[index+1:]
-}
-
-func filenameOfIndexerID(indexerID string) string {
-	index := strings.IndexByte(indexerID, '_')
-	if index == -1 {
-		log.Error("Unexpected ID in repo indexer: %s", indexerID)
-	}
-	return indexerID[index+1:]
-}
-
-// IndexerData represents data stored in the code indexer
-type IndexerData struct {
-	RepoID int64
-}
-
-var indexerQueue *queue.WorkerPoolQueue[*IndexerData]
-
-func index(ctx context.Context, indexer Indexer, repoID int64) error {
+func index(ctx context.Context, indexer internal.Indexer, repoID int64) error {
 	repo, err := repo_model.GetRepositoryByID(ctx, repoID)
 	if repo_model.IsErrRepoNotExist(err) {
-		return indexer.Delete(repoID)
+		return indexer.Delete(ctx, repoID)
 	}
 	if err != nil {
 		return err
@@ -139,7 +94,7 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error {
 // Init initialize the repo indexer
 func Init() {
 	if !setting.Indexer.RepoIndexerEnabled {
-		indexer.Close()
+		(*globalIndexer.Load()).Close()
 		return
 	}
 
@@ -153,7 +108,7 @@ func Init() {
 		}
 		cancel()
 		log.Debug("Closing repository indexer")
-		indexer.Close()
+		(*globalIndexer.Load()).Close()
 		log.Info("PID: %d Repository Indexer closed", os.Getpid())
 		finished()
 	})
@@ -163,13 +118,8 @@ func Init() {
 	// Create the Queue
 	switch setting.Indexer.RepoType {
 	case "bleve", "elasticsearch":
-		handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
-			idx, err := indexer.get()
-			if idx == nil || err != nil {
-				log.Warn("Codes indexer handler: indexer is not ready, retry later.")
-				return items
-			}
-
+		handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) {
+			indexer := *globalIndexer.Load()
 			for _, indexerData := range items {
 				log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
 
@@ -188,11 +138,7 @@ func Init() {
 					code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105)
 				*/
 				if err := index(ctx, indexer, indexerData.RepoID); err != nil {
-					if !idx.Ping() {
-						log.Error("Code indexer handler: indexer is unavailable.")
-						unhandled = append(unhandled, indexerData)
-						continue
-					}
+					unhandled = append(unhandled, indexerData)
 					if !setting.IsInTesting {
 						log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err)
 					}
@@ -213,8 +159,8 @@ func Init() {
 		pprof.SetGoroutineLabels(ctx)
 		start := time.Now()
 		var (
-			rIndexer Indexer
-			populate bool
+			rIndexer internal.Indexer
+			existed  bool
 			err      error
 		)
 		switch setting.Indexer.RepoType {
@@ -228,10 +174,11 @@ func Init() {
 				}
 			}()
 
-			rIndexer, populate, err = NewBleveIndexer(setting.Indexer.RepoPath)
+			rIndexer = bleve.NewIndexer(setting.Indexer.RepoPath)
+			existed, err = rIndexer.Init(ctx)
 			if err != nil {
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
 				close(waitChannel)
 				log.Fatal("PID: %d Unable to initialize the bleve Repository Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.RepoPath, err)
 			}
@@ -245,23 +192,31 @@ func Init() {
 				}
 			}()
 
-			rIndexer, populate, err = NewElasticSearchIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName)
+			rIndexer = elasticsearch.NewIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName)
 			if err != nil {
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
+				close(waitChannel)
+				log.Fatal("PID: %d Unable to create the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
+			}
+			existed, err = rIndexer.Init(ctx)
+			if err != nil {
+				cancel()
+				(*globalIndexer.Load()).Close()
 				close(waitChannel)
 				log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
 			}
+
 		default:
 			log.Fatal("PID: %d Unknown Indexer type: %s", os.Getpid(), setting.Indexer.RepoType)
 		}
 
-		indexer.set(rIndexer)
+		globalIndexer.Store(&rIndexer)
 
 		// Start processing the queue
 		go graceful.GetManager().RunWithCancel(indexerQueue)
 
-		if populate {
+		if !existed { // populate the index because it's created for the first time
 			go graceful.GetManager().RunWithShutdownContext(populateRepoIndexer)
 		}
 		select {
@@ -283,18 +238,18 @@ func Init() {
 			case <-graceful.GetManager().IsShutdown():
 				log.Warn("Shutdown before Repository Indexer completed initialization")
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
 			case duration, ok := <-waitChannel:
 				if !ok {
 					log.Warn("Repository Indexer Initialization failed")
 					cancel()
-					indexer.Close()
+					(*globalIndexer.Load()).Close()
 					return
 				}
 				log.Info("Repository Indexer Initialization took %v", duration)
 			case <-time.After(timeout):
 				cancel()
-				indexer.Close()
+				(*globalIndexer.Load()).Close()
 				log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
 			}
 		}()
@@ -303,21 +258,15 @@ func Init() {
 
 // UpdateRepoIndexer update a repository's entries in the indexer
 func UpdateRepoIndexer(repo *repo_model.Repository) {
-	indexData := &IndexerData{RepoID: repo.ID}
+	indexData := &internal.IndexerData{RepoID: repo.ID}
 	if err := indexerQueue.Push(indexData); err != nil {
 		log.Error("Update repo index data %v failed: %v", indexData, err)
 	}
 }
 
 // IsAvailable checks if issue indexer is available
-func IsAvailable() bool {
-	idx, err := indexer.get()
-	if err != nil {
-		log.Error("IsAvailable(): unable to get indexer: %v", err)
-		return false
-	}
-
-	return idx.Ping()
+func IsAvailable(ctx context.Context) bool {
+	return (*globalIndexer.Load()).Ping(ctx) == nil
 }
 
 // populateRepoIndexer populate the repo indexer with pre-existing data. This
@@ -368,7 +317,7 @@ func populateRepoIndexer(ctx context.Context) {
 				return
 			default:
 			}
-			if err := indexerQueue.Push(&IndexerData{RepoID: id}); err != nil {
+			if err := indexerQueue.Push(&internal.IndexerData{RepoID: id}); err != nil {
 				log.Error("indexerQueue.Push: %v", err)
 				return
 			}

modules/indexer/code/indexer_test.go

@@ -5,11 +5,15 @@ package code
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/indexer/code/bleve"
+	"code.gitea.io/gitea/modules/indexer/code/elasticsearch"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 
 	_ "code.gitea.io/gitea/models"
 
@@ -22,7 +26,7 @@ func TestMain(m *testing.M) {
 	})
 }
 
-func testIndexer(name string, t *testing.T, indexer Indexer) {
+func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 	t.Run(name, func(t *testing.T) {
 		var repoID int64 = 1
 		err := index(git.DefaultContext, indexer, repoID)
@@ -81,6 +85,48 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
 			})
 		}
 
-		assert.NoError(t, indexer.Delete(repoID))
+		assert.NoError(t, indexer.Delete(context.Background(), repoID))
 	})
 }
+
+func TestBleveIndexAndSearch(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+
+	dir := t.TempDir()
+
+	idx := bleve.NewIndexer(dir)
+	_, err := idx.Init(context.Background())
+	if err != nil {
+		assert.Fail(t, "Unable to create bleve indexer Error: %v", err)
+		if idx != nil {
+			idx.Close()
+		}
+		return
+	}
+	defer idx.Close()
+
+	testIndexer("beleve", t, idx)
+}
+
+func TestESIndexAndSearch(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+
+	u := os.Getenv("TEST_INDEXER_CODE_ES_URL")
+	if u == "" {
+		t.SkipNow()
+		return
+	}
+
+	indexer := elasticsearch.NewIndexer(u, "gitea_codes")
+	if _, err := indexer.Init(context.Background()); err != nil {
+		assert.Fail(t, "Unable to init ES indexer Error: %v", err)
+		if indexer != nil {
+			indexer.Close()
+		}
+		return
+	}
+
+	defer indexer.Close()
+
+	testIndexer("elastic_search", t, indexer)
+}

modules/indexer/code/internal/indexer.go

@@ -0,0 +1,43 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/indexer/internal"
+)
+
+// Indexer defines an interface to index and search code contents
+type Indexer interface {
+	internal.Indexer
+	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error
+	Delete(ctx context.Context, repoID int64) error
+	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
+}
+
+// NewDummyIndexer returns a dummy indexer
+func NewDummyIndexer() Indexer {
+	return &dummyIndexer{
+		Indexer: internal.NewDummyIndexer(),
+	}
+}
+
+type dummyIndexer struct {
+	internal.Indexer
+}
+
+func (d *dummyIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error {
+	return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+	return 0, nil, nil, fmt.Errorf("indexer is not ready")
+}

modules/indexer/code/internal/model.go

@@ -0,0 +1,44 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import "code.gitea.io/gitea/modules/timeutil"
+
+type FileUpdate struct {
+	Filename string
+	BlobSha  string
+	Size     int64
+	Sized    bool
+}
+
+// RepoChanges changes (file additions/updates/removals) to a repo
+type RepoChanges struct {
+	Updates          []FileUpdate
+	RemovedFilenames []string
+}
+
+// IndexerData represents data stored in the code indexer
+type IndexerData struct {
+	RepoID int64
+}
+
+// SearchResult result of performing a search in a repo
+type SearchResult struct {
+	RepoID      int64
+	StartIndex  int
+	EndIndex    int
+	Filename    string
+	Content     string
+	CommitID    string
+	UpdatedUnix timeutil.TimeStamp
+	Language    string
+	Color       string
+}
+
+// SearchResultLanguages result of top languages count in search results
+type SearchResultLanguages struct {
+	Language string
+	Color    string
+	Count    int
+}

modules/indexer/code/internal/util.go

@@ -0,0 +1,32 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"strings"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func FilenameIndexerID(repoID int64, filename string) string {
+	return internal.Base36(repoID) + "_" + filename
+}
+
+func ParseIndexerID(indexerID string) (int64, string) {
+	index := strings.IndexByte(indexerID, '_')
+	if index == -1 {
+		log.Error("Unexpected ID in repo indexer: %s", indexerID)
+	}
+	repoID, _ := internal.ParseBase36(indexerID[:index])
+	return repoID, indexerID[index+1:]
+}
+
+func FilenameOfIndexerID(indexerID string) string {
+	index := strings.IndexByte(indexerID, '_')
+	if index == -1 {
+		log.Error("Unexpected ID in repo indexer: %s", indexerID)
+	}
+	return indexerID[index+1:]
+}

modules/indexer/code/search.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/highlight"
+	"code.gitea.io/gitea/modules/indexer/code/internal"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -25,6 +26,8 @@ type Result struct {
 	FormattedLines string
 }
 
+type SearchResultLanguages = internal.SearchResultLanguages
+
 func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
 	startIndex := selectionStartIndex
 	numLinesBefore := 0
@@ -61,7 +64,7 @@ func writeStrings(buf *bytes.Buffer, strs ...string) error {
 	return nil
 }
 
-func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, error) {
+func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Result, error) {
 	startLineNum := 1 + strings.Count(result.Content[:startIndex], "\n")
 
 	var formattedLinesBuffer bytes.Buffer
@@ -109,12 +112,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
 }
 
 // PerformSearch perform a search on a repository
-func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
+func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*internal.SearchResultLanguages, error) {
 	if len(keyword) == 0 {
 		return 0, nil, nil, nil
 	}
 
-	total, results, resultLanguages, err := indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
+	total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
 	if err != nil {
 		return 0, nil, nil, err
 	}

modules/indexer/code/wrapped.go

@@ -1,104 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package code
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	repo_model "code.gitea.io/gitea/models/repo"
-	"code.gitea.io/gitea/modules/log"
-)
-
-var indexer = newWrappedIndexer()
-
-// ErrWrappedIndexerClosed is the error returned if the indexer was closed before it was ready
-var ErrWrappedIndexerClosed = fmt.Errorf("Indexer closed before ready")
-
-type wrappedIndexer struct {
-	internal Indexer
-	lock     sync.RWMutex
-	cond     *sync.Cond
-	closed   bool
-}
-
-func newWrappedIndexer() *wrappedIndexer {
-	w := &wrappedIndexer{}
-	w.cond = sync.NewCond(w.lock.RLocker())
-	return w
-}
-
-func (w *wrappedIndexer) set(indexer Indexer) {
-	w.lock.Lock()
-	defer w.lock.Unlock()
-	if w.closed {
-		// Too late!
-		indexer.Close()
-	}
-	w.internal = indexer
-	w.cond.Broadcast()
-}
-
-func (w *wrappedIndexer) get() (Indexer, error) {
-	w.lock.RLock()
-	defer w.lock.RUnlock()
-	if w.internal == nil {
-		if w.closed {
-			return nil, ErrWrappedIndexerClosed
-		}
-		w.cond.Wait()
-		if w.closed {
-			return nil, ErrWrappedIndexerClosed
-		}
-	}
-	return w.internal, nil
-}
-
-// Ping checks if elastic is available
-func (w *wrappedIndexer) Ping() bool {
-	indexer, err := w.get()
-	if err != nil {
-		log.Warn("Failed to get indexer: %v", err)
-		return false
-	}
-	return indexer.Ping()
-}
-
-func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
-	indexer, err := w.get()
-	if err != nil {
-		return err
-	}
-	return indexer.Index(ctx, repo, sha, changes)
-}
-
-func (w *wrappedIndexer) Delete(repoID int64) error {
-	indexer, err := w.get()
-	if err != nil {
-		return err
-	}
-	return indexer.Delete(repoID)
-}
-
-func (w *wrappedIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
-	indexer, err := w.get()
-	if err != nil {
-		return 0, nil, nil, err
-	}
-	return indexer.Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
-}
-
-func (w *wrappedIndexer) Close() {
-	w.lock.Lock()
-	defer w.lock.Unlock()
-	if w.closed {
-		return
-	}
-	w.closed = true
-	w.cond.Broadcast()
-	if w.internal != nil {
-		w.internal.Close()
-	}
-}

modules/indexer/internal/base32.go

@@ -0,0 +1,21 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+	"fmt"
+	"strconv"
+)
+
+func Base36(i int64) string {
+	return strconv.FormatInt(i, 36)
+}
+
+func ParseBase36(s string) (int64, error) {
+	i, err := strconv.ParseInt(s, 36, 64)
+	if err != nil {
+		return 0, fmt.Errorf("invalid base36 integer %q: %w", s, err)
+	}
+	return i, nil
+}

modules/indexer/internal/bleve/indexer.go

@@ -0,0 +1,103 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/ethantkoenig/rupture"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic bleve indexer implementation
+type Indexer struct {
+	Indexer bleve.Index
+
+	indexDir      string
+	version       int
+	mappingGetter MappingGetter
+}
+
+type MappingGetter func() (mapping.IndexMapping, error)
+
+func NewIndexer(indexDir string, version int, mappingGetter func() (mapping.IndexMapping, error)) *Indexer {
+	return &Indexer{
+		indexDir:      indexDir,
+		version:       version,
+		mappingGetter: mappingGetter,
+	}
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+	if i == nil {
+		return false, fmt.Errorf("cannot init nil indexer")
+	}
+
+	if i.Indexer != nil {
+		return false, fmt.Errorf("indexer is already initialized")
+	}
+
+	indexer, version, err := openIndexer(i.indexDir, i.version)
+	if err != nil {
+		return false, err
+	}
+	if indexer != nil {
+		i.Indexer = indexer
+		return true, nil
+	}
+
+	if version != 0 {
+		log.Warn("Found older bleve index with version %d, Gitea will remove it and rebuild", version)
+	}
+
+	indexMapping, err := i.mappingGetter()
+	if err != nil {
+		return false, err
+	}
+
+	indexer, err = bleve.New(i.indexDir, indexMapping)
+	if err != nil {
+		return false, err
+	}
+
+	if err = rupture.WriteIndexMetadata(i.indexDir, &rupture.IndexMetadata{
+		Version: i.version,
+	}); err != nil {
+		return false, err
+	}
+
+	i.Indexer = indexer
+
+	return false, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(_ context.Context) error {
+	if i == nil {
+		return fmt.Errorf("cannot ping nil indexer")
+	}
+	if i.Indexer == nil {
+		return fmt.Errorf("indexer is not initialized")
+	}
+	return nil
+}
+
+func (i *Indexer) Close() {
+	if i == nil {
+		return
+	}
+
+	if err := i.Indexer.Close(); err != nil {
+		log.Error("Failed to close bleve indexer in %q: %v", i.indexDir, err)
+	}
+	i.Indexer = nil
+}

modules/indexer/internal/bleve/util.go

@@ -0,0 +1,49 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"errors"
+	"os"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/index/upsidedown"
+	"github.com/ethantkoenig/rupture"
+)
+
+// openIndexer open the index at the specified path, checking for metadata
+// updates and bleve version updates.  If index needs to be created (or
+// re-created), returns (nil, nil)
+func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
+	_, err := os.Stat(path)
+	if err != nil && os.IsNotExist(err) {
+		return nil, 0, nil
+	} else if err != nil {
+		return nil, 0, err
+	}
+
+	metadata, err := rupture.ReadIndexMetadata(path)
+	if err != nil {
+		return nil, 0, err
+	}
+	if metadata.Version < latestVersion {
+		// the indexer is using a previous version, so we should delete it and
+		// re-populate
+		return nil, metadata.Version, util.RemoveAll(path)
+	}
+
+	index, err := bleve.Open(path)
+	if err != nil {
+		if errors.Is(err, upsidedown.IncompatibleVersion) {
+			log.Warn("Indexer was built with a previous version of bleve, deleting and rebuilding")
+			return nil, 0, util.RemoveAll(path)
+		}
+		return nil, 0, err
+	}
+
+	return index, 0, nil
+}

modules/indexer/internal/db/indexer.go

@@ -0,0 +1,34 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic db indexer implementation
+type Indexer struct{}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+	// Return true to indicate that the index was opened/existed.
+	// So that the indexer will not try to populate the index, the data is already there.
+	return true, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(_ context.Context) error {
+	// No need to ping database to check if it is available.
+	// If the database goes down, Gitea will go down, so nobody will care if the indexer is available.
+	return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+	// nothing to do
+}

modules/indexer/internal/elasticsearch/indexer.go

@@ -0,0 +1,92 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/indexer/internal"
+
+	"github.com/olivere/elastic/v7"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic elasticsearch indexer implementation
+type Indexer struct {
+	Client *elastic.Client
+
+	url       string
+	indexName string
+	version   int
+	mapping   string
+}
+
+func NewIndexer(url, indexName string, version int, mapping string) *Indexer {
+	return &Indexer{
+		url:       url,
+		indexName: indexName,
+		version:   version,
+		mapping:   mapping,
+	}
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(ctx context.Context) (bool, error) {
+	if i == nil {
+		return false, fmt.Errorf("cannot init nil indexer")
+	}
+	if i.Client != nil {
+		return false, fmt.Errorf("indexer is already initialized")
+	}
+
+	client, err := i.initClient()
+	if err != nil {
+		return false, err
+	}
+	i.Client = client
+
+	exists, err := i.Client.IndexExists(i.VersionedIndexName()).Do(ctx)
+	if err != nil {
+		return false, err
+	}
+	if exists {
+		return true, nil
+	}
+
+	if err := i.createIndex(ctx); err != nil {
+		return false, err
+	}
+
+	return exists, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(ctx context.Context) error {
+	if i == nil {
+		return fmt.Errorf("cannot ping nil indexer")
+	}
+	if i.Client == nil {
+		return fmt.Errorf("indexer is not initialized")
+	}
+
+	resp, err := i.Client.ClusterHealth().Do(ctx)
+	if err != nil {
+		return err
+	}
+	if resp.Status != "green" {
+		// see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
+		return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status)
+	}
+	return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+	if i == nil {
+		return
+	}
+	i.Client = nil
+}