From ff77a928ae4146dedec16e0ac3d52878f0d71b38 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 01:13:29 +0200 Subject: [PATCH 01/10] ci: improve cache reuse and drop redundant build work - e2e and checks-backend: enable build-cache restore so they hit the seeded gobuild cache. - pgsql/sqlite/mysql/mssql: replace `make backend` with `make generate-go`. Integration tests build their own test binary and never invoke the gitea executable; only bindata generation is needed. - unit-tests-gogit: narrow to packages with gogit/nogogit-tagged files via a new `test-backend-gogit` Makefile target. Other packages produce identical compiled output regardless of the gogit tag, so retesting them was busywork. - cache-seeder: stop the lint job from competing with the gobuild job on the shared non-rotated gobuild key. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/cache-seeder.yml | 2 ++ .github/workflows/pull-compliance.yml | 1 - .github/workflows/pull-db-tests.yml | 14 +++++--------- .github/workflows/pull-e2e-tests.yml | 1 - Makefile | 6 ++++++ 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cache-seeder.yml b/.github/workflows/cache-seeder.yml index 733077cc80..5aa0a98bfc 100644 --- a/.github/workflows/cache-seeder.yml +++ b/.github/workflows/cache-seeder.yml @@ -64,6 +64,8 @@ jobs: - uses: ./.github/actions/go-cache with: cache-name: ${{ matrix.job }} + # the gobuild job already saves the shared gobuild cache key; let only it win. + build-cache: "false" lint-cache: "true" - run: make deps-backend deps-tools - run: make ${{ matrix.target }} diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml index 51e1c805e6..801c77d56d 100644 --- a/.github/workflows/pull-compliance.yml +++ b/.github/workflows/pull-compliance.yml @@ -78,7 +78,6 @@ jobs: - uses: ./.github/actions/go-cache with: cache-name: checks-backend - build-cache: "false" - run: make deps-backend deps-tools - run: make --always-make checks-backend # ensure the "go-licenses" make target runs diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index a3f4efdc16..a5184baf67 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -52,8 +52,7 @@ jobs: cache-name: pgsql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' - - run: make deps-backend - - run: make backend + - run: make deps-backend generate-go env: TAGS: bindata - name: run migration tests @@ -81,8 +80,7 @@ jobs: - uses: ./.github/actions/go-cache with: cache-name: sqlite - - run: make deps-backend - - run: make backend + - run: make deps-backend generate-go env: TAGS: bindata gogit GOEXPERIMENT: @@ -161,7 +159,7 @@ jobs: TAGS: bindata GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - name: unit-tests-gogit - run: make test-backend test-check + run: make test-backend-gogit test-check env: GOTEST_FLAGS: -race -timeout=20m TAGS: bindata gogit @@ -209,8 +207,7 @@ jobs: cache-name: mysql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' - - run: make deps-backend - - run: make backend + - run: make deps-backend generate-go env: TAGS: bindata - name: run migration tests @@ -250,8 +247,7 @@ jobs: cache-name: mssql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql devstoreaccount1.azurite.local" | sudo tee -a /etc/hosts' - - run: make deps-backend - - run: make backend + - run: make deps-backend generate-go env: TAGS: bindata - run: GITEA_TEST_DATABASE=mssql make test-migration diff --git a/.github/workflows/pull-e2e-tests.yml b/.github/workflows/pull-e2e-tests.yml index f81026a5ae..0c2a2634f1 100644 --- a/.github/workflows/pull-e2e-tests.yml +++ b/.github/workflows/pull-e2e-tests.yml @@ -28,7 +28,6 @@ jobs: - uses: ./.github/actions/go-cache with: cache-name: e2e - build-cache: "false" - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: diff --git a/Makefile b/Makefile index 27b2c30295..e16a31399f 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,7 @@ LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64,linux/r GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration,$(shell $(GO) list ./... | grep -v /vendor/)) MIGRATE_TEST_PACKAGES ?= $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) +GO_GOGIT_TEST_PACKAGES ?= code.gitea.io/gitea/modules/git/... code.gitea.io/gitea/modules/gitrepo/... code.gitea.io/gitea/modules/lfs/... FRONTEND_SOURCES := $(shell find web_src/js web_src/css -type f) FRONTEND_CONFIGS := vite.config.ts tailwind.config.ts @@ -381,6 +382,11 @@ test-backend: ## test backend files @echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)'..." @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $(GO_TEST_PACKAGES) +.PHONY: test-backend-gogit +test-backend-gogit: ## test gogit-affected packages only + @echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over gogit-affected packages..." + @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $(GO_GOGIT_TEST_PACKAGES) + .PHONY: test-frontend test-frontend: node_modules ## test frontend files pnpm exec vitest From ad9b6337572b51b78004b7bed1d9e9d799396aff Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 01:26:34 +0200 Subject: [PATCH 02/10] ci: restore make backend in db-test jobs The previous commit replaced `make backend` with `make generate-go` on the assumption that the gitea executable was unused by the integration tests. It is used: integration tests install git pre-receive hooks that shell out to the binary, so `git push` operations during tests fail with "No such file or directory". Reverts that part of the previous change; the other cache-reuse tweaks remain. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/pull-db-tests.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index a5184baf67..358f8b8eac 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -52,7 +52,8 @@ jobs: cache-name: pgsql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' - - run: make deps-backend generate-go + - run: make deps-backend + - run: make backend env: TAGS: bindata - name: run migration tests @@ -80,7 +81,8 @@ jobs: - uses: ./.github/actions/go-cache with: cache-name: sqlite - - run: make deps-backend generate-go + - run: make deps-backend + - run: make backend env: TAGS: bindata gogit GOEXPERIMENT: @@ -207,7 +209,8 @@ jobs: cache-name: mysql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' - - run: make deps-backend generate-go + - run: make deps-backend + - run: make backend env: TAGS: bindata - name: run migration tests @@ -247,7 +250,8 @@ jobs: cache-name: mssql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql devstoreaccount1.azurite.local" | sudo tee -a /etc/hosts' - - run: make deps-backend generate-go + - run: make deps-backend + - run: make backend env: TAGS: bindata - run: GITEA_TEST_DATABASE=mssql make test-migration From 8d6615f939faeb1cb32b235ddc720ee1f83d2ec7 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 01:35:03 +0200 Subject: [PATCH 03/10] ci: shard integration tests across 2 runners per database Splits each database integration job (pgsql/sqlite/mysql/mssql) into two parallel matrix shards. Test names are enumerated from the integration source and partitioned round-robin (~301/302 of 603 tests per shard); names that don't match the shard are filtered out via -test.run. Migration tests (~50-90 s, fast, sequential) only run on shard 1. The original job names (test-pgsql, test-sqlite, test-mysql, test-mssql) are kept as one-step aggregator jobs that depend on the shards job and report success only when all shards passed. This keeps any branch-protection rule referencing those names valid. Source-based enumeration is used because the test binary's -test.list calls TestMain, which boots the full Gitea environment and panics without a configured database. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/pull-db-tests.yml | 66 ++++++++++++++++++++++++++--- Makefile | 4 ++ tools/test-integration-shard.sh | 25 +++++++++++ 3 files changed, 90 insertions(+), 5 deletions(-) create mode 100755 tools/test-integration-shard.sh diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 358f8b8eac..f21cdf8135 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -14,10 +14,14 @@ jobs: files-changed: uses: ./.github/workflows/files-changed.yml - test-pgsql: + test-pgsql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] services: pgsql: image: postgres:14 @@ -57,6 +61,7 @@ jobs: env: TAGS: bindata - name: run migration tests + if: matrix.shard == 1 run: GITEA_TEST_DATABASE=pgsql make test-migration - name: run tests run: GITEA_TEST_DATABASE=pgsql make test-integration @@ -66,11 +71,24 @@ jobs: GOTEST_FLAGS: -race -timeout=40m TAGS: bindata gogit TEST_LDAP: 1 + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: 2 - test-sqlite: + test-pgsql: + needs: [files-changed, test-pgsql-shards] + if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') + runs-on: ubuntu-latest + steps: + - run: '[ "${{ needs.test-pgsql-shards.result }}" = "success" ]' + + test-sqlite-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 @@ -87,6 +105,7 @@ jobs: TAGS: bindata gogit GOEXPERIMENT: - name: run migration tests + if: matrix.shard == 1 run: GITEA_TEST_DATABASE=sqlite make test-migration env: TAGS: bindata gogit @@ -98,6 +117,15 @@ jobs: GOTEST_FLAGS: -timeout=40m TAGS: bindata gogit GOEXPERIMENT: + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: 2 + + test-sqlite: + needs: [files-changed, test-sqlite-shards] + if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') + runs-on: ubuntu-latest + steps: + - run: '[ "${{ needs.test-sqlite-shards.result }}" = "success" ]' test-unit: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' @@ -168,10 +196,14 @@ jobs: GOEXPERIMENT: GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - test-mysql: + test-mysql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] services: mysql: # the bitnami mysql image has more options than the official one, it's easier to customize @@ -214,17 +246,31 @@ jobs: env: TAGS: bindata - name: run migration tests + if: matrix.shard == 1 run: GITEA_TEST_DATABASE=mysql make test-migration - name: run tests run: GITEA_TEST_DATABASE=mysql make test-integration env: TAGS: bindata TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: 2 - test-mssql: + test-mysql: + needs: [files-changed, test-mysql-shards] + if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') + runs-on: ubuntu-latest + steps: + - run: '[ "${{ needs.test-mysql-shards.result }}" = "success" ]' + + test-mssql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] services: mssql: image: mcr.microsoft.com/mssql/server:2019-latest @@ -254,9 +300,19 @@ jobs: - run: make backend env: TAGS: bindata - - run: GITEA_TEST_DATABASE=mssql make test-migration + - if: matrix.shard == 1 + run: GITEA_TEST_DATABASE=mssql make test-migration - name: run tests run: GITEA_TEST_DATABASE=mssql make test-integration timeout-minutes: 50 env: TAGS: bindata + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: 2 + + test-mssql: + needs: [files-changed, test-mssql-shards] + if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') + runs-on: ubuntu-latest + steps: + - run: '[ "${{ needs.test-mssql-shards.result }}" = "success" ]' diff --git a/Makefile b/Makefile index e16a31399f..7f06ef39ff 100644 --- a/Makefile +++ b/Makefile @@ -450,7 +450,11 @@ test-integration: @# would flood output per passing test. testcache can't help these tests anyway — @# they mutate the work directory, so cache inputs change between runs. $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -c code.gitea.io/gitea/tests/integration -o ./test-integration-$(GITEA_TEST_DATABASE).test +ifdef TEST_SHARD + ./tools/test-integration-shard.sh ./test-integration-$(GITEA_TEST_DATABASE).test +else ./test-integration-$(GITEA_TEST_DATABASE).test +endif .PHONY: test-integration\#% test-integration\#%: diff --git a/tools/test-integration-shard.sh b/tools/test-integration-shard.sh new file mode 100755 index 0000000000..1f72dd5094 --- /dev/null +++ b/tools/test-integration-shard.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +# Run a deterministic shard of the integration test binary. Test names are +# enumerated from source — running the binary with -test.list isn't viable +# because TestMain boots the full Gitea environment and would panic without +# a configured database. + +binary=$1 +shard=${TEST_SHARD:?missing TEST_SHARD} +total=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS} + +names=$(grep -hE '^func Test[A-Z][A-Za-z0-9_]*\(' tests/integration/*.go \ + | sed -E 's/^func (Test[A-Z][A-Za-z0-9_]*).*/\1/' \ + | sort -u \ + | awk -v s="$shard" -v t="$total" 'NR % t == (s - 1) % t') + +if [ -z "$names" ]; then + echo "shard $shard/$total has no tests assigned" >&2 + exit 0 +fi + +pattern=$(echo "$names" | paste -sd '|' -) +echo "Running shard $shard/$total ($(echo "$names" | wc -l | tr -d ' ') tests)" +exec "$binary" -test.run "^($pattern)\$" From 4037cd51c3eda399693052bca1a0c43f9bc74954 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 01:41:51 +0200 Subject: [PATCH 04/10] ci: derive gogit test packages dynamically from imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hardcoded list of three top-level paths (modules/git, modules/gitrepo, modules/lfs) with an at-build-time derivation. tools/find-gogit-test-pkgs.sh queries `go list` for every package whose own code or test code imports any of the gogit-affected modules — currently 72 packages — and feeds that list into `make test-backend-gogit`. This restores coverage that the static list dropped: callers like models/git, services/repository, modules/repository, etc., whose unit tests transitively exercise gogit/nogogit code paths via the modules/git API. Co-Authored-By: Claude (Opus 4.7) --- Makefile | 8 ++++---- tools/find-gogit-test-pkgs.sh | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100755 tools/find-gogit-test-pkgs.sh diff --git a/Makefile b/Makefile index 7f06ef39ff..717817c390 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,6 @@ LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64,linux/r GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration,$(shell $(GO) list ./... | grep -v /vendor/)) MIGRATE_TEST_PACKAGES ?= $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) -GO_GOGIT_TEST_PACKAGES ?= code.gitea.io/gitea/modules/git/... code.gitea.io/gitea/modules/gitrepo/... code.gitea.io/gitea/modules/lfs/... FRONTEND_SOURCES := $(shell find web_src/js web_src/css -type f) FRONTEND_CONFIGS := vite.config.ts tailwind.config.ts @@ -383,9 +382,10 @@ test-backend: ## test backend files @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $(GO_TEST_PACKAGES) .PHONY: test-backend-gogit -test-backend-gogit: ## test gogit-affected packages only - @echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over gogit-affected packages..." - @$(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $(GO_GOGIT_TEST_PACKAGES) +test-backend-gogit: ## test packages whose code or tests import the gogit-affected modules + @pkgs=$$(./tools/find-gogit-test-pkgs.sh '$(TAGS)'); \ + echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over $$(echo $$pkgs | wc -w) gogit-affected packages..."; \ + $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs .PHONY: test-frontend test-frontend: node_modules ## test frontend files diff --git a/tools/find-gogit-test-pkgs.sh b/tools/find-gogit-test-pkgs.sh new file mode 100755 index 0000000000..59525644ad --- /dev/null +++ b/tools/find-gogit-test-pkgs.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +# Print packages whose own code or test code imports any of the gogit-affected +# modules (modules/git, modules/gitrepo, modules/lfs). These are the packages +# whose tests can observe behavioral differences between the bindata and +# bindata+gogit tag sets. + +tags=${1:?usage: $0 TAGS} + +go list -tags "$tags" -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}|{{range .Imports}}{{.}};{{end}}{{range .TestImports}}{{.}};{{end}}{{range .XTestImports}}{{.}};{{end}}{{end}}' ./... \ + | awk -F'|' '$2 ~ /code\.gitea\.io\/gitea\/modules\/(git|gitrepo|lfs)([\.\/;]|$)/ { print $1 }' \ + | sort -u From 9b6cd194938f93c047a5cbeb5898e92772d302a3 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 01:49:25 +0200 Subject: [PATCH 05/10] ci: 3-shard db tests, split test-unit into bindata/gogit jobs - pgsql/sqlite/mysql/mssql: increase shards from 2 to 3, dropping per-job wall time from ~13 min to ~9-10 min on the slowest (pgsql). - test-unit: split into test-unit-bindata and test-unit-gogit running in parallel, with a test-unit aggregator preserving the existing check name. Each gets its own rotated cache (cache-name unit and unit-gogit) so testcache accumulates per variant. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/pull-db-tests.yml | 82 +++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index f21cdf8135..9c78245ef6 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] services: pgsql: image: postgres:14 @@ -72,7 +72,7 @@ jobs: TAGS: bindata gogit TEST_LDAP: 1 TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: 2 + TEST_TOTAL_SHARDS: 3 test-pgsql: needs: [files-changed, test-pgsql-shards] @@ -88,7 +88,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 @@ -118,7 +118,7 @@ jobs: TAGS: bindata gogit GOEXPERIMENT: TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: 2 + TEST_TOTAL_SHARDS: 3 test-sqlite: needs: [files-changed, test-sqlite-shards] @@ -127,7 +127,7 @@ jobs: steps: - run: '[ "${{ needs.test-sqlite-shards.result }}" = "success" ]' - test-unit: + test-unit-bindata: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest @@ -188,6 +188,63 @@ jobs: GOTEST_FLAGS: -race -timeout=20m TAGS: bindata GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} + + test-unit-gogit: + if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' + needs: files-changed + runs-on: ubuntu-latest + services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + env: + discovery.type: single-node + xpack.security.enabled: false + ports: + - "9200:9200" + meilisearch: + image: getmeili/meilisearch:v1 + env: + MEILI_ENV: development # disable auth + ports: + - "7700:7700" + redis: + image: redis + options: >- # wait until redis has started + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + ports: + - 6379:6379 + minio: + image: bitnamilegacy/minio:2021.12.29 + env: + MINIO_ACCESS_KEY: 123456 + MINIO_SECRET_KEY: 12345678 + ports: + - "9000:9000" + devstoreaccount1.azurite.local: # https://github.com/Azure/Azurite/issues/1583 + image: mcr.microsoft.com/azure-storage/azurite:latest + ports: + - 10000:10000 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: go.mod + check-latest: true + cache: false + - uses: ./.github/actions/go-cache + with: + cache-name: unit-gogit + build-cache-rotate: "true" + - name: Add hosts to /etc/hosts + run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 minio devstoreaccount1.azurite.local mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts' + - run: make deps-backend + - run: make backend + env: + TAGS: bindata gogit + GOEXPERIMENT: - name: unit-tests-gogit run: make test-backend-gogit test-check env: @@ -196,6 +253,13 @@ jobs: GOEXPERIMENT: GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} + test-unit: + needs: [files-changed, test-unit-bindata, test-unit-gogit] + if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') + runs-on: ubuntu-latest + steps: + - run: '[ "${{ needs.test-unit-bindata.result }}" = "success" ] && [ "${{ needs.test-unit-gogit.result }}" = "success" ]' + test-mysql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed @@ -203,7 +267,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] services: mysql: # the bitnami mysql image has more options than the official one, it's easier to customize @@ -254,7 +318,7 @@ jobs: TAGS: bindata TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: 2 + TEST_TOTAL_SHARDS: 3 test-mysql: needs: [files-changed, test-mysql-shards] @@ -270,7 +334,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] services: mssql: image: mcr.microsoft.com/mssql/server:2019-latest @@ -308,7 +372,7 @@ jobs: env: TAGS: bindata TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: 2 + TEST_TOTAL_SHARDS: 3 test-mssql: needs: [files-changed, test-mssql-shards] From b266c64d241db89b77c3e09a8dbecd734f57b218 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 02:12:39 +0200 Subject: [PATCH 06/10] ci: fix LFS test pollution and exclude integration packages from gogit pass Two CI failures from sharding/splitting: 1. TestAPILFSNotStarted and TestAPILFSLocksNotStarted set setting.LFS.StartServer = false but never restore it. In the pre-sharding sequential run a later test happened to set it back to true, but with sharding TestLFSRender and TestChangeRepoFilesForUpdateWithFileRename can land in a different shard and run with LFS disabled. Use test.MockVariableValue so the change is scoped to the test that makes it. 2. find-gogit-test-pkgs.sh enumerated every package whose tests import the gogit modules, including tests/integration, tests/integration/migration-test, and models/migrations/... These need a real DB / dedicated harness and have to be tested separately; mirror GO_TEST_PACKAGES' filter to exclude them. Co-Authored-By: Claude (Opus 4.7) --- tests/integration/api_repo_lfs_locks_test.go | 7 ++++--- tests/integration/api_repo_lfs_test.go | 15 +++++---------- tools/find-gogit-test-pkgs.sh | 3 +++ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/integration/api_repo_lfs_locks_test.go b/tests/integration/api_repo_lfs_locks_test.go index 460b32a7dd..0b48bd00cc 100644 --- a/tests/integration/api_repo_lfs_locks_test.go +++ b/tests/integration/api_repo_lfs_locks_test.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" @@ -22,7 +23,7 @@ import ( func TestAPILFSLocksNotStarted(t *testing.T) { defer tests.PrepareTestEnv(t)() - setting.LFS.StartServer = false + defer test.MockVariableValue(&setting.LFS.StartServer, false)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) @@ -38,7 +39,7 @@ func TestAPILFSLocksNotStarted(t *testing.T) { func TestAPILFSLocksNotLogin(t *testing.T) { defer tests.PrepareTestEnv(t)() - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) @@ -51,7 +52,7 @@ func TestAPILFSLocksNotLogin(t *testing.T) { func TestAPILFSLocksLogged(t *testing.T) { defer tests.PrepareTestEnv(t)() - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) // in org 3 user4 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 4}) // in org 3 diff --git a/tests/integration/api_repo_lfs_test.go b/tests/integration/api_repo_lfs_test.go index 47bf244ee6..0f13bb23a8 100644 --- a/tests/integration/api_repo_lfs_test.go +++ b/tests/integration/api_repo_lfs_test.go @@ -28,8 +28,7 @@ import ( func TestAPILFSNotStarted(t *testing.T) { defer tests.PrepareTestEnv(t)() - - setting.LFS.StartServer = false + defer test.MockVariableValue(&setting.LFS.StartServer, false)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) @@ -48,8 +47,7 @@ func TestAPILFSNotStarted(t *testing.T) { func TestAPILFSMediaType(t *testing.T) { defer tests.PrepareTestEnv(t)() - - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) @@ -72,8 +70,7 @@ func createLFSTestRepository(t *testing.T, repoName string) *repo_model.Reposito func TestAPILFSBatch(t *testing.T) { defer tests.PrepareTestEnv(t)() - - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() repo := createLFSTestRepository(t, "lfs-batch-repo") @@ -326,8 +323,7 @@ func TestAPILFSBatch(t *testing.T) { func TestAPILFSUpload(t *testing.T) { defer tests.PrepareTestEnv(t)() - - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() repo := createLFSTestRepository(t, "lfs-upload-repo") oid := storeObjectInRepo(t, repo.ID, "dummy3") @@ -428,8 +424,7 @@ func TestAPILFSUpload(t *testing.T) { func TestAPILFSVerify(t *testing.T) { defer tests.PrepareTestEnv(t)() - - setting.LFS.StartServer = true + defer test.MockVariableValue(&setting.LFS.StartServer, true)() repo := createLFSTestRepository(t, "lfs-verify-repo") oid := storeObjectInRepo(t, repo.ID, "dummy3") diff --git a/tools/find-gogit-test-pkgs.sh b/tools/find-gogit-test-pkgs.sh index 59525644ad..9011f07e66 100755 --- a/tools/find-gogit-test-pkgs.sh +++ b/tools/find-gogit-test-pkgs.sh @@ -8,6 +8,9 @@ set -euo pipefail tags=${1:?usage: $0 TAGS} +# Exclusions mirror the Makefile's GO_TEST_PACKAGES filter — these packages +# need a real database / dedicated harness and are tested separately. go list -tags "$tags" -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}|{{range .Imports}}{{.}};{{end}}{{range .TestImports}}{{.}};{{end}}{{range .XTestImports}}{{.}};{{end}}{{end}}' ./... \ | awk -F'|' '$2 ~ /code\.gitea\.io\/gitea\/modules\/(git|gitrepo|lfs)([\.\/;]|$)/ { print $1 }' \ + | grep -vE '^code\.gitea\.io/gitea/(models/migrations(/|$)|tests(/integration(/migration-test)?)?$)' \ | sort -u From 28e72633e0afcfe2b30b8ae4cd3dda32af2f87c1 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 05:45:05 +0200 Subject: [PATCH 07/10] ci: rename test-unit-bindata, simplify and address review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename: - test-unit-bindata -> test-unit-nogogit. Both jobs run with the bindata tag; the meaningful difference is the gogit codepath, and the codebase already uses the nogogit/gogit suffix on its build-tagged source files. test-unit-gogit cleanup: - The gogit-affected package set (modules/git/gitrepo/lfs and direct importers) doesn't touch elasticsearch/meilisearch/redis/minio/ azurite, so drop those services and the matching /etc/hosts step. Shard runner robustness (Copilot review feedback): - Validate TEST_SHARD/TEST_TOTAL_SHARDS are positive ints with shard in [1, total]; exit 2 on bad input. - Tighten the test-name grep to require `*testing.T` or `*testing.TB` arg, dropping the TestMain false-positive. - Force LC_ALL=C sort so the partition is deterministic regardless of the runner's locale. - Empty assignment now exits 1 instead of silently passing. find-gogit-test-pkgs.sh: - Mirror the Makefile's GO_TEST_PACKAGES exclusions (drop models/migrations/..., tests/integration, tests/integration/migration-test). - Tighten the comment header — the script returns packages WITH TESTS, not all callers. - Drop dead alternatives from the import-match regex. Makefile test-backend-gogit: - Fail when the script fails or returns no packages instead of silently running `go test` with no args. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/pull-db-tests.yml | 45 ++++------------------------- Makefile | 5 ++-- tools/find-gogit-test-pkgs.sh | 14 +++++---- tools/test-integration-shard.sh | 20 +++++++++---- 4 files changed, 33 insertions(+), 51 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 9c78245ef6..c6b1d5195d 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -127,7 +127,7 @@ jobs: steps: - run: '[ "${{ needs.test-sqlite-shards.result }}" = "success" ]' - test-unit-bindata: + test-unit-nogogit: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest @@ -193,40 +193,9 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest - services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 - env: - discovery.type: single-node - xpack.security.enabled: false - ports: - - "9200:9200" - meilisearch: - image: getmeili/meilisearch:v1 - env: - MEILI_ENV: development # disable auth - ports: - - "7700:7700" - redis: - image: redis - options: >- # wait until redis has started - --health-cmd "redis-cli ping" - --health-interval 5s - --health-timeout 3s - --health-retries 10 - ports: - - 6379:6379 - minio: - image: bitnamilegacy/minio:2021.12.29 - env: - MINIO_ACCESS_KEY: 123456 - MINIO_SECRET_KEY: 12345678 - ports: - - "9000:9000" - devstoreaccount1.azurite.local: # https://github.com/Azure/Azurite/issues/1583 - image: mcr.microsoft.com/azure-storage/azurite:latest - ports: - - 10000:10000 + # the gogit-affected package set (modules/git, modules/gitrepo, modules/lfs and + # their direct importers) doesn't touch elasticsearch/meilisearch/redis/minio/ + # azurite — no services needed. steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 @@ -238,8 +207,6 @@ jobs: with: cache-name: unit-gogit build-cache-rotate: "true" - - name: Add hosts to /etc/hosts - run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 minio devstoreaccount1.azurite.local mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend - run: make backend env: @@ -254,11 +221,11 @@ jobs: GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} test-unit: - needs: [files-changed, test-unit-bindata, test-unit-gogit] + needs: [files-changed, test-unit-nogogit, test-unit-gogit] if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') runs-on: ubuntu-latest steps: - - run: '[ "${{ needs.test-unit-bindata.result }}" = "success" ] && [ "${{ needs.test-unit-gogit.result }}" = "success" ]' + - run: '[ "${{ needs.test-unit-nogogit.result }}" = "success" ] && [ "${{ needs.test-unit-gogit.result }}" = "success" ]' test-mysql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' diff --git a/Makefile b/Makefile index 717817c390..4179d91cdb 100644 --- a/Makefile +++ b/Makefile @@ -383,8 +383,9 @@ test-backend: ## test backend files .PHONY: test-backend-gogit test-backend-gogit: ## test packages whose code or tests import the gogit-affected modules - @pkgs=$$(./tools/find-gogit-test-pkgs.sh '$(TAGS)'); \ - echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over $$(echo $$pkgs | wc -w) gogit-affected packages..."; \ + @pkgs=$$(./tools/find-gogit-test-pkgs.sh '$(TAGS)') && \ + if [ -z "$$pkgs" ]; then echo "no gogit-affected packages found" >&2; exit 1; fi && \ + echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over $$(echo $$pkgs | wc -w) gogit-affected packages..." && \ $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs .PHONY: test-frontend diff --git a/tools/find-gogit-test-pkgs.sh b/tools/find-gogit-test-pkgs.sh index 9011f07e66..83809a6a05 100755 --- a/tools/find-gogit-test-pkgs.sh +++ b/tools/find-gogit-test-pkgs.sh @@ -1,16 +1,20 @@ #!/bin/bash set -euo pipefail -# Print packages whose own code or test code imports any of the gogit-affected -# modules (modules/git, modules/gitrepo, modules/lfs). These are the packages -# whose tests can observe behavioral differences between the bindata and -# bindata+gogit tag sets. +# Print packages with tests whose own code or test code imports any of the +# gogit-affected modules (modules/git, modules/gitrepo, modules/lfs). These +# are the packages whose tests can observe behavioral differences between +# the bindata and bindata+gogit tag sets. +# +# Packages without tests are intentionally skipped — they're compiled +# transitively by their consumers, so any tag-related compile error would +# already surface in those consumers' test builds. tags=${1:?usage: $0 TAGS} # Exclusions mirror the Makefile's GO_TEST_PACKAGES filter — these packages # need a real database / dedicated harness and are tested separately. go list -tags "$tags" -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}|{{range .Imports}}{{.}};{{end}}{{range .TestImports}}{{.}};{{end}}{{range .XTestImports}}{{.}};{{end}}{{end}}' ./... \ - | awk -F'|' '$2 ~ /code\.gitea\.io\/gitea\/modules\/(git|gitrepo|lfs)([\.\/;]|$)/ { print $1 }' \ + | awk -F'|' '$2 ~ /code\.gitea\.io\/gitea\/modules\/(git|gitrepo|lfs)[\/;]/ { print $1 }' \ | grep -vE '^code\.gitea\.io/gitea/(models/migrations(/|$)|tests(/integration(/migration-test)?)?$)' \ | sort -u diff --git a/tools/test-integration-shard.sh b/tools/test-integration-shard.sh index 1f72dd5094..14828edbf4 100755 --- a/tools/test-integration-shard.sh +++ b/tools/test-integration-shard.sh @@ -10,14 +10,24 @@ binary=$1 shard=${TEST_SHARD:?missing TEST_SHARD} total=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS} -names=$(grep -hE '^func Test[A-Z][A-Za-z0-9_]*\(' tests/integration/*.go \ +if ! [[ "$total" =~ ^[1-9][0-9]*$ ]]; then + echo "TEST_TOTAL_SHARDS must be a positive integer, got: $total" >&2 + exit 2 +fi +if ! [[ "$shard" =~ ^[1-9][0-9]*$ ]] || [ "$shard" -gt "$total" ]; then + echo "TEST_SHARD must be in [1, $total], got: $shard" >&2 + exit 2 +fi + +# match `func Test*(t *testing.T|TB)` only — excludes TestMain (takes *testing.M) +names=$(grep -hE '^func Test[A-Z][A-Za-z0-9_]*\([a-zA-Z_][a-zA-Z0-9_]* \*testing\.(T|TB)\)' tests/integration/*.go \ | sed -E 's/^func (Test[A-Z][A-Za-z0-9_]*).*/\1/' \ - | sort -u \ - | awk -v s="$shard" -v t="$total" 'NR % t == (s - 1) % t') + | LC_ALL=C sort -u \ + | awk -v r=$((shard - 1)) -v t="$total" 'NR % t == r') if [ -z "$names" ]; then - echo "shard $shard/$total has no tests assigned" >&2 - exit 0 + echo "no tests assigned to shard $shard/$total — likely a misconfiguration" >&2 + exit 1 fi pattern=$(echo "$names" | paste -sd '|' -) From 8337e1c5f3a4d95d8fa47d43d85efbc20eafc51e Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:05:51 +0200 Subject: [PATCH 08/10] ci: combine unit shards (3-way), drop rotation, use make generate-go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces test-unit-nogogit + test-unit-gogit with a single test-unit-shards matrix (3-shard). Each shard runs both the bindata and bindata-gogit test subsets — round-robin partition of GO_TEST_PACKAGES (123 each) and find-gogit-test-pkgs.sh (22-23 each). Combined work split 3 ways gives each shard ~7:46 wall (vs 10:55/7:38 today). PRs no longer write rotated unit caches: build-cache-rotate is dropped, so the shared seeded gobuild key is restored but not re-saved per PR push. Trade-off: cold testcache on every push (vs warm-on-rerun before). Frees ~3 GB of rotated-cache pressure on the 10 GB cap. Unit shards swap `make backend` for `make generate-go` — only the bindata codegen is needed; the gitea executable's link step (~10-15s) is wasted on unit tests since they don't shell out to the binary (db integration tests do — those keep `make backend`). New shared tools/partition-by-shard.sh handles validation + round-robin partitioning; tools/test-integration-shard.sh now uses it. New Makefile targets: test-backend-shard, test-backend-gogit-shard. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/pull-db-tests.yml | 45 +++++++++-------------------- Makefile | 14 +++++++++ tools/partition-by-shard.sh | 20 +++++++++++++ tools/test-integration-shard.sh | 18 ++---------- 4 files changed, 51 insertions(+), 46 deletions(-) create mode 100755 tools/partition-by-shard.sh diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index c6b1d5195d..b382be55ca 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -127,10 +127,14 @@ jobs: steps: - run: '[ "${{ needs.test-sqlite-shards.result }}" = "success" ]' - test-unit-nogogit: + test-unit-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 @@ -175,57 +179,36 @@ jobs: - uses: ./.github/actions/go-cache with: cache-name: unit - build-cache-rotate: "true" - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 minio devstoreaccount1.azurite.local mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend - - run: make backend + - run: make generate-go env: TAGS: bindata - name: unit-tests - run: make test-backend test-check + run: make test-backend-shard test-check env: + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: 3 GOTEST_FLAGS: -race -timeout=20m TAGS: bindata GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - - test-unit-gogit: - if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' - needs: files-changed - runs-on: ubuntu-latest - # the gogit-affected package set (modules/git, modules/gitrepo, modules/lfs and - # their direct importers) doesn't touch elasticsearch/meilisearch/redis/minio/ - # azurite — no services needed. - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache - with: - cache-name: unit-gogit - build-cache-rotate: "true" - - run: make deps-backend - - run: make backend - env: - TAGS: bindata gogit - GOEXPERIMENT: - name: unit-tests-gogit - run: make test-backend-gogit test-check + run: make test-backend-gogit-shard test-check env: + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: 3 GOTEST_FLAGS: -race -timeout=20m TAGS: bindata gogit GOEXPERIMENT: GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} test-unit: - needs: [files-changed, test-unit-nogogit, test-unit-gogit] + needs: [files-changed, test-unit-shards] if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true') runs-on: ubuntu-latest steps: - - run: '[ "${{ needs.test-unit-nogogit.result }}" = "success" ] && [ "${{ needs.test-unit-gogit.result }}" = "success" ]' + - run: '[ "${{ needs.test-unit-shards.result }}" = "success" ]' test-mysql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' diff --git a/Makefile b/Makefile index 4179d91cdb..3c6b6905f5 100644 --- a/Makefile +++ b/Makefile @@ -388,6 +388,20 @@ test-backend-gogit: ## test packages whose code or tests import the gogit-affect echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over $$(echo $$pkgs | wc -w) gogit-affected packages..." && \ $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs +.PHONY: test-backend-shard +test-backend-shard: ## run the TEST_SHARD/TEST_TOTAL_SHARDS slice of test-backend + @pkgs=$$(echo "$(GO_TEST_PACKAGES)" | tr ' ' '\n' | ./tools/partition-by-shard.sh | tr '\n' ' ') && \ + if [ -z "$$pkgs" ]; then echo "shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS has no test-backend packages" >&2; exit 1; fi && \ + echo "Running shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS of test-backend ($$(echo $$pkgs | wc -w) packages)..." && \ + $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs + +.PHONY: test-backend-gogit-shard +test-backend-gogit-shard: ## run the TEST_SHARD/TEST_TOTAL_SHARDS slice of test-backend-gogit + @pkgs=$$(./tools/find-gogit-test-pkgs.sh '$(TAGS)' | ./tools/partition-by-shard.sh) && \ + if [ -z "$$pkgs" ]; then echo "shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS has no gogit-affected packages" >&2; exit 1; fi && \ + echo "Running shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS of test-backend-gogit ($$(echo $$pkgs | wc -w) packages)..." && \ + $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs + .PHONY: test-frontend test-frontend: node_modules ## test frontend files pnpm exec vitest diff --git a/tools/partition-by-shard.sh b/tools/partition-by-shard.sh new file mode 100755 index 0000000000..7e55368f27 --- /dev/null +++ b/tools/partition-by-shard.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +# Print the TEST_SHARD/TEST_TOTAL_SHARDS slice of stdin (newline-separated +# items), partitioned round-robin by line number after a deterministic sort. +# Required env: TEST_SHARD (1..TEST_TOTAL_SHARDS), TEST_TOTAL_SHARDS (>= 1). + +shard=${TEST_SHARD:?missing TEST_SHARD} +total=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS} + +if ! [[ "$total" =~ ^[1-9][0-9]*$ ]]; then + echo "TEST_TOTAL_SHARDS must be a positive integer, got: $total" >&2 + exit 2 +fi +if ! [[ "$shard" =~ ^[1-9][0-9]*$ ]] || [ "$shard" -gt "$total" ]; then + echo "TEST_SHARD must be in [1, $total], got: $shard" >&2 + exit 2 +fi + +LC_ALL=C sort -u | awk -v r=$((shard - 1)) -v t="$total" 'NR % t == r' diff --git a/tools/test-integration-shard.sh b/tools/test-integration-shard.sh index 14828edbf4..f51543bb5f 100755 --- a/tools/test-integration-shard.sh +++ b/tools/test-integration-shard.sh @@ -7,29 +7,17 @@ set -euo pipefail # a configured database. binary=$1 -shard=${TEST_SHARD:?missing TEST_SHARD} -total=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS} - -if ! [[ "$total" =~ ^[1-9][0-9]*$ ]]; then - echo "TEST_TOTAL_SHARDS must be a positive integer, got: $total" >&2 - exit 2 -fi -if ! [[ "$shard" =~ ^[1-9][0-9]*$ ]] || [ "$shard" -gt "$total" ]; then - echo "TEST_SHARD must be in [1, $total], got: $shard" >&2 - exit 2 -fi # match `func Test*(t *testing.T|TB)` only — excludes TestMain (takes *testing.M) names=$(grep -hE '^func Test[A-Z][A-Za-z0-9_]*\([a-zA-Z_][a-zA-Z0-9_]* \*testing\.(T|TB)\)' tests/integration/*.go \ | sed -E 's/^func (Test[A-Z][A-Za-z0-9_]*).*/\1/' \ - | LC_ALL=C sort -u \ - | awk -v r=$((shard - 1)) -v t="$total" 'NR % t == r') + | ./tools/partition-by-shard.sh) if [ -z "$names" ]; then - echo "no tests assigned to shard $shard/$total — likely a misconfiguration" >&2 + echo "no tests assigned to shard $TEST_SHARD/$TEST_TOTAL_SHARDS — likely a misconfiguration" >&2 exit 1 fi pattern=$(echo "$names" | paste -sd '|' -) -echo "Running shard $shard/$total ($(echo "$names" | wc -l | tr -d ' ') tests)" +echo "Running shard $TEST_SHARD/$TEST_TOTAL_SHARDS ($(echo "$names" | wc -l | tr -d ' ') tests)" exec "$binary" -test.run "^($pattern)\$" From 5c2bcdb64c5d7533361af6bd384814bb9545c178 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:24:56 +0200 Subject: [PATCH 09/10] ci: bump test-unit-shards from 3 to 4 The 3-shard combined unit run came in at 11:42 wall (slowest shard), slightly worse than the prior parallel-jobs setup (10:55). The per-shard race-instrumented compile happens cold for each job and doesn't shard, so per-shard test work was ~67% higher than a clean 3-way split would suggest. N=4 amortizes that overhead across more runners. Co-Authored-By: Claude (Opus 4.7) --- .github/workflows/pull-db-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index b382be55ca..bca502ffe8 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4] services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 @@ -189,7 +189,7 @@ jobs: run: make test-backend-shard test-check env: TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: 3 + TEST_TOTAL_SHARDS: 4 GOTEST_FLAGS: -race -timeout=20m TAGS: bindata GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} @@ -197,7 +197,7 @@ jobs: run: make test-backend-gogit-shard test-check env: TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: 3 + TEST_TOTAL_SHARDS: 4 GOTEST_FLAGS: -race -timeout=20m TAGS: bindata gogit GOEXPERIMENT: From 4cab370652b5dc556c35689cd457224c28067a3e Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 9 May 2026 06:52:20 +0200 Subject: [PATCH 10/10] ci: address review feedback on shard scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - partition-by-shard.sh: fix off-by-one. `NR % t == r` placed item 1 in shard `total`, item 2 in shard 1 — shifted by one. Switch to `(NR-1) % t == r` so shard 1 owns items 1, 1+t, 1+2t, ... as documented. - test-integration-shard.sh: the `[A-Z]` after `Test` rejected `Test_Foo`-style names (28 such tests in tests/integration). Drop the [A-Z] and broaden the comment to match the actual regex. - test-integration-shard.sh: drop `*testing.TB` alternative — there are zero matching tests in this codebase, so the simpler regex is exact. - find-gogit-test-pkgs.sh: merge the `grep -vE` exclusion into the awk filter. Avoids a pipefail edge case where empty grep output would abort the pipeline before the friendly empty-list message fires; also one fewer process per invocation. - Makefile test-integration: fail fast at Make level if TEST_SHARD is set without TEST_TOTAL_SHARDS, instead of letting the script's ${X:?...} surface a less obvious error. All locals in the new bash scripts use UPPER_CASE to match the existing tools/test-e2e.sh convention. Co-Authored-By: Claude (Opus 4.7) --- Makefile | 11 ++++------- tools/find-gogit-test-pkgs.sh | 18 +++++++++++++----- tools/partition-by-shard.sh | 19 ++++++++++++------- tools/test-integration-shard.sh | 19 +++++++------------ 4 files changed, 36 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 3c6b6905f5..bf5b29ca53 100644 --- a/Makefile +++ b/Makefile @@ -384,22 +384,16 @@ test-backend: ## test backend files .PHONY: test-backend-gogit test-backend-gogit: ## test packages whose code or tests import the gogit-affected modules @pkgs=$$(./tools/find-gogit-test-pkgs.sh '$(TAGS)') && \ - if [ -z "$$pkgs" ]; then echo "no gogit-affected packages found" >&2; exit 1; fi && \ - echo "Running go test with $(GOTEST_FLAGS) -tags '$(TAGS)' over $$(echo $$pkgs | wc -w) gogit-affected packages..." && \ $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs .PHONY: test-backend-shard test-backend-shard: ## run the TEST_SHARD/TEST_TOTAL_SHARDS slice of test-backend - @pkgs=$$(echo "$(GO_TEST_PACKAGES)" | tr ' ' '\n' | ./tools/partition-by-shard.sh | tr '\n' ' ') && \ - if [ -z "$$pkgs" ]; then echo "shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS has no test-backend packages" >&2; exit 1; fi && \ - echo "Running shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS of test-backend ($$(echo $$pkgs | wc -w) packages)..." && \ + @pkgs=$$(echo "$(GO_TEST_PACKAGES)" | tr ' ' '\n' | ./tools/partition-by-shard.sh) && \ $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs .PHONY: test-backend-gogit-shard test-backend-gogit-shard: ## run the TEST_SHARD/TEST_TOTAL_SHARDS slice of test-backend-gogit @pkgs=$$(./tools/find-gogit-test-pkgs.sh '$(TAGS)' | ./tools/partition-by-shard.sh) && \ - if [ -z "$$pkgs" ]; then echo "shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS has no gogit-affected packages" >&2; exit 1; fi && \ - echo "Running shard $$TEST_SHARD/$$TEST_TOTAL_SHARDS of test-backend-gogit ($$(echo $$pkgs | wc -w) packages)..." && \ $(GO) test $(GOTEST_FLAGS) -tags='$(TAGS)' $$pkgs .PHONY: test-frontend @@ -466,6 +460,9 @@ test-integration: @# they mutate the work directory, so cache inputs change between runs. $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -c code.gitea.io/gitea/tests/integration -o ./test-integration-$(GITEA_TEST_DATABASE).test ifdef TEST_SHARD +ifndef TEST_TOTAL_SHARDS + $(error TEST_TOTAL_SHARDS must be set when TEST_SHARD is set) +endif ./tools/test-integration-shard.sh ./test-integration-$(GITEA_TEST_DATABASE).test else ./test-integration-$(GITEA_TEST_DATABASE).test diff --git a/tools/find-gogit-test-pkgs.sh b/tools/find-gogit-test-pkgs.sh index 83809a6a05..1d60a55a06 100755 --- a/tools/find-gogit-test-pkgs.sh +++ b/tools/find-gogit-test-pkgs.sh @@ -10,11 +10,19 @@ set -euo pipefail # transitively by their consumers, so any tag-related compile error would # already surface in those consumers' test builds. -tags=${1:?usage: $0 TAGS} +TAGS=${1:?usage: $0 TAGS} # Exclusions mirror the Makefile's GO_TEST_PACKAGES filter — these packages # need a real database / dedicated harness and are tested separately. -go list -tags "$tags" -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}|{{range .Imports}}{{.}};{{end}}{{range .TestImports}}{{.}};{{end}}{{range .XTestImports}}{{.}};{{end}}{{end}}' ./... \ - | awk -F'|' '$2 ~ /code\.gitea\.io\/gitea\/modules\/(git|gitrepo|lfs)[\/;]/ { print $1 }' \ - | grep -vE '^code\.gitea\.io/gitea/(models/migrations(/|$)|tests(/integration(/migration-test)?)?$)' \ - | sort -u +OUT=$(go list -tags "$TAGS" -f '{{if or .TestGoFiles .XTestGoFiles}}{{.ImportPath}}|{{range .Imports}}{{.}};{{end}}{{range .TestImports}}{{.}};{{end}}{{range .XTestImports}}{{.}};{{end}}{{end}}' ./... \ + | awk -F'|' ' + $2 ~ /code\.gitea\.io\/gitea\/modules\/(git|gitrepo|lfs)[\/;]/ && + $1 !~ /^code\.gitea\.io\/gitea\/(models\/migrations(\/|$)|tests(\/integration(\/migration-test)?)?$)/ { + print $1 + }' \ + | sort -u) +if [ -z "$OUT" ]; then + echo "no gogit-affected packages found" >&2 + exit 1 +fi +echo "$OUT" diff --git a/tools/partition-by-shard.sh b/tools/partition-by-shard.sh index 7e55368f27..acb5139251 100755 --- a/tools/partition-by-shard.sh +++ b/tools/partition-by-shard.sh @@ -5,16 +5,21 @@ set -euo pipefail # items), partitioned round-robin by line number after a deterministic sort. # Required env: TEST_SHARD (1..TEST_TOTAL_SHARDS), TEST_TOTAL_SHARDS (>= 1). -shard=${TEST_SHARD:?missing TEST_SHARD} -total=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS} +SHARD=${TEST_SHARD:?missing TEST_SHARD} +TOTAL=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS} -if ! [[ "$total" =~ ^[1-9][0-9]*$ ]]; then - echo "TEST_TOTAL_SHARDS must be a positive integer, got: $total" >&2 +if ! [[ "$TOTAL" =~ ^[1-9][0-9]*$ ]]; then + echo "TEST_TOTAL_SHARDS must be a positive integer, got: $TOTAL" >&2 exit 2 fi -if ! [[ "$shard" =~ ^[1-9][0-9]*$ ]] || [ "$shard" -gt "$total" ]; then - echo "TEST_SHARD must be in [1, $total], got: $shard" >&2 +if ! [[ "$SHARD" =~ ^[1-9][0-9]*$ ]] || [ "$SHARD" -gt "$TOTAL" ]; then + echo "TEST_SHARD must be in [1, $TOTAL], got: $SHARD" >&2 exit 2 fi -LC_ALL=C sort -u | awk -v r=$((shard - 1)) -v t="$total" 'NR % t == r' +OUT=$(LC_ALL=C sort -u | awk -v r=$((SHARD - 1)) -v t="$TOTAL" '(NR - 1) % t == r') +if [ -z "$OUT" ]; then + echo "shard $SHARD/$TOTAL has no items assigned" >&2 + exit 1 +fi +echo "$OUT" diff --git a/tools/test-integration-shard.sh b/tools/test-integration-shard.sh index f51543bb5f..6f007cebd8 100755 --- a/tools/test-integration-shard.sh +++ b/tools/test-integration-shard.sh @@ -6,18 +6,13 @@ set -euo pipefail # because TestMain boots the full Gitea environment and would panic without # a configured database. -binary=$1 +BINARY=$1 -# match `func Test*(t *testing.T|TB)` only — excludes TestMain (takes *testing.M) -names=$(grep -hE '^func Test[A-Z][A-Za-z0-9_]*\([a-zA-Z_][a-zA-Z0-9_]* \*testing\.(T|TB)\)' tests/integration/*.go \ - | sed -E 's/^func (Test[A-Z][A-Za-z0-9_]*).*/\1/' \ +# match `func Test...(t *testing.T)` only — `*testing.M` excludes TestMain +NAMES=$(grep -hE '^func Test[A-Za-z0-9_]*\([a-zA-Z_][a-zA-Z0-9_]* \*testing\.T\)' tests/integration/*.go \ + | sed -E 's/^func (Test[A-Za-z0-9_]*).*/\1/' \ | ./tools/partition-by-shard.sh) -if [ -z "$names" ]; then - echo "no tests assigned to shard $TEST_SHARD/$TEST_TOTAL_SHARDS — likely a misconfiguration" >&2 - exit 1 -fi - -pattern=$(echo "$names" | paste -sd '|' -) -echo "Running shard $TEST_SHARD/$TEST_TOTAL_SHARDS ($(echo "$names" | wc -l | tr -d ' ') tests)" -exec "$binary" -test.run "^($pattern)\$" +PATTERN=$(echo "$NAMES" | paste -sd '|' -) +echo "Running shard $TEST_SHARD/$TEST_TOTAL_SHARDS ($(echo "$NAMES" | wc -l | tr -d ' ') tests)" +exec "$BINARY" -test.run "^($PATTERN)\$"