mirror of
https://github.com/go-gitea/gitea.git
synced 2026-05-10 09:41:52 +02:00
ci: shard integration tests across 2 runners per database
Splits each database integration job (pgsql/sqlite/mysql/mssql) into two parallel matrix shards. Test names are enumerated from the integration source and partitioned round-robin (~301/302 of 603 tests per shard); names that don't match the shard are filtered out via -test.run. Migration tests (~50-90 s, fast, sequential) only run on shard 1. The original job names (test-pgsql, test-sqlite, test-mysql, test-mssql) are kept as one-step aggregator jobs that depend on the shards job and report success only when all shards passed. This keeps any branch-protection rule referencing those names valid. Source-based enumeration is used because the test binary's -test.list calls TestMain, which boots the full Gitea environment and panics without a configured database. Co-Authored-By: Claude (Opus 4.7) <noreply@anthropic.com>
This commit is contained in:
silverwind
parent
ad9b633757
commit
8d6615f939
66
.github/workflows/pull-db-tests.yml
vendored
66
.github/workflows/pull-db-tests.yml
vendored
@ -14,10 +14,14 @@ jobs:
|
||||
files-changed:
|
||||
uses: ./.github/workflows/files-changed.yml
|
||||
|
||||
test-pgsql:
|
||||
test-pgsql-shards:
|
||||
if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true'
|
||||
needs: files-changed
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
services:
|
||||
pgsql:
|
||||
image: postgres:14
|
||||
@ -57,6 +61,7 @@ jobs:
|
||||
env:
|
||||
TAGS: bindata
|
||||
- name: run migration tests
|
||||
if: matrix.shard == 1
|
||||
run: GITEA_TEST_DATABASE=pgsql make test-migration
|
||||
- name: run tests
|
||||
run: GITEA_TEST_DATABASE=pgsql make test-integration
|
||||
@ -66,11 +71,24 @@ jobs:
|
||||
GOTEST_FLAGS: -race -timeout=40m
|
||||
TAGS: bindata gogit
|
||||
TEST_LDAP: 1
|
||||
TEST_SHARD: ${{ matrix.shard }}
|
||||
TEST_TOTAL_SHARDS: 2
|
||||
|
||||
test-sqlite:
|
||||
test-pgsql:
|
||||
needs: [files-changed, test-pgsql-shards]
|
||||
if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: '[ "${{ needs.test-pgsql-shards.result }}" = "success" ]'
|
||||
|
||||
test-sqlite-shards:
|
||||
if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true'
|
||||
needs: files-changed
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
@ -87,6 +105,7 @@ jobs:
|
||||
TAGS: bindata gogit
|
||||
GOEXPERIMENT:
|
||||
- name: run migration tests
|
||||
if: matrix.shard == 1
|
||||
run: GITEA_TEST_DATABASE=sqlite make test-migration
|
||||
env:
|
||||
TAGS: bindata gogit
|
||||
@ -98,6 +117,15 @@ jobs:
|
||||
GOTEST_FLAGS: -timeout=40m
|
||||
TAGS: bindata gogit
|
||||
GOEXPERIMENT:
|
||||
TEST_SHARD: ${{ matrix.shard }}
|
||||
TEST_TOTAL_SHARDS: 2
|
||||
|
||||
test-sqlite:
|
||||
needs: [files-changed, test-sqlite-shards]
|
||||
if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: '[ "${{ needs.test-sqlite-shards.result }}" = "success" ]'
|
||||
|
||||
test-unit:
|
||||
if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true'
|
||||
@ -168,10 +196,14 @@ jobs:
|
||||
GOEXPERIMENT:
|
||||
GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }}
|
||||
|
||||
test-mysql:
|
||||
test-mysql-shards:
|
||||
if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true'
|
||||
needs: files-changed
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
services:
|
||||
mysql:
|
||||
# the bitnami mysql image has more options than the official one, it's easier to customize
|
||||
@ -214,17 +246,31 @@ jobs:
|
||||
env:
|
||||
TAGS: bindata
|
||||
- name: run migration tests
|
||||
if: matrix.shard == 1
|
||||
run: GITEA_TEST_DATABASE=mysql make test-migration
|
||||
- name: run tests
|
||||
run: GITEA_TEST_DATABASE=mysql make test-integration
|
||||
env:
|
||||
TAGS: bindata
|
||||
TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200"
|
||||
TEST_SHARD: ${{ matrix.shard }}
|
||||
TEST_TOTAL_SHARDS: 2
|
||||
|
||||
test-mssql:
|
||||
test-mysql:
|
||||
needs: [files-changed, test-mysql-shards]
|
||||
if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: '[ "${{ needs.test-mysql-shards.result }}" = "success" ]'
|
||||
|
||||
test-mssql-shards:
|
||||
if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true'
|
||||
needs: files-changed
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
services:
|
||||
mssql:
|
||||
image: mcr.microsoft.com/mssql/server:2019-latest
|
||||
@ -254,9 +300,19 @@ jobs:
|
||||
- run: make backend
|
||||
env:
|
||||
TAGS: bindata
|
||||
- run: GITEA_TEST_DATABASE=mssql make test-migration
|
||||
- if: matrix.shard == 1
|
||||
run: GITEA_TEST_DATABASE=mssql make test-migration
|
||||
- name: run tests
|
||||
run: GITEA_TEST_DATABASE=mssql make test-integration
|
||||
timeout-minutes: 50
|
||||
env:
|
||||
TAGS: bindata
|
||||
TEST_SHARD: ${{ matrix.shard }}
|
||||
TEST_TOTAL_SHARDS: 2
|
||||
|
||||
test-mssql:
|
||||
needs: [files-changed, test-mssql-shards]
|
||||
if: always() && (needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: '[ "${{ needs.test-mssql-shards.result }}" = "success" ]'
|
||||
|
||||
4
Makefile
4
Makefile
@ -450,7 +450,11 @@ test-integration:
|
||||
@# would flood output per passing test. testcache can't help these tests anyway —
|
||||
@# they mutate the work directory, so cache inputs change between runs.
|
||||
$(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -c code.gitea.io/gitea/tests/integration -o ./test-integration-$(GITEA_TEST_DATABASE).test
|
||||
ifdef TEST_SHARD
|
||||
./tools/test-integration-shard.sh ./test-integration-$(GITEA_TEST_DATABASE).test
|
||||
else
|
||||
./test-integration-$(GITEA_TEST_DATABASE).test
|
||||
endif
|
||||
|
||||
.PHONY: test-integration\#%
|
||||
test-integration\#%:
|
||||
|
||||
25
tools/test-integration-shard.sh
Executable file
25
tools/test-integration-shard.sh
Executable file
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Run a deterministic shard of the integration test binary. Test names are
|
||||
# enumerated from source — running the binary with -test.list isn't viable
|
||||
# because TestMain boots the full Gitea environment and would panic without
|
||||
# a configured database.
|
||||
|
||||
binary=$1
|
||||
shard=${TEST_SHARD:?missing TEST_SHARD}
|
||||
total=${TEST_TOTAL_SHARDS:?missing TEST_TOTAL_SHARDS}
|
||||
|
||||
names=$(grep -hE '^func Test[A-Z][A-Za-z0-9_]*\(' tests/integration/*.go \
|
||||
| sed -E 's/^func (Test[A-Z][A-Za-z0-9_]*).*/\1/' \
|
||||
| sort -u \
|
||||
| awk -v s="$shard" -v t="$total" 'NR % t == (s - 1) % t')
|
||||
|
||||
if [ -z "$names" ]; then
|
||||
echo "shard $shard/$total has no tests assigned" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
pattern=$(echo "$names" | paste -sd '|' -)
|
||||
echo "Running shard $shard/$total ($(echo "$names" | wc -l | tr -d ' ') tests)"
|
||||
exec "$binary" -test.run "^($pattern)\$"
|
||||
Loading…
x
Reference in New Issue
Block a user