diff --git a/models/git/commit_status.go b/models/git/commit_status.go index 1b7ba58d00b..256096c828b 100644 --- a/models/git/commit_status.go +++ b/models/git/commit_status.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/url" + "slices" "strconv" "strings" "time" @@ -35,11 +36,11 @@ import ( // CommitStatus holds a single Status of a single Commit type CommitStatus struct { ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"UNIQUE(repo_sha_index)"` + SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"` Index int64 `xorm:"INDEX UNIQUE(repo_sha_index)"` - RepoID int64 `xorm:"INDEX UNIQUE(repo_sha_index)"` Repo *repo_model.Repository `xorm:"-"` State commitstatus.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"` - SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"` // TargetURL points to the commit status page reported by a CI system // If Gitea Actions is used, it is a relative link like "{RepoLink}/actions/runs/{RunID}/jobs{JobID}" @@ -420,46 +421,48 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map return repoStatuses, nil } -// GetLatestCommitStatusForRepoCommitIDs returns all statuses with a unique context for a given list of repo-sha pairs -func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, commitIDs []string) (map[string][]*CommitStatus, error) { - type result struct { +// GetLatestCommitStatusForRepoCommitIDs returns the commit statuses with a unique context for a given list of repo-sha pairs +// If the provided commit IDs are too many, only the first part and the last part of the commit IDs will be queried. +func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, allCommitIDs []string) (map[string][]*CommitStatus, error) { + const maxCommitIDs = 500 + const maxBatchSize = 200 + queryCommitIDs := allCommitIDs + if len(allCommitIDs) > maxCommitIDs { + // The commit IDs are usually from "commits list" or "compare" page (create a PR or compare commits), nobody can read so many commits at once. + // The commit IDs are usually sorted by time, so we can take the first half and the last half of the commit IDs to get the latest statuses. + log.Warn("GetLatestCommitStatusForRepoCommitIDs: too many commit IDs (%d) for repo %d, truncating to %d", len(allCommitIDs), repoID, maxCommitIDs) + queryCommitIDs = allCommitIDs[:maxCommitIDs/2] + queryCommitIDs = append(queryCommitIDs, allCommitIDs[len(allCommitIDs)-maxCommitIDs/2:]...) + } + + baseSql := func() db.Session { + return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID) + } + + type shaMaxIndexResult struct { Index int64 SHA string } - - getBase := func() db.Session { - return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID) - } - results := make([]result, 0, len(commitIDs)) - - conds := make([]builder.Cond, 0, len(commitIDs)) - for _, sha := range commitIDs { - conds = append(conds, builder.Eq{"sha": sha}) - } - sess := getBase().And(builder.Or(conds...)). - Select("max( `index` ) as `index`, sha"). - GroupBy("context_hash, sha").OrderBy("max( `index` ) desc") - - err := sess.Find(&results) + shaMaxIndexResults := make([]*shaMaxIndexResult, 0, len(allCommitIDs)) + err := baseSql().And(builder.In("sha", queryCommitIDs)). + Select("max(`index`) as `index`, sha"). + GroupBy("context_hash, sha"). + Find(&shaMaxIndexResults) if err != nil { return nil, err } repoStatuses := make(map[string][]*CommitStatus) - - if len(results) > 0 { - statuses := make([]*CommitStatus, 0, len(results)) - - conds = make([]builder.Cond, 0, len(results)) - for _, result := range results { - conds = append(conds, builder.Eq{"`index`": result.Index, "sha": result.SHA}) + for chunk := range slices.Chunk(shaMaxIndexResults, maxBatchSize) { + statuses := make([]*CommitStatus, 0, len(chunk)) + condIndexSha := make([]builder.Cond, 0, len(chunk)) + for _, res := range chunk { + condIndexSha = append(condIndexSha, builder.Eq{"`index`": res.Index, "sha": res.SHA}) } - err = getBase().And(builder.Or(conds...)).Find(&statuses) + err = baseSql().And(builder.Or(condIndexSha...)).Find(&statuses) if err != nil { return nil, err } - - // Group the statuses by commit for _, status := range statuses { repoStatuses[status.SHA] = append(repoStatuses[status.SHA], status) } diff --git a/models/git/commit_status_test.go b/models/git/commit_status_test.go index a85a2ff5de1..8b1c250cf28 100644 --- a/models/git/commit_status_test.go +++ b/models/git/commit_status_test.go @@ -4,6 +4,7 @@ package git_test import ( + "fmt" "testing" "time" @@ -22,33 +23,71 @@ func TestGetCommitStatuses(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - sha1 := "1234123412341234123412341234123412341234" // the mocked commit ID in test fixtures - statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{ - ListOptions: db.ListOptions{Page: 1, PageSize: 50}, - RepoID: repo1.ID, - SHA: sha1, - }) - assert.NoError(t, err) - assert.Equal(t, 5, int(maxResults)) - var indexes []int64 - for _, status := range statuses { - indexes = append(indexes, status.Index) - } - assert.Equal(t, []int64{5, 4, 3, 2, 1}, indexes) - assert.Equal(t, "deploy/awesomeness", statuses[0].Context) - assert.Equal(t, commitstatus.CommitStatusError, statuses[0].State) - assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/statuses/1234123412341234123412341234123412341234", statuses[0].APIURL(t.Context())) + t.Run("CommitStatusOptions", func(t *testing.T) { + statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{ + ListOptions: db.ListOptions{Page: 1, PageSize: 50}, + RepoID: repo1.ID, + SHA: sha1, + }) + assert.NoError(t, err) + assert.Equal(t, 5, int(maxResults)) + var indexes []int64 + for _, status := range statuses { + indexes = append(indexes, status.Index) + } + assert.Equal(t, []int64{5, 4, 3, 2, 1}, indexes) + assert.Equal(t, "deploy/awesomeness", statuses[0].Context) + assert.Equal(t, commitstatus.CommitStatusError, statuses[0].State) + assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/statuses/1234123412341234123412341234123412341234", statuses[0].APIURL(t.Context())) - statuses, maxResults, err = db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{ - ListOptions: db.ListOptions{Page: 2, PageSize: 50}, - RepoID: repo1.ID, - SHA: sha1, + statuses, maxResults, err = db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{ + ListOptions: db.ListOptions{Page: 2, PageSize: 50}, + RepoID: repo1.ID, + SHA: sha1, + }) + assert.NoError(t, err) + assert.Equal(t, 5, int(maxResults)) + assert.Empty(t, statuses) + }) + + t.Run("GetCountLatestCommitStatus", func(t *testing.T) { + commitStatuses, err := git_model.GetLatestCommitStatus(t.Context(), repo1.ID, sha1, db.ListOptions{ + Page: 1, + PageSize: 2, + }) + assert.NoError(t, err) + assert.Len(t, commitStatuses, 2) + assert.Equal(t, commitstatus.CommitStatusFailure, commitStatuses[0].State) + assert.Equal(t, "ci/awesomeness", commitStatuses[0].Context) + assert.Equal(t, commitstatus.CommitStatusError, commitStatuses[1].State) + assert.Equal(t, "deploy/awesomeness", commitStatuses[1].Context) + + count, err := git_model.CountLatestCommitStatus(t.Context(), repo1.ID, sha1) + assert.NoError(t, err) + assert.EqualValues(t, 3, count) + }) + + t.Run("GetLatestCommitStatusForRepoCommitIDs", func(t *testing.T) { + commitIDs := []string{sha1} + // SQLite has a limit of 1000 for WHERE expression variables per query, use a larger slice to test + for i := range 2000 { + commitIDs = append(commitIDs, fmt.Sprintf("%040x", i+1)) + } + statuses, err := git_model.GetLatestCommitStatusForRepoCommitIDs(t.Context(), repo1.ID, commitIDs) + assert.NoError(t, err) + assert.Len(t, statuses[sha1], 3) + latestIndexes := make(map[string]int64) + for _, status := range statuses[sha1] { + latestIndexes[status.Context] = status.Index + } + assert.Equal(t, map[string]int64{ + "ci/awesomeness": 4, + "cov/awesomeness": 3, + "deploy/awesomeness": 5, + }, latestIndexes) }) - assert.NoError(t, err) - assert.Equal(t, 5, int(maxResults)) - assert.Empty(t, statuses) } func Test_CalcCommitStatus(t *testing.T) { @@ -247,26 +286,3 @@ func TestCommitStatusesApplyDoerPermission(t *testing.T) { assert.Empty(t, statuses[1].TargetURL) assert.Equal(t, "https://mycicd.org/1", statuses[2].TargetURL) } - -func TestGetCountLatestCommitStatus(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - - sha1 := "1234123412341234123412341234123412341234" // the mocked commit ID in test fixtures - - commitStatuses, err := git_model.GetLatestCommitStatus(t.Context(), repo1.ID, sha1, db.ListOptions{ - Page: 1, - PageSize: 2, - }) - assert.NoError(t, err) - assert.Len(t, commitStatuses, 2) - assert.Equal(t, commitstatus.CommitStatusFailure, commitStatuses[0].State) - assert.Equal(t, "ci/awesomeness", commitStatuses[0].Context) - assert.Equal(t, commitstatus.CommitStatusError, commitStatuses[1].State) - assert.Equal(t, "deploy/awesomeness", commitStatuses[1].Context) - - count, err := git_model.CountLatestCommitStatus(t.Context(), repo1.ID, sha1) - assert.NoError(t, err) - assert.EqualValues(t, 3, count) -}