fix(models): chunk commit status queries to avoid SQLite expression depth limit (#39611)

Fixes https://github.com/go-gitea/gitea/issues/39606

Refactor GetLatestCommitStatusForRepoCommitIDs to 
query commit IDs by batch to avoid generating a large SQL

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

models/git/commit_status.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -35,11 +36,11 @@ import (
 // CommitStatus holds a single Status of a single Commit
 type CommitStatus struct {
 	ID     int64                          `xorm:"pk autoincr"`
+	RepoID int64                          `xorm:"UNIQUE(repo_sha_index)"`
+	SHA    string                         `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"`
 	Index  int64                          `xorm:"INDEX UNIQUE(repo_sha_index)"`
-	RepoID int64                          `xorm:"INDEX UNIQUE(repo_sha_index)"`
 	Repo   *repo_model.Repository         `xorm:"-"`
 	State  commitstatus.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
-	SHA    string                         `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"`
 
 	// TargetURL points to the commit status page reported by a CI system
 	// If Gitea Actions is used, it is a relative link like "{RepoLink}/actions/runs/{RunID}/jobs{JobID}"
@@ -420,46 +421,48 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map
 	return repoStatuses, nil
 }
 
-// GetLatestCommitStatusForRepoCommitIDs returns all statuses with a unique context for a given list of repo-sha pairs
+// GetLatestCommitStatusForRepoCommitIDs returns the commit statuses with a unique context for a given list of repo-sha pairs
-func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, commitIDs []string) (map[string][]*CommitStatus, error) {
+// If the provided commit IDs are too many, only the first part and the last part of the commit IDs will be queried.
-	type result struct {
+func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, allCommitIDs []string) (map[string][]*CommitStatus, error) {
+	const maxCommitIDs = 500
+	const maxBatchSize = 200
+	queryCommitIDs := allCommitIDs
+	if len(allCommitIDs) > maxCommitIDs {
+		// The commit IDs are usually from "commits list" or "compare" page (create a PR or compare commits), nobody can read so many commits at once.
+		// The commit IDs are usually sorted by time, so we can take the first half and the last half of the commit IDs to get the latest statuses.
+		log.Warn("GetLatestCommitStatusForRepoCommitIDs: too many commit IDs (%d) for repo %d, truncating to %d", len(allCommitIDs), repoID, maxCommitIDs)
+		queryCommitIDs = allCommitIDs[:maxCommitIDs/2]
+		queryCommitIDs = append(queryCommitIDs, allCommitIDs[len(allCommitIDs)-maxCommitIDs/2:]...)
+	}
+
+	baseSql := func() db.Session {
+		return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
+	}
+
+	type shaMaxIndexResult struct {
 		Index int64
 		SHA   string
 	}
-
+	shaMaxIndexResults := make([]*shaMaxIndexResult, 0, len(allCommitIDs))
-	getBase := func() db.Session {
+	err := baseSql().And(builder.In("sha", queryCommitIDs)).
-		return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
+		Select("max(`index`) as `index`, sha").
-	}
+		GroupBy("context_hash, sha").
-	results := make([]result, 0, len(commitIDs))
+		Find(&shaMaxIndexResults)
-
-	conds := make([]builder.Cond, 0, len(commitIDs))
-	for _, sha := range commitIDs {
-		conds = append(conds, builder.Eq{"sha": sha})
-	}
-	sess := getBase().And(builder.Or(conds...)).
-		Select("max( `index` ) as `index`, sha").
-		GroupBy("context_hash, sha").OrderBy("max( `index` ) desc")
-
-	err := sess.Find(&results)
 	if err != nil {
 		return nil, err
 	}
 
 	repoStatuses := make(map[string][]*CommitStatus)
-
+	for chunk := range slices.Chunk(shaMaxIndexResults, maxBatchSize) {
-	if len(results) > 0 {
+		statuses := make([]*CommitStatus, 0, len(chunk))
-		statuses := make([]*CommitStatus, 0, len(results))
+		condIndexSha := make([]builder.Cond, 0, len(chunk))
-
+		for _, res := range chunk {
-		conds = make([]builder.Cond, 0, len(results))
+			condIndexSha = append(condIndexSha, builder.Eq{"`index`": res.Index, "sha": res.SHA})
-		for _, result := range results {
-			conds = append(conds, builder.Eq{"`index`": result.Index, "sha": result.SHA})
 		}
-		err = getBase().And(builder.Or(conds...)).Find(&statuses)
+		err = baseSql().And(builder.Or(condIndexSha...)).Find(&statuses)
 		if err != nil {
 			return nil, err
 		}
-
-		// Group the statuses by commit
 		for _, status := range statuses {
 			repoStatuses[status.SHA] = append(repoStatuses[status.SHA], status)
 		}

models/git/commit_status_test.go

@@ -4,6 +4,7 @@
 package git_test
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -22,33 +23,71 @@ func TestGetCommitStatuses(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 
 	repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
-
 	sha1 := "1234123412341234123412341234123412341234" // the mocked commit ID in test fixtures
 
-	statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
+	t.Run("CommitStatusOptions", func(t *testing.T) {
-		ListOptions: db.ListOptions{Page: 1, PageSize: 50},
+		statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
-		RepoID:      repo1.ID,
+			ListOptions: db.ListOptions{Page: 1, PageSize: 50},
-		SHA:         sha1,
+			RepoID:      repo1.ID,
-	})
+			SHA:         sha1,
-	assert.NoError(t, err)
+		})
-	assert.Equal(t, 5, int(maxResults))
+		assert.NoError(t, err)
-	var indexes []int64
+		assert.Equal(t, 5, int(maxResults))
-	for _, status := range statuses {
+		var indexes []int64
-		indexes = append(indexes, status.Index)
+		for _, status := range statuses {
-	}
+			indexes = append(indexes, status.Index)
-	assert.Equal(t, []int64{5, 4, 3, 2, 1}, indexes)
+		}
-	assert.Equal(t, "deploy/awesomeness", statuses[0].Context)
+		assert.Equal(t, []int64{5, 4, 3, 2, 1}, indexes)
-	assert.Equal(t, commitstatus.CommitStatusError, statuses[0].State)
+		assert.Equal(t, "deploy/awesomeness", statuses[0].Context)
-	assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/statuses/1234123412341234123412341234123412341234", statuses[0].APIURL(t.Context()))
+		assert.Equal(t, commitstatus.CommitStatusError, statuses[0].State)
+		assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/statuses/1234123412341234123412341234123412341234", statuses[0].APIURL(t.Context()))
 
-	statuses, maxResults, err = db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
+		statuses, maxResults, err = db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
-		ListOptions: db.ListOptions{Page: 2, PageSize: 50},
+			ListOptions: db.ListOptions{Page: 2, PageSize: 50},
-		RepoID:      repo1.ID,
+			RepoID:      repo1.ID,
-		SHA:         sha1,
+			SHA:         sha1,
+		})
+		assert.NoError(t, err)
+		assert.Equal(t, 5, int(maxResults))
+		assert.Empty(t, statuses)
+	})
+
+	t.Run("GetCountLatestCommitStatus", func(t *testing.T) {
+		commitStatuses, err := git_model.GetLatestCommitStatus(t.Context(), repo1.ID, sha1, db.ListOptions{
+			Page:     1,
+			PageSize: 2,
+		})
+		assert.NoError(t, err)
+		assert.Len(t, commitStatuses, 2)
+		assert.Equal(t, commitstatus.CommitStatusFailure, commitStatuses[0].State)
+		assert.Equal(t, "ci/awesomeness", commitStatuses[0].Context)
+		assert.Equal(t, commitstatus.CommitStatusError, commitStatuses[1].State)
+		assert.Equal(t, "deploy/awesomeness", commitStatuses[1].Context)
+
+		count, err := git_model.CountLatestCommitStatus(t.Context(), repo1.ID, sha1)
+		assert.NoError(t, err)
+		assert.EqualValues(t, 3, count)
+	})
+
+	t.Run("GetLatestCommitStatusForRepoCommitIDs", func(t *testing.T) {
+		commitIDs := []string{sha1}
+		// SQLite has a limit of 1000 for WHERE expression variables per query, use a larger slice to test
+		for i := range 2000 {
+			commitIDs = append(commitIDs, fmt.Sprintf("%040x", i+1))
+		}
+		statuses, err := git_model.GetLatestCommitStatusForRepoCommitIDs(t.Context(), repo1.ID, commitIDs)
+		assert.NoError(t, err)
+		assert.Len(t, statuses[sha1], 3)
+		latestIndexes := make(map[string]int64)
+		for _, status := range statuses[sha1] {
+			latestIndexes[status.Context] = status.Index
+		}
+		assert.Equal(t, map[string]int64{
+			"ci/awesomeness":     4,
+			"cov/awesomeness":    3,
+			"deploy/awesomeness": 5,
+		}, latestIndexes)
 	})
-	assert.NoError(t, err)
-	assert.Equal(t, 5, int(maxResults))
-	assert.Empty(t, statuses)
 }
 
 func Test_CalcCommitStatus(t *testing.T) {
@@ -247,26 +286,3 @@ func TestCommitStatusesApplyDoerPermission(t *testing.T) {
 	assert.Empty(t, statuses[1].TargetURL)
 	assert.Equal(t, "https://mycicd.org/1", statuses[2].TargetURL)
 }
-
-func TestGetCountLatestCommitStatus(t *testing.T) {
-	assert.NoError(t, unittest.PrepareTestDatabase())
-
-	repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
-
-	sha1 := "1234123412341234123412341234123412341234" // the mocked commit ID in test fixtures
-
-	commitStatuses, err := git_model.GetLatestCommitStatus(t.Context(), repo1.ID, sha1, db.ListOptions{
-		Page:     1,
-		PageSize: 2,
-	})
-	assert.NoError(t, err)
-	assert.Len(t, commitStatuses, 2)
-	assert.Equal(t, commitstatus.CommitStatusFailure, commitStatuses[0].State)
-	assert.Equal(t, "ci/awesomeness", commitStatuses[0].Context)
-	assert.Equal(t, commitstatus.CommitStatusError, commitStatuses[1].State)
-	assert.Equal(t, "deploy/awesomeness", commitStatuses[1].Context)
-
-	count, err := git_model.CountLatestCommitStatus(t.Context(), repo1.ID, sha1)
-	assert.NoError(t, err)
-	assert.EqualValues(t, 3, count)
-}