From 67881ae99abc673954d56436730af313c93ee48f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 30 Nov 2022 21:39:02 +0800 Subject: [PATCH 01/17] Skip initing disabled storages (#21985) If `Attachment` or `Packages` are disabled, we don't have to init the storages for them. --- modules/storage/helper.go | 32 +++++++++++++++++++++ modules/storage/helper_test.go | 50 ++++++++++++++++++++++++++++++++ modules/storage/storage.go | 52 +++++++++++++++++----------------- 3 files changed, 108 insertions(+), 26 deletions(-) create mode 100644 modules/storage/helper_test.go diff --git a/modules/storage/helper.go b/modules/storage/helper.go index 5aaa2a9e64..1ab99d98b3 100644 --- a/modules/storage/helper.go +++ b/modules/storage/helper.go @@ -4,6 +4,10 @@ package storage import ( + "fmt" + "io" + "net/url" + "os" "reflect" "code.gitea.io/gitea/modules/json" @@ -61,3 +65,31 @@ func toConfig(exemplar, cfg interface{}) (interface{}, error) { } return newVal.Elem().Interface(), nil } + +var uninitializedStorage = discardStorage("uninitialized storage") + +type discardStorage string + +func (s discardStorage) Open(_ string) (Object, error) { + return nil, fmt.Errorf("%s", s) +} + +func (s discardStorage) Save(_ string, _ io.Reader, _ int64) (int64, error) { + return 0, fmt.Errorf("%s", s) +} + +func (s discardStorage) Stat(_ string) (os.FileInfo, error) { + return nil, fmt.Errorf("%s", s) +} + +func (s discardStorage) Delete(_ string) error { + return fmt.Errorf("%s", s) +} + +func (s discardStorage) URL(_, _ string) (*url.URL, error) { + return nil, fmt.Errorf("%s", s) +} + +func (s discardStorage) IterateObjects(_ func(string, Object) error) error { + return fmt.Errorf("%s", s) +} diff --git a/modules/storage/helper_test.go b/modules/storage/helper_test.go new file mode 100644 index 0000000000..7d74671c54 --- /dev/null +++ b/modules/storage/helper_test.go @@ -0,0 +1,50 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_discardStorage(t *testing.T) { + tests := []discardStorage{ + uninitializedStorage, + discardStorage("empty"), + } + for _, tt := range tests { + t.Run(string(tt), func(t *testing.T) { + { + got, err := tt.Open("path") + assert.Nil(t, got) + assert.Error(t, err, string(tt)) + } + { + got, err := tt.Save("path", bytes.NewReader([]byte{0}), 1) + assert.Equal(t, int64(0), got) + assert.Error(t, err, string(tt)) + } + { + got, err := tt.Stat("path") + assert.Nil(t, got) + assert.Error(t, err, string(tt)) + } + { + err := tt.Delete("path") + assert.Error(t, err, string(tt)) + } + { + got, err := tt.URL("path", "name") + assert.Nil(t, got) + assert.Errorf(t, err, string(tt)) + } + { + err := tt.IterateObjects(func(_ string, _ Object) error { return nil }) + assert.Error(t, err, string(tt)) + } + }) + } +} diff --git a/modules/storage/storage.go b/modules/storage/storage.go index a7d3b9ce1f..35f1527172 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -110,46 +110,38 @@ func SaveFrom(objStorage ObjectStorage, p string, callback func(w io.Writer) err var ( // Attachments represents attachments storage - Attachments ObjectStorage + Attachments ObjectStorage = uninitializedStorage // LFS represents lfs storage - LFS ObjectStorage + LFS ObjectStorage = uninitializedStorage // Avatars represents user avatars storage - Avatars ObjectStorage + Avatars ObjectStorage = uninitializedStorage // RepoAvatars represents repository avatars storage - RepoAvatars ObjectStorage + RepoAvatars ObjectStorage = uninitializedStorage // RepoArchives represents repository archives storage - RepoArchives ObjectStorage + RepoArchives ObjectStorage = uninitializedStorage // Packages represents packages storage - Packages ObjectStorage + Packages ObjectStorage = uninitializedStorage ) // Init init the stoarge func Init() error { - if err := initAttachments(); err != nil { - return err + for _, f := range []func() error{ + initAttachments, + initAvatars, + initRepoAvatars, + initLFS, + initRepoArchives, + initPackages, + } { + if err := f(); err != nil { + return err + } } - - if err := initAvatars(); err != nil { - return err - } - - if err := initRepoAvatars(); err != nil { - return err - } - - if err := initLFS(); err != nil { - return err - } - - if err := initRepoArchives(); err != nil { - return err - } - - return initPackages() + return nil } // NewStorage takes a storage type and some config and returns an ObjectStorage or an error @@ -172,6 +164,10 @@ func initAvatars() (err error) { } func initAttachments() (err error) { + if !setting.Attachment.Enabled { + Attachments = discardStorage("Attachment isn't enabled") + return nil + } log.Info("Initialising Attachment storage with type: %s", setting.Attachment.Storage.Type) Attachments, err = NewStorage(setting.Attachment.Storage.Type, &setting.Attachment.Storage) return err @@ -196,6 +192,10 @@ func initRepoArchives() (err error) { } func initPackages() (err error) { + if !setting.Packages.Enabled { + Packages = discardStorage("Packages isn't enabled") + return nil + } log.Info("Initialising Packages storage with type: %s", setting.Packages.Storage.Type) Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage) return err From b2c4870481329889a76ea2421152647d36dd8374 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 1 Dec 2022 00:41:49 +0800 Subject: [PATCH 02/17] Fix parallel creating commit status bug with tests (#21911) This PR is a follow up of #21469 Co-authored-by: Lauris BH --- models/db/index.go | 5 -- models/git/commit_status.go | 109 +++++++++---------------- tests/integration/repo_commits_test.go | 31 +++++++ 3 files changed, 71 insertions(+), 74 deletions(-) diff --git a/models/db/index.go b/models/db/index.go index 46be74e91e..ca664c9cf1 100644 --- a/models/db/index.go +++ b/models/db/index.go @@ -23,11 +23,6 @@ var ( ErrGetResourceIndexFailed = errors.New("get resource index failed") ) -const ( - // MaxDupIndexAttempts max retry times to create index - MaxDupIndexAttempts = 3 -) - // SyncMaxResourceIndex sync the max index with the resource func SyncMaxResourceIndex(ctx context.Context, tableName string, groupID, maxIndex int64) (err error) { e := GetEngine(ctx) diff --git a/models/git/commit_status.go b/models/git/commit_status.go index 411fbbe536..6353bb2fa9 100644 --- a/models/git/commit_status.go +++ b/models/git/commit_status.go @@ -6,6 +6,7 @@ package git import ( "context" "crypto/sha1" + "errors" "fmt" "net/url" "strings" @@ -48,79 +49,49 @@ func init() { db.RegisterModel(new(CommitStatusIndex)) } -// upsertCommitStatusIndex the function will not return until it acquires the lock or receives an error. -func upsertCommitStatusIndex(ctx context.Context, repoID int64, sha string) (err error) { - // An atomic UPSERT operation (INSERT/UPDATE) is the only operation - // that ensures that the key is actually locked. - switch { - case setting.Database.UseSQLite3 || setting.Database.UsePostgreSQL: - _, err = db.Exec(ctx, "INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+ - "VALUES (?,?,1) ON CONFLICT (repo_id,sha) DO UPDATE SET max_index = `commit_status_index`.max_index+1", - repoID, sha) - case setting.Database.UseMySQL: - _, err = db.Exec(ctx, "INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+ - "VALUES (?,?,1) ON DUPLICATE KEY UPDATE max_index = max_index+1", - repoID, sha) - case setting.Database.UseMSSQL: - // https://weblogs.sqlteam.com/dang/2009/01/31/upsert-race-condition-with-merge/ - _, err = db.Exec(ctx, "MERGE `commit_status_index` WITH (HOLDLOCK) as target "+ - "USING (SELECT ? AS repo_id, ? AS sha) AS src "+ - "ON src.repo_id = target.repo_id AND src.sha = target.sha "+ - "WHEN MATCHED THEN UPDATE SET target.max_index = target.max_index+1 "+ - "WHEN NOT MATCHED THEN INSERT (repo_id, sha, max_index) "+ - "VALUES (src.repo_id, src.sha, 1);", - repoID, sha) - default: - return fmt.Errorf("database type not supported") - } - return err -} - // GetNextCommitStatusIndex retried 3 times to generate a resource index -func GetNextCommitStatusIndex(repoID int64, sha string) (int64, error) { - for i := 0; i < db.MaxDupIndexAttempts; i++ { - idx, err := getNextCommitStatusIndex(repoID, sha) - if err == db.ErrResouceOutdated { - continue - } +func GetNextCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) { + e := db.GetEngine(ctx) + + // try to update the max_index to next value, and acquire the write-lock for the record + res, err := e.Exec("UPDATE `commit_status_index` SET max_index=max_index+1 WHERE repo_id=? AND sha=?", repoID, sha) + if err != nil { + return 0, err + } + affected, err := res.RowsAffected() + if err != nil { + return 0, err + } + if affected == 0 { + // this slow path is only for the first time of creating a resource index + _, errIns := e.Exec("INSERT INTO `commit_status_index` (repo_id, sha, max_index) VALUES (?, ?, 0)", repoID, sha) + res, err = e.Exec("UPDATE `commit_status_index` SET max_index=max_index+1 WHERE repo_id=? AND sha=?", repoID, sha) if err != nil { return 0, err } - return idx, nil - } - return 0, db.ErrGetResourceIndexFailed -} - -// getNextCommitStatusIndex return the next index -func getNextCommitStatusIndex(repoID int64, sha string) (int64, error) { - ctx, commiter, err := db.TxContext(db.DefaultContext) - if err != nil { - return 0, err - } - defer commiter.Close() - - var preIdx int64 - _, err = db.GetEngine(ctx).SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id = ? AND sha = ?", repoID, sha).Get(&preIdx) - if err != nil { - return 0, err + affected, err = res.RowsAffected() + if err != nil { + return 0, err + } + // if the update still can not update any records, the record must not exist and there must be some errors (insert error) + if affected == 0 { + if errIns == nil { + return 0, errors.New("impossible error when GetNextCommitStatusIndex, insert and update both succeeded but no record is updated") + } + return 0, errIns + } } - if err := upsertCommitStatusIndex(ctx, repoID, sha); err != nil { - return 0, err - } - - var curIdx int64 - has, err := db.GetEngine(ctx).SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id = ? AND sha = ? AND max_index=?", repoID, sha, preIdx+1).Get(&curIdx) + // now, the new index is in database (protected by the transaction and write-lock) + var newIdx int64 + has, err := e.SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id=? AND sha=?", repoID, sha).Get(&newIdx) if err != nil { return 0, err } if !has { - return 0, db.ErrResouceOutdated + return 0, errors.New("impossible error when GetNextCommitStatusIndex, upsert succeeded but no record can be selected") } - if err := commiter.Commit(); err != nil { - return 0, err - } - return curIdx, nil + return newIdx, nil } func (status *CommitStatus) loadAttributes(ctx context.Context) (err error) { @@ -290,18 +261,18 @@ func NewCommitStatus(opts NewCommitStatusOptions) error { return fmt.Errorf("NewCommitStatus[%s, %s]: no user specified", repoPath, opts.SHA) } - // Get the next Status Index - idx, err := GetNextCommitStatusIndex(opts.Repo.ID, opts.SHA) - if err != nil { - return fmt.Errorf("generate commit status index failed: %w", err) - } - ctx, committer, err := db.TxContext(db.DefaultContext) if err != nil { return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", opts.Repo.ID, opts.Creator.ID, opts.SHA, err) } defer committer.Close() + // Get the next Status Index + idx, err := GetNextCommitStatusIndex(ctx, opts.Repo.ID, opts.SHA) + if err != nil { + return fmt.Errorf("generate commit status index failed: %w", err) + } + opts.CommitStatus.Description = strings.TrimSpace(opts.CommitStatus.Description) opts.CommitStatus.Context = strings.TrimSpace(opts.CommitStatus.Context) opts.CommitStatus.TargetURL = strings.TrimSpace(opts.CommitStatus.TargetURL) @@ -315,7 +286,7 @@ func NewCommitStatus(opts NewCommitStatusOptions) error { // Insert new CommitStatus if _, err = db.GetEngine(ctx).Insert(opts.CommitStatus); err != nil { - return fmt.Errorf("Insert CommitStatus[%s, %s]: %w", repoPath, opts.SHA, err) + return fmt.Errorf("insert CommitStatus[%s, %s]: %w", repoPath, opts.SHA, err) } return committer.Commit() diff --git a/tests/integration/repo_commits_test.go b/tests/integration/repo_commits_test.go index 438e1c2cc4..3840c508dc 100644 --- a/tests/integration/repo_commits_test.go +++ b/tests/integration/repo_commits_test.go @@ -4,9 +4,11 @@ package integration import ( + "fmt" "net/http" "net/http/httptest" "path" + "sync" "testing" "code.gitea.io/gitea/modules/json" @@ -114,3 +116,32 @@ func TestRepoCommitsWithStatusFailure(t *testing.T) { func TestRepoCommitsWithStatusWarning(t *testing.T) { doTestRepoCommitWithStatus(t, "warning", "gitea-exclamation", "yellow") } + +func TestRepoCommitsStatusParallel(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + session := loginUser(t, "user2") + + // Request repository commits page + req := NewRequest(t, "GET", "/user2/repo1/commits/branch/master") + resp := session.MakeRequest(t, req, http.StatusOK) + + doc := NewHTMLParser(t, resp.Body) + // Get first commit URL + commitURL, exists := doc.doc.Find("#commits-table tbody tr td.sha a").Attr("href") + assert.True(t, exists) + assert.NotEmpty(t, commitURL) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func(t *testing.T, i int) { + t.Run(fmt.Sprintf("ParallelCreateStatus_%d", i), func(t *testing.T) { + runBody := doAPICreateCommitStatus(NewAPITestContext(t, "user2", "repo1"), path.Base(commitURL), api.CommitStatusState("pending")) + runBody(t) + wg.Done() + }) + }(t, i) + } + wg.Wait() +} From 4e5d4d0073df7bffba95107ea969f1fdeb165830 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 1 Dec 2022 17:02:04 +0800 Subject: [PATCH 03/17] Skip initing LFS storage if disabled (#21996) A complement to #21985. I overlooked it because the name of the switch is `StartServer`, not `Enabled`. I believe the weird name is a legacy, but renaming is out of scope. --- modules/storage/storage.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 35f1527172..671e0ce565 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -174,6 +174,10 @@ func initAttachments() (err error) { } func initLFS() (err error) { + if !setting.LFS.StartServer { + LFS = discardStorage("LFS isn't enabled") + return nil + } log.Info("Initialising LFS storage with type: %s", setting.LFS.Storage.Type) LFS, err = NewStorage(setting.LFS.Storage.Type, &setting.LFS.Storage) return err From f9cbf5a1bcd38810432e9ca16270e1f7ccc63605 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 1 Dec 2022 19:56:04 +0800 Subject: [PATCH 04/17] Util type to parse ref name (#21969) Provide a new type to make it easier to parse a ref name. Actually, it's picked up from #21937, to make the origin PR lighter. Co-authored-by: Lunny Xiao --- modules/git/ref.go | 63 +++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/modules/git/ref.go b/modules/git/ref.go index cd8d268184..47cc04b7fb 100644 --- a/modules/git/ref.go +++ b/modules/git/ref.go @@ -55,40 +55,57 @@ func (ref *Reference) Commit() (*Commit, error) { // ShortName returns the short name of the reference func (ref *Reference) ShortName() string { - if ref == nil { - return "" - } - if strings.HasPrefix(ref.Name, BranchPrefix) { - return strings.TrimPrefix(ref.Name, BranchPrefix) - } - if strings.HasPrefix(ref.Name, TagPrefix) { - return strings.TrimPrefix(ref.Name, TagPrefix) - } - if strings.HasPrefix(ref.Name, RemotePrefix) { - return strings.TrimPrefix(ref.Name, RemotePrefix) - } - if strings.HasPrefix(ref.Name, PullPrefix) && strings.IndexByte(ref.Name[pullLen:], '/') > -1 { - return ref.Name[pullLen : strings.IndexByte(ref.Name[pullLen:], '/')+pullLen] - } - - return ref.Name + return RefName(ref.Name).ShortName() } // RefGroup returns the group type of the reference func (ref *Reference) RefGroup() string { - if ref == nil { - return "" + return RefName(ref.Name).RefGroup() +} + +// RefName represents a git reference name +type RefName string + +func (ref RefName) IsBranch() bool { + return strings.HasPrefix(string(ref), BranchPrefix) +} + +func (ref RefName) IsTag() bool { + return strings.HasPrefix(string(ref), TagPrefix) +} + +// ShortName returns the short name of the reference name +func (ref RefName) ShortName() string { + refName := string(ref) + if strings.HasPrefix(refName, BranchPrefix) { + return strings.TrimPrefix(refName, BranchPrefix) } - if strings.HasPrefix(ref.Name, BranchPrefix) { + if strings.HasPrefix(refName, TagPrefix) { + return strings.TrimPrefix(refName, TagPrefix) + } + if strings.HasPrefix(refName, RemotePrefix) { + return strings.TrimPrefix(refName, RemotePrefix) + } + if strings.HasPrefix(refName, PullPrefix) && strings.IndexByte(refName[pullLen:], '/') > -1 { + return refName[pullLen : strings.IndexByte(refName[pullLen:], '/')+pullLen] + } + + return refName +} + +// RefGroup returns the group type of the reference +func (ref RefName) RefGroup() string { + refName := string(ref) + if strings.HasPrefix(refName, BranchPrefix) { return "heads" } - if strings.HasPrefix(ref.Name, TagPrefix) { + if strings.HasPrefix(refName, TagPrefix) { return "tags" } - if strings.HasPrefix(ref.Name, RemotePrefix) { + if strings.HasPrefix(refName, RemotePrefix) { return "remotes" } - if strings.HasPrefix(ref.Name, PullPrefix) && strings.IndexByte(ref.Name[pullLen:], '/') > -1 { + if strings.HasPrefix(refName, PullPrefix) && strings.IndexByte(refName[pullLen:], '/') > -1 { return "pull" } return "" From f0bd219a5e153ddcf6806334da700aac46fcefad Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 1 Dec 2022 22:44:38 +0100 Subject: [PATCH 05/17] Update chroma to v2.4.0 (#22000) Did a few cursory tests, seems to work well. --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ca8c79c689..5e090a98b4 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/NYTimes/gziphandler v1.1.1 github.com/PuerkitoBio/goquery v1.8.0 - github.com/alecthomas/chroma/v2 v2.3.0 + github.com/alecthomas/chroma/v2 v2.4.0 github.com/blevesearch/bleve/v2 v2.3.4 github.com/buildkite/terminal-to-html/v3 v3.7.0 github.com/caddyserver/certmagic v0.17.2 diff --git a/go.sum b/go.sum index d6748eae34..097d03780c 100644 --- a/go.sum +++ b/go.sum @@ -160,9 +160,10 @@ github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/alecthomas/assert/v2 v2.2.0 h1:f6L/b7KE2bfA+9O4FL3CM/xJccDEwPVYd5fALBiuwvw= github.com/alecthomas/chroma/v2 v2.2.0/go.mod h1:vf4zrexSH54oEjJ7EdB65tGNHmH3pGZmVkgTP5RHvAs= -github.com/alecthomas/chroma/v2 v2.3.0 h1:83xfxrnjv8eK+Cf8qZDzNo3PPF9IbTWHs7z28GY6D0U= -github.com/alecthomas/chroma/v2 v2.3.0/go.mod h1:mZxeWZlxP2Dy+/8cBob2PYd8O2DwNAzave5AY7A2eQw= +github.com/alecthomas/chroma/v2 v2.4.0 h1:Loe2ZjT5x3q1bcWwemqyqEi8p11/IV/ncFCeLYDpWC4= +github.com/alecthomas/chroma/v2 v2.4.0/go.mod h1:6kHzqF5O6FUSJzBXW7fXELjb+e+7OXW4UpoPqMO7IBQ= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= @@ -830,6 +831,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= From 64973cf18fcd21179fb7612b15914691ee24c657 Mon Sep 17 00:00:00 2001 From: zeripath Date: Thu, 1 Dec 2022 23:56:51 +0000 Subject: [PATCH 06/17] Use path not filepath in template filenames (#21993) Paths in git are always separated by `/` not `\` - therefore we should `path` and not `filepath` Fix #21987 Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao Co-authored-by: Lauris BH --- modules/issue/template/unmarshal.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/issue/template/unmarshal.go b/modules/issue/template/unmarshal.go index f43a83fb62..8cae8d4c42 100644 --- a/modules/issue/template/unmarshal.go +++ b/modules/issue/template/unmarshal.go @@ -6,7 +6,7 @@ package template import ( "fmt" "io" - "path/filepath" + "path" "strconv" "code.gitea.io/gitea/modules/git" @@ -43,7 +43,7 @@ func Unmarshal(filename string, content []byte) (*api.IssueTemplate, error) { // UnmarshalFromEntry parses out a valid template from the blob in entry func UnmarshalFromEntry(entry *git.TreeEntry, dir string) (*api.IssueTemplate, error) { - return unmarshalFromEntry(entry, filepath.Join(dir, entry.Name())) + return unmarshalFromEntry(entry, path.Join(dir, entry.Name())) // Filepaths in Git are ALWAYS '/' separated do not use filepath here } // UnmarshalFromCommit parses out a valid template from the commit @@ -108,7 +108,7 @@ func unmarshal(filename string, content []byte) (*api.IssueTemplate, error) { // It could be a valid markdown with two horizontal lines, or an invalid markdown with wrong metadata. it.Content = string(content) - it.Name = filepath.Base(it.FileName) + it.Name = path.Base(it.FileName) // paths in Git are always '/' separated - do not use filepath! it.About, _ = util.SplitStringAtByteN(it.Content, 80) } else { it.Content = templateBody From f7ade6de7c1c4991c650d6b96e6407053e6bdae7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 2 Dec 2022 11:15:36 +0800 Subject: [PATCH 07/17] Fix generate index failure possibility on postgres (#21998) @wxiaoguang Please review Co-authored-by: silverwind --- models/db/index.go | 20 ++++++++++++++++++++ models/git/commit_status.go | 18 ++++++++++++++++++ tests/integration/repo_commits_test.go | 4 ++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/models/db/index.go b/models/db/index.go index ca664c9cf1..f840a62c89 100644 --- a/models/db/index.go +++ b/models/db/index.go @@ -7,6 +7,9 @@ import ( "context" "errors" "fmt" + "strconv" + + "code.gitea.io/gitea/modules/setting" ) // ResourceIndex represents a resource index which could be used as issue/release and others @@ -55,8 +58,25 @@ func SyncMaxResourceIndex(ctx context.Context, tableName string, groupID, maxInd return nil } +func postgresGetNextResourceIndex(ctx context.Context, tableName string, groupID int64) (int64, error) { + res, err := GetEngine(ctx).Query(fmt.Sprintf("INSERT INTO %s (group_id, max_index) "+ + "VALUES (?,1) ON CONFLICT (group_id) DO UPDATE SET max_index = %s.max_index+1 RETURNING max_index", + tableName, tableName), groupID) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, ErrGetResourceIndexFailed + } + return strconv.ParseInt(string(res[0]["max_index"]), 10, 64) +} + // GetNextResourceIndex generates a resource index, it must run in the same transaction where the resource is created func GetNextResourceIndex(ctx context.Context, tableName string, groupID int64) (int64, error) { + if setting.Database.UsePostgreSQL { + return postgresGetNextResourceIndex(ctx, tableName, groupID) + } + e := GetEngine(ctx) // try to update the max_index to next value, and acquire the write-lock for the record diff --git a/models/git/commit_status.go b/models/git/commit_status.go index 6353bb2fa9..928f1771fe 100644 --- a/models/git/commit_status.go +++ b/models/git/commit_status.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/url" + "strconv" "strings" "time" @@ -49,8 +50,25 @@ func init() { db.RegisterModel(new(CommitStatusIndex)) } +func postgresGetCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) { + res, err := db.GetEngine(ctx).Query("INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+ + "VALUES (?,?,1) ON CONFLICT (repo_id, sha) DO UPDATE SET max_index = `commit_status_index`.max_index+1 RETURNING max_index", + repoID, sha) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, db.ErrGetResourceIndexFailed + } + return strconv.ParseInt(string(res[0]["max_index"]), 10, 64) +} + // GetNextCommitStatusIndex retried 3 times to generate a resource index func GetNextCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) { + if setting.Database.UsePostgreSQL { + return postgresGetCommitStatusIndex(ctx, repoID, sha) + } + e := db.GetEngine(ctx) // try to update the max_index to next value, and acquire the write-lock for the record diff --git a/tests/integration/repo_commits_test.go b/tests/integration/repo_commits_test.go index 3840c508dc..0d794cec75 100644 --- a/tests/integration/repo_commits_test.go +++ b/tests/integration/repo_commits_test.go @@ -135,8 +135,8 @@ func TestRepoCommitsStatusParallel(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) - go func(t *testing.T, i int) { - t.Run(fmt.Sprintf("ParallelCreateStatus_%d", i), func(t *testing.T) { + go func(parentT *testing.T, i int) { + parentT.Run(fmt.Sprintf("ParallelCreateStatus_%d", i), func(t *testing.T) { runBody := doAPICreateCommitStatus(NewAPITestContext(t, "user2", "repo1"), path.Base(commitURL), api.CommitStatusState("pending")) runBody(t) wg.Done() From 665d02efaf6ebaadf3ffb5b412ea77c4502f0baf Mon Sep 17 00:00:00 2001 From: Mark Ormesher Date: Fri, 2 Dec 2022 03:39:19 +0000 Subject: [PATCH 08/17] Remove duplicate "Actions" label in mobile view (#21974) Closes #21973. The "Actions" button on the commit view page is labelled twice in mobile view. No other buttons on the page have a `mobile-only` extra label, so this PR removes it. Before: ![before](https://user-images.githubusercontent.com/6496999/204540002-75baa08a-6c06-4b39-847b-34272e09d71e.PNG) After: ![after](https://user-images.githubusercontent.com/6496999/204539991-a0607765-d5e2-4b1a-84c9-a3e16cbc674e.PNG) Co-authored-by: Lunny Xiao --- templates/repo/commit_page.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl index 2dcd02ad98..d4839e8930 100644 --- a/templates/repo/commit_page.tmpl +++ b/templates/repo/commit_page.tmpl @@ -27,7 +27,7 @@ {{if and ($.Permission.CanWrite $.UnitTypeCode) (not $.Repository.IsArchived) (not .IsDeleted)}}{{- /* */ -}}