From 00cc84e37cd352b3ca73f881b9b51ece92e4793b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 26 Dec 2025 22:55:30 -0800 Subject: [PATCH 1/8] remove nolint (#36252) --- modules/markup/mdstripper/mdstripper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/mdstripper/mdstripper.go b/modules/markup/mdstripper/mdstripper.go index 5a6504416a..19b852a3ee 100644 --- a/modules/markup/mdstripper/mdstripper.go +++ b/modules/markup/mdstripper/mdstripper.go @@ -46,7 +46,7 @@ func (r *stripRenderer) Render(w io.Writer, source []byte, doc ast.Node) error { coalesce := prevSibIsText r.processString( w, - v.Text(source), //nolint:staticcheck // Text is deprecated + v.Value(source), coalesce) if v.SoftLineBreak() { r.doubleSpace(w) From 19e1997ee21a989cf330a02991db4a8a3242d03f Mon Sep 17 00:00:00 2001 From: Ivan Tkatchev Date: Sat, 27 Dec 2025 15:33:08 +0300 Subject: [PATCH 2/8] Add an option to automatically verify SSH keys from LDAP (#35927) This pull request adds an option to automatically verify SSH keys from LDAP authentication sources. This allows a correct authentication and verification workflow for LDAP-enabled organizations; under normal circumstances SSH keys in LDAP are not managed by users manually. --- cmd/admin_auth_ldap.go | 7 +++++++ models/asymkey/ssh_key.go | 11 ++++++----- options/locale/locale_en-US.json | 1 + routers/api/v1/user/key.go | 2 +- routers/web/admin/auths.go | 1 + routers/web/auth/oauth_signin_sync.go | 2 +- routers/web/user/setting/keys.go | 2 +- services/asymkey/commit_test.go | 2 +- services/asymkey/ssh_key_test.go | 2 +- services/auth/source/ldap/source.go | 1 + services/auth/source/ldap/source_authenticate.go | 4 ++-- services/auth/source/ldap/source_sync.go | 4 ++-- services/forms/auth_form.go | 1 + templates/admin/auth/edit.tmpl | 6 ++++++ templates/admin/auth/source/ldap.tmpl | 6 ++++++ 15 files changed, 38 insertions(+), 14 deletions(-) diff --git a/cmd/admin_auth_ldap.go b/cmd/admin_auth_ldap.go index 069ad6600c..c9be5abb37 100644 --- a/cmd/admin_auth_ldap.go +++ b/cmd/admin_auth_ldap.go @@ -94,6 +94,10 @@ func commonLdapCLIFlags() []cli.Flag { Name: "public-ssh-key-attribute", Usage: "The attribute of the user’s LDAP record containing the user’s public ssh key.", }, + &cli.BoolFlag{ + Name: "ssh-keys-are-verified", + Usage: "Set to true to automatically flag SSH keys in LDAP as verified.", + }, &cli.BoolFlag{ Name: "skip-local-2fa", Usage: "Set to true to skip local 2fa for users authenticated by this source", @@ -294,6 +298,9 @@ func parseLdapConfig(c *cli.Command, config *ldap.Source) error { if c.IsSet("public-ssh-key-attribute") { config.AttributeSSHPublicKey = c.String("public-ssh-key-attribute") } + if c.IsSet("ssh-keys-are-verified") { + config.SSHKeysAreVerified = c.Bool("ssh-keys-are-verified") + } if c.IsSet("avatar-attribute") { config.AttributeAvatar = c.String("avatar-attribute") } diff --git a/models/asymkey/ssh_key.go b/models/asymkey/ssh_key.go index d77b5d46a7..98784b36bd 100644 --- a/models/asymkey/ssh_key.go +++ b/models/asymkey/ssh_key.go @@ -84,7 +84,7 @@ func addKey(ctx context.Context, key *PublicKey) (err error) { } // AddPublicKey adds new public key to database and authorized_keys file. -func AddPublicKey(ctx context.Context, ownerID int64, name, content string, authSourceID int64) (*PublicKey, error) { +func AddPublicKey(ctx context.Context, ownerID int64, name, content string, authSourceID int64, verified bool) (*PublicKey, error) { log.Trace(content) fingerprint, err := CalcFingerprint(content) @@ -115,6 +115,7 @@ func AddPublicKey(ctx context.Context, ownerID int64, name, content string, auth Mode: perm.AccessModeWrite, Type: KeyTypeUser, LoginSourceID: authSourceID, + Verified: verified, } if err = addKey(ctx, key); err != nil { return nil, fmt.Errorf("addKey: %w", err) @@ -298,7 +299,7 @@ func deleteKeysMarkedForDeletion(ctx context.Context, keys []string) (bool, erro } // AddPublicKeysBySource add a users public keys. Returns true if there are changes. -func AddPublicKeysBySource(ctx context.Context, usr *user_model.User, s *auth.Source, sshPublicKeys []string) bool { +func AddPublicKeysBySource(ctx context.Context, usr *user_model.User, s *auth.Source, sshPublicKeys []string, verified bool) bool { var sshKeysNeedUpdate bool for _, sshKey := range sshPublicKeys { var err error @@ -317,7 +318,7 @@ func AddPublicKeysBySource(ctx context.Context, usr *user_model.User, s *auth.So marshalled = marshalled[:len(marshalled)-1] sshKeyName := fmt.Sprintf("%s-%s", s.Name, ssh.FingerprintSHA256(out)) - if _, err := AddPublicKey(ctx, usr.ID, sshKeyName, marshalled, s.ID); err != nil { + if _, err := AddPublicKey(ctx, usr.ID, sshKeyName, marshalled, s.ID, verified); err != nil { if IsErrKeyAlreadyExist(err) { log.Trace("AddPublicKeysBySource[%s]: Public SSH Key %s already exists for user", sshKeyName, usr.Name) } else { @@ -336,7 +337,7 @@ func AddPublicKeysBySource(ctx context.Context, usr *user_model.User, s *auth.So } // SynchronizePublicKeys updates a user's public keys. Returns true if there are changes. -func SynchronizePublicKeys(ctx context.Context, usr *user_model.User, s *auth.Source, sshPublicKeys []string) bool { +func SynchronizePublicKeys(ctx context.Context, usr *user_model.User, s *auth.Source, sshPublicKeys []string, verified bool) bool { var sshKeysNeedUpdate bool log.Trace("synchronizePublicKeys[%s]: Handling Public SSH Key synchronization for user %s", s.Name, usr.Name) @@ -381,7 +382,7 @@ func SynchronizePublicKeys(ctx context.Context, usr *user_model.User, s *auth.So newKeys = append(newKeys, key) } } - if AddPublicKeysBySource(ctx, usr, s, newKeys) { + if AddPublicKeysBySource(ctx, usr, s, newKeys, verified) { sshKeysNeedUpdate = true } diff --git a/options/locale/locale_en-US.json b/options/locale/locale_en-US.json index e56e3a299d..0fb95606b3 100644 --- a/options/locale/locale_en-US.json +++ b/options/locale/locale_en-US.json @@ -3067,6 +3067,7 @@ "admin.auths.attribute_mail": "Email Attribute", "admin.auths.attribute_ssh_public_key": "Public SSH Key Attribute", "admin.auths.attribute_avatar": "Avatar Attribute", + "admin.auths.ssh_keys_are_verified": "SSH keys in LDAP are considered as verified", "admin.auths.attributes_in_bind": "Fetch Attributes in Bind DN Context", "admin.auths.allow_deactivate_all": "Allow an empty search result to deactivate all users", "admin.auths.use_paged_search": "Use Paged Search", diff --git a/routers/api/v1/user/key.go b/routers/api/v1/user/key.go index aa69245e49..08aa182ca1 100644 --- a/routers/api/v1/user/key.go +++ b/routers/api/v1/user/key.go @@ -211,7 +211,7 @@ func CreateUserPublicKey(ctx *context.APIContext, form api.CreateKeyOption, uid return } - key, err := asymkey_model.AddPublicKey(ctx, uid, form.Title, content, 0) + key, err := asymkey_model.AddPublicKey(ctx, uid, form.Title, content, 0, false) if err != nil { repo.HandleAddKeyError(ctx, err) return diff --git a/routers/web/admin/auths.go b/routers/web/admin/auths.go index fb1a7d9524..3407789f2f 100644 --- a/routers/web/admin/auths.go +++ b/routers/web/admin/auths.go @@ -136,6 +136,7 @@ func parseLDAPConfig(form forms.AuthenticationForm) *ldap.Source { AttributesInBind: form.AttributesInBind, AttributeSSHPublicKey: form.AttributeSSHPublicKey, AttributeAvatar: form.AttributeAvatar, + SSHKeysAreVerified: form.SSHKeysAreVerified, SearchPageSize: pageSize, Filter: form.Filter, GroupsEnabled: form.GroupsEnabled, diff --git a/routers/web/auth/oauth_signin_sync.go b/routers/web/auth/oauth_signin_sync.go index 86d1966024..2f7a8eab58 100644 --- a/routers/web/auth/oauth_signin_sync.go +++ b/routers/web/auth/oauth_signin_sync.go @@ -86,7 +86,7 @@ func oauth2UpdateSSHPubIfNeed(ctx *context.Context, authSource *auth.Source, got if err != nil { return err } - if !asymkey_model.SynchronizePublicKeys(ctx, user, authSource, sshKeys) { + if !asymkey_model.SynchronizePublicKeys(ctx, user, authSource, sshKeys, false) { return nil } return asymkey_service.RewriteAllPublicKeys(ctx) diff --git a/routers/web/user/setting/keys.go b/routers/web/user/setting/keys.go index 13aa4a471b..999bb76683 100644 --- a/routers/web/user/setting/keys.go +++ b/routers/web/user/setting/keys.go @@ -187,7 +187,7 @@ func KeysPost(ctx *context.Context) { return } - if _, err = asymkey_model.AddPublicKey(ctx, ctx.Doer.ID, form.Title, content, 0); err != nil { + if _, err = asymkey_model.AddPublicKey(ctx, ctx.Doer.ID, form.Title, content, 0, false); err != nil { ctx.Data["HasSSHError"] = true switch { case asymkey_model.IsErrKeyAlreadyExist(err): diff --git a/services/asymkey/commit_test.go b/services/asymkey/commit_test.go index 6edba1e90a..2dd08b5dd4 100644 --- a/services/asymkey/commit_test.go +++ b/services/asymkey/commit_test.go @@ -31,7 +31,7 @@ func TestParseCommitWithSSHSignature(t *testing.T) { // AAAEDWqPHTH51xb4hy1y1f1VeWL/2A9Q0b6atOyv5fx8x5prpPrMXSg9qTx04jPNPWRcHs // utyxWjThIpzcaO68yWVnAAAAEXVzZXIyQGV4YW1wbGUuY29tAQIDBA== // -----END OPENSSH PRIVATE KEY----- - sshPubKey, err := asymkey_model.AddPublicKey(t.Context(), 999, "user-ssh-key-any-name", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpPrMXSg9qTx04jPNPWRcHsutyxWjThIpzcaO68yWVn", 0) + sshPubKey, err := asymkey_model.AddPublicKey(t.Context(), 999, "user-ssh-key-any-name", "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILpPrMXSg9qTx04jPNPWRcHsutyxWjThIpzcaO68yWVn", 0, false) require.NoError(t, err) _, err = db.GetEngine(t.Context()).ID(sshPubKey.ID).Cols("verified").Update(&asymkey_model.PublicKey{Verified: true}) require.NoError(t, err) diff --git a/services/asymkey/ssh_key_test.go b/services/asymkey/ssh_key_test.go index 3605bd1e64..b052050dc6 100644 --- a/services/asymkey/ssh_key_test.go +++ b/services/asymkey/ssh_key_test.go @@ -66,7 +66,7 @@ ssh-dss AAAAB3NzaC1kc3MAAACBAOChCC7lf6Uo9n7BmZ6M8St19PZf4Tn59NriyboW2x/DZuYAz3ib for i, kase := range testCases { s.ID = int64(i) + 20 - asymkey_model.AddPublicKeysBySource(t.Context(), user, s, []string{kase.keyString}) + asymkey_model.AddPublicKeysBySource(t.Context(), user, s, []string{kase.keyString}, false) keys, err := db.Find[asymkey_model.PublicKey](t.Context(), asymkey_model.FindPublicKeyOptions{ OwnerID: user.ID, LoginSourceID: s.ID, diff --git a/services/auth/source/ldap/source.go b/services/auth/source/ldap/source.go index 2362cad8aa..81d4b5446b 100644 --- a/services/auth/source/ldap/source.go +++ b/services/auth/source/ldap/source.go @@ -44,6 +44,7 @@ type Source struct { AttributesInBind bool // fetch attributes in bind context (not user) AttributeSSHPublicKey string // LDAP SSH Public Key attribute AttributeAvatar string + SSHKeysAreVerified bool // true if SSH keys in LDAP are verified SearchPageSize uint32 // Search with paging page size Filter string // Query filter to validate entry AdminFilter string // Query filter to check if user is admin diff --git a/services/auth/source/ldap/source_authenticate.go b/services/auth/source/ldap/source_authenticate.go index 4463bcc054..582841aebe 100644 --- a/services/auth/source/ldap/source_authenticate.go +++ b/services/auth/source/ldap/source_authenticate.go @@ -73,7 +73,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u } if user != nil { - if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, user, source.AuthSource, sr.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, user, source.AuthSource, sr.SSHPublicKey, source.SSHKeysAreVerified) { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return user, err } @@ -99,7 +99,7 @@ func (source *Source) Authenticate(ctx context.Context, user *user_model.User, u return user, err } - if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(ctx, user, source.AuthSource, sr.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(ctx, user, source.AuthSource, sr.SSHPublicKey, source.SSHKeysAreVerified) { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return user, err } diff --git a/services/auth/source/ldap/source_sync.go b/services/auth/source/ldap/source_sync.go index 7b401c5c96..0c5fdac674 100644 --- a/services/auth/source/ldap/source_sync.go +++ b/services/auth/source/ldap/source_sync.go @@ -135,7 +135,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { if err == nil && isAttributeSSHPublicKeySet { log.Trace("SyncExternalUsers[%s]: Adding LDAP Public SSH Keys for user %s", source.AuthSource.Name, usr.Name) - if asymkey_model.AddPublicKeysBySource(ctx, usr, source.AuthSource, su.SSHPublicKey) { + if asymkey_model.AddPublicKeysBySource(ctx, usr, source.AuthSource, su.SSHPublicKey, source.SSHKeysAreVerified) { sshKeysNeedUpdate = true } } @@ -145,7 +145,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { } } else if updateExisting { // Synchronize SSH Public Key if that attribute is set - if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, usr, source.AuthSource, su.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(ctx, usr, source.AuthSource, su.SSHPublicKey, source.SSHKeysAreVerified) { sshKeysNeedUpdate = true } diff --git a/services/forms/auth_form.go b/services/forms/auth_form.go index 886110236c..95965b5f29 100644 --- a/services/forms/auth_form.go +++ b/services/forms/auth_form.go @@ -34,6 +34,7 @@ type AuthenticationForm struct { AttributeMail string AttributeSSHPublicKey string AttributeAvatar string + SSHKeysAreVerified bool AttributesInBind bool UsePagedSearch bool SearchPageSize int diff --git a/templates/admin/auth/edit.tmpl b/templates/admin/auth/edit.tmpl index d29a52b76b..56f9e1b9cd 100644 --- a/templates/admin/auth/edit.tmpl +++ b/templates/admin/auth/edit.tmpl @@ -112,6 +112,12 @@ +
+
+ + +
+
diff --git a/templates/admin/auth/source/ldap.tmpl b/templates/admin/auth/source/ldap.tmpl index 9754aed55a..e5852daa3d 100644 --- a/templates/admin/auth/source/ldap.tmpl +++ b/templates/admin/auth/source/ldap.tmpl @@ -80,6 +80,12 @@
+
+
+ + +
+
From 83527d3f8a8f2bca8f6105188a6c2b11620c91d2 Mon Sep 17 00:00:00 2001 From: Gregorius Bima Kharisma Wicaksana <51526537+bimakw@users.noreply.github.com> Date: Sun, 28 Dec 2025 00:05:24 +0700 Subject: [PATCH 3/8] Support closing keywords with URL references (#36221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR adds support for closing keywords (`closes`, `fixes`, `reopens`, etc.) with full URL references in markdown links. **Before:** - `closes #123` ✅ works - `closes org/repo#123` ✅ works - `Closes [this issue](https://gitea.io/user/repo/issues/123)` ❌ didn't work - `Fixes [#456](https://gitea.io/org/project/issues/456)` ❌ didn't work **After:** All of the above now work correctly. ## Problem When users reference issues using full URLs in markdown links (e.g., `Closes [this issue](https://gitea.io/user/repo/issues/123)`), the closing keywords were not detected. This was because the URL processing code explicitly stated: ```go // Note: closing/reopening keywords not supported with URLs ``` Both methods of writing the reference render the same in the UI, so users expected the closing keywords to behave the same. ## Solution The fix works by: 1. Passing the original (unstripped) content to `findAllIssueReferencesBytes` 2. When processing URL links from markdown, finding the URL position in the original content 3. For markdown links `[text](url)`, finding the opening bracket `[` position 4. Using that position to detect closing keywords before the link ## Testing Added test cases for: - `Closes [this issue](url)` - single URL with closing keyword - `This fixes [#456](url)` - keyword in middle of text - `Reopens [PR](url)` - reopen keyword with pull request URL - Multiple URLs where only one has a closing keyword All existing tests continue to pass. Fixes #27549 --- modules/references/references.go | 30 +++++++++++--- modules/references/references_test.go | 56 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/modules/references/references.go b/modules/references/references.go index 592bd4cbe4..ef3568ebea 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -248,7 +248,7 @@ func FindAllIssueReferencesMarkdown(content string) []IssueReference { func findAllIssueReferencesMarkdown(content string) []*rawReference { bcontent, links := mdstripper.StripMarkdownBytes([]byte(content)) - return findAllIssueReferencesBytes(bcontent, links) + return findAllIssueReferencesBytes(bcontent, links, []byte(content)) } func convertFullHTMLReferencesToShortRefs(re *regexp.Regexp, contentBytes *[]byte) { @@ -326,7 +326,7 @@ func FindAllIssueReferences(content string) []IssueReference { } else { log.Debug("No GiteaIssuePullPattern pattern") } - return rawToIssueReferenceList(findAllIssueReferencesBytes(contentBytes, []string{})) + return rawToIssueReferenceList(findAllIssueReferencesBytes(contentBytes, []string{}, nil)) } // FindRenderizableReferenceNumeric returns the first unvalidated reference found in a string. @@ -406,7 +406,8 @@ func FindRenderizableReferenceAlphanumeric(content string) *RenderizableReferenc } // FindAllIssueReferencesBytes returns a list of unvalidated references found in a byte slice. -func findAllIssueReferencesBytes(content []byte, links []string) []*rawReference { +// originalContent is optional and used to detect closing/reopening keywords for URL references. +func findAllIssueReferencesBytes(content []byte, links []string, originalContent []byte) []*rawReference { ret := make([]*rawReference, 0, 10) pos := 0 @@ -470,10 +471,27 @@ func findAllIssueReferencesBytes(content []byte, links []string) []*rawReference default: continue } - // Note: closing/reopening keywords not supported with URLs - bytes := []byte(parts[1] + "/" + parts[2] + sep + parts[4]) - if ref := getCrossReference(bytes, 0, len(bytes), true, false); ref != nil { + refBytes := []byte(parts[1] + "/" + parts[2] + sep + parts[4]) + if ref := getCrossReference(refBytes, 0, len(refBytes), true, false); ref != nil { ref.refLocation = nil + // Detect closing/reopening keywords by finding the URL position in original content + if originalContent != nil { + if idx := bytes.Index(originalContent, []byte(link)); idx > 0 { + // For markdown links [text](url), find the opening bracket before the URL + // to properly detect keywords like "closes [text](url)" + searchStart := idx + if idx >= 2 && originalContent[idx-1] == '(' { + // Find the matching '[' for this markdown link + bracketIdx := bytes.LastIndex(originalContent[:idx-1], []byte{'['}) + if bracketIdx >= 0 { + searchStart = bracketIdx + } + } + action, location := findActionKeywords(originalContent, searchStart) + ref.action = action + ref.actionLocation = location + } + } ret = append(ret, ref) } } diff --git a/modules/references/references_test.go b/modules/references/references_test.go index a15ae99f79..5922a9f5a9 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -227,6 +227,62 @@ func TestFindAllIssueReferences(t *testing.T) { testFixtures(t, fixtures, "default") + // Test closing/reopening keywords with URLs (issue #27549) + // Uses the same AppURL as testFixtures (https://gitea.com:3000/) + urlFixtures := []testFixture{ + { + "Closes [this issue](https://gitea.com:3000/user/repo/issues/123)", + []testResult{ + {123, "user", "repo", "123", false, XRefActionCloses, nil, &RefSpan{Start: 0, End: 6}, ""}, + }, + }, + { + "This fixes [#456](https://gitea.com:3000/org/project/issues/456)", + []testResult{ + {456, "org", "project", "456", false, XRefActionCloses, nil, &RefSpan{Start: 5, End: 10}, ""}, + }, + }, + { + "Reopens [PR](https://gitea.com:3000/owner/repo/pulls/789)", + []testResult{ + {789, "owner", "repo", "789", true, XRefActionReopens, nil, &RefSpan{Start: 0, End: 7}, ""}, + }, + }, + { + "See [issue](https://gitea.com:3000/user/repo/issues/100) but closes [another](https://gitea.com:3000/user/repo/issues/200)", + []testResult{ + {100, "user", "repo", "100", false, XRefActionNone, nil, nil, ""}, + {200, "user", "repo", "200", false, XRefActionCloses, nil, &RefSpan{Start: 61, End: 67}, ""}, + }, + }, + } + + testFixtures(t, urlFixtures, "url-keywords") + + // Test bare URLs (not markdown links) with closing keywords + // These use FindAllIssueReferences (non-markdown) which converts full URLs to short refs first + setting.AppURL = "https://gitea.com:3000/" + bareURLTests := []struct { + name string + input string + expected XRefAction + }{ + {"Fixes bare URL", "Fixes https://gitea.com:3000/org/project/issues/456", XRefActionCloses}, + {"Fixes with colon", "Fixes: https://gitea.com:3000/org/project/issues/456", XRefActionCloses}, + {"Closes bare URL", "Closes https://gitea.com:3000/user/repo/issues/123", XRefActionCloses}, + {"Closes with colon", "Closes: https://gitea.com:3000/user/repo/issues/123", XRefActionCloses}, + } + + for _, tt := range bareURLTests { + t.Run(tt.name, func(t *testing.T) { + refs := FindAllIssueReferences(tt.input) + assert.Len(t, refs, 1, "Expected 1 reference for: %s", tt.input) + if len(refs) > 0 { + assert.Equal(t, tt.expected, refs[0].Action, "Expected action %v for: %s", tt.expected, tt.input) + } + }) + } + type alnumFixture struct { input string issue string From c7b3cdf7b184f6baed50849eeaaf5d0cc79954a3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 28 Dec 2025 03:24:28 -0800 Subject: [PATCH 4/8] Use gitrepo's push function (#36245) extract from #36186 --- modules/git/repo.go | 9 ++++++++- services/repository/generate.go | 23 ----------------------- services/repository/init.go | 14 ++++++++------ 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/modules/git/repo.go b/modules/git/repo.go index 579accf92e..bea599e22e 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -186,6 +186,7 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { // PushOptions options when push to remote type PushOptions struct { Remote string + LocalRefName string Branch string Force bool ForceWithLease string @@ -207,7 +208,13 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { } remoteBranchArgs := []string{opts.Remote} if len(opts.Branch) > 0 { - remoteBranchArgs = append(remoteBranchArgs, opts.Branch) + var refspec string + if opts.LocalRefName != "" { + refspec = fmt.Sprintf("%s:%s", opts.LocalRefName, opts.Branch) + } else { + refspec = opts.Branch + } + remoteBranchArgs = append(remoteBranchArgs, refspec) } cmd.AddDashesAndList(remoteBranchArgs...) diff --git a/services/repository/generate.go b/services/repository/generate.go index b2913cd110..bc37bc7bfe 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -21,7 +21,6 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/glob" "code.gitea.io/gitea/modules/log" @@ -216,19 +215,6 @@ func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, } func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error { - commitTimeStr := time.Now().Format(time.RFC3339) - authorSig := repo.Owner.NewGitSig() - - // Because this may call hooks we should pass in the environment - env := append(os.Environ(), - "GIT_AUTHOR_NAME="+authorSig.Name, - "GIT_AUTHOR_EMAIL="+authorSig.Email, - "GIT_AUTHOR_DATE="+commitTimeStr, - "GIT_COMMITTER_NAME="+authorSig.Name, - "GIT_COMMITTER_EMAIL="+authorSig.Email, - "GIT_COMMITTER_DATE="+commitTimeStr, - ) - // Clone to temporary path and do the init commit. if err := gitrepo.CloneRepoToLocal(ctx, templateRepo, tmpDir, git.CloneRepoOptions{ Depth: 1, @@ -264,15 +250,6 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return err } - if stdout, _, err := gitcmd.NewCommand("remote", "add", "origin"). - AddDynamicArguments(repo.RepoPath()). - WithDir(tmpDir). - WithEnv(env). - RunStdString(ctx); err != nil { - log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err) - return fmt.Errorf("git remote add: %w", err) - } - if err = git.AddTemplateSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { return fmt.Errorf("failed to add submodules: %v", err) } diff --git a/services/repository/init.go b/services/repository/init.go index 51cc113d63..6aeb5ec644 100644 --- a/services/repository/init.go +++ b/services/repository/init.go @@ -11,7 +11,9 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -71,12 +73,12 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi defaultBranch = setting.Repository.DefaultBranch } - if stdout, _, err := gitcmd.NewCommand("push", "origin"). - AddDynamicArguments("HEAD:" + defaultBranch). - WithDir(tmpPath). - WithEnv(repo_module.InternalPushingEnvironment(u, repo)). - RunStdString(ctx); err != nil { - log.Error("Failed to push back to HEAD: Stdout: %s\nError: %v", stdout, err) + if err := gitrepo.PushFromLocal(ctx, tmpPath, repo, git.PushOptions{ + LocalRefName: "HEAD", + Branch: defaultBranch, + Env: repo_module.InternalPushingEnvironment(u, repo), + }); err != nil { + log.Error("Failed to push back to HEAD Error: %v", err) return fmt.Errorf("git push: %w", err) } From 85dd16b3fc573d292137ef41dca61ad8dbf5b75f Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Mon, 29 Dec 2025 00:43:22 +0000 Subject: [PATCH 5/8] [skip ci] Updated translations via Crowdin --- options/locale/locale_zh-CN.json | 1 + 1 file changed, 1 insertion(+) diff --git a/options/locale/locale_zh-CN.json b/options/locale/locale_zh-CN.json index 2d031d2d58..9b4309ecea 100644 --- a/options/locale/locale_zh-CN.json +++ b/options/locale/locale_zh-CN.json @@ -3067,6 +3067,7 @@ "admin.auths.attribute_mail": "电子邮箱属性", "admin.auths.attribute_ssh_public_key": "SSH公钥属性", "admin.auths.attribute_avatar": "头像属性", + "admin.auths.ssh_keys_are_verified": "LDAP中的 SSH 密钥被视为已验证", "admin.auths.attributes_in_bind": "从 Bind DN 中拉取属性信息", "admin.auths.allow_deactivate_all": "允许在搜索结果为空时停用所有用户", "admin.auths.use_paged_search": "使用分页搜索", From d0cb198c8974921b2ed5f8c399dba92718d5fa74 Mon Sep 17 00:00:00 2001 From: alphazeba <33792307+alphazeba@users.noreply.github.com> Date: Sun, 28 Dec 2025 23:51:10 -0800 Subject: [PATCH 6/8] fix: prevent 100% width radio buttons (#36262) as part of [Remove fomantic form module](https://github.com/go-gitea/gitea/commit/eddf8759926911c465b249de5f6d68c052a539e0#diff-c34b74004deb63fb4f8a8549ef9d822b9839db0b69ae2c0cdacc05ce3d5d5682) radio buttons get caught in crossfire and recieve `width: 100%` this is particularly noticeable on the `user/settings/applications` page which has many radio buttons. This continues using an opt out `input:not([type="checkbox"], [type="radio"])` to prevent this. Signed-off-by: alphazeba <33792307+alphazeba@users.noreply.github.com> --- web_src/css/modules/form.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/css/modules/form.css b/web_src/css/modules/form.css index 49e4012306..09d247cf17 100644 --- a/web_src/css/modules/form.css +++ b/web_src/css/modules/form.css @@ -33,7 +33,7 @@ } .ui.form textarea, -.ui.form input { +.ui.form input:not([type="checkbox"], [type="radio"]) { width: 100%; vertical-align: top; } From 0ad94dfc70a7d45169989cac499f809b7df5a25e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 29 Dec 2025 10:19:42 -0800 Subject: [PATCH 7/8] Move catfile batch to a sub package of git module (#36232) --- modules/git/batch.go | 47 --- modules/git/batch_reader.go | 323 ++---------------- modules/git/blob_nogogit.go | 11 +- modules/git/catfile/batch.go | 178 ++++++++++ modules/git/catfile/reader.go | 211 ++++++++++++ .../languagestats/language_stats_nogogit.go | 5 +- modules/git/pipeline/lfs_nogogit.go | 5 +- modules/git/repo_base_nogogit.go | 34 +- modules/git/repo_branch_nogogit.go | 12 +- modules/git/repo_commit_nogogit.go | 29 +- modules/git/repo_tag_nogogit.go | 11 +- modules/git/repo_tree_nogogit.go | 6 +- modules/git/tree_entry_nogogit.go | 6 +- modules/git/tree_nogogit.go | 4 +- modules/gitrepo/cat_file.go | 6 +- modules/indexer/code/bleve/bleve.go | 15 +- .../code/elasticsearch/elasticsearch.go | 9 +- 17 files changed, 500 insertions(+), 412 deletions(-) delete mode 100644 modules/git/batch.go create mode 100644 modules/git/catfile/batch.go create mode 100644 modules/git/catfile/reader.go diff --git a/modules/git/batch.go b/modules/git/batch.go deleted file mode 100644 index f9e1748b54..0000000000 --- a/modules/git/batch.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bufio" - "context" -) - -type Batch struct { - cancel context.CancelFunc - Reader *bufio.Reader - Writer WriteCloserError -} - -// NewBatch creates a new batch for the given repository, the Close must be invoked before release the batch -func NewBatch(ctx context.Context, repoPath string) (*Batch, error) { - // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repoPath); err != nil { - return nil, err - } - - var batch Batch - batch.Writer, batch.Reader, batch.cancel = catFileBatch(ctx, repoPath) - return &batch, nil -} - -func NewBatchCheck(ctx context.Context, repoPath string) (*Batch, error) { - // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := ensureValidGitRepository(ctx, repoPath); err != nil { - return nil, err - } - - var check Batch - check.Writer, check.Reader, check.cancel = catFileBatchCheck(ctx, repoPath) - return &check, nil -} - -func (b *Batch) Close() { - if b.cancel != nil { - b.cancel() - b.Reader = nil - b.Writer = nil - b.cancel = nil - } -} diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index b5cec130d5..3d612f5549 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -5,320 +5,53 @@ package git import ( "bufio" - "bytes" - "context" - "io" - "math" - "strconv" - "strings" + "errors" - "code.gitea.io/gitea/modules/git/gitcmd" - "code.gitea.io/gitea/modules/log" - - "github.com/djherbis/buffer" - "github.com/djherbis/nio/v3" + "code.gitea.io/gitea/modules/git/catfile" ) -// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function -type WriteCloserError interface { - io.WriteCloser - CloseWithError(err error) error -} - -// ensureValidGitRepository runs git rev-parse in the repository path - thus ensuring that the repository is a valid repository. -// Run before opening git cat-file. -// This is needed otherwise the git cat-file will hang for invalid repositories. -func ensureValidGitRepository(ctx context.Context, repoPath string) error { - stderr := strings.Builder{} - err := gitcmd.NewCommand("rev-parse"). - WithDir(repoPath). - WithStderr(&stderr). - Run(ctx) - if err != nil { - return gitcmd.ConcatenateError(err, (&stderr).String()) - } - return nil -} - -// catFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, a stdout reader and cancel function -func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { - batchStdinReader, batchStdinWriter := io.Pipe() - batchStdoutReader, batchStdoutWriter := io.Pipe() - ctx, ctxCancel := context.WithCancel(ctx) - closed := make(chan struct{}) - cancel := func() { - ctxCancel() - _ = batchStdoutReader.Close() - _ = batchStdinWriter.Close() - <-closed - } - - // Ensure cancel is called as soon as the provided context is cancelled - go func() { - <-ctx.Done() - cancel() - }() - - go func() { - stderr := strings.Builder{} - err := gitcmd.NewCommand("cat-file", "--batch-check"). - WithDir(repoPath). - WithStdin(batchStdinReader). - WithStdout(batchStdoutWriter). - WithStderr(&stderr). - WithUseContextTimeout(true). - Run(ctx) - if err != nil { - _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - } else { - _ = batchStdoutWriter.Close() - _ = batchStdinReader.Close() - } - close(closed) - }() - - // For simplicities sake we'll use a buffered reader to read from the cat-file --batch-check - batchReader := bufio.NewReader(batchStdoutReader) - - return batchStdinWriter, batchReader, cancel -} - -// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, a stdout reader and cancel function -func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { - // We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. - // so let's create a batch stdin and stdout - batchStdinReader, batchStdinWriter := io.Pipe() - batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024)) - ctx, ctxCancel := context.WithCancel(ctx) - closed := make(chan struct{}) - cancel := func() { - ctxCancel() - _ = batchStdinWriter.Close() - _ = batchStdoutReader.Close() - <-closed - } - - // Ensure cancel is called as soon as the provided context is cancelled - go func() { - <-ctx.Done() - cancel() - }() - - go func() { - stderr := strings.Builder{} - err := gitcmd.NewCommand("cat-file", "--batch"). - WithDir(repoPath). - WithStdin(batchStdinReader). - WithStdout(batchStdoutWriter). - WithStderr(&stderr). - WithUseContextTimeout(true). - Run(ctx) - if err != nil { - _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, (&stderr).String())) - } else { - _ = batchStdoutWriter.Close() - _ = batchStdinReader.Close() - } - close(closed) - }() - - // For simplicities sake we'll us a buffered reader to read from the cat-file --batch - batchReader := bufio.NewReaderSize(batchStdoutReader, 32*1024) - - return batchStdinWriter, batchReader, cancel -} - -// ReadBatchLine reads the header line from cat-file --batch -// We expect: SP SP LF -// then leaving the rest of the stream " LF" to be read +// ReadBatchLine reads the header line from cat-file --batch while preserving the traditional return signature. func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { - typ, err = rd.ReadString('\n') - if err != nil { - return sha, typ, size, err - } - if len(typ) == 1 { - typ, err = rd.ReadString('\n') - if err != nil { - return sha, typ, size, err - } - } - idx := strings.IndexByte(typ, ' ') - if idx < 0 { - log.Debug("missing space typ: %s", typ) - return sha, typ, size, ErrNotExist{ID: string(sha)} - } - sha = []byte(typ[:idx]) - typ = typ[idx+1:] - - idx = strings.IndexByte(typ, ' ') - if idx < 0 { - return sha, typ, size, ErrNotExist{ID: string(sha)} - } - - sizeStr := typ[idx+1 : len(typ)-1] - typ = typ[:idx] - - size, err = strconv.ParseInt(sizeStr, 10, 64) - return sha, typ, size, err + sha, typ, size, err = catfile.ReadBatchLine(rd) + return sha, typ, size, convertCatfileError(err, sha) } // ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream. func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) { - var id string - var n int64 -headerLoop: - for { - line, err := rd.ReadBytes('\n') - if err != nil { - return "", err - } - n += int64(len(line)) - idx := bytes.Index(line, []byte{' '}) - if idx < 0 { - continue - } - - if string(line[:idx]) == "object" { - id = string(line[idx+1 : len(line)-1]) - break headerLoop - } - } - - // Discard the rest of the tag - return id, DiscardFull(rd, size-n+1) + return catfile.ReadTagObjectID(rd, size) } // ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream. func ReadTreeID(rd *bufio.Reader, size int64) (string, error) { - var id string - var n int64 -headerLoop: - for { - line, err := rd.ReadBytes('\n') - if err != nil { - return "", err - } - n += int64(len(line)) - idx := bytes.Index(line, []byte{' '}) - if idx < 0 { - continue - } - - if string(line[:idx]) == "tree" { - id = string(line[idx+1 : len(line)-1]) - break headerLoop - } - } - - // Discard the rest of the commit - return id, DiscardFull(rd, size-n+1) + return catfile.ReadTreeID(rd, size) } -// git tree files are a list: -// SP NUL -// -// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools -// Therefore we need some method to convert these binary hashes to hex hashes - -// constant hextable to help quickly convert between binary and hex representation -const hextable = "0123456789abcdef" - -// BinToHexHeash converts a binary Hash into a hex encoded one. Input and output can be the -// same byte slice to support in place conversion without allocations. -// This is at least 100x quicker that hex.EncodeToString +// BinToHex converts a binary hash into a hex encoded one. func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { - for i := objectFormat.FullLength()/2 - 1; i >= 0; i-- { - v := sha[i] - vhi, vlo := v>>4, v&0x0f - shi, slo := hextable[vhi], hextable[vlo] - out[i*2], out[i*2+1] = shi, slo - } - return out + return catfile.BinToHex(objectFormat, sha, out) } -// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream -// This carefully avoids allocations - except where fnameBuf is too small. -// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations -// -// Each line is composed of: -// SP NUL -// -// We don't attempt to convert the raw HASH to save a lot of time +// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream. func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { - var readBytes []byte - - // Read the Mode & fname - readBytes, err = rd.ReadSlice('\x00') - if err != nil { - return mode, fname, sha, n, err - } - idx := bytes.IndexByte(readBytes, ' ') - if idx < 0 { - log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes) - return mode, fname, sha, n, &ErrNotExist{} - } - - n += idx + 1 - copy(modeBuf, readBytes[:idx]) - if len(modeBuf) >= idx { - modeBuf = modeBuf[:idx] - } else { - modeBuf = append(modeBuf, readBytes[len(modeBuf):idx]...) - } - mode = modeBuf - - readBytes = readBytes[idx+1:] - - // Deal with the fname - copy(fnameBuf, readBytes) - if len(fnameBuf) > len(readBytes) { - fnameBuf = fnameBuf[:len(readBytes)] - } else { - fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) - } - for err == bufio.ErrBufferFull { - readBytes, err = rd.ReadSlice('\x00') - fnameBuf = append(fnameBuf, readBytes...) - } - n += len(fnameBuf) - if err != nil { - return mode, fname, sha, n, err - } - fnameBuf = fnameBuf[:len(fnameBuf)-1] - fname = fnameBuf - - // Deal with the binary hash - idx = 0 - length := objectFormat.FullLength() / 2 - for idx < length { - var read int - read, err = rd.Read(shaBuf[idx:length]) - n += read - if err != nil { - return mode, fname, sha, n, err - } - idx += read - } - sha = shaBuf - return mode, fname, sha, n, err + mode, fname, sha, n, err = catfile.ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf) + return mode, fname, sha, n, convertCatfileError(err, nil) } +// DiscardFull discards the requested number of bytes from the buffered reader. func DiscardFull(rd *bufio.Reader, discard int64) error { - if discard > math.MaxInt32 { - n, err := rd.Discard(math.MaxInt32) - discard -= int64(n) - if err != nil { - return err - } - } - for discard > 0 { - n, err := rd.Discard(int(discard)) - discard -= int64(n) - if err != nil { - return err - } - } - return nil + return catfile.DiscardFull(rd, discard) +} + +func convertCatfileError(err error, defaultID []byte) error { + if err == nil { + return nil + } + var notFound catfile.ErrObjectNotFound + if errors.As(err, ¬Found) { + if notFound.ID == "" && len(defaultID) > 0 { + notFound.ID = string(defaultID) + } + return ErrNotExist{ID: notFound.ID} + } + return err } diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go index af3ce376d6..88e2be792b 100644 --- a/modules/git/blob_nogogit.go +++ b/modules/git/blob_nogogit.go @@ -26,12 +26,13 @@ type Blob struct { // DataAsync gets a ReadCloser for the contents of a blob without reading it all. // Calling the Close function on the result will discard all unread output. func (b *Blob) DataAsync() (io.ReadCloser, error) { - wr, rd, cancel, err := b.repo.CatFileBatch(b.repo.Ctx) + batch, cancel, err := b.repo.CatFileBatch(b.repo.Ctx) if err != nil { return nil, err } - _, err = wr.Write([]byte(b.ID.String() + "\n")) + rd := batch.Reader() + _, err = batch.Writer().Write([]byte(b.ID.String() + "\n")) if err != nil { cancel() return nil, err @@ -67,18 +68,18 @@ func (b *Blob) Size() int64 { return b.size } - wr, rd, cancel, err := b.repo.CatFileBatchCheck(b.repo.Ctx) + batch, cancel, err := b.repo.CatFileBatchCheck(b.repo.Ctx) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) return 0 } defer cancel() - _, err = wr.Write([]byte(b.ID.String() + "\n")) + _, err = batch.Writer().Write([]byte(b.ID.String() + "\n")) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) return 0 } - _, _, b.size, err = ReadBatchLine(rd) + _, _, b.size, err = ReadBatchLine(batch.Reader()) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) return 0 diff --git a/modules/git/catfile/batch.go b/modules/git/catfile/batch.go new file mode 100644 index 0000000000..1facb8946e --- /dev/null +++ b/modules/git/catfile/batch.go @@ -0,0 +1,178 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package catfile + +import ( + "bufio" + "context" + "io" + "strings" + + "code.gitea.io/gitea/modules/git/gitcmd" + + "github.com/djherbis/buffer" + "github.com/djherbis/nio/v3" +) + +// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function +type WriteCloserError interface { + io.WriteCloser + CloseWithError(err error) error +} + +type Batch interface { + Writer() WriteCloserError + Reader() *bufio.Reader + Close() +} + +// batch represents an active `git cat-file --batch` or `--batch-check` invocation +// paired with the pipes that feed/read from it. Call Close to release resources. +type batch struct { + cancel context.CancelFunc + reader *bufio.Reader + writer WriteCloserError +} + +// NewBatch creates a new cat-file --batch process for the provided repository path. +// The returned Batch must be closed once the caller has finished with it. +func NewBatch(ctx context.Context, repoPath string) (Batch, error) { + if err := EnsureValidGitRepository(ctx, repoPath); err != nil { + return nil, err + } + + var batch batch + batch.writer, batch.reader, batch.cancel = catFileBatch(ctx, repoPath) + return &batch, nil +} + +// NewBatchCheck creates a cat-file --batch-check process for the provided repository path. +// The returned Batch must be closed once the caller has finished with it. +func NewBatchCheck(ctx context.Context, repoPath string) (Batch, error) { + if err := EnsureValidGitRepository(ctx, repoPath); err != nil { + return nil, err + } + + var check batch + check.writer, check.reader, check.cancel = catFileBatchCheck(ctx, repoPath) + return &check, nil +} + +func (b *batch) Writer() WriteCloserError { + return b.writer +} + +func (b *batch) Reader() *bufio.Reader { + return b.reader +} + +// Close stops the underlying git cat-file process and releases held resources. +func (b *batch) Close() { + if b == nil || b.cancel == nil { + return + } + b.cancel() + b.reader = nil + b.writer = nil + b.cancel = nil +} + +// EnsureValidGitRepository runs `git rev-parse` in the repository path to make sure +// the directory is a valid git repository. This avoids git cat-file hanging indefinitely +// when invoked in invalid paths. +func EnsureValidGitRepository(ctx context.Context, repoPath string) error { + stder := strings.Builder{} + err := gitcmd.NewCommand("rev-parse"). + WithDir(repoPath). + WithStderr(&stder). + Run(ctx) + if err != nil { + return gitcmd.ConcatenateError(err, stder.String()) + } + return nil +} + +// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe, +// a stdout reader and a cancel function. +func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024)) + ctx, ctxCancel := context.WithCancel(ctx) + closed := make(chan struct{}) + cancel := func() { + ctxCancel() + _ = batchStdinWriter.Close() + _ = batchStdoutReader.Close() + <-closed + } + + go func() { + <-ctx.Done() + cancel() + }() + + go func() { + stder := strings.Builder{} + err := gitcmd.NewCommand("cat-file", "--batch"). + WithDir(repoPath). + WithStdin(batchStdinReader). + WithStdout(batchStdoutWriter). + WithStderr(&stder). + WithUseContextTimeout(true). + Run(ctx) + if err != nil { + _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + } else { + _ = batchStdoutWriter.Close() + _ = batchStdinReader.Close() + } + close(closed) + }() + + batchReader := bufio.NewReaderSize(batchStdoutReader, 32*1024) + return batchStdinWriter, batchReader, cancel +} + +// catFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe, +// a stdout reader and cancel function. +func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) { + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := io.Pipe() + ctx, ctxCancel := context.WithCancel(ctx) + closed := make(chan struct{}) + cancel := func() { + ctxCancel() + _ = batchStdoutReader.Close() + _ = batchStdinWriter.Close() + <-closed + } + + go func() { + <-ctx.Done() + cancel() + }() + + go func() { + stder := strings.Builder{} + err := gitcmd.NewCommand("cat-file", "--batch-check"). + WithDir(repoPath). + WithStdin(batchStdinReader). + WithStdout(batchStdoutWriter). + WithStderr(&stder). + WithUseContextTimeout(true). + Run(ctx) + if err != nil { + _ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + _ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, stder.String())) + } else { + _ = batchStdoutWriter.Close() + _ = batchStdinReader.Close() + } + close(closed) + }() + + batchReader := bufio.NewReader(batchStdoutReader) + return batchStdinWriter, batchReader, cancel +} diff --git a/modules/git/catfile/reader.go b/modules/git/catfile/reader.go new file mode 100644 index 0000000000..1785cc4cc0 --- /dev/null +++ b/modules/git/catfile/reader.go @@ -0,0 +1,211 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package catfile + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "math" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/log" +) + +// ErrObjectNotFound indicates that the requested object could not be read from cat-file +type ErrObjectNotFound struct { + ID string +} + +func (err ErrObjectNotFound) Error() string { + return fmt.Sprintf("catfile: object does not exist [id: %s]", err.ID) +} + +// IsErrObjectNotFound reports whether err is an ErrObjectNotFound +func IsErrObjectNotFound(err error) bool { + var target ErrObjectNotFound + return errors.As(err, &target) +} + +// ObjectFormat abstracts the minimal information needed from git.ObjectFormat. +type ObjectFormat interface { + FullLength() int +} + +// ReadBatchLine reads the header line from cat-file --batch. It expects the format +// " SP SP LF" and leaves the reader positioned at the start of +// the object contents (which must be fully consumed by the caller). +func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { + typ, err = rd.ReadString('\n') + if err != nil { + return sha, typ, size, err + } + if len(typ) == 1 { + typ, err = rd.ReadString('\n') + if err != nil { + return sha, typ, size, err + } + } + idx := strings.IndexByte(typ, ' ') + if idx < 0 { + return sha, typ, size, ErrObjectNotFound{} + } + sha = []byte(typ[:idx]) + typ = typ[idx+1:] + + idx = strings.IndexByte(typ, ' ') + if idx < 0 { + return sha, typ, size, ErrObjectNotFound{ID: string(sha)} + } + + sizeStr := typ[idx+1 : len(typ)-1] + typ = typ[:idx] + + size, err = strconv.ParseInt(sizeStr, 10, 64) + return sha, typ, size, err +} + +// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest. +func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) { + var id string + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "object" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + return id, DiscardFull(rd, size-n+1) +} + +// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the commit content. +func ReadTreeID(rd *bufio.Reader, size int64) (string, error) { + var id string + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "tree" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + return id, DiscardFull(rd, size-n+1) +} + +// hextable helps quickly convert between binary and hex representation +const hextable = "0123456789abcdef" + +// BinToHex converts a binary hash into a hex encoded one. Input and output can be the +// same byte slice to support in-place conversion without allocations. +func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte { + for i := objectFormat.FullLength()/2 - 1; i >= 0; i-- { + v := sha[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + out[i*2], out[i*2+1] = shi, slo + } + return out +} + +// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream and avoids allocations +// where possible. Each line is composed of: +// SP NUL +func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { + var readBytes []byte + + readBytes, err = rd.ReadSlice('\x00') + if err != nil { + return mode, fname, sha, n, err + } + idx := bytes.IndexByte(readBytes, ' ') + if idx < 0 { + log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes) + return mode, fname, sha, n, ErrObjectNotFound{} + } + + n += idx + 1 + copy(modeBuf, readBytes[:idx]) + if len(modeBuf) >= idx { + modeBuf = modeBuf[:idx] + } else { + modeBuf = append(modeBuf, readBytes[len(modeBuf):idx]...) + } + mode = modeBuf + + readBytes = readBytes[idx+1:] + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) + } + for err == bufio.ErrBufferFull { + readBytes, err = rd.ReadSlice('\x00') + fnameBuf = append(fnameBuf, readBytes...) + } + n += len(fnameBuf) + if err != nil { + return mode, fname, sha, n, err + } + fnameBuf = fnameBuf[:len(fnameBuf)-1] + fname = fnameBuf + + idx = 0 + length := objectFormat.FullLength() / 2 + for idx < length { + var read int + read, err = rd.Read(shaBuf[idx:length]) + n += read + if err != nil { + return mode, fname, sha, n, err + } + idx += read + } + sha = shaBuf + return mode, fname, sha, n, err +} + +// DiscardFull discards the requested amount of bytes from the buffered reader regardless of its internal limit. +func DiscardFull(rd *bufio.Reader, discard int64) error { + if discard > math.MaxInt32 { + n, err := rd.Discard(math.MaxInt32) + discard -= int64(n) + if err != nil { + return err + } + } + for discard > 0 { + n, err := rd.Discard(int(discard)) + discard -= int64(n) + if err != nil { + return err + } + } + return nil +} diff --git a/modules/git/languagestats/language_stats_nogogit.go b/modules/git/languagestats/language_stats_nogogit.go index 94cf9fff8c..da291ae848 100644 --- a/modules/git/languagestats/language_stats_nogogit.go +++ b/modules/git/languagestats/language_stats_nogogit.go @@ -22,20 +22,21 @@ import ( func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // so let's create a batch stdin and stdout - batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() writeID := func(id string) error { - _, err := batchStdinWriter.Write([]byte(id + "\n")) + _, err := batch.Writer().Write([]byte(id + "\n")) return err } if err := writeID(commitID); err != nil { return nil, err } + batchReader := batch.Reader() shaBytes, typ, size, err := git.ReadBatchLine(batchReader) if typ != "commit" { log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index 4881a2be64..6f1a860c1d 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -47,12 +47,15 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err // Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. // so let's create a batch stdin and stdout - batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() + batchStdinWriter := batch.Writer() + batchReader := batch.Reader() + // We'll use a scanner for the revList because it's simpler than a bufio.Reader scan := bufio.NewScanner(revListReader) trees := [][]byte{} diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 17c71da5ef..97a43b90fd 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -7,10 +7,10 @@ package git import ( - "bufio" "context" "path/filepath" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -24,10 +24,10 @@ type Repository struct { tagCache *ObjectCache[*Tag] batchInUse bool - batch *Batch + batch catfile.Batch checkInUse bool - check *Batch + check catfile.Batch Ctx context.Context LastCommitCache *LastCommitCache @@ -57,53 +57,53 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { } // CatFileBatch obtains a CatFileBatch for this repository -func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { +func (repo *Repository) CatFileBatch(ctx context.Context) (catfile.Batch, func(), error) { if repo.batch == nil { var err error - repo.batch, err = NewBatch(ctx, repo.Path) + repo.batch, err = catfile.NewBatch(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } } if !repo.batchInUse { repo.batchInUse = true - return repo.batch.Writer, repo.batch.Reader, func() { + return repo.batch, func() { repo.batchInUse = false }, nil } log.Debug("Opening temporary cat file batch for: %s", repo.Path) - tempBatch, err := NewBatch(ctx, repo.Path) + tempBatch, err := catfile.NewBatch(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return tempBatch.Writer, tempBatch.Reader, tempBatch.Close, nil + return tempBatch, tempBatch.Close, nil } // CatFileBatchCheck obtains a CatFileBatchCheck for this repository -func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func(), error) { +func (repo *Repository) CatFileBatchCheck(ctx context.Context) (catfile.Batch, func(), error) { if repo.check == nil { var err error - repo.check, err = NewBatchCheck(ctx, repo.Path) + repo.check, err = catfile.NewBatchCheck(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } } if !repo.checkInUse { repo.checkInUse = true - return repo.check.Writer, repo.check.Reader, func() { + return repo.check, func() { repo.checkInUse = false }, nil } log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) - tempBatchCheck, err := NewBatchCheck(ctx, repo.Path) + tempBatchCheck, err := catfile.NewBatchCheck(ctx, repo.Path) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return tempBatchCheck.Writer, tempBatchCheck.Reader, tempBatchCheck.Close, nil + return tempBatchCheck, tempBatchCheck.Close, nil } func (repo *Repository) Close() error { diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index f1b26b06ab..09873fb2c6 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -23,18 +23,18 @@ func (repo *Repository) IsObjectExist(name string) bool { return false } - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } defer cancel() - _, err = wr.Write([]byte(name + "\n")) + _, err = batch.Writer().Write([]byte(name + "\n")) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } - sha, _, _, err := ReadBatchLine(rd) + sha, _, _, err := ReadBatchLine(batch.Reader()) return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name))) } @@ -44,18 +44,18 @@ func (repo *Repository) IsReferenceExist(name string) bool { return false } - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } defer cancel() - _, err = wr.Write([]byte(name + "\n")) + _, err = batch.Writer().Write([]byte(name + "\n")) if err != nil { log.Debug("Error writing to CatFileBatchCheck %v", err) return false } - _, _, _, err = ReadBatchLine(rd) + _, _, _, err = ReadBatchLine(batch.Reader()) return err == nil } diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index 3f27833fa6..a3d728eb6d 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -6,11 +6,11 @@ package git import ( - "bufio" "errors" "io" "strings" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/log" ) @@ -37,16 +37,16 @@ func (repo *Repository) ResolveReference(name string) (string, error) { // GetRefCommitID returns the last commit ID string of given reference (branch or tag). func (repo *Repository) GetRefCommitID(name string) (string, error) { - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { return "", err } defer cancel() - _, err = wr.Write([]byte(name + "\n")) + _, err = batch.Writer().Write([]byte(name + "\n")) if err != nil { return "", err } - shaBs, _, _, err := ReadBatchLine(rd) + shaBs, _, _, err := ReadBatchLine(batch.Reader()) if IsErrNotExist(err) { return "", ErrNotExist{name, ""} } @@ -56,7 +56,7 @@ func (repo *Repository) GetRefCommitID(name string) (string, error) { // IsCommitExist returns true if given commit exists in current repository. func (repo *Repository) IsCommitExist(name string) bool { - if err := ensureValidGitRepository(repo.Ctx, repo.Path); err != nil { + if err := catfile.EnsureValidGitRepository(repo.Ctx, repo.Path); err != nil { log.Error("IsCommitExist: %v", err) return false } @@ -68,18 +68,19 @@ func (repo *Repository) IsCommitExist(name string) bool { } func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { - wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() - _, _ = wr.Write([]byte(id.String() + "\n")) + _, _ = batch.Writer().Write([]byte(id.String() + "\n")) - return repo.getCommitFromBatchReader(wr, rd, id) + return repo.getCommitFromBatchReader(batch, id) } -func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio.Reader, id ObjectID) (*Commit, error) { +func (repo *Repository) getCommitFromBatchReader(batch catfile.Batch, id ObjectID) (*Commit, error) { + rd := batch.Reader() _, typ, size, err := ReadBatchLine(rd) if err != nil { if errors.Is(err, io.EOF) || IsErrNotExist(err) { @@ -107,11 +108,11 @@ func (repo *Repository) getCommitFromBatchReader(wr WriteCloserError, rd *bufio. return nil, err } - if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { + if _, err := batch.Writer().Write([]byte(tag.Object.String() + "\n")); err != nil { return nil, err } - commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) + commit, err := repo.getCommitFromBatchReader(batch, tag.Object) if err != nil { return nil, err } @@ -152,16 +153,16 @@ func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { } } - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { return nil, err } defer cancel() - _, err = wr.Write([]byte(commitID + "\n")) + _, err = batch.Writer().Write([]byte(commitID + "\n")) if err != nil { return nil, err } - sha, _, _, err := ReadBatchLine(rd) + sha, _, _, err := ReadBatchLine(batch.Reader()) if err != nil { if IsErrNotExist(err) { return nil, ErrNotExist{commitID, ""} diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index 5f79b68a9a..88d9edcbd8 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -24,16 +24,16 @@ func (repo *Repository) IsTagExist(name string) bool { // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) func (repo *Repository) GetTagType(id ObjectID) (string, error) { - wr, rd, cancel, err := repo.CatFileBatchCheck(repo.Ctx) + batch, cancel, err := repo.CatFileBatchCheck(repo.Ctx) if err != nil { return "", err } defer cancel() - _, err = wr.Write([]byte(id.String() + "\n")) + _, err = batch.Writer().Write([]byte(id.String() + "\n")) if err != nil { return "", err } - _, typ, _, err := ReadBatchLine(rd) + _, typ, _, err := ReadBatchLine(batch.Reader()) if err != nil { if IsErrNotExist(err) { return "", ErrNotExist{ID: id.String()} @@ -88,13 +88,14 @@ func (repo *Repository) getTag(tagID ObjectID, name string) (*Tag, error) { } // The tag is an annotated tag with a message. - wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() - if _, err := wr.Write([]byte(tagID.String() + "\n")); err != nil { + rd := batch.Reader() + if _, err := batch.Writer().Write([]byte(tagID.String() + "\n")); err != nil { return nil, err } _, typ, size, err := ReadBatchLine(rd) diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index 1954f85162..e6e2ee9fa0 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -10,12 +10,14 @@ import ( ) func (repo *Repository) getTree(id ObjectID) (*Tree, error) { - wr, rd, cancel, err := repo.CatFileBatch(repo.Ctx) + batch, cancel, err := repo.CatFileBatch(repo.Ctx) if err != nil { return nil, err } defer cancel() + wr := batch.Writer() + rd := batch.Reader() _, _ = wr.Write([]byte(id.String() + "\n")) // ignore the SHA @@ -39,7 +41,7 @@ func (repo *Repository) getTree(id ObjectID) (*Tree, error) { if _, err := wr.Write([]byte(tag.Object.String() + "\n")); err != nil { return nil, err } - commit, err := repo.getCommitFromBatchReader(wr, rd, tag.Object) + commit, err := repo.getCommitFromBatchReader(batch, tag.Object) if err != nil { return nil, err } diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index fd2f3c567f..0ea7aeed9d 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -15,18 +15,18 @@ func (te *TreeEntry) Size() int64 { return te.size } - wr, rd, cancel, err := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx) + batch, cancel, err := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) return 0 } defer cancel() - _, err = wr.Write([]byte(te.ID.String() + "\n")) + _, err = batch.Writer().Write([]byte(te.ID.String() + "\n")) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) return 0 } - _, _, te.size, err = ReadBatchLine(rd) + _, _, te.size, err = ReadBatchLine(batch.Reader()) if err != nil { log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) return 0 diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index d0ddb1d041..b8561dd352 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -27,12 +27,14 @@ func (t *Tree) ListEntries() (Entries, error) { } if t.repo != nil { - wr, rd, cancel, err := t.repo.CatFileBatch(t.repo.Ctx) + batch, cancel, err := t.repo.CatFileBatch(t.repo.Ctx) if err != nil { return nil, err } defer cancel() + wr := batch.Writer() + rd := batch.Reader() _, _ = wr.Write([]byte(t.ID.String() + "\n")) _, typ, sz, err := ReadBatchLine(rd) if err != nil { diff --git a/modules/gitrepo/cat_file.go b/modules/gitrepo/cat_file.go index c6ac74756f..0e5fc9951c 100644 --- a/modules/gitrepo/cat_file.go +++ b/modules/gitrepo/cat_file.go @@ -6,9 +6,9 @@ package gitrepo import ( "context" - "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/catfile" ) -func NewBatch(ctx context.Context, repo Repository) (*git.Batch, error) { - return git.NewBatch(ctx, repoPath(repo)) +func NewBatch(ctx context.Context, repo Repository) (catfile.Batch, error) { + return catfile.NewBatch(ctx, repoPath(repo)) } diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 5f6a7f6082..a3727bd0cb 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -4,7 +4,6 @@ package bleve import ( - "bufio" "context" "fmt" "io" @@ -16,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer" @@ -151,7 +151,7 @@ func NewIndexer(indexDir string) *Indexer { } } -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string, +func (b *Indexer) addUpdate(ctx context.Context, catfileBatch catfile.Batch, commitSha string, update internal.FileUpdate, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch, ) error { // Ignore vendored files in code search @@ -177,10 +177,11 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return b.addDelete(update.Filename, repo, batch) } - if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { + if _, err := catfileBatch.Writer().Write([]byte(update.BlobSha + "\n")); err != nil { return err } + batchReader := catfileBatch.Reader() _, _, size, err = git.ReadBatchLine(batchReader) if err != nil { return err @@ -218,18 +219,18 @@ func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error { batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize) if len(changes.Updates) > 0 { - gitBatch, err := gitrepo.NewBatch(ctx, repo) + catfileBatch, err := gitrepo.NewBatch(ctx, repo) if err != nil { return err } - defer gitBatch.Close() + defer catfileBatch.Close() for _, update := range changes.Updates { - if err := b.addUpdate(ctx, gitBatch.Writer, gitBatch.Reader, sha, update, repo, batch); err != nil { + if err := b.addUpdate(ctx, catfileBatch, sha, update, repo, batch); err != nil { return err } } - gitBatch.Close() + catfileBatch.Close() } for _, filename := range changes.RemovedFilenames { if err := b.addDelete(filename, repo, batch); err != nil { diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index a7027051d2..653df0bd11 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -4,7 +4,6 @@ package elasticsearch import ( - "bufio" "context" "fmt" "io" @@ -15,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/catfile" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/indexer" @@ -139,7 +139,7 @@ const ( }` ) -func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { +func (b *Indexer) addUpdate(ctx context.Context, batch catfile.Batch, sha string, update internal.FileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { return nil, nil @@ -162,10 +162,11 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil } - if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil { + if _, err := batch.Writer().Write([]byte(update.BlobSha + "\n")); err != nil { return nil, err } + batchReader := batch.Reader() _, _, size, err = git.ReadBatchLine(batchReader) if err != nil { return nil, err @@ -217,7 +218,7 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st defer batch.Close() for _, update := range changes.Updates { - updateReqs, err := b.addUpdate(ctx, batch.Writer, batch.Reader, sha, update, repo) + updateReqs, err := b.addUpdate(ctx, batch, sha, update, repo) if err != nil { return err } From 17715693005678e3145c19e95b0282bc2bea0853 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 29 Dec 2025 10:49:54 -0800 Subject: [PATCH 8/8] Some refactor for repo path (#36251) - Use `gitrepo.IsRepositoryExist` instead of `util.IsExit` or `util.IsDir` - Use `gitrepo.OpenRepository` instead of `git.OpenRepository` - Use `gitrepo.DeleteRepository` instead of `util.RemoveAll` - Use `gitrepo.RenameRepository` instead of `util.Rename` --- models/migrations/v1_21/v276.go | 19 ++++++------------- models/migrations/v1_9/v82.go | 17 +++-------------- modules/gitrepo/gitrepo.go | 10 +++++++++- routers/api/v1/admin/adopt.go | 10 +++++----- routers/web/admin/repos.go | 6 +++--- routers/web/user/setting/adopt.go | 10 +++------- services/migrations/gitea_uploader.go | 9 +-------- services/repository/adopt.go | 11 +++++------ services/repository/transfer.go | 9 ++------- 9 files changed, 37 insertions(+), 64 deletions(-) diff --git a/models/migrations/v1_21/v276.go b/models/migrations/v1_21/v276.go index 3ab7e22cd0..be24b31902 100644 --- a/models/migrations/v1_21/v276.go +++ b/models/migrations/v1_21/v276.go @@ -5,14 +5,10 @@ package v1_21 import ( "context" - "fmt" - "path/filepath" - "strings" - "code.gitea.io/gitea/modules/git" - giturl "code.gitea.io/gitea/modules/git/url" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "xorm.io/xorm" ) @@ -163,16 +159,13 @@ func migratePushMirrors(x *xorm.Engine) error { } func getRemoteAddress(ownerName, repoName, remoteName string) (string, error) { - repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ownerName), strings.ToLower(repoName)+".git") - if exist, _ := util.IsExist(repoPath); !exist { + ctx := context.Background() + relativePath := repo_model.RelativePath(ownerName, repoName) + if exist, _ := gitrepo.IsRepositoryExist(ctx, repo_model.StorageRepo(relativePath)); !exist { return "", nil } - remoteURL, err := git.GetRemoteAddress(context.Background(), repoPath, remoteName) - if err != nil { - return "", fmt.Errorf("get remote %s's address of %s/%s failed: %v", remoteName, ownerName, repoName, err) - } - u, err := giturl.ParseGitURL(remoteURL) + u, err := gitrepo.GitRemoteGetURL(ctx, repo_model.StorageRepo(relativePath), remoteName) if err != nil { return "", err } diff --git a/models/migrations/v1_9/v82.go b/models/migrations/v1_9/v82.go index f0307bf07a..8796b0563d 100644 --- a/models/migrations/v1_9/v82.go +++ b/models/migrations/v1_9/v82.go @@ -6,11 +6,10 @@ package v1_9 import ( "context" "fmt" - "path/filepath" - "strings" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/gitrepo" "xorm.io/xorm" ) @@ -34,16 +33,6 @@ func FixReleaseSha1OnReleaseTable(ctx context.Context, x *xorm.Engine) error { Name string } - // UserPath returns the path absolute path of user repositories. - UserPath := func(userName string) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(userName)) - } - - // RepoPath returns repository path by given user and repository name. - RepoPath := func(userName, repoName string) string { - return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".git") - } - // Update release sha1 const batchSize = 100 sess := x.NewSession() @@ -99,7 +88,7 @@ func FixReleaseSha1OnReleaseTable(ctx context.Context, x *xorm.Engine) error { userCache[repo.OwnerID] = user } - gitRepo, err = git.OpenRepository(ctx, RepoPath(user.Name, repo.Name)) + gitRepo, err = gitrepo.OpenRepository(ctx, repo_model.StorageRepo(repo_model.RelativePath(user.Name, repo.Name))) if err != nil { return err } diff --git a/modules/gitrepo/gitrepo.go b/modules/gitrepo/gitrepo.go index 3a9b0a1c89..535d72ed98 100644 --- a/modules/gitrepo/gitrepo.go +++ b/modules/gitrepo/gitrepo.go @@ -80,7 +80,12 @@ func DeleteRepository(ctx context.Context, repo Repository) error { // RenameRepository renames a repository's name on disk func RenameRepository(ctx context.Context, repo, newRepo Repository) error { - if err := util.Rename(repoPath(repo), repoPath(newRepo)); err != nil { + dstDir := repoPath(newRepo) + if err := os.MkdirAll(filepath.Dir(dstDir), os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %w", filepath.Dir(dstDir), err) + } + + if err := util.Rename(repoPath(repo), dstDir); err != nil { return fmt.Errorf("rename repository directory: %w", err) } return nil @@ -116,5 +121,8 @@ func RemoveRepoFileOrDir(ctx context.Context, repo Repository, relativeFileOrDir func CreateRepoFile(ctx context.Context, repo Repository, relativeFilePath string) (io.WriteCloser, error) { absoluteFilePath := filepath.Join(repoPath(repo), relativeFilePath) + if err := os.MkdirAll(filepath.Dir(absoluteFilePath), os.ModePerm); err != nil { + return nil, err + } return os.Create(absoluteFilePath) } diff --git a/routers/api/v1/admin/adopt.go b/routers/api/v1/admin/adopt.go index c2efed7490..92711409f0 100644 --- a/routers/api/v1/admin/adopt.go +++ b/routers/api/v1/admin/adopt.go @@ -8,7 +8,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/routers/api/v1/utils" "code.gitea.io/gitea/services/context" repo_service "code.gitea.io/gitea/services/repository" @@ -99,12 +99,12 @@ func AdoptRepository(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + exist, err := gitrepo.IsRepositoryExist(ctx, repo_model.StorageRepo(repo_model.RelativePath(ctxUser.Name, repoName))) if err != nil { ctx.APIErrorInternal(err) return } - if has || !isDir { + if has || !exist { ctx.APIErrorNotFound() return } @@ -161,12 +161,12 @@ func DeleteUnadoptedRepository(ctx *context.APIContext) { ctx.APIErrorInternal(err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + exist, err := gitrepo.IsRepositoryExist(ctx, repo_model.StorageRepo(repo_model.RelativePath(ctxUser.Name, repoName))) if err != nil { ctx.APIErrorInternal(err) return } - if has || !isDir { + if has || !exist { ctx.APIErrorNotFound() return } diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 1bc8abb88c..424219815c 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -11,10 +11,10 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/web/explore" "code.gitea.io/gitea/services/context" repo_service "code.gitea.io/gitea/services/repository" @@ -134,12 +134,12 @@ func AdoptOrDeleteRepository(ctx *context.Context) { ctx.ServerError("IsRepositoryExist", err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + exist, err := gitrepo.IsRepositoryExist(ctx, repo_model.StorageRepo(repo_model.RelativePath(ctxUser.Name, repoName))) if err != nil { ctx.ServerError("IsDir", err) return } - if has || !isDir { + if has || !exist { // Fallthrough to failure mode } else if action == "adopt" { if _, err := repo_service.AdoptRepository(ctx, ctx.Doer, ctxUser, repo_service.CreateRepoOptions{ diff --git a/routers/web/user/setting/adopt.go b/routers/web/user/setting/adopt.go index 171c1933d4..abf9d8c6db 100644 --- a/routers/web/user/setting/adopt.go +++ b/routers/web/user/setting/adopt.go @@ -4,12 +4,9 @@ package setting import ( - "path/filepath" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context" repo_service "code.gitea.io/gitea/services/repository" ) @@ -27,7 +24,6 @@ func AdoptOrDeleteRepository(ctx *context.Context) { action := ctx.FormString("action") ctxUser := ctx.Doer - root := user_model.UserPath(ctxUser.LowerName) // check not a repo has, err := repo_model.IsRepositoryModelExist(ctx, ctxUser, dir) @@ -36,12 +32,12 @@ func AdoptOrDeleteRepository(ctx *context.Context) { return } - isDir, err := util.IsDir(filepath.Join(root, dir+".git")) + exist, err := gitrepo.IsRepositoryExist(ctx, repo_model.StorageRepo(repo_model.RelativePath(ctxUser.Name, dir))) if err != nil { ctx.ServerError("IsDir", err) return } - if has || !isDir { + if has || !exist { // Fallthrough to failure mode } else if action == "adopt" && allowAdopt { if _, err := repo_service.AdoptRepository(ctx, ctxUser, ctxUser, repo_service.CreateRepoOptions{ diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 96c2655b3a..4b50e86b12 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -8,8 +8,6 @@ import ( "context" "fmt" "io" - "os" - "path/filepath" "strconv" "strings" "time" @@ -589,12 +587,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba } defer ret.Close() - pullDir := filepath.Join(g.repo.RepoPath(), "pulls") - if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { - return err - } - - f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) + f, err := gitrepo.CreateRepoFile(ctx, g.repo, fmt.Sprintf("pulls/%d.patch", pr.Number)) if err != nil { return err } diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 8d8e59b053..18d70d1bee 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -23,7 +23,6 @@ import ( "code.gitea.io/gitea/modules/optional" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" ) @@ -214,13 +213,13 @@ func DeleteUnadoptedRepository(ctx context.Context, doer, u *user_model.User, re return err } - repoPath := repo_model.RepoPath(u.Name, repoName) - isExist, err := util.IsExist(repoPath) + relativePath := repo_model.RelativePath(u.Name, repoName) + exist, err := gitrepo.IsRepositoryExist(ctx, repo_model.StorageRepo(relativePath)) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", relativePath, err) return err } - if !isExist { + if !exist { return repo_model.ErrRepoNotExist{ OwnerName: u.Name, Name: repoName, @@ -236,7 +235,7 @@ func DeleteUnadoptedRepository(ctx context.Context, doer, u *user_model.User, re } } - return util.RemoveAll(repoPath) + return gitrepo.DeleteRepository(ctx, repo_model.StorageRepo(relativePath)) } type unadoptedRepositories struct { diff --git a/services/repository/transfer.go b/services/repository/transfer.go index af477fc7f1..a601ee6f16 100644 --- a/services/repository/transfer.go +++ b/services/repository/transfer.go @@ -6,7 +6,6 @@ package repository import ( "context" "fmt" - "os" "strings" "code.gitea.io/gitea/models/db" @@ -291,12 +290,8 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName } // Rename remote repository to new path and delete local copy. - dir := user_model.UserPath(newOwner.Name) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %w", dir, err) - } - - if err := util.Rename(repo_model.RepoPath(oldOwner.Name, repo.Name), repo_model.RepoPath(newOwner.Name, repo.Name)); err != nil { + oldRelativePath, newRelativePath := repo_model.RelativePath(oldOwner.Name, repo.Name), repo_model.RelativePath(newOwner.Name, repo.Name) + if err := gitrepo.RenameRepository(ctx, repo_model.StorageRepo(oldRelativePath), repo_model.StorageRepo(newRelativePath)); err != nil { return fmt.Errorf("rename repository directory: %w", err) } repoRenamed = true