diff --git a/.github/workflows/cron-licenses.yml b/.github/workflows/cron-licenses.yml index 12f52289b6..a8be1ffa59 100644 --- a/.github/workflows/cron-licenses.yml +++ b/.github/workflows/cron-licenses.yml @@ -9,8 +9,10 @@ jobs: cron-licenses: runs-on: ubuntu-latest if: github.repository == 'go-gitea/gitea' + permissions: + contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/cron-translations.yml b/.github/workflows/cron-translations.yml index ae2238ad2d..3a012e9876 100644 --- a/.github/workflows/cron-translations.yml +++ b/.github/workflows/cron-translations.yml @@ -9,8 +9,10 @@ jobs: crowdin-pull: runs-on: ubuntu-latest if: github.repository == 'go-gitea/gitea' + permissions: + contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: crowdin/github-action@v1 with: upload_sources: true diff --git a/.github/workflows/files-changed.yml b/.github/workflows/files-changed.yml index b21341a277..d18ee6e998 100644 --- a/.github/workflows/files-changed.yml +++ b/.github/workflows/files-changed.yml @@ -24,6 +24,8 @@ jobs: detect: runs-on: ubuntu-latest timeout-minutes: 3 + permissions: + contents: read outputs: backend: ${{ steps.changes.outputs.backend }} frontend: ${{ steps.changes.outputs.frontend }} @@ -34,7 +36,7 @@ jobs: swagger: ${{ steps.changes.outputs.swagger }} yaml: ${{ steps.changes.outputs.yaml }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dorny/paths-filter@v3 id: changes with: diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml index f73772e934..c146b439e0 100644 --- a/.github/workflows/pull-compliance.yml +++ b/.github/workflows/pull-compliance.yml @@ -10,13 +10,17 @@ concurrency: jobs: files-changed: uses: ./.github/workflows/files-changed.yml + permissions: + contents: read lint-backend: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -30,8 +34,10 @@ jobs: if: needs.files-changed.outputs.templates == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@v6 - run: uv python install 3.12 - uses: pnpm/action-setup@v4 @@ -46,8 +52,10 @@ jobs: if: needs.files-changed.outputs.yaml == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@v6 - run: uv python install 3.12 - run: make deps-py @@ -57,8 +65,10 @@ jobs: if: needs.files-changed.outputs.swagger == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v5 with: @@ -70,8 +80,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.frontend == 'true' || needs.files-changed.outputs.actions == 'true' || needs.files-changed.outputs.docs == 'true' || needs.files-changed.outputs.templates == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -82,8 +94,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -99,8 +113,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -114,8 +130,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -127,8 +145,10 @@ jobs: if: needs.files-changed.outputs.frontend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v5 with: @@ -143,8 +163,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -175,8 +197,10 @@ jobs: if: needs.files-changed.outputs.docs == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v5 with: @@ -188,8 +212,10 @@ jobs: if: needs.files-changed.outputs.actions == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 21ec76b48e..66f48d5af8 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -10,11 +10,15 @@ concurrency: jobs: files-changed: uses: ./.github/workflows/files-changed.yml + permissions: + contents: read test-pgsql: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: pgsql: image: postgres:14 @@ -38,7 +42,7 @@ jobs: ports: - "9000:9000" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -65,8 +69,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -90,6 +96,8 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: elasticsearch: image: elasticsearch:7.5.0 @@ -124,7 +132,7 @@ jobs: ports: - 10000:10000 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -152,6 +160,8 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: mysql: # the bitnami mysql image has more options than the official one, it's easier to customize @@ -177,7 +187,7 @@ jobs: - "587:587" - "993:993" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -203,6 +213,8 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: mssql: image: mcr.microsoft.com/mssql/server:2019-latest @@ -217,7 +229,7 @@ jobs: ports: - 10000:10000 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/pull-docker-dryrun.yml b/.github/workflows/pull-docker-dryrun.yml index 9c9dd2ffe6..1cd1ba31dd 100644 --- a/.github/workflows/pull-docker-dryrun.yml +++ b/.github/workflows/pull-docker-dryrun.yml @@ -10,13 +10,17 @@ concurrency: jobs: files-changed: uses: ./.github/workflows/files-changed.yml + permissions: + contents: read container: if: needs.files-changed.outputs.docker == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: docker/setup-buildx-action@v3 - name: Build regular container image uses: docker/build-push-action@v5 diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index ada4c18d33..c8ce0aa787 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -11,8 +11,10 @@ concurrency: jobs: nightly-binary: runs-on: namespace-profile-gitea-release-binary + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force @@ -56,12 +58,14 @@ jobs: - name: upload binaries to s3 run: | aws s3 sync dist/release s3://${{ secrets.AWS_S3_BUCKET }}/gitea/${{ steps.clean_name.outputs.branch }} --no-progress + nightly-container: runs-on: namespace-profile-gitea-release-docker permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force diff --git a/.github/workflows/release-tag-rc.yml b/.github/workflows/release-tag-rc.yml index 35558933e0..ef36e55a94 100644 --- a/.github/workflows/release-tag-rc.yml +++ b/.github/workflows/release-tag-rc.yml @@ -12,8 +12,10 @@ concurrency: jobs: binary: runs-on: namespace-profile-gitea-release-binary + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force @@ -66,12 +68,14 @@ jobs: gh release create ${{ github.ref_name }} --title ${{ github.ref_name }} --draft --notes-from-tag dist/release/* env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + container: runs-on: namespace-profile-gitea-release-docker permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force diff --git a/.github/workflows/release-tag-version.yml b/.github/workflows/release-tag-version.yml index 56426d3bc3..a3838de3c0 100644 --- a/.github/workflows/release-tag-version.yml +++ b/.github/workflows/release-tag-version.yml @@ -15,9 +15,10 @@ jobs: binary: runs-on: namespace-profile-gitea-release-binary permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force @@ -70,12 +71,14 @@ jobs: gh release create ${{ github.ref_name }} --title ${{ github.ref_name }} --notes-from-tag dist/release/* env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + container: runs-on: namespace-profile-gitea-release-docker permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force diff --git a/.golangci.yml b/.golangci.yml index 2f1587a1e6..45083d5fd2 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,10 +6,12 @@ linters: default: none enable: - bidichk + - bodyclose - depguard - dupl - errcheck - forbidigo + - gocheckcompilerdirectives - gocritic - govet - ineffassign diff --git a/cmd/admin_auth_ldap_test.go b/cmd/admin_auth_ldap_test.go index 2da7ebc573..0e3e465e81 100644 --- a/cmd/admin_auth_ldap_test.go +++ b/cmd/admin_auth_ldap_test.go @@ -233,7 +233,7 @@ func TestAddLdapBindDn(t *testing.T) { }, getAuthSourceByID: func(ctx context.Context, id int64) (*auth.Source, error) { assert.FailNow(t, "getAuthSourceByID called", "case %d: should not call getAuthSourceByID", n) - return nil, nil + return nil, nil //nolint:nilnil // mock function covering improper behavior }, } @@ -463,7 +463,7 @@ func TestAddLdapSimpleAuth(t *testing.T) { }, getAuthSourceByID: func(ctx context.Context, id int64) (*auth.Source, error) { assert.FailNow(t, "getAuthSourceById called", "case %d: should not call getAuthSourceByID", n) - return nil, nil + return nil, nil //nolint:nilnil // mock function covering improper behavior }, } diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 2ade845590..475c62ab6c 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -733,6 +733,9 @@ LEVEL = Info ;DISABLE_CORE_PROTECT_NTFS=false ;; Disable the usage of using partial clones for git. ;DISABLE_PARTIAL_CLONE = false +;; Set the similarity threshold passed to git commands via `--find-renames=`. +;; Default is 50%, the same as git. Must be a integer percentage between 0% and 100%. +;DIFF_RENAME_SIMILARITY_THRESHOLD = 50% ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Git Operation timeout in seconds diff --git a/go.mod b/go.mod index 6806e76ffc..130820a55c 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/ProtonMail/go-crypto v1.3.0 github.com/PuerkitoBio/goquery v1.10.3 github.com/SaveTheRbtz/zstd-seekable-format-go/pkg v0.8.0 - github.com/alecthomas/chroma/v2 v2.20.0 + github.com/alecthomas/chroma/v2 v2.21.0 github.com/aws/aws-sdk-go-v2/credentials v1.18.10 github.com/aws/aws-sdk-go-v2/service/codecommit v1.32.2 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb diff --git a/go.sum b/go.sum index 86fe782ae7..92013e195a 100644 --- a/go.sum +++ b/go.sum @@ -98,11 +98,11 @@ github.com/SaveTheRbtz/zstd-seekable-format-go/pkg v0.8.0/go.mod h1:1HmmMEVsr+0R github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.2.0/go.mod h1:vf4zrexSH54oEjJ7EdB65tGNHmH3pGZmVkgTP5RHvAs= -github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= -github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= +github.com/alecthomas/chroma/v2 v2.21.0 h1:YVW9qQAFnQm2OFPPFQg6G/TpMxKSsUr/KUPDi/BEqtY= +github.com/alecthomas/chroma/v2 v2.21.0/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= -github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= -github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= diff --git a/models/issues/pull_list.go b/models/issues/pull_list.go index 84f9f6166d..19d727ecbd 100644 --- a/models/issues/pull_list.go +++ b/models/issues/pull_list.go @@ -14,6 +14,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "xorm.io/builder" @@ -324,12 +325,26 @@ func (prs PullRequestList) LoadReviews(ctx context.Context) (ReviewList, error) // HasMergedPullRequestInRepo returns whether the user(poster) has merged pull-request in the repo func HasMergedPullRequestInRepo(ctx context.Context, repoID, posterID int64) (bool, error) { - return db.GetEngine(ctx). + return HasMergedPullRequestInRepoBefore(ctx, repoID, posterID, 0, 0) +} + +// HasMergedPullRequestInRepoBefore returns whether the user has a merged PR before a timestamp (0 = no limit) +func HasMergedPullRequestInRepoBefore(ctx context.Context, repoID, posterID int64, beforeUnix timeutil.TimeStamp, excludePullID int64) (bool, error) { + sess := db.GetEngine(ctx). Join("INNER", "pull_request", "pull_request.issue_id = issue.id"). Where("repo_id=?", repoID). And("poster_id=?", posterID). And("is_pull=?", true). - And("pull_request.has_merged=?", true). + And("pull_request.has_merged=?", true) + + if beforeUnix > 0 { + sess.And("pull_request.merged_unix < ?", beforeUnix) + } + if excludePullID > 0 { + sess.And("pull_request.id != ?", excludePullID) + } + + return sess. Select("issue.id"). Limit(1). Get(new(Issue)) diff --git a/models/perm/access/repo_permission.go b/models/perm/access/repo_permission.go index f5b7c9ef33..f1c1c3f552 100644 --- a/models/perm/access/repo_permission.go +++ b/models/perm/access/repo_permission.go @@ -281,8 +281,14 @@ func GetActionsUserRepoPermission(ctx context.Context, repo *repo_model.Reposito if !actionsCfg.IsCollaborativeOwner(taskRepo.OwnerID) || !taskRepo.IsPrivate { // The task repo can access the current repo only if the task repo is private and // the owner of the task repo is a collaborative owner of the current repo. - // FIXME allow public repo read access if tokenless pull is enabled // FIXME should owner's visibility also be considered here? + + // check permission like simple user but limit to read-only + perm, err = GetUserRepoPermission(ctx, repo, user_model.NewActionsUser()) + if err != nil { + return perm, err + } + perm.AccessMode = min(perm.AccessMode, perm_model.AccessModeRead) return perm, nil } // Cross-repo access is always read-only diff --git a/models/repo/repo.go b/models/repo/repo.go index 819356dfad..605a9e0f3f 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -869,16 +869,6 @@ func GetRepositoriesMapByIDs(ctx context.Context, ids []int64) (map[int64]*Repos return repos, db.GetEngine(ctx).In("id", ids).Find(&repos) } -// IsRepositoryModelOrDirExist returns true if the repository with given name under user has already existed. -func IsRepositoryModelOrDirExist(ctx context.Context, u *user_model.User, repoName string) (bool, error) { - has, err := IsRepositoryModelExist(ctx, u, repoName) - if err != nil { - return false, err - } - isDir, err := util.IsDir(RepoPath(u.Name, repoName)) - return has || isDir, err -} - func IsRepositoryModelExist(ctx context.Context, u *user_model.User, repoName string) (bool, error) { return db.GetEngine(ctx).Get(&Repository{ OwnerID: u.ID, diff --git a/models/repo/update.go b/models/repo/update.go index 3228ae11a4..bf560cf695 100644 --- a/models/repo/update.go +++ b/models/repo/update.go @@ -9,8 +9,6 @@ import ( "time" "code.gitea.io/gitea/models/db" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -106,35 +104,6 @@ func (err ErrRepoFilesAlreadyExist) Unwrap() error { return util.ErrAlreadyExist } -// CheckCreateRepository check if doer could create a repository in new owner -func CheckCreateRepository(ctx context.Context, doer, owner *user_model.User, name string, overwriteOrAdopt bool) error { - if !doer.CanCreateRepoIn(owner) { - return ErrReachLimitOfRepo{owner.MaxRepoCreation} - } - - if err := IsUsableRepoName(name); err != nil { - return err - } - - has, err := IsRepositoryModelOrDirExist(ctx, owner, name) - if err != nil { - return fmt.Errorf("IsRepositoryExist: %w", err) - } else if has { - return ErrRepoAlreadyExist{owner.Name, name} - } - - repoPath := RepoPath(owner.Name, name) - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if !overwriteOrAdopt && isExist { - return ErrRepoFilesAlreadyExist{owner.Name, name} - } - return nil -} - // UpdateRepoSize updates the repository size, calculating it using getDirectorySize func UpdateRepoSize(ctx context.Context, repoID, gitSize, lfsSize int64) error { _, err := db.GetEngine(ctx).ID(repoID).Cols("size", "git_size", "lfs_size").NoAutoTime().Update(&Repository{ diff --git a/models/user/search.go b/models/user/search.go index db4b07f64a..36d1d3913b 100644 --- a/models/user/search.go +++ b/models/user/search.go @@ -18,6 +18,23 @@ import ( "xorm.io/xorm" ) +// AdminUserOrderByMap represents all possible admin user search orders +// This should only be used for admin API endpoints as we should not expose "updated" ordering which could expose recent user activity including logins. +var AdminUserOrderByMap = map[string]map[string]db.SearchOrderBy{ + "asc": { + "name": db.SearchOrderByAlphabetically, + "created": db.SearchOrderByOldest, + "updated": db.SearchOrderByLeastUpdated, + "id": db.SearchOrderByID, + }, + "desc": { + "name": db.SearchOrderByAlphabeticallyReverse, + "created": db.SearchOrderByNewest, + "updated": db.SearchOrderByRecentUpdated, + "id": db.SearchOrderByIDReverse, + }, +} + // SearchUserOptions contains the options for searching type SearchUserOptions struct { db.ListOptions diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 597ce5120c..b156654973 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -5,12 +5,10 @@ package charset import ( "bytes" - "fmt" "io" "strings" "unicode/utf8" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -23,60 +21,39 @@ import ( var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} type ConvertOpts struct { - KeepBOM bool + KeepBOM bool + ErrorReplacement []byte + ErrorReturnOrigin bool } +var ToUTF8WithFallbackReaderPrefetchSize = 16 * 1024 + // ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader { - buf := make([]byte, 2048) + buf := make([]byte, ToUTF8WithFallbackReaderPrefetchSize) n, err := util.ReadAtMost(rd, buf) if err != nil { - return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd) - } - - charsetLabel, err := DetectEncoding(buf[:n]) - if err != nil || charsetLabel == "UTF-8" { - return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd) - } - - encoding, _ := charset.Lookup(charsetLabel) - if encoding == nil { + // read error occurs, don't do any processing return io.MultiReader(bytes.NewReader(buf[:n]), rd) } - return transform.NewReader( - io.MultiReader( - bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), - rd, - ), - encoding.NewDecoder(), - ) -} - -// ToUTF8 converts content to UTF8 encoding -func ToUTF8(content []byte, opts ConvertOpts) (string, error) { - charsetLabel, err := DetectEncoding(content) - if err != nil { - return "", err - } else if charsetLabel == "UTF-8" { - return string(MaybeRemoveBOM(content, opts)), nil + charsetLabel, _ := DetectEncoding(buf[:n]) + if charsetLabel == "UTF-8" { + // is utf-8, try to remove BOM and read it as-is + return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd) } encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { - return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel) + // unknown charset, don't do any processing + return io.MultiReader(bytes.NewReader(buf[:n]), rd) } - // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose much data. - result, n, err := transform.Bytes(encoding.NewDecoder(), content) - if err != nil { - result = append(result, content[n:]...) - } - - result = MaybeRemoveBOM(result, opts) - - return string(result), err + // convert from charset to utf-8 + return transform.NewReader( + io.MultiReader(bytes.NewReader(buf[:n]), rd), + encoding.NewDecoder(), + ) } // ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible @@ -85,73 +62,84 @@ func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte { return bs } -// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible -func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte { - charsetLabel, err := DetectEncoding(content) - if err != nil || charsetLabel == "UTF-8" { - return MaybeRemoveBOM(content, opts) +func ToUTF8DropErrors(content []byte) []byte { + return ToUTF8(content, ConvertOpts{ErrorReplacement: []byte{' '}}) +} + +func ToUTF8(content []byte, opts ConvertOpts) []byte { + charsetLabel, _ := DetectEncoding(content) + if charsetLabel == "UTF-8" { + return maybeRemoveBOM(content, opts) } encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { + setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel) return content } - // We ignore any non-decodable parts from the file. - // Some parts might be lost var decoded []byte decoder := encoding.NewDecoder() idx := 0 - for { + for idx < len(content) { result, n, err := transform.Bytes(decoder, content[idx:]) decoded = append(decoded, result...) if err == nil { break } - decoded = append(decoded, ' ') - idx = idx + n + 1 - if idx >= len(content) { - break + if opts.ErrorReturnOrigin { + return content } + if opts.ErrorReplacement == nil { + decoded = append(decoded, content[idx+n]) + } else { + decoded = append(decoded, opts.ErrorReplacement...) + } + idx += n + 1 } - - return MaybeRemoveBOM(decoded, opts) + return maybeRemoveBOM(decoded, opts) } -// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false -func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte { +// maybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false +func maybeRemoveBOM(content []byte, opts ConvertOpts) []byte { if opts.KeepBOM { return content } - if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) { - return content[3:] - } - return content + return bytes.TrimPrefix(content, UTF8BOM) } // DetectEncoding detect the encoding of content -func DetectEncoding(content []byte) (string, error) { +// it always returns a detected or guessed "encoding" string, no matter error happens or not +func DetectEncoding(content []byte) (encoding string, _ error) { // First we check if the content represents valid utf8 content excepting a truncated character at the end. // Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do - // instead we walk backwards from the end to trim off a the incomplete character + // instead we walk backwards from the end to trim off the incomplete character toValidate := content end := len(toValidate) - 1 - if end < 0 { - // no-op - } else if toValidate[end]>>5 == 0b110 { - // Incomplete 1 byte extension e.g. © which has been truncated to - toValidate = toValidate[:end] - } else if end > 0 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>4 == 0b1110 { - // Incomplete 2 byte extension e.g. ⛔ <9b><94> which has been truncated to <9b> - toValidate = toValidate[:end-1] - } else if end > 1 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>6 == 0b10 && toValidate[end-2]>>3 == 0b11110 { - // Incomplete 3 byte extension e.g. 💩 <9f><92> which has been truncated to <9f><92> - toValidate = toValidate[:end-2] + // U+0000 U+007F 0yyyzzzz + // U+0080 U+07FF 110xxxyy 10yyzzzz + // U+0800 U+FFFF 1110wwww 10xxxxyy 10yyzzzz + // U+010000 U+10FFFF 11110uvv 10vvwwww 10xxxxyy 10yyzzzz + cnt := 0 + for end >= 0 && cnt < 4 { + c := toValidate[end] + if c>>5 == 0b110 || c>>4 == 0b1110 || c>>3 == 0b11110 { + // a leading byte + toValidate = toValidate[:end] + break + } else if c>>6 == 0b10 { + // a continuation byte + end-- + } else { + // not an utf-8 byte + break + } + cnt++ } + if utf8.Valid(toValidate) { - log.Debug("Detected encoding: utf-8 (fast)") return "UTF-8", nil } @@ -160,7 +148,7 @@ func DetectEncoding(content []byte) (string, error) { if len(content) < 1024 { // Check if original content is valid if _, err := textDetector.DetectBest(content); err != nil { - return "", err + return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err } times := 1024 / len(content) detectContent = make([]byte, 0, times*len(content)) @@ -171,14 +159,10 @@ func DetectEncoding(content []byte) (string, error) { detectContent = content } - // Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break + // Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie-break results, err := textDetector.DetectAll(detectContent) if err != nil { - if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 { - log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) - return setting.Repository.AnsiCharset, nil - } - return "", err + return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err } topConfidence := results[0].Confidence @@ -201,11 +185,9 @@ func DetectEncoding(content []byte) (string, error) { } // FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument - if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 { - log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) + if topResult.Charset != "UTF-8" && setting.Repository.AnsiCharset != "" { return setting.Repository.AnsiCharset, err } - log.Debug("Detected encoding: %s", topResult.Charset) - return topResult.Charset, err + return topResult.Charset, nil } diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index cd2e3b9aaa..0314abc347 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -4,108 +4,89 @@ package charset import ( - "bytes" "io" + "os" "strings" "testing" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) -func resetDefaultCharsetsOrder() { - defaultDetectedCharsetsOrder := make([]string, 0, len(setting.Repository.DetectedCharsetsOrder)) - for _, charset := range setting.Repository.DetectedCharsetsOrder { - defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder, strings.ToLower(strings.TrimSpace(charset))) - } +func TestMain(m *testing.M) { setting.Repository.DetectedCharsetScore = map[string]int{} - i := 0 - for _, charset := range defaultDetectedCharsetsOrder { - canonicalCharset := strings.ToLower(strings.TrimSpace(charset)) - if _, has := setting.Repository.DetectedCharsetScore[canonicalCharset]; !has { - setting.Repository.DetectedCharsetScore[canonicalCharset] = i - i++ - } + for i, charset := range setting.Repository.DetectedCharsetsOrder { + setting.Repository.DetectedCharsetScore[strings.ToLower(charset)] = i } + os.Exit(m.Run()) } func TestMaybeRemoveBOM(t *testing.T) { - res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res := maybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) - res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res = maybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) } func TestToUTF8(t *testing.T) { - resetDefaultCharsetsOrder() - // Note: golang compiler seems so behave differently depending on the current // locale, so some conversions might behave differently. For that reason, we don't // depend on particular conversions but in expected behaviors. - res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, "ABC", res) + res := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) + assert.Equal(t, "ABC", string(res)) // "áéíóú" - res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) + res = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // "áéíóú" - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba, }, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, }, ConvertOpts{}) - assert.NoError(t, err) stringMustStartWith(t, "Hola,", res) stringMustEndWith(t, "AAA.", res) - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, }, ConvertOpts{}) - assert.NoError(t, err) stringMustStartWith(t, "Hola,", res) stringMustEndWith(t, "AAA.", res) - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, }, ConvertOpts{}) - assert.NoError(t, err) stringMustStartWith(t, "Hola,", res) stringMustEndWith(t, "AAA.", res) // Japanese (Shift-JIS) // 日属秘ぞしちゅ。 - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42, }, ConvertOpts{}) - assert.NoError(t, err) assert.Equal(t, []byte{ 0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, 0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, - }, - []byte(res)) + }, res) - res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res)) + res = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) + assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) } func TestToUTF8WithFallback(t *testing.T) { - resetDefaultCharsetsOrder() // "ABC" res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) assert.Equal(t, []byte{0x41, 0x42, 0x43}, res) @@ -152,54 +133,58 @@ func TestToUTF8WithFallback(t *testing.T) { } func TestToUTF8DropErrors(t *testing.T) { - resetDefaultCharsetsOrder() // "ABC" - res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) + res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}) assert.Equal(t, []byte{0x41, 0x42, 0x43}, res) // "áéíóú" - res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // UTF8 BOM + "áéíóú" - res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // "Hola, así cómo ños" - res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}) assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8]) assert.Equal(t, []byte{0x73}, res[len(res)-1:]) // "Hola, así cómo " minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20} - res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}) // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those assert.Equal(t, minmatch, res[0:len(minmatch)]) - res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}) // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those assert.Equal(t, minmatch, res[0:len(minmatch)]) // Japanese (Shift-JIS) // "日属秘ぞしちゅ。" - res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}) assert.Equal(t, []byte{ 0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, 0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, }, res) - res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}) assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) } func TestDetectEncoding(t *testing.T) { - resetDefaultCharsetsOrder() testSuccess := func(b []byte, expected string) { encoding, err := DetectEncoding(b) assert.NoError(t, err) assert.Equal(t, expected, encoding) } + + // invalid bytes + encoding, err := DetectEncoding([]byte{0xfa}) + assert.Error(t, err) + assert.Equal(t, "UTF-8", encoding) + // utf-8 b := []byte("just some ascii") testSuccess(b, "UTF-8") @@ -214,169 +199,49 @@ func TestDetectEncoding(t *testing.T) { // iso-8859-1: dcor b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a} - encoding, err := DetectEncoding(b) + encoding, err = DetectEncoding(b) assert.NoError(t, err) assert.Contains(t, encoding, "ISO-8859-1") - old := setting.Repository.AnsiCharset - setting.Repository.AnsiCharset = "placeholder" - defer func() { - setting.Repository.AnsiCharset = old - }() - testSuccess(b, "placeholder") - - // invalid bytes - b = []byte{0xfa} - _, err = DetectEncoding(b) - assert.Error(t, err) + defer test.MockVariableValue(&setting.Repository.AnsiCharset, "MyEncoding")() + testSuccess(b, "MyEncoding") } -func stringMustStartWith(t *testing.T, expected, value string) { - assert.Equal(t, expected, value[:len(expected)]) +func stringMustStartWith(t *testing.T, expected string, value []byte) { + assert.Equal(t, expected, string(value[:len(expected)])) } -func stringMustEndWith(t *testing.T, expected, value string) { - assert.Equal(t, expected, value[len(value)-len(expected):]) +func stringMustEndWith(t *testing.T, expected string, value []byte) { + assert.Equal(t, expected, string(value[len(value)-len(expected):])) } func TestToUTF8WithFallbackReader(t *testing.T) { - resetDefaultCharsetsOrder() + test.MockVariableValue(&ToUTF8WithFallbackReaderPrefetchSize) - for testLen := range 2048 { - pattern := " test { () }\n" - input := "" - for len(input) < testLen { - input += pattern - } - input = input[:testLen] - input += "// Выключаем" - rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{}) + block := "aá啊🤔" + runes := []rune(block) + assert.Len(t, string(runes[0]), 1) + assert.Len(t, string(runes[1]), 2) + assert.Len(t, string(runes[2]), 3) + assert.Len(t, string(runes[3]), 4) + + content := strings.Repeat(block, 2) + for i := 1; i < len(content); i++ { + encoding, err := DetectEncoding([]byte(content[:i])) + assert.NoError(t, err) + assert.Equal(t, "UTF-8", encoding) + + ToUTF8WithFallbackReaderPrefetchSize = i + rd := ToUTF8WithFallbackReader(strings.NewReader(content), ConvertOpts{}) r, _ := io.ReadAll(rd) - assert.Equalf(t, input, string(r), "testing string len=%d", testLen) + assert.Equal(t, content, string(r)) + } + for _, r := range runes { + content = "abc abc " + string(r) + string(r) + string(r) + for i := 0; i < len(content); i++ { + encoding, err := DetectEncoding([]byte(content[:i])) + assert.NoError(t, err) + assert.Equal(t, "UTF-8", encoding) + } } - - truncatedOneByteExtension := failFastBytes - encoding, _ := DetectEncoding(truncatedOneByteExtension) - assert.Equal(t, "UTF-8", encoding) - - truncatedTwoByteExtension := failFastBytes - truncatedTwoByteExtension[len(failFastBytes)-1] = 0x9b - truncatedTwoByteExtension[len(failFastBytes)-2] = 0xe2 - - encoding, _ = DetectEncoding(truncatedTwoByteExtension) - assert.Equal(t, "UTF-8", encoding) - - truncatedThreeByteExtension := failFastBytes - truncatedThreeByteExtension[len(failFastBytes)-1] = 0x92 - truncatedThreeByteExtension[len(failFastBytes)-2] = 0x9f - truncatedThreeByteExtension[len(failFastBytes)-3] = 0xf0 - - encoding, _ = DetectEncoding(truncatedThreeByteExtension) - assert.Equal(t, "UTF-8", encoding) -} - -var failFastBytes = []byte{ - 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x74, 0x6f, - 0x6f, 0x6c, 0x73, 0x2e, 0x61, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x64, 0x65, 0x66, 0x73, 0x2e, 0x63, 0x6f, 0x6e, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4f, 0x73, 0x0a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67, - 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, - 0x74, 0x2e, 0x67, 0x72, 0x61, 0x64, 0x6c, 0x65, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x75, 0x6e, 0x2e, 0x42, - 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x0a, 0x0a, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x64, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, - 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, - 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x2d, 0x64, 0x6f, 0x63, 0x73, 0x22, 0x29, - 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x64, 0x62, - 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x66, - 0x73, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x6d, 0x71, 0x22, 0x29, 0x29, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, - 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2d, 0x61, 0x75, 0x74, 0x68, 0x2d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x65, 0x72, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, - 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x68, 0x61, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, - 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x29, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, - 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x77, 0x65, 0x62, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, - 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, - 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x6f, 0x70, - 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, - 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x63, 0x74, 0x75, 0x61, 0x74, 0x6f, 0x72, 0x22, 0x29, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, - 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x22, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, - 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2d, 0x61, 0x6c, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, - 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x65, 0x72, 0x2d, 0x73, 0x6c, 0x65, 0x75, 0x74, 0x68, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, - 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, - 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x72, 0x65, 0x74, 0x72, 0x79, 0x3a, - 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x63, 0x68, 0x2e, 0x71, - 0x6f, 0x73, 0x2e, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x3a, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x2d, 0x63, - 0x6c, 0x61, 0x73, 0x73, 0x69, 0x63, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x69, 0x6f, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65, - 0x74, 0x65, 0x72, 0x3a, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x2d, 0x72, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x72, 0x79, 0x2d, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6b, 0x6f, 0x74, - 0x6c, 0x69, 0x6e, 0x28, 0x22, 0x73, 0x74, 0x64, 0x6c, 0x69, 0x62, 0x22, 0x29, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, - 0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, - 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, - 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, - 0x65, 0x73, 0x74, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, - 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, - 0x74, 0x65, 0x73, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a, - 0x61, 0x72, 0x20, 0x62, 0x79, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x28, 0x4a, 0x61, 0x72, 0x3a, 0x3a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x69, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x74, 0x28, 0x22, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x64, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x76, 0x61, 0x6c, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68, - 0x20, 0x62, 0x79, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x67, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x73, 0x28, 0x22, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x2d, 0x50, 0x61, 0x74, 0x68, 0x22, 0x20, 0x74, 0x6f, 0x20, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, - 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x20, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, - 0x20, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x2f, 0x2b, 0x22, 0x2e, 0x74, 0x6f, 0x52, 0x65, 0x67, 0x65, 0x78, 0x28, 0x29, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x20, 0x66, 0x75, 0x6e, 0x20, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x3a, 0x20, 0x53, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70, - 0x61, 0x74, 0x68, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x6f, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x28, 0x22, 0x20, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x74, 0x2e, 0x74, 0x6f, 0x55, 0x52, 0x49, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x55, - 0x52, 0x4c, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, - 0x61, 0x63, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x22, 0x2f, - 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x74, 0x61, 0x73, - 0x6b, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x3c, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x3e, 0x28, 0x22, 0x62, - 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x4f, - 0x73, 0x2e, 0x69, 0x73, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x28, 0x4f, 0x73, 0x2e, 0x46, 0x41, 0x4d, 0x49, 0x4c, 0x59, - 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x28, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x74, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x28, 0x22, 0x6d, 0x61, 0x69, - 0x6e, 0x22, 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x20, 0x7b, 0x20, 0x69, 0x74, 0x2e, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x20, - 0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a, 0x61, 0x72, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0xd0, } diff --git a/modules/charset/escape.go b/modules/charset/escape.go index 92e417d1f7..167683a298 100644 --- a/modules/charset/escape.go +++ b/modules/charset/escape.go @@ -20,14 +20,17 @@ import ( // RuneNBSP is the codepoint for NBSP const RuneNBSP = 0xa0 -// EscapeControlHTML escapes the unicode control sequences in a provided html document +// EscapeControlHTML escapes the Unicode control sequences in a provided html document func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) { + if !setting.UI.AmbiguousUnicodeDetection { + return &EscapeStatus{}, html + } sb := &strings.Builder{} escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader return escaped, template.HTML(sb.String()) } -// EscapeControlReader escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus +// EscapeControlReader escapes the Unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) { if !setting.UI.AmbiguousUnicodeDetection { _, err = io.Copy(writer, reader) diff --git a/modules/git/blame.go b/modules/git/blame.go deleted file mode 100644 index 601be96f05..0000000000 --- a/modules/git/blame.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "bufio" - "bytes" - "context" - "io" - "os" - - "code.gitea.io/gitea/modules/git/gitcmd" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -// BlamePart represents block of blame - continuous lines with one sha -type BlamePart struct { - Sha string - Lines []string - PreviousSha string - PreviousPath string -} - -// BlameReader returns part of file blame one by one -type BlameReader struct { - output io.WriteCloser - reader io.ReadCloser - bufferedReader *bufio.Reader - done chan error - lastSha *string - ignoreRevsFile string - objectFormat ObjectFormat - cleanupFuncs []func() -} - -func (r *BlameReader) UsesIgnoreRevs() bool { - return r.ignoreRevsFile != "" -} - -// NextPart returns next part of blame (sequential code lines with the same commit) -func (r *BlameReader) NextPart() (*BlamePart, error) { - var blamePart *BlamePart - - if r.lastSha != nil { - blamePart = &BlamePart{ - Sha: *r.lastSha, - Lines: make([]string, 0), - } - } - - const previousHeader = "previous " - var lineBytes []byte - var isPrefix bool - var err error - - for err != io.EOF { - lineBytes, isPrefix, err = r.bufferedReader.ReadLine() - if err != nil && err != io.EOF { - return blamePart, err - } - - if len(lineBytes) == 0 { - // isPrefix will be false - continue - } - - var objectID string - objectFormatLength := r.objectFormat.FullLength() - - if len(lineBytes) > objectFormatLength && lineBytes[objectFormatLength] == ' ' && r.objectFormat.IsValid(string(lineBytes[0:objectFormatLength])) { - objectID = string(lineBytes[0:objectFormatLength]) - } - if len(objectID) > 0 { - if blamePart == nil { - blamePart = &BlamePart{ - Sha: objectID, - Lines: make([]string, 0), - } - } - - if blamePart.Sha != objectID { - r.lastSha = &objectID - // need to munch to end of line... - for isPrefix { - _, isPrefix, err = r.bufferedReader.ReadLine() - if err != nil && err != io.EOF { - return blamePart, err - } - } - return blamePart, nil - } - } else if lineBytes[0] == '\t' { - blamePart.Lines = append(blamePart.Lines, string(lineBytes[1:])) - } else if bytes.HasPrefix(lineBytes, []byte(previousHeader)) { - offset := len(previousHeader) // already includes a space - blamePart.PreviousSha = string(lineBytes[offset : offset+objectFormatLength]) - offset += objectFormatLength + 1 // +1 for space - blamePart.PreviousPath = string(lineBytes[offset:]) - } - - // need to munch to end of line... - for isPrefix { - _, isPrefix, err = r.bufferedReader.ReadLine() - if err != nil && err != io.EOF { - return blamePart, err - } - } - } - - r.lastSha = nil - - return blamePart, nil -} - -// Close BlameReader - don't run NextPart after invoking that -func (r *BlameReader) Close() error { - if r.bufferedReader == nil { - return nil - } - - err := <-r.done - r.bufferedReader = nil - _ = r.reader.Close() - _ = r.output.Close() - for _, cleanup := range r.cleanupFuncs { - if cleanup != nil { - cleanup() - } - } - return err -} - -// CreateBlameReader creates reader for given repository, commit and file -func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) { - var ignoreRevsFileName string - var ignoreRevsFileCleanup func() - defer func() { - if err != nil && ignoreRevsFileCleanup != nil { - ignoreRevsFileCleanup() - } - }() - - cmd := gitcmd.NewCommand("blame", "--porcelain") - - if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { - ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit) - if err != nil && !IsErrNotExist(err) { - return nil, err - } - if ignoreRevsFileName != "" { - // Possible improvement: use --ignore-revs-file /dev/stdin on unix - // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. - cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) - } - } - - cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) - - done := make(chan error, 1) - reader, stdout, err := os.Pipe() - if err != nil { - return nil, err - } - go func() { - stderr := bytes.Buffer{} - // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" - err := cmd.WithDir(repoPath). - WithUseContextTimeout(true). - WithStdout(stdout). - WithStderr(&stderr). - Run(ctx) - done <- err - _ = stdout.Close() - if err != nil { - log.Error("Error running git blame (dir: %v): %v, stderr: %v", repoPath, err, stderr.String()) - } - }() - - bufferedReader := bufio.NewReader(reader) - return &BlameReader{ - output: stdout, - reader: reader, - bufferedReader: bufferedReader, - done: done, - ignoreRevsFile: ignoreRevsFileName, - objectFormat: objectFormat, - cleanupFuncs: []func(){ignoreRevsFileCleanup}, - }, nil -} - -func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func(), error) { - entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") - if err != nil { - return "", nil, err - } - - r, err := entry.Blob().DataAsync() - if err != nil { - return "", nil, err - } - defer r.Close() - - f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") - if err != nil { - return "", nil, err - } - filename := f.Name() - _, err = io.Copy(f, r) - _ = f.Close() - if err != nil { - cleanup() - return "", nil, err - } - - return filename, cleanup, nil -} diff --git a/modules/git/commit.go b/modules/git/commit.go index af09697018..1917a72bbf 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -323,14 +323,6 @@ func GetFullCommitID(ctx context.Context, repoPath, shortID string) (string, err return strings.TrimSpace(commitID), nil } -// GetRepositoryDefaultPublicGPGKey returns the default public key for this commit -func (c *Commit) GetRepositoryDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, error) { - if c.repo == nil { - return nil, nil - } - return c.repo.GetDefaultPublicGPGKey(forceUpdate) -} - func IsStringLikelyCommitID(objFmt ObjectFormat, s string, minLength ...int) bool { maxLen := 64 // sha256 if objFmt != nil { diff --git a/modules/git/diff.go b/modules/git/diff.go index 437b26eb05..309d8f4615 100644 --- a/modules/git/diff.go +++ b/modules/git/diff.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) // RawDiffType type of a raw diff. @@ -32,20 +33,6 @@ func GetRawDiff(repo *Repository, commitID string, diffType RawDiffType, writer return GetRepoRawDiffForFile(repo, "", commitID, diffType, "", writer) } -// GetReverseRawDiff dumps the reverse diff results of repository in given commit ID to io.Writer. -func GetReverseRawDiff(ctx context.Context, repoPath, commitID string, writer io.Writer) error { - stderr := new(bytes.Buffer) - if err := gitcmd.NewCommand("show", "--pretty=format:revert %H%n", "-R"). - AddDynamicArguments(commitID). - WithDir(repoPath). - WithStdout(writer). - WithStderr(stderr). - Run(ctx); err != nil { - return fmt.Errorf("Run: %w - %s", err, stderr) - } - return nil -} - // GetRepoRawDiffForFile dumps diff results of file in given commit ID to io.Writer according given repository func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diffType RawDiffType, file string, writer io.Writer) error { commit, err := repo.GetCommit(endCommit) @@ -61,7 +48,9 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff switch diffType { case RawDiffNormal: if len(startCommit) != 0 { - cmd.AddArguments("diff", "-M").AddDynamicArguments(startCommit, endCommit).AddDashesAndList(files...) + cmd.AddArguments("diff"). + AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold). + AddDynamicArguments(startCommit, endCommit).AddDashesAndList(files...) } else if commit.ParentCount() == 0 { cmd.AddArguments("show").AddDynamicArguments(endCommit).AddDashesAndList(files...) } else { @@ -69,7 +58,9 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff if err != nil { return err } - cmd.AddArguments("diff", "-M").AddDynamicArguments(c.ID.String(), endCommit).AddDashesAndList(files...) + cmd.AddArguments("diff"). + AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold). + AddDynamicArguments(c.ID.String(), endCommit).AddDashesAndList(files...) } case RawDiffPatch: if len(startCommit) != 0 { diff --git a/modules/git/gpg.go b/modules/git/gpg.go new file mode 100644 index 0000000000..dbc5569309 --- /dev/null +++ b/modules/git/gpg.go @@ -0,0 +1,102 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + + "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/process" +) + +// GPGSettings represents the default GPG settings for this repository +type GPGSettings struct { + Sign bool + KeyID string + Email string + Name string + PublicKeyContent string + Format string +} + +// LoadPublicKeyContent will load the key from gpg +func (gpgSettings *GPGSettings) LoadPublicKeyContent() error { + if gpgSettings.PublicKeyContent != "" { + return nil + } + + if gpgSettings.Format == SigningKeyFormatSSH { + content, err := os.ReadFile(gpgSettings.KeyID) + if err != nil { + return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err) + } + gpgSettings.PublicKeyContent = string(content) + return nil + } + content, stderr, err := process.GetManager().Exec( + "gpg -a --export", + "gpg", "-a", "--export", gpgSettings.KeyID) + if err != nil { + return fmt.Errorf("unable to get default signing key: %s, %s, %w", gpgSettings.KeyID, stderr, err) + } + gpgSettings.PublicKeyContent = content + return nil +} + +var ( + loadPublicGPGKeyMutex sync.RWMutex + globalGPGSettings *GPGSettings +) + +// GetDefaultPublicGPGKey will return and cache the default public GPG settings +func GetDefaultPublicGPGKey(ctx context.Context, forceUpdate bool) (*GPGSettings, error) { + if !forceUpdate { + loadPublicGPGKeyMutex.RLock() + if globalGPGSettings != nil { + defer loadPublicGPGKeyMutex.RUnlock() + return globalGPGSettings, nil + } + loadPublicGPGKeyMutex.RUnlock() + } + + loadPublicGPGKeyMutex.Lock() + defer loadPublicGPGKeyMutex.Unlock() + + if globalGPGSettings != nil && !forceUpdate { + return globalGPGSettings, nil + } + + globalGPGSettings = &GPGSettings{ + Sign: true, + } + + value, _, _ := gitcmd.NewCommand("config", "--global", "--get", "commit.gpgsign").RunStdString(ctx) + sign, valid := ParseBool(strings.TrimSpace(value)) + if !sign || !valid { + globalGPGSettings.Sign = false + return globalGPGSettings, nil + } + + signingKey, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.signingkey").RunStdString(ctx) + globalGPGSettings.KeyID = strings.TrimSpace(signingKey) + + format, _, _ := gitcmd.NewCommand("config", "--global", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(ctx) + globalGPGSettings.Format = strings.TrimSpace(format) + + defaultEmail, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.email").RunStdString(ctx) + globalGPGSettings.Email = strings.TrimSpace(defaultEmail) + + defaultName, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.name").RunStdString(ctx) + globalGPGSettings.Name = strings.TrimSpace(defaultName) + + if err := globalGPGSettings.LoadPublicKeyContent(); err != nil { + return nil, err + } + return globalGPGSettings, nil +} diff --git a/modules/git/key.go b/modules/git/key.go index 39e79ddbe0..9d51704595 100644 --- a/modules/git/key.go +++ b/modules/git/key.go @@ -32,23 +32,23 @@ func (s *SigningKey) String() string { } // GetSigningKey returns the KeyID and git Signature for the repo -func GetSigningKey(ctx context.Context, repoPath string) (*SigningKey, *Signature) { +func GetSigningKey(ctx context.Context) (*SigningKey, *Signature) { if setting.Repository.Signing.SigningKey == "none" { return nil, nil } if setting.Repository.Signing.SigningKey == "default" || setting.Repository.Signing.SigningKey == "" { // Can ignore the error here as it means that commit.gpgsign is not set - value, _, _ := gitcmd.NewCommand("config", "--get", "commit.gpgsign").WithDir(repoPath).RunStdString(ctx) + value, _, _ := gitcmd.NewCommand("config", "--global", "--get", "commit.gpgsign").RunStdString(ctx) sign, valid := ParseBool(strings.TrimSpace(value)) if !sign || !valid { return nil, nil } - format, _, _ := gitcmd.NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").WithDir(repoPath).RunStdString(ctx) - signingKey, _, _ := gitcmd.NewCommand("config", "--get", "user.signingkey").WithDir(repoPath).RunStdString(ctx) - signingName, _, _ := gitcmd.NewCommand("config", "--get", "user.name").WithDir(repoPath).RunStdString(ctx) - signingEmail, _, _ := gitcmd.NewCommand("config", "--get", "user.email").WithDir(repoPath).RunStdString(ctx) + format, _, _ := gitcmd.NewCommand("config", "--global", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(ctx) + signingKey, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.signingkey").RunStdString(ctx) + signingName, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.name").RunStdString(ctx) + signingEmail, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.email").RunStdString(ctx) if strings.TrimSpace(signingKey) == "" { return nil, nil diff --git a/modules/git/repo.go b/modules/git/repo.go index 7e86b10de9..579accf92e 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -20,16 +20,6 @@ import ( "code.gitea.io/gitea/modules/proxy" ) -// GPGSettings represents the default GPG settings for this repository -type GPGSettings struct { - Sign bool - KeyID string - Email string - Name string - PublicKeyContent string - Format string -} - const prettyLogFormat = `--pretty=format:%H` func (repo *Repository) ShowPrettyFormatLogToList(ctx context.Context, revisionRange string) ([]*Commit, error) { @@ -123,6 +113,8 @@ type CloneRepoOptions struct { Depth int Filter string SkipTLSVerify bool + SingleBranch bool + Env []string } // Clone clones original repository to target path. @@ -157,6 +149,9 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { if opts.Filter != "" { cmd.AddArguments("--filter").AddDynamicArguments(opts.Filter) } + if opts.SingleBranch { + cmd.AddArguments("--single-branch") + } if len(opts.Branch) > 0 { cmd.AddArguments("-b").AddDynamicArguments(opts.Branch) } @@ -167,13 +162,17 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { } envs := os.Environ() - u, err := url.Parse(from) - if err == nil { - envs = proxy.EnvWithProxy(u) + if opts.Env != nil { + envs = opts.Env + } else { + u, err := url.Parse(from) + if err == nil { + envs = proxy.EnvWithProxy(u) + } } stderr := new(bytes.Buffer) - if err = cmd. + if err := cmd. WithTimeout(opts.Timeout). WithEnv(envs). WithStdout(io.Discard). @@ -186,18 +185,21 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { // PushOptions options when push to remote type PushOptions struct { - Remote string - Branch string - Force bool - Mirror bool - Env []string - Timeout time.Duration + Remote string + Branch string + Force bool + ForceWithLease string + Mirror bool + Env []string + Timeout time.Duration } // Push pushs local commits to given remote branch. func Push(ctx context.Context, repoPath string, opts PushOptions) error { cmd := gitcmd.NewCommand("push") - if opts.Force { + if opts.ForceWithLease != "" { + cmd.AddOptionFormat("--force-with-lease=%s", opts.ForceWithLease) + } else if opts.Force { cmd.AddArguments("-f") } if opts.Mirror { @@ -225,14 +227,3 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { return nil } - -// GetLatestCommitTime returns time for latest commit in repository (across all branches) -func GetLatestCommitTime(ctx context.Context, repoPath string) (time.Time, error) { - cmd := gitcmd.NewCommand("for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)") - stdout, _, err := cmd.WithDir(repoPath).RunStdString(ctx) - if err != nil { - return time.Time{}, err - } - commitTime := strings.TrimSpace(stdout) - return time.Parse("Mon Jan _2 15:04:05 2006 -0700", commitTime) -} diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index e0d0b45372..986264fd93 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -32,7 +32,6 @@ type Repository struct { gogitRepo *gogit.Repository gogitStorage *filesystem.Storage - gpgSettings *GPGSettings Ctx context.Context LastCommitCache *LastCommitCache diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index 4091e70846..17c71da5ef 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -23,8 +23,6 @@ type Repository struct { tagCache *ObjectCache[*Tag] - gpgSettings *GPGSettings - batchInUse bool batch *Batch diff --git a/modules/git/repo_gpg.go b/modules/git/repo_gpg.go deleted file mode 100644 index eb1e71e30a..0000000000 --- a/modules/git/repo_gpg.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -import ( - "fmt" - "os" - "strings" - - "code.gitea.io/gitea/modules/git/gitcmd" - "code.gitea.io/gitea/modules/process" -) - -// LoadPublicKeyContent will load the key from gpg -func (gpgSettings *GPGSettings) LoadPublicKeyContent() error { - if gpgSettings.Format == SigningKeyFormatSSH { - content, err := os.ReadFile(gpgSettings.KeyID) - if err != nil { - return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err) - } - gpgSettings.PublicKeyContent = string(content) - return nil - } - content, stderr, err := process.GetManager().Exec( - "gpg -a --export", - "gpg", "-a", "--export", gpgSettings.KeyID) - if err != nil { - return fmt.Errorf("unable to get default signing key: %s, %s, %w", gpgSettings.KeyID, stderr, err) - } - gpgSettings.PublicKeyContent = content - return nil -} - -// GetDefaultPublicGPGKey will return and cache the default public GPG settings for this repository -func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, error) { - if repo.gpgSettings != nil && !forceUpdate { - return repo.gpgSettings, nil - } - - gpgSettings := &GPGSettings{ - Sign: true, - } - - value, _, _ := gitcmd.NewCommand("config", "--get", "commit.gpgsign").WithDir(repo.Path).RunStdString(repo.Ctx) - sign, valid := ParseBool(strings.TrimSpace(value)) - if !sign || !valid { - gpgSettings.Sign = false - repo.gpgSettings = gpgSettings - return gpgSettings, nil - } - - signingKey, _, _ := gitcmd.NewCommand("config", "--get", "user.signingkey").WithDir(repo.Path).RunStdString(repo.Ctx) - gpgSettings.KeyID = strings.TrimSpace(signingKey) - - format, _, _ := gitcmd.NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").WithDir(repo.Path).RunStdString(repo.Ctx) - gpgSettings.Format = strings.TrimSpace(format) - - defaultEmail, _, _ := gitcmd.NewCommand("config", "--get", "user.email").WithDir(repo.Path).RunStdString(repo.Ctx) - gpgSettings.Email = strings.TrimSpace(defaultEmail) - - defaultName, _, _ := gitcmd.NewCommand("config", "--get", "user.name").WithDir(repo.Path).RunStdString(repo.Ctx) - gpgSettings.Name = strings.TrimSpace(defaultName) - - if err := gpgSettings.LoadPublicKeyContent(); err != nil { - return nil, err - } - repo.gpgSettings = gpgSettings - return repo.gpgSettings, nil -} diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go index 26ee3a091a..776c297a34 100644 --- a/modules/git/repo_test.go +++ b/modules/git/repo_test.go @@ -10,16 +10,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGetLatestCommitTime(t *testing.T) { - bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") - lct, err := GetLatestCommitTime(t.Context(), bareRepo1Path) - assert.NoError(t, err) - // Time is Sun Nov 13 16:40:14 2022 +0100 - // which is the time of commit - // ce064814f4a0d337b333e646ece456cd39fab612 (refs/heads/master) - assert.EqualValues(t, 1668354014, lct.Unix()) -} - func TestRepoIsEmpty(t *testing.T) { emptyRepo2Path := filepath.Join(testReposDir, "repo2_empty") repo, err := OpenRepository(t.Context(), emptyRepo2Path) diff --git a/modules/gitrepo/blame.go b/modules/gitrepo/blame.go index 3ce808d9b3..bd64c748d4 100644 --- a/modules/gitrepo/blame.go +++ b/modules/gitrepo/blame.go @@ -4,9 +4,16 @@ package gitrepo import ( + "bufio" + "bytes" "context" + "io" + "os" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) func LineBlame(ctx context.Context, repo Repository, revision, file string, line uint) (string, error) { @@ -16,3 +23,204 @@ func LineBlame(ctx context.Context, repo Repository, revision, file string, line AddOptionValues("-p", revision). AddDashesAndList(file)) } + +// BlamePart represents block of blame - continuous lines with one sha +type BlamePart struct { + Sha string + Lines []string + PreviousSha string + PreviousPath string +} + +// BlameReader returns part of file blame one by one +type BlameReader struct { + output io.WriteCloser + reader io.ReadCloser + bufferedReader *bufio.Reader + done chan error + lastSha *string + ignoreRevsFile string + objectFormat git.ObjectFormat + cleanupFuncs []func() +} + +func (r *BlameReader) UsesIgnoreRevs() bool { + return r.ignoreRevsFile != "" +} + +// NextPart returns next part of blame (sequential code lines with the same commit) +func (r *BlameReader) NextPart() (*BlamePart, error) { + var blamePart *BlamePart + + if r.lastSha != nil { + blamePart = &BlamePart{ + Sha: *r.lastSha, + Lines: make([]string, 0), + } + } + + const previousHeader = "previous " + var lineBytes []byte + var isPrefix bool + var err error + + for err != io.EOF { + lineBytes, isPrefix, err = r.bufferedReader.ReadLine() + if err != nil && err != io.EOF { + return blamePart, err + } + + if len(lineBytes) == 0 { + // isPrefix will be false + continue + } + + var objectID string + objectFormatLength := r.objectFormat.FullLength() + + if len(lineBytes) > objectFormatLength && lineBytes[objectFormatLength] == ' ' && r.objectFormat.IsValid(string(lineBytes[0:objectFormatLength])) { + objectID = string(lineBytes[0:objectFormatLength]) + } + if len(objectID) > 0 { + if blamePart == nil { + blamePart = &BlamePart{ + Sha: objectID, + Lines: make([]string, 0), + } + } + + if blamePart.Sha != objectID { + r.lastSha = &objectID + // need to munch to end of line... + for isPrefix { + _, isPrefix, err = r.bufferedReader.ReadLine() + if err != nil && err != io.EOF { + return blamePart, err + } + } + return blamePart, nil + } + } else if lineBytes[0] == '\t' { + blamePart.Lines = append(blamePart.Lines, string(lineBytes[1:])) + } else if bytes.HasPrefix(lineBytes, []byte(previousHeader)) { + offset := len(previousHeader) // already includes a space + blamePart.PreviousSha = string(lineBytes[offset : offset+objectFormatLength]) + offset += objectFormatLength + 1 // +1 for space + blamePart.PreviousPath = string(lineBytes[offset:]) + } + + // need to munch to end of line... + for isPrefix { + _, isPrefix, err = r.bufferedReader.ReadLine() + if err != nil && err != io.EOF { + return blamePart, err + } + } + } + + r.lastSha = nil + + return blamePart, nil +} + +// Close BlameReader - don't run NextPart after invoking that +func (r *BlameReader) Close() error { + if r.bufferedReader == nil { + return nil + } + + err := <-r.done + r.bufferedReader = nil + _ = r.reader.Close() + _ = r.output.Close() + for _, cleanup := range r.cleanupFuncs { + if cleanup != nil { + cleanup() + } + } + return err +} + +// CreateBlameReader creates reader for given repository, commit and file +func CreateBlameReader(ctx context.Context, objectFormat git.ObjectFormat, repo Repository, commit *git.Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) { + var ignoreRevsFileName string + var ignoreRevsFileCleanup func() + defer func() { + if err != nil && ignoreRevsFileCleanup != nil { + ignoreRevsFileCleanup() + } + }() + + cmd := gitcmd.NewCommand("blame", "--porcelain") + + if git.DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { + ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit) + if err != nil && !git.IsErrNotExist(err) { + return nil, err + } + if ignoreRevsFileName != "" { + // Possible improvement: use --ignore-revs-file /dev/stdin on unix + // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. + cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) + } + } + + cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) + + done := make(chan error, 1) + reader, stdout, err := os.Pipe() + if err != nil { + return nil, err + } + go func() { + stderr := bytes.Buffer{} + // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" + err := RunCmd(ctx, repo, cmd.WithUseContextTimeout(true). + WithStdout(stdout). + WithStderr(&stderr), + ) + done <- err + _ = stdout.Close() + if err != nil { + log.Error("Error running git blame (dir: %v): %v, stderr: %v", repoPath, err, stderr.String()) + } + }() + + bufferedReader := bufio.NewReader(reader) + return &BlameReader{ + output: stdout, + reader: reader, + bufferedReader: bufferedReader, + done: done, + ignoreRevsFile: ignoreRevsFileName, + objectFormat: objectFormat, + cleanupFuncs: []func(){ignoreRevsFileCleanup}, + }, nil +} + +func tryCreateBlameIgnoreRevsFile(commit *git.Commit) (string, func(), error) { + entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") + if err != nil { + return "", nil, err + } + + r, err := entry.Blob().DataAsync() + if err != nil { + return "", nil, err + } + defer r.Close() + + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") + if err != nil { + return "", nil, err + } + filename := f.Name() + _, err = io.Copy(f, r) + _ = f.Close() + if err != nil { + cleanup() + return "", nil, err + } + + return filename, cleanup, nil +} diff --git a/modules/git/blame_sha256_test.go b/modules/gitrepo/blame_sha256_test.go similarity index 88% rename from modules/git/blame_sha256_test.go rename to modules/gitrepo/blame_sha256_test.go index c0a97bed3b..e92931d596 100644 --- a/modules/git/blame_sha256_test.go +++ b/modules/gitrepo/blame_sha256_test.go @@ -1,12 +1,13 @@ // Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package gitrepo import ( "context" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -17,13 +18,14 @@ func TestReadingBlameOutputSha256(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - if isGogit { + if git.DefaultFeatures().UsingGogit { t.Skip("Skipping test since gogit does not support sha256") return } t.Run("Without .git-blame-ignore-revs", func(t *testing.T) { - repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls_sha256") + storage := &mockRepository{path: "repo5_pulls_sha256"} + repo, err := OpenRepository(ctx, storage) assert.NoError(t, err) defer repo.Close() @@ -47,7 +49,7 @@ func TestReadingBlameOutputSha256(t *testing.T) { } for _, bypass := range []bool{false, true} { - blameReader, err := CreateBlameReader(ctx, Sha256ObjectFormat, "./tests/repos/repo5_pulls_sha256", commit, "README.md", bypass) + blameReader, err := CreateBlameReader(ctx, git.Sha256ObjectFormat, storage, commit, "README.md", bypass) assert.NoError(t, err) assert.NotNil(t, blameReader) defer blameReader.Close() @@ -68,7 +70,8 @@ func TestReadingBlameOutputSha256(t *testing.T) { }) t.Run("With .git-blame-ignore-revs", func(t *testing.T) { - repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame_sha256") + storage := &mockRepository{path: "repo6_blame_sha256"} + repo, err := OpenRepository(ctx, storage) assert.NoError(t, err) defer repo.Close() @@ -131,7 +134,7 @@ func TestReadingBlameOutputSha256(t *testing.T) { for _, c := range cases { commit, err := repo.GetCommit(c.CommitID) assert.NoError(t, err) - blameReader, err := CreateBlameReader(ctx, objectFormat, "./tests/repos/repo6_blame_sha256", commit, "blame.txt", c.Bypass) + blameReader, err := CreateBlameReader(ctx, objectFormat, storage, commit, "blame.txt", c.Bypass) assert.NoError(t, err) assert.NotNil(t, blameReader) defer blameReader.Close() diff --git a/modules/git/blame_test.go b/modules/gitrepo/blame_test.go similarity index 89% rename from modules/git/blame_test.go rename to modules/gitrepo/blame_test.go index 809d6fbcf7..0307a5fd33 100644 --- a/modules/git/blame_test.go +++ b/modules/gitrepo/blame_test.go @@ -1,12 +1,13 @@ // Copyright 2019 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package git +package gitrepo import ( "context" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -18,10 +19,10 @@ func TestReadingBlameOutput(t *testing.T) { defer cancel() t.Run("Without .git-blame-ignore-revs", func(t *testing.T) { - repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls") + storage := &mockRepository{path: "repo5_pulls"} + repo, err := OpenRepository(ctx, storage) assert.NoError(t, err) defer repo.Close() - commit, err := repo.GetCommit("f32b0a9dfd09a60f616f29158f772cedd89942d2") assert.NoError(t, err) @@ -42,7 +43,7 @@ func TestReadingBlameOutput(t *testing.T) { } for _, bypass := range []bool{false, true} { - blameReader, err := CreateBlameReader(ctx, Sha1ObjectFormat, "./tests/repos/repo5_pulls", commit, "README.md", bypass) + blameReader, err := CreateBlameReader(ctx, git.Sha1ObjectFormat, storage, commit, "README.md", bypass) assert.NoError(t, err) assert.NotNil(t, blameReader) defer blameReader.Close() @@ -63,7 +64,8 @@ func TestReadingBlameOutput(t *testing.T) { }) t.Run("With .git-blame-ignore-revs", func(t *testing.T) { - repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame") + storage := &mockRepository{path: "repo6_blame"} + repo, err := OpenRepository(ctx, storage) assert.NoError(t, err) defer repo.Close() @@ -127,7 +129,7 @@ func TestReadingBlameOutput(t *testing.T) { commit, err := repo.GetCommit(c.CommitID) assert.NoError(t, err) - blameReader, err := CreateBlameReader(ctx, objectFormat, "./tests/repos/repo6_blame", commit, "blame.txt", c.Bypass) + blameReader, err := CreateBlameReader(ctx, objectFormat, storage, commit, "blame.txt", c.Bypass) assert.NoError(t, err) assert.NotNil(t, blameReader) defer blameReader.Close() diff --git a/modules/gitrepo/clone.go b/modules/gitrepo/clone.go index 8c437f657c..a0e4cc814c 100644 --- a/modules/gitrepo/clone.go +++ b/modules/gitrepo/clone.go @@ -18,3 +18,7 @@ func CloneExternalRepo(ctx context.Context, fromRemoteURL string, toRepo Reposit func CloneRepoToLocal(ctx context.Context, fromRepo Repository, toLocalPath string, opts git.CloneRepoOptions) error { return git.Clone(ctx, repoPath(fromRepo), toLocalPath, opts) } + +func Clone(ctx context.Context, fromRepo, toRepo Repository, opts git.CloneRepoOptions) error { + return git.Clone(ctx, repoPath(fromRepo), repoPath(toRepo), opts) +} diff --git a/modules/gitrepo/commit.go b/modules/gitrepo/commit.go index e0a87ac10b..da0f3b85a2 100644 --- a/modules/gitrepo/commit.go +++ b/modules/gitrepo/commit.go @@ -7,6 +7,7 @@ import ( "context" "strconv" "strings" + "time" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" @@ -94,3 +95,18 @@ func AllCommitsCount(ctx context.Context, repo Repository, hidePRRefs bool, file return strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) } + +func GetFullCommitID(ctx context.Context, repo Repository, shortID string) (string, error) { + return git.GetFullCommitID(ctx, repoPath(repo), shortID) +} + +// GetLatestCommitTime returns time for latest commit in repository (across all branches) +func GetLatestCommitTime(ctx context.Context, repo Repository) (time.Time, error) { + stdout, err := RunCmdString(ctx, repo, + gitcmd.NewCommand("for-each-ref", "--sort=-committerdate", git.BranchPrefix, "--count", "1", "--format=%(committerdate)")) + if err != nil { + return time.Time{}, err + } + commitTime := strings.TrimSpace(stdout) + return time.Parse("Mon Jan _2 15:04:05 2006 -0700", commitTime) +} diff --git a/modules/gitrepo/commit_test.go b/modules/gitrepo/commit_test.go index 93483f3e0d..05cedc39ef 100644 --- a/modules/gitrepo/commit_test.go +++ b/modules/gitrepo/commit_test.go @@ -33,3 +33,13 @@ func TestCommitsCountWithoutBase(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(2), commitsCount) } + +func TestGetLatestCommitTime(t *testing.T) { + bareRepo1 := &mockRepository{path: "repo1_bare"} + lct, err := GetLatestCommitTime(t.Context(), bareRepo1) + assert.NoError(t, err) + // Time is Sun Nov 13 16:40:14 2022 +0100 + // which is the time of commit + // ce064814f4a0d337b333e646ece456cd39fab612 (refs/heads/master) + assert.EqualValues(t, 1668354014, lct.Unix()) +} diff --git a/modules/gitrepo/diff.go b/modules/gitrepo/diff.go index c98c3ffcfe..ad7f24762f 100644 --- a/modules/gitrepo/diff.go +++ b/modules/gitrepo/diff.go @@ -4,8 +4,10 @@ package gitrepo import ( + "bytes" "context" "fmt" + "io" "regexp" "strconv" @@ -60,3 +62,15 @@ func parseDiffStat(stdout string) (numFiles, totalAdditions, totalDeletions int, } return numFiles, totalAdditions, totalDeletions, err } + +// GetReverseRawDiff dumps the reverse diff results of repository in given commit ID to io.Writer. +func GetReverseRawDiff(ctx context.Context, repo Repository, commitID string, writer io.Writer) error { + stderr := new(bytes.Buffer) + if err := RunCmd(ctx, repo, gitcmd.NewCommand("show", "--pretty=format:revert %H%n", "-R"). + AddDynamicArguments(commitID). + WithStdout(writer). + WithStderr(stderr)); err != nil { + return fmt.Errorf("GetReverseRawDiff: %w - %s", err, stderr) + } + return nil +} diff --git a/modules/gitrepo/gitrepo.go b/modules/gitrepo/gitrepo.go index 4dd03c18fe..3a9b0a1c89 100644 --- a/modules/gitrepo/gitrepo.go +++ b/modules/gitrepo/gitrepo.go @@ -98,3 +98,23 @@ func UpdateServerInfo(ctx context.Context, repo Repository) error { func GetRepoFS(repo Repository) fs.FS { return os.DirFS(repoPath(repo)) } + +func IsRepoFileExist(ctx context.Context, repo Repository, relativeFilePath string) (bool, error) { + absoluteFilePath := filepath.Join(repoPath(repo), relativeFilePath) + return util.IsExist(absoluteFilePath) +} + +func IsRepoDirExist(ctx context.Context, repo Repository, relativeDirPath string) (bool, error) { + absoluteDirPath := filepath.Join(repoPath(repo), relativeDirPath) + return util.IsDir(absoluteDirPath) +} + +func RemoveRepoFileOrDir(ctx context.Context, repo Repository, relativeFileOrDirPath string) error { + absoluteFilePath := filepath.Join(repoPath(repo), relativeFileOrDirPath) + return util.Remove(absoluteFilePath) +} + +func CreateRepoFile(ctx context.Context, repo Repository, relativeFilePath string) (io.WriteCloser, error) { + absoluteFilePath := filepath.Join(repoPath(repo), relativeFilePath) + return os.Create(absoluteFilePath) +} diff --git a/modules/gitrepo/push.go b/modules/gitrepo/push.go index 18808cac24..920c317f79 100644 --- a/modules/gitrepo/push.go +++ b/modules/gitrepo/push.go @@ -9,6 +9,19 @@ import ( "code.gitea.io/gitea/modules/git" ) -func Push(ctx context.Context, repo Repository, opts git.PushOptions) error { +// PushToExternal pushes a managed repository to an external remote. +func PushToExternal(ctx context.Context, repo Repository, opts git.PushOptions) error { return git.Push(ctx, repoPath(repo), opts) } + +// Push pushes from one managed repository to another managed repository. +func Push(ctx context.Context, fromRepo, toRepo Repository, opts git.PushOptions) error { + opts.Remote = repoPath(toRepo) + return git.Push(ctx, repoPath(fromRepo), opts) +} + +// PushFromLocal pushes from a local path to a managed repository. +func PushFromLocal(ctx context.Context, fromLocalPath string, toRepo Repository, opts git.PushOptions) error { + opts.Remote = repoPath(toRepo) + return git.Push(ctx, fromLocalPath, opts) +} diff --git a/modules/gitrepo/signing.go b/modules/gitrepo/signing.go index c50978d15a..2f77758d8c 100644 --- a/modules/gitrepo/signing.go +++ b/modules/gitrepo/signing.go @@ -9,6 +9,6 @@ import ( "code.gitea.io/gitea/modules/git" ) -func GetSigningKey(ctx context.Context, repo Repository) (*git.SigningKey, *git.Signature) { - return git.GetSigningKey(ctx, repoPath(repo)) +func GetSigningKey(ctx context.Context) (*git.SigningKey, *git.Signature) { + return git.GetSigningKey(ctx) } diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index 77e47fdf48..2b13e9c4ce 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -56,7 +56,39 @@ func NewContext() { }) } -// Code returns a HTML version of code string with chroma syntax highlighting classes and the matched lexer name +// UnsafeSplitHighlightedLines splits highlighted code into lines preserving HTML tags +// It always includes '\n', '\n' can appear at the end of each line or in the middle of HTML tags +// The '\n' is necessary for copying code from web UI to preserve original code lines +// ATTENTION: It uses the unsafe conversion between string and []byte for performance reason +// DO NOT make any modification to the returned [][]byte slice items +func UnsafeSplitHighlightedLines(code template.HTML) (ret [][]byte) { + buf := util.UnsafeStringToBytes(string(code)) + lineCount := bytes.Count(buf, []byte("\n")) + 1 + ret = make([][]byte, 0, lineCount) + nlTagClose := []byte("\n 0 { + ret = append(ret, buf) + } + return ret + } + // Chroma highlighting output sometimes have "" right after \n, sometimes before. + // * "text\n" + // * "text\n" + if bytes.HasPrefix(buf[pos:], nlTagClose) { + pos1 := bytes.IndexByte(buf[pos:], '>') + if pos1 != -1 { + pos += pos1 + } + } + ret = append(ret, buf[:pos+1]) + buf = buf[pos+1:] + } +} + +// Code returns an HTML version of code string with chroma syntax highlighting classes and the matched lexer name func Code(fileName, language, code string) (output template.HTML, lexerName string) { NewContext() diff --git a/modules/highlight/highlight_test.go b/modules/highlight/highlight_test.go index b36de98c5c..52873427a8 100644 --- a/modules/highlight/highlight_test.go +++ b/modules/highlight/highlight_test.go @@ -181,3 +181,21 @@ c=2`), }) } } + +func TestUnsafeSplitHighlightedLines(t *testing.T) { + ret := UnsafeSplitHighlightedLines("") + assert.Empty(t, ret) + + ret = UnsafeSplitHighlightedLines("a") + assert.Len(t, ret, 1) + assert.Equal(t, "a", string(ret[0])) + + ret = UnsafeSplitHighlightedLines("\n") + assert.Len(t, ret, 1) + assert.Equal(t, "\n", string(ret[0])) + + ret = UnsafeSplitHighlightedLines("a\nb\n") + assert.Len(t, ret, 2) + assert.Equal(t, "a\n", string(ret[0])) + assert.Equal(t, "b\n", string(ret[1])) +} diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index b4c5e7fe1e..2d66a86a8b 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -19,7 +19,6 @@ import ( charsetModule "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/httpcache" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" @@ -109,11 +108,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byt } if isPlain { - charset, err := charsetModule.DetectEncoding(mineBuf) - if err != nil { - log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err) - charset = "utf-8" - } + charset, _ := charsetModule.DetectEncoding(mineBuf) opts.ContentTypeCharset = strings.ToLower(charset) } diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index bdb477ce6e..5f6a7f6082 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -203,7 +203,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro RepoID: repo.ID, CommitID: commitSha, Filename: update.Filename, - Content: string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), + Content: string(charset.ToUTF8DropErrors(fileContents)), Language: analyze.GetCodeLanguage(update.Filename, fileContents), UpdatedAt: time.Now().UTC(), }) diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index b2eb301a5d..a7027051d2 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -191,7 +191,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro Doc(map[string]any{ "repo_id": repo.ID, "filename": update.Filename, - "content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), + "content": string(charset.ToUTF8DropErrors(fileContents)), "commit_id": sha, "language": analyze.GetCodeLanguage(update.Filename, fileContents), "updated_at": timeutil.TimeStampNow(), diff --git a/modules/indexer/issues/elasticsearch/elasticsearch_test.go b/modules/indexer/issues/elasticsearch/elasticsearch_test.go index dc329c07dd..cb9ed3889d 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch_test.go @@ -30,7 +30,11 @@ func TestElasticsearchIndexer(t *testing.T) { require.Eventually(t, func() bool { resp, err := http.Get(url) - return err == nil && resp.StatusCode == http.StatusOK + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK }, time.Minute, time.Second, "Expected elasticsearch to be up") indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix())) diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index a32cbdd6de..81a27487bb 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -36,7 +36,11 @@ func TestMeilisearchIndexer(t *testing.T) { require.Eventually(t, func() bool { resp, err := http.Get(url) - return err == nil && resp.StatusCode == http.StatusOK + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK }, time.Minute, time.Second, "Expected meilisearch to be up") indexer := NewIndexer(url, key, fmt.Sprintf("test_meilisearch_indexer_%d", time.Now().Unix())) diff --git a/modules/markup/markdown/markdown_math_test.go b/modules/markup/markdown/markdown_math_test.go index a75f18d36a..9e368cb689 100644 --- a/modules/markup/markdown/markdown_math_test.go +++ b/modules/markup/markdown/markdown_math_test.go @@ -30,6 +30,10 @@ func TestMathRender(t *testing.T) { "$ a $", `

a

` + nl, }, + { + "$a$$b$", + `

ab

` + nl, + }, { "$a$ $b$", `

a b

` + nl, @@ -59,7 +63,7 @@ func TestMathRender(t *testing.T) { `

a$b $a a$b b$

` + nl, }, { - "a$x$", + "a$x$", // Pattern: "word$other$" The real world example is: "Price is between US$1 and US$2.", so don't parse this. `

a$x$

` + nl, }, { @@ -70,6 +74,10 @@ func TestMathRender(t *testing.T) { "$a$ ($b$) [$c$] {$d$}", `

a (b) [$c$] {$d$}

` + nl, }, + { + "[$a$](link)", + `

a

` + nl, + }, { "$$a$$", `

a

` + nl, diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index a711d1e1cd..564861df90 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -54,6 +54,10 @@ func isAlphanumeric(b byte) bool { return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') } +func isInMarkdownLinkText(block text.Reader, lineAfter []byte) bool { + return block.PrecendingCharacter() == '[' && bytes.HasPrefix(lineAfter, []byte("](")) +} + // Parse parses the current line and returns a result of parsing. func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { line, _ := block.PeekLine() @@ -115,7 +119,9 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. } // check valid ending character isValidEndingChar := isPunctuation(succeedingCharacter) || isParenthesesClose(succeedingCharacter) || - succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0 + succeedingCharacter == ' ' || succeedingCharacter == '\n' || succeedingCharacter == 0 || + succeedingCharacter == '$' || + isInMarkdownLinkText(block, line[i+len(stopMark):]) if checkSurrounding && !isValidEndingChar { break } diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 11b5123c27..cc7695726b 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -62,7 +62,28 @@ type PackageMetadata struct { Author User `json:"author"` ReadmeFilename string `json:"readmeFilename,omitempty"` Users map[string]bool `json:"users,omitempty"` - License string `json:"license,omitempty"` + License License `json:"license,omitempty"` +} + +type License string + +func (l *License) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *l = License(value) + case '{': + var values map[string]any + if err := json.Unmarshal(data, &values); err != nil { + return err + } + value, _ := values["type"].(string) + *l = License(value) + } + return nil } // PackageMetadataVersion documentation: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version @@ -74,7 +95,7 @@ type PackageMetadataVersion struct { Description string `json:"description"` Author User `json:"author"` Homepage string `json:"homepage,omitempty"` - License string `json:"license,omitempty"` + License License `json:"license,omitempty"` Repository Repository `json:"repository"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index 806377a52b..40c50de91f 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/json" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParsePackage(t *testing.T) { @@ -291,11 +292,36 @@ func TestParsePackage(t *testing.T) { assert.Equal(t, packageDescription, p.Metadata.Readme) assert.Equal(t, packageAuthor, p.Metadata.Author) assert.Equal(t, packageBin, p.Metadata.Bin["bin"]) - assert.Equal(t, "MIT", p.Metadata.License) + assert.Equal(t, "MIT", string(p.Metadata.License)) assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL) assert.Contains(t, p.Metadata.Dependencies, "package") assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) assert.Equal(t, repository.Type, p.Metadata.Repository.Type) assert.Equal(t, repository.URL, p.Metadata.Repository.URL) }) + + t.Run("ValidLicenseMap", func(t *testing.T) { + packageJSON := `{ + "versions": { + "0.1.1": { + "name": "dev-null", + "version": "0.1.1", + "license": { + "type": "MIT" + }, + "dist": { + "integrity": "sha256-" + } + } + }, + "_attachments": { + "foo": { + "data": "AAAA" + } + } +}` + p, err := ParsePackage(strings.NewReader(packageJSON)) + require.NoError(t, err) + require.Equal(t, "MIT", string(p.Metadata.License)) + }) } diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index 362d0470d5..e6bbcb1177 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -12,7 +12,7 @@ type Metadata struct { Name string `json:"name,omitempty"` Description string `json:"description,omitempty"` Author string `json:"author,omitempty"` - License string `json:"license,omitempty"` + License License `json:"license,omitempty"` ProjectURL string `json:"project_url,omitempty"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index b6f9f07f98..57dc23b17f 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -337,14 +337,14 @@ func LogStartupProblem(skip int, level log.Level, format string, args ...any) { func deprecatedSetting(rootCfg ConfigProvider, oldSection, oldKey, newSection, newKey, version string) { if rootCfg.Section(oldSection).HasKey(oldKey) { - LogStartupProblem(1, log.ERROR, "Deprecation: config option `[%s].%s` presents, please use `[%s].%s` instead because this fallback will be/has been removed in %s", oldSection, oldKey, newSection, newKey, version) + LogStartupProblem(1, log.ERROR, "Deprecation: config option `[%s].%s` present, please use `[%s].%s` instead because this fallback will be/has been removed in %s", oldSection, oldKey, newSection, newKey, version) } } // deprecatedSettingDB add a hint that the configuration has been moved to database but still kept in app.ini func deprecatedSettingDB(rootCfg ConfigProvider, oldSection, oldKey string) { if rootCfg.Section(oldSection).HasKey(oldKey) { - LogStartupProblem(1, log.ERROR, "Deprecation: config option `[%s].%s` presents but it won't take effect because it has been moved to admin panel -> config setting", oldSection, oldKey) + LogStartupProblem(1, log.ERROR, "Deprecation: config option `[%s].%s` present but it won't take effect because it has been moved to admin panel -> config setting", oldSection, oldKey) } } diff --git a/modules/setting/git.go b/modules/setting/git.go index 48a4e7f30d..318f2c0cac 100644 --- a/modules/setting/git.go +++ b/modules/setting/git.go @@ -5,6 +5,7 @@ package setting import ( "path/filepath" + "regexp" "strings" "time" @@ -17,20 +18,21 @@ var Git = struct { HomePath string DisableDiffHighlight bool - MaxGitDiffLines int - MaxGitDiffLineCharacters int - MaxGitDiffFiles int - CommitsRangeSize int // CommitsRangeSize the default commits range size - BranchesRangeSize int // BranchesRangeSize the default branches range size - VerbosePush bool - VerbosePushDelay time.Duration - GCArgs []string `ini:"GC_ARGS" delim:" "` - EnableAutoGitWireProtocol bool - PullRequestPushMessage bool - LargeObjectThreshold int64 - DisableCoreProtectNTFS bool - DisablePartialClone bool - Timeout struct { + MaxGitDiffLines int + MaxGitDiffLineCharacters int + MaxGitDiffFiles int + CommitsRangeSize int // CommitsRangeSize the default commits range size + BranchesRangeSize int // BranchesRangeSize the default branches range size + VerbosePush bool + VerbosePushDelay time.Duration + GCArgs []string `ini:"GC_ARGS" delim:" "` + EnableAutoGitWireProtocol bool + PullRequestPushMessage bool + LargeObjectThreshold int64 + DisableCoreProtectNTFS bool + DisablePartialClone bool + DiffRenameSimilarityThreshold string + Timeout struct { Default int Migrate int Mirror int @@ -39,19 +41,20 @@ var Git = struct { GC int `ini:"GC"` } `ini:"git.timeout"` }{ - DisableDiffHighlight: false, - MaxGitDiffLines: 1000, - MaxGitDiffLineCharacters: 5000, - MaxGitDiffFiles: 100, - CommitsRangeSize: 50, - BranchesRangeSize: 20, - VerbosePush: true, - VerbosePushDelay: 5 * time.Second, - GCArgs: []string{}, - EnableAutoGitWireProtocol: true, - PullRequestPushMessage: true, - LargeObjectThreshold: 1024 * 1024, - DisablePartialClone: false, + DisableDiffHighlight: false, + MaxGitDiffLines: 1000, + MaxGitDiffLineCharacters: 5000, + MaxGitDiffFiles: 100, + CommitsRangeSize: 50, + BranchesRangeSize: 20, + VerbosePush: true, + VerbosePushDelay: 5 * time.Second, + GCArgs: []string{}, + EnableAutoGitWireProtocol: true, + PullRequestPushMessage: true, + LargeObjectThreshold: 1024 * 1024, + DisablePartialClone: false, + DiffRenameSimilarityThreshold: "50%", Timeout: struct { Default int Migrate int @@ -117,4 +120,9 @@ func loadGitFrom(rootCfg ConfigProvider) { } else { Git.HomePath = filepath.Clean(Git.HomePath) } + + // validate for a integer percentage between 0% and 100% + if !regexp.MustCompile(`^([0-9]|[1-9][0-9]|100)%$`).MatchString(Git.DiffRenameSimilarityThreshold) { + log.Fatal("Invalid git.DIFF_RENAME_SIMILARITY_THRESHOLD: %s", Git.DiffRenameSimilarityThreshold) + } } diff --git a/modules/setting/server.go b/modules/setting/server.go index cedca32da9..a865e942a6 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -370,6 +370,10 @@ func loadServerFrom(rootCfg ConfigProvider) { } } + // TODO: GOLANG-HTTP-TMPDIR: Some Golang packages (like "http") use os.TempDir() to create temporary files when uploading files. + // So ideally we should set the TMPDIR environment variable to make them use our managed temp directory. + // But there is no clear place to set it currently, for example: when running "install" page, the AppDataPath is not ready yet, then AppDataTempDir won't work + EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(filepath.Join(AppWorkPath, "data/tmp/pprof")) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e14997801f..dc60d99bd6 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -240,4 +240,5 @@ func PanicInDevOrTesting(msg string, a ...any) { if !IsProd || IsInTesting { panic(fmt.Sprintf(msg, a...)) } + log.Error(msg, a...) } diff --git a/modules/structs/repo.go b/modules/structs/repo.go index c1c85837fc..47973a5f6a 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -292,6 +292,21 @@ type RenameBranchRepoOption struct { Name string `json:"name" binding:"Required;GitRefName;MaxSize(100)"` } +// UpdateBranchRepoOption options when updating a branch reference in a repository +// swagger:model +type UpdateBranchRepoOption struct { + // New commit SHA (or any ref) the branch should point to + // + // required: true + NewCommitID string `json:"new_commit_id" binding:"Required"` + + // Expected old commit SHA of the branch; if provided it must match the current tip + OldCommitID string `json:"old_commit_id"` + + // Force update even if the change is not a fast-forward + Force bool `json:"force"` +} + // TransferRepoOption options when transfer a repository's ownership // swagger:model type TransferRepoOption struct { diff --git a/modules/web/routemock.go b/modules/web/routemock.go index e85b0db738..68d19475e9 100644 --- a/modules/web/routemock.go +++ b/modules/web/routemock.go @@ -46,11 +46,15 @@ func RouterMockPoint(pointName string) func(next http.Handler) http.Handler { // // Then the mock function will be executed as a middleware at the mock point. // It only takes effect in testing mode (setting.IsInTesting == true). -func RouteMock(pointName string, h any) { +func RouteMock(pointName string, h any) func() { if _, ok := routeMockPoints[pointName]; !ok { panic("route mock point not found: " + pointName) } + old := routeMockPoints[pointName] routeMockPoints[pointName] = toHandlerProvider(h) + return func() { + routeMockPoints[pointName] = old + } } // RouteMockReset resets all mock points (no mock anymore) diff --git a/modules/web/router.go b/modules/web/router.go index 5812ff69d4..5374f82a23 100644 --- a/modules/web/router.go +++ b/modules/web/router.go @@ -55,7 +55,7 @@ func NewRouter() *Router { // Use supports two middlewares func (r *Router) Use(middlewares ...any) { for _, m := range middlewares { - if m != nil { + if !isNilOrFuncNil(m) { r.chiRouter.Use(toHandlerProvider(m)) } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 9990d30c2c..56b9acb6b6 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1294,6 +1294,7 @@ commit = Commit release = Release releases = Releases tag = Tag +git_tag = Git Tag released_this = released this tagged_this = tagged this file.title = %s at %s @@ -2755,6 +2756,13 @@ release.add_tag_msg = Use the title and content of release as tag message. release.add_tag = Create Tag Only release.releases_for = Releases for %s release.tags_for = Tags for %s +release.notes = Release notes +release.generate_notes = Generate release notes +release.generate_notes_desc = Automatically add merged pull requests and a changelog link for this release. +release.previous_tag = Previous tag +release.generate_notes_tag_not_found = Tag "%s" does not exist in this repository. +release.generate_notes_target_not_found = The release target "%s" cannot be found. +release.generate_notes_missing_tag = Enter a tag name to generate release notes. branch.name = Branch Name branch.already_exists = A branch named "%s" already exists. diff --git a/options/locale/locale_fr-FR.ini b/options/locale/locale_fr-FR.ini index 221abb5d1f..886b3955bd 100644 --- a/options/locale/locale_fr-FR.ini +++ b/options/locale/locale_fr-FR.ini @@ -215,6 +215,7 @@ more=Plus buttons.heading.tooltip=Ajouter un en-tête buttons.bold.tooltip=Ajouter du texte en gras buttons.italic.tooltip=Ajouter du texte en italique +buttons.strikethrough.tooltip=Ajouté un texte barré buttons.quote.tooltip=Citer le texte buttons.code.tooltip=Ajouter du code buttons.link.tooltip=Ajouter un lien @@ -1354,8 +1355,11 @@ editor.this_file_locked=Le fichier est verrouillé editor.must_be_on_a_branch=Vous devez être sur une branche pour appliquer ou proposer des modifications à ce fichier. editor.fork_before_edit=Vous devez faire bifurquer ce dépôt pour appliquer ou proposer des modifications à ce fichier. editor.delete_this_file=Supprimer le fichier +editor.delete_this_directory=Supprimer le répertoire editor.must_have_write_access=Vous devez avoir un accès en écriture pour appliquer ou proposer des modifications à ce fichier. editor.file_delete_success=Le fichier "%s" a été supprimé. +editor.directory_delete_success=Le répertoire « %s » a été supprimé. +editor.delete_directory=Supprimer le répertoire « %s » editor.name_your_file=Nommez votre fichier… editor.filename_help=Ajoutez un dossier en entrant son nom suivi d'une barre oblique ('/'). Supprimez un dossier avec un retour arrière au début du champ. editor.or=ou @@ -1482,6 +1486,7 @@ projects.column.new_submit=Créer une colonne projects.column.new=Nouvelle colonne projects.column.set_default=Définir par défaut projects.column.set_default_desc=Les tickets et demandes d’ajout non-catégorisés seront placés dans cette colonne. +projects.column.default_column_hint=Les nouveaux tickets ajoutés à ce projet seront ajoutés dans cette colonne projects.column.delete=Supprimer la colonne projects.column.deletion_desc=La suppression d’une colonne déplace tous ses tickets dans la colonne par défaut. Continuer ? projects.column.color=Couleur diff --git a/options/locale/locale_ga-IE.ini b/options/locale/locale_ga-IE.ini index 6b9ae41e9b..6f348b1b71 100644 --- a/options/locale/locale_ga-IE.ini +++ b/options/locale/locale_ga-IE.ini @@ -215,6 +215,7 @@ more=Níos mó buttons.heading.tooltip=Cuir ceannteideal leis buttons.bold.tooltip=Cuir téacs trom leis buttons.italic.tooltip=Cuir téacs iodálach leis +buttons.strikethrough.tooltip=Cuir téacs trína chéile buttons.quote.tooltip=Téacs luaigh buttons.code.tooltip=Cuir cód leis buttons.link.tooltip=Cuir nasc leis diff --git a/options/locale/locale_pt-PT.ini b/options/locale/locale_pt-PT.ini index 0b2e57ea00..f0a5f2142a 100644 --- a/options/locale/locale_pt-PT.ini +++ b/options/locale/locale_pt-PT.ini @@ -215,6 +215,7 @@ more=Mais buttons.heading.tooltip=Adicionar cabeçalho buttons.bold.tooltip=Adicionar texto em negrito buttons.italic.tooltip=Adicionar texto em itálico +buttons.strikethrough.tooltip=Adicionar texto rasurado buttons.quote.tooltip=Citar texto buttons.code.tooltip=Adicionar código-fonte buttons.link.tooltip=Adicionar uma ligação diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 73e5b41f4e..99fe329cbd 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -215,6 +215,7 @@ more=更多的 buttons.heading.tooltip=添加标题 buttons.bold.tooltip=添加粗体文本 buttons.italic.tooltip=添加斜体文本 +buttons.strikethrough.tooltip=添加划线文本 buttons.quote.tooltip=引用文本 buttons.code.tooltip=添加代码 buttons.link.tooltip=添加链接 @@ -1355,8 +1356,11 @@ editor.this_file_locked=文件已锁定 editor.must_be_on_a_branch=您必须在某个分支上才能对此文件进行修改操作。 editor.fork_before_edit=您必须派生这个仓库才能对此文件进行修改操作。 editor.delete_this_file=删除文件 +editor.delete_this_directory=删除目录 editor.must_have_write_access=您必须具有写权限才能对此文件进行修改操作。 editor.file_delete_success=文件「%s」已删除。 +editor.directory_delete_success=目录「%s」已被删除。 +editor.delete_directory=删除目录「%s」 editor.name_your_file=命名文件… editor.filename_help=通过键入名称后跟斜线 ("/") 来添加目录。通过在输入框的开头键入「退格」来删除目录。 editor.or=或 diff --git a/package.json b/package.json index 9f848ac3ce..9566e42e88 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "chartjs-adapter-dayjs-4": "1.0.4", "chartjs-plugin-zoom": "2.2.0", "clippie": "4.1.9", + "compare-versions": "6.1.1", "cropperjs": "1.6.2", "css-loader": "7.1.2", "dayjs": "1.11.19", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6c00cd081..7d6b00f675 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: clippie: specifier: 4.1.9 version: 4.1.9 + compare-versions: + specifier: 6.1.1 + version: 6.1.1 cropperjs: specifier: 1.6.2 version: 1.6.2 @@ -1870,6 +1873,9 @@ packages: resolution: {integrity: sha512-buhp5kePrmda3vhc5B9t7pUQXAb2Tnd0qgpkIhPhkHXxJpiPJ11H0ZEU0oBpJ2QztSbzG/ZxMj/CHsYJqRHmyg==} engines: {node: '>= 12.0.0'} + compare-versions@6.1.1: + resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -5704,6 +5710,8 @@ snapshots: comment-parser@1.4.1: {} + compare-versions@6.1.1: {} + concat-map@0.0.1: {} confbox@0.1.8: {} diff --git a/routers/api/v1/admin/user.go b/routers/api/v1/admin/user.go index 6f1e2eb120..6bed410642 100644 --- a/routers/api/v1/admin/user.go +++ b/routers/api/v1/admin/user.go @@ -414,22 +414,116 @@ func SearchUsers(ctx *context.APIContext) { // in: query // description: page size of results // type: integer + // - name: sort + // in: query + // description: sort users by attribute. Supported values are + // "name", "created", "updated" and "id". + // Default is "name" + // type: string + // - name: order + // in: query + // description: sort order, either "asc" (ascending) or "desc" (descending). + // Default is "asc", ignored if "sort" is not specified. + // type: string + // - name: q + // in: query + // description: search term (username, full name, email) + // type: string + // - name: visibility + // in: query + // description: visibility filter. Supported values are + // "public", "limited" and "private". + // type: string + // - name: is_active + // in: query + // description: filter active users + // type: boolean + // - name: is_admin + // in: query + // description: filter admin users + // type: boolean + // - name: is_restricted + // in: query + // description: filter restricted users + // type: boolean + // - name: is_2fa_enabled + // in: query + // description: filter 2FA enabled users + // type: boolean + // - name: is_prohibit_login + // in: query + // description: filter login prohibited users + // type: boolean // responses: // "200": // "$ref": "#/responses/UserList" // "403": // "$ref": "#/responses/forbidden" + // "422": + // "$ref": "#/responses/validationError" listOptions := utils.GetListOptions(ctx) - users, maxResults, err := user_model.SearchUsers(ctx, user_model.SearchUserOptions{ - Actor: ctx.Doer, - Types: []user_model.UserType{user_model.UserTypeIndividual}, - LoginName: ctx.FormTrim("login_name"), - SourceID: ctx.FormInt64("source_id"), - OrderBy: db.SearchOrderByAlphabetically, - ListOptions: listOptions, - }) + orderBy := db.SearchOrderByAlphabetically + sortMode := ctx.FormString("sort") + if len(sortMode) > 0 { + sortOrder := ctx.FormString("order") + if len(sortOrder) == 0 { + sortOrder = "asc" + } + if searchModeMap, ok := user_model.AdminUserOrderByMap[sortOrder]; ok { + if order, ok := searchModeMap[sortMode]; ok { + orderBy = order + } else { + ctx.APIError(http.StatusUnprocessableEntity, fmt.Errorf("Invalid sort mode: \"%s\"", sortMode)) + return + } + } else { + ctx.APIError(http.StatusUnprocessableEntity, fmt.Errorf("Invalid sort order: \"%s\"", sortOrder)) + return + } + } + + var visible []api.VisibleType + visibilityParam := ctx.FormString("visibility") + if len(visibilityParam) > 0 { + if visibility, ok := api.VisibilityModes[visibilityParam]; ok { + visible = []api.VisibleType{visibility} + } else { + ctx.APIError(http.StatusUnprocessableEntity, fmt.Errorf("Invalid visibility: \"%s\"", visibilityParam)) + return + } + } + + searchOpts := user_model.SearchUserOptions{ + Actor: ctx.Doer, + Types: []user_model.UserType{user_model.UserTypeIndividual}, + LoginName: ctx.FormTrim("login_name"), + SourceID: ctx.FormInt64("source_id"), + Keyword: ctx.FormTrim("q"), + Visible: visible, + OrderBy: orderBy, + ListOptions: listOptions, + SearchByEmail: true, + } + + if ctx.FormString("is_active") != "" { + searchOpts.IsActive = optional.Some(ctx.FormBool("is_active")) + } + if ctx.FormString("is_admin") != "" { + searchOpts.IsAdmin = optional.Some(ctx.FormBool("is_admin")) + } + if ctx.FormString("is_restricted") != "" { + searchOpts.IsRestricted = optional.Some(ctx.FormBool("is_restricted")) + } + if ctx.FormString("is_2fa_enabled") != "" { + searchOpts.IsTwoFactorEnabled = optional.Some(ctx.FormBool("is_2fa_enabled")) + } + if ctx.FormString("is_prohibit_login") != "" { + searchOpts.IsProhibitLogin = optional.Some(ctx.FormBool("is_prohibit_login")) + } + + users, maxResults, err := user_model.SearchUsers(ctx, searchOpts) if err != nil { ctx.APIErrorInternal(err) return diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e07685759..fcf9e73057 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -152,7 +152,7 @@ func repoAssignment() func(ctx *context.APIContext) { if err != nil { if user_model.IsErrUserNotExist(err) { if redirectUserID, err := user_model.LookupUserRedirect(ctx, userName); err == nil { - context.RedirectToUser(ctx.Base, userName, redirectUserID) + context.RedirectToUser(ctx.Base, ctx.Doer, userName, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { ctx.APIErrorNotFound("GetUserByName", err) } else { @@ -612,7 +612,7 @@ func orgAssignment(args ...bool) func(ctx *context.APIContext) { if organization.IsErrOrgNotExist(err) { redirectUserID, err := user_model.LookupUserRedirect(ctx, ctx.PathParam("org")) if err == nil { - context.RedirectToUser(ctx.Base, ctx.PathParam("org"), redirectUserID) + context.RedirectToUser(ctx.Base, ctx.Doer, ctx.PathParam("org"), redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { ctx.APIErrorNotFound("GetOrgByName", err) } else { @@ -1242,6 +1242,7 @@ func Routes() *web.Router { m.Get("/*", repo.GetBranch) m.Delete("/*", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, repo.DeleteBranch) m.Post("", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, bind(api.CreateBranchRepoOption{}), repo.CreateBranch) + m.Put("/*", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, bind(api.UpdateBranchRepoOption{}), repo.UpdateBranch) m.Patch("/*", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, bind(api.RenameBranchRepoOption{}), repo.RenameBranch) }, context.ReferencesGitRepo(), reqRepoReader(unit.TypeCode)) m.Group("/branch_protections", func() { diff --git a/routers/api/v1/misc/signing.go b/routers/api/v1/misc/signing.go index db70e04b8f..6e1a9a09b2 100644 --- a/routers/api/v1/misc/signing.go +++ b/routers/api/v1/misc/signing.go @@ -10,13 +10,8 @@ import ( ) func getSigningKey(ctx *context.APIContext, expectedFormat string) { - // if the handler is in the repo's route group, get the repo's signing key - // otherwise, get the global signing key - path := "" - if ctx.Repo != nil && ctx.Repo.Repository != nil { - path = ctx.Repo.Repository.RepoPath() - } - content, format, err := asymkey_service.PublicSigningKey(ctx, path) + // get the global signing key + content, format, err := asymkey_service.PublicSigningKey(ctx) if err != nil { ctx.APIErrorInternal(err) return diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index b9060e9cbd..4624d7e738 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -380,6 +380,81 @@ func ListBranches(ctx *context.APIContext) { ctx.JSON(http.StatusOK, apiBranches) } +// UpdateBranch moves a branch reference to a new commit. +func UpdateBranch(ctx *context.APIContext) { + // swagger:operation PUT /repos/{owner}/{repo}/branches/{branch} repository repoUpdateBranch + // --- + // summary: Update a branch reference to a new commit + // consumes: + // - application/json + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: branch + // in: path + // description: name of the branch + // type: string + // required: true + // - name: body + // in: body + // schema: + // "$ref": "#/definitions/UpdateBranchRepoOption" + // responses: + // "204": + // "$ref": "#/responses/empty" + // "403": + // "$ref": "#/responses/forbidden" + // "404": + // "$ref": "#/responses/notFound" + // "409": + // "$ref": "#/responses/conflict" + // "422": + // "$ref": "#/responses/validationError" + + opt := web.GetForm(ctx).(*api.UpdateBranchRepoOption) + + branchName := ctx.PathParam("*") + repo := ctx.Repo.Repository + + if repo.IsEmpty { + ctx.APIError(http.StatusNotFound, "Git Repository is empty.") + return + } + + if repo.IsMirror { + ctx.APIError(http.StatusForbidden, "Git Repository is a mirror.") + return + } + + // permission check has been done in api.go + if err := repo_service.UpdateBranch(ctx, repo, ctx.Repo.GitRepo, ctx.Doer, branchName, opt.NewCommitID, opt.OldCommitID, opt.Force); err != nil { + switch { + case git_model.IsErrBranchNotExist(err): + ctx.APIErrorNotFound(err) + case errors.Is(err, util.ErrInvalidArgument): + ctx.APIError(http.StatusUnprocessableEntity, err) + case git.IsErrPushRejected(err): + rej := err.(*git.ErrPushRejected) + ctx.APIError(http.StatusForbidden, rej.Message) + default: + ctx.APIErrorInternal(err) + } + return + } + + ctx.Status(http.StatusNoContent) +} + // RenameBranch renames a repository's branch. func RenameBranch(ctx *context.APIContext) { // swagger:operation PATCH /repos/{owner}/{repo}/branches/{branch} repository repoRenameBranch diff --git a/routers/api/v1/swagger/options.go b/routers/api/v1/swagger/options.go index b80a9c14ba..310839374b 100644 --- a/routers/api/v1/swagger/options.go +++ b/routers/api/v1/swagger/options.go @@ -147,6 +147,8 @@ type swaggerParameterBodies struct { // in:body CreateBranchRepoOption api.CreateBranchRepoOption + // in:body + UpdateBranchRepoOption api.UpdateBranchRepoOption // in:body CreateBranchProtectionOption api.CreateBranchProtectionOption diff --git a/routers/api/v1/user/helper.go b/routers/api/v1/user/helper.go index f49bbbd6db..de3ec089df 100644 --- a/routers/api/v1/user/helper.go +++ b/routers/api/v1/user/helper.go @@ -16,7 +16,7 @@ func GetUserByPathParam(ctx *context.APIContext, name string) *user_model.User { if err != nil { if user_model.IsErrUserNotExist(err) { if redirectUserID, err2 := user_model.LookupUserRedirect(ctx, username); err2 == nil { - context.RedirectToUser(ctx.Base, username, redirectUserID) + context.RedirectToUser(ctx.Base, ctx.Doer, username, redirectUserID) } else { ctx.APIErrorNotFound("GetUserByName", err) } diff --git a/routers/common/middleware.go b/routers/common/middleware.go index bfa258b976..6bf430d361 100644 --- a/routers/common/middleware.go +++ b/routers/common/middleware.go @@ -72,8 +72,13 @@ func RequestContextHandler() func(h http.Handler) http.Handler { req = req.WithContext(cache.WithCacheContext(ctx)) ds.SetContextValue(httplib.RequestContextKey, req) ds.AddCleanUp(func() { - if req.MultipartForm != nil { - _ = req.MultipartForm.RemoveAll() // remove the temp files buffered to tmp directory + // TODO: GOLANG-HTTP-TMPDIR: Golang saves the uploaded files to temp directory (TMPDIR) when parsing multipart-form. + // The "req" might have changed due to the new "req.WithContext" calls + // For example: in NewBaseContext, a new "req" with context is created, and the multipart-form is parsed there. + // So we always use the latest "req" from the data store. + ctxReq := ds.GetContextValue(httplib.RequestContextKey).(*http.Request) + if ctxReq.MultipartForm != nil { + _ = ctxReq.MultipartForm.RemoveAll() // remove the temp files buffered to tmp directory } }) next.ServeHTTP(respWriter, req) diff --git a/routers/web/repo/blame.go b/routers/web/repo/blame.go index 0eebff6aa8..0e95a9d023 100644 --- a/routers/web/repo/blame.go +++ b/routers/web/repo/blame.go @@ -4,8 +4,9 @@ package repo import ( + "bytes" "fmt" - gotemplate "html/template" + "html/template" "net/http" "net/url" "path" @@ -16,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/languagestats" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -25,18 +27,17 @@ import ( ) type blameRow struct { - RowNumber int - Avatar gotemplate.HTML - RepoLink string - PartSha string + RowNumber int + + Avatar template.HTML PreviousSha string PreviousShaURL string - IsFirstCommit bool CommitURL string CommitMessage string - CommitSince gotemplate.HTML - Code gotemplate.HTML - EscapeStatus *charset.EscapeStatus + CommitSince template.HTML + + Code template.HTML + EscapeStatus *charset.EscapeStatus } // RefBlame render blame page @@ -99,7 +100,7 @@ func RefBlame(ctx *context.Context) { } type blameResult struct { - Parts []*git.BlamePart + Parts []*gitrepo.BlamePart UsesIgnoreRevs bool FaultyIgnoreRevsFile bool } @@ -107,7 +108,7 @@ type blameResult struct { func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git.Commit, file string, bypassBlameIgnore bool) (*blameResult, error) { objectFormat := ctx.Repo.GetObjectFormat() - blameReader, err := git.CreateBlameReader(ctx, objectFormat, repo.RepoPath(), commit, file, bypassBlameIgnore) + blameReader, err := gitrepo.CreateBlameReader(ctx, objectFormat, repo, commit, file, bypassBlameIgnore) if err != nil { return nil, err } @@ -123,7 +124,7 @@ func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git if len(r.Parts) == 0 && r.UsesIgnoreRevs { // try again without ignored revs - blameReader, err = git.CreateBlameReader(ctx, objectFormat, repo.RepoPath(), commit, file, true) + blameReader, err = gitrepo.CreateBlameReader(ctx, objectFormat, repo, commit, file, true) if err != nil { return nil, err } @@ -143,12 +144,12 @@ func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git return r, nil } -func fillBlameResult(br *git.BlameReader, r *blameResult) error { +func fillBlameResult(br *gitrepo.BlameReader, r *blameResult) error { r.UsesIgnoreRevs = br.UsesIgnoreRevs() - previousHelper := make(map[string]*git.BlamePart) + previousHelper := make(map[string]*gitrepo.BlamePart) - r.Parts = make([]*git.BlamePart, 0, 5) + r.Parts = make([]*gitrepo.BlamePart, 0, 5) for { blamePart, err := br.NextPart() if err != nil { @@ -173,7 +174,7 @@ func fillBlameResult(br *git.BlameReader, r *blameResult) error { return nil } -func processBlameParts(ctx *context.Context, blameParts []*git.BlamePart) map[string]*user_model.UserCommit { +func processBlameParts(ctx *context.Context, blameParts []*gitrepo.BlamePart) map[string]*user_model.UserCommit { // store commit data by SHA to look up avatar info etc commitNames := make(map[string]*user_model.UserCommit) // and as blameParts can reference the same commits multiple @@ -220,76 +221,64 @@ func processBlameParts(ctx *context.Context, blameParts []*git.BlamePart) map[st return commitNames } -func renderBlame(ctx *context.Context, blameParts []*git.BlamePart, commitNames map[string]*user_model.UserCommit) { - repoLink := ctx.Repo.RepoLink +func renderBlameFillFirstBlameRow(repoLink string, avatarUtils *templates.AvatarUtils, part *gitrepo.BlamePart, commit *user_model.UserCommit, br *blameRow) { + if commit.User != nil { + br.Avatar = avatarUtils.Avatar(commit.User, 18) + } else { + br.Avatar = avatarUtils.AvatarByEmail(commit.Author.Email, commit.Author.Name, 18) + } + br.PreviousSha = part.PreviousSha + br.PreviousShaURL = fmt.Sprintf("%s/blame/commit/%s/%s", repoLink, url.PathEscape(part.PreviousSha), util.PathEscapeSegments(part.PreviousPath)) + br.CommitURL = fmt.Sprintf("%s/commit/%s", repoLink, url.PathEscape(part.Sha)) + br.CommitMessage = commit.CommitMessage + br.CommitSince = templates.TimeSince(commit.Author.When) +} + +func renderBlame(ctx *context.Context, blameParts []*gitrepo.BlamePart, commitNames map[string]*user_model.UserCommit) { language, err := languagestats.GetFileLanguage(ctx, ctx.Repo.GitRepo, ctx.Repo.CommitID, ctx.Repo.TreePath) if err != nil { log.Error("Unable to get file language for %-v:%s. Error: %v", ctx.Repo.Repository, ctx.Repo.TreePath, err) } - lines := make([]string, 0) + buf := &bytes.Buffer{} rows := make([]*blameRow, 0) + avatarUtils := templates.NewAvatarUtils(ctx) + rowNumber := 0 // will be 1-based + for _, part := range blameParts { + for partLineIdx, line := range part.Lines { + rowNumber++ + + br := &blameRow{RowNumber: rowNumber} + rows = append(rows, br) + + if int64(buf.Len()) < setting.UI.MaxDisplayFileSize { + buf.WriteString(line) + buf.WriteByte('\n') + } + + if partLineIdx == 0 { + renderBlameFillFirstBlameRow(ctx.Repo.RepoLink, avatarUtils, part, commitNames[part.Sha], br) + } + } + } + escapeStatus := &charset.EscapeStatus{} - var lexerName string - - avatarUtils := templates.NewAvatarUtils(ctx) - i := 0 - commitCnt := 0 - for _, part := range blameParts { - for index, line := range part.Lines { - i++ - lines = append(lines, line) - - br := &blameRow{ - RowNumber: i, - } - - commit := commitNames[part.Sha] - if index == 0 { - // Count commit number - commitCnt++ - - // User avatar image - commitSince := templates.TimeSince(commit.Author.When) - - var avatar string - if commit.User != nil { - avatar = string(avatarUtils.Avatar(commit.User, 18)) - } else { - avatar = string(avatarUtils.AvatarByEmail(commit.Author.Email, commit.Author.Name, 18, "tw-mr-2")) - } - - br.Avatar = gotemplate.HTML(avatar) - br.RepoLink = repoLink - br.PartSha = part.Sha - br.PreviousSha = part.PreviousSha - br.PreviousShaURL = fmt.Sprintf("%s/blame/commit/%s/%s", repoLink, url.PathEscape(part.PreviousSha), util.PathEscapeSegments(part.PreviousPath)) - br.CommitURL = fmt.Sprintf("%s/commit/%s", repoLink, url.PathEscape(part.Sha)) - br.CommitMessage = commit.CommitMessage - br.CommitSince = commitSince - } - - if i != len(lines)-1 { - line += "\n" - } - line, lexerNameForLine := highlight.Code(path.Base(ctx.Repo.TreePath), language, line) - - // set lexer name to the first detected lexer. this is certainly suboptimal and - // we should instead highlight the whole file at once - if lexerName == "" { - lexerName = lexerNameForLine - } - - br.EscapeStatus, br.Code = charset.EscapeControlHTML(line, ctx.Locale) - rows = append(rows, br) - escapeStatus = escapeStatus.Or(br.EscapeStatus) + bufContent := buf.Bytes() + bufContent = charset.ToUTF8(bufContent, charset.ConvertOpts{}) + highlighted, lexerName := highlight.Code(path.Base(ctx.Repo.TreePath), language, util.UnsafeBytesToString(bufContent)) + unsafeLines := highlight.UnsafeSplitHighlightedLines(highlighted) + for i, br := range rows { + var line template.HTML + if i < len(rows) { + line = template.HTML(util.UnsafeBytesToString(unsafeLines[i])) } + br.EscapeStatus, br.Code = charset.EscapeControlHTML(line, ctx.Locale) + escapeStatus = escapeStatus.Or(br.EscapeStatus) } ctx.Data["EscapeStatus"] = escapeStatus ctx.Data["BlameRows"] = rows - ctx.Data["CommitCnt"] = commitCnt ctx.Data["LexerName"] = lexerName } diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index f21f568231..2b0ba9072d 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -15,6 +15,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" repo_module "code.gitea.io/gitea/modules/repository" @@ -133,8 +134,7 @@ func RestoreBranchPost(ctx *context.Context) { return } - if err := git.Push(ctx, ctx.Repo.Repository.RepoPath(), git.PushOptions{ - Remote: ctx.Repo.Repository.RepoPath(), + if err := gitrepo.Push(ctx, ctx.Repo.Repository, ctx.Repo.Repository, git.PushOptions{ Branch: fmt.Sprintf("%s:%s%s", deletedBranch.CommitID, git.BranchPrefix, deletedBranch.Name), Env: repo_module.PushingEnvironment(ctx.Doer, ctx.Repo.Repository), }); err != nil { diff --git a/routers/web/repo/editor.go b/routers/web/repo/editor.go index 983249a6d2..048c9f3d4a 100644 --- a/routers/web/repo/editor.go +++ b/routers/web/repo/editor.go @@ -317,11 +317,7 @@ func EditFile(ctx *context.Context) { ctx.ServerError("ReadAll", err) return } - if content, err := charset.ToUTF8(buf, charset.ConvertOpts{KeepBOM: true}); err != nil { - ctx.Data["FileContent"] = string(buf) - } else { - ctx.Data["FileContent"] = content - } + ctx.Data["FileContent"] = string(charset.ToUTF8(buf, charset.ConvertOpts{KeepBOM: true, ErrorReturnOrigin: true})) } } diff --git a/routers/web/repo/editor_cherry_pick.go b/routers/web/repo/editor_cherry_pick.go index 32e3c58e87..ca0e19517a 100644 --- a/routers/web/repo/editor_cherry_pick.go +++ b/routers/web/repo/editor_cherry_pick.go @@ -9,6 +9,7 @@ import ( "strings" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/forms" @@ -35,9 +36,7 @@ func CherryPick(ctx *context.Context) { ctx.Data["commit_message"] = "revert " + cherryPickCommit.Message() } else { ctx.Data["CherryPickType"] = "cherry-pick" - splits := strings.SplitN(cherryPickCommit.Message(), "\n", 2) - ctx.Data["commit_summary"] = splits[0] - ctx.Data["commit_message"] = splits[1] + ctx.Data["commit_summary"], ctx.Data["commit_message"], _ = strings.Cut(cherryPickCommit.Message(), "\n") } ctx.HTML(http.StatusOK, tplCherryPick) @@ -66,7 +65,7 @@ func CherryPickPost(ctx *context.Context) { // Drop through to the "apply" method buf := &bytes.Buffer{} if parsed.form.Revert { - err = git.GetReverseRawDiff(ctx, ctx.Repo.Repository.RepoPath(), fromCommitID, buf) + err = gitrepo.GetReverseRawDiff(ctx, ctx.Repo.Repository, fromCommitID, buf) } else { err = git.GetRawDiff(ctx.Repo.GitRepo, fromCommitID, "patch", buf) } diff --git a/routers/web/repo/editor_util.go b/routers/web/repo/editor_util.go index f910f0bd40..07bcb474f0 100644 --- a/routers/web/repo/editor_util.go +++ b/routers/web/repo/editor_util.go @@ -13,6 +13,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -102,8 +103,7 @@ func getUniqueRepositoryName(ctx context.Context, ownerID int64, name string) st } func editorPushBranchToForkedRepository(ctx context.Context, doer *user_model.User, baseRepo *repo_model.Repository, baseBranchName string, targetRepo *repo_model.Repository, targetBranchName string) error { - return git.Push(ctx, baseRepo.RepoPath(), git.PushOptions{ - Remote: targetRepo.RepoPath(), + return gitrepo.Push(ctx, baseRepo, targetRepo, git.PushOptions{ Branch: baseBranchName + ":" + targetBranchName, Env: repo_module.PushingEnvironment(doer, targetRepo), }) diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index edad756b6b..a3cb88e76a 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models/renderhelper" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/htmlutil" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup/markdown" @@ -110,7 +111,7 @@ func NewComment(ctx *context.Context) { ctx.ServerError("Unable to load base repo", err) return } - prHeadCommitID, err := git.GetFullCommitID(ctx, pull.BaseRepo.RepoPath(), prHeadRef) + prHeadCommitID, err := gitrepo.GetFullCommitID(ctx, pull.BaseRepo, prHeadRef) if err != nil { ctx.ServerError("Get head commit Id of pr fail", err) return @@ -127,7 +128,7 @@ func NewComment(ctx *context.Context) { return } headBranchRef := git.RefNameFromBranch(pull.HeadBranch) - headBranchCommitID, err := git.GetFullCommitID(ctx, pull.HeadRepo.RepoPath(), headBranchRef.String()) + headBranchCommitID, err := gitrepo.GetFullCommitID(ctx, pull.HeadRepo, headBranchRef.String()) if err != nil { ctx.ServerError("Get head commit Id of head branch fail", err) return @@ -141,8 +142,7 @@ func NewComment(ctx *context.Context) { if prHeadCommitID != headBranchCommitID { // force push to base repo - err := git.Push(ctx, pull.HeadRepo.RepoPath(), git.PushOptions{ - Remote: pull.BaseRepo.RepoPath(), + err := gitrepo.Push(ctx, pull.HeadRepo, pull.BaseRepo, git.PushOptions{ Branch: pull.HeadBranch + ":" + prHeadRef, Force: true, Env: repo_module.InternalPushingEnvironment(pull.Issue.Poster, pull.BaseRepo), diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go index ea15e90e5c..8f4adb2ad2 100644 --- a/routers/web/repo/migrate.go +++ b/routers/web/repo/migrate.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/forms" "code.gitea.io/gitea/services/migrations" + repo_service "code.gitea.io/gitea/services/repository" "code.gitea.io/gitea/services/task" ) @@ -237,7 +238,7 @@ func MigratePost(ctx *context.Context) { opts.AWSSecretAccessKey = form.AWSSecretAccessKey } - err = repo_model.CheckCreateRepository(ctx, ctx.Doer, ctxUser, opts.RepoName, false) + err = repo_service.CheckCreateRepository(ctx, ctx.Doer, ctxUser, opts.RepoName, false) if err != nil { handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form) return diff --git a/routers/web/repo/release.go b/routers/web/repo/release.go index 33e0dc5889..1b36dc4d44 100644 --- a/routers/web/repo/release.go +++ b/routers/web/repo/release.go @@ -392,6 +392,32 @@ func NewRelease(ctx *context.Context) { ctx.HTML(http.StatusOK, tplReleaseNew) } +// GenerateReleaseNotes builds release notes content for the given tag and base. +func GenerateReleaseNotes(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.GenerateReleaseNotesForm) + + if ctx.HasError() { + ctx.JSONError(ctx.GetErrMsg()) + return + } + + content, err := release_service.GenerateReleaseNotes(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, release_service.GenerateReleaseNotesOptions{ + TagName: form.TagName, + TagTarget: form.TagTarget, + PreviousTag: form.PreviousTag, + }) + if err != nil { + if errTr := util.ErrorAsTranslatable(err); errTr != nil { + ctx.JSONError(errTr.Translate(ctx.Locale)) + } else { + ctx.ServerError("GenerateReleaseNotes", err) + } + return + } + + ctx.JSON(http.StatusOK, map[string]any{"content": content}) +} + // NewReleasePost response for creating a release func NewReleasePost(ctx *context.Context) { newReleaseCommon(ctx) @@ -520,11 +546,13 @@ func NewReleasePost(ctx *context.Context) { // EditRelease render release edit page func EditRelease(ctx *context.Context) { + newReleaseCommon(ctx) + if ctx.Written() { + return + } + ctx.Data["Title"] = ctx.Tr("repo.release.edit_release") - ctx.Data["PageIsReleaseList"] = true ctx.Data["PageIsEditRelease"] = true - ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled - upload.AddUploadContext(ctx, "release") tagName := ctx.PathParam("*") rel, err := repo_model.GetRelease(ctx, ctx.Repo.Repository.ID, tagName) @@ -565,8 +593,13 @@ func EditRelease(ctx *context.Context) { // EditReleasePost response for edit release func EditReleasePost(ctx *context.Context) { form := web.GetForm(ctx).(*forms.EditReleaseForm) + + newReleaseCommon(ctx) + if ctx.Written() { + return + } + ctx.Data["Title"] = ctx.Tr("repo.release.edit_release") - ctx.Data["PageIsReleaseList"] = true ctx.Data["PageIsEditRelease"] = true tagName := ctx.PathParam("*") diff --git a/routers/web/repo/setting/lfs.go b/routers/web/repo/setting/lfs.go index a558231df1..c7a19062d2 100644 --- a/routers/web/repo/setting/lfs.go +++ b/routers/web/repo/setting/lfs.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/git/pipeline" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -112,7 +113,7 @@ func LFSLocks(ctx *context.Context) { } defer cleanup() - if err := git.Clone(ctx, ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{ + if err := gitrepo.CloneRepoToLocal(ctx, ctx.Repo.Repository, tmpBasePath, git.CloneRepoOptions{ Bare: true, Shared: true, }); err != nil { diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index 60eb35f56d..0c73c1490f 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -61,7 +61,7 @@ func SettingsCtxData(ctx *context.Context) { ctx.Data["MinimumMirrorInterval"] = setting.Mirror.MinInterval ctx.Data["CanConvertFork"] = ctx.Repo.Repository.IsFork && ctx.Doer.CanCreateRepoIn(ctx.Repo.Repository.Owner) - signing, _ := gitrepo.GetSigningKey(ctx, ctx.Repo.Repository) + signing, _ := gitrepo.GetSigningKey(ctx) ctx.Data["SigningKeyAvailable"] = signing != nil ctx.Data["SigningSettings"] = setting.Repository.Signing ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled @@ -104,7 +104,7 @@ func SettingsPost(ctx *context.Context) { ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval ctx.Data["MinimumMirrorInterval"] = setting.Mirror.MinInterval - signing, _ := gitrepo.GetSigningKey(ctx, ctx.Repo.Repository) + signing, _ := gitrepo.GetSigningKey(ctx) ctx.Data["SigningKeyAvailable"] = signing != nil ctx.Data["SigningSettings"] = setting.Repository.Signing ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled diff --git a/routers/web/web.go b/routers/web/web.go index 1c01eb63a1..ea70d8bb62 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -227,6 +227,8 @@ func ctxDataSet(args ...any) func(ctx *context.Context) { } } +const RouterMockPointBeforeWebRoutes = "before-web-routes" + // Routes returns all web routes func Routes() *web.Router { routes := web.NewRouter() @@ -285,7 +287,7 @@ func Routes() *web.Router { webRoutes := web.NewRouter() webRoutes.Use(mid...) - webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive(), common.QoS()) + webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive(), common.QoS(), web.RouterMockPoint(RouterMockPointBeforeWebRoutes)) routes.Mount("", webRoutes) return routes } @@ -1402,6 +1404,7 @@ func registerWebRoutes(m *web.Router) { m.Group("/releases", func() { m.Get("/new", repo.NewRelease) m.Post("/new", web.Bind(forms.NewReleaseForm{}), repo.NewReleasePost) + m.Post("/generate-notes", web.Bind(forms.GenerateReleaseNotesForm{}), repo.GenerateReleaseNotes) m.Post("/delete", repo.DeleteRelease) m.Post("/attachments", repo.UploadReleaseAttachment) m.Post("/attachments/remove", repo.DeleteAttachment) diff --git a/services/asymkey/commit.go b/services/asymkey/commit.go index 54ef052a50..6286588a60 100644 --- a/services/asymkey/commit.go +++ b/services/asymkey/commit.go @@ -162,7 +162,7 @@ func parseCommitWithGPGSignature(ctx context.Context, c *git.Commit, committer * } } - defaultGPGSettings, err := c.GetRepositoryDefaultPublicGPGKey(false) + defaultGPGSettings, err := git.GetDefaultPublicGPGKey(ctx, false) if err != nil { log.Error("Error getting default public gpg key: %v", err) } else if defaultGPGSettings == nil { diff --git a/services/asymkey/sign.go b/services/asymkey/sign.go index 1ed05ba287..eb6e461346 100644 --- a/services/asymkey/sign.go +++ b/services/asymkey/sign.go @@ -108,34 +108,34 @@ func IsErrWontSign(err error) bool { return ok } -// PublicSigningKey gets the public signing key within a provided repository directory -func PublicSigningKey(ctx context.Context, repoPath string) (content, format string, err error) { - signingKey, _ := git.GetSigningKey(ctx, repoPath) +// PublicSigningKey gets the public signing key of the entire instance +func PublicSigningKey(ctx context.Context) (content, format string, err error) { + signingKey, _ := git.GetSigningKey(ctx) if signingKey == nil { return "", "", nil } if signingKey.Format == git.SigningKeyFormatSSH { content, err := os.ReadFile(signingKey.KeyID) if err != nil { - log.Error("Unable to read SSH public key file in %s: %s, %v", repoPath, signingKey, err) + log.Error("Unable to read SSH public key file: %s, %v", signingKey, err) return "", signingKey.Format, err } return string(content), signingKey.Format, nil } - content, stderr, err := process.GetManager().ExecDir(ctx, -1, repoPath, + content, stderr, err := process.GetManager().ExecDir(ctx, -1, setting.Git.HomePath, "gpg --export -a", "gpg", "--export", "-a", signingKey.KeyID) if err != nil { - log.Error("Unable to get default signing key in %s: %s, %s, %v", repoPath, signingKey, stderr, err) + log.Error("Unable to get default signing key: %s, %s, %v", signingKey, stderr, err) return "", signingKey.Format, err } return content, signingKey.Format, nil } // SignInitialCommit determines if we should sign the initial commit to this repository -func SignInitialCommit(ctx context.Context, repoPath string, u *user_model.User) (bool, *git.SigningKey, *git.Signature, error) { +func SignInitialCommit(ctx context.Context, u *user_model.User) (bool, *git.SigningKey, *git.Signature, error) { rules := signingModeFromStrings(setting.Repository.Signing.InitialCommit) - signingKey, sig := git.GetSigningKey(ctx, repoPath) + signingKey, sig := git.GetSigningKey(ctx) if signingKey == nil { return false, nil, nil, &ErrWontSign{noKey} } @@ -171,7 +171,7 @@ Loop: // SignWikiCommit determines if we should sign the commits to this repository wiki func SignWikiCommit(ctx context.Context, repo *repo_model.Repository, u *user_model.User) (bool, *git.SigningKey, *git.Signature, error) { rules := signingModeFromStrings(setting.Repository.Signing.Wiki) - signingKey, sig := gitrepo.GetSigningKey(ctx, repo.WikiStorageRepo()) + signingKey, sig := gitrepo.GetSigningKey(ctx) if signingKey == nil { return false, nil, nil, &ErrWontSign{noKey} } @@ -222,9 +222,9 @@ Loop: } // SignCRUDAction determines if we should sign a CRUD commit to this repository -func SignCRUDAction(ctx context.Context, repoPath string, u *user_model.User, tmpBasePath, parentCommit string) (bool, *git.SigningKey, *git.Signature, error) { +func SignCRUDAction(ctx context.Context, u *user_model.User, tmpBasePath, parentCommit string) (bool, *git.SigningKey, *git.Signature, error) { rules := signingModeFromStrings(setting.Repository.Signing.CRUDActions) - signingKey, sig := git.GetSigningKey(ctx, repoPath) + signingKey, sig := git.GetSigningKey(ctx) if signingKey == nil { return false, nil, nil, &ErrWontSign{noKey} } @@ -288,7 +288,7 @@ func SignMerge(ctx context.Context, pr *issues_model.PullRequest, u *user_model. } repo := pr.BaseRepo - signingKey, signer := gitrepo.GetSigningKey(ctx, repo) + signingKey, signer := gitrepo.GetSigningKey(ctx) if signingKey == nil { return false, nil, nil, &ErrWontSign{noKey} } diff --git a/services/context/base.go b/services/context/base.go index de839ede81..8bd66bed09 100644 --- a/services/context/base.go +++ b/services/context/base.go @@ -43,8 +43,10 @@ type Base struct { Locale translation.Locale } +var ParseMultipartFormMaxMemory = int64(32 << 20) + func (b *Base) ParseMultipartForm() bool { - err := b.Req.ParseMultipartForm(32 << 20) + err := b.Req.ParseMultipartForm(ParseMultipartFormMaxMemory) if err != nil { // TODO: all errors caused by client side should be ignored (connection closed). if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { diff --git a/services/context/context_response.go b/services/context/context_response.go index 3f64fc7352..bb896024b1 100644 --- a/services/context/context_response.go +++ b/services/context/context_response.go @@ -20,15 +20,27 @@ import ( "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/web/middleware" ) // RedirectToUser redirect to a differently-named user -func RedirectToUser(ctx *Base, userName string, redirectUserID int64) { +func RedirectToUser(ctx *Base, doer *user_model.User, userName string, redirectUserID int64) { user, err := user_model.GetUserByID(ctx, redirectUserID) if err != nil { - ctx.HTTPError(http.StatusInternalServerError, "unable to get user") + if user_model.IsErrUserNotExist(err) { + ctx.HTTPError(http.StatusNotFound, "user does not exist") + } else { + ctx.HTTPError(http.StatusInternalServerError, "unable to get user") + } + return + } + + // Handle Visibility + if user.Visibility != structs.VisibleTypePublic && doer == nil { + // We must be signed in to see limited or private organizations + ctx.HTTPError(http.StatusNotFound, "user does not exist") return } diff --git a/services/context/csrf.go b/services/context/csrf.go index f190465bdb..aa99f34b03 100644 --- a/services/context/csrf.go +++ b/services/context/csrf.go @@ -118,7 +118,7 @@ func (c *csrfProtector) PrepareForSessionUser(ctx *Context) { if uidChanged { _ = ctx.Session.Set(c.opt.oldSessionKey, c.id) } else if cookieToken != "" { - // If cookie token presents, re-use existing unexpired token, else generate a new one. + // If cookie token present, re-use existing unexpired token, else generate a new one. if issueTime, ok := ParseCsrfToken(cookieToken); ok { dur := time.Since(issueTime) // issueTime is not a monotonic-clock, the server time may change a lot to an early time. if dur >= -CsrfTokenRegenerationInterval && dur <= CsrfTokenRegenerationInterval { diff --git a/services/context/org.go b/services/context/org.go index 1cd8923178..4c64ff72a9 100644 --- a/services/context/org.go +++ b/services/context/org.go @@ -49,7 +49,7 @@ func GetOrganizationByParams(ctx *Context) { if organization.IsErrOrgNotExist(err) { redirectUserID, err := user_model.LookupUserRedirect(ctx, orgName) if err == nil { - RedirectToUser(ctx.Base, orgName, redirectUserID) + RedirectToUser(ctx.Base, ctx.Doer, orgName, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { ctx.NotFound(err) } else { @@ -70,8 +70,9 @@ type OrgAssignmentOptions struct { } // OrgAssignment returns a middleware to handle organization assignment -func OrgAssignment(opts OrgAssignmentOptions) func(ctx *Context) { +func OrgAssignment(orgAssignmentOpts OrgAssignmentOptions) func(ctx *Context) { return func(ctx *Context) { + opts := orgAssignmentOpts // it must be a copy, because the values will be changed var err error if ctx.ContextUser == nil { // if Organization is not defined, get it from params diff --git a/services/context/repo.go b/services/context/repo.go index 64b8695236..e70e83e233 100644 --- a/services/context/repo.go +++ b/services/context/repo.go @@ -140,7 +140,7 @@ func PrepareCommitFormOptions(ctx *Context, doer *user_model.User, targetRepo *r protectionRequireSigned = protectedBranch.RequireSignedCommits } - willSign, signKey, _, err := asymkey_service.SignCRUDAction(ctx, targetRepo.RepoPath(), doer, targetRepo.RepoPath(), refName.String()) + willSign, signKey, _, err := asymkey_service.SignCRUDAction(ctx, doer, targetRepo.RepoPath(), refName.String()) wontSignReason := "" if asymkey_service.IsErrWontSign(err) { wontSignReason = string(err.(*asymkey_service.ErrWontSign).Reason) @@ -443,7 +443,7 @@ func RepoAssignment(ctx *Context) { } if redirectUserID, err := user_model.LookupUserRedirect(ctx, userName); err == nil { - RedirectToUser(ctx.Base, userName, redirectUserID) + RedirectToUser(ctx.Base, ctx.Doer, userName, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { ctx.NotFound(nil) } else { diff --git a/services/context/user.go b/services/context/user.go index f1a3035ee9..19c055e2a3 100644 --- a/services/context/user.go +++ b/services/context/user.go @@ -69,7 +69,7 @@ func userAssignment(ctx *Base, doer *user_model.User, errCb func(int, any)) (con if err != nil { if user_model.IsErrUserNotExist(err) { if redirectUserID, err := user_model.LookupUserRedirect(ctx, username); err == nil { - RedirectToUser(ctx, username, redirectUserID) + RedirectToUser(ctx, doer, username, redirectUserID) } else if user_model.IsErrUserRedirectNotExist(err) { errCb(http.StatusNotFound, err) } else { diff --git a/services/doctor/checkOldArchives.go b/services/doctor/checkOldArchives.go index 390dfb43aa..fa1a6ccb1d 100644 --- a/services/doctor/checkOldArchives.go +++ b/services/doctor/checkOldArchives.go @@ -5,12 +5,10 @@ package doctor import ( "context" - "os" - "path/filepath" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" ) func checkOldArchives(ctx context.Context, logger log.Logger, autofix bool) error { @@ -21,18 +19,18 @@ func checkOldArchives(ctx context.Context, logger log.Logger, autofix bool) erro return nil } - p := filepath.Join(repo.RepoPath(), "archives") - isDir, err := util.IsDir(p) + isDir, err := gitrepo.IsRepoDirExist(ctx, repo, "archives") if err != nil { - log.Warn("check if %s is directory failed: %v", p, err) + log.Warn("check if %s is directory failed: %v", repo.FullName(), err) } if isDir { numRepos++ if autofix { - if err := os.RemoveAll(p); err == nil { + err := gitrepo.RemoveRepoFileOrDir(ctx, repo, "archives") + if err == nil { numReposUpdated++ } else { - log.Warn("remove %s failed: %v", p, err) + log.Warn("remove %s failed: %v", repo.FullName(), err) } } } diff --git a/services/doctor/misc.go b/services/doctor/misc.go index ce7eea1dcc..8765cfa025 100644 --- a/services/doctor/misc.go +++ b/services/doctor/misc.go @@ -6,9 +6,7 @@ package doctor import ( "context" "fmt" - "os" "os/exec" - "path/filepath" "strings" "code.gitea.io/gitea/models" @@ -20,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" lru "github.com/hashicorp/golang-lru/v2" "xorm.io/builder" @@ -142,10 +139,10 @@ func checkDaemonExport(ctx context.Context, logger log.Logger, autofix bool) err } // Create/Remove git-daemon-export-ok for git-daemon... - daemonExportFile := filepath.Join(repo.RepoPath(), `git-daemon-export-ok`) - isExist, err := util.IsExist(daemonExportFile) + daemonExportFile := `git-daemon-export-ok` + isExist, err := gitrepo.IsRepoFileExist(ctx, repo, daemonExportFile) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) + log.Error("Unable to check if %s:%s exists. Error: %v", repo.FullName(), daemonExportFile, err) return err } isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic @@ -154,12 +151,12 @@ func checkDaemonExport(ctx context.Context, logger log.Logger, autofix bool) err numNeedUpdate++ if autofix { if !isPublic && isExist { - if err = util.Remove(daemonExportFile); err != nil { - log.Error("Failed to remove %s: %v", daemonExportFile, err) + if err = gitrepo.RemoveRepoFileOrDir(ctx, repo, daemonExportFile); err != nil { + log.Error("Failed to remove %s:%s: %v", repo.FullName(), daemonExportFile, err) } } else if isPublic && !isExist { - if f, err := os.Create(daemonExportFile); err != nil { - log.Error("Failed to create %s: %v", daemonExportFile, err) + if f, err := gitrepo.CreateRepoFile(ctx, repo, daemonExportFile); err != nil { + log.Error("Failed to create %s:%s: %v", repo.FullName(), daemonExportFile, err) } else { f.Close() } @@ -190,16 +187,16 @@ func checkCommitGraph(ctx context.Context, logger log.Logger, autofix bool) erro commitGraphExists := func() (bool, error) { // Check commit-graph exists - commitGraphFile := filepath.Join(repo.RepoPath(), `objects/info/commit-graph`) - isExist, err := util.IsExist(commitGraphFile) + commitGraphFile := `objects/info/commit-graph` + isExist, err := gitrepo.IsRepoFileExist(ctx, repo, commitGraphFile) if err != nil { logger.Error("Unable to check if %s exists. Error: %v", commitGraphFile, err) return false, err } if !isExist { - commitGraphsDir := filepath.Join(repo.RepoPath(), `objects/info/commit-graphs`) - isExist, err = util.IsExist(commitGraphsDir) + commitGraphsDir := `objects/info/commit-graphs` + isExist, err = gitrepo.IsRepoDirExist(ctx, repo, commitGraphsDir) if err != nil { logger.Error("Unable to check if %s exists. Error: %v", commitGraphsDir, err) return false, err @@ -215,7 +212,7 @@ func checkCommitGraph(ctx context.Context, logger log.Logger, autofix bool) erro if !isExist { numNeedUpdate++ if autofix { - if err := git.WriteCommitGraph(ctx, repo.RepoPath()); err != nil { + if err := gitrepo.WriteCommitGraph(ctx, repo); err != nil { logger.Error("Unable to write commit-graph in %s. Error: %v", repo.FullName(), err) return err } diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 6820521ba3..2d33d2b42b 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -638,6 +638,19 @@ func (f *NewReleaseForm) Validate(req *http.Request, errs binding.Errors) bindin return middleware.Validate(errs, ctx.Data, f, ctx.Locale) } +// GenerateReleaseNotesForm retrieves release notes recommendations. +type GenerateReleaseNotesForm struct { + TagName string `form:"tag_name" binding:"Required;GitRefName;MaxSize(255)"` + TagTarget string `form:"tag_target" binding:"MaxSize(255)"` + PreviousTag string `form:"previous_tag" binding:"MaxSize(255)"` +} + +// Validate validates the fields +func (f *GenerateReleaseNotesForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetValidateContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} + // EditReleaseForm form for changing release type EditReleaseForm struct { Title string `form:"title" binding:"Required;MaxSize(255)"` diff --git a/services/gitdiff/git_diff_tree.go b/services/gitdiff/git_diff_tree.go index 4649c24af3..2a3c7c9445 100644 --- a/services/gitdiff/git_diff_tree.go +++ b/services/gitdiff/git_diff_tree.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) type DiffTree struct { @@ -56,7 +57,9 @@ func runGitDiffTree(ctx context.Context, gitRepo *git.Repository, useMergeBase b return nil, err } - cmd := gitcmd.NewCommand("diff-tree", "--raw", "-r", "--find-renames", "--root") + cmd := gitcmd.NewCommand("diff-tree", "--raw", "-r", "--root"). + AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold) + if useMergeBase { cmd.AddArguments("--merge-base") } diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 6e15f71609..17eb3d4280 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -835,11 +835,11 @@ parsingLoop: if buffer.Len() == 0 { continue } - charsetLabel, err := charset.DetectEncoding(buffer.Bytes()) - if charsetLabel != "UTF-8" && err == nil { - encoding, _ := stdcharset.Lookup(charsetLabel) - if encoding != nil { - diffLineTypeDecoders[lineType] = encoding.NewDecoder() + charsetLabel, _ := charset.DetectEncoding(buffer.Bytes()) + if charsetLabel != "UTF-8" { + charsetEncoding, _ := stdcharset.Lookup(charsetLabel) + if charsetEncoding != nil { + diffLineTypeDecoders[lineType] = charsetEncoding.NewDecoder() } } } @@ -1225,8 +1225,9 @@ func getDiffBasic(ctx context.Context, gitRepo *git.Repository, opts *DiffOption } cmdDiff := gitcmd.NewCommand(). - AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/", "-M"). - AddArguments(opts.WhitespaceBehavior...) + AddArguments("diff", "--src-prefix=\\a/", "--dst-prefix=\\b/"). + AddArguments(opts.WhitespaceBehavior...). + AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold) // In git 2.31, git diff learned --skip-to which we can use to shortcut skip to file // so if we are using at least this version of git we don't have to tell ParsePatch to do @@ -1325,10 +1326,10 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit shouldFullFileHighlight := !setting.Git.DisableDiffHighlight && attrDiff.Value() == "" if shouldFullFileHighlight { if limitedContent.LeftContent != nil && limitedContent.LeftContent.buf.Len() < MaxDiffHighlightEntireFileSize { - diffFile.highlightedLeftLines = highlightCodeLines(diffFile, true /* left */, limitedContent.LeftContent.buf.String()) + diffFile.highlightedLeftLines = highlightCodeLines(diffFile, true /* left */, limitedContent.LeftContent.buf.Bytes()) } if limitedContent.RightContent != nil && limitedContent.RightContent.buf.Len() < MaxDiffHighlightEntireFileSize { - diffFile.highlightedRightLines = highlightCodeLines(diffFile, false /* right */, limitedContent.RightContent.buf.String()) + diffFile.highlightedRightLines = highlightCodeLines(diffFile, false /* right */, limitedContent.RightContent.buf.Bytes()) } } } @@ -1336,10 +1337,11 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit return diff, nil } -func highlightCodeLines(diffFile *DiffFile, isLeft bool, content string) map[int]template.HTML { +func highlightCodeLines(diffFile *DiffFile, isLeft bool, rawContent []byte) map[int]template.HTML { + content := util.UnsafeBytesToString(charset.ToUTF8(rawContent, charset.ConvertOpts{})) highlightedNewContent, _ := highlight.Code(diffFile.Name, diffFile.Language, content) - splitLines := strings.Split(string(highlightedNewContent), "\n") - lines := make(map[int]template.HTML, len(splitLines)) + unsafeLines := highlight.UnsafeSplitHighlightedLines(highlightedNewContent) + lines := make(map[int]template.HTML, len(unsafeLines)) // only save the highlighted lines we need, but not the whole file, to save memory for _, sec := range diffFile.Sections { for _, ln := range sec.Lines { @@ -1349,8 +1351,8 @@ func highlightCodeLines(diffFile *DiffFile, isLeft bool, content string) map[int } if lineIdx >= 1 { idx := lineIdx - 1 - if idx < len(splitLines) { - lines[idx] = template.HTML(splitLines[idx]) + if idx < len(unsafeLines) { + lines[idx] = template.HTML(util.UnsafeBytesToString(unsafeLines[idx])) } } } diff --git a/services/gitdiff/gitdiff_test.go b/services/gitdiff/gitdiff_test.go index 721ae0dfc7..a94dad8b63 100644 --- a/services/gitdiff/gitdiff_test.go +++ b/services/gitdiff/gitdiff_test.go @@ -5,6 +5,7 @@ package gitdiff import ( + "html/template" "strconv" "strings" "testing" @@ -1106,3 +1107,41 @@ func TestDiffLine_GetExpandDirection(t *testing.T) { assert.Equal(t, c.direction, c.diffLine.GetExpandDirection(), "case %s expected direction: %s", c.name, c.direction) } } + +func TestHighlightCodeLines(t *testing.T) { + t.Run("CharsetDetecting", func(t *testing.T) { + diffFile := &DiffFile{ + Name: "a.c", + Language: "c", + Sections: []*DiffSection{ + { + Lines: []*DiffLine{{LeftIdx: 1}}, + }, + }, + } + ret := highlightCodeLines(diffFile, true, []byte("// abc\xcc def\xcd")) // ISO-8859-1 bytes + assert.Equal(t, "// abcÌ defÍ\n", string(ret[0])) + }) + + t.Run("LeftLines", func(t *testing.T) { + diffFile := &DiffFile{ + Name: "a.c", + Language: "c", + Sections: []*DiffSection{ + { + Lines: []*DiffLine{ + {LeftIdx: 1}, + {LeftIdx: 2}, + {LeftIdx: 3}, + }, + }, + }, + } + const nl = "\n" + ret := highlightCodeLines(diffFile, true, []byte("a\nb\n")) + assert.Equal(t, map[int]template.HTML{ + 0: `a` + nl, + 1: `b`, + }, ret) + }) +} diff --git a/services/gitdiff/highlightdiff_test.go b/services/gitdiff/highlightdiff_test.go index aebe38ae7c..0df2e29d13 100644 --- a/services/gitdiff/highlightdiff_test.go +++ b/services/gitdiff/highlightdiff_test.go @@ -25,12 +25,12 @@ func TestDiffWithHighlight(t *testing.T) { t.Run("CleanUp", func(t *testing.T) { hcd := newHighlightCodeDiff() - codeA := template.HTML(`this is updated comment`) + codeA := template.HTML(`this is a comment`) + codeB := template.HTML(`this is updated comment`) outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB) - assert.Equal(t, `a comment`, string(outDel)) + assert.Equal(t, `this is a comment`, string(outDel)) outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB) - assert.Equal(t, `updated comment`, string(outAdd)) + assert.Equal(t, `this is updated comment`, string(outAdd)) }) t.Run("OpenCloseTags", func(t *testing.T) { diff --git a/services/mailer/sender/sender.go b/services/mailer/sender/sender.go index e470c2f2b3..30c6feaf7a 100644 --- a/services/mailer/sender/sender.go +++ b/services/mailer/sender/sender.go @@ -4,10 +4,8 @@ package sender import ( + "errors" "io" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" ) type Sender interface { @@ -16,23 +14,18 @@ type Sender interface { var Send = send -func send(sender Sender, msgs ...*Message) error { - if setting.MailService == nil { - log.Error("Mailer: Send is being invoked but mail service hasn't been initialized") - return nil +func send(sender Sender, msg *Message) error { + m := msg.ToMessage() + froms := m.GetFrom() + to, err := m.GetRecipients() + if err != nil { + return err } - for _, msg := range msgs { - m := msg.ToMessage() - froms := m.GetFrom() - to, err := m.GetRecipients() - if err != nil { - return err - } - // TODO: implement sending from multiple addresses - if err := sender.Send(froms[0].Address, to, m); err != nil { - return err - } + // TODO: implement sending from multiple addresses + if len(froms) == 0 { + // FIXME: no idea why sometimes the "froms" can be empty, need to figure out the root problem + return errors.New("no FROM specified") } - return nil + return sender.Send(froms[0].Address, to, m) } diff --git a/services/markup/renderhelper_codepreview_test.go b/services/markup/renderhelper_codepreview_test.go index 63e7f4d3bd..6665f0d009 100644 --- a/services/markup/renderhelper_codepreview_test.go +++ b/services/markup/renderhelper_codepreview_test.go @@ -40,7 +40,7 @@ func TestRenderHelperCodePreview(t *testing.T) {
# repo1
-
+
diff --git a/services/migrations/dump.go b/services/migrations/dump.go index f9309e5e6a..b1c5695854 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -306,14 +306,15 @@ func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.R if err != nil { return err } + defer rc.Close() } else { resp, err := http.Get(*asset.DownloadURL) if err != nil { return err } + defer resp.Body.Close() rc = resp.Body } - defer rc.Close() fw, err := os.Create(attachPath) if err != nil { diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index bb1760e889..fb985ee9d5 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -27,6 +27,7 @@ func TestGiteaDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name()) } + defer resp.Body.Close() ctx := t.Context() downloader, err := NewGiteaDownloader(ctx, "https://gitea.com", "gitea/test_repo", "", "", giteaToken) require.NoError(t, err, "NewGiteaDownloader error occur") diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index a9ae89a2a3..fef1053ec8 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -30,6 +30,7 @@ func TestGitlabDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't access test repo, skipping %s", t.Name()) } + defer resp.Body.Close() ctx := t.Context() downloader, err := NewGitlabDownloader(ctx, "https://gitlab.com", "gitea/test_repo", gitlabPersonalAccessToken) if err != nil { diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index 503b669f8e..de7351b5bf 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -27,6 +27,7 @@ func TestGogsDownloadRepo(t *testing.T) { t.Skipf("visit test repo failed, ignored") return } + defer resp.Body.Close() ctx := t.Context() downloader := NewGogsDownloader(ctx, "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO") repo, err := downloader.GetRepoInfo(ctx) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index eba9c79df5..bd7e52cc3d 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -16,6 +16,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" system_model "code.gitea.io/gitea/models/system" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" @@ -327,6 +328,9 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba messenger("repo.migrate.migrating_issues") issueBatchSize := uploader.MaxBatchInsertSize("issue") + // because when the migrating is running, some issues maybe removed, so after the next page + // some of issue maybe duplicated, so we need to record the inserted issue indexes + mapInsertedIssueIndexes := container.Set[int64]{} for i := 1; ; i++ { issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize) if err != nil { @@ -336,6 +340,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba log.Warn("migrating issues is not supported, ignored") break } + for i := 0; i < len(issues); i++ { + if mapInsertedIssueIndexes.Contains(issues[i].Number) { + issues = append(issues[:i], issues[i+1:]...) + i-- + continue + } + mapInsertedIssueIndexes.Add(issues[i].Number) + } if err := uploader.CreateIssues(ctx, issues...); err != nil { return err @@ -381,6 +393,7 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba log.Trace("migrating pull requests and comments") messenger("repo.migrate.migrating_pulls") prBatchSize := uploader.MaxBatchInsertSize("pullrequest") + mapInsertedPRIndexes := container.Set[int64]{} for i := 1; ; i++ { prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize) if err != nil { @@ -390,6 +403,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba log.Warn("migrating pull requests is not supported, ignored") break } + for i := 0; i < len(prs); i++ { + if mapInsertedPRIndexes.Contains(prs[i].Number) { + prs = append(prs[:i], prs[i+1:]...) + i-- + continue + } + mapInsertedPRIndexes.Add(prs[i].Number) + } if err := uploader.CreatePullRequests(ctx, prs...); err != nil { return err diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index 9917bdae3c..a30e36c8b8 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -77,19 +77,19 @@ type OneDevDownloader struct { } // NewOneDevDownloader creates a new downloader -func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader { +func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader { + httpTransport := NewMigrationHTTPTransport() downloader := &OneDevDownloader{ baseURL: baseURL, repoPath: repoPath, client: &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - if len(username) > 0 && len(password) > 0 { + Transport: roundTripperFunc( + func(req *http.Request) (*http.Response, error) { + if username != "" && password != "" { req.SetBasicAuth(username, password) } - return nil, nil - }, - }, + return httpTransport.RoundTrip(req.WithContext(ctx)) + }), }, userMap: make(map[int64]*onedevUser), milestoneMap: make(map[int64]string), diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 3319e19851..9e93272d38 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -19,6 +19,7 @@ func TestOneDevDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't access test repo, skipping %s", t.Name()) } + defer resp.Body.Close() u, _ := url.Parse("https://code.onedev.io") ctx := t.Context() diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index da58bbd1b6..f9c40049db 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -449,7 +449,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err) return false } - _ = m.GetRepository(ctx) // force load repository of mirror + repo := m.GetRepository(ctx) // force load repository of mirror ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing Mirror %s/%s", m.Repo.OwnerName, m.Repo.Name)) defer finished() @@ -515,12 +515,12 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { } // Push commits - oldCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.oldCommitID) + oldCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.oldCommitID) if err != nil { log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err) continue } - newCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.newCommitID) + newCommitID, err := gitrepo.GetFullCommitID(ctx, repo, result.newCommitID) if err != nil { log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err) continue @@ -560,7 +560,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { } if !isEmpty { // Get latest commit date and update to current repository updated time - commitDate, err := git.GetLatestCommitTime(ctx, m.Repo.RepoPath()) + commitDate, err := gitrepo.GetLatestCommitTime(ctx, m.Repo) if err != nil { log.Error("SyncMirrors [repo: %-v]: unable to GetLatestCommitDate: %v", m.Repo, err) return false diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index b61345e830..bae189ba87 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -153,7 +153,7 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { log.Trace("Pushing %s mirror[%d] remote %s", storageRepo.RelativePath(), m.ID, m.RemoteName) envs := proxy.EnvWithProxy(remoteURL.URL) - if err := gitrepo.Push(ctx, storageRepo, git.PushOptions{ + if err := gitrepo.PushToExternal(ctx, storageRepo, git.PushOptions{ Remote: m.RemoteName, Force: true, Mirror: true, diff --git a/services/pull/check.go b/services/pull/check.go index 5b28ec9658..5978a57aec 100644 --- a/services/pull/check.go +++ b/services/pull/check.go @@ -295,7 +295,7 @@ func getMergeCommit(ctx context.Context, pr *issues_model.PullRequest) (*git.Com // If merge-base successfully exits then prHeadRef is an ancestor of pr.BaseBranch // Find the head commit id - prHeadCommitID, err := git.GetFullCommitID(ctx, pr.BaseRepo.RepoPath(), prHeadRef) + prHeadCommitID, err := gitrepo.GetFullCommitID(ctx, pr.BaseRepo, prHeadRef) if err != nil { return nil, fmt.Errorf("GetFullCommitID(%s) in %s: %w", prHeadRef, pr.BaseRepo.FullName(), err) } diff --git a/services/pull/compare.go b/services/pull/compare.go index 2c4b77a772..fbdb17cfdd 100644 --- a/services/pull/compare.go +++ b/services/pull/compare.go @@ -33,7 +33,7 @@ func GetCompareInfo(ctx context.Context, baseRepo, headRepo *repo_model.Reposito ) // We don't need a temporary remote for same repository. - if headGitRepo.Path != baseRepo.RepoPath() { + if baseRepo.ID != headRepo.ID { // Add a temporary remote tmpRemote = strconv.FormatInt(time.Now().UnixNano(), 10) if err = gitrepo.GitRemoteAdd(ctx, headRepo, tmpRemote, baseRepo.RepoPath()); err != nil { @@ -48,14 +48,14 @@ func GetCompareInfo(ctx context.Context, baseRepo, headRepo *repo_model.Reposito compareInfo := new(CompareInfo) - compareInfo.HeadCommitID, err = git.GetFullCommitID(ctx, headGitRepo.Path, headBranch) + compareInfo.HeadCommitID, err = gitrepo.GetFullCommitID(ctx, headRepo, headBranch) if err != nil { compareInfo.HeadCommitID = headBranch } compareInfo.MergeBase, remoteBranch, err = headGitRepo.GetMergeBase(tmpRemote, baseBranch, headBranch) if err == nil { - compareInfo.BaseCommitID, err = git.GetFullCommitID(ctx, headGitRepo.Path, remoteBranch) + compareInfo.BaseCommitID, err = gitrepo.GetFullCommitID(ctx, headRepo, remoteBranch) if err != nil { compareInfo.BaseCommitID = remoteBranch } @@ -77,7 +77,7 @@ func GetCompareInfo(ctx context.Context, baseRepo, headRepo *repo_model.Reposito } } else { compareInfo.Commits = []*git.Commit{} - compareInfo.MergeBase, err = git.GetFullCommitID(ctx, headGitRepo.Path, remoteBranch) + compareInfo.MergeBase, err = gitrepo.GetFullCommitID(ctx, headRepo, remoteBranch) if err != nil { compareInfo.MergeBase = remoteBranch } diff --git a/services/pull/pull.go b/services/pull/pull.go index 04f48f0565..ecc0b2c7ce 100644 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -570,13 +570,11 @@ func pushToBaseRepoHelper(ctx context.Context, pr *issues_model.PullRequest, pre log.Error("Unable to load head repository for PR[%d] Error: %v", pr.ID, err) return err } - headRepoPath := pr.HeadRepo.RepoPath() if err := pr.LoadBaseRepo(ctx); err != nil { log.Error("Unable to load base repository for PR[%d] Error: %v", pr.ID, err) return err } - baseRepoPath := pr.BaseRepo.RepoPath() if err = pr.LoadIssue(ctx); err != nil { return fmt.Errorf("unable to load issue %d for pr %d: %w", pr.IssueID, pr.ID, err) @@ -587,8 +585,7 @@ func pushToBaseRepoHelper(ctx context.Context, pr *issues_model.PullRequest, pre gitRefName := pr.GetGitHeadRefName() - if err := git.Push(ctx, headRepoPath, git.PushOptions{ - Remote: baseRepoPath, + if err := gitrepo.Push(ctx, pr.HeadRepo, pr.BaseRepo, git.PushOptions{ Branch: prefixHeadBranch + pr.HeadBranch + ":" + gitRefName, Force: true, // Use InternalPushingEnvironment here because we know that pre-receive and post-receive do not run on a refs/pulls/... diff --git a/services/release/notes.go b/services/release/notes.go new file mode 100644 index 0000000000..c9dc75af70 --- /dev/null +++ b/services/release/notes.go @@ -0,0 +1,188 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package release + +import ( + "cmp" + "context" + "fmt" + "slices" + "strings" + + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/util" +) + +// GenerateReleaseNotesOptions describes how to build release notes content. +type GenerateReleaseNotesOptions struct { + TagName string + TagTarget string + PreviousTag string +} + +// GenerateReleaseNotes builds the markdown snippet for release notes. +func GenerateReleaseNotes(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, opts GenerateReleaseNotesOptions) (string, error) { + headCommit, err := resolveHeadCommit(gitRepo, opts.TagName, opts.TagTarget) + if err != nil { + return "", err + } + + if opts.PreviousTag == "" { + // no previous tag, usually due to there is no tag in the repo, use the same content as GitHub + content := fmt.Sprintf("**Full Changelog**: %s/commits/tag/%s\n", repo.HTMLURL(ctx), util.PathEscapeSegments(opts.TagName)) + return content, nil + } + + baseCommit, err := gitRepo.GetCommit(opts.PreviousTag) + if err != nil { + return "", util.ErrorWrapTranslatable(util.ErrNotExist, "repo.release.generate_notes_tag_not_found", opts.TagName) + } + + commits, err := gitRepo.CommitsBetweenIDs(headCommit.ID.String(), baseCommit.ID.String()) + if err != nil { + return "", fmt.Errorf("CommitsBetweenIDs: %w", err) + } + + prs, err := collectPullRequestsFromCommits(ctx, repo.ID, commits) + if err != nil { + return "", err + } + + contributors, newContributors, err := collectContributors(ctx, repo.ID, prs) + if err != nil { + return "", err + } + + content := buildReleaseNotesContent(ctx, repo, opts.TagName, opts.PreviousTag, prs, contributors, newContributors) + return content, nil +} + +func resolveHeadCommit(gitRepo *git.Repository, tagName, tagTarget string) (*git.Commit, error) { + ref := tagName + if !gitRepo.IsTagExist(tagName) { + ref = tagTarget + } + + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return nil, util.ErrorWrapTranslatable(util.ErrNotExist, "repo.release.generate_notes_target_not_found", ref) + } + return commit, nil +} + +func collectPullRequestsFromCommits(ctx context.Context, repoID int64, commits []*git.Commit) ([]*issues_model.PullRequest, error) { + prs := make([]*issues_model.PullRequest, 0, len(commits)) + + for _, commit := range commits { + pr, err := issues_model.GetPullRequestByMergedCommit(ctx, repoID, commit.ID.String()) + if err != nil { + if issues_model.IsErrPullRequestNotExist(err) { + continue + } + return nil, fmt.Errorf("GetPullRequestByMergedCommit: %w", err) + } + + if err = pr.LoadIssue(ctx); err != nil { + return nil, fmt.Errorf("LoadIssue: %w", err) + } + if err = pr.Issue.LoadAttributes(ctx); err != nil { + return nil, fmt.Errorf("LoadIssueAttributes: %w", err) + } + + prs = append(prs, pr) + } + + slices.SortFunc(prs, func(a, b *issues_model.PullRequest) int { + if cmpRes := cmp.Compare(b.MergedUnix, a.MergedUnix); cmpRes != 0 { + return cmpRes + } + return cmp.Compare(b.Issue.Index, a.Issue.Index) + }) + + return prs, nil +} + +func buildReleaseNotesContent(ctx context.Context, repo *repo_model.Repository, tagName, baseRef string, prs []*issues_model.PullRequest, contributors []*user_model.User, newContributors []*issues_model.PullRequest) string { + var builder strings.Builder + builder.WriteString("## What's Changed\n") + + for _, pr := range prs { + prURL := pr.Issue.HTMLURL(ctx) + builder.WriteString(fmt.Sprintf("* %s in [#%d](%s)\n", pr.Issue.Title, pr.Issue.Index, prURL)) + } + + builder.WriteString("\n") + + if len(contributors) > 0 { + builder.WriteString("## Contributors\n") + for _, contributor := range contributors { + builder.WriteString(fmt.Sprintf("* @%s\n", contributor.Name)) + } + builder.WriteString("\n") + } + + if len(newContributors) > 0 { + builder.WriteString("## New Contributors\n") + for _, contributor := range newContributors { + prURL := contributor.Issue.HTMLURL(ctx) + builder.WriteString(fmt.Sprintf("* @%s made their first contribution in [#%d](%s)\n", contributor.Issue.Poster.Name, contributor.Issue.Index, prURL)) + } + builder.WriteString("\n") + } + + builder.WriteString("**Full Changelog**: ") + compareURL := fmt.Sprintf("%s/compare/%s...%s", repo.HTMLURL(ctx), util.PathEscapeSegments(baseRef), util.PathEscapeSegments(tagName)) + builder.WriteString(fmt.Sprintf("[%s...%s](%s)", baseRef, tagName, compareURL)) + builder.WriteByte('\n') + return builder.String() +} + +func collectContributors(ctx context.Context, repoID int64, prs []*issues_model.PullRequest) ([]*user_model.User, []*issues_model.PullRequest, error) { + contributors := make([]*user_model.User, 0, len(prs)) + newContributors := make([]*issues_model.PullRequest, 0, len(prs)) + seenContributors := container.Set[int64]{} + seenNew := container.Set[int64]{} + + for _, pr := range prs { + poster := pr.Issue.Poster + posterID := poster.ID + + if posterID == 0 { + // Migrated PRs may not have a linked local user (PosterID == 0). Skip them for now. + continue + } + + if !seenContributors.Contains(posterID) { + contributors = append(contributors, poster) + seenContributors.Add(posterID) + } + + if seenNew.Contains(posterID) { + continue + } + + isFirst, err := isFirstContribution(ctx, repoID, posterID, pr) + if err != nil { + return nil, nil, err + } + if isFirst { + seenNew.Add(posterID) + newContributors = append(newContributors, pr) + } + } + + return contributors, newContributors, nil +} + +func isFirstContribution(ctx context.Context, repoID, posterID int64, pr *issues_model.PullRequest) (bool, error) { + hasMergedBefore, err := issues_model.HasMergedPullRequestInRepoBefore(ctx, repoID, posterID, pr.MergedUnix, pr.ID) + if err != nil { + return false, fmt.Errorf("check merged PRs for contributor: %w", err) + } + return !hasMergedBefore, nil +} diff --git a/services/release/notes_test.go b/services/release/notes_test.go new file mode 100644 index 0000000000..211c364deb --- /dev/null +++ b/services/release/notes_test.go @@ -0,0 +1,97 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package release + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateReleaseNotes(t *testing.T) { + unittest.PrepareTestEnv(t) + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + gitRepo, err := gitrepo.OpenRepository(t.Context(), repo) + require.NoError(t, err) + + t.Run("ChangeLogsWithPRs", func(t *testing.T) { + mergedCommit := "90c1019714259b24fb81711d4416ac0f18667dfa" + createMergedPullRequest(t, repo, mergedCommit, 5) + + content, err := GenerateReleaseNotes(t.Context(), repo, gitRepo, GenerateReleaseNotesOptions{ + TagName: "v1.2.0", + TagTarget: "DefaultBranch", + PreviousTag: "v1.1", + }) + require.NoError(t, err) + + assert.Equal(t, `## What's Changed +* Release notes test pull request in [#6](https://try.gitea.io/user2/repo1/pulls/6) + +## Contributors +* @user5 + +## New Contributors +* @user5 made their first contribution in [#6](https://try.gitea.io/user2/repo1/pulls/6) + +**Full Changelog**: [v1.1...v1.2.0](https://try.gitea.io/user2/repo1/compare/v1.1...v1.2.0) +`, content) + }) + + t.Run("NoPreviousTag", func(t *testing.T) { + content, err := GenerateReleaseNotes(t.Context(), repo, gitRepo, GenerateReleaseNotesOptions{ + TagName: "v1.2.0", + TagTarget: "DefaultBranch", + }) + require.NoError(t, err) + assert.Equal(t, "**Full Changelog**: https://try.gitea.io/user2/repo1/commits/tag/v1.2.0\n", content) + }) +} + +func createMergedPullRequest(t *testing.T, repo *repo_model.Repository, mergeCommit string, posterID int64) *issues_model.PullRequest { + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: posterID}) + + issue := &issues_model.Issue{ + RepoID: repo.ID, + Repo: repo, + Poster: user, + PosterID: user.ID, + Title: "Release notes test pull request", + Content: "content", + } + + pr := &issues_model.PullRequest{ + HeadRepoID: repo.ID, + BaseRepoID: repo.ID, + HeadBranch: repo.DefaultBranch, + BaseBranch: repo.DefaultBranch, + Status: issues_model.PullRequestStatusMergeable, + Flow: issues_model.PullRequestFlowGithub, + } + + require.NoError(t, issues_model.NewPullRequest(t.Context(), repo, issue, nil, nil, pr)) + + pr.HasMerged = true + pr.MergedCommitID = mergeCommit + pr.MergedUnix = timeutil.TimeStampNow() + _, err := db.GetEngine(t.Context()). + ID(pr.ID). + Cols("has_merged", "merged_commit_id", "merged_unix"). + Update(pr) + require.NoError(t, err) + + require.NoError(t, pr.LoadIssue(t.Context())) + require.NoError(t, pr.Issue.LoadAttributes(t.Context())) + return pr +} diff --git a/services/repository/branch.go b/services/repository/branch.go index 0a2fd30620..142073eabe 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -385,8 +385,7 @@ func CreateNewBranchFromCommit(ctx context.Context, doer *user_model.User, repo return err } - if err := git.Push(ctx, repo.RepoPath(), git.PushOptions{ - Remote: repo.RepoPath(), + if err := gitrepo.Push(ctx, repo, repo, git.PushOptions{ Branch: fmt.Sprintf("%s:%s%s", commitID, git.BranchPrefix, branchName), Env: repo_module.PushingEnvironment(doer, repo), }); err != nil { @@ -483,6 +482,63 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_m return "", nil } +// UpdateBranch moves a branch reference to the provided commit. permission check should be done before calling this function. +func UpdateBranch(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, doer *user_model.User, branchName, newCommitID, expectedOldCommitID string, force bool) error { + branch, err := git_model.GetBranch(ctx, repo.ID, branchName) + if err != nil { + return err + } + if branch.IsDeleted { + return git_model.ErrBranchNotExist{ + BranchName: branchName, + } + } + + if expectedOldCommitID != "" { + expectedID, err := gitRepo.ConvertToGitID(expectedOldCommitID) + if err != nil { + return fmt.Errorf("ConvertToGitID(old): %w", err) + } + if expectedID.String() != branch.CommitID { + return util.NewInvalidArgumentErrorf("branch commit does not match [expected: %s, given: %s]", expectedID.String(), branch.CommitID) + } + } + + newID, err := gitRepo.ConvertToGitID(newCommitID) + if err != nil { + return fmt.Errorf("ConvertToGitID(new): %w", err) + } + newCommit, err := gitRepo.GetCommit(newID.String()) + if err != nil { + return err + } + + if newCommit.ID.String() == branch.CommitID { + return nil + } + + isForcePush, err := newCommit.IsForcePush(branch.CommitID) + if err != nil { + return err + } + if isForcePush && !force { + return util.NewInvalidArgumentErrorf("Force push %s need a confirm force parameter", branchName) + } + + pushOpts := git.PushOptions{ + Branch: fmt.Sprintf("%s:%s%s", newCommit.ID.String(), git.BranchPrefix, branchName), + Env: repo_module.PushingEnvironment(doer, repo), + Force: isForcePush || force, + } + + if expectedOldCommitID != "" { + pushOpts.ForceWithLease = fmt.Sprintf("%s:%s", git.BranchPrefix+branchName, branch.CommitID) + } + + // branch protection will be checked in the pre received hook, so that we don't need any check here + return gitrepo.Push(ctx, repo, repo, pushOpts) +} + var ErrBranchIsDefault = util.ErrorWrap(util.ErrPermissionDenied, "branch is default") func CanDeleteBranch(ctx context.Context, repo *repo_model.Repository, branchName string, doer *user_model.User) error { diff --git a/services/repository/create.go b/services/repository/create.go index 0b57db988b..7439fc8f08 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -69,9 +69,10 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir ) // Clone to temporary path and do the init commit. - if stdout, _, err := gitcmd.NewCommand("clone").AddDynamicArguments(repo.RepoPath(), tmpDir). - WithEnv(env).RunStdString(ctx); err != nil { - log.Error("Failed to clone from %v into %s: stdout: %s\nError: %v", repo, tmpDir, stdout, err) + if err := gitrepo.CloneRepoToLocal(ctx, repo, tmpDir, git.CloneRepoOptions{ + Env: env, + }); err != nil { + log.Error("Failed to clone from %v into %s\nError: %v", repo, tmpDir, err) return fmt.Errorf("git clone: %w", err) } diff --git a/services/repository/files/patch.go b/services/repository/files/patch.go index 8fe6bb917b..5361091c90 100644 --- a/services/repository/files/patch.go +++ b/services/repository/files/patch.go @@ -95,7 +95,7 @@ func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_mode } } if protectedBranch != nil && protectedBranch.RequireSignedCommits { - _, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), opts.OldBranch) + _, _, _, err := asymkey_service.SignCRUDAction(ctx, doer, repo.RepoPath(), opts.OldBranch) if err != nil { if !asymkey_service.IsErrWontSign(err) { return err diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 731f23855d..cb39abfd6e 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -18,6 +18,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -54,12 +55,11 @@ func (t *TemporaryUploadRepository) Close() { // Clone the base repository to our path and set branch as the HEAD func (t *TemporaryUploadRepository) Clone(ctx context.Context, branch string, bare bool) error { - cmd := gitcmd.NewCommand("clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) - if bare { - cmd.AddArguments("--bare") - } - - if _, _, err := cmd.RunStdString(ctx); err != nil { + if err := gitrepo.CloneRepoToLocal(ctx, t.repo, t.basePath, git.CloneRepoOptions{ + Bare: bare, + Branch: branch, + Shared: true, + }); err != nil { stderr := err.Error() if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { return git.ErrBranchNotExist{ @@ -303,9 +303,9 @@ func (t *TemporaryUploadRepository) CommitTree(ctx context.Context, opts *Commit var key *git.SigningKey var signer *git.Signature if opts.ParentCommitID != "" { - sign, key, signer, _ = asymkey_service.SignCRUDAction(ctx, t.repo.RepoPath(), opts.DoerUser, t.basePath, opts.ParentCommitID) + sign, key, signer, _ = asymkey_service.SignCRUDAction(ctx, opts.DoerUser, t.basePath, opts.ParentCommitID) } else { - sign, key, signer, _ = asymkey_service.SignInitialCommit(ctx, t.repo.RepoPath(), opts.DoerUser) + sign, key, signer, _ = asymkey_service.SignInitialCommit(ctx, opts.DoerUser) } if sign { if key.Format != "" { @@ -362,8 +362,7 @@ func (t *TemporaryUploadRepository) CommitTree(ctx context.Context, opts *Commit func (t *TemporaryUploadRepository) Push(ctx context.Context, doer *user_model.User, commitHash, branch string, force bool) error { // Because calls hooks we need to pass in the environment env := repo_module.PushingEnvironment(doer, t.repo) - if err := git.Push(ctx, t.basePath, git.PushOptions{ - Remote: t.repo.RepoPath(), + if err := gitrepo.PushFromLocal(ctx, t.basePath, t.repo, git.PushOptions{ Branch: strings.TrimSpace(commitHash) + ":" + git.BranchPrefix + strings.TrimSpace(branch), Env: env, Force: force, diff --git a/services/repository/files/update.go b/services/repository/files/update.go index 4830f711fc..967c4d928e 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -686,7 +686,7 @@ func VerifyBranchProtection(ctx context.Context, repo *repo_model.Repository, do } } if protectedBranch.RequireSignedCommits { - _, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), branchName) + _, _, _, err := asymkey_service.SignCRUDAction(ctx, doer, repo.RepoPath(), branchName) if err != nil { if !asymkey_service.IsErrWontSign(err) { return err diff --git a/services/repository/fork.go b/services/repository/fork.go index 2380666afb..f92af65605 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -15,7 +15,6 @@ import ( "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -147,15 +146,16 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork } // 3 - Clone the repository - cloneCmd := gitcmd.NewCommand("clone", "--bare") - if opts.SingleBranch != "" { - cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) + cloneOpts := git.CloneRepoOptions{ + Bare: true, + Timeout: 10 * time.Minute, } - var stdout []byte - if stdout, _, err = cloneCmd.AddDynamicArguments(opts.BaseRepo.RepoPath(), repo.RepoPath()). - WithTimeout(10 * time.Minute). - RunStdBytes(ctx); err != nil { - log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) + if opts.SingleBranch != "" { + cloneOpts.SingleBranch = true + cloneOpts.Branch = opts.SingleBranch + } + if err = gitrepo.Clone(ctx, opts.BaseRepo, repo, cloneOpts); err != nil { + log.Error("Fork Repository (git clone) Failed for %v (from %v):\nError: %v", repo, opts.BaseRepo, err) return nil, fmt.Errorf("git clone: %w", err) } diff --git a/services/repository/generate.go b/services/repository/generate.go index caf15265a0..b2913cd110 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -177,7 +177,7 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t } generatedContent := generateExpansion(ctx, string(content), templateRepo, generateRepo) - substSubPath := filepath.Clean(filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo))) + substSubPath := filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo)) newLocalPath := filepath.Join(tmpDir, substSubPath) regular, err := util.IsRegularFile(newLocalPath) if canWrite := regular || errors.Is(err, fs.ErrNotExist); !canWrite { @@ -230,8 +230,7 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r ) // Clone to temporary path and do the init commit. - templateRepoPath := templateRepo.RepoPath() - if err := git.Clone(ctx, templateRepoPath, tmpDir, git.CloneRepoOptions{ + if err := gitrepo.CloneRepoToLocal(ctx, templateRepo, tmpDir, git.CloneRepoOptions{ Depth: 1, Branch: templateRepo.DefaultBranch, }); err != nil { @@ -359,5 +358,5 @@ func filePathSanitize(s string) string { } fields[i] = field } - return filepath.FromSlash(strings.Join(fields, "/")) + return filepath.Clean(filepath.FromSlash(strings.Trim(strings.Join(fields, "/"), "/"))) } diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index 9c01911ded..432de4dc59 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -54,19 +54,24 @@ text/*.txt } func TestFilePathSanitize(t *testing.T) { - assert.Equal(t, "test_CON", filePathSanitize("test_CON")) - assert.Equal(t, "test CON", filePathSanitize("test CON ")) - assert.Equal(t, "__/traverse/__", filePathSanitize(".. /traverse/ ..")) - assert.Equal(t, "./__/a/_git/b_", filePathSanitize("./../a/.git/ b: ")) + // path clean + assert.Equal(t, "a", filePathSanitize("//a/")) + assert.Equal(t, "_a", filePathSanitize(`\a`)) + assert.Equal(t, "__/a/__", filePathSanitize(".. /a/ ..")) + assert.Equal(t, "__/a/_git/b_", filePathSanitize("./../a/.git/ b: ")) + + // Windows reserved names assert.Equal(t, "_", filePathSanitize("CoN")) assert.Equal(t, "_", filePathSanitize("LpT1")) assert.Equal(t, "_", filePathSanitize("CoM1")) + assert.Equal(t, "test_CON", filePathSanitize("test_CON")) + assert.Equal(t, "test CON", filePathSanitize("test CON ")) + + // special chars assert.Equal(t, "_", filePathSanitize("\u0000")) - assert.Equal(t, "目标", filePathSanitize("目标")) - // unlike filepath.Clean, it only sanitizes, doesn't change the separator layout - assert.Equal(t, "", filePathSanitize("")) //nolint:testifylint // for easy reading + assert.Equal(t, ".", filePathSanitize("")) assert.Equal(t, ".", filePathSanitize(".")) - assert.Equal(t, "/", filePathSanitize("/")) + assert.Equal(t, ".", filePathSanitize("/")) } func TestProcessGiteaTemplateFile(t *testing.T) { diff --git a/services/repository/init.go b/services/repository/init.go index 8d9decf811..51cc113d63 100644 --- a/services/repository/init.go +++ b/services/repository/init.go @@ -41,7 +41,7 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi cmd := gitcmd.NewCommand("commit", "--message=Initial commit"). AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email) - sign, key, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) + sign, key, signer, _ := asymkey_service.SignInitialCommit(ctx, u) if sign { if key.Format != "" { cmd.AddConfig("gpg.format", key.Format) diff --git a/services/repository/merge_upstream.go b/services/repository/merge_upstream.go index 8d6f11372c..692b801303 100644 --- a/services/repository/merge_upstream.go +++ b/services/repository/merge_upstream.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/modules/util" @@ -33,8 +34,7 @@ func MergeUpstream(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_ return "up-to-date", nil } - err = git.Push(ctx, repo.BaseRepo.RepoPath(), git.PushOptions{ - Remote: repo.RepoPath(), + err = gitrepo.Push(ctx, repo.BaseRepo, repo, git.PushOptions{ Branch: fmt.Sprintf("%s:%s", divergingInfo.BaseBranchName, branch), Env: repo_module.PushingEnvironment(doer, repo), }) diff --git a/services/repository/migrate.go b/services/repository/migrate.go index acac6fd9ad..8f515326ad 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -74,8 +74,6 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts migration.MigrateOptions, httpTransport *http.Transport, ) (*repo_model.Repository, error) { - repoPath := repo.RepoPath() - if u.IsOrganization() { t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx) if err != nil { @@ -92,7 +90,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("failed to remove existing repo dir %q, err: %w", repo.FullName(), err) } - if err := git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ + if err := gitrepo.CloneExternalRepo(ctx, opts.CloneAddr, repo, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -104,7 +102,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("clone error: %w", err) } - if err := git.WriteCommitGraph(ctx, repoPath); err != nil { + if err := gitrepo.WriteCommitGraph(ctx, repo); err != nil { return repo, err } diff --git a/services/repository/repository.go b/services/repository/repository.go index acc5ce56cf..4d07cb0e38 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -7,8 +7,6 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" "strings" activities_model "code.gitea.io/gitea/models/activities" @@ -28,7 +26,6 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" ) @@ -251,9 +248,8 @@ func CheckDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error } // Create/Remove git-daemon-export-ok for git-daemon... - daemonExportFile := filepath.Join(repo.RepoPath(), `git-daemon-export-ok`) - - isExist, err := util.IsExist(daemonExportFile) + daemonExportFile := `git-daemon-export-ok` + isExist, err := gitrepo.IsRepoFileExist(ctx, repo, daemonExportFile) if err != nil { log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) return err @@ -261,11 +257,11 @@ func CheckDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic if !isPublic && isExist { - if err = util.Remove(daemonExportFile); err != nil { + if err = gitrepo.RemoveRepoFileOrDir(ctx, repo, daemonExportFile); err != nil { log.Error("Failed to remove %s: %v", daemonExportFile, err) } } else if isPublic && !isExist { - if f, err := os.Create(daemonExportFile); err != nil { + if f, err := gitrepo.CreateRepoFile(ctx, repo, daemonExportFile); err != nil { log.Error("Failed to create %s: %v", daemonExportFile, err) } else { f.Close() @@ -345,3 +341,31 @@ func HasWiki(ctx context.Context, repo *repo_model.Repository) bool { } return hasWiki && err == nil } + +// CheckCreateRepository check if doer could create a repository in new owner +func CheckCreateRepository(ctx context.Context, doer, owner *user_model.User, name string, overwriteOrAdopt bool) error { + if !doer.CanCreateRepoIn(owner) { + return repo_model.ErrReachLimitOfRepo{Limit: owner.MaxRepoCreation} + } + + if err := repo_model.IsUsableRepoName(name); err != nil { + return err + } + + has, err := repo_model.IsRepositoryModelExist(ctx, owner, name) + if err != nil { + return err + } else if has { + return repo_model.ErrRepoAlreadyExist{Uname: owner.Name, Name: name} + } + repo := repo_model.StorageRepo(repo_model.RelativePath(owner.Name, name)) + isExist, err := gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.RelativePath(), err) + return err + } + if !overwriteOrAdopt && isExist { + return repo_model.ErrRepoFilesAlreadyExist{Uname: owner.Name, Name: name} + } + return nil +} diff --git a/services/repository/transfer.go b/services/repository/transfer.go index 98307a447a..af477fc7f1 100644 --- a/services/repository/transfer.go +++ b/services/repository/transfer.go @@ -90,6 +90,17 @@ func AcceptTransferOwnership(ctx context.Context, repo *repo_model.Repository, d return nil } +// isRepositoryModelOrDirExist returns true if the repository with given name under user has already existed. +func isRepositoryModelOrDirExist(ctx context.Context, u *user_model.User, repoName string) (bool, error) { + has, err := repo_model.IsRepositoryModelExist(ctx, u, repoName) + if err != nil { + return false, err + } + repo := repo_model.StorageRepo(repo_model.RelativePath(u.Name, repoName)) + isExist, err := gitrepo.IsRepositoryExist(ctx, repo) + return has || isExist, err +} + // transferOwnership transfers all corresponding repository items from old user to new one. func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName string, repo *repo_model.Repository, teams []*organization.Team) (err error) { repoRenamed := false @@ -143,7 +154,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName newOwnerName = newOwner.Name // ensure capitalisation matches // Check if new owner has repository with same name. - if has, err := repo_model.IsRepositoryModelOrDirExist(ctx, newOwner, repo.Name); err != nil { + if has, err := isRepositoryModelOrDirExist(ctx, newOwner, repo.Name); err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { return repo_model.ErrRepoAlreadyExist{ @@ -345,7 +356,7 @@ func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newR return err } - has, err := repo_model.IsRepositoryModelOrDirExist(ctx, repo.Owner, newRepoName) + has, err := isRepositoryModelOrDirExist(ctx, repo.Owner, newRepoName) if err != nil { return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index 6a57a9a63e..5f74817ef3 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -25,8 +25,6 @@ import ( repo_service "code.gitea.io/gitea/services/repository" ) -const DefaultRemote = "origin" - func getWikiWorkingLockKey(repoID int64) string { return fmt.Sprintf("wiki_working_%d", repoID) } @@ -214,8 +212,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return err } - if err := git.Push(gitRepo.Ctx, basePath, git.PushOptions{ - Remote: DefaultRemote, + if err := gitrepo.PushFromLocal(gitRepo.Ctx, basePath, repo.WikiStorageRepo(), git.PushOptions{ Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, repo.DefaultWikiBranch), Env: repo_module.FullPushingEnvironment( doer, @@ -333,8 +330,7 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return err } - if err := git.Push(gitRepo.Ctx, basePath, git.PushOptions{ - Remote: DefaultRemote, + if err := gitrepo.PushFromLocal(gitRepo.Ctx, basePath, repo.WikiStorageRepo(), git.PushOptions{ Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, repo.DefaultWikiBranch), Env: repo_module.FullPushingEnvironment( doer, diff --git a/templates/package/content/pypi.tmpl b/templates/package/content/pypi.tmpl index 2625c160fe..15d8971eaa 100644 --- a/templates/package/content/pypi.tmpl +++ b/templates/package/content/pypi.tmpl @@ -4,7 +4,7 @@
-
pip install --index-url  --extra-index-url https://pypi.org/ {{.PackageDescriptor.Package.Name}}
+
pip install --index-url  --extra-index-url https://pypi.org/simple {{.PackageDescriptor.Package.Name}}
diff --git a/templates/repo/blame.tmpl b/templates/repo/blame.tmpl index c4d9f0741f..9cd4b2a122 100644 --- a/templates/repo/blame.tmpl +++ b/templates/repo/blame.tmpl @@ -38,7 +38,7 @@ {{range $row := .BlameRows}} - +
diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 089cdf2ccd..6d23186d08 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -96,7 +96,7 @@
{{else if eq .Type 2}}
- {{svg "octicon-circle-slash"}} + {{svg "octicon-issue-closed"}} {{if not .OriginalAuthor}} {{template "shared/user/avatarlink" dict "user" .Poster}} {{end}} diff --git a/templates/repo/release/new.tmpl b/templates/repo/release/new.tmpl index 109a18fa0e..44f496e73d 100644 --- a/templates/repo/release/new.tmpl +++ b/templates/repo/release/new.tmpl @@ -12,17 +12,25 @@ {{end}} {{template "base/alert" .}} -
+ + {{.CsrfTokenHtml}} -
-
- {{if .PageIsEditRelease}} - {{.tag_name}}@{{.tag_target}} - {{else}} - - -
- @ +
+ + {{if .PageIsEditRelease}} + + + {{.tag_name}} @ {{.tag_target}} + {{else}} +
+ +
+ @
-
- {{ctx.Locale.Tr "repo.release.tag_helper"}} -
- {{end}} -
+
+
+ {{ctx.Locale.Tr "repo.release.tag_helper"}} +
+ {{end}}
-
-
- + +
+ + +
+ +
+ + +
+
+ {{template "shared/combomarkdowneditor" (dict + "MarkdownPreviewInRepo" $.Repository + "MarkdownPreviewMode" "comment" + "TextareaName" "content" + "TextareaContent" .content + "TextareaPlaceholder" (ctx.Locale.Tr "repo.release.message") + "DropzoneParentContainer" "form" + )}} +
+ + {{range .attachments}} +
+
+ + + {{.Size | FileSize}} + + {{svg "octicon-info"}} + +
+ + {{ctx.Locale.Tr "remove"}} +
+ {{end}} + {{if .IsAttachmentEnabled}}
- {{template "shared/combomarkdowneditor" (dict - "MarkdownPreviewInRepo" $.Repository - "MarkdownPreviewMode" "comment" - "TextareaName" "content" - "TextareaContent" .content - "TextareaPlaceholder" (ctx.Locale.Tr "repo.release.message") - "DropzoneParentContainer" "form" - )}} + {{template "repo/upload" .}}
- {{range .attachments}} -
-
- - - {{.Size | FileSize}} - - {{svg "octicon-info"}} - -
- - {{ctx.Locale.Tr "remove"}} - + {{end}} + + {{if not .PageIsEditRelease}} +
+
+ +
- {{end}} - {{if .IsAttachmentEnabled}} -
- {{template "repo/upload" .}} -
- {{end}} +
+ {{else}} + + {{end}} + +
+
+ + +
+
{{ctx.Locale.Tr "repo.release.prerelease_helper"}}
-
-
-
- {{if not .PageIsEditRelease}} -
-
- - -
-
+ +
+ {{if .PageIsEditRelease}} + + {{ctx.Locale.Tr "repo.release.cancel"}} + + + {{ctx.Locale.Tr "repo.release.delete_release"}} + + {{if .IsDraft}} + + {{else}} - + {{end}} -
-
- - -
-
- {{ctx.Locale.Tr "repo.release.prerelease_helper"}} -
-
- {{if .PageIsEditRelease}} - - {{ctx.Locale.Tr "repo.release.cancel"}} - - - {{ctx.Locale.Tr "repo.release.delete_release"}} - - {{if .IsDraft}} - - - {{else}} - - {{end}} - {{else}} - {{if .ShowCreateTagOnlyButton}} - - {{end}} - - - {{end}} -
-
+ {{else}} + {{if .ShowCreateTagOnlyButton}} + + {{end}} + + + {{end}}
-{{if .PageIsEditRelease}} -