diff --git a/docs/content/administration/repo-size-limit.en-us.md b/docs/content/administration/repo-size-limit.en-us.md deleted file mode 100644 index 7e35531294..0000000000 --- a/docs/content/administration/repo-size-limit.en-us.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -date: "2023-05-24T13:00:00+00:00" -title: "Per repository size limit" -slug: "repo-size-limit" -weight: 12 -toc: false -draft: false -aliases: - - /en-us/repo-size-limit -menu: - sidebar: - parent: "administration" - name: "Per repository size limit" - weight: 12 - identifier: "repo-size-limit" ---- - -# Gitea per repository size limit setup - -To use Gitea's experimental built-in per repository size limit support, Administrator must update the `app.ini` file: - -```ini -;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit -;; "false" means no limit will be enforced, even if specified on a repository -ENABLE_SIZE_LIMIT = true - -;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit -;; If repository has it's own limit set in UI it will override the global setting -;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ... -REPO_SIZE_LIMIT = 500 MB - -This setting is persistent. - -The size limitation is triggered when repository `disk size` + `new commit size` > `defined repository size limit` - -If size limitation is triggered the feature would prevent commits that increase repository size on disk -of gitea server and allow those that decrease it - -# Gitea per repository size limit setup in UI - -1. For Gitea admin it is possible during runtime to enable/disable limit size feature, change the global size limit on the fly. -**This setting is not persistent across restarts** - -`Admin panel/Site settings` -> `Repository management` - -Persistance can be achieved if the limit is maintained by editing `app.ini` file - -2. The individually set per repository limit in `Settings` of the -repository would take precedence over global limit when the size limit -feature is enabled. Only admin can modify those limits - -**Note**: Size checking for large repositories is time consuming operation so time of push under size limit might increase up to a minute depending on your server hardware diff --git a/models/migrations/v1_26/v999.go b/models/migrations/v1_26/v999.go index 4831f4d972..5fc9774cdb 100644 --- a/models/migrations/v1_26/v999.go +++ b/models/migrations/v1_26/v999.go @@ -9,8 +9,9 @@ import ( func AddSizeLimitOnRepo(x *xorm.Engine) error { type Repository struct { - ID int64 `xorm:"pk autoincr"` - SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` + ID int64 `xorm:"pk autoincr"` + SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` + LFSSizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` } return x.Sync2(new(Repository)) diff --git a/models/repo/repo.go b/models/repo/repo.go index b1d67119a7..02333b13e9 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -206,6 +206,7 @@ type Repository struct { EnableSizeLimit bool `xorm:"NOT NULL DEFAULT true"` GitSize int64 `xorm:"NOT NULL DEFAULT 0"` LFSSize int64 `xorm:"NOT NULL DEFAULT 0"` + LFSSizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` @@ -647,12 +648,46 @@ func (repo *Repository) IsRepoSizeOversized(additionalSize int64) bool { return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.GitSize+additionalSize > repo.GetActualSizeLimit() } -// RepoSizeLimitEnabled return true if size limit checking is enabled and limit is non zero for this specific repository +// ShouldCheckRepoSize returns true if size limit checking is enabled and limit is non zero for this specific repository // this is used to enable size checking during pre-receive hook -func (repo *Repository) IsRepoSizeLimitEnabled() bool { +func (repo *Repository) ShouldCheckRepoSize() bool { return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 } +// GetActualLFSSizeLimit returns repository LFS size limit in bytes +// or global LFS size limit setting if per repository LFS size limit is not set +func (repo *Repository) GetActualLFSSizeLimit() int64 { + lfsSizeLimit := repo.LFSSizeLimit + if setting.LFSSizeLimit > 0 && lfsSizeLimit == 0 { + lfsSizeLimit = setting.LFSSizeLimit + } + return lfsSizeLimit +} + +// ShouldCheckLFSSize returns true if LFS size limit checking is enabled for this repository +func (repo *Repository) ShouldCheckLFSSize() bool { + return setting.EnableSizeLimit && repo.GetActualLFSSizeLimit() > 0 +} + +// IsLFSSizeOversized returns true if adding additionalSize would exceed the LFS size limit +func (repo *Repository) IsLFSSizeOversized(additionalSize int64) bool { + return repo.ShouldCheckLFSSize() && + repo.LFSSize+additionalSize > repo.GetActualLFSSizeLimit() +} + +// IsRepoAndLFSSizeOversized checks if combined repo + LFS size exceeds repo size limit +// This is used when LFS_SIZE_IN_REPO_SIZE is enabled +func (repo *Repository) IsRepoAndLFSSizeOversized(additionalGitSize, additionalLFSSize int64) bool { + if !setting.LFSSizeInRepoSize || !setting.EnableSizeLimit { + return false + } + limit := repo.GetActualSizeLimit() + if limit == 0 { + return false + } + return (repo.GitSize+additionalGitSize)+(repo.LFSSize+additionalLFSSize) > limit +} + // CanCreateBranch returns true if repository meets the requirements for creating new branches. func (repo *Repository) CanCreateBranch() bool { return !repo.IsMirror diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 9374751300..6e99cb4090 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -276,13 +276,17 @@ var ( RepoRootPath string ScriptType = "bash" - EnableSizeLimit = true - RepoSizeLimit int64 + EnableSizeLimit = true + RepoSizeLimit int64 + LFSSizeLimit int64 + LFSSizeInRepoSize bool ) -func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) error { +func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit, lfsSizeLimit int64, lfsSizeInRepoSize bool) error { EnableSizeLimit = enableSizeLimit RepoSizeLimit = repoSizeLimit + LFSSizeLimit = lfsSizeLimit + LFSSizeInRepoSize = lfsSizeInRepoSize sec := CfgProvider.Section("repository") if EnableSizeLimit { sec.Key("ENABLE_SIZE_LIMIT").SetValue("true") @@ -291,6 +295,12 @@ func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) erro } sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit))) + sec.Key("LFS_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(LFSSizeLimit))) + if lfsSizeInRepoSize { + sec.Key("LFS_SIZE_IN_REPO_SIZE").SetValue("true") + } else { + sec.Key("LFS_SIZE_IN_REPO_SIZE").SetValue("false") + } return nil } @@ -303,6 +313,9 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { v, _ := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0")) RepoSizeLimit = int64(v) + v, _ = humanize.ParseBytes(sec.Key("LFS_SIZE_LIMIT").MustString("0")) + LFSSizeLimit = int64(v) + LFSSizeInRepoSize = sec.Key("LFS_SIZE_IN_REPO_SIZE").MustBool(false) Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool() Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 976c094a71..0dcc0096da 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -155,6 +155,8 @@ type CreateRepoOption struct { ObjectFormatName string `json:"object_format_name" binding:"MaxSize(6)"` // SizeLimit of the repository SizeLimit int64 `json:"size_limit"` + // LFSSizeLimit of the repository + LFSSizeLimit int64 `json:"lfs_size_limit"` } // EditRepoOption options when editing a repository's properties @@ -227,6 +229,8 @@ type EditRepoOption struct { Archived *bool `json:"archived,omitempty"` // SizeLimit of the repository. SizeLimit *int64 `json:"size_limit,omitempty"` + // LFSSizeLimit of the repository. + LFSSizeLimit *int64 `json:"lfs_size_limit,omitempty"` // set to a string like `8h30m0s` to set the mirror interval time MirrorInterval *string `json:"mirror_interval,omitempty"` // enable prune - remove obsolete remote-tracking references when mirroring diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cf1d3c3234..1a1cf1e92c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1053,6 +1053,9 @@ repo_name_profile_private_hint = .profile-private is a special repository that y repo_name_helper = Good repository names use short, memorable and unique keywords. A repository named ".profile" or ".profile-private" could be used to add a README.md for the user/organization profile. repo_size = Repository Size repo_size_limit = Repository Size Limit +lfs_size = LFS Size +lfs_size_limit = LFS Size Limit +lfs_size_limit_helper = Override global LFS size limit. Leave empty to use global setting. template = Template template_select = Select a template. template_helper = Make repository a template @@ -1187,6 +1190,9 @@ form.name_reserved = The repository name "%s" is reserved. form.name_pattern_not_allowed = The pattern "%s" is not allowed in a repository name. form.repo_size_limit_negative = Repository size limitation cannot be negative. form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation. +form.invalid_lfs_size_limit = Invalid LFS size limit format. +form.lfs_size_limit_negative = LFS size limitation cannot be negative. +form.lfs_size_limit_only_by_admins = Only administrators can change the LFS size limitation. need_auth = Authorization migrate_options = Migration Options @@ -3453,6 +3459,10 @@ config.repository_config = Repository Configuration config.enable_size_limit = Enable Size Limit config.repo_size_limit = Default Repository Size Limit config.repo_size_limit.desc = Maximum repository size in MiB. Leave empty for no limit. +config.lfs_size_limit = Default LFS Size Limit +config.lfs_size_limit.desc = Maximum LFS size in MiB. Leave empty for no limit. +config.lfs_size_in_repo_size = Count LFS in Repository Size +config.lfs_size_in_repo_size.desc = When enabled, LFS size is included in repository size calculations config.global_repo_size_limit_manage_panel = Manage Global Repository Size Limit config.update_settings = Update Settings config.invalid_repo_size = Invalid repository size %s diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index d926b67e8f..ada70dda4c 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -264,6 +264,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *user_model.User, opt api.Cre IsTemplate: opt.Template, ObjectFormatName: opt.ObjectFormatName, SizeLimit: opt.SizeLimit, + LFSSizeLimit: opt.LFSSizeLimit, }) if err != nil { if repo_model.IsErrRepoAlreadyExist(err) { @@ -753,6 +754,9 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err if opts.SizeLimit != nil { repo.SizeLimit = *opts.SizeLimit } + if opts.LFSSizeLimit != nil { + repo.LFSSizeLimit = *opts.LFSSizeLimit + } if err := repo_service.UpdateRepository(ctx, repo, visibilityChanged); err != nil { ctx.APIErrorInternal(err) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 3e67979412..7cc1c10f2d 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -4,11 +4,13 @@ package private import ( + "bytes" "errors" "fmt" "net/http" "os" "path/filepath" + "regexp" "strconv" "strings" "sync" @@ -28,6 +30,7 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/services/agit" @@ -120,9 +123,7 @@ func calculateSizeOfObject(ctx *gitea_context.PrivateContext, dir string, env [] return 0, err } - var errParse error - var objectSize int64 - objectSize, errParse = strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) + objectSize, errParse := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) if errParse != nil { log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) return 0, errParse @@ -136,24 +137,14 @@ func calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCo // Calculate size of objects that were added for objectID := range newCommitObjects { if _, exists := oldCommitObjects[objectID]; !exists { - // objectID is not referenced in the list of objects of old commit so it is a new object - // Calculate its size and add it to the addedSize addedSize += commitObjectsSizes[objectID] } - // We might check here if new object is not already in the rest of repo to be precise - // However our goal is to prevent growth of repository so on determination of addedSize - // We can skip this preciseness, addedSize will be more then real addedSize - // TODO - do not count size of object that is referenced in other part of repo but not referenced neither in old nor new commit - // git will not add the object twice } // Calculate size of objects that were removed for objectID := range oldCommitObjects { if _, exists := newCommitObjects[objectID]; !exists { - // objectID is not referenced in the list of new commit objects so it was possibly removed if _, exists := otherCommitObjects[objectID]; !exists { - // objectID is not referenced in rest of the objects of the repository so it was removed - // Calculate its size and add it to the removedSize removedSize += commitObjectsSizes[objectID] } } @@ -241,7 +232,6 @@ func loadObjectSizesFromPack(ctx *gitea_context.PrivateContext, dir string, env, } // Second field has object type - // If object type is not known filter it out and do not process objectType := fields[1] if objectType != "commit" && objectType != "tree" && objectType != "blob" && objectType != "tag" { continue @@ -292,8 +282,6 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, dir string, e i := 0 for _, objectID := range objectIDs { _, exists := objectsSizes[objectID] - - // If object doesn't yet have size in objectsSizes add it for further processing if !exists { reducedObjectIDs[i%numWorkers] = append(reducedObjectIDs[i%numWorkers], objectID) i++ @@ -321,7 +309,6 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, dir string, e ran = true }) if !ran { - // This is not the first error – count it as a subsequent error. atomic.AddInt64(&errCount, 1) } } @@ -333,36 +320,25 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, dir string, e }(&reducedObjectIDs[(w-1)%numWorkers]) } - // Wait for all workers to finish processing. wg.Wait() if errExec == nil { return nil } - - // If there were additional errors, wrap the first one with a summary. if n := atomic.LoadInt64(&errCount); n > 0 { return fmt.Errorf("%w (and %d subsequent similar errors)", errExec, n) } - - // Only one error occurred. return errExec } // loadObjectsSizesViaBatch uses hashes from objectIDs and uses pre-opened `git cat-file --batch-check` command to slice and return each object sizes // This function can't be used for new commit objects. -// It speeds up loading object sizes from existing git database of the repository avoiding -// multiple `git cat-files -s` func loadObjectsSizesViaBatch(ctx *gitea_context.PrivateContext, repoPath string, objectIDs []string, objectsSizes map[string]int64) error { var i int32 reducedObjectIDs := make([]string, 0, len(objectIDs)) - - // Loop over all objectIDs and find which ones are missing size information for _, objectID := range objectIDs { _, exists := objectsSizes[objectID] - - // If object doesn't yet have size in objectsSizes add it for further processing if !exists { reducedObjectIDs = append(reducedObjectIDs, objectID) } @@ -420,15 +396,177 @@ func parseSize(sizeStr string) (int64, error) { return size, nil } +/* +LFS pointer scanning (fast-ish, bounded) + +We look for pointer blobs (small, <= 4KiB) and parse: + + oid sha256:<64hex> + size + +This lets us compute: +- incomingNewToRepoLFS: pointers that are new vs old AND not referenced in "other" parts of repo +- removedLFSSize: pointers removed vs new AND not referenced in "other" +*/ +var ( + lfsPointerMarker = []byte("version https://git-lfs.github.com/spec/v1") + lfsOIDRe = regexp.MustCompile(`(?m)^oid sha256:([0-9a-f]{64})$`) + lfsSizeRe = regexp.MustCompile(`(?m)^size ([0-9]+)$`) +) + +func sumLFSSizes(m map[string]int64) int64 { + var s int64 + for _, v := range m { + s += v + } + return s +} + +// scanLFSPointersFromObjectIDs finds LFS pointer blobs among objectIDs and returns map[oid]size. +// It only reads small blobs via cat-file, so it stays bounded. +// scanLFSPointersFromObjectIDs finds LFS pointer blobs among objectIDs and returns map[oid]size. +// It only reads small blobs via cat-file, so it stays bounded. +func scanLFSPointersFromObjectIDs(ctx *gitea_context.PrivateContext, repoPath string, env, objectIDs []string, maxBlobSize int64) (map[string]int64, error) { + out := make(map[string]int64) + if len(objectIDs) == 0 { + return out, nil + } + + // 1) batch-check: filter small blobs only + var input bytes.Buffer + for _, oid := range objectIDs { + if oid == "" { + continue + } + input.WriteString(oid) + input.WriteByte('\n') + } + + // Feed stdin via WithStdin, RunStdBytes takes only context.Context in your version + checkCmd := gitcmd.NewCommand("cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)"). + WithDir(repoPath). + WithEnv(env). + WithStdin(bytes.NewReader(input.Bytes())) + + checkBytes, _, err := checkCmd.RunStdBytes(ctx) + if err != nil { + return out, err + } + + smallBlobs := make([]string, 0, 1024) + for line := range bytes.SplitSeq(checkBytes, []byte{'\n'}) { + // " blob " + fields := bytes.Fields(line) + if len(fields) != 3 { + continue + } + if !bytes.Equal(fields[1], []byte("blob")) { + continue + } + size, perr := strconv.ParseInt(string(fields[2]), 10, 64) + if perr != nil { + continue + } + if size <= maxBlobSize { + smallBlobs = append(smallBlobs, string(fields[0])) + } + } + + if len(smallBlobs) == 0 { + return out, nil + } + + // 2) batch: read contents of small blobs, parse LFS pointers + var input2 bytes.Buffer + for _, oid := range smallBlobs { + input2.WriteString(oid) + input2.WriteByte('\n') + } + + catCmd := gitcmd.NewCommand("cat-file", "--batch"). + WithDir(repoPath). + WithEnv(env). + WithStdin(bytes.NewReader(input2.Bytes())) + + catBytes, _, err := catCmd.RunStdBytes(ctx) + if err != nil { + return out, err + } + + data := catBytes + i := 0 + for i < len(data) { + j := bytes.IndexByte(data[i:], '\n') + if j < 0 { + break + } + j += i + header := data[i:j] + i = j + 1 + + hf := bytes.Fields(header) + if len(hf) < 3 { + break + } + blobSize, perr := strconv.ParseInt(string(hf[2]), 10, 64) + if perr != nil || blobSize < 0 { + break + } + if i+int(blobSize) > len(data) { + break + } + + content := data[i : i+int(blobSize)] + i += int(blobSize) + + if i < len(data) && data[i] == '\n' { + i++ + } + + if !bytes.Contains(content, lfsPointerMarker) { + continue + } + + oidm := lfsOIDRe.FindSubmatch(content) + if len(oidm) != 2 { + continue + } + sizem := lfsSizeRe.FindSubmatch(content) + if len(sizem) != 2 { + continue + } + + oid := string(oidm[1]) + sz, perr := strconv.ParseInt(string(sizem[1]), 10, 64) + if perr != nil || sz < 0 { + continue + } + + if prev, ok := out[oid]; !ok || sz > prev { + out[oid] = sz + } + } + + return out, nil +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + // HookPreReceive checks whether a individual commit is acceptable func HookPreReceive(ctx *gitea_context.PrivateContext) { startTime := time.Now() + const maxLFSPointerBlobSize = int64(4096) opts := web.GetForm(ctx).(*private.HookOptions) ourCtx := &preReceiveContext{ PrivateContext: ctx, - env: generateGitEnv(opts), // Generate git environment for checking commits + env: generateGitEnv(opts), opts: opts, } @@ -436,14 +574,23 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { var addedSize int64 var removedSize int64 + + // LFS sizes derived from pointers + var incomingNewToRepoLFS int64 // best proxy for “incoming LFS objects” + var removedLFSSize int64 + var addedLFSSize int64 // new-vs-old pointers (can include already-known-to-repo) + var isRepoOversized bool var pushSize *git.CountObject var repoSize *git.CountObject var err error var duration time.Duration - if repo.IsRepoSizeLimitEnabled() { - // Calculating total size of the repo using `git count-objects` + needGitDelta := repo.ShouldCheckRepoSize() + needLFSDelta := repo.ShouldCheckLFSSize() || setting.LFSSizeInRepoSize + + // Only do CountObjects (push/repo) when we're doing the repo-size limit at all + if needGitDelta { repoSize, err = git.CountObjects(ctx, repo.RepoPath()) if err != nil { log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) @@ -453,7 +600,6 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { return } - // Calculating total size of the push using `git count-objects` pushSize, err = git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) if err != nil { log.Error("Unable to get push size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) @@ -463,13 +609,11 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { return } - // Cache whether the repository would breach the size limit after the operation isRepoOversized = repo.IsRepoSizeOversized(pushSize.Size + pushSize.SizePack) log.Warn("Push counts %+v", pushSize) log.Warn("Repo counts %+v", repoSize) } - // Iterate across the provided old commit IDs for i := range opts.OldCommitIDs { oldCommitID := opts.OldCommitIDs[i] newCommitID := opts.NewCommitIDs[i] @@ -477,15 +621,28 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { log.Trace("Processing old commit: %s, new commit: %s, ref: %s", oldCommitID, newCommitID, refFullName) - // If operation is in potential breach of size limit prepare data for analysis - if isRepoOversized { + // Deep work is only needed if: + // - repo is oversized (git deep path), OR + // - we need LFS delta (LFS limit enabled OR combined mode enabled) + if isRepoOversized || needLFSDelta { var gitObjects string var errLoop error + var errLFS error - // Create cache of objects in old commit - // if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail + // Keep pointer maps so we can compute delta at the end + var oldLFSPtrs, otherLFSPtrs, newLFSPtrs map[string]int64 + + // Only allocate object-size cache if we'll actually do git delta calc + var commitObjectsSizes map[string]int64 + if isRepoOversized { + commitObjectsSizes = make(map[string]int64) + } + + // OLD commit objects if oldCommitID != "0000000000000000000000000000000000000000" { - gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects").AddDynamicArguments(oldCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) + gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects"). + AddDynamicArguments(oldCommitID). + WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) if err != nil { log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -495,14 +652,26 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { } } - commitObjectsSizes := make(map[string]int64) oldCommitObjects := convertObjectsToMap(gitObjects) objectIDs := convertObjectsToSlice(gitObjects) - // Create cache of objects that are in the repository but not part of old or new commit - // if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail + // LFS pointers for OLD (only if needed) + oldLFSPtrs = map[string]int64{} + if needLFSDelta { + oldLFSPtrs, errLFS = scanLFSPointersFromObjectIDs(ctx, repo.RepoPath(), ourCtx.env, objectIDs, maxLFSPointerBlobSize) + if errLFS != nil { + log.Error("Unable to scan old commit LFS pointers for %s in %-v: %v", oldCommitID, repo, errLFS) + oldLFSPtrs = map[string]int64{} + } else { + log.Trace("LFS(old): pointers=%d total=%s", len(oldLFSPtrs), base.FileSize(sumLFSSizes(oldLFSPtrs))) + } + } + + // OTHER objects (repo excluding old+new) if oldCommitID == "0000000000000000000000000000000000000000" { - gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all").AddDynamicArguments("^" + newCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) + gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all"). + AddDynamicArguments("^" + newCommitID). + WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) if err != nil { log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -511,7 +680,9 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { return } } else { - gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) + gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all"). + AddDynamicArguments("^"+oldCommitID, "^"+newCommitID). + WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) if err != nil { log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -523,28 +694,42 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { otherCommitObjects := convertObjectsToMap(gitObjects) objectIDs = append(objectIDs, convertObjectsToSlice(gitObjects)...) - // Unfortunately `git cat-file --check-batch` shows full object size - // so we would load compressed sizes from pack file via `git verify-pack -v` if there are pack files in repo - // The result would still miss items that are loose as individual objects (not part of pack files) - if repoSize.InPack > 0 { - errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), nil, objectIDs, commitObjectsSizes) - if errLoop != nil { - log.Error("Unable to get sizes of objects from the pack in %-v Error: %v", repo, errLoop) + + // LFS pointers for OTHER (only if needed) + otherLFSPtrs = map[string]int64{} + if needLFSDelta { + otherLFSPtrs, errLFS = scanLFSPointersFromObjectIDs(ctx, repo.RepoPath(), ourCtx.env, objectIDs, maxLFSPointerBlobSize) + if errLFS != nil { + log.Error("Unable to scan other-objects LFS pointers for repo %-v: %v", repo, errLFS) + otherLFSPtrs = map[string]int64{} + } else { + log.Trace("LFS(other): pointers=%d total=%s", len(otherLFSPtrs), base.FileSize(sumLFSSizes(otherLFSPtrs))) } } - // Load loose objects that are missing - errLoop = loadObjectsSizesViaBatch(ctx, repo.RepoPath(), objectIDs, commitObjectsSizes) - if errLoop != nil { - log.Error("Unable to get sizes of objects that are missing in both old %s and new commits %s in %-v Error: %v", oldCommitID, newCommitID, repo, errLoop) - ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to get sizes of objects missing in both old and new commit and those in old commit: %v", err), - }) - return + // Load sizes of OLD+OTHER objects (existing in DB): pack + batch (git deep only) + if isRepoOversized { + if repoSize != nil && repoSize.InPack > 0 { + errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), nil, objectIDs, commitObjectsSizes) + if errLoop != nil { + log.Error("Unable to get sizes of objects from the pack in %-v Error: %v", repo, errLoop) + } + } + + errLoop = loadObjectsSizesViaBatch(ctx, repo.RepoPath(), objectIDs, commitObjectsSizes) + if errLoop != nil { + log.Error("Unable to get sizes of objects that are missing in both old %s and new commits %s in %-v Error: %v", oldCommitID, newCommitID, repo, errLoop) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to get sizes of objects missing in both old and new commit and those in old commit: %v", errLoop), + }) + return + } } - // Create cache of objects in new commit - gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects").AddDynamicArguments(newCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) + // NEW commit objects + gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects"). + AddDynamicArguments(newCommitID). + WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx) if err != nil { log.Error("Unable to list objects in new commit %s in %-v Error: %v", newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ @@ -555,25 +740,67 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { newCommitObjects := convertObjectsToMap(gitObjects) objectIDs = convertObjectsToSlice(gitObjects) - // Unfortunately `git cat-file --check-batch` doesn't work on objects not yet accepted into git database - // so the sizes will be calculated through pack file `git verify-pack -v` if there are pack files - // The result would still miss items that were sent loose as individual objects (not part of pack files) - if pushSize.InPack > 0 { - errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes) - if errLoop != nil { - log.Error("Unable to get sizes of objects from the pack in new commit %s in %-v Error: %v", newCommitID, repo, errLoop) + + // LFS pointers for NEW (only if needed) + newLFSPtrs = map[string]int64{} + if needLFSDelta { + newLFSPtrs, errLFS = scanLFSPointersFromObjectIDs(ctx, repo.RepoPath(), ourCtx.env, objectIDs, maxLFSPointerBlobSize) + if errLFS != nil { + log.Error("Unable to scan new commit LFS pointers for %s in %-v: %v", newCommitID, repo, errLFS) + newLFSPtrs = map[string]int64{} + } else { + log.Trace("LFS(new): pointers=%d total=%s", len(newLFSPtrs), base.FileSize(sumLFSSizes(newLFSPtrs))) } } - // After loading everything we could from pack file, objects could have been sent as loose bunch as well - // We need to load them individually with `git cat-file -s` on any object that is missing from accumulated size cache commitObjectsSizes - errLoop = loadObjectsSizesViaCatFile(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes) - if errLoop != nil { - log.Error("Unable to get sizes of objects in new commit %s in %-v Error: %v", newCommitID, repo, errLoop) + // Load sizes of NEW objects (may be in quarantine packs, etc.) (git deep only) + if isRepoOversized { + if pushSize != nil && pushSize.InPack > 0 { + errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes) + if errLoop != nil { + log.Error("Unable to get sizes of objects from the pack in new commit %s in %-v Error: %v", newCommitID, repo, errLoop) + } + } + + errLoop = loadObjectsSizesViaCatFile(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes) + if errLoop != nil { + log.Error("Unable to get sizes of objects in new commit %s in %-v Error: %v", newCommitID, repo, errLoop) + } + + // Git object delta (git deep only) + addedSize, removedSize = calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects, commitObjectsSizes) } - // Calculate size that was added and removed by the new commit - addedSize, removedSize = calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects, commitObjectsSizes) + // LFS delta based on pointer presence (LFS deep only) + if needLFSDelta { + for oid, sz := range newLFSPtrs { + if _, inOld := oldLFSPtrs[oid]; !inOld { + addedLFSSize += sz + if _, inOther := otherLFSPtrs[oid]; !inOther { + incomingNewToRepoLFS += sz + } + } + } + + for oid, sz := range oldLFSPtrs { + if _, inNew := newLFSPtrs[oid]; inNew { + continue + } + if _, inOther := otherLFSPtrs[oid]; inOther { + continue + } + removedLFSSize += sz + } + + log.Trace( + "LFS(delta): incoming-new-to-repo=%s added(vs old)=%s removed=%s current(repo.LFSSize)=%s predicted=%s", + base.FileSize(incomingNewToRepoLFS), + base.FileSize(addedLFSSize), + base.FileSize(removedLFSSize), + base.FileSize(repo.LFSSize), + base.FileSize(repo.LFSSize+incomingNewToRepoLFS-removedLFSSize), + ) + } } switch { @@ -591,18 +818,101 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { } } - if repo.IsRepoSizeLimitEnabled() { - duration = time.Since(startTime) - log.Warn("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %d, push size: %d, repo size: %d. Took %s seconds.", addedSize, removedSize, repo.GetActualSizeLimit(), pushSize.Size+pushSize.SizePack, repo.GitSize, duration) + // --------- Final accounting + enforcement (one timing) --------- + duration = time.Since(startTime) + + currentGit := repo.GitSize + currentLFS := repo.LFSSize + currentCombined := currentGit + currentLFS + + gitDelta := addedSize - removedSize + predictedGitAfter := currentGit + gitDelta + + lfsDelta := incomingNewToRepoLFS - removedLFSSize + predictedLFSAfter := currentLFS + lfsDelta + + predictedCombinedAfter := predictedGitAfter + if setting.LFSSizeInRepoSize { + predictedCombinedAfter = predictedGitAfter + predictedLFSAfter } - // If total of commits add more size then they remove and we are in a potential breach of size limit -- abort - if (addedSize > removedSize) && isRepoOversized { - log.Warn("Forbidden: new repo size %s would be over limitation of %s. Push size: %s. Took %s seconds. addedSize: %s. removedSize: %s", base.FileSize(repo.GitSize+addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size+pushSize.SizePack), duration, base.FileSize(addedSize), base.FileSize(removedSize)) - ctx.JSON(http.StatusForbidden, private.Response{ - UserMsg: "New repository size is over limitation of " + base.FileSize(repo.GetActualSizeLimit()), - }) - return + // Avoid nil panic when repo-size check is disabled but LFS delta is enabled (combined mode / LFS limit) + pushBytes := int64(0) + if pushSize != nil { + pushBytes = maxInt64(0, pushSize.Size+pushSize.SizePack) + } + + // One summary line (time included here only) + if repo.ShouldCheckRepoSize() || repo.ShouldCheckLFSSize() { + log.Warn( + "SizeCheck summary: took=%s repo=%s/%s git(pred=%s cur=%s delta=%s) lfs(pred=%s cur=%s delta=%s) combined(pred=%s) limits(repo=%s lfs=%s) LFSSizeInRepoSize=%v push=%s", + duration, + repo.OwnerName, repo.Name, + base.FileSize(predictedGitAfter), base.FileSize(currentGit), base.FileSize(gitDelta), + base.FileSize(predictedLFSAfter), base.FileSize(currentLFS), base.FileSize(lfsDelta), + base.FileSize(predictedCombinedAfter), + base.FileSize(repo.GetActualSizeLimit()), + base.FileSize(repo.GetActualLFSSizeLimit()), + setting.LFSSizeInRepoSize, + base.FileSize(pushBytes), + ) + } + + // 1) LFS-only limit: compare against predicted LFS after push + if repo.ShouldCheckLFSSize() { + lfsLimit := repo.GetActualLFSSizeLimit() + if lfsLimit > 0 && predictedLFSAfter > lfsLimit && predictedLFSAfter > currentLFS { + log.Warn("Forbidden: LFS limit exceeded: %s > %s for repo %-v", + base.FileSize(predictedLFSAfter), + base.FileSize(lfsLimit), + repo, + ) + ctx.JSON(http.StatusForbidden, private.Response{ + UserMsg: fmt.Sprintf("LFS size limit exceeded: %s > then limit of %s", + base.FileSize(predictedLFSAfter), + base.FileSize(lfsLimit), + ), + }) + return + } + } + + // 2) Repo (git) size limit when NOT counting LFS into repo size + if repo.ShouldCheckRepoSize() && !setting.LFSSizeInRepoSize { + limit := repo.GetActualSizeLimit() + if limit > 0 && predictedGitAfter > limit { + log.Warn("Forbidden: repository size limit exceeded: %s > %s for repo %-v", + base.FileSize(predictedGitAfter), + base.FileSize(limit), + repo, + ) + ctx.JSON(http.StatusForbidden, private.Response{ + UserMsg: fmt.Sprintf("Repository size limit exceeded: %s > then limit of %s", + base.FileSize(predictedGitAfter), + base.FileSize(limit), + ), + }) + return + } + } + + // 3) Combined limit when LFS is counted in repo size + if setting.LFSSizeInRepoSize && repo.ShouldCheckRepoSize() { + limit := repo.GetActualSizeLimit() + if limit > 0 && predictedCombinedAfter > limit && predictedCombinedAfter > currentCombined { + log.Warn("Forbidden: combined repo and LFS size limit exceeded: %s > %s for repo %-v", + base.FileSize(predictedCombinedAfter), + base.FileSize(limit), + repo, + ) + ctx.JSON(http.StatusForbidden, private.Response{ + UserMsg: fmt.Sprintf("Combined repository+LFS size limit exceeded: %s > then limit of %s", + base.FileSize(predictedCombinedAfter), + base.FileSize(limit), + ), + }) + return + } } ctx.PlainText(http.StatusOK, "ok") @@ -637,17 +947,11 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r return } - // Allow pushes to non-protected branches if protectBranch == nil { return } protectBranch.Repo = repo - // This ref is a protected branch. - // - // First of all we need to enforce absolutely: - // - // 1. Detect and prevent deletion of the branch if newCommitID == objectFormat.EmptyObjectID().String() { log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo) ctx.JSON(http.StatusForbidden, private.Response{ @@ -658,7 +962,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r isForcePush := false - // 2. Disallow force pushes to protected branches if oldCommitID != objectFormat.EmptyObjectID().String() { output, err := gitrepo.RunCmdString(ctx, repo, @@ -685,7 +988,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r } } - // 3. Enforce require signed commits if protectBranch.RequireSignedCommits { err := verifyCommits(oldCommitID, newCommitID, gitRepo, ctx.env) if err != nil { @@ -705,9 +1007,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r } } - // Now there are several tests which can be overridden: - // - // 4. Check protected file patterns - this is overridable from the UI changedProtectedfiles := false protectedFilePath := "" @@ -728,10 +1027,8 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r } } - // 5. Check if the doer is allowed to push (and force-push if the incoming push is a force-push) var canPush bool if ctx.opts.DeployKeyID != 0 { - // This flag is only ever true if protectBranch.CanForcePush is true if isForcePush { canPush = !changedProtectedfiles && protectBranch.CanPush && (!protectBranch.EnableForcePushAllowlist || protectBranch.ForcePushAllowlistDeployKeys) } else { @@ -753,13 +1050,8 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r } } - // 6. If we're not allowed to push directly if !canPush { - // Is this is a merge from the UI/API? if ctx.opts.PullRequestID == 0 { - // 6a. If we're not merging from the UI/API then there are two ways we got here: - // - // We are changing a protected file and we're not allowed to do that if changedProtectedfiles { log.Warn("Forbidden: Branch: %s in %-v is protected from changing file %s", branchName, repo, protectedFilePath) ctx.JSON(http.StatusForbidden, private.Response{ @@ -768,7 +1060,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r return } - // Allow commits that only touch unprotected files globs := protectBranch.GetUnprotectedFilePatterns() if len(globs) > 0 { unprotectedFilesOnly, err := pull_service.CheckUnprotectedFiles(gitRepo, branchName, oldCommitID, newCommitID, globs, ctx.env) @@ -780,12 +1071,10 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r return } if unprotectedFilesOnly { - // Commit only touches unprotected files, this is allowed return } } - // Or we're simply not able to push to this protected branch if isForcePush { log.Warn("Forbidden: User %d is not allowed to force-push to protected branch: %s in %-v", ctx.opts.UserID, branchName, repo) ctx.JSON(http.StatusForbidden, private.Response{ @@ -799,9 +1088,7 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r }) return } - // 6b. Merge (from UI or API) - // Get the PR, user and permissions for the user in the repository pr, err := issues_model.GetPullRequestByID(ctx, ctx.opts.PullRequestID) if err != nil { log.Error("Unable to get PullRequest %d Error: %v", ctx.opts.PullRequestID, err) @@ -811,14 +1098,10 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r return } - // although we should have called `loadPusherAndPermission` before, here we call it explicitly again because we need to access ctx.user below if !ctx.loadPusherAndPermission() { - // if error occurs, loadPusherAndPermission had written the error response return } - // Now check if the user is allowed to merge PRs for this repository - // Note: we can use ctx.perm and ctx.user directly as they will have been loaded above allowedMerge, err := pull_service.IsUserAllowedToMerge(ctx, pr, ctx.userPerm, ctx.user) if err != nil { log.Error("Error calculating if allowed to merge: %v", err) @@ -836,12 +1119,10 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r return } - // If we're an admin for the repository we can ignore status checks, reviews and override protected files if ctx.userPerm.IsAdmin() { return } - // Now if we're not an admin - we can't overwrite protected files so fail now if changedProtectedfiles { log.Warn("Forbidden: Branch: %s in %-v is protected from changing file %s", branchName, repo, protectedFilePath) ctx.JSON(http.StatusForbidden, private.Response{ @@ -850,7 +1131,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r return } - // Check all status checks and reviews are ok if err := pull_service.CheckPullBranchProtections(ctx, pr, true); err != nil { if errors.Is(err, pull_service.ErrNotReadyToMerge) { log.Warn("Forbidden: User %d is not allowed push to protected branch %s in %-v and pr #%d is not ready to be merged: %s", ctx.opts.UserID, branchName, repo, pr.Index, err.Error()) @@ -940,16 +1220,13 @@ func preReceiveFor(ctx *preReceiveContext, refFullName git.RefName) { func generateGitEnv(opts *private.HookOptions) (env []string) { env = os.Environ() if opts.GitAlternativeObjectDirectories != "" { - env = append(env, - private.GitAlternativeObjectDirectories+"="+opts.GitAlternativeObjectDirectories) + env = append(env, private.GitAlternativeObjectDirectories+"="+opts.GitAlternativeObjectDirectories) } if opts.GitObjectDirectory != "" { - env = append(env, - private.GitObjectDirectory+"="+opts.GitObjectDirectory) + env = append(env, private.GitObjectDirectory+"="+opts.GitObjectDirectory) } if opts.GitQuarantinePath != "" { - env = append(env, - private.GitQuarantinePath+"="+opts.GitQuarantinePath) + env = append(env, private.GitQuarantinePath+"="+opts.GitQuarantinePath) } return env } diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 5d367fd0fb..4fce654b96 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -35,6 +35,8 @@ func Repos(ctx *context.Context) { ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit ctx.Data["RepoSizeLimit"] = base.FileSize(setting.RepoSizeLimit) + ctx.Data["LFSSizeLimit"] = base.FileSize(setting.LFSSizeLimit) + ctx.Data["LFSSizeInRepoSize"] = setting.LFSSizeInRepoSize explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ Private: true, @@ -64,6 +66,7 @@ func UpdateRepoPost(ctx *context.Context) { ctx.Data["EnableSizeLimit"] = form.EnableSizeLimit ctx.Data["RepoSizeLimit"] = form.RepoSizeLimit + ctx.Data["LFSSizeInRepoSize"] = form.LFSSizeInRepoSize if err != nil { ctx.Data["Err_Repo_Size_Limit"] = err.Error() @@ -76,7 +79,21 @@ func UpdateRepoPost(ctx *context.Context) { return } - err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit) + lfsSizeLimit, err := base.GetFileSize(form.LFSSizeLimit) + ctx.Data["LFSSizeLimit"] = form.LFSSizeLimit + + if err != nil { + ctx.Data["Err_LFS_Size_Limit"] = err.Error() + explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ + Private: true, + PageSize: setting.UI.Admin.RepoPagingNum, + TplName: tplRepos, + OnlyShowRelevant: false, + }) + return + } + + err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit, lfsSizeLimit, form.LFSSizeInRepoSize) if err != nil { ctx.Data["Err_Repo_Size_Save"] = err.Error() explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index b023223d4b..eca3991754 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -63,7 +63,9 @@ func SettingsCtxData(ctx *context.Context) { ctx.Data["CanConvertFork"] = ctx.Repo.Repository.IsFork && ctx.Doer.CanCreateRepoIn(ctx.Repo.Repository.Owner) ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.IsRepoSizeOversized(ctx.Repo.Repository.GetActualSizeLimit() / 10) // less than 10% left ctx.Data["ActualSizeLimit"] = ctx.Repo.Repository.GetActualSizeLimit() + ctx.Data["ActualLFSSizeLimit"] = ctx.Repo.Repository.GetActualLFSSizeLimit() ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit + ctx.Data["LFSSizeInRepoSize"] = setting.LFSSizeInRepoSize signing, _ := gitrepo.GetSigningKey(ctx, ctx.Repo.Repository) ctx.Data["SigningKeyAvailable"] = signing != nil @@ -237,6 +239,28 @@ func handleSettingsPostUpdate(ctx *context.Context) { } repo.SizeLimit = repoSizeLimit + // Handle LFS size limit (admin-only) + var lfsSizeLimit int64 + if form.LFSSizeLimit != "" { + lfsSizeLimit, err = base.GetFileSize(form.LFSSizeLimit) + if err != nil { + ctx.Data["Err_LFSSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.invalid_lfs_size_limit"), tplSettingsOptions, &form) + return + } + } + if lfsSizeLimit < 0 { + ctx.Data["Err_LFSSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.lfs_size_limit_negative"), tplSettingsOptions, &form) + return + } + if !ctx.Doer.IsAdmin && repo.LFSSizeLimit != lfsSizeLimit { + ctx.Data["Err_LFSSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.lfs_size_limit_only_by_admins"), tplSettingsOptions, &form) + return + } + repo.LFSSizeLimit = lfsSizeLimit + if err := repo_service.UpdateRepository(ctx, repo, false); err != nil { ctx.ServerError("UpdateRepository", err) return diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index bc399f4576..4b5bebe7e0 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -20,8 +20,10 @@ import ( // UpdateGlobalRepoFrom for updating global repository setting type UpdateGlobalRepoFrom struct { - RepoSizeLimit string - EnableSizeLimit bool + RepoSizeLimit string + LFSSizeLimit string + EnableSizeLimit bool + LFSSizeInRepoSize bool } // CreateRepoForm form for creating repository @@ -114,6 +116,7 @@ type RepoSettingForm struct { Template bool EnablePrune bool RepoSizeLimit string + LFSSizeLimit string // Advanced settings EnableCode bool diff --git a/services/lfs/server.go b/services/lfs/server.go index 81991de434..6f27e45214 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -5,6 +5,7 @@ package lfs import ( stdCtx "context" + "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -28,6 +29,7 @@ import ( "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/auth/httpauth" + "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/json" lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" @@ -184,6 +186,25 @@ func DownloadHandler(ctx *context.Context) { } } +// ---- tracing helpers ---- + +func traceBatchDecision(rc *requestContext, op, msg string, args ...any) { + prefix := fmt.Sprintf("LFS[BATCH][%s/%s][op=%s] ", rc.User, rc.Repo, op) + log.Trace(prefix+msg, args...) +} + +func batchReqID(br *lfs_module.BatchRequest) string { + h := sha1.New() + h.Write([]byte(br.Operation)) + for _, o := range br.Objects { + h.Write([]byte(o.Oid)) + h.Write([]byte{0}) + h.Write([]byte(strconv.FormatInt(o.Size, 10))) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil))[:10] +} + // BatchHandler provides the batch api func BatchHandler(ctx *context.Context) { var br lfs_module.BatchRequest @@ -206,6 +227,9 @@ func BatchHandler(ctx *context.Context) { } rc := getRequestContext(ctx) + reqID := batchReqID(&br) + + log.Trace("LFS[BATCH][%s/%s] req=%s op=%s objects=%d", rc.User, rc.Repo, reqID, br.Operation, len(br.Objects)) repository := getAuthenticatedRepository(ctx, rc, isUpload) if repository == nil { @@ -217,8 +241,98 @@ func BatchHandler(ctx *context.Context) { return } + // Create content store once, reuse for tracing + normal logic below. contentStore := lfs_module.NewContentStore() + currCombinedTotal := repository.GitSize + repository.LFSSize + + // Baseline repo stats and limits + traceBatchDecision(rc, br.Operation, + "req=%s auth=%t isUpload=%t repoID=%d sizes: git=%s lfs=%s combined=%s limits: repo=%s lfs=%s LFSSizeInRepoSize=%v", + reqID, + ctx.IsSigned || ctx.Doer != nil, + isUpload, + repository.ID, + base.FileSize(repository.GitSize), + base.FileSize(repository.LFSSize), + base.FileSize(currCombinedTotal), + base.FileSize(repository.GetActualSizeLimit()), + base.FileSize(repository.GetActualLFSSizeLimit()), + setting.LFSSizeInRepoSize, + ) + + // Check LFS size limits for upload operations + if isUpload && (repository.ShouldCheckLFSSize() || (setting.LFSSizeInRepoSize && repository.ShouldCheckRepoSize())) { + // Sum sizes of objects that are NEW TO THIS REPO (no meta row) + var incomingNewToRepoLFS int64 + var invalidCount, metaMissingCount, metaPresentCount, storeExistsCount int64 + + for _, p := range br.Objects { + if !p.IsValid() { + invalidCount++ + continue + } + + meta, _ := git_model.GetLFSMetaObjectByOid(ctx, repository.ID, p.Oid) + + exists, _ := contentStore.Exists(p) + if exists { + storeExistsCount++ + } + + if meta == nil { + metaMissingCount++ + incomingNewToRepoLFS += p.Size + } else { + metaPresentCount++ + } + } + + predictedLFS := repository.LFSSize + incomingNewToRepoLFS + predictedTotal := repository.GitSize + predictedLFS + + traceBatchDecision(rc, br.Operation, + "req=%s accounting: objects=%d invalid=%d metaMissing=%d metaPresent=%d storeExists=%d incomingNewToRepoLFS=%s predictedLFS=%s predictedTotal=%s", + reqID, + len(br.Objects), + invalidCount, + metaMissingCount, + metaPresentCount, + storeExistsCount, + base.FileSize(incomingNewToRepoLFS), + base.FileSize(predictedLFS), + base.FileSize(predictedTotal), + ) + + // LFS-only limit if we are over, but size doesn't increase allow + if repository.ShouldCheckLFSSize() && predictedLFS > repository.GetActualLFSSizeLimit() && predictedLFS > repository.LFSSize { + traceBatchDecision(rc, br.Operation, + "req=%s DECISION=FORBID reason=LFS_LIMIT predictedLFS=%s limit=%s", + reqID, base.FileSize(predictedLFS), base.FileSize(repository.GetActualLFSSizeLimit()), + ) + writeStatusMessage(ctx, http.StatusForbidden, + fmt.Sprintf("LFS size %s would exceed limit %s", + base.FileSize(predictedLFS), base.FileSize(repository.GetActualLFSSizeLimit()))) + return + } + + // Combined limit (conservative: ignores git delta/removals) if we are over, but combined size doesn't increase allow + if setting.LFSSizeInRepoSize && repository.ShouldCheckRepoSize() { + if predictedTotal > repository.GetActualSizeLimit() && predictedTotal > currCombinedTotal { + traceBatchDecision(rc, br.Operation, + "req=%s DECISION=FORBID reason=COMBINED_LIMIT predictedTotal=%s limit=%s", + reqID, base.FileSize(predictedTotal), base.FileSize(repository.GetActualSizeLimit()), + ) + writeStatusMessage(ctx, http.StatusForbidden, + fmt.Sprintf("Repository size %s after LFS addition would exceed limit %s", + base.FileSize(predictedTotal), base.FileSize(repository.GetActualSizeLimit()))) + return + } + } + + traceBatchDecision(rc, br.Operation, "req=%s DECISION=ALLOW size-check passed", reqID) + } + var responseObjects []*lfs_module.ObjectResponse for _, p := range br.Objects { diff --git a/services/repository/create.go b/services/repository/create.go index 4b6fe70ae4..8eb2082738 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -53,6 +53,7 @@ type CreateRepoOptions struct { MirrorInterval string ObjectFormatName string SizeLimit int64 + LFSSizeLimit int64 } func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir string, opts CreateRepoOptions) error { @@ -249,6 +250,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, SizeLimit: opts.SizeLimit, + LFSSizeLimit: opts.LFSSizeLimit, IsMirror: opts.IsMirror, DefaultBranch: opts.DefaultBranch, DefaultWikiBranch: setting.Repository.DefaultBranch, diff --git a/templates/admin/repo/list.tmpl b/templates/admin/repo/list.tmpl index e4605f4f0f..adebc70521 100644 --- a/templates/admin/repo/list.tmpl +++ b/templates/admin/repo/list.tmpl @@ -12,10 +12,20 @@ +
+ +
+ +
+
+
+ + +
diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index ee61e902b8..ca498380d3 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -20,12 +20,27 @@ {{end}} +
+ + {{FileSize .Repository.LFSSize}} + {{if and .ActualLFSSizeLimit .EnableSizeLimit}} + / {{FileSize .ActualLFSSizeLimit}} + {{end}} + +
+
+ + +
diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index 29239c3cb8..fa315aa259 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -483,3 +483,18 @@ func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) f ctx.Session.MakeRequest(t, req, 200) } } + +func doAPISetRepoLFSSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) { + return func(t *testing.T) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s", + owner, repo) + req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{LFSSizeLimit: &size}). + AddTokenAuth(ctx.Token) + + if ctx.ExpectedCode != 0 { + ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) + return + } + ctx.Session.MakeRequest(t, req, 200) + } +} diff --git a/tests/integration/git_general_test.go b/tests/integration/git_general_test.go index ba8e26a340..025f493c9c 100644 --- a/tests/integration/git_general_test.go +++ b/tests/integration/git_general_test.go @@ -87,7 +87,7 @@ func testGitGeneral(t *testing.T, u *url.URL) { t.Run("SizeLimit", func(t *testing.T) { dstForkedPath := t.TempDir() - setting.SaveGlobalRepositorySetting(true, 0) + setting.SaveGlobalRepositorySetting(true, 0, 0, false) t.Run("Under", func(t *testing.T) { defer tests.PrintCurrentTest(t)() doCommitAndPush(t, testFileSizeSmall, dstPath, "data-file-") @@ -115,7 +115,7 @@ func testGitGeneral(t *testing.T, u *url.URL) { doRebaseCommitAndPush(t, dstPath, lastCommitID) newRepoSize := doGetRemoteRepoSizeViaAPI(t, forkedUserCtx) assert.LessOrEqual(t, newRepoSize, oldRepoSize) - setting.SaveGlobalRepositorySetting(false, 0) + setting.SaveGlobalRepositorySetting(false, 0, 0, false) }) }) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "test/head")) diff --git a/tests/integration/lfs_size_limit_test.go b/tests/integration/lfs_size_limit_test.go new file mode 100644 index 0000000000..3ce68d7967 --- /dev/null +++ b/tests/integration/lfs_size_limit_test.go @@ -0,0 +1,413 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/gitcmd" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +func base62(n int64) string { + if n == 0 { + return string(alphabet[0]) + } + var buf [11]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = alphabet[n%62] + n /= 62 + } + return string(buf[i:]) +} + +// ID5: 5-char ID derived from unix seconds. +func ID5() string { + sec := time.Now().Unix() + s := base62(sec) + if len(s) < 5 { + return fmt.Sprintf("%05s", s) + } + return s[len(s)-5:] +} + +// TestLFSSizeLimit exercises the pre-receive size limit logic. +// Runs sequentially (no t.Parallel) because it mutates GLOBAL settings. +// +// Enable/disable via: +// +// setting.SaveGlobalRepositorySetting(enabled, repoLimit, lfsLimit, lfsSizeInRepoSize) +// +// Scenarios: +// - repo-only limit: blocks git blobs but not LFS objects (when lfsSizeInRepoSize=false, lfsLimit=0) +// - LFS-only limit: blocks LFS objects but not git blobs +// - combined repo+LFS (LFS counted into repo size): blocks LFS when it would exceed repo limit +// - per-repo LFS override wins over global default (retry same push after increasing per-repo LFS limit) +func TestLFSSizeLimit(t *testing.T) { + onGiteaRun(t, testLFSSizeLimit) +} + +func testLFSSizeLimit(t *testing.T, baseURL *url.URL) { + // Always disable at the end so we don't leak to other integration tests. + t.Cleanup(func() { + setting.SaveGlobalRepositorySetting(false, 0, 0, false) + }) + + if !setting.LFS.StartServer { + t.Skip("LFS server disabled") + } + // ---- sizes ---- + // Repo-size checks are not precise (compression), so use very different sizes vs limit. + const ( + repoLimitBytes = int64(64 * 1024) // 64KiB + gitUnderBytes = int(8 * 1024) // 8KiB + gitOverBytes = int(4 * 1024 * 1024) // 4MiB (far above limit) + lfsBigBytes = int(1 * 1024 * 1024) // 1MiB (should still pass repo-only when not counted) + ) + + // LFS-only checks are precise. + const ( + lfsLimitBytes = int64(64 * 1024) // 64KiB + lfsUnderBytes = int(32 * 1024) // 32KiB + lfsOverBytes = int(128 * 1024) // 128KiB + ) + + // Combined repo+LFS (LFS counted into repo size). + const ( + combinedRepoLimitBytes = int64(128 * 1024) // 128KiB + combinedLFSUnderBytes = int(64 * 1024) // 64KiB + combinedLFSOverBytes = int(512 * 1024) // 512KiB + ) + + // ---- helpers ---- + + newLimitRepo := func(t *testing.T, suffix string) APITestContext { + t.Helper() + + // Make a unique repo name without relying on tests.GetTestUID / CreateRepo. + repoName := fmt.Sprintf( + "lfsl-%s-%s", + suffix, + ID5(), + ) + + ctx := NewAPITestContext( + t, + "user2", + repoName, + auth_model.AccessTokenScopeWriteRepository, + auth_model.AccessTokenScopeWriteUser, + ) + + // Create via API (same style as git_general_test.go) + doAPICreateRepository(ctx, false)(t) + return ctx + } + + cloneHTTP := func(t *testing.T, ctx APITestContext) string { + t.Helper() + + u := *baseURL // copy + u.Path = ctx.GitPath() + u.User = url.UserPassword(ctx.Username, userPassword) + + dst := t.TempDir() + doGitClone(dst, &u)(t) + return dst + } + + commitSignature := func() *git.Signature { + return &git.Signature{ + Email: "user2@example.com", + Name: "User Two", + When: time.Now(), + } + } + + configureLFSForPrefix := func(t *testing.T, repoPath, prefix string) { + t.Helper() + + // git lfs install + err := gitcmd.NewCommand("lfs").AddArguments("install").WithDir(repoPath).Run(t.Context()) + require.NoError(t, err) + + // track prefix* + _, _, err = gitcmd.NewCommand("lfs"). + AddArguments("track"). + AddDynamicArguments(prefix + "*"). + WithDir(repoPath). + RunStdString(t.Context()) + require.NoError(t, err) + + // commit .gitattributes + err = git.AddChanges(t.Context(), repoPath, false, ".gitattributes") + require.NoError(t, err) + + sig := commitSignature() + err = git.CommitChanges(t.Context(), repoPath, git.CommitChangesOptions{ + Committer: sig, + Author: sig, + Message: "configure LFS tracking", + }) + require.NoError(t, err) + } + + pushCurrentBranch := func(t *testing.T, repoPath string) error { + t.Helper() + _, _, err := gitcmd.NewCommand("push", "origin", "master").WithDir(repoPath).RunStdString(t.Context()) + return err + } + + // Push a git blob by creating a new random file (random-ish content reduces compression effects). + pushGitBlob := func(t *testing.T, ctx APITestContext, size int) error { + t.Helper() + + repoPath := cloneHTTP(t, ctx) + + _, err := generateCommitWithNewData( + t.Context(), + size, + repoPath, + "user2@example.com", + "User Two", + "git-data-", + ) + require.NoError(t, err) + + return pushCurrentBranch(t, repoPath) + } + + // Push an LFS object by tracking prefix* and then committing a file created by generateCommitWithNewData. + pushLFSObjectOnce := func(t *testing.T, ctx APITestContext, size int) error { + t.Helper() + + repoPath := cloneHTTP(t, ctx) + + const prefix = "lfs-data-" + configureLFSForPrefix(t, repoPath, prefix) + + _, err := generateCommitWithNewData( + t.Context(), + size, + repoPath, + "user2@example.com", + "User Two", + prefix, + ) + require.NoError(t, err) + + return pushCurrentBranch(t, repoPath) + } + + // Prepare a single LFS commit in a clone and return the clone path, so we can: + // - push (expect fail) + // - change limits + // - push SAME COMMIT again (expect pass) + prepareLFSPushForRetry := func(t *testing.T, ctx APITestContext, size int) string { + t.Helper() + + repoPath := cloneHTTP(t, ctx) + + const prefix = "lfs-retry-" + configureLFSForPrefix(t, repoPath, prefix) + + _, err := generateCommitWithNewData( + t.Context(), + size, + repoPath, + "user2@example.com", + "User Two", + prefix, + ) + require.NoError(t, err) + + // Ensure the repoPath is used (avoid accidental cleanup / unused). + _, err = os.Stat(filepath.Join(repoPath, ".git")) + require.NoError(t, err) + + return repoPath + } + + runGlobalThenPerRepo := func( + t *testing.T, + name string, + global func(t *testing.T), + perRepo func(t *testing.T), + ) { + t.Helper() + t.Run(name+"/Global", func(t *testing.T) { global(t) }) + t.Run(name+"/PerRepo", func(t *testing.T) { perRepo(t) }) + } + + // ---- tests ---- + + runGlobalThenPerRepo(t, + "GlobalAndRepoOnlyLimit_BlocksGitButNotLFS", + func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Global: repo limit ON, LFS limit OFF, LFS not counted into repo size. + setting.SaveGlobalRepositorySetting(true, repoLimitBytes, 0, false) + + ctx := newLimitRepo(t, "ggit") + + // push over on git (NOK) + err := pushGitBlob(t, ctx, gitOverBytes) + assert.Error(t, err, "git file well over repo limit should be rejected") + + // push LFS (OK) + err = pushLFSObjectOnce(t, ctx, lfsBigBytes) + assert.NoError(t, err, "LFS push should be allowed when repo-only limit is enabled and LFSSizeInRepoSize=false") + + // push under on git (OK) + err = pushGitBlob(t, ctx, gitUnderBytes) + assert.NoError(t, err, "small git file should be accepted if under repo limit") + }, + func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Per-repo: enable checking globally but do not set global limits. + setting.SaveGlobalRepositorySetting(true, 0, 0, false) + + ctx := newLimitRepo(t, "rgit") + + // set per-repo repo limit + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(ctx, ctx.Username, ctx.Reponame, repoLimitBytes)) + + // push over on git (NOK) + err := pushGitBlob(t, ctx, gitOverBytes) + assert.Error(t, err, "git file well over per-repo repo limit should be rejected") + + // push LFS (OK) + err = pushLFSObjectOnce(t, ctx, lfsBigBytes) + assert.NoError(t, err, "LFS push should be allowed when only per-repo repo limit is set and LFSSizeInRepoSize=false") + + // push under on git (OK) + err = pushGitBlob(t, ctx, gitUnderBytes) + assert.NoError(t, err, "small git file should be accepted under per-repo repo limit") + }, + ) + + runGlobalThenPerRepo(t, + "GlobalAndLFSOnlyLimit_BlocksLFSButNotGit", + func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Global: LFS limit ON, repo limit OFF, LFS not counted into repo size. + setting.SaveGlobalRepositorySetting(true, 0, lfsLimitBytes, false) + + ctx := newLimitRepo(t, "glfs") + + // push on git (OK) + err := pushGitBlob(t, ctx, gitOverBytes) + assert.NoError(t, err, "git push should be allowed when repo limit is 0") + + // push over on LFS (NOK) + err = pushLFSObjectOnce(t, ctx, lfsOverBytes) + assert.Error(t, err, "LFS push above global LFS limit must be rejected") + + // push under on LFS (OK) (under is last) + err = pushLFSObjectOnce(t, ctx, lfsUnderBytes) + assert.NoError(t, err, "LFS push under global LFS limit must be accepted") + }, + func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Per-repo: enable checking globally but do not set global limits. + setting.SaveGlobalRepositorySetting(true, 0, 0, false) + + ctx := newLimitRepo(t, "rlfs") + + // set per-repo LFS limit + t.Run("APISetRepoLFSSizeLimit", doAPISetRepoLFSSizeLimit(ctx, ctx.Username, ctx.Reponame, lfsLimitBytes)) + + // push on git (OK) + err := pushGitBlob(t, ctx, gitOverBytes) + assert.NoError(t, err, "git push should be allowed when repo limit is 0") + + // push over on LFS (NOK) + err = pushLFSObjectOnce(t, ctx, lfsOverBytes) + assert.Error(t, err, "LFS push above per-repo LFS limit must be rejected") + }, + ) + + runGlobalThenPerRepo(t, + "GlobalOrCombinedRepoAndLFSLimits_BlocksLFSandGit", + func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Global: repo limit ON, no LFS-specific limit, but LFS counted into repo size. + setting.SaveGlobalRepositorySetting(true, combinedRepoLimitBytes, 0, true) + + ctx := newLimitRepo(t, "gc") + + // push over via LFS (NOK) + err := pushLFSObjectOnce(t, ctx, combinedLFSOverBytes) + assert.Error(t, err, "LFS push must be rejected when it would exceed repo limit and LFSSizeInRepoSize=true") + + // push under via LFS (OK) + err = pushLFSObjectOnce(t, ctx, combinedLFSUnderBytes) + assert.NoError(t, err, "LFS push under repo limit must be accepted when LFSSizeInRepoSize=true") + }, + func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Per-repo: enable checking globally with LFSSizeInRepoSize=true but no global limits. + setting.SaveGlobalRepositorySetting(true, 0, 0, true) + + ctx := newLimitRepo(t, "rc") + + // set per-repo repo limit + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(ctx, ctx.Username, ctx.Reponame, combinedRepoLimitBytes)) + + // push over via LFS (NOK) + err := pushLFSObjectOnce(t, ctx, combinedLFSOverBytes) + assert.Error(t, err, "LFS push must be rejected when it would exceed per-repo repo limit and LFSSizeInRepoSize=true") + + // push under via LFS (OK) + err = pushLFSObjectOnce(t, ctx, combinedLFSUnderBytes) + assert.NoError(t, err, "LFS push under per-repo repo limit must be accepted when LFSSizeInRepoSize=true") + }, + ) + + t.Run("PerRepoLFSOverride_WinsOverGlobalDefault", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Global: strict LFS limit, no repo limit. + setting.SaveGlobalRepositorySetting(true, 0, lfsLimitBytes, false) + + ctx := newLimitRepo(t, "rwing") + + // Prepare one LFS commit and keep the clone so we can retry pushing SAME commit. + repoPath := prepareLFSPushForRetry(t, ctx, lfsOverBytes) + + // First push must fail. + err := pushCurrentBranch(t, repoPath) + assert.Error(t, err, "push must fail under strict global LFS limit") + + // Increase per-repo LFS limit to allow the previously rejected object. + // Use a clearly larger limit than the file size. + newPerRepoLimit := int64(lfsOverBytes) * 4 + t.Run("APISetRepoLFSSizeLimit", doAPISetRepoLFSSizeLimit(ctx, ctx.Username, ctx.Reponame, newPerRepoLimit)) + + // Retry pushing SAME commit must succeed now. + err = pushCurrentBranch(t, repoPath) + assert.NoError(t, err, "per-repo LFS limit must override global default and allow previously rejected push") + }) +}