mirror of
https://github.com/go-gitea/gitea.git
synced 2026-08-01 03:04:48 +02:00
LFSSize and related features
This commit is contained in:
@@ -1,52 +0,0 @@
|
||||
---
|
||||
date: "2023-05-24T13:00:00+00:00"
|
||||
title: "Per repository size limit"
|
||||
slug: "repo-size-limit"
|
||||
weight: 12
|
||||
toc: false
|
||||
draft: false
|
||||
aliases:
|
||||
- /en-us/repo-size-limit
|
||||
menu:
|
||||
sidebar:
|
||||
parent: "administration"
|
||||
name: "Per repository size limit"
|
||||
weight: 12
|
||||
identifier: "repo-size-limit"
|
||||
---
|
||||
|
||||
# Gitea per repository size limit setup
|
||||
|
||||
To use Gitea's experimental built-in per repository size limit support, Administrator must update the `app.ini` file:
|
||||
|
||||
```ini
|
||||
;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit
|
||||
;; "false" means no limit will be enforced, even if specified on a repository
|
||||
ENABLE_SIZE_LIMIT = true
|
||||
|
||||
;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit
|
||||
;; If repository has it's own limit set in UI it will override the global setting
|
||||
;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ...
|
||||
REPO_SIZE_LIMIT = 500 MB
|
||||
|
||||
This setting is persistent.
|
||||
|
||||
The size limitation is triggered when repository `disk size` + `new commit size` > `defined repository size limit`
|
||||
|
||||
If size limitation is triggered the feature would prevent commits that increase repository size on disk
|
||||
of gitea server and allow those that decrease it
|
||||
|
||||
# Gitea per repository size limit setup in UI
|
||||
|
||||
1. For Gitea admin it is possible during runtime to enable/disable limit size feature, change the global size limit on the fly.
|
||||
**This setting is not persistent across restarts**
|
||||
|
||||
`Admin panel/Site settings` -> `Repository management`
|
||||
|
||||
Persistance can be achieved if the limit is maintained by editing `app.ini` file
|
||||
|
||||
2. The individually set per repository limit in `Settings` of the
|
||||
repository would take precedence over global limit when the size limit
|
||||
feature is enabled. Only admin can modify those limits
|
||||
|
||||
**Note**: Size checking for large repositories is time consuming operation so time of push under size limit might increase up to a minute depending on your server hardware
|
||||
@@ -9,8 +9,9 @@ import (
|
||||
|
||||
func AddSizeLimitOnRepo(x *xorm.Engine) error {
|
||||
type Repository struct {
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
|
||||
ID int64 `xorm:"pk autoincr"`
|
||||
SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
|
||||
LFSSizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
|
||||
}
|
||||
|
||||
return x.Sync2(new(Repository))
|
||||
|
||||
+37
-2
@@ -206,6 +206,7 @@ type Repository struct {
|
||||
EnableSizeLimit bool `xorm:"NOT NULL DEFAULT true"`
|
||||
GitSize int64 `xorm:"NOT NULL DEFAULT 0"`
|
||||
LFSSize int64 `xorm:"NOT NULL DEFAULT 0"`
|
||||
LFSSizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
|
||||
CodeIndexerStatus *RepoIndexerStatus `xorm:"-"`
|
||||
StatsIndexerStatus *RepoIndexerStatus `xorm:"-"`
|
||||
IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"`
|
||||
@@ -647,12 +648,46 @@ func (repo *Repository) IsRepoSizeOversized(additionalSize int64) bool {
|
||||
return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.GitSize+additionalSize > repo.GetActualSizeLimit()
|
||||
}
|
||||
|
||||
// RepoSizeLimitEnabled return true if size limit checking is enabled and limit is non zero for this specific repository
|
||||
// ShouldCheckRepoSize returns true if size limit checking is enabled and limit is non zero for this specific repository
|
||||
// this is used to enable size checking during pre-receive hook
|
||||
func (repo *Repository) IsRepoSizeLimitEnabled() bool {
|
||||
func (repo *Repository) ShouldCheckRepoSize() bool {
|
||||
return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0
|
||||
}
|
||||
|
||||
// GetActualLFSSizeLimit returns repository LFS size limit in bytes
|
||||
// or global LFS size limit setting if per repository LFS size limit is not set
|
||||
func (repo *Repository) GetActualLFSSizeLimit() int64 {
|
||||
lfsSizeLimit := repo.LFSSizeLimit
|
||||
if setting.LFSSizeLimit > 0 && lfsSizeLimit == 0 {
|
||||
lfsSizeLimit = setting.LFSSizeLimit
|
||||
}
|
||||
return lfsSizeLimit
|
||||
}
|
||||
|
||||
// ShouldCheckLFSSize returns true if LFS size limit checking is enabled for this repository
|
||||
func (repo *Repository) ShouldCheckLFSSize() bool {
|
||||
return setting.EnableSizeLimit && repo.GetActualLFSSizeLimit() > 0
|
||||
}
|
||||
|
||||
// IsLFSSizeOversized returns true if adding additionalSize would exceed the LFS size limit
|
||||
func (repo *Repository) IsLFSSizeOversized(additionalSize int64) bool {
|
||||
return repo.ShouldCheckLFSSize() &&
|
||||
repo.LFSSize+additionalSize > repo.GetActualLFSSizeLimit()
|
||||
}
|
||||
|
||||
// IsRepoAndLFSSizeOversized checks if combined repo + LFS size exceeds repo size limit
|
||||
// This is used when LFS_SIZE_IN_REPO_SIZE is enabled
|
||||
func (repo *Repository) IsRepoAndLFSSizeOversized(additionalGitSize, additionalLFSSize int64) bool {
|
||||
if !setting.LFSSizeInRepoSize || !setting.EnableSizeLimit {
|
||||
return false
|
||||
}
|
||||
limit := repo.GetActualSizeLimit()
|
||||
if limit == 0 {
|
||||
return false
|
||||
}
|
||||
return (repo.GitSize+additionalGitSize)+(repo.LFSSize+additionalLFSSize) > limit
|
||||
}
|
||||
|
||||
// CanCreateBranch returns true if repository meets the requirements for creating new branches.
|
||||
func (repo *Repository) CanCreateBranch() bool {
|
||||
return !repo.IsMirror
|
||||
|
||||
@@ -276,13 +276,17 @@ var (
|
||||
RepoRootPath string
|
||||
ScriptType = "bash"
|
||||
|
||||
EnableSizeLimit = true
|
||||
RepoSizeLimit int64
|
||||
EnableSizeLimit = true
|
||||
RepoSizeLimit int64
|
||||
LFSSizeLimit int64
|
||||
LFSSizeInRepoSize bool
|
||||
)
|
||||
|
||||
func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) error {
|
||||
func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit, lfsSizeLimit int64, lfsSizeInRepoSize bool) error {
|
||||
EnableSizeLimit = enableSizeLimit
|
||||
RepoSizeLimit = repoSizeLimit
|
||||
LFSSizeLimit = lfsSizeLimit
|
||||
LFSSizeInRepoSize = lfsSizeInRepoSize
|
||||
sec := CfgProvider.Section("repository")
|
||||
if EnableSizeLimit {
|
||||
sec.Key("ENABLE_SIZE_LIMIT").SetValue("true")
|
||||
@@ -291,6 +295,12 @@ func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) erro
|
||||
}
|
||||
|
||||
sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit)))
|
||||
sec.Key("LFS_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(LFSSizeLimit)))
|
||||
if lfsSizeInRepoSize {
|
||||
sec.Key("LFS_SIZE_IN_REPO_SIZE").SetValue("true")
|
||||
} else {
|
||||
sec.Key("LFS_SIZE_IN_REPO_SIZE").SetValue("false")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -303,6 +313,9 @@ func loadRepositoryFrom(rootCfg ConfigProvider) {
|
||||
|
||||
v, _ := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0"))
|
||||
RepoSizeLimit = int64(v)
|
||||
v, _ = humanize.ParseBytes(sec.Key("LFS_SIZE_LIMIT").MustString("0"))
|
||||
LFSSizeLimit = int64(v)
|
||||
LFSSizeInRepoSize = sec.Key("LFS_SIZE_IN_REPO_SIZE").MustBool(false)
|
||||
|
||||
Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool()
|
||||
Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool()
|
||||
|
||||
@@ -155,6 +155,8 @@ type CreateRepoOption struct {
|
||||
ObjectFormatName string `json:"object_format_name" binding:"MaxSize(6)"`
|
||||
// SizeLimit of the repository
|
||||
SizeLimit int64 `json:"size_limit"`
|
||||
// LFSSizeLimit of the repository
|
||||
LFSSizeLimit int64 `json:"lfs_size_limit"`
|
||||
}
|
||||
|
||||
// EditRepoOption options when editing a repository's properties
|
||||
@@ -227,6 +229,8 @@ type EditRepoOption struct {
|
||||
Archived *bool `json:"archived,omitempty"`
|
||||
// SizeLimit of the repository.
|
||||
SizeLimit *int64 `json:"size_limit,omitempty"`
|
||||
// LFSSizeLimit of the repository.
|
||||
LFSSizeLimit *int64 `json:"lfs_size_limit,omitempty"`
|
||||
// set to a string like `8h30m0s` to set the mirror interval time
|
||||
MirrorInterval *string `json:"mirror_interval,omitempty"`
|
||||
// enable prune - remove obsolete remote-tracking references when mirroring
|
||||
|
||||
@@ -1053,6 +1053,9 @@ repo_name_profile_private_hint = .profile-private is a special repository that y
|
||||
repo_name_helper = Good repository names use short, memorable and unique keywords. A repository named ".profile" or ".profile-private" could be used to add a README.md for the user/organization profile.
|
||||
repo_size = Repository Size
|
||||
repo_size_limit = Repository Size Limit
|
||||
lfs_size = LFS Size
|
||||
lfs_size_limit = LFS Size Limit
|
||||
lfs_size_limit_helper = Override global LFS size limit. Leave empty to use global setting.
|
||||
template = Template
|
||||
template_select = Select a template.
|
||||
template_helper = Make repository a template
|
||||
@@ -1187,6 +1190,9 @@ form.name_reserved = The repository name "%s" is reserved.
|
||||
form.name_pattern_not_allowed = The pattern "%s" is not allowed in a repository name.
|
||||
form.repo_size_limit_negative = Repository size limitation cannot be negative.
|
||||
form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation.
|
||||
form.invalid_lfs_size_limit = Invalid LFS size limit format.
|
||||
form.lfs_size_limit_negative = LFS size limitation cannot be negative.
|
||||
form.lfs_size_limit_only_by_admins = Only administrators can change the LFS size limitation.
|
||||
|
||||
need_auth = Authorization
|
||||
migrate_options = Migration Options
|
||||
@@ -3453,6 +3459,10 @@ config.repository_config = Repository Configuration
|
||||
config.enable_size_limit = Enable Size Limit
|
||||
config.repo_size_limit = Default Repository Size Limit
|
||||
config.repo_size_limit.desc = Maximum repository size in MiB. Leave empty for no limit.
|
||||
config.lfs_size_limit = Default LFS Size Limit
|
||||
config.lfs_size_limit.desc = Maximum LFS size in MiB. Leave empty for no limit.
|
||||
config.lfs_size_in_repo_size = Count LFS in Repository Size
|
||||
config.lfs_size_in_repo_size.desc = When enabled, LFS size is included in repository size calculations
|
||||
config.global_repo_size_limit_manage_panel = Manage Global Repository Size Limit
|
||||
config.update_settings = Update Settings
|
||||
config.invalid_repo_size = Invalid repository size %s
|
||||
|
||||
@@ -264,6 +264,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *user_model.User, opt api.Cre
|
||||
IsTemplate: opt.Template,
|
||||
ObjectFormatName: opt.ObjectFormatName,
|
||||
SizeLimit: opt.SizeLimit,
|
||||
LFSSizeLimit: opt.LFSSizeLimit,
|
||||
})
|
||||
if err != nil {
|
||||
if repo_model.IsErrRepoAlreadyExist(err) {
|
||||
@@ -753,6 +754,9 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
|
||||
if opts.SizeLimit != nil {
|
||||
repo.SizeLimit = *opts.SizeLimit
|
||||
}
|
||||
if opts.LFSSizeLimit != nil {
|
||||
repo.LFSSizeLimit = *opts.LFSSizeLimit
|
||||
}
|
||||
|
||||
if err := repo_service.UpdateRepository(ctx, repo, visibilityChanged); err != nil {
|
||||
ctx.APIErrorInternal(err)
|
||||
|
||||
+398
-121
@@ -4,11 +4,13 @@
|
||||
package private
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -28,6 +30,7 @@ import (
|
||||
"code.gitea.io/gitea/modules/gitrepo"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/private"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"code.gitea.io/gitea/modules/web"
|
||||
"code.gitea.io/gitea/services/agit"
|
||||
@@ -120,9 +123,7 @@ func calculateSizeOfObject(ctx *gitea_context.PrivateContext, dir string, env []
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var errParse error
|
||||
var objectSize int64
|
||||
objectSize, errParse = strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64)
|
||||
objectSize, errParse := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64)
|
||||
if errParse != nil {
|
||||
log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID)
|
||||
return 0, errParse
|
||||
@@ -136,24 +137,14 @@ func calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCo
|
||||
// Calculate size of objects that were added
|
||||
for objectID := range newCommitObjects {
|
||||
if _, exists := oldCommitObjects[objectID]; !exists {
|
||||
// objectID is not referenced in the list of objects of old commit so it is a new object
|
||||
// Calculate its size and add it to the addedSize
|
||||
addedSize += commitObjectsSizes[objectID]
|
||||
}
|
||||
// We might check here if new object is not already in the rest of repo to be precise
|
||||
// However our goal is to prevent growth of repository so on determination of addedSize
|
||||
// We can skip this preciseness, addedSize will be more then real addedSize
|
||||
// TODO - do not count size of object that is referenced in other part of repo but not referenced neither in old nor new commit
|
||||
// git will not add the object twice
|
||||
}
|
||||
|
||||
// Calculate size of objects that were removed
|
||||
for objectID := range oldCommitObjects {
|
||||
if _, exists := newCommitObjects[objectID]; !exists {
|
||||
// objectID is not referenced in the list of new commit objects so it was possibly removed
|
||||
if _, exists := otherCommitObjects[objectID]; !exists {
|
||||
// objectID is not referenced in rest of the objects of the repository so it was removed
|
||||
// Calculate its size and add it to the removedSize
|
||||
removedSize += commitObjectsSizes[objectID]
|
||||
}
|
||||
}
|
||||
@@ -241,7 +232,6 @@ func loadObjectSizesFromPack(ctx *gitea_context.PrivateContext, dir string, env,
|
||||
}
|
||||
|
||||
// Second field has object type
|
||||
// If object type is not known filter it out and do not process
|
||||
objectType := fields[1]
|
||||
if objectType != "commit" && objectType != "tree" && objectType != "blob" && objectType != "tag" {
|
||||
continue
|
||||
@@ -292,8 +282,6 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, dir string, e
|
||||
i := 0
|
||||
for _, objectID := range objectIDs {
|
||||
_, exists := objectsSizes[objectID]
|
||||
|
||||
// If object doesn't yet have size in objectsSizes add it for further processing
|
||||
if !exists {
|
||||
reducedObjectIDs[i%numWorkers] = append(reducedObjectIDs[i%numWorkers], objectID)
|
||||
i++
|
||||
@@ -321,7 +309,6 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, dir string, e
|
||||
ran = true
|
||||
})
|
||||
if !ran {
|
||||
// This is not the first error – count it as a subsequent error.
|
||||
atomic.AddInt64(&errCount, 1)
|
||||
}
|
||||
}
|
||||
@@ -333,36 +320,25 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, dir string, e
|
||||
}(&reducedObjectIDs[(w-1)%numWorkers])
|
||||
}
|
||||
|
||||
// Wait for all workers to finish processing.
|
||||
wg.Wait()
|
||||
|
||||
if errExec == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If there were additional errors, wrap the first one with a summary.
|
||||
if n := atomic.LoadInt64(&errCount); n > 0 {
|
||||
return fmt.Errorf("%w (and %d subsequent similar errors)", errExec, n)
|
||||
}
|
||||
|
||||
// Only one error occurred.
|
||||
return errExec
|
||||
}
|
||||
|
||||
// loadObjectsSizesViaBatch uses hashes from objectIDs and uses pre-opened `git cat-file --batch-check` command to slice and return each object sizes
|
||||
// This function can't be used for new commit objects.
|
||||
// It speeds up loading object sizes from existing git database of the repository avoiding
|
||||
// multiple `git cat-files -s`
|
||||
func loadObjectsSizesViaBatch(ctx *gitea_context.PrivateContext, repoPath string, objectIDs []string, objectsSizes map[string]int64) error {
|
||||
var i int32
|
||||
|
||||
reducedObjectIDs := make([]string, 0, len(objectIDs))
|
||||
|
||||
// Loop over all objectIDs and find which ones are missing size information
|
||||
for _, objectID := range objectIDs {
|
||||
_, exists := objectsSizes[objectID]
|
||||
|
||||
// If object doesn't yet have size in objectsSizes add it for further processing
|
||||
if !exists {
|
||||
reducedObjectIDs = append(reducedObjectIDs, objectID)
|
||||
}
|
||||
@@ -420,15 +396,177 @@ func parseSize(sizeStr string) (int64, error) {
|
||||
return size, nil
|
||||
}
|
||||
|
||||
/*
|
||||
LFS pointer scanning (fast-ish, bounded)
|
||||
|
||||
We look for pointer blobs (small, <= 4KiB) and parse:
|
||||
|
||||
oid sha256:<64hex>
|
||||
size <bytes>
|
||||
|
||||
This lets us compute:
|
||||
- incomingNewToRepoLFS: pointers that are new vs old AND not referenced in "other" parts of repo
|
||||
- removedLFSSize: pointers removed vs new AND not referenced in "other"
|
||||
*/
|
||||
var (
|
||||
lfsPointerMarker = []byte("version https://git-lfs.github.com/spec/v1")
|
||||
lfsOIDRe = regexp.MustCompile(`(?m)^oid sha256:([0-9a-f]{64})$`)
|
||||
lfsSizeRe = regexp.MustCompile(`(?m)^size ([0-9]+)$`)
|
||||
)
|
||||
|
||||
func sumLFSSizes(m map[string]int64) int64 {
|
||||
var s int64
|
||||
for _, v := range m {
|
||||
s += v
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// scanLFSPointersFromObjectIDs finds LFS pointer blobs among objectIDs and returns map[oid]size.
|
||||
// It only reads small blobs via cat-file, so it stays bounded.
|
||||
// scanLFSPointersFromObjectIDs finds LFS pointer blobs among objectIDs and returns map[oid]size.
|
||||
// It only reads small blobs via cat-file, so it stays bounded.
|
||||
func scanLFSPointersFromObjectIDs(ctx *gitea_context.PrivateContext, repoPath string, env, objectIDs []string, maxBlobSize int64) (map[string]int64, error) {
|
||||
out := make(map[string]int64)
|
||||
if len(objectIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// 1) batch-check: filter small blobs only
|
||||
var input bytes.Buffer
|
||||
for _, oid := range objectIDs {
|
||||
if oid == "" {
|
||||
continue
|
||||
}
|
||||
input.WriteString(oid)
|
||||
input.WriteByte('\n')
|
||||
}
|
||||
|
||||
// Feed stdin via WithStdin, RunStdBytes takes only context.Context in your version
|
||||
checkCmd := gitcmd.NewCommand("cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)").
|
||||
WithDir(repoPath).
|
||||
WithEnv(env).
|
||||
WithStdin(bytes.NewReader(input.Bytes()))
|
||||
|
||||
checkBytes, _, err := checkCmd.RunStdBytes(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
smallBlobs := make([]string, 0, 1024)
|
||||
for line := range bytes.SplitSeq(checkBytes, []byte{'\n'}) {
|
||||
// "<sha> blob <size>"
|
||||
fields := bytes.Fields(line)
|
||||
if len(fields) != 3 {
|
||||
continue
|
||||
}
|
||||
if !bytes.Equal(fields[1], []byte("blob")) {
|
||||
continue
|
||||
}
|
||||
size, perr := strconv.ParseInt(string(fields[2]), 10, 64)
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
if size <= maxBlobSize {
|
||||
smallBlobs = append(smallBlobs, string(fields[0]))
|
||||
}
|
||||
}
|
||||
|
||||
if len(smallBlobs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// 2) batch: read contents of small blobs, parse LFS pointers
|
||||
var input2 bytes.Buffer
|
||||
for _, oid := range smallBlobs {
|
||||
input2.WriteString(oid)
|
||||
input2.WriteByte('\n')
|
||||
}
|
||||
|
||||
catCmd := gitcmd.NewCommand("cat-file", "--batch").
|
||||
WithDir(repoPath).
|
||||
WithEnv(env).
|
||||
WithStdin(bytes.NewReader(input2.Bytes()))
|
||||
|
||||
catBytes, _, err := catCmd.RunStdBytes(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
data := catBytes
|
||||
i := 0
|
||||
for i < len(data) {
|
||||
j := bytes.IndexByte(data[i:], '\n')
|
||||
if j < 0 {
|
||||
break
|
||||
}
|
||||
j += i
|
||||
header := data[i:j]
|
||||
i = j + 1
|
||||
|
||||
hf := bytes.Fields(header)
|
||||
if len(hf) < 3 {
|
||||
break
|
||||
}
|
||||
blobSize, perr := strconv.ParseInt(string(hf[2]), 10, 64)
|
||||
if perr != nil || blobSize < 0 {
|
||||
break
|
||||
}
|
||||
if i+int(blobSize) > len(data) {
|
||||
break
|
||||
}
|
||||
|
||||
content := data[i : i+int(blobSize)]
|
||||
i += int(blobSize)
|
||||
|
||||
if i < len(data) && data[i] == '\n' {
|
||||
i++
|
||||
}
|
||||
|
||||
if !bytes.Contains(content, lfsPointerMarker) {
|
||||
continue
|
||||
}
|
||||
|
||||
oidm := lfsOIDRe.FindSubmatch(content)
|
||||
if len(oidm) != 2 {
|
||||
continue
|
||||
}
|
||||
sizem := lfsSizeRe.FindSubmatch(content)
|
||||
if len(sizem) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
oid := string(oidm[1])
|
||||
sz, perr := strconv.ParseInt(string(sizem[1]), 10, 64)
|
||||
if perr != nil || sz < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if prev, ok := out[oid]; !ok || sz > prev {
|
||||
out[oid] = sz
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func maxInt64(a, b int64) int64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// HookPreReceive checks whether a individual commit is acceptable
|
||||
func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
startTime := time.Now()
|
||||
const maxLFSPointerBlobSize = int64(4096)
|
||||
|
||||
opts := web.GetForm(ctx).(*private.HookOptions)
|
||||
|
||||
ourCtx := &preReceiveContext{
|
||||
PrivateContext: ctx,
|
||||
env: generateGitEnv(opts), // Generate git environment for checking commits
|
||||
env: generateGitEnv(opts),
|
||||
opts: opts,
|
||||
}
|
||||
|
||||
@@ -436,14 +574,23 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
|
||||
var addedSize int64
|
||||
var removedSize int64
|
||||
|
||||
// LFS sizes derived from pointers
|
||||
var incomingNewToRepoLFS int64 // best proxy for “incoming LFS objects”
|
||||
var removedLFSSize int64
|
||||
var addedLFSSize int64 // new-vs-old pointers (can include already-known-to-repo)
|
||||
|
||||
var isRepoOversized bool
|
||||
var pushSize *git.CountObject
|
||||
var repoSize *git.CountObject
|
||||
var err error
|
||||
var duration time.Duration
|
||||
|
||||
if repo.IsRepoSizeLimitEnabled() {
|
||||
// Calculating total size of the repo using `git count-objects`
|
||||
needGitDelta := repo.ShouldCheckRepoSize()
|
||||
needLFSDelta := repo.ShouldCheckLFSSize() || setting.LFSSizeInRepoSize
|
||||
|
||||
// Only do CountObjects (push/repo) when we're doing the repo-size limit at all
|
||||
if needGitDelta {
|
||||
repoSize, err = git.CountObjects(ctx, repo.RepoPath())
|
||||
if err != nil {
|
||||
log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err)
|
||||
@@ -453,7 +600,6 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculating total size of the push using `git count-objects`
|
||||
pushSize, err = git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env)
|
||||
if err != nil {
|
||||
log.Error("Unable to get push size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err)
|
||||
@@ -463,13 +609,11 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
return
|
||||
}
|
||||
|
||||
// Cache whether the repository would breach the size limit after the operation
|
||||
isRepoOversized = repo.IsRepoSizeOversized(pushSize.Size + pushSize.SizePack)
|
||||
log.Warn("Push counts %+v", pushSize)
|
||||
log.Warn("Repo counts %+v", repoSize)
|
||||
}
|
||||
|
||||
// Iterate across the provided old commit IDs
|
||||
for i := range opts.OldCommitIDs {
|
||||
oldCommitID := opts.OldCommitIDs[i]
|
||||
newCommitID := opts.NewCommitIDs[i]
|
||||
@@ -477,15 +621,28 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
|
||||
log.Trace("Processing old commit: %s, new commit: %s, ref: %s", oldCommitID, newCommitID, refFullName)
|
||||
|
||||
// If operation is in potential breach of size limit prepare data for analysis
|
||||
if isRepoOversized {
|
||||
// Deep work is only needed if:
|
||||
// - repo is oversized (git deep path), OR
|
||||
// - we need LFS delta (LFS limit enabled OR combined mode enabled)
|
||||
if isRepoOversized || needLFSDelta {
|
||||
var gitObjects string
|
||||
var errLoop error
|
||||
var errLFS error
|
||||
|
||||
// Create cache of objects in old commit
|
||||
// if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail
|
||||
// Keep pointer maps so we can compute delta at the end
|
||||
var oldLFSPtrs, otherLFSPtrs, newLFSPtrs map[string]int64
|
||||
|
||||
// Only allocate object-size cache if we'll actually do git delta calc
|
||||
var commitObjectsSizes map[string]int64
|
||||
if isRepoOversized {
|
||||
commitObjectsSizes = make(map[string]int64)
|
||||
}
|
||||
|
||||
// OLD commit objects
|
||||
if oldCommitID != "0000000000000000000000000000000000000000" {
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects").AddDynamicArguments(oldCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects").
|
||||
AddDynamicArguments(oldCommitID).
|
||||
WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
if err != nil {
|
||||
log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err)
|
||||
ctx.JSON(http.StatusInternalServerError, private.Response{
|
||||
@@ -495,14 +652,26 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
}
|
||||
}
|
||||
|
||||
commitObjectsSizes := make(map[string]int64)
|
||||
oldCommitObjects := convertObjectsToMap(gitObjects)
|
||||
objectIDs := convertObjectsToSlice(gitObjects)
|
||||
|
||||
// Create cache of objects that are in the repository but not part of old or new commit
|
||||
// if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail
|
||||
// LFS pointers for OLD (only if needed)
|
||||
oldLFSPtrs = map[string]int64{}
|
||||
if needLFSDelta {
|
||||
oldLFSPtrs, errLFS = scanLFSPointersFromObjectIDs(ctx, repo.RepoPath(), ourCtx.env, objectIDs, maxLFSPointerBlobSize)
|
||||
if errLFS != nil {
|
||||
log.Error("Unable to scan old commit LFS pointers for %s in %-v: %v", oldCommitID, repo, errLFS)
|
||||
oldLFSPtrs = map[string]int64{}
|
||||
} else {
|
||||
log.Trace("LFS(old): pointers=%d total=%s", len(oldLFSPtrs), base.FileSize(sumLFSSizes(oldLFSPtrs)))
|
||||
}
|
||||
}
|
||||
|
||||
// OTHER objects (repo excluding old+new)
|
||||
if oldCommitID == "0000000000000000000000000000000000000000" {
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all").AddDynamicArguments("^" + newCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all").
|
||||
AddDynamicArguments("^" + newCommitID).
|
||||
WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
if err != nil {
|
||||
log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err)
|
||||
ctx.JSON(http.StatusInternalServerError, private.Response{
|
||||
@@ -511,7 +680,9 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects", "--all").
|
||||
AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).
|
||||
WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
if err != nil {
|
||||
log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err)
|
||||
ctx.JSON(http.StatusInternalServerError, private.Response{
|
||||
@@ -523,28 +694,42 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
|
||||
otherCommitObjects := convertObjectsToMap(gitObjects)
|
||||
objectIDs = append(objectIDs, convertObjectsToSlice(gitObjects)...)
|
||||
// Unfortunately `git cat-file --check-batch` shows full object size
|
||||
// so we would load compressed sizes from pack file via `git verify-pack -v` if there are pack files in repo
|
||||
// The result would still miss items that are loose as individual objects (not part of pack files)
|
||||
if repoSize.InPack > 0 {
|
||||
errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), nil, objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects from the pack in %-v Error: %v", repo, errLoop)
|
||||
|
||||
// LFS pointers for OTHER (only if needed)
|
||||
otherLFSPtrs = map[string]int64{}
|
||||
if needLFSDelta {
|
||||
otherLFSPtrs, errLFS = scanLFSPointersFromObjectIDs(ctx, repo.RepoPath(), ourCtx.env, objectIDs, maxLFSPointerBlobSize)
|
||||
if errLFS != nil {
|
||||
log.Error("Unable to scan other-objects LFS pointers for repo %-v: %v", repo, errLFS)
|
||||
otherLFSPtrs = map[string]int64{}
|
||||
} else {
|
||||
log.Trace("LFS(other): pointers=%d total=%s", len(otherLFSPtrs), base.FileSize(sumLFSSizes(otherLFSPtrs)))
|
||||
}
|
||||
}
|
||||
|
||||
// Load loose objects that are missing
|
||||
errLoop = loadObjectsSizesViaBatch(ctx, repo.RepoPath(), objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects that are missing in both old %s and new commits %s in %-v Error: %v", oldCommitID, newCommitID, repo, errLoop)
|
||||
ctx.JSON(http.StatusInternalServerError, private.Response{
|
||||
Err: fmt.Sprintf("Fail to get sizes of objects missing in both old and new commit and those in old commit: %v", err),
|
||||
})
|
||||
return
|
||||
// Load sizes of OLD+OTHER objects (existing in DB): pack + batch (git deep only)
|
||||
if isRepoOversized {
|
||||
if repoSize != nil && repoSize.InPack > 0 {
|
||||
errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), nil, objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects from the pack in %-v Error: %v", repo, errLoop)
|
||||
}
|
||||
}
|
||||
|
||||
errLoop = loadObjectsSizesViaBatch(ctx, repo.RepoPath(), objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects that are missing in both old %s and new commits %s in %-v Error: %v", oldCommitID, newCommitID, repo, errLoop)
|
||||
ctx.JSON(http.StatusInternalServerError, private.Response{
|
||||
Err: fmt.Sprintf("Fail to get sizes of objects missing in both old and new commit and those in old commit: %v", errLoop),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Create cache of objects in new commit
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects").AddDynamicArguments(newCommitID).WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
// NEW commit objects
|
||||
gitObjects, _, err = gitcmd.NewCommand("rev-list", "--objects").
|
||||
AddDynamicArguments(newCommitID).
|
||||
WithDir(repo.RepoPath()).WithEnv(ourCtx.env).RunStdString(ctx)
|
||||
if err != nil {
|
||||
log.Error("Unable to list objects in new commit %s in %-v Error: %v", newCommitID, repo, err)
|
||||
ctx.JSON(http.StatusInternalServerError, private.Response{
|
||||
@@ -555,25 +740,67 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
|
||||
newCommitObjects := convertObjectsToMap(gitObjects)
|
||||
objectIDs = convertObjectsToSlice(gitObjects)
|
||||
// Unfortunately `git cat-file --check-batch` doesn't work on objects not yet accepted into git database
|
||||
// so the sizes will be calculated through pack file `git verify-pack -v` if there are pack files
|
||||
// The result would still miss items that were sent loose as individual objects (not part of pack files)
|
||||
if pushSize.InPack > 0 {
|
||||
errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects from the pack in new commit %s in %-v Error: %v", newCommitID, repo, errLoop)
|
||||
|
||||
// LFS pointers for NEW (only if needed)
|
||||
newLFSPtrs = map[string]int64{}
|
||||
if needLFSDelta {
|
||||
newLFSPtrs, errLFS = scanLFSPointersFromObjectIDs(ctx, repo.RepoPath(), ourCtx.env, objectIDs, maxLFSPointerBlobSize)
|
||||
if errLFS != nil {
|
||||
log.Error("Unable to scan new commit LFS pointers for %s in %-v: %v", newCommitID, repo, errLFS)
|
||||
newLFSPtrs = map[string]int64{}
|
||||
} else {
|
||||
log.Trace("LFS(new): pointers=%d total=%s", len(newLFSPtrs), base.FileSize(sumLFSSizes(newLFSPtrs)))
|
||||
}
|
||||
}
|
||||
|
||||
// After loading everything we could from pack file, objects could have been sent as loose bunch as well
|
||||
// We need to load them individually with `git cat-file -s` on any object that is missing from accumulated size cache commitObjectsSizes
|
||||
errLoop = loadObjectsSizesViaCatFile(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects in new commit %s in %-v Error: %v", newCommitID, repo, errLoop)
|
||||
// Load sizes of NEW objects (may be in quarantine packs, etc.) (git deep only)
|
||||
if isRepoOversized {
|
||||
if pushSize != nil && pushSize.InPack > 0 {
|
||||
errLoop = loadObjectSizesFromPack(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects from the pack in new commit %s in %-v Error: %v", newCommitID, repo, errLoop)
|
||||
}
|
||||
}
|
||||
|
||||
errLoop = loadObjectsSizesViaCatFile(ctx, repo.RepoPath(), ourCtx.env, objectIDs, commitObjectsSizes)
|
||||
if errLoop != nil {
|
||||
log.Error("Unable to get sizes of objects in new commit %s in %-v Error: %v", newCommitID, repo, errLoop)
|
||||
}
|
||||
|
||||
// Git object delta (git deep only)
|
||||
addedSize, removedSize = calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects, commitObjectsSizes)
|
||||
}
|
||||
|
||||
// Calculate size that was added and removed by the new commit
|
||||
addedSize, removedSize = calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects, commitObjectsSizes)
|
||||
// LFS delta based on pointer presence (LFS deep only)
|
||||
if needLFSDelta {
|
||||
for oid, sz := range newLFSPtrs {
|
||||
if _, inOld := oldLFSPtrs[oid]; !inOld {
|
||||
addedLFSSize += sz
|
||||
if _, inOther := otherLFSPtrs[oid]; !inOther {
|
||||
incomingNewToRepoLFS += sz
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for oid, sz := range oldLFSPtrs {
|
||||
if _, inNew := newLFSPtrs[oid]; inNew {
|
||||
continue
|
||||
}
|
||||
if _, inOther := otherLFSPtrs[oid]; inOther {
|
||||
continue
|
||||
}
|
||||
removedLFSSize += sz
|
||||
}
|
||||
|
||||
log.Trace(
|
||||
"LFS(delta): incoming-new-to-repo=%s added(vs old)=%s removed=%s current(repo.LFSSize)=%s predicted=%s",
|
||||
base.FileSize(incomingNewToRepoLFS),
|
||||
base.FileSize(addedLFSSize),
|
||||
base.FileSize(removedLFSSize),
|
||||
base.FileSize(repo.LFSSize),
|
||||
base.FileSize(repo.LFSSize+incomingNewToRepoLFS-removedLFSSize),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -591,18 +818,101 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
|
||||
}
|
||||
}
|
||||
|
||||
if repo.IsRepoSizeLimitEnabled() {
|
||||
duration = time.Since(startTime)
|
||||
log.Warn("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %d, push size: %d, repo size: %d. Took %s seconds.", addedSize, removedSize, repo.GetActualSizeLimit(), pushSize.Size+pushSize.SizePack, repo.GitSize, duration)
|
||||
// --------- Final accounting + enforcement (one timing) ---------
|
||||
duration = time.Since(startTime)
|
||||
|
||||
currentGit := repo.GitSize
|
||||
currentLFS := repo.LFSSize
|
||||
currentCombined := currentGit + currentLFS
|
||||
|
||||
gitDelta := addedSize - removedSize
|
||||
predictedGitAfter := currentGit + gitDelta
|
||||
|
||||
lfsDelta := incomingNewToRepoLFS - removedLFSSize
|
||||
predictedLFSAfter := currentLFS + lfsDelta
|
||||
|
||||
predictedCombinedAfter := predictedGitAfter
|
||||
if setting.LFSSizeInRepoSize {
|
||||
predictedCombinedAfter = predictedGitAfter + predictedLFSAfter
|
||||
}
|
||||
|
||||
// If total of commits add more size then they remove and we are in a potential breach of size limit -- abort
|
||||
if (addedSize > removedSize) && isRepoOversized {
|
||||
log.Warn("Forbidden: new repo size %s would be over limitation of %s. Push size: %s. Took %s seconds. addedSize: %s. removedSize: %s", base.FileSize(repo.GitSize+addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size+pushSize.SizePack), duration, base.FileSize(addedSize), base.FileSize(removedSize))
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
UserMsg: "New repository size is over limitation of " + base.FileSize(repo.GetActualSizeLimit()),
|
||||
})
|
||||
return
|
||||
// Avoid nil panic when repo-size check is disabled but LFS delta is enabled (combined mode / LFS limit)
|
||||
pushBytes := int64(0)
|
||||
if pushSize != nil {
|
||||
pushBytes = maxInt64(0, pushSize.Size+pushSize.SizePack)
|
||||
}
|
||||
|
||||
// One summary line (time included here only)
|
||||
if repo.ShouldCheckRepoSize() || repo.ShouldCheckLFSSize() {
|
||||
log.Warn(
|
||||
"SizeCheck summary: took=%s repo=%s/%s git(pred=%s cur=%s delta=%s) lfs(pred=%s cur=%s delta=%s) combined(pred=%s) limits(repo=%s lfs=%s) LFSSizeInRepoSize=%v push=%s",
|
||||
duration,
|
||||
repo.OwnerName, repo.Name,
|
||||
base.FileSize(predictedGitAfter), base.FileSize(currentGit), base.FileSize(gitDelta),
|
||||
base.FileSize(predictedLFSAfter), base.FileSize(currentLFS), base.FileSize(lfsDelta),
|
||||
base.FileSize(predictedCombinedAfter),
|
||||
base.FileSize(repo.GetActualSizeLimit()),
|
||||
base.FileSize(repo.GetActualLFSSizeLimit()),
|
||||
setting.LFSSizeInRepoSize,
|
||||
base.FileSize(pushBytes),
|
||||
)
|
||||
}
|
||||
|
||||
// 1) LFS-only limit: compare against predicted LFS after push
|
||||
if repo.ShouldCheckLFSSize() {
|
||||
lfsLimit := repo.GetActualLFSSizeLimit()
|
||||
if lfsLimit > 0 && predictedLFSAfter > lfsLimit && predictedLFSAfter > currentLFS {
|
||||
log.Warn("Forbidden: LFS limit exceeded: %s > %s for repo %-v",
|
||||
base.FileSize(predictedLFSAfter),
|
||||
base.FileSize(lfsLimit),
|
||||
repo,
|
||||
)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
UserMsg: fmt.Sprintf("LFS size limit exceeded: %s > then limit of %s",
|
||||
base.FileSize(predictedLFSAfter),
|
||||
base.FileSize(lfsLimit),
|
||||
),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Repo (git) size limit when NOT counting LFS into repo size
|
||||
if repo.ShouldCheckRepoSize() && !setting.LFSSizeInRepoSize {
|
||||
limit := repo.GetActualSizeLimit()
|
||||
if limit > 0 && predictedGitAfter > limit {
|
||||
log.Warn("Forbidden: repository size limit exceeded: %s > %s for repo %-v",
|
||||
base.FileSize(predictedGitAfter),
|
||||
base.FileSize(limit),
|
||||
repo,
|
||||
)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
UserMsg: fmt.Sprintf("Repository size limit exceeded: %s > then limit of %s",
|
||||
base.FileSize(predictedGitAfter),
|
||||
base.FileSize(limit),
|
||||
),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 3) Combined limit when LFS is counted in repo size
|
||||
if setting.LFSSizeInRepoSize && repo.ShouldCheckRepoSize() {
|
||||
limit := repo.GetActualSizeLimit()
|
||||
if limit > 0 && predictedCombinedAfter > limit && predictedCombinedAfter > currentCombined {
|
||||
log.Warn("Forbidden: combined repo and LFS size limit exceeded: %s > %s for repo %-v",
|
||||
base.FileSize(predictedCombinedAfter),
|
||||
base.FileSize(limit),
|
||||
repo,
|
||||
)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
UserMsg: fmt.Sprintf("Combined repository+LFS size limit exceeded: %s > then limit of %s",
|
||||
base.FileSize(predictedCombinedAfter),
|
||||
base.FileSize(limit),
|
||||
),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ctx.PlainText(http.StatusOK, "ok")
|
||||
@@ -637,17 +947,11 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
return
|
||||
}
|
||||
|
||||
// Allow pushes to non-protected branches
|
||||
if protectBranch == nil {
|
||||
return
|
||||
}
|
||||
protectBranch.Repo = repo
|
||||
|
||||
// This ref is a protected branch.
|
||||
//
|
||||
// First of all we need to enforce absolutely:
|
||||
//
|
||||
// 1. Detect and prevent deletion of the branch
|
||||
if newCommitID == objectFormat.EmptyObjectID().String() {
|
||||
log.Warn("Forbidden: Branch: %s in %-v is protected from deletion", branchName, repo)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
@@ -658,7 +962,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
|
||||
isForcePush := false
|
||||
|
||||
// 2. Disallow force pushes to protected branches
|
||||
if oldCommitID != objectFormat.EmptyObjectID().String() {
|
||||
output, err := gitrepo.RunCmdString(ctx,
|
||||
repo,
|
||||
@@ -685,7 +988,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Enforce require signed commits
|
||||
if protectBranch.RequireSignedCommits {
|
||||
err := verifyCommits(oldCommitID, newCommitID, gitRepo, ctx.env)
|
||||
if err != nil {
|
||||
@@ -705,9 +1007,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
}
|
||||
}
|
||||
|
||||
// Now there are several tests which can be overridden:
|
||||
//
|
||||
// 4. Check protected file patterns - this is overridable from the UI
|
||||
changedProtectedfiles := false
|
||||
protectedFilePath := ""
|
||||
|
||||
@@ -728,10 +1027,8 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Check if the doer is allowed to push (and force-push if the incoming push is a force-push)
|
||||
var canPush bool
|
||||
if ctx.opts.DeployKeyID != 0 {
|
||||
// This flag is only ever true if protectBranch.CanForcePush is true
|
||||
if isForcePush {
|
||||
canPush = !changedProtectedfiles && protectBranch.CanPush && (!protectBranch.EnableForcePushAllowlist || protectBranch.ForcePushAllowlistDeployKeys)
|
||||
} else {
|
||||
@@ -753,13 +1050,8 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
}
|
||||
}
|
||||
|
||||
// 6. If we're not allowed to push directly
|
||||
if !canPush {
|
||||
// Is this is a merge from the UI/API?
|
||||
if ctx.opts.PullRequestID == 0 {
|
||||
// 6a. If we're not merging from the UI/API then there are two ways we got here:
|
||||
//
|
||||
// We are changing a protected file and we're not allowed to do that
|
||||
if changedProtectedfiles {
|
||||
log.Warn("Forbidden: Branch: %s in %-v is protected from changing file %s", branchName, repo, protectedFilePath)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
@@ -768,7 +1060,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
return
|
||||
}
|
||||
|
||||
// Allow commits that only touch unprotected files
|
||||
globs := protectBranch.GetUnprotectedFilePatterns()
|
||||
if len(globs) > 0 {
|
||||
unprotectedFilesOnly, err := pull_service.CheckUnprotectedFiles(gitRepo, branchName, oldCommitID, newCommitID, globs, ctx.env)
|
||||
@@ -780,12 +1071,10 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
return
|
||||
}
|
||||
if unprotectedFilesOnly {
|
||||
// Commit only touches unprotected files, this is allowed
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Or we're simply not able to push to this protected branch
|
||||
if isForcePush {
|
||||
log.Warn("Forbidden: User %d is not allowed to force-push to protected branch: %s in %-v", ctx.opts.UserID, branchName, repo)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
@@ -799,9 +1088,7 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
})
|
||||
return
|
||||
}
|
||||
// 6b. Merge (from UI or API)
|
||||
|
||||
// Get the PR, user and permissions for the user in the repository
|
||||
pr, err := issues_model.GetPullRequestByID(ctx, ctx.opts.PullRequestID)
|
||||
if err != nil {
|
||||
log.Error("Unable to get PullRequest %d Error: %v", ctx.opts.PullRequestID, err)
|
||||
@@ -811,14 +1098,10 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
return
|
||||
}
|
||||
|
||||
// although we should have called `loadPusherAndPermission` before, here we call it explicitly again because we need to access ctx.user below
|
||||
if !ctx.loadPusherAndPermission() {
|
||||
// if error occurs, loadPusherAndPermission had written the error response
|
||||
return
|
||||
}
|
||||
|
||||
// Now check if the user is allowed to merge PRs for this repository
|
||||
// Note: we can use ctx.perm and ctx.user directly as they will have been loaded above
|
||||
allowedMerge, err := pull_service.IsUserAllowedToMerge(ctx, pr, ctx.userPerm, ctx.user)
|
||||
if err != nil {
|
||||
log.Error("Error calculating if allowed to merge: %v", err)
|
||||
@@ -836,12 +1119,10 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
return
|
||||
}
|
||||
|
||||
// If we're an admin for the repository we can ignore status checks, reviews and override protected files
|
||||
if ctx.userPerm.IsAdmin() {
|
||||
return
|
||||
}
|
||||
|
||||
// Now if we're not an admin - we can't overwrite protected files so fail now
|
||||
if changedProtectedfiles {
|
||||
log.Warn("Forbidden: Branch: %s in %-v is protected from changing file %s", branchName, repo, protectedFilePath)
|
||||
ctx.JSON(http.StatusForbidden, private.Response{
|
||||
@@ -850,7 +1131,6 @@ func preReceiveBranch(ctx *preReceiveContext, oldCommitID, newCommitID string, r
|
||||
return
|
||||
}
|
||||
|
||||
// Check all status checks and reviews are ok
|
||||
if err := pull_service.CheckPullBranchProtections(ctx, pr, true); err != nil {
|
||||
if errors.Is(err, pull_service.ErrNotReadyToMerge) {
|
||||
log.Warn("Forbidden: User %d is not allowed push to protected branch %s in %-v and pr #%d is not ready to be merged: %s", ctx.opts.UserID, branchName, repo, pr.Index, err.Error())
|
||||
@@ -940,16 +1220,13 @@ func preReceiveFor(ctx *preReceiveContext, refFullName git.RefName) {
|
||||
func generateGitEnv(opts *private.HookOptions) (env []string) {
|
||||
env = os.Environ()
|
||||
if opts.GitAlternativeObjectDirectories != "" {
|
||||
env = append(env,
|
||||
private.GitAlternativeObjectDirectories+"="+opts.GitAlternativeObjectDirectories)
|
||||
env = append(env, private.GitAlternativeObjectDirectories+"="+opts.GitAlternativeObjectDirectories)
|
||||
}
|
||||
if opts.GitObjectDirectory != "" {
|
||||
env = append(env,
|
||||
private.GitObjectDirectory+"="+opts.GitObjectDirectory)
|
||||
env = append(env, private.GitObjectDirectory+"="+opts.GitObjectDirectory)
|
||||
}
|
||||
if opts.GitQuarantinePath != "" {
|
||||
env = append(env,
|
||||
private.GitQuarantinePath+"="+opts.GitQuarantinePath)
|
||||
env = append(env, private.GitQuarantinePath+"="+opts.GitQuarantinePath)
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ func Repos(ctx *context.Context) {
|
||||
|
||||
ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit
|
||||
ctx.Data["RepoSizeLimit"] = base.FileSize(setting.RepoSizeLimit)
|
||||
ctx.Data["LFSSizeLimit"] = base.FileSize(setting.LFSSizeLimit)
|
||||
ctx.Data["LFSSizeInRepoSize"] = setting.LFSSizeInRepoSize
|
||||
|
||||
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
|
||||
Private: true,
|
||||
@@ -64,6 +66,7 @@ func UpdateRepoPost(ctx *context.Context) {
|
||||
|
||||
ctx.Data["EnableSizeLimit"] = form.EnableSizeLimit
|
||||
ctx.Data["RepoSizeLimit"] = form.RepoSizeLimit
|
||||
ctx.Data["LFSSizeInRepoSize"] = form.LFSSizeInRepoSize
|
||||
|
||||
if err != nil {
|
||||
ctx.Data["Err_Repo_Size_Limit"] = err.Error()
|
||||
@@ -76,7 +79,21 @@ func UpdateRepoPost(ctx *context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit)
|
||||
lfsSizeLimit, err := base.GetFileSize(form.LFSSizeLimit)
|
||||
ctx.Data["LFSSizeLimit"] = form.LFSSizeLimit
|
||||
|
||||
if err != nil {
|
||||
ctx.Data["Err_LFS_Size_Limit"] = err.Error()
|
||||
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
|
||||
Private: true,
|
||||
PageSize: setting.UI.Admin.RepoPagingNum,
|
||||
TplName: tplRepos,
|
||||
OnlyShowRelevant: false,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit, lfsSizeLimit, form.LFSSizeInRepoSize)
|
||||
if err != nil {
|
||||
ctx.Data["Err_Repo_Size_Save"] = err.Error()
|
||||
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
|
||||
|
||||
@@ -63,7 +63,9 @@ func SettingsCtxData(ctx *context.Context) {
|
||||
ctx.Data["CanConvertFork"] = ctx.Repo.Repository.IsFork && ctx.Doer.CanCreateRepoIn(ctx.Repo.Repository.Owner)
|
||||
ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.IsRepoSizeOversized(ctx.Repo.Repository.GetActualSizeLimit() / 10) // less than 10% left
|
||||
ctx.Data["ActualSizeLimit"] = ctx.Repo.Repository.GetActualSizeLimit()
|
||||
ctx.Data["ActualLFSSizeLimit"] = ctx.Repo.Repository.GetActualLFSSizeLimit()
|
||||
ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit
|
||||
ctx.Data["LFSSizeInRepoSize"] = setting.LFSSizeInRepoSize
|
||||
|
||||
signing, _ := gitrepo.GetSigningKey(ctx, ctx.Repo.Repository)
|
||||
ctx.Data["SigningKeyAvailable"] = signing != nil
|
||||
@@ -237,6 +239,28 @@ func handleSettingsPostUpdate(ctx *context.Context) {
|
||||
}
|
||||
repo.SizeLimit = repoSizeLimit
|
||||
|
||||
// Handle LFS size limit (admin-only)
|
||||
var lfsSizeLimit int64
|
||||
if form.LFSSizeLimit != "" {
|
||||
lfsSizeLimit, err = base.GetFileSize(form.LFSSizeLimit)
|
||||
if err != nil {
|
||||
ctx.Data["Err_LFSSizeLimit"] = true
|
||||
ctx.RenderWithErr(ctx.Tr("repo.form.invalid_lfs_size_limit"), tplSettingsOptions, &form)
|
||||
return
|
||||
}
|
||||
}
|
||||
if lfsSizeLimit < 0 {
|
||||
ctx.Data["Err_LFSSizeLimit"] = true
|
||||
ctx.RenderWithErr(ctx.Tr("repo.form.lfs_size_limit_negative"), tplSettingsOptions, &form)
|
||||
return
|
||||
}
|
||||
if !ctx.Doer.IsAdmin && repo.LFSSizeLimit != lfsSizeLimit {
|
||||
ctx.Data["Err_LFSSizeLimit"] = true
|
||||
ctx.RenderWithErr(ctx.Tr("repo.form.lfs_size_limit_only_by_admins"), tplSettingsOptions, &form)
|
||||
return
|
||||
}
|
||||
repo.LFSSizeLimit = lfsSizeLimit
|
||||
|
||||
if err := repo_service.UpdateRepository(ctx, repo, false); err != nil {
|
||||
ctx.ServerError("UpdateRepository", err)
|
||||
return
|
||||
|
||||
@@ -20,8 +20,10 @@ import (
|
||||
|
||||
// UpdateGlobalRepoFrom for updating global repository setting
|
||||
type UpdateGlobalRepoFrom struct {
|
||||
RepoSizeLimit string
|
||||
EnableSizeLimit bool
|
||||
RepoSizeLimit string
|
||||
LFSSizeLimit string
|
||||
EnableSizeLimit bool
|
||||
LFSSizeInRepoSize bool
|
||||
}
|
||||
|
||||
// CreateRepoForm form for creating repository
|
||||
@@ -114,6 +116,7 @@ type RepoSettingForm struct {
|
||||
Template bool
|
||||
EnablePrune bool
|
||||
RepoSizeLimit string
|
||||
LFSSizeLimit string
|
||||
|
||||
// Advanced settings
|
||||
EnableCode bool
|
||||
|
||||
@@ -5,6 +5,7 @@ package lfs
|
||||
|
||||
import (
|
||||
stdCtx "context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
"code.gitea.io/gitea/models/unit"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/auth/httpauth"
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
lfs_module "code.gitea.io/gitea/modules/lfs"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
@@ -184,6 +186,25 @@ func DownloadHandler(ctx *context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- tracing helpers ----
|
||||
|
||||
func traceBatchDecision(rc *requestContext, op, msg string, args ...any) {
|
||||
prefix := fmt.Sprintf("LFS[BATCH][%s/%s][op=%s] ", rc.User, rc.Repo, op)
|
||||
log.Trace(prefix+msg, args...)
|
||||
}
|
||||
|
||||
func batchReqID(br *lfs_module.BatchRequest) string {
|
||||
h := sha1.New()
|
||||
h.Write([]byte(br.Operation))
|
||||
for _, o := range br.Objects {
|
||||
h.Write([]byte(o.Oid))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(strconv.FormatInt(o.Size, 10)))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:10]
|
||||
}
|
||||
|
||||
// BatchHandler provides the batch api
|
||||
func BatchHandler(ctx *context.Context) {
|
||||
var br lfs_module.BatchRequest
|
||||
@@ -206,6 +227,9 @@ func BatchHandler(ctx *context.Context) {
|
||||
}
|
||||
|
||||
rc := getRequestContext(ctx)
|
||||
reqID := batchReqID(&br)
|
||||
|
||||
log.Trace("LFS[BATCH][%s/%s] req=%s op=%s objects=%d", rc.User, rc.Repo, reqID, br.Operation, len(br.Objects))
|
||||
|
||||
repository := getAuthenticatedRepository(ctx, rc, isUpload)
|
||||
if repository == nil {
|
||||
@@ -217,8 +241,98 @@ func BatchHandler(ctx *context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Create content store once, reuse for tracing + normal logic below.
|
||||
contentStore := lfs_module.NewContentStore()
|
||||
|
||||
currCombinedTotal := repository.GitSize + repository.LFSSize
|
||||
|
||||
// Baseline repo stats and limits
|
||||
traceBatchDecision(rc, br.Operation,
|
||||
"req=%s auth=%t isUpload=%t repoID=%d sizes: git=%s lfs=%s combined=%s limits: repo=%s lfs=%s LFSSizeInRepoSize=%v",
|
||||
reqID,
|
||||
ctx.IsSigned || ctx.Doer != nil,
|
||||
isUpload,
|
||||
repository.ID,
|
||||
base.FileSize(repository.GitSize),
|
||||
base.FileSize(repository.LFSSize),
|
||||
base.FileSize(currCombinedTotal),
|
||||
base.FileSize(repository.GetActualSizeLimit()),
|
||||
base.FileSize(repository.GetActualLFSSizeLimit()),
|
||||
setting.LFSSizeInRepoSize,
|
||||
)
|
||||
|
||||
// Check LFS size limits for upload operations
|
||||
if isUpload && (repository.ShouldCheckLFSSize() || (setting.LFSSizeInRepoSize && repository.ShouldCheckRepoSize())) {
|
||||
// Sum sizes of objects that are NEW TO THIS REPO (no meta row)
|
||||
var incomingNewToRepoLFS int64
|
||||
var invalidCount, metaMissingCount, metaPresentCount, storeExistsCount int64
|
||||
|
||||
for _, p := range br.Objects {
|
||||
if !p.IsValid() {
|
||||
invalidCount++
|
||||
continue
|
||||
}
|
||||
|
||||
meta, _ := git_model.GetLFSMetaObjectByOid(ctx, repository.ID, p.Oid)
|
||||
|
||||
exists, _ := contentStore.Exists(p)
|
||||
if exists {
|
||||
storeExistsCount++
|
||||
}
|
||||
|
||||
if meta == nil {
|
||||
metaMissingCount++
|
||||
incomingNewToRepoLFS += p.Size
|
||||
} else {
|
||||
metaPresentCount++
|
||||
}
|
||||
}
|
||||
|
||||
predictedLFS := repository.LFSSize + incomingNewToRepoLFS
|
||||
predictedTotal := repository.GitSize + predictedLFS
|
||||
|
||||
traceBatchDecision(rc, br.Operation,
|
||||
"req=%s accounting: objects=%d invalid=%d metaMissing=%d metaPresent=%d storeExists=%d incomingNewToRepoLFS=%s predictedLFS=%s predictedTotal=%s",
|
||||
reqID,
|
||||
len(br.Objects),
|
||||
invalidCount,
|
||||
metaMissingCount,
|
||||
metaPresentCount,
|
||||
storeExistsCount,
|
||||
base.FileSize(incomingNewToRepoLFS),
|
||||
base.FileSize(predictedLFS),
|
||||
base.FileSize(predictedTotal),
|
||||
)
|
||||
|
||||
// LFS-only limit if we are over, but size doesn't increase allow
|
||||
if repository.ShouldCheckLFSSize() && predictedLFS > repository.GetActualLFSSizeLimit() && predictedLFS > repository.LFSSize {
|
||||
traceBatchDecision(rc, br.Operation,
|
||||
"req=%s DECISION=FORBID reason=LFS_LIMIT predictedLFS=%s limit=%s",
|
||||
reqID, base.FileSize(predictedLFS), base.FileSize(repository.GetActualLFSSizeLimit()),
|
||||
)
|
||||
writeStatusMessage(ctx, http.StatusForbidden,
|
||||
fmt.Sprintf("LFS size %s would exceed limit %s",
|
||||
base.FileSize(predictedLFS), base.FileSize(repository.GetActualLFSSizeLimit())))
|
||||
return
|
||||
}
|
||||
|
||||
// Combined limit (conservative: ignores git delta/removals) if we are over, but combined size doesn't increase allow
|
||||
if setting.LFSSizeInRepoSize && repository.ShouldCheckRepoSize() {
|
||||
if predictedTotal > repository.GetActualSizeLimit() && predictedTotal > currCombinedTotal {
|
||||
traceBatchDecision(rc, br.Operation,
|
||||
"req=%s DECISION=FORBID reason=COMBINED_LIMIT predictedTotal=%s limit=%s",
|
||||
reqID, base.FileSize(predictedTotal), base.FileSize(repository.GetActualSizeLimit()),
|
||||
)
|
||||
writeStatusMessage(ctx, http.StatusForbidden,
|
||||
fmt.Sprintf("Repository size %s after LFS addition would exceed limit %s",
|
||||
base.FileSize(predictedTotal), base.FileSize(repository.GetActualSizeLimit())))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
traceBatchDecision(rc, br.Operation, "req=%s DECISION=ALLOW size-check passed", reqID)
|
||||
}
|
||||
|
||||
var responseObjects []*lfs_module.ObjectResponse
|
||||
|
||||
for _, p := range br.Objects {
|
||||
|
||||
@@ -53,6 +53,7 @@ type CreateRepoOptions struct {
|
||||
MirrorInterval string
|
||||
ObjectFormatName string
|
||||
SizeLimit int64
|
||||
LFSSizeLimit int64
|
||||
}
|
||||
|
||||
func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir string, opts CreateRepoOptions) error {
|
||||
@@ -249,6 +250,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User,
|
||||
IsEmpty: !opts.AutoInit,
|
||||
TrustModel: opts.TrustModel,
|
||||
SizeLimit: opts.SizeLimit,
|
||||
LFSSizeLimit: opts.LFSSizeLimit,
|
||||
IsMirror: opts.IsMirror,
|
||||
DefaultBranch: opts.DefaultBranch,
|
||||
DefaultWikiBranch: setting.Repository.DefaultBranch,
|
||||
|
||||
@@ -12,10 +12,20 @@
|
||||
<input name="enable_size_limit" type="checkbox" {{if .EnableSizeLimit}}checked{{end}}>
|
||||
</div>
|
||||
</div>
|
||||
<div class="inline field">
|
||||
<label>{{ctx.Locale.Tr "admin.config.lfs_size_in_repo_size"}}</label>
|
||||
<div class="ui checkbox">
|
||||
<input name="lfs_size_in_repo_size" type="checkbox" {{if .LFSSizeInRepoSize}}checked{{end}} data-tooltip-content='{{ctx.Locale.Tr "admin.config.lfs_size_in_repo_size.desc"}}'>
|
||||
</div>
|
||||
</div>
|
||||
<div class="inline field {{if .Err_Repo_Size_Limit}}error{{end}}">
|
||||
<label for="repo_size_limit">{{ctx.Locale.Tr "admin.config.repo_size_limit"}}</label>
|
||||
<input id="repo_size_limit" name="repo_size_limit" value="{{.RepoSizeLimit}}" data-tooltip-content='{{ctx.Locale.Tr "admin.config.repo_size_limit.desc"}}'>
|
||||
</div>
|
||||
<div class="inline field {{if .Err_LFS_Size_Limit}}error{{end}}">
|
||||
<label for="lfs_size_limit">{{ctx.Locale.Tr "admin.config.lfs_size_limit"}}</label>
|
||||
<input id="lfs_size_limit" name="lfs_size_limit" value="{{.LFSSizeLimit}}" data-tooltip-content='{{ctx.Locale.Tr "admin.config.lfs_size_limit.desc"}}'>
|
||||
</div>
|
||||
<div class="field">
|
||||
<button class="ui green button">{{ctx.Locale.Tr "admin.config.update_settings"}}</button>
|
||||
</div>
|
||||
|
||||
@@ -20,12 +20,27 @@
|
||||
{{end}}
|
||||
</span>
|
||||
</div>
|
||||
<div class="inline field">
|
||||
<label>{{ctx.Locale.Tr "repo.lfs_size"}}</label>
|
||||
<span>{{FileSize .Repository.LFSSize}}
|
||||
{{if and .ActualLFSSizeLimit .EnableSizeLimit}}
|
||||
/ {{FileSize .ActualLFSSizeLimit}}
|
||||
{{end}}
|
||||
</span>
|
||||
</div>
|
||||
<div class="field {{if .Err_RepoSizeLimit}}error{{end}}" {{if not .IsAdmin}}style="display:none;"{{end}}>
|
||||
<label for="repo_size_limit">{{ctx.Locale.Tr "repo.repo_size_limit"}}</label>
|
||||
<input id="repo_size_limit" name="repo_size_limit"
|
||||
{{if not .EnableSizeLimit}}class="ui text light grey"{{end}}
|
||||
type="text" value="{{if .Repository.SizeLimit}}{{FileSize .Repository.SizeLimit}}{{end}}">
|
||||
</div>
|
||||
<div class="field {{if .Err_LFSSizeLimit}}error{{end}}" {{if not .IsAdmin}}style="display:none;"{{end}}>
|
||||
<label for="lfs_size_limit">{{ctx.Locale.Tr "repo.lfs_size_limit"}}</label>
|
||||
<input id="lfs_size_limit" name="lfs_size_limit"
|
||||
{{if not .EnableSizeLimit}}class="ui text light grey"{{end}}
|
||||
type="text" value="{{if .Repository.LFSSizeLimit}}{{FileSize .Repository.LFSSizeLimit}}{{end}}"
|
||||
data-tooltip-content='{{ctx.Locale.Tr "repo.lfs_size_limit_helper"}}'>
|
||||
</div>
|
||||
<div class="inline field">
|
||||
<label>{{ctx.Locale.Tr "repo.template"}}</label>
|
||||
<div class="ui checkbox">
|
||||
|
||||
@@ -483,3 +483,18 @@ func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) f
|
||||
ctx.Session.MakeRequest(t, req, 200)
|
||||
}
|
||||
}
|
||||
|
||||
func doAPISetRepoLFSSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) {
|
||||
return func(t *testing.T) {
|
||||
urlStr := fmt.Sprintf("/api/v1/repos/%s/%s",
|
||||
owner, repo)
|
||||
req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{LFSSizeLimit: &size}).
|
||||
AddTokenAuth(ctx.Token)
|
||||
|
||||
if ctx.ExpectedCode != 0 {
|
||||
ctx.Session.MakeRequest(t, req, ctx.ExpectedCode)
|
||||
return
|
||||
}
|
||||
ctx.Session.MakeRequest(t, req, 200)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func testGitGeneral(t *testing.T, u *url.URL) {
|
||||
|
||||
t.Run("SizeLimit", func(t *testing.T) {
|
||||
dstForkedPath := t.TempDir()
|
||||
setting.SaveGlobalRepositorySetting(true, 0)
|
||||
setting.SaveGlobalRepositorySetting(true, 0, 0, false)
|
||||
t.Run("Under", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
doCommitAndPush(t, testFileSizeSmall, dstPath, "data-file-")
|
||||
@@ -115,7 +115,7 @@ func testGitGeneral(t *testing.T, u *url.URL) {
|
||||
doRebaseCommitAndPush(t, dstPath, lastCommitID)
|
||||
newRepoSize := doGetRemoteRepoSizeViaAPI(t, forkedUserCtx)
|
||||
assert.LessOrEqual(t, newRepoSize, oldRepoSize)
|
||||
setting.SaveGlobalRepositorySetting(false, 0)
|
||||
setting.SaveGlobalRepositorySetting(false, 0, 0, false)
|
||||
})
|
||||
})
|
||||
t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "test/head"))
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
// Copyright 2017 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
auth_model "code.gitea.io/gitea/models/auth"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/git/gitcmd"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/tests"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
func base62(n int64) string {
|
||||
if n == 0 {
|
||||
return string(alphabet[0])
|
||||
}
|
||||
var buf [11]byte
|
||||
i := len(buf)
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = alphabet[n%62]
|
||||
n /= 62
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
|
||||
// ID5: 5-char ID derived from unix seconds.
|
||||
func ID5() string {
|
||||
sec := time.Now().Unix()
|
||||
s := base62(sec)
|
||||
if len(s) < 5 {
|
||||
return fmt.Sprintf("%05s", s)
|
||||
}
|
||||
return s[len(s)-5:]
|
||||
}
|
||||
|
||||
// TestLFSSizeLimit exercises the pre-receive size limit logic.
|
||||
// Runs sequentially (no t.Parallel) because it mutates GLOBAL settings.
|
||||
//
|
||||
// Enable/disable via:
|
||||
//
|
||||
// setting.SaveGlobalRepositorySetting(enabled, repoLimit, lfsLimit, lfsSizeInRepoSize)
|
||||
//
|
||||
// Scenarios:
|
||||
// - repo-only limit: blocks git blobs but not LFS objects (when lfsSizeInRepoSize=false, lfsLimit=0)
|
||||
// - LFS-only limit: blocks LFS objects but not git blobs
|
||||
// - combined repo+LFS (LFS counted into repo size): blocks LFS when it would exceed repo limit
|
||||
// - per-repo LFS override wins over global default (retry same push after increasing per-repo LFS limit)
|
||||
func TestLFSSizeLimit(t *testing.T) {
|
||||
onGiteaRun(t, testLFSSizeLimit)
|
||||
}
|
||||
|
||||
func testLFSSizeLimit(t *testing.T, baseURL *url.URL) {
|
||||
// Always disable at the end so we don't leak to other integration tests.
|
||||
t.Cleanup(func() {
|
||||
setting.SaveGlobalRepositorySetting(false, 0, 0, false)
|
||||
})
|
||||
|
||||
if !setting.LFS.StartServer {
|
||||
t.Skip("LFS server disabled")
|
||||
}
|
||||
// ---- sizes ----
|
||||
// Repo-size checks are not precise (compression), so use very different sizes vs limit.
|
||||
const (
|
||||
repoLimitBytes = int64(64 * 1024) // 64KiB
|
||||
gitUnderBytes = int(8 * 1024) // 8KiB
|
||||
gitOverBytes = int(4 * 1024 * 1024) // 4MiB (far above limit)
|
||||
lfsBigBytes = int(1 * 1024 * 1024) // 1MiB (should still pass repo-only when not counted)
|
||||
)
|
||||
|
||||
// LFS-only checks are precise.
|
||||
const (
|
||||
lfsLimitBytes = int64(64 * 1024) // 64KiB
|
||||
lfsUnderBytes = int(32 * 1024) // 32KiB
|
||||
lfsOverBytes = int(128 * 1024) // 128KiB
|
||||
)
|
||||
|
||||
// Combined repo+LFS (LFS counted into repo size).
|
||||
const (
|
||||
combinedRepoLimitBytes = int64(128 * 1024) // 128KiB
|
||||
combinedLFSUnderBytes = int(64 * 1024) // 64KiB
|
||||
combinedLFSOverBytes = int(512 * 1024) // 512KiB
|
||||
)
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
newLimitRepo := func(t *testing.T, suffix string) APITestContext {
|
||||
t.Helper()
|
||||
|
||||
// Make a unique repo name without relying on tests.GetTestUID / CreateRepo.
|
||||
repoName := fmt.Sprintf(
|
||||
"lfsl-%s-%s",
|
||||
suffix,
|
||||
ID5(),
|
||||
)
|
||||
|
||||
ctx := NewAPITestContext(
|
||||
t,
|
||||
"user2",
|
||||
repoName,
|
||||
auth_model.AccessTokenScopeWriteRepository,
|
||||
auth_model.AccessTokenScopeWriteUser,
|
||||
)
|
||||
|
||||
// Create via API (same style as git_general_test.go)
|
||||
doAPICreateRepository(ctx, false)(t)
|
||||
return ctx
|
||||
}
|
||||
|
||||
cloneHTTP := func(t *testing.T, ctx APITestContext) string {
|
||||
t.Helper()
|
||||
|
||||
u := *baseURL // copy
|
||||
u.Path = ctx.GitPath()
|
||||
u.User = url.UserPassword(ctx.Username, userPassword)
|
||||
|
||||
dst := t.TempDir()
|
||||
doGitClone(dst, &u)(t)
|
||||
return dst
|
||||
}
|
||||
|
||||
commitSignature := func() *git.Signature {
|
||||
return &git.Signature{
|
||||
Email: "user2@example.com",
|
||||
Name: "User Two",
|
||||
When: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
configureLFSForPrefix := func(t *testing.T, repoPath, prefix string) {
|
||||
t.Helper()
|
||||
|
||||
// git lfs install
|
||||
err := gitcmd.NewCommand("lfs").AddArguments("install").WithDir(repoPath).Run(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
// track prefix*
|
||||
_, _, err = gitcmd.NewCommand("lfs").
|
||||
AddArguments("track").
|
||||
AddDynamicArguments(prefix + "*").
|
||||
WithDir(repoPath).
|
||||
RunStdString(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
// commit .gitattributes
|
||||
err = git.AddChanges(t.Context(), repoPath, false, ".gitattributes")
|
||||
require.NoError(t, err)
|
||||
|
||||
sig := commitSignature()
|
||||
err = git.CommitChanges(t.Context(), repoPath, git.CommitChangesOptions{
|
||||
Committer: sig,
|
||||
Author: sig,
|
||||
Message: "configure LFS tracking",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
pushCurrentBranch := func(t *testing.T, repoPath string) error {
|
||||
t.Helper()
|
||||
_, _, err := gitcmd.NewCommand("push", "origin", "master").WithDir(repoPath).RunStdString(t.Context())
|
||||
return err
|
||||
}
|
||||
|
||||
// Push a git blob by creating a new random file (random-ish content reduces compression effects).
|
||||
pushGitBlob := func(t *testing.T, ctx APITestContext, size int) error {
|
||||
t.Helper()
|
||||
|
||||
repoPath := cloneHTTP(t, ctx)
|
||||
|
||||
_, err := generateCommitWithNewData(
|
||||
t.Context(),
|
||||
size,
|
||||
repoPath,
|
||||
"user2@example.com",
|
||||
"User Two",
|
||||
"git-data-",
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return pushCurrentBranch(t, repoPath)
|
||||
}
|
||||
|
||||
// Push an LFS object by tracking prefix* and then committing a file created by generateCommitWithNewData.
|
||||
pushLFSObjectOnce := func(t *testing.T, ctx APITestContext, size int) error {
|
||||
t.Helper()
|
||||
|
||||
repoPath := cloneHTTP(t, ctx)
|
||||
|
||||
const prefix = "lfs-data-"
|
||||
configureLFSForPrefix(t, repoPath, prefix)
|
||||
|
||||
_, err := generateCommitWithNewData(
|
||||
t.Context(),
|
||||
size,
|
||||
repoPath,
|
||||
"user2@example.com",
|
||||
"User Two",
|
||||
prefix,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return pushCurrentBranch(t, repoPath)
|
||||
}
|
||||
|
||||
// Prepare a single LFS commit in a clone and return the clone path, so we can:
|
||||
// - push (expect fail)
|
||||
// - change limits
|
||||
// - push SAME COMMIT again (expect pass)
|
||||
prepareLFSPushForRetry := func(t *testing.T, ctx APITestContext, size int) string {
|
||||
t.Helper()
|
||||
|
||||
repoPath := cloneHTTP(t, ctx)
|
||||
|
||||
const prefix = "lfs-retry-"
|
||||
configureLFSForPrefix(t, repoPath, prefix)
|
||||
|
||||
_, err := generateCommitWithNewData(
|
||||
t.Context(),
|
||||
size,
|
||||
repoPath,
|
||||
"user2@example.com",
|
||||
"User Two",
|
||||
prefix,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Ensure the repoPath is used (avoid accidental cleanup / unused).
|
||||
_, err = os.Stat(filepath.Join(repoPath, ".git"))
|
||||
require.NoError(t, err)
|
||||
|
||||
return repoPath
|
||||
}
|
||||
|
||||
runGlobalThenPerRepo := func(
|
||||
t *testing.T,
|
||||
name string,
|
||||
global func(t *testing.T),
|
||||
perRepo func(t *testing.T),
|
||||
) {
|
||||
t.Helper()
|
||||
t.Run(name+"/Global", func(t *testing.T) { global(t) })
|
||||
t.Run(name+"/PerRepo", func(t *testing.T) { perRepo(t) })
|
||||
}
|
||||
|
||||
// ---- tests ----
|
||||
|
||||
runGlobalThenPerRepo(t,
|
||||
"GlobalAndRepoOnlyLimit_BlocksGitButNotLFS",
|
||||
func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Global: repo limit ON, LFS limit OFF, LFS not counted into repo size.
|
||||
setting.SaveGlobalRepositorySetting(true, repoLimitBytes, 0, false)
|
||||
|
||||
ctx := newLimitRepo(t, "ggit")
|
||||
|
||||
// push over on git (NOK)
|
||||
err := pushGitBlob(t, ctx, gitOverBytes)
|
||||
assert.Error(t, err, "git file well over repo limit should be rejected")
|
||||
|
||||
// push LFS (OK)
|
||||
err = pushLFSObjectOnce(t, ctx, lfsBigBytes)
|
||||
assert.NoError(t, err, "LFS push should be allowed when repo-only limit is enabled and LFSSizeInRepoSize=false")
|
||||
|
||||
// push under on git (OK)
|
||||
err = pushGitBlob(t, ctx, gitUnderBytes)
|
||||
assert.NoError(t, err, "small git file should be accepted if under repo limit")
|
||||
},
|
||||
func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Per-repo: enable checking globally but do not set global limits.
|
||||
setting.SaveGlobalRepositorySetting(true, 0, 0, false)
|
||||
|
||||
ctx := newLimitRepo(t, "rgit")
|
||||
|
||||
// set per-repo repo limit
|
||||
t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(ctx, ctx.Username, ctx.Reponame, repoLimitBytes))
|
||||
|
||||
// push over on git (NOK)
|
||||
err := pushGitBlob(t, ctx, gitOverBytes)
|
||||
assert.Error(t, err, "git file well over per-repo repo limit should be rejected")
|
||||
|
||||
// push LFS (OK)
|
||||
err = pushLFSObjectOnce(t, ctx, lfsBigBytes)
|
||||
assert.NoError(t, err, "LFS push should be allowed when only per-repo repo limit is set and LFSSizeInRepoSize=false")
|
||||
|
||||
// push under on git (OK)
|
||||
err = pushGitBlob(t, ctx, gitUnderBytes)
|
||||
assert.NoError(t, err, "small git file should be accepted under per-repo repo limit")
|
||||
},
|
||||
)
|
||||
|
||||
runGlobalThenPerRepo(t,
|
||||
"GlobalAndLFSOnlyLimit_BlocksLFSButNotGit",
|
||||
func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Global: LFS limit ON, repo limit OFF, LFS not counted into repo size.
|
||||
setting.SaveGlobalRepositorySetting(true, 0, lfsLimitBytes, false)
|
||||
|
||||
ctx := newLimitRepo(t, "glfs")
|
||||
|
||||
// push on git (OK)
|
||||
err := pushGitBlob(t, ctx, gitOverBytes)
|
||||
assert.NoError(t, err, "git push should be allowed when repo limit is 0")
|
||||
|
||||
// push over on LFS (NOK)
|
||||
err = pushLFSObjectOnce(t, ctx, lfsOverBytes)
|
||||
assert.Error(t, err, "LFS push above global LFS limit must be rejected")
|
||||
|
||||
// push under on LFS (OK) (under is last)
|
||||
err = pushLFSObjectOnce(t, ctx, lfsUnderBytes)
|
||||
assert.NoError(t, err, "LFS push under global LFS limit must be accepted")
|
||||
},
|
||||
func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Per-repo: enable checking globally but do not set global limits.
|
||||
setting.SaveGlobalRepositorySetting(true, 0, 0, false)
|
||||
|
||||
ctx := newLimitRepo(t, "rlfs")
|
||||
|
||||
// set per-repo LFS limit
|
||||
t.Run("APISetRepoLFSSizeLimit", doAPISetRepoLFSSizeLimit(ctx, ctx.Username, ctx.Reponame, lfsLimitBytes))
|
||||
|
||||
// push on git (OK)
|
||||
err := pushGitBlob(t, ctx, gitOverBytes)
|
||||
assert.NoError(t, err, "git push should be allowed when repo limit is 0")
|
||||
|
||||
// push over on LFS (NOK)
|
||||
err = pushLFSObjectOnce(t, ctx, lfsOverBytes)
|
||||
assert.Error(t, err, "LFS push above per-repo LFS limit must be rejected")
|
||||
},
|
||||
)
|
||||
|
||||
runGlobalThenPerRepo(t,
|
||||
"GlobalOrCombinedRepoAndLFSLimits_BlocksLFSandGit",
|
||||
func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Global: repo limit ON, no LFS-specific limit, but LFS counted into repo size.
|
||||
setting.SaveGlobalRepositorySetting(true, combinedRepoLimitBytes, 0, true)
|
||||
|
||||
ctx := newLimitRepo(t, "gc")
|
||||
|
||||
// push over via LFS (NOK)
|
||||
err := pushLFSObjectOnce(t, ctx, combinedLFSOverBytes)
|
||||
assert.Error(t, err, "LFS push must be rejected when it would exceed repo limit and LFSSizeInRepoSize=true")
|
||||
|
||||
// push under via LFS (OK)
|
||||
err = pushLFSObjectOnce(t, ctx, combinedLFSUnderBytes)
|
||||
assert.NoError(t, err, "LFS push under repo limit must be accepted when LFSSizeInRepoSize=true")
|
||||
},
|
||||
func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Per-repo: enable checking globally with LFSSizeInRepoSize=true but no global limits.
|
||||
setting.SaveGlobalRepositorySetting(true, 0, 0, true)
|
||||
|
||||
ctx := newLimitRepo(t, "rc")
|
||||
|
||||
// set per-repo repo limit
|
||||
t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(ctx, ctx.Username, ctx.Reponame, combinedRepoLimitBytes))
|
||||
|
||||
// push over via LFS (NOK)
|
||||
err := pushLFSObjectOnce(t, ctx, combinedLFSOverBytes)
|
||||
assert.Error(t, err, "LFS push must be rejected when it would exceed per-repo repo limit and LFSSizeInRepoSize=true")
|
||||
|
||||
// push under via LFS (OK)
|
||||
err = pushLFSObjectOnce(t, ctx, combinedLFSUnderBytes)
|
||||
assert.NoError(t, err, "LFS push under per-repo repo limit must be accepted when LFSSizeInRepoSize=true")
|
||||
},
|
||||
)
|
||||
|
||||
t.Run("PerRepoLFSOverride_WinsOverGlobalDefault", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
// Global: strict LFS limit, no repo limit.
|
||||
setting.SaveGlobalRepositorySetting(true, 0, lfsLimitBytes, false)
|
||||
|
||||
ctx := newLimitRepo(t, "rwing")
|
||||
|
||||
// Prepare one LFS commit and keep the clone so we can retry pushing SAME commit.
|
||||
repoPath := prepareLFSPushForRetry(t, ctx, lfsOverBytes)
|
||||
|
||||
// First push must fail.
|
||||
err := pushCurrentBranch(t, repoPath)
|
||||
assert.Error(t, err, "push must fail under strict global LFS limit")
|
||||
|
||||
// Increase per-repo LFS limit to allow the previously rejected object.
|
||||
// Use a clearly larger limit than the file size.
|
||||
newPerRepoLimit := int64(lfsOverBytes) * 4
|
||||
t.Run("APISetRepoLFSSizeLimit", doAPISetRepoLFSSizeLimit(ctx, ctx.Username, ctx.Reponame, newPerRepoLimit))
|
||||
|
||||
// Retry pushing SAME commit must succeed now.
|
||||
err = pushCurrentBranch(t, repoPath)
|
||||
assert.NoError(t, err, "per-repo LFS limit must override global default and allow previously rejected push")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user