Mirrors/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-06-12 22:14:48 +02:00
Code Issues Activity

Merge remote-tracking branch 'upstream/main' into limit-repo-size

Browse Source
This commit is contained in:
DmitryFrolovTri 2022-12-16 16:04:52 +00:00
commit 4256bdcb72
13 changed files with 356 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
MAINTAINERS
View File

@ -44,8 +44,7 @@ Janis Estelmann <admin@oldschoolhack.me> (@KN4CK3R)
Steven Kriegler <sk.bunsenbrenner@gmail.com> (@justusbunsi)
Jimmy Praet <jimmy.praet@telenet.be> (@jpraet)
Leon Hofmeister <dev.lh@web.de> (@delvh)
silentcode <silentcode@senga.org> (@silentcodeg)
Wim <wim@42.be> (@42wim)
xinyu <xinyu@nerv.org.cn> (@penlinux)
Xinyu Zhou <i@sourcehut.net> (@xin-u)
Jason Song <i@wolfogre.com> (@wolfogre)
Yarden Shoham <hrsi88@gmail.com> (@yardenshoham)

54
models/git/lfs.go
View File

@ -6,6 +6,7 @@ package git
import (
"context"
"fmt"
"time"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/perm"
@ -14,6 +15,7 @@ import (
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@ -180,6 +182,12 @@ func GetLFSMetaObjectByOid(repoID int64, oid string) (*LFSMetaObject, error) {
// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error.
func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) {
return RemoveLFSMetaObjectByOidFn(repoID, oid, nil)
}
// RemoveLFSMetaObjectByOidFn removes a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. It will run Fn with the current count within the transaction
func RemoveLFSMetaObjectByOidFn(repoID int64, oid string, fn func(count int64) error) (int64, error) {
if len(oid) == 0 {
return 0, ErrLFSObjectNotExist
}
@ -200,6 +208,12 @@ func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) {
return count, err
}
if fn != nil {
if err := fn(count); err != nil {
return count, err
}
}
return count, committer.Commit()
}
@ -319,3 +333,43 @@ func GetRepoLFSSize(ctx context.Context, repoID int64) (int64, error) {
}
return lfsSize, nil
}
type IterateLFSMetaObjectsForRepoOptions struct {
OlderThan time.Time
}
// IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo
func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject, int64) error, opts *IterateLFSMetaObjectsForRepoOptions) error {
var start int
batchSize := setting.Database.IterateBufferSize
engine := db.GetEngine(ctx)
type CountLFSMetaObject struct {
Count int64
LFSMetaObject
}
for {
beans := make([]*CountLFSMetaObject, 0, batchSize)
// SELECT `lfs_meta_object`.*, COUNT(`l1`.id) as `count` FROM lfs_meta_object INNER JOIN lfs_meta_object AS l1 ON l1.oid = lfs_meta_object.oid WHERE lfs_meta_object.repository_id = ? GROUP BY lfs_meta_object.id
sess := engine.Select("`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`").
Join("INNER", "`lfs_meta_object` AS l1", "`lfs_meta_object`.oid = `l1`.oid").
Where("`lfs_meta_object`.repository_id = ?", repoID)
if !opts.OlderThan.IsZero() {
sess.And("`lfs_meta_object`.created_unix < ?", opts.OlderThan)
}
sess.GroupBy("`lfs_meta_object`.id")
if err := sess.Limit(batchSize, start).Find(&beans); err != nil {
return err
}
if len(beans) == 0 {
return nil
}
start += len(beans)
for _, bean := range beans {
if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil {
return err
}
}
}
}

7
models/packages/package_version.go
View File

@ -302,9 +302,14 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
cond := opts.toConds().
And(builder.Expr("pv2.id IS NULL"))
joinCond := builder.Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))")
if !opts.IsInternal.IsNone() {
joinCond = joinCond.And(builder.Eq{"pv2.is_internal": opts.IsInternal.IsTrue()})
}
sess := db.GetEngine(ctx).
Table("package_version").
Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))").
Join("LEFT", "package_version pv2", joinCond).
Join("INNER", "package", "package.id = package_version.package_id").
Where(cond)

37
modules/doctor/lfs.go Normal file
View File

@ -0,0 +1,37 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/repository"
)
func init() {
Register(&Check{
Title: "Garbage collect LFS",
Name: "gc-lfs",
IsDefault: false,
Run: garbageCollectLFSCheck,
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
}
func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error {
if !setting.LFS.StartServer {
return fmt.Errorf("LFS support is disabled")
}
if err := repository.GarbageCollectLFSMetaObjects(ctx, logger, autofix); err != nil {
return err
}
return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix)
}

7
modules/setting/setting.go
View File

@ -465,6 +465,13 @@ func getAppPath() (string, error) {
appPath, err = exec.LookPath(os.Args[0])
}
if err != nil {
// FIXME: Once we switch to go 1.19 use !errors.Is(err, exec.ErrDot)
if !strings.Contains(err.Error(), "cannot run executable found relative to current directory") {
return "", err
}
appPath, err = filepath.Abs(os.Args[0])
}
if err != nil {
return "", err
}

5
routers/api/packages/npm/npm.go
View File

@ -405,8 +405,9 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo
func PackageSearch(ctx *context.Context) {
pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{
OwnerID: ctx.Package.Owner.ID,
Type: packages_model.TypeNpm,
OwnerID: ctx.Package.Owner.ID,
Type: packages_model.TypeNpm,
IsInternal: util.OptionalBoolFalse,
Name: packages_model.SearchValue{
ExactMatch: false,
Value: ctx.FormTrim("text"),

211
routers/web/repo/view.go
View File

@ -241,18 +241,19 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string)
return readmeFile, readmeTreelink
}
func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) {
ctx.Data["RawFileLink"] = ""
ctx.Data["ReadmeInList"] = true
ctx.Data["ReadmeExist"] = true
ctx.Data["FileIsSymlink"] = readmeFile.isSymlink
type fileInfo struct {
isTextFile bool
isLFSFile bool
fileSize int64
lfsMeta *lfs.Pointer
st typesniffer.SniffedType
}
dataRc, err := readmeFile.blob.DataAsync()
func getFileReader(repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) {
dataRc, err := blob.DataAsync()
if err != nil {
ctx.ServerError("Data", err)
return
return nil, nil, nil, err
}
defer dataRc.Close()
buf := make([]byte, 1024)
n, _ := util.ReadAtMost(dataRc, buf)
@ -261,67 +262,75 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin
st := typesniffer.DetectContentType(buf)
isTextFile := st.IsText()
ctx.Data["FileIsText"] = isTextFile
ctx.Data["FileName"] = readmeFile.name
fileSize := int64(0)
isLFSFile := false
ctx.Data["IsLFSFile"] = false
// FIXME: what happens when README file is an image?
if isTextFile && setting.LFS.StartServer {
pointer, _ := lfs.ReadPointerFromBuffer(buf)
if pointer.IsValid() {
meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid)
if err != nil && err != git_model.ErrLFSObjectNotExist {
ctx.ServerError("GetLFSMetaObject", err)
return
}
if meta != nil {
ctx.Data["IsLFSFile"] = true
isLFSFile = true
// OK read the lfs object
var err error
dataRc, err = lfs.ReadMetaObject(pointer)
if err != nil {
ctx.ServerError("ReadMetaObject", err)
return
}
defer dataRc.Close()
buf = make([]byte, 1024)
n, err = util.ReadAtMost(dataRc, buf)
if err != nil {
ctx.ServerError("Data", err)
return
}
buf = buf[:n]
st = typesniffer.DetectContentType(buf)
isTextFile = st.IsText()
ctx.Data["IsTextFile"] = isTextFile
fileSize = meta.Size
ctx.Data["FileSize"] = meta.Size
filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(meta.Oid), url.PathEscape(filenameBase64))
}
}
if !isTextFile || !setting.LFS.StartServer {
return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil
}
if !isTextFile {
pointer, _ := lfs.ReadPointerFromBuffer(buf)
if !pointer.IsValid() { // fallback to plain file
return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil
}
meta, err := git_model.GetLFSMetaObjectByOid(repoID, pointer.Oid)
if err != git_model.ErrLFSObjectNotExist { // fallback to plain file
return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil
}
dataRc.Close()
if err != nil {
return nil, nil, nil, err
}
dataRc, err = lfs.ReadMetaObject(pointer)
if err != nil {
return nil, nil, nil, err
}
buf = make([]byte, 1024)
n, err = util.ReadAtMost(dataRc, buf)
if err != nil {
dataRc.Close()
return nil, nil, nil, err
}
buf = buf[:n]
st = typesniffer.DetectContentType(buf)
return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil
}
func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) {
ctx.Data["RawFileLink"] = ""
ctx.Data["ReadmeInList"] = true
ctx.Data["ReadmeExist"] = true
ctx.Data["FileIsSymlink"] = readmeFile.isSymlink
buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, readmeFile.blob)
if err != nil {
ctx.ServerError("getFileReader", err)
return
}
defer dataRc.Close()
ctx.Data["FileIsText"] = fInfo.isTextFile
ctx.Data["FileName"] = readmeFile.name
ctx.Data["IsLFSFile"] = fInfo.isLFSFile
if fInfo.isLFSFile {
filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(fInfo.lfsMeta.Oid), url.PathEscape(filenameBase64))
}
if !fInfo.isTextFile {
return
}
if !isLFSFile {
fileSize = readmeFile.blob.Size()
}
if fileSize >= setting.UI.MaxDisplayFileSize {
if fInfo.fileSize >= setting.UI.MaxDisplayFileSize {
// Pretend that this is a normal text file to display 'This file is too large to be shown'
ctx.Data["IsFileTooLarge"] = true
ctx.Data["IsTextFile"] = true
ctx.Data["FileSize"] = fileSize
ctx.Data["FileSize"] = fInfo.fileSize
return
}
@ -362,16 +371,14 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
ctx.Data["IsViewFile"] = true
ctx.Data["HideRepoInfo"] = true
blob := entry.Blob()
dataRc, err := blob.DataAsync()
buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, blob)
if err != nil {
ctx.ServerError("DataAsync", err)
ctx.ServerError("getFileReader", err)
return
}
defer dataRc.Close()
ctx.Data["Title"] = ctx.Tr("repo.file.title", ctx.Repo.Repository.Name+"/"+path.Base(ctx.Repo.TreePath), ctx.Repo.RefName)
fileSize := blob.Size()
ctx.Data["FileIsSymlink"] = entry.IsLink()
ctx.Data["FileName"] = blob.Name()
ctx.Data["RawFileLink"] = rawLink + "/" + util.PathEscapeSegments(ctx.Repo.TreePath)
@ -381,69 +388,27 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
ctx.Data["FileError"] = editorconfigErr
}
buf := make([]byte, 1024)
n, _ := util.ReadAtMost(dataRc, buf)
buf = buf[:n]
st := typesniffer.DetectContentType(buf)
isTextFile := st.IsText()
isLFSFile := false
isDisplayingSource := ctx.FormString("display") == "source"
isDisplayingRendered := !isDisplayingSource
// Check for LFS meta file
if isTextFile && setting.LFS.StartServer {
pointer, _ := lfs.ReadPointerFromBuffer(buf)
if pointer.IsValid() {
meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid)
if err != nil && err != git_model.ErrLFSObjectNotExist {
ctx.ServerError("GetLFSMetaObject", err)
return
}
if meta != nil {
isLFSFile = true
// OK read the lfs object
var err error
dataRc, err = lfs.ReadMetaObject(pointer)
if err != nil {
ctx.ServerError("ReadMetaObject", err)
return
}
defer dataRc.Close()
buf = make([]byte, 1024)
n, err = util.ReadAtMost(dataRc, buf)
if err != nil {
ctx.ServerError("Data", err)
return
}
buf = buf[:n]
st = typesniffer.DetectContentType(buf)
isTextFile = st.IsText()
fileSize = meta.Size
ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath)
}
}
if fInfo.isLFSFile {
ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath)
}
isRepresentableAsText := st.IsRepresentableAsText()
isRepresentableAsText := fInfo.st.IsRepresentableAsText()
if !isRepresentableAsText {
// If we can't show plain text, always try to render.
isDisplayingSource = false
isDisplayingRendered = true
}
ctx.Data["IsLFSFile"] = isLFSFile
ctx.Data["FileSize"] = fileSize
ctx.Data["IsTextFile"] = isTextFile
ctx.Data["IsLFSFile"] = fInfo.isLFSFile
ctx.Data["FileSize"] = fInfo.fileSize
ctx.Data["IsTextFile"] = fInfo.isTextFile
ctx.Data["IsRepresentableAsText"] = isRepresentableAsText
ctx.Data["IsDisplayingSource"] = isDisplayingSource
ctx.Data["IsDisplayingRendered"] = isDisplayingRendered
isTextSource := isTextFile || isDisplayingSource
isTextSource := fInfo.isTextFile || isDisplayingSource
ctx.Data["IsTextSource"] = isTextSource
if isTextSource {
ctx.Data["CanCopyContent"] = true
@ -468,7 +433,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
// Assume file is not editable first.
if isLFSFile {
if fInfo.isLFSFile {
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files")
} else if !isRepresentableAsText {
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files")
@ -476,13 +441,13 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
switch {
case isRepresentableAsText:
if st.IsSvgImage() {
if fInfo.st.IsSvgImage() {
ctx.Data["IsImageFile"] = true
ctx.Data["CanCopyContent"] = true
ctx.Data["HasSourceRenderedToggle"] = true
}
if fileSize >= setting.UI.MaxDisplayFileSize {
if fInfo.fileSize >= setting.UI.MaxDisplayFileSize {
ctx.Data["IsFileTooLarge"] = true
break
}
@ -589,7 +554,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
ctx.Data["FileContent"] = fileContent
ctx.Data["LineEscapeStatus"] = statuses
}
if !isLFSFile {
if !fInfo.isLFSFile {
if ctx.Repo.CanEnableEditor(ctx.Doer) {
if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID {
ctx.Data["CanEditFile"] = false
@ -605,17 +570,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
}
case st.IsPDF():
case fInfo.st.IsPDF():
ctx.Data["IsPDFFile"] = true
case st.IsVideo():
case fInfo.st.IsVideo():
ctx.Data["IsVideoFile"] = true
case st.IsAudio():
case fInfo.st.IsAudio():
ctx.Data["IsAudioFile"] = true
case st.IsImage() && (setting.UI.SVG.Enabled || !st.IsSvgImage()):
case fInfo.st.IsImage() && (setting.UI.SVG.Enabled || !fInfo.st.IsSvgImage()):
ctx.Data["IsImageFile"] = true
ctx.Data["CanCopyContent"] = true
default:
if fileSize >= setting.UI.MaxDisplayFileSize {
if fInfo.fileSize >= setting.UI.MaxDisplayFileSize {
ctx.Data["IsFileTooLarge"] = true
break
}

2
services/cron/tasks_basic.go
View File

@ -63,7 +63,7 @@ func registerRepoHealthCheck() {
for _, arg := range rhcConfig.Args {
args = append(args, git.CmdArg(arg))
}
return repo_service.GitFsck(ctx, rhcConfig.Timeout, args)
return repo_service.GitFsckRepos(ctx, rhcConfig.Timeout, args)
})
}

3
services/pull/patch.go
View File

@ -53,6 +53,8 @@ var patchErrorSuffices = []string{
": patch does not apply",
": already exists in working directory",
"unrecognized input",
": No such file or directory",
": does not exist in index",
}
// TestPatch will test whether a simple patch will apply
@ -416,6 +418,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo *
scanner := bufio.NewScanner(stderrReader)
for scanner.Scan() {
line := scanner.Text()
log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line)
if strings.HasPrefix(line, prefix) {
conflict = true
filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0])

2
services/repository/adopt.go
View File

@ -337,7 +337,7 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in
}
repoNamesToCheck = append(repoNamesToCheck, name)
if len(repoNamesToCheck) > setting.Database.IterateBufferSize {
if len(repoNamesToCheck) >= setting.Database.IterateBufferSize {
if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil {
return err
}

86
services/repository/check.go
View File

@ -22,8 +22,8 @@ import (
"xorm.io/builder"
)
// GitFsck calls 'git fsck' to check repository health.
func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) error {
// GitFsckRepos calls 'git fsck' to check repository health.
func GitFsckRepos(ctx context.Context, timeout time.Duration, args []git.CmdArg) error {
log.Trace("Doing: GitFsck")
if err := db.Iterate(
@ -35,15 +35,7 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro
return db.ErrCancelledf("before fsck of %s", repo.FullName())
default:
}
log.Trace("Running health check on repository %v", repo)
repoPath := repo.RepoPath()
if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil {
log.Warn("Failed to health check repository (%v): %v", repo, err)
if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
}
return nil
return GitFsckRepo(ctx, repo, timeout, args)
},
); err != nil {
log.Trace("Error: GitFsck: %v", err)
@ -54,6 +46,19 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro
return nil
}
// GitFsckRepo calls 'git fsck' to check an individual repository's health.
func GitFsckRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error {
log.Trace("Running health check on repository %-v", repo)
repoPath := repo.RepoPath()
if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil {
log.Warn("Failed to health check repository (%-v): %v", repo, err)
if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
}
return nil
}
// GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository
func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) error {
log.Trace("Doing: GitGcRepos")
@ -68,33 +73,7 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg)
return db.ErrCancelledf("before GC of %s", repo.FullName())
default:
}
log.Trace("Running git gc on %v", repo)
command := git.NewCommand(ctx, args...).
SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName()))
var stdout string
var err error
stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()})
if err != nil {
log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
if err = system_model.CreateRepositoryNotice(desc); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
}
// Now update the size of the repository
if err := repo_module.UpdateRepoSize(ctx, repo); err != nil {
log.Error("Updating size as part of garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
if err = system_model.CreateRepositoryNotice(desc); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
}
return nil
return GitGcRepo(ctx, repo, timeout, args)
},
); err != nil {
return err
@ -104,6 +83,37 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg)
return nil
}
// GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository
func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error {
log.Trace("Running git gc on %-v", repo)
command := git.NewCommand(ctx, args...).
SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName()))
var stdout string
var err error
stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()})
if err != nil {
log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
if err = system_model.CreateRepositoryNotice(desc); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
}
// Now update the size of the repository
if err := repo_module.UpdateRepoSize(ctx, repo); err != nil {
log.Error("Updating size as part of garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err)
desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err)
if err = system_model.CreateRepositoryNotice(desc); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err)
}
return nil
}
func gatherMissingRepoRecords(ctx context.Context) ([]*repo_model.Repository, error) {
repos := make([]*repo_model.Repository, 0, 10)
if err := db.Iterate(

105
services/repository/lfs.go Normal file
View File

@ -0,0 +1,105 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"context"
"fmt"
"time"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"xorm.io/builder"
)
func GarbageCollectLFSMetaObjects(ctx context.Context, logger log.Logger, autofix bool) error {
log.Trace("Doing: GarbageCollectLFSMetaObjects")
if err := db.Iterate(
ctx,
builder.And(builder.Gt{"id": 0}),
func(ctx context.Context, repo *repo_model.Repository) error {
return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, logger, autofix)
},
); err != nil {
return err
}
log.Trace("Finished: GarbageCollectLFSMetaObjects")
return nil
}
func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, logger log.Logger, autofix bool) error {
if logger != nil {
logger.Info("Checking %-v", repo)
}
total, orphaned, collected, deleted := 0, 0, 0, 0
if logger != nil {
defer func() {
if orphaned == 0 {
logger.Info("Found %d total LFSMetaObjects in %-v", total, repo)
} else if !autofix {
logger.Info("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo)
} else {
logger.Info("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted)
}
}()
}
gitRepo, err := git.OpenRepository(ctx, repo.RepoPath())
if err != nil {
log.Error("Unable to open git repository %-v: %v", repo, err)
return err
}
defer gitRepo.Close()
store := lfs.NewContentStore()
return git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error {
total++
pointerSha := git.ComputeBlobHash([]byte(metaObject.Pointer.StringContent()))
if gitRepo.IsObjectExist(pointerSha.String()) {
return nil
}
orphaned++
if !autofix {
return nil
}
// Non-existent pointer file
_, err = git_model.RemoveLFSMetaObjectByOidFn(repo.ID, metaObject.Oid, func(count int64) error {
if count > 0 {
return nil
}
if err := store.Delete(metaObject.RelativePath()); err != nil {
log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err)
}
deleted++
return nil
})
if err != nil {
return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err)
}
collected++
return nil
}, &git_model.IterateLFSMetaObjectsForRepoOptions{
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
// objects.
//
// It is likely that a week is potentially excessive but it should definitely be enough that any
// unassociated LFS object is genuinely unassociated.
OlderThan: time.Now().Add(-24 * 7 * time.Hour),
})
}

3
web_src/less/features/projects.less
View File

@ -12,7 +12,8 @@
margin: 0 .5rem !important;
padding: .5rem !important;
width: 320px;
height: 60vh;
height: calc(100vh - 450px);
min-height: 60vh;
overflow-y: scroll;
flex: 0 0 auto;
overflow: visible;