Merge remote-tracking branch 'upstream/main' into limit-repo-size

modules/convert/pull_review.go

@@ -39,6 +39,7 @@ func ToPullReview(ctx context.Context, r *issues_model.Review, doer *user_model.
 		Dismissed:         r.Dismissed,
 		CodeCommentsCount: r.GetCodeCommentsCount(),
 		Submitted:         r.CreatedUnix.AsTime(),
+		Updated:           r.UpdatedUnix.AsTime(),
 		HTMLURL:           r.HTMLURL(),
 		HTMLPullURL:       r.Issue.HTMLURL(),
 	}

modules/doctor/heads.go

@@ -19,11 +19,9 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool)
 	numReposUpdated := 0
 	err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
 		numRepos++
-		runOpts := &git.RunOpts{Dir: repo.RepoPath()}
+		_, _, defaultBranchErr := git.NewCommand(ctx, "rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
 
-		_, _, defaultBranchErr := git.NewCommand(ctx, "rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(runOpts)
-
-		head, _, headErr := git.NewCommand(ctx, "symbolic-ref", "--short", "HEAD").RunStdString(runOpts)
+		head, _, headErr := git.NewCommand(ctx, "symbolic-ref", "--short", "HEAD").RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
 
 		// what we expect: default branch is valid, and HEAD points to it
 		if headErr == nil && defaultBranchErr == nil && head == repo.DefaultBranch {
@@ -49,7 +47,7 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool)
 		}
 
 		// otherwise, let's try fixing HEAD
-		err := git.NewCommand(ctx, "symbolic-ref").AddDashesAndList("HEAD", git.BranchPrefix+repo.DefaultBranch).Run(runOpts)
+		err := git.NewCommand(ctx, "symbolic-ref").AddDashesAndList("HEAD", git.BranchPrefix+repo.DefaultBranch).Run(&git.RunOpts{Dir: repo.RepoPath()})
 		if err != nil {
 			logger.Warn("Failed to fix HEAD for %s/%s: %v", repo.OwnerName, repo.Name, err)
 			return nil
@@ -65,7 +63,7 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool)
 		logger.Info("Out of %d repos, HEADs for %d are now fixed and HEADS for %d are still broken", numRepos, numReposUpdated, numDefaultBranchesBroken+numHeadsBroken-numReposUpdated)
 	} else {
 		if numHeadsBroken == 0 && numDefaultBranchesBroken == 0 {
-			logger.Info("All %d repos have their HEADs in the correct state")
+			logger.Info("All %d repos have their HEADs in the correct state", numRepos)
 		} else {
 			if numHeadsBroken == 0 && numDefaultBranchesBroken != 0 {
 				logger.Critical("Default branches are broken for %d/%d repos", numDefaultBranchesBroken, numRepos)

modules/doctor/storage.go

@@ -6,71 +6,255 @@ package doctor
 
 import (
 	"context"
+	"errors"
+	"io/fs"
+	"strings"
 
-	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/git"
+	"code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
 	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/util"
 )
 
-func checkAttachmentStorageFiles(logger log.Logger, autofix bool) error {
-	var total, garbageNum int
-	var deletePaths []string
-	if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error {
+type commonStorageCheckOptions struct {
+	storer     storage.ObjectStorage
+	isOrphaned func(path string, obj storage.Object, stat fs.FileInfo) (bool, error)
+	name       string
+}
+
+func commonCheckStorage(ctx context.Context, logger log.Logger, autofix bool, opts *commonStorageCheckOptions) error {
+	totalCount, orphanedCount := 0, 0
+	totalSize, orphanedSize := int64(0), int64(0)
+
+	var pathsToDelete []string
+	if err := opts.storer.IterateObjects(func(p string, obj storage.Object) error {
 		defer obj.Close()
 
-		total++
+		totalCount++
 		stat, err := obj.Stat()
 		if err != nil {
 			return err
 		}
-		exist, err := repo_model.ExistAttachmentsByUUID(stat.Name())
+		totalSize += stat.Size()
+
+		orphaned, err := opts.isOrphaned(p, obj, stat)
 		if err != nil {
 			return err
 		}
-		if !exist {
-			garbageNum++
+		if orphaned {
+			orphanedCount++
+			orphanedSize += stat.Size()
 			if autofix {
-				deletePaths = append(deletePaths, p)
+				pathsToDelete = append(pathsToDelete, p)
 			}
 		}
 		return nil
 	}); err != nil {
-		logger.Error("storage.Attachments.IterateObjects failed: %v", err)
+		logger.Error("Error whilst iterating %s storage: %v", opts.name, err)
 		return err
 	}
 
-	if garbageNum > 0 {
+	if orphanedCount > 0 {
 		if autofix {
 			var deletedNum int
-			for _, p := range deletePaths {
-				if err := storage.Attachments.Delete(p); err != nil {
-					log.Error("Delete attachment %s failed: %v", p, err)
+			for _, p := range pathsToDelete {
+				if err := opts.storer.Delete(p); err != nil {
+					log.Error("Error whilst deleting %s from %s storage: %v", p, opts.name, err)
 				} else {
 					deletedNum++
 				}
 			}
-			logger.Info("%d missed information attachment detected, %d deleted.", garbageNum, deletedNum)
+			logger.Info("Deleted %d/%d orphaned %s(s)", deletedNum, orphanedCount, opts.name)
 		} else {
-			logger.Warn("Checked %d attachment, %d missed information.", total, garbageNum)
+			logger.Warn("Found %d/%d (%s/%s) orphaned %s(s)", orphanedCount, totalCount, base.FileSize(orphanedSize), base.FileSize(totalSize), opts.name)
 		}
+	} else {
+		logger.Info("Found %d (%s) %s(s)", totalCount, base.FileSize(totalSize), opts.name)
 	}
 	return nil
 }
 
-func checkStorageFiles(ctx context.Context, logger log.Logger, autofix bool) error {
-	if err := storage.Init(); err != nil {
-		logger.Error("storage.Init failed: %v", err)
-		return err
+type checkStorageOptions struct {
+	All          bool
+	Attachments  bool
+	LFS          bool
+	Avatars      bool
+	RepoAvatars  bool
+	RepoArchives bool
+	Packages     bool
+}
+
+// checkStorage will return a doctor check function to check the requested storage types for "orphaned" stored object/files and optionally delete them
+func checkStorage(opts *checkStorageOptions) func(ctx context.Context, logger log.Logger, autofix bool) error {
+	return func(ctx context.Context, logger log.Logger, autofix bool) error {
+		if err := storage.Init(); err != nil {
+			logger.Error("storage.Init failed: %v", err)
+			return err
+		}
+
+		if opts.Attachments || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Attachments,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := repo.ExistAttachmentsByUUID(ctx, stat.Name())
+						return !exists, err
+					},
+					name: "attachment",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.LFS || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.LFS,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						// The oid of an LFS stored object is the name but with all the path.Separators removed
+						oid := strings.ReplaceAll(path, "/", "")
+						exists, err := git.ExistsLFSObject(ctx, oid)
+						return !exists, err
+					},
+					name: "LFS file",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.Avatars || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Avatars,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := user.ExistsWithAvatarAtStoragePath(ctx, path)
+						return !exists, err
+					},
+					name: "avatar",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.RepoAvatars || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.RepoAvatars,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := repo.ExistsWithAvatarAtStoragePath(ctx, path)
+						return !exists, err
+					},
+					name: "repo avatar",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.RepoArchives || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.RepoAvatars,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := repo.ExistsRepoArchiverWithStoragePath(ctx, path)
+						if err == nil || errors.Is(err, util.ErrInvalidArgument) {
+							// invalid arguments mean that the object is not a valid repo archiver and it should be removed
+							return !exists, nil
+						}
+						return !exists, err
+					},
+					name: "repo archive",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.Packages || opts.All {
+			if err := commonCheckStorage(ctx, logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Packages,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						key, err := packages_module.RelativePathToKey(path)
+						if err != nil {
+							// If there is an error here then the relative path does not match a valid package
+							// Therefore it is orphaned by default
+							return true, nil
+						}
+
+						exists, err := packages.ExistPackageBlobWithSHA(ctx, string(key))
+
+						return !exists, err
+					},
+					name: "package blob",
+				}); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	}
-	return checkAttachmentStorageFiles(logger, autofix)
 }
 
 func init() {
 	Register(&Check{
-		Title:                      "Check if there is garbage storage files",
+		Title:                      "Check if there are orphaned storage files",
 		Name:                       "storages",
 		IsDefault:                  false,
-		Run:                        checkStorageFiles,
+		Run:                        checkStorage(&checkStorageOptions{All: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned attachments in storage",
+		Name:                       "storage-attachments",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{Attachments: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned lfs files in storage",
+		Name:                       "storage-lfs",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{LFS: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned avatars in storage",
+		Name:                       "storage-avatars",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{Avatars: true, RepoAvatars: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned archives in storage",
+		Name:                       "storage-archives",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{RepoArchives: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned package blobs in storage",
+		Name:                       "storage-packages",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{Packages: true}),
 		AbortIfFailed:              false,
 		SkipDatabaseInitialization: false,
 		Priority:                   1,

modules/git/command.go

@@ -202,8 +202,11 @@ func (c *Command) Run(opts *RunOpts) error {
 	if opts == nil {
 		opts = &RunOpts{}
 	}
-	if opts.Timeout <= 0 {
-		opts.Timeout = defaultCommandExecutionTimeout
+
+	// We must not change the provided options
+	timeout := opts.Timeout
+	if timeout <= 0 {
+		timeout = defaultCommandExecutionTimeout
 	}
 
 	if len(opts.Dir) == 0 {
@@ -238,7 +241,7 @@ func (c *Command) Run(opts *RunOpts) error {
 	if opts.UseContextTimeout {
 		ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc)
 	} else {
-		ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, opts.Timeout, desc)
+		ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc)
 	}
 	defer finished()
 
@@ -339,9 +342,20 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS
 	}
 	stdoutBuf := &bytes.Buffer{}
 	stderrBuf := &bytes.Buffer{}
-	opts.Stdout = stdoutBuf
-	opts.Stderr = stderrBuf
-	err := c.Run(opts)
+
+	// We must not change the provided options as it could break future calls - therefore make a copy.
+	newOpts := &RunOpts{
+		Env:               opts.Env,
+		Timeout:           opts.Timeout,
+		UseContextTimeout: opts.UseContextTimeout,
+		Dir:               opts.Dir,
+		Stdout:            stdoutBuf,
+		Stderr:            stderrBuf,
+		Stdin:             opts.Stdin,
+		PipelineFunc:      opts.PipelineFunc,
+	}
+
+	err := c.Run(newOpts)
 	stderr = stderrBuf.Bytes()
 	if err != nil {
 		return nil, stderr, &runStdError{err: err, stderr: bytesToString(stderr)}

modules/git/repo_archive.go

@@ -38,6 +38,18 @@ func (a ArchiveType) String() string {
 	return "unknown"
 }
 
+func ToArchiveType(s string) ArchiveType {
+	switch s {
+	case "zip":
+		return ZIP
+	case "tar.gz":
+		return TARGZ
+	case "bundle":
+		return BUNDLE
+	}
+	return 0
+}
+
 // CreateArchive create archive content to the target path
 func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, target io.Writer, usePrefix bool, commitID string) error {
 	if format.String() == "unknown" {

modules/packages/content_store.go

@@ -7,8 +7,10 @@ package packages
 import (
 	"io"
 	"path"
+	"strings"
 
 	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/util"
 )
 
 // BlobHash256Key is the key to address a blob content
@@ -45,3 +47,13 @@ func (s *ContentStore) Delete(key BlobHash256Key) error {
 func KeyToRelativePath(key BlobHash256Key) string {
 	return path.Join(string(key)[0:2], string(key)[2:4], string(key))
 }
+
+// RelativePathToKey converts a relative path aa/bb/aabb000000... to the sha256 key aabb000000...
+func RelativePathToKey(relativePath string) (BlobHash256Key, error) {
+	parts := strings.SplitN(relativePath, "/", 3)
+	if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 || len(parts[2]) < 4 || parts[0]+parts[1] != parts[2][0:4] {
+		return "", util.ErrInvalidArgument
+	}
+
+	return BlobHash256Key(parts[2]), nil
+}

modules/repository/collaborator.go

@@ -37,7 +37,7 @@ func addCollaborator(ctx context.Context, repo *repo_model.Repository, u *user_m
 
 // AddCollaborator adds new collaboration to a repository with default access mode.
 func AddCollaborator(repo *repo_model.Repository, u *user_model.User) error {
-	return db.WithTx(func(ctx context.Context) error {
+	return db.WithTx(db.DefaultContext, func(ctx context.Context) error {
 		return addCollaborator(ctx, repo, u)
 	})
 }

modules/repository/create.go

@@ -213,7 +213,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m
 
 	var rollbackRepo *repo_model.Repository
 
-	if err := db.WithTx(func(ctx context.Context) error {
+	if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
 		if err := CreateRepositoryByExample(ctx, doer, u, repo, false); err != nil {
 			return err
 		}

modules/repository/repo.go

@@ -6,6 +6,7 @@ package repository
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -79,6 +80,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 		Timeout:       migrateTimeout,
 		SkipTLSVerify: setting.Migrations.SkipTLSVerify,
 	}); err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return repo, fmt.Errorf("Clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err)
+		}
 		return repo, fmt.Errorf("Clone: %w", err)
 	}
 
@@ -169,7 +173,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 		}
 	}
 
-	ctx, committer, err := db.TxContext()
+	ctx, committer, err := db.TxContext(db.DefaultContext)
 	if err != nil {
 		return nil, err
 	}
@@ -485,7 +489,7 @@ func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository)
 	if err != nil {
 		return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
 	}
-	err = db.WithTx(func(ctx context.Context) error {
+	err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
 		//
 		// clear out existing releases
 		//

modules/structs/pull_review.go

@@ -40,6 +40,8 @@ type PullReview struct {
 	CodeCommentsCount int             `json:"comments_count"`
 	// swagger:strfmt date-time
 	Submitted time.Time `json:"submitted_at"`
+	// swagger:strfmt date-time
+	Updated time.Time `json:"updated_at"`
 
 	HTMLURL     string `json:"html_url"`
 	HTMLPullURL string `json:"pull_request_url"`