Merge branch 'main' of https://github.com/go-gitea/gitea into workflow-webhook-api

services/actions/cleanup.go

@@ -9,14 +9,17 @@ import (
 	"time"
 
 	actions_model "code.gitea.io/gitea/models/actions"
+	"code.gitea.io/gitea/models/db"
 	actions_module "code.gitea.io/gitea/modules/actions"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/builder"
 )
 
-// Cleanup removes expired actions logs, data and artifacts
+// Cleanup removes expired actions logs, data, artifacts and used ephemeral runners
 func Cleanup(ctx context.Context) error {
 	// clean up expired artifacts
 	if err := CleanupArtifacts(ctx); err != nil {
@@ -28,6 +31,11 @@ func Cleanup(ctx context.Context) error {
 		return fmt.Errorf("cleanup logs: %w", err)
 	}
 
+	// clean up old ephemeral runners
+	if err := CleanupEphemeralRunners(ctx); err != nil {
+		return fmt.Errorf("cleanup old ephemeral runners: %w", err)
+	}
+
 	return nil
 }
 
@@ -123,3 +131,20 @@ func CleanupLogs(ctx context.Context) error {
 	log.Info("Removed %d logs", count)
 	return nil
 }
+
+// CleanupEphemeralRunners removes used ephemeral runners which are no longer able to process jobs
+func CleanupEphemeralRunners(ctx context.Context) error {
+	subQuery := builder.Select("`action_runner`.id").
+		From(builder.Select("*").From("`action_runner`"), "`action_runner`"). // mysql needs this redundant subquery
+		Join("INNER", "`action_task`", "`action_task`.`runner_id` = `action_runner`.`id`").
+		Where(builder.Eq{"`action_runner`.`ephemeral`": true}).
+		And(builder.NotIn("`action_task`.`status`", actions_model.StatusWaiting, actions_model.StatusRunning, actions_model.StatusBlocked))
+	b := builder.Delete(builder.In("id", subQuery)).From("`action_runner`")
+	res, err := db.GetEngine(ctx).Exec(b)
+	if err != nil {
+		return fmt.Errorf("find runners: %w", err)
+	}
+	affected, _ := res.RowsAffected()
+	log.Info("Removed %d runners", affected)
+	return nil
+}

services/actions/notifier.go

@@ -535,7 +535,7 @@ func (n *actionsNotifier) PushCommits(ctx context.Context, pusher *user_model.Us
 	ctx = withMethod(ctx, "PushCommits")
 
 	apiPusher := convert.ToUser(ctx, pusher, nil)
-	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL())
+	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo)
 	if err != nil {
 		log.Error("commits.ToAPIPayloadCommits failed: %v", err)
 		return
@@ -596,7 +596,7 @@ func (n *actionsNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode
 	ctx = withMethod(ctx, "SyncPushCommits")
 
 	apiPusher := convert.ToUser(ctx, pusher, nil)
-	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL())
+	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo)
 	if err != nil {
 		log.Error("commits.ToAPIPayloadCommits failed: %v", err)
 		return

services/actions/task.go

@@ -23,6 +23,26 @@ func PickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv
 		actionTask *actions_model.ActionTask
 	)
 
+	if runner.Ephemeral {
+		var task actions_model.ActionTask
+		has, err := db.GetEngine(ctx).Where("runner_id = ?", runner.ID).Get(&task)
+		// Let the runner retry the request, do not allow to proceed
+		if err != nil {
+			return nil, false, err
+		}
+		if has {
+			if task.Status == actions_model.StatusWaiting || task.Status == actions_model.StatusRunning || task.Status == actions_model.StatusBlocked {
+				return nil, false, nil
+			}
+			// task has been finished, remove it
+			_, err = db.DeleteByID[actions_model.ActionRunner](ctx, runner.ID)
+			if err != nil {
+				return nil, false, err
+			}
+			return nil, false, fmt.Errorf("runner has been removed")
+		}
+	}
+
 	if err := db.WithTx(ctx, func(ctx context.Context) error {
 		t, ok, err := actions_model.CreateTaskForRunner(ctx, runner)
 		if err != nil {

services/actions/variables.go

@@ -13,7 +13,7 @@ import (
 	secret_service "code.gitea.io/gitea/services/secrets"
 )
 
-func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data string) (*actions_model.ActionVariable, error) {
+func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data, description string) (*actions_model.ActionVariable, error) {
 	if err := secret_service.ValidateName(name); err != nil {
 		return nil, err
 	}
@@ -22,7 +22,7 @@ func CreateVariable(ctx context.Context, ownerID, repoID int64, name, data strin
 		return nil, err
 	}
 
-	v, err := actions_model.InsertVariable(ctx, ownerID, repoID, name, util.ReserveLineBreakForTextarea(data))
+	v, err := actions_model.InsertVariable(ctx, ownerID, repoID, name, util.ReserveLineBreakForTextarea(data), description)
 	if err != nil {
 		return nil, err
 	}
@@ -41,7 +41,7 @@ func UpdateVariableNameData(ctx context.Context, variable *actions_model.ActionV
 
 	variable.Data = util.ReserveLineBreakForTextarea(variable.Data)
 
-	return actions_model.UpdateVariableCols(ctx, variable, "name", "data")
+	return actions_model.UpdateVariableCols(ctx, variable, "name", "data", "description")
 }
 
 func DeleteVariableByID(ctx context.Context, variableID int64) error {

services/agit/agit.go

@@ -13,6 +13,7 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
 	"code.gitea.io/gitea/modules/setting"
@@ -56,10 +57,10 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git.
 
 		baseBranchName := opts.RefFullNames[i].ForBranchName()
 		currentTopicBranch := ""
-		if !gitRepo.IsBranchExist(baseBranchName) {
+		if !gitrepo.IsBranchExist(ctx, repo, baseBranchName) {
 			// try match refs/for/<target-branch>/<topic-branch>
 			for p, v := range baseBranchName {
-				if v == '/' && gitRepo.IsBranchExist(baseBranchName[:p]) && p != len(baseBranchName)-1 {
+				if v == '/' && gitrepo.IsBranchExist(ctx, repo, baseBranchName[:p]) && p != len(baseBranchName)-1 {
 					currentTopicBranch = baseBranchName[p+1:]
 					baseBranchName = baseBranchName[:p]
 					break

services/automerge/automerge.go

@@ -248,13 +248,13 @@ func handlePullRequestAutoMerge(pullID int64, sha string) {
 
 	switch pr.Flow {
 	case issues_model.PullRequestFlowGithub:
-		headBranchExist := headGitRepo.IsBranchExist(pr.HeadBranch)
-		if pr.HeadRepo == nil || !headBranchExist {
+		headBranchExist := pr.HeadRepo != nil && gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch)
+		if !headBranchExist {
 			log.Warn("Head branch of auto merge %-v does not exist [HeadRepoID: %d, Branch: %s]", pr, pr.HeadRepoID, pr.HeadBranch)
 			return
 		}
 	case issues_model.PullRequestFlowAGit:
-		headBranchExist := git.IsReferenceExist(ctx, baseGitRepo.Path, pr.GetGitRefName())
+		headBranchExist := gitrepo.IsReferenceExist(ctx, pr.BaseRepo, pr.GetGitRefName())
 		if !headBranchExist {
 			log.Warn("Head branch of auto merge %-v does not exist [HeadRepoID: %d, Branch(Agit): %s]", pr, pr.HeadRepoID, pr.HeadBranch)
 			return

services/context/api.go

@@ -304,14 +304,14 @@ func RepoRefForAPI(next http.Handler) http.Handler {
 		refName, _, _ := getRefNameLegacy(ctx.Base, ctx.Repo, ctx.PathParam("*"), ctx.FormTrim("ref"))
 		var err error
 
-		if ctx.Repo.GitRepo.IsBranchExist(refName) {
+		if gitrepo.IsBranchExist(ctx, ctx.Repo.Repository, refName) {
 			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName)
 			if err != nil {
 				ctx.APIErrorInternal(err)
 				return
 			}
 			ctx.Repo.CommitID = ctx.Repo.Commit.ID.String()
-		} else if ctx.Repo.GitRepo.IsTagExist(refName) {
+		} else if gitrepo.IsTagExist(ctx, ctx.Repo.Repository, refName) {
 			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refName)
 			if err != nil {
 				ctx.APIErrorInternal(err)

services/context/repo.go

@@ -814,7 +814,7 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {
 		reqPath := ctx.PathParam("*")
 		if reqPath == "" {
 			refShortName = ctx.Repo.Repository.DefaultBranch
-			if !ctx.Repo.GitRepo.IsBranchExist(refShortName) {
+			if !gitrepo.IsBranchExist(ctx, ctx.Repo.Repository, refShortName) {
 				brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 1)
 				if err == nil && len(brs) != 0 {
 					refShortName = brs[0].Name
@@ -854,7 +854,7 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {
 				return
 			}
 
-			if refType == git.RefTypeBranch && ctx.Repo.GitRepo.IsBranchExist(refShortName) {
+			if refType == git.RefTypeBranch && gitrepo.IsBranchExist(ctx, ctx.Repo.Repository, refShortName) {
 				ctx.Repo.BranchName = refShortName
 				ctx.Repo.RefFullName = git.RefNameFromBranch(refShortName)
 
@@ -864,7 +864,7 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {
 					return
 				}
 				ctx.Repo.CommitID = ctx.Repo.Commit.ID.String()
-			} else if refType == git.RefTypeTag && ctx.Repo.GitRepo.IsTagExist(refShortName) {
+			} else if refType == git.RefTypeTag && gitrepo.IsTagExist(ctx, ctx.Repo.Repository, refShortName) {
 				ctx.Repo.RefFullName = git.RefNameFromTag(refShortName)
 
 				ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refShortName)

services/contexttest/context_tests.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/cache"
+	git_module "code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/reqctx"
 	"code.gitea.io/gitea/modules/session"
@@ -30,6 +31,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func mockRequest(t *testing.T, reqPath string) *http.Request {
@@ -85,7 +87,7 @@ func MockAPIContext(t *testing.T, reqPath string) (*context.APIContext, *httptes
 	base := context.NewBaseContext(resp, req)
 	base.Data = middleware.GetContextData(req.Context())
 	base.Locale = &translation.MockLocale{}
-	ctx := &context.APIContext{Base: base}
+	ctx := &context.APIContext{Base: base, Repo: &context.Repository{}}
 	chiCtx := chi.NewRouteContext()
 	ctx.SetContextValue(chi.RouteCtxKey, chiCtx)
 	return ctx, resp
@@ -106,13 +108,13 @@ func MockPrivateContext(t *testing.T, reqPath string) (*context.PrivateContext,
 // LoadRepo load a repo into a test context.
 func LoadRepo(t *testing.T, ctx gocontext.Context, repoID int64) {
 	var doer *user_model.User
-	repo := &context.Repository{}
+	var repo *context.Repository
 	switch ctx := ctx.(type) {
 	case *context.Context:
-		ctx.Repo = repo
+		repo = ctx.Repo
 		doer = ctx.Doer
 	case *context.APIContext:
-		ctx.Repo = repo
+		repo = ctx.Repo
 		doer = ctx.Doer
 	default:
 		assert.FailNow(t, "context is not *context.Context or *context.APIContext")
@@ -140,15 +142,17 @@ func LoadRepoCommit(t *testing.T, ctx gocontext.Context) {
 	}
 
 	gitRepo, err := gitrepo.OpenRepository(ctx, repo.Repository)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	defer gitRepo.Close()
-	branch, err := gitRepo.GetHEADBranch()
-	assert.NoError(t, err)
-	assert.NotNil(t, branch)
-	if branch != nil {
-		repo.Commit, err = gitRepo.GetBranchCommit(branch.Name)
-		assert.NoError(t, err)
+
+	if repo.RefFullName == "" {
+		repo.RefFullName = git_module.RefNameFromBranch(repo.Repository.DefaultBranch)
 	}
+	if repo.RefFullName.IsPull() {
+		repo.BranchName = repo.RefFullName.ShortName()
+	}
+	repo.Commit, err = gitRepo.GetCommit(repo.RefFullName.String())
+	require.NoError(t, err)
 }
 
 // LoadUser load a user into a test context

services/doctor/misc.go

@@ -18,7 +18,6 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
@@ -50,14 +49,14 @@ func checkScriptType(ctx context.Context, logger log.Logger, autofix bool) error
 
 func checkHooks(ctx context.Context, logger log.Logger, autofix bool) error {
 	if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
-		results, err := repository.CheckDelegateHooks(repo.RepoPath())
+		results, err := gitrepo.CheckDelegateHooksForRepo(ctx, repo)
 		if err != nil {
 			logger.Critical("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err)
 			return fmt.Errorf("Unable to check delegate hooks for repo %-v. ERROR: %w", repo, err)
 		}
 		if len(results) > 0 && autofix {
 			logger.Warn("Regenerated hooks for %s", repo.FullName())
-			if err := repository.CreateDelegateHooks(repo.RepoPath()); err != nil {
+			if err := gitrepo.CreateDelegateHooksForRepo(ctx, repo); err != nil {
 				logger.Critical("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err)
 				return fmt.Errorf("Unable to recreate delegate hooks for %-v. ERROR: %w", repo, err)
 			}

services/forms/user_form.go

@@ -323,8 +323,9 @@ func (f *AddKeyForm) Validate(req *http.Request, errs binding.Errors) binding.Er
 
 // AddSecretForm for adding secrets
 type AddSecretForm struct {
-	Name string `binding:"Required;MaxSize(255)"`
-	Data string `binding:"Required;MaxSize(65535)"`
+	Name        string `binding:"Required;MaxSize(255)"`
+	Data        string `binding:"Required;MaxSize(65535)"`
+	Description string `binding:"MaxSize(65535)"`
 }
 
 // Validate validates the fields
@@ -334,8 +335,9 @@ func (f *AddSecretForm) Validate(req *http.Request, errs binding.Errors) binding
 }
 
 type EditVariableForm struct {
-	Name string `binding:"Required;MaxSize(255)"`
-	Data string `binding:"Required;MaxSize(65535)"`
+	Name        string `binding:"Required;MaxSize(255)"`
+	Data        string `binding:"Required;MaxSize(65535)"`
+	Description string `binding:"MaxSize(65535)"`
 }
 
 func (f *EditVariableForm) Validate(req *http.Request, errs binding.Errors) binding.Errors {

services/gitdiff/gitdiff.go

@@ -1253,6 +1253,8 @@ func GetDiffForRender(ctx context.Context, gitRepo *git.Repository, opts *DiffOp
 				if language.Has() {
 					diffFile.Language = language.Value()
 				}
+			} else {
+				checker = nil // CheckPath fails, it's not impossible to "check" anymore
 			}
 		}
 

services/mirror/mirror_pull.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	giturl "code.gitea.io/gitea/modules/git/url"
 	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/globallock"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/process"
@@ -425,6 +426,10 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
 	return parseRemoteUpdateOutput(output, m.GetRemoteName()), true
 }
 
+func getRepoPullMirrorLockKey(repoID int64) string {
+	return fmt.Sprintf("repo_pull_mirror_%d", repoID)
+}
+
 // SyncPullMirror starts the sync of the pull mirror and schedules the next run.
 func SyncPullMirror(ctx context.Context, repoID int64) bool {
 	log.Trace("SyncMirrors [repo_id: %v]", repoID)
@@ -437,6 +442,13 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
 		log.Error("PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2))
 	}()
 
+	releaser, err := globallock.Lock(ctx, getRepoPullMirrorLockKey(repoID))
+	if err != nil {
+		log.Error("globallock.Lock(): %v", err)
+		return false
+	}
+	defer releaser()
+
 	m, err := repo_model.GetMirrorByRepoID(ctx, repoID)
 	if err != nil {
 		log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err)

services/pull/commit_status.go

@@ -10,7 +10,6 @@ import (
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
 	issues_model "code.gitea.io/gitea/models/issues"
-	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/structs"
@@ -131,10 +130,10 @@ func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullR
 	}
 	defer closer.Close()
 
-	if pr.Flow == issues_model.PullRequestFlowGithub && !headGitRepo.IsBranchExist(pr.HeadBranch) {
+	if pr.Flow == issues_model.PullRequestFlowGithub && !gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) {
 		return "", errors.New("Head branch does not exist, can not merge")
 	}
-	if pr.Flow == issues_model.PullRequestFlowAGit && !git.IsReferenceExist(ctx, headGitRepo.Path, pr.GetGitRefName()) {
+	if pr.Flow == issues_model.PullRequestFlowAGit && !gitrepo.IsReferenceExist(ctx, pr.HeadRepo, pr.GetGitRefName()) {
 		return "", errors.New("Head branch does not exist, can not merge")
 	}
 

services/pull/protected_branch.go

@@ -8,7 +8,7 @@ import (
 
 	git_model "code.gitea.io/gitea/models/git"
 	repo_model "code.gitea.io/gitea/models/repo"
-	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
 )
 
 func CreateOrUpdateProtectedBranch(ctx context.Context, repo *repo_model.Repository,
@@ -22,7 +22,8 @@ func CreateOrUpdateProtectedBranch(ctx context.Context, repo *repo_model.Reposit
 	isPlainRule := !git_model.IsRuleNameSpecial(protectBranch.RuleName)
 	var isBranchExist bool
 	if isPlainRule {
-		isBranchExist = git.IsBranchExist(ctx, repo.RepoPath(), protectBranch.RuleName)
+		// TODO: read the database directly to check if the branch exists
+		isBranchExist = gitrepo.IsBranchExist(ctx, repo, protectBranch.RuleName)
 	}
 
 	if isBranchExist {

services/pull/pull.go

@@ -491,7 +491,7 @@ func AddTestPullRequestTask(opts TestPullRequestOptions) {
 		for _, pr := range prs {
 			divergence, err := GetDiverging(ctx, pr)
 			if err != nil {
-				if git_model.IsErrBranchNotExist(err) && !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) {
+				if git_model.IsErrBranchNotExist(err) && !gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) {
 					log.Warn("Cannot test PR %s/%d: head_branch %s no longer exists", pr.BaseRepo.Name, pr.IssueID, pr.HeadBranch)
 				} else {
 					log.Error("GetDiverging: %v", err)

services/pull/temp_repo.go

@@ -15,6 +15,7 @@ import (
 	issues_model "code.gitea.io/gitea/models/issues"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/log"
 	repo_module "code.gitea.io/gitea/modules/repository"
 )
@@ -181,7 +182,7 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest)
 	if err := git.NewCommand("fetch").AddArguments(fetchArgs...).AddDynamicArguments(remoteRepoName, headBranch+":"+trackingBranch).
 		Run(ctx, prCtx.RunOpts()); err != nil {
 		cancel()
-		if !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) {
+		if !gitrepo.IsBranchExist(ctx, pr.HeadRepo, pr.HeadBranch) {
 			return nil, nil, git_model.ErrBranchNotExist{
 				BranchName: pr.HeadBranch,
 			}

services/release/release.go

@@ -77,7 +77,7 @@ func createTag(ctx context.Context, gitRepo *git.Repository, rel *repo_model.Rel
 	var created bool
 	// Only actual create when publish.
 	if !rel.IsDraft {
-		if !gitRepo.IsTagExist(rel.TagName) {
+		if !gitrepo.IsTagExist(ctx, rel.Repo, rel.TagName) {
 			if err := rel.LoadAttributes(ctx); err != nil {
 				log.Error("LoadAttributes: %v", err)
 				return false, err

services/repository/adopt.go

@@ -115,7 +115,7 @@ func adoptRepository(ctx context.Context, repo *repo_model.Repository, defaultBr
 		return fmt.Errorf("adoptRepository: path does not already exist: %s", repo.FullName())
 	}
 
-	if err := repo_module.CreateDelegateHooks(repo.RepoPath()); err != nil {
+	if err := gitrepo.CreateDelegateHooksForRepo(ctx, repo); err != nil {
 		return fmt.Errorf("createDelegateHooks: %w", err)
 	}
 

services/repository/branch.go

@@ -410,11 +410,11 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_m
 		return "target_exist", nil
 	}
 
-	if gitRepo.IsBranchExist(to) {
+	if gitrepo.IsBranchExist(ctx, repo, to) {
 		return "target_exist", nil
 	}
 
-	if !gitRepo.IsBranchExist(from) {
+	if !gitrepo.IsBranchExist(ctx, repo, from) {
 		return "from_not_exist", nil
 	}
 
@@ -618,12 +618,12 @@ func AddAllRepoBranchesToSyncQueue(ctx context.Context) error {
 	return nil
 }
 
-func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, newBranchName string) error {
+func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newBranchName string) error {
 	if repo.DefaultBranch == newBranchName {
 		return nil
 	}
 
-	if !gitRepo.IsBranchExist(newBranchName) {
+	if !gitrepo.IsBranchExist(ctx, repo, newBranchName) {
 		return git_model.ErrBranchNotExist{
 			BranchName: newBranchName,
 		}

services/repository/create.go

@@ -277,7 +277,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt
 		}
 
 		if err = initRepository(ctx, doer, repo, opts); err != nil {
-			if err2 := util.RemoveAll(repo.RepoPath()); err2 != nil {
+			if err2 := gitrepo.DeleteRepository(ctx, repo); err2 != nil {
 				log.Error("initRepository: %v", err)
 				return fmt.Errorf(
 					"delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2)

services/repository/delete.go

@@ -23,6 +23,7 @@ import (
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/models/webhook"
 	actions_module "code.gitea.io/gitea/modules/actions"
+	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
@@ -289,8 +290,13 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID
 	// we delete the file but the database rollback, the repository will be broken.
 
 	// Remove repository files.
-	repoPath := repo.RepoPath()
-	system_model.RemoveAllWithNotice(ctx, "Delete repository files", repoPath)
+	if err := gitrepo.DeleteRepository(ctx, repo); err != nil {
+		desc := fmt.Sprintf("Delete repository files [%s]: %v", repo.FullName(), err)
+		// Note we use the db.DefaultContext here rather than passing in a context as the context may be cancelled
+		if err = system_model.CreateNotice(db.DefaultContext, system_model.NoticeRepository, desc); err != nil {
+			log.Error("CreateRepositoryNotice: %v", err)
+		}
+	}
 
 	// Remove wiki files
 	if repo.HasWiki() {

services/repository/files/tree.go

@@ -7,9 +7,13 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"path"
+	"sort"
+	"strings"
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
@@ -118,3 +122,98 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git
 	}
 	return tree, nil
 }
+
+func entryModeString(entryMode git.EntryMode) string {
+	switch entryMode {
+	case git.EntryModeBlob:
+		return "blob"
+	case git.EntryModeExec:
+		return "exec"
+	case git.EntryModeSymlink:
+		return "symlink"
+	case git.EntryModeCommit:
+		return "commit" // submodule
+	case git.EntryModeTree:
+		return "tree"
+	}
+	return "unknown"
+}
+
+type TreeViewNode struct {
+	EntryName    string          `json:"entryName"`
+	EntryMode    string          `json:"entryMode"`
+	FullPath     string          `json:"fullPath"`
+	SubmoduleURL string          `json:"submoduleUrl,omitempty"`
+	Children     []*TreeViewNode `json:"children,omitempty"`
+}
+
+func (node *TreeViewNode) sortLevel() int {
+	return util.Iif(node.EntryMode == "tree" || node.EntryMode == "commit", 0, 1)
+}
+
+func newTreeViewNodeFromEntry(ctx context.Context, commit *git.Commit, parentDir string, entry *git.TreeEntry) *TreeViewNode {
+	node := &TreeViewNode{
+		EntryName: entry.Name(),
+		EntryMode: entryModeString(entry.Mode()),
+		FullPath:  path.Join(parentDir, entry.Name()),
+	}
+
+	if node.EntryMode == "commit" {
+		if subModule, err := commit.GetSubModule(node.FullPath); err != nil {
+			log.Error("GetSubModule: %v", err)
+		} else if subModule != nil {
+			submoduleFile := git.NewCommitSubmoduleFile(subModule.URL, entry.ID.String())
+			webLink := submoduleFile.SubmoduleWebLink(ctx)
+			node.SubmoduleURL = webLink.CommitWebLink
+		}
+	}
+
+	return node
+}
+
+// sortTreeViewNodes list directory first and with alpha sequence
+func sortTreeViewNodes(nodes []*TreeViewNode) {
+	sort.Slice(nodes, func(i, j int) bool {
+		a, b := nodes[i].sortLevel(), nodes[j].sortLevel()
+		if a != b {
+			return a < b
+		}
+		return nodes[i].EntryName < nodes[j].EntryName
+	})
+}
+
+func listTreeNodes(ctx context.Context, commit *git.Commit, tree *git.Tree, treePath, subPath string) ([]*TreeViewNode, error) {
+	entries, err := tree.ListEntries()
+	if err != nil {
+		return nil, err
+	}
+
+	subPathDirName, subPathRemaining, _ := strings.Cut(subPath, "/")
+	nodes := make([]*TreeViewNode, 0, len(entries))
+	for _, entry := range entries {
+		node := newTreeViewNodeFromEntry(ctx, commit, treePath, entry)
+		nodes = append(nodes, node)
+		if entry.IsDir() && subPathDirName == entry.Name() {
+			subTreePath := treePath + "/" + node.EntryName
+			if subTreePath[0] == '/' {
+				subTreePath = subTreePath[1:]
+			}
+			subNodes, err := listTreeNodes(ctx, commit, entry.Tree(), subTreePath, subPathRemaining)
+			if err != nil {
+				log.Error("listTreeNodes: %v", err)
+			} else {
+				node.Children = subNodes
+			}
+		}
+	}
+	sortTreeViewNodes(nodes)
+	return nodes, nil
+}
+
+func GetTreeViewNodes(ctx context.Context, commit *git.Commit, treePath, subPath string) ([]*TreeViewNode, error) {
+	entry, err := commit.GetTreeEntryByPath(treePath)
+	if err != nil {
+		return nil, err
+	}
+	return listTreeNodes(ctx, commit, entry.Tree(), treePath, subPath)
+}

services/repository/files/tree_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/git"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/services/contexttest"
 
@@ -50,3 +51,51 @@ func TestGetTreeBySHA(t *testing.T) {
 
 	assert.EqualValues(t, expectedTree, tree)
 }
+
+func TestGetTreeViewNodes(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+	ctx, _ := contexttest.MockContext(t, "user2/repo1")
+	ctx.Repo.RefFullName = git.RefNameFromBranch("sub-home-md-img-check")
+	contexttest.LoadRepo(t, ctx, 1)
+	contexttest.LoadRepoCommit(t, ctx)
+	contexttest.LoadUser(t, ctx, 2)
+	contexttest.LoadGitRepo(t, ctx)
+	defer ctx.Repo.GitRepo.Close()
+
+	treeNodes, err := GetTreeViewNodes(ctx, ctx.Repo.Commit, "", "")
+	assert.NoError(t, err)
+	assert.Equal(t, []*TreeViewNode{
+		{
+			EntryName: "docs",
+			EntryMode: "tree",
+			FullPath:  "docs",
+		},
+	}, treeNodes)
+
+	treeNodes, err = GetTreeViewNodes(ctx, ctx.Repo.Commit, "", "docs/README.md")
+	assert.NoError(t, err)
+	assert.Equal(t, []*TreeViewNode{
+		{
+			EntryName: "docs",
+			EntryMode: "tree",
+			FullPath:  "docs",
+			Children: []*TreeViewNode{
+				{
+					EntryName: "README.md",
+					EntryMode: "blob",
+					FullPath:  "docs/README.md",
+				},
+			},
+		},
+	}, treeNodes)
+
+	treeNodes, err = GetTreeViewNodes(ctx, ctx.Repo.Commit, "docs", "README.md")
+	assert.NoError(t, err)
+	assert.Equal(t, []*TreeViewNode{
+		{
+			EntryName: "README.md",
+			EntryMode: "blob",
+			FullPath:  "docs/README.md",
+		},
+	}, treeNodes)
+}

services/repository/fork.go

@@ -116,7 +116,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
 
 		// As the transaction will be failed and hence database changes will be destroyed we only need
 		// to delete the related repository on the filesystem
-		if errDelete := util.RemoveAll(repo.RepoPath()); errDelete != nil {
+		if errDelete := gitrepo.DeleteRepository(ctx, repo); errDelete != nil {
 			log.Error("Failed to remove fork repo")
 		}
 	}
@@ -154,8 +154,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
 		if opts.SingleBranch != "" {
 			cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch)
 		}
-		repoPath := repo_model.RepoPath(owner.Name, repo.Name)
-		if stdout, _, err := cloneCmd.AddDynamicArguments(oldRepoPath, repoPath).
+		if stdout, _, err := cloneCmd.AddDynamicArguments(oldRepoPath, repo.RepoPath()).
 			RunStdBytes(txCtx, &git.RunOpts{Timeout: 10 * time.Minute}); err != nil {
 			log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err)
 			return fmt.Errorf("git clone: %w", err)
@@ -166,12 +165,12 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
 		}
 
 		if stdout, _, err := git.NewCommand("update-server-info").
-			RunStdString(txCtx, &git.RunOpts{Dir: repoPath}); err != nil {
+			RunStdString(txCtx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil {
 			log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err)
 			return fmt.Errorf("git update-server-info: %w", err)
 		}
 
-		if err = repo_module.CreateDelegateHooks(repoPath); err != nil {
+		if err = gitrepo.CreateDelegateHooksForRepo(ctx, repo); err != nil {
 			return fmt.Errorf("createDelegateHooks: %w", err)
 		}
 

services/repository/hooks.go

@@ -12,7 +12,6 @@ import (
 	"code.gitea.io/gitea/models/webhook"
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/log"
-	repo_module "code.gitea.io/gitea/modules/repository"
 
 	"xorm.io/builder"
 )
@@ -32,11 +31,11 @@ func SyncRepositoryHooks(ctx context.Context) error {
 			default:
 			}
 
-			if err := repo_module.CreateDelegateHooks(repo.RepoPath()); err != nil {
+			if err := gitrepo.CreateDelegateHooksForRepo(ctx, repo); err != nil {
 				return fmt.Errorf("SyncRepositoryHook: %w", err)
 			}
 			if repo.HasWiki() {
-				if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil {
+				if err := gitrepo.CreateDelegateHooksForWiki(ctx, repo); err != nil {
 					return fmt.Errorf("SyncRepositoryHook: %w", err)
 				}
 			}

services/repository/migrate.go

@@ -15,6 +15,7 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migration"
@@ -264,17 +265,16 @@ func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error {
 
 // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
 func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo_model.Repository, error) {
-	repoPath := repo.RepoPath()
-	if err := repo_module.CreateDelegateHooks(repoPath); err != nil {
+	if err := gitrepo.CreateDelegateHooksForRepo(ctx, repo); err != nil {
 		return repo, fmt.Errorf("createDelegateHooks: %w", err)
 	}
 	if repo.HasWiki() {
-		if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil {
+		if err := gitrepo.CreateDelegateHooksForWiki(ctx, repo); err != nil {
 			return repo, fmt.Errorf("createDelegateHooks.(wiki): %w", err)
 		}
 	}
 
-	_, _, err := git.NewCommand("remote", "rm", "origin").RunStdString(ctx, &git.RunOpts{Dir: repoPath})
+	_, _, err := git.NewCommand("remote", "rm", "origin").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()})
 	if err != nil && !git.IsRemoteNotExistError(err) {
 		return repo, fmt.Errorf("CleanUpMigrateInfo: %w", err)
 	}

services/repository/transfer.go

@@ -17,6 +17,7 @@ import (
 	project_model "code.gitea.io/gitea/models/project"
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/globallock"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/util"
@@ -335,8 +336,7 @@ func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newR
 		}
 	}
 
-	newRepoPath := repo_model.RepoPath(repo.Owner.Name, newRepoName)
-	if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil {
+	if err = gitrepo.RenameRepository(ctx, repo, newRepoName); err != nil {
 		return fmt.Errorf("rename repository directory: %w", err)
 	}
 

services/secrets/secrets.go

@@ -10,7 +10,7 @@ import (
 	secret_model "code.gitea.io/gitea/models/secret"
 )
 
-func CreateOrUpdateSecret(ctx context.Context, ownerID, repoID int64, name, data string) (*secret_model.Secret, bool, error) {
+func CreateOrUpdateSecret(ctx context.Context, ownerID, repoID int64, name, data, description string) (*secret_model.Secret, bool, error) {
 	if err := ValidateName(name); err != nil {
 		return nil, false, err
 	}
@@ -25,14 +25,14 @@ func CreateOrUpdateSecret(ctx context.Context, ownerID, repoID int64, name, data
 	}
 
 	if len(s) == 0 {
-		s, err := secret_model.InsertEncryptedSecret(ctx, ownerID, repoID, name, data)
+		s, err := secret_model.InsertEncryptedSecret(ctx, ownerID, repoID, name, data, description)
 		if err != nil {
 			return nil, false, err
 		}
 		return s, true, nil
 	}
 
-	if err := secret_model.UpdateSecret(ctx, s[0].ID, data); err != nil {
+	if err := secret_model.UpdateSecret(ctx, s[0].ID, data, description); err != nil {
 		return nil, false, err
 	}
 

services/webhook/notifier.go

@@ -604,7 +604,7 @@ func (m *webhookNotifier) IssueChangeMilestone(ctx context.Context, doer *user_m
 
 func (m *webhookNotifier) PushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) {
 	apiPusher := convert.ToUser(ctx, pusher, nil)
-	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL())
+	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo)
 	if err != nil {
 		log.Error("commits.ToAPIPayloadCommits failed: %v", err)
 		return
@@ -843,7 +843,7 @@ func (m *webhookNotifier) DeleteRelease(ctx context.Context, doer *user_model.Us
 
 func (m *webhookNotifier) SyncPushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) {
 	apiPusher := convert.ToUser(ctx, pusher, nil)
-	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo.RepoPath(), repo.HTMLURL())
+	apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo)
 	if err != nil {
 		log.Error("commits.ToAPIPayloadCommits failed: %v", err)
 		return
@@ -867,7 +867,7 @@ func (m *webhookNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode
 
 func (m *webhookNotifier) CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, commit *repository.PushCommit, sender *user_model.User, status *git_model.CommitStatus) {
 	apiSender := convert.ToUser(ctx, sender, nil)
-	apiCommit, err := repository.ToAPIPayloadCommit(ctx, map[string]*user_model.User{}, repo.RepoPath(), repo.HTMLURL(), commit)
+	apiCommit, err := repository.ToAPIPayloadCommit(ctx, map[string]*user_model.User{}, repo, commit)
 	if err != nil {
 		log.Error("commits.ToAPIPayloadCommits failed: %v", err)
 		return

services/wiki/wiki.go

@@ -41,7 +41,7 @@ func InitWiki(ctx context.Context, repo *repo_model.Repository) error {
 
 	if err := git.InitRepository(ctx, repo.WikiPath(), true, repo.ObjectFormatName); err != nil {
 		return fmt.Errorf("InitRepository: %w", err)
-	} else if err = repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil {
+	} else if err = gitrepo.CreateDelegateHooksForWiki(ctx, repo); err != nil {
 		return fmt.Errorf("createDelegateHooks: %w", err)
 	} else if _, _, err = git.NewCommand("symbolic-ref", "HEAD").AddDynamicArguments(git.BranchPrefix+repo.DefaultWikiBranch).RunStdString(ctx, &git.RunOpts{Dir: repo.WikiPath()}); err != nil {
 		return fmt.Errorf("unable to set default wiki branch to %q: %w", repo.DefaultWikiBranch, err)
@@ -100,7 +100,7 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model
 		return fmt.Errorf("InitWiki: %w", err)
 	}
 
-	hasDefaultBranch := git.IsBranchExist(ctx, repo.WikiPath(), repo.DefaultWikiBranch)
+	hasDefaultBranch := gitrepo.IsWikiBranchExist(ctx, repo, repo.DefaultWikiBranch)
 
 	basePath, err := repo_module.CreateTemporaryPath("update-wiki")
 	if err != nil {