diff --git a/modules/repository/create.go b/modules/repository/create.go index 63be5cac25..8240600be1 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -332,6 +332,26 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return fmt.Errorf("updateSize: GetLFSMetaObjects: %w", err) } + if setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && size+lfsSize > repo.GetActualSizeLimit() { + // return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit()) + + _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push + if err != nil { + return fmt.Errorf("updateSize: Git Reflog Failed: %w", err) + } + + _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push + if err != nil { + return fmt.Errorf("updateSize: Git GC Failed: %w", err) + } + + size, err = getDirectorySize(repo.RepoPath()) + if err != nil { + return fmt.Errorf("updateSize: %w", err) + } + // return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit()) + } + return repo_model.UpdateRepoSize(ctx, repo.ID, size+lfsSize) } diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 3813ce66b4..a4e1ad94f6 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -204,6 +204,10 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { // If operation is in potential breach of size limit prepare data for analysis if isRepoOversized { + // ctx.JSON(http.StatusForbidden, private.Response{ + // UserMsg: fmt.Sprintf("oldCommitID(%s) newCommitID(%s)", oldCommitID, newCommitID), + // }) + // Objects that are in newCommitID but not in oldCommitID are added addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { @@ -246,7 +250,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { if (addedSize > removedSize) && isRepoOversized { // Check next size if we are not deleting a reference log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) ctx.JSON(http.StatusForbidden, private.Response{ - UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), + UserMsg: fmt.Sprintf("Repository size is over limitation of %s addedSize(%d) removedSize(%d) repo(%s)", base.FileSize(repo.GetActualSizeLimit()), addedSize, removedSize, repo.RepoPath()), }) return } diff --git a/services/repository/push.go b/services/repository/push.go index c7ea8f336e..a6ba284bc7 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -93,9 +93,9 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } defer gitRepo.Close() - if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } + // if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + // log.Error("Failed to update size for repository: %v", err) + // } addTags := make([]string, 0, len(optsList)) delTags := make([]string, 0, len(optsList)) @@ -293,6 +293,10 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { return fmt.Errorf("UpdateRepositoryUpdatedTime: %w", err) } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + return nil } diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index cdff57f21f..d2bb383a71 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -35,7 +35,7 @@ import ( ) const ( - littleSize = 1024 // 1ko + littleSize = 1024 * 10 // 1ko bigSize = 128 * 1024 * 1024 // 128Mo ) @@ -60,7 +60,8 @@ func testGit(t *testing.T, u *url.URL) { dstPath := t.TempDir() - dstForkedPath := t.TempDir() + // dstForkedPath := t.TempDir() + dstForkedPath4Reduce := t.TempDir() t.Run("CreateRepoInDifferentUser", doAPICreateRepository(forkedUserCtx, false)) t.Run("AddUserAsCollaborator", doAPIAddCollaborator(forkedUserCtx, httpContext.Username, perm.AccessModeRead)) @@ -104,6 +105,38 @@ func testGit(t *testing.T, u *url.URL) { defer tests.PrintCurrentTest(t)() // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) + + t.Run("ReduceRepoSize", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + u.Path = forkedUserCtx.GitPath() + u.User = url.UserPassword(forkedUserCtx.Username, userPassword) + + t.Run("Clone", doGitClone(dstForkedPath4Reduce, u)) + fmt.Fprintf(os.Stdout, "dstForkedPath4Reduce = %s\n", dstForkedPath4Reduce) + doCommitAndPush(t, littleSize, dstForkedPath4Reduce, "data-file-") + bigOne := doCommitAndPush(t, bigSize, dstForkedPath4Reduce, "my-file-") + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame) + assert.NoError(t, err) + orgGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce) + fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize) + + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize/2)) + fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize) + + //Delete Object from Oversized Repository, This should be Accepdted + doDeleteObjectAndClean(t, dstForkedPath4Reduce, bigOne) + + repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame) + assert.NoError(t, err) + reducedGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce) + + fmt.Fprintf(os.Stdout, "Reduced GitSize = %d\n", reducedGitRepoSize) + + assert.Less(t, reducedGitRepoSize, orgGitRepoSize, "Repo size is not reduced.") + + setting.SaveGlobalRepositorySetting(false, 0) + }) + // TODO delete branch // TODO delete tag // TODO add big commit that will be over with the push @@ -325,6 +358,20 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } +func doDeleteObjectAndClean(t *testing.T, repoPath, filename string) { + var err error + err = deleteCommit(repoPath, "user4@example.com", "User Four", filename) + assert.NoError(t, err) + _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + assert.NoError(t, err) + + // _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + // assert.NoError(t, err) + // _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + // assert.NoError(t, err) + +} + func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) @@ -341,6 +388,34 @@ func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix s return name } +func deleteCommit(repoPath, email, fullName, filename string) error { + + var err error + globalArgs := git.AllowLFSFiltersArgs() + + cmd := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename) + + _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) // Push + if err != nil { + return err + } + + return git.CommitChangesWithArgs(repoPath, globalArgs, git.CommitChangesOptions{ + Committer: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Author: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Message: fmt.Sprintf("Testing commit @ %v", time.Now()), + }) + +} + func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { // Generate random file bufSize := 4 * 1024