Mirrors/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-06-24 13:51:24 +02:00
Code Issues Activity

Update for TestScript to Reduce RepoSize

Browse Source
This commit is contained in:
truecode112 2023-05-23 08:18:41 +03:00
parent 2cbdee0104
commit 08c65ffb17
4 changed files with 109 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
modules/repository/create.go
View File

@ -332,6 +332,26 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error {
return fmt.Errorf("updateSize: GetLFSMetaObjects: %w", err)
}
if setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && size+lfsSize > repo.GetActualSizeLimit() {
// return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit())
_, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push
if err != nil {
return fmt.Errorf("updateSize: Git Reflog Failed: %w", err)
}
_, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push
if err != nil {
return fmt.Errorf("updateSize: Git GC Failed: %w", err)
}
size, err = getDirectorySize(repo.RepoPath())
if err != nil {
return fmt.Errorf("updateSize: %w", err)
}
// return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit())
}
return repo_model.UpdateRepoSize(ctx, repo.ID, size+lfsSize)
}

6
routers/private/hook_pre_receive.go
View File

@ -204,6 +204,10 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
// If operation is in potential breach of size limit prepare data for analysis
if isRepoOversized {
// ctx.JSON(http.StatusForbidden, private.Response{
// UserMsg: fmt.Sprintf("oldCommitID(%s) newCommitID(%s)", oldCommitID, newCommitID),
// })
// Objects that are in newCommitID but not in oldCommitID are added
addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env})
if err != nil {
@ -246,7 +250,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
if (addedSize > removedSize) && isRepoOversized { // Check next size if we are not deleting a reference
log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit()))
ctx.JSON(http.StatusForbidden, private.Response{
UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())),
UserMsg: fmt.Sprintf("Repository size is over limitation of %s addedSize(%d) removedSize(%d) repo(%s)", base.FileSize(repo.GetActualSizeLimit()), addedSize, removedSize, repo.RepoPath()),
})
return
}

10
services/repository/push.go
View File

@ -93,9 +93,9 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
}
defer gitRepo.Close()
if err = repo_module.UpdateRepoSize(ctx, repo); err != nil {
log.Error("Failed to update size for repository: %v", err)
}
// if err = repo_module.UpdateRepoSize(ctx, repo); err != nil {
// log.Error("Failed to update size for repository: %v", err)
// }
addTags := make([]string, 0, len(optsList))
delTags := make([]string, 0, len(optsList))
@ -293,6 +293,10 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
return fmt.Errorf("UpdateRepositoryUpdatedTime: %w", err)
}
if err = repo_module.UpdateRepoSize(ctx, repo); err != nil {
log.Error("Failed to update size for repository: %v", err)
}
return nil
}

79
tests/integration/git_test.go
View File

@ -35,7 +35,7 @@ import (
)
const (
littleSize = 1024 // 1ko
littleSize = 1024 * 10 // 1ko
bigSize = 128 * 1024 * 1024 // 128Mo
)
@ -60,7 +60,8 @@ func testGit(t *testing.T, u *url.URL) {
dstPath := t.TempDir()
dstForkedPath := t.TempDir()
// dstForkedPath := t.TempDir()
dstForkedPath4Reduce := t.TempDir()
t.Run("CreateRepoInDifferentUser", doAPICreateRepository(forkedUserCtx, false))
t.Run("AddUserAsCollaborator", doAPIAddCollaborator(forkedUserCtx, httpContext.Username, perm.AccessModeRead))
@ -104,6 +105,38 @@ func testGit(t *testing.T, u *url.URL) {
defer tests.PrintCurrentTest(t)()
// TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-")
})
t.Run("ReduceRepoSize", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
u.Path = forkedUserCtx.GitPath()
u.User = url.UserPassword(forkedUserCtx.Username, userPassword)
t.Run("Clone", doGitClone(dstForkedPath4Reduce, u))
fmt.Fprintf(os.Stdout, "dstForkedPath4Reduce = %s\n", dstForkedPath4Reduce)
doCommitAndPush(t, littleSize, dstForkedPath4Reduce, "data-file-")
bigOne := doCommitAndPush(t, bigSize, dstForkedPath4Reduce, "my-file-")
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame)
assert.NoError(t, err)
orgGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce)
fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize)
t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize/2))
fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize)
//Delete Object from Oversized Repository, This should be Accepdted
doDeleteObjectAndClean(t, dstForkedPath4Reduce, bigOne)
repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame)
assert.NoError(t, err)
reducedGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce)
fmt.Fprintf(os.Stdout, "Reduced GitSize = %d\n", reducedGitRepoSize)
assert.Less(t, reducedGitRepoSize, orgGitRepoSize, "Repo size is not reduced.")
setting.SaveGlobalRepositorySetting(false, 0)
})
// TODO delete branch
// TODO delete tag
// TODO add big commit that will be over with the push
@ -325,6 +358,20 @@ func lockFileTest(t *testing.T, filename, repoPath string) {
assert.NoError(t, err)
}
func doDeleteObjectAndClean(t *testing.T, repoPath, filename string) {
var err error
err = deleteCommit(repoPath, "user4@example.com", "User Four", filename)
assert.NoError(t, err)
_, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
assert.NoError(t, err)
// _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
// assert.NoError(t, err)
// _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
// assert.NoError(t, err)
}
func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string {
name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix)
assert.NoError(t, err)
@ -341,6 +388,34 @@ func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix s
return name
}
func deleteCommit(repoPath, email, fullName, filename string) error {
var err error
globalArgs := git.AllowLFSFiltersArgs()
cmd := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename)
_, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) // Push
if err != nil {
return err
}
return git.CommitChangesWithArgs(repoPath, globalArgs, git.CommitChangesOptions{
Committer: &git.Signature{
Email: email,
Name: fullName,
When: time.Now(),
},
Author: &git.Signature{
Email: email,
Name: fullName,
When: time.Now(),
},
Message: fmt.Sprintf("Testing commit @ %v", time.Now()),
})
}
func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) {
// Generate random file
bufSize := 4 * 1024