0
0
mirror of https://github.com/go-gitea/gitea.git synced 2026-05-14 00:01:05 +02:00

added batch processing and other fixes

This commit is contained in:
karthikbhandary2 2026-03-07 16:34:32 +05:30
parent a5092e5aa7
commit 20e3fe40d7
3 changed files with 117 additions and 21 deletions

View File

@ -514,49 +514,76 @@ func DeleteOrgRepos(ctx *context.APIContext) {
// required: true // required: true
// responses: // responses:
// "202": // "202":
// description: Deletion started // "$ref": "#/responses/empty"
// "204": // "204":
// description: No repositories to delete // "$ref": "#/responses/empty"
// "403": // "403":
// "$ref": "#/responses/forbidden" // "$ref": "#/responses/forbidden"
// "404": // "404":
// "$ref": "#/responses/notFound" // "$ref": "#/responses/notFound"
org := ctx.Org.Organization org := ctx.Org.Organization
repos, err := repo_model.GetOrgRepositories(ctx, org.ID) orgID := org.ID
doer := ctx.Doer
// Check if org has any repos
count, err := db.GetEngine(ctx).Where("owner_id = ?", orgID).Count(new(repo_model.Repository))
if err != nil { if err != nil {
ctx.APIErrorInternal(err) ctx.APIErrorInternal(err)
return return
} }
if len(repos) == 0 { if count == 0 {
ctx.Status(http.StatusNoContent) ctx.Status(http.StatusNoContent)
return return
} }
doer := ctx.Doer
// Start deletion in background with detached context // Start deletion in background with detached context
go func() { go func() {
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
log.Error("Panic during org repo deletion: %v", r) desc := fmt.Sprintf("Panic during org repo deletion for org ID %d: %v", orgID, r)
if noticeErr := system_model.CreateNotice(graceful.GetManager().HammerContext(), system_model.NoticeRepository, desc); noticeErr != nil {
log.Error("Failed to create notice for panic: %v", noticeErr)
}
} }
}() }()
// Use HammerContext so deletion continues even if client disconnects // Use HammerContext so deletion continues even if client disconnects
bgCtx := graceful.GetManager().HammerContext() bgCtx := graceful.GetManager().HammerContext()
for _, repo := range repos { const batchSize = 50
if err := repo_service.DeleteRepository(bgCtx, doer, repo, true); err != nil {
desc := fmt.Sprintf("Failed to delete repository %s (ID: %d) in org %s: %v", repo.Name, repo.ID, org.Name, err) for {
repos := make([]*repo_model.Repository, 0, batchSize)
// Always fetch from offset 0 since we're deleting as we go
err := db.GetEngine(bgCtx).Where("owner_id = ?", orgID).
Limit(batchSize, 0).
Find(&repos)
if err != nil {
desc := fmt.Sprintf("Failed to fetch repositories for org ID %d: %v", orgID, err)
if noticeErr := system_model.CreateNotice(bgCtx, system_model.NoticeRepository, desc); noticeErr != nil { if noticeErr := system_model.CreateNotice(bgCtx, system_model.NoticeRepository, desc); noticeErr != nil {
log.Error("Failed to create notice for repo deletion failure: %v", noticeErr) log.Error("Failed to create notice for repo fetch failure: %v", noticeErr)
}
break
}
// exit the loop when there are no more repos to delete
if len(repos) == 0 {
break
}
for _, repo := range repos {
if err := repo_service.DeleteRepository(bgCtx, doer, repo, true); err != nil {
desc := fmt.Sprintf("Failed to delete repository %s (ID: %d) in org ID %d: %v", repo.Name, repo.ID, orgID, err)
if noticeErr := system_model.CreateNotice(bgCtx, system_model.NoticeRepository, desc); noticeErr != nil {
log.Error("Failed to create notice for repo deletion failure: %v", noticeErr)
}
} else {
log.Info("Successfully deleted repository %s (ID: %d) in org ID %d", repo.Name, repo.ID, orgID)
} }
} else {
log.Info("Successfully deleted repository %s (ID: %d) in org %s", repo.Name, repo.ID, org.Name)
} }
} }
log.Info("Completed deletion of %d repositories in org %s", len(repos), org.Name) log.Info("Completed deletion of repositories in org ID %d", orgID)
}() }()
ctx.Status(http.StatusAccepted) ctx.Status(http.StatusAccepted)

View File

@ -3549,10 +3549,10 @@
], ],
"responses": { "responses": {
"202": { "202": {
"description": "Deletion started" "$ref": "#/responses/empty"
}, },
"204": { "204": {
"description": "No repositories to delete" "$ref": "#/responses/empty"
}, },
"403": { "403": {
"$ref": "#/responses/forbidden" "$ref": "#/responses/forbidden"

View File

@ -255,9 +255,8 @@ func TestAPIOrgGeneral(t *testing.T) {
} }
func TestAPIDeleteOrgRepos(t *testing.T) { func TestAPIDeleteOrgRepos(t *testing.T) {
defer tests.PrepareTestEnv(t)()
t.Run("Delete all repos successfully", func(t *testing.T) { t.Run("Delete all repos successfully", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
// Create test org with owner // Create test org with owner
@ -284,7 +283,8 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
MakeRequest(t, req, http.StatusAccepted) MakeRequest(t, req, http.StatusAccepted)
}) })
t.Run("Verify response structure", func(t *testing.T) { t.Run("Verify delete status code", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
session := loginUser(t, "user1") session := loginUser(t, "user1")
@ -310,6 +310,7 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
}) })
t.Run("Fail without permissions", func(t *testing.T) { t.Run("Fail without permissions", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
// user2 is owner of org3 // user2 is owner of org3
@ -332,6 +333,7 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
}) })
t.Run("No system notice created on successful deletion", func(t *testing.T) { t.Run("No system notice created on successful deletion", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
session := loginUser(t, "user1") session := loginUser(t, "user1")
@ -356,8 +358,23 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/orgs/%s/repos", orgName)).AddTokenAuth(token) req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/orgs/%s/repos", orgName)).AddTokenAuth(token)
MakeRequest(t, req, http.StatusAccepted) MakeRequest(t, req, http.StatusAccepted)
// Wait for background deletion to complete // Wait for background deletion to complete (poll until done)
time.Sleep(2 * time.Second) org := unittest.AssertExistsAndLoadBean(t, &org_model.Organization{Name: orgName})
maxWait := 10 * time.Second
checkInterval := 200 * time.Millisecond
elapsed := time.Duration(0)
for elapsed < maxWait {
time.Sleep(checkInterval)
elapsed += checkInterval
remainingRepos, err := repo_model.GetOrgRepositories(t.Context(), org.ID)
assert.NoError(t, err)
if len(remainingRepos) == 0 {
break
}
}
// Check if notices were created (should be 0 for successful deletions) // Check if notices were created (should be 0 for successful deletions)
finalNotices := unittest.GetCount(t, &system_model.Notice{Type: system_model.NoticeRepository}) finalNotices := unittest.GetCount(t, &system_model.Notice{Type: system_model.NoticeRepository})
@ -365,6 +382,7 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
}) })
t.Run("Returns 204 when repos already deleted", func(t *testing.T) { t.Run("Returns 204 when repos already deleted", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
session := loginUser(t, "user1") session := loginUser(t, "user1")
@ -397,6 +415,7 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
}) })
t.Run("Returns 204 when no repos exist", func(t *testing.T) { t.Run("Returns 204 when no repos exist", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)() defer tests.PrintCurrentTest(t)()
session := loginUser(t, "user1") session := loginUser(t, "user1")
@ -412,4 +431,54 @@ func TestAPIDeleteOrgRepos(t *testing.T) {
req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/orgs/%s/repos", orgName)).AddTokenAuth(token) req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/orgs/%s/repos", orgName)).AddTokenAuth(token)
MakeRequest(t, req, http.StatusNoContent) MakeRequest(t, req, http.StatusNoContent)
}) })
t.Run("Pagination works for large org", func(t *testing.T) {
defer tests.PrepareTestEnv(t)()
defer tests.PrintCurrentTest(t)()
session := loginUser(t, "user1")
token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteOrganization, auth_model.AccessTokenScopeWriteRepository)
orgName := "test_pagination_org"
req := NewRequestWithJSON(t, "POST", "/api/v1/orgs", &api.CreateOrgOption{
UserName: orgName,
}).AddTokenAuth(token)
MakeRequest(t, req, http.StatusCreated)
// Create 75 repos to test pagination (batch size is 50)
for i := range 75 {
repoName := fmt.Sprintf("test_repo_%d", i)
req := NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/org/%s/repos", orgName), &api.CreateRepoOption{
Name: repoName,
}).AddTokenAuth(token)
MakeRequest(t, req, http.StatusCreated)
}
// Delete all repos - should return 202 Accepted
req = NewRequest(t, "DELETE", fmt.Sprintf("/api/v1/orgs/%s/repos", orgName)).AddTokenAuth(token)
MakeRequest(t, req, http.StatusAccepted)
// Wait for background deletion to complete (poll until done)
org := unittest.AssertExistsAndLoadBean(t, &org_model.Organization{Name: orgName})
maxWait := 30 * time.Second
checkInterval := 500 * time.Millisecond
elapsed := time.Duration(0)
for elapsed < maxWait {
time.Sleep(checkInterval)
elapsed += checkInterval
remainingRepos, err := repo_model.GetOrgRepositories(t.Context(), org.ID)
assert.NoError(t, err)
if len(remainingRepos) == 0 {
break
}
}
// Verify all repos were deleted
remainingRepos, err := repo_model.GetOrgRepositories(t.Context(), org.ID)
assert.NoError(t, err)
assert.Empty(t, remainingRepos, "Org is empty")
})
} }