From f59ce777728cf84ccdf711491772104a0fc0dd6e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 18 Jan 2023 23:52:04 +0800 Subject: [PATCH 1/7] Display unreferenced packages total size in package admin panel (#22498) --- models/packages/package_blob.go | 13 +++++++++++-- models/packages/package_file.go | 4 ++-- options/locale/locale_en-US.ini | 1 + routers/web/admin/packages.go | 13 ++++++++++--- services/packages/packages.go | 4 ++-- templates/admin/packages/list.tmpl | 4 +++- 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/models/packages/package_blob.go b/models/packages/package_blob.go index 3b4a1ecf18..a55109af96 100644 --- a/models/packages/package_blob.go +++ b/models/packages/package_blob.go @@ -85,7 +85,16 @@ func DeleteBlobByID(ctx context.Context, blobID int64) error { } // GetTotalBlobSize returns the total blobs size in bytes -func GetTotalBlobSize() (int64, error) { - return db.GetEngine(db.DefaultContext). +func GetTotalBlobSize(ctx context.Context) (int64, error) { + return db.GetEngine(ctx). + SumInt(&PackageBlob{}, "size") +} + +// GetTotalUnreferencedBlobSize returns the total size of all unreferenced blobs in bytes +func GetTotalUnreferencedBlobSize(ctx context.Context) (int64, error) { + return db.GetEngine(ctx). + Table("package_blob"). + Join("LEFT", "package_file", "package_file.blob_id = package_blob.id"). + Where("package_file.id IS NULL"). SumInt(&PackageBlob{}, "size") } diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 7f794836dc..97e7a0d407 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -199,9 +199,9 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag return pfs, count, err } -// CalculateBlobSize sums up all blob sizes matching the search options. +// CalculateFileSize sums up all blob sizes matching the search options. // It does NOT respect the deduplication of blobs. -func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) { +func CalculateFileSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) { return db.GetEngine(ctx). Table("package_file"). Where(opts.toConds()). diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 9d1d279b0e..43a8aeb08e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2645,6 +2645,7 @@ repos.size = Size packages.package_manage_panel = Package Management packages.total_size = Total Size: %s +packages.unreferenced_size = Unreferenced Size: %s packages.owner = Owner packages.creator = Creator packages.name = Name diff --git a/routers/web/admin/packages.go b/routers/web/admin/packages.go index 7c6d1ed840..88fb47ca01 100644 --- a/routers/web/admin/packages.go +++ b/routers/web/admin/packages.go @@ -51,12 +51,18 @@ func Packages(ctx *context.Context) { return } - totalBlobSize, err := packages_model.GetTotalBlobSize() + totalBlobSize, err := packages_model.GetTotalBlobSize(ctx) if err != nil { ctx.ServerError("GetTotalBlobSize", err) return } + totalUnreferencedBlobSize, err := packages_model.GetTotalUnreferencedBlobSize(ctx) + if err != nil { + ctx.ServerError("CalculateBlobSize", err) + return + } + ctx.Data["Title"] = ctx.Tr("packages.title") ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminPackages"] = true @@ -65,8 +71,9 @@ func Packages(ctx *context.Context) { ctx.Data["AvailableTypes"] = packages_model.TypeList ctx.Data["SortType"] = sort ctx.Data["PackageDescriptors"] = pds - ctx.Data["Total"] = total - ctx.Data["TotalBlobSize"] = totalBlobSize + ctx.Data["TotalCount"] = total + ctx.Data["TotalBlobSize"] = totalBlobSize - totalUnreferencedBlobSize + ctx.Data["TotalUnreferencedBlobSize"] = totalUnreferencedBlobSize pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5) pager.AddParamString("q", query) diff --git a/services/packages/packages.go b/services/packages/packages.go index 49f5a2fac4..410e73c048 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -361,11 +361,11 @@ func checkSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p } if setting.Packages.LimitTotalOwnerSize > -1 { - totalSize, err := packages_model.CalculateBlobSize(ctx, &packages_model.PackageFileSearchOptions{ + totalSize, err := packages_model.CalculateFileSize(ctx, &packages_model.PackageFileSearchOptions{ OwnerID: owner.ID, }) if err != nil { - log.Error("CalculateBlobSize failed: %v", err) + log.Error("CalculateFileSize failed: %v", err) return err } if totalSize+uploadSize > setting.Packages.LimitTotalOwnerSize { diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index c39f5fc128..4c96d2bf10 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -4,7 +4,9 @@
{{template "base/alert" .}}

- {{.locale.Tr "admin.packages.package_manage_panel"}} ({{.locale.Tr "admin.total" .Total}}, {{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}}) + {{.locale.Tr "admin.packages.package_manage_panel"}} ({{.locale.Tr "admin.total" .TotalCount}}, + {{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}}, + {{.locale.Tr "admin.packages.unreferenced_size" (FileSize .TotalUnreferencedBlobSize)}})

From 326d29dce0cf2dcd5478f297d899f098aefaadaa Mon Sep 17 00:00:00 2001 From: Sybren <122987084+drsybren@users.noreply.github.com> Date: Wed, 18 Jan 2023 17:57:16 +0100 Subject: [PATCH 2/7] Reliable selection of admin user (#22509) When importing a repository via `gitea restore-repo`, external users will get remapped to an admin user. This admin user is obtained via `users.GetAdminUser()`, which unfortunately picks a more-or-less random admin to return. This makes it hard to predict which admin user will get assigned. This patch orders the admin by ascending ID before choosing the first one, i.e. it picks the admin with the lowest ID. Even though it would be nicer to have full control over which user is chosen, this at least gives us a predictable result. --- models/user/user.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/user/user.go b/models/user/user.go index 825223201b..a2c54a4429 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -1233,7 +1233,10 @@ func GetUserByOpenID(uri string) (*User, error) { // GetAdminUser returns the first administrator func GetAdminUser() (*User, error) { var admin User - has, err := db.GetEngine(db.DefaultContext).Where("is_admin=?", true).Get(&admin) + has, err := db.GetEngine(db.DefaultContext). + Where("is_admin=?", true). + Asc("id"). // Reliably get the admin with the lowest ID. + Get(&admin) if err != nil { return nil, err } else if !has { From 4804900ac935c5f9d4e47c0b0827be4c232bd0e5 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 19 Jan 2023 02:19:38 +0800 Subject: [PATCH 3/7] Load asciicast css async (#22502) Load asciicast css asynchronously. Related to #22448. --- web_src/js/markup/asciicast.js | 5 ++++- web_src/less/markup/asciicast.less | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/web_src/js/markup/asciicast.js b/web_src/js/markup/asciicast.js index d77c05b7aa..902cfcb731 100644 --- a/web_src/js/markup/asciicast.js +++ b/web_src/js/markup/asciicast.js @@ -2,7 +2,10 @@ export async function renderAsciinemaPlayer() { const els = document.querySelectorAll('.asciinema-player-container'); if (!els.length) return; - const player = await import(/* webpackChunkName: "asciinema-player" */'asciinema-player'); + const [player] = await Promise.all([ + import(/* webpackChunkName: "asciinema-player" */'asciinema-player'), + import(/* webpackChunkName: "asciinema-player" */'asciinema-player/dist/bundle/asciinema-player.css'), + ]); for (const el of els) { player.create(el.getAttribute('data-asciinema-player-src'), el, { diff --git a/web_src/less/markup/asciicast.less b/web_src/less/markup/asciicast.less index 468f0b4f3f..a52b2ae12e 100644 --- a/web_src/less/markup/asciicast.less +++ b/web_src/less/markup/asciicast.less @@ -1,5 +1,3 @@ -@import "../asciinema-player/dist/bundle/asciinema-player.css"; - .asciinema-player-container { width: 100%; height: auto; From aa87b3690081261379c6b93488cdd3f3f7b1ef31 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 18 Jan 2023 20:54:57 +0100 Subject: [PATCH 4/7] Fix 500 error viewing pull request when fork has pull requests disabled (#22512) Swallow error just like in #20839, for the case where there is no protected branch. Fixes #20826 for me, though I can't tell if this now covers all cases. --- services/pull/update.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/pull/update.go b/services/pull/update.go index 9e29f63c7c..ede89bcdff 100644 --- a/services/pull/update.go +++ b/services/pull/update.go @@ -118,6 +118,9 @@ func IsUserAllowedToUpdate(ctx context.Context, pull *issues_model.PullRequest, } prUnit, err := pr.BaseRepo.GetUnit(ctx, unit.TypePullRequests) if err != nil { + if repo_model.IsErrUnitTypeNotExist(err) { + return false, false, nil + } log.Error("pr.BaseRepo.GetUnit(unit.TypePullRequests): %v", err) return false, false, err } From e0a8965208f17c8f037e5489d28f1a070e2d8b47 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 18 Jan 2023 22:50:22 +0100 Subject: [PATCH 5/7] Fix invalid issue branch reference if not specified in template (#22513) When an issue template does not contain a ref, it would end up with an invalid `ref/heads/` value instead of having no branch referenced . --- routers/web/repo/issue.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index b081092c57..59ab717a1d 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -784,7 +784,8 @@ func setTemplateIfExists(ctx *context.Context, ctxDataKey string, possibleFiles } } - if !strings.HasPrefix(template.Ref, "refs/") { // Assume that the ref intended is always a branch - for tags users should use refs/tags/ + + if template.Ref != "" && !strings.HasPrefix(template.Ref, "refs/") { // Assume that the ref intended is always a branch - for tags users should use refs/tags/ template.Ref = git.BranchPrefix + template.Ref } ctx.Data["HasSelectedLabel"] = len(labelIDs) > 0 From cdf53fa4a7f3f7094eec8742529783ae7ceffc8f Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 19 Jan 2023 01:24:38 +0100 Subject: [PATCH 6/7] Fix issue not auto-closing when it includes a reference to a branch (#22514) Ensure branch prefix is stripped away for both when comparing the branch name. --- services/issue/commit.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/issue/commit.go b/services/issue/commit.go index db31fc66bb..c3d2e853bb 100644 --- a/services/issue/commit.go +++ b/services/issue/commit.go @@ -18,6 +18,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/repository" @@ -175,7 +176,8 @@ func UpdateIssuesCommit(doer *user_model.User, repo *repo_model.Repository, comm if !repo.CloseIssuesViaCommitInAnyBranch { // If the issue was specified to be in a particular branch, don't allow commits in other branches to close it if refIssue.Ref != "" { - if branchName != refIssue.Ref { + issueBranchName := strings.TrimPrefix(refIssue.Ref, git.BranchPrefix) + if branchName != issueBranchName { continue } // Otherwise, only process commits to the default branch From 151b1a9508d9a407163edbf3d726b4785afef5ce Mon Sep 17 00:00:00 2001 From: Sybren <122987084+drsybren@users.noreply.github.com> Date: Thu, 19 Jan 2023 03:14:56 +0100 Subject: [PATCH 7/7] Support importing comment types (#22510) This commit adds support for specifying comment types when importing with `gitea restore-repo`. It makes it possible to import issue changes, such as "title changed" or "assigned user changed". An earlier version of this pull request was made by Matti Ranta, in https://future.projects.blender.org/blender-migration/gitea-bf/pulls/3 There are two changes with regard to Matti's original code: 1. The comment type was an `int64` in Matti's code, and is now using a string. This makes it possible to use `comment_type: title`, which is more reliable and future-proof than an index into an internal list in the Gitea Go code. 2. Matti's code also had support for including labels, but in a way that would require knowing the database ID of the labels before the import even starts, which is impossible. This can be solved by using label names instead of IDs; for simplicity I I left that out of this PR. --- models/issues/comment.go | 9 +++++++++ models/issues/comment_test.go | 7 +++++++ modules/migration/comment.go | 2 ++ services/migrations/gitea_uploader.go | 23 +++++++++++++++++++++-- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/models/issues/comment.go b/models/issues/comment.go index 87e6b0a229..91dc128277 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -175,6 +175,15 @@ func (t CommentType) String() string { return commentStrings[t] } +func AsCommentType(typeName string) CommentType { + for index, name := range commentStrings { + if typeName == name { + return CommentType(index) + } + } + return CommentTypeUnknown +} + // RoleDescriptor defines comment tag type type RoleDescriptor int diff --git a/models/issues/comment_test.go b/models/issues/comment_test.go index 0d0570ae34..f1232729f1 100644 --- a/models/issues/comment_test.go +++ b/models/issues/comment_test.go @@ -62,3 +62,10 @@ func TestFetchCodeComments(t *testing.T) { assert.NoError(t, err) assert.Len(t, res, 1) } + +func TestAsCommentType(t *testing.T) { + assert.Equal(t, issues_model.CommentTypeUnknown, issues_model.AsCommentType("")) + assert.Equal(t, issues_model.CommentTypeUnknown, issues_model.AsCommentType("nonsense")) + assert.Equal(t, issues_model.CommentTypeComment, issues_model.AsCommentType("comment")) + assert.Equal(t, issues_model.CommentTypePRUnScheduledToAutoMerge, issues_model.AsCommentType("pull_cancel_scheduled_merge")) +} diff --git a/modules/migration/comment.go b/modules/migration/comment.go index 27ecaa830a..f994e972ed 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -17,6 +17,7 @@ type Commentable interface { type Comment struct { IssueIndex int64 `yaml:"issue_index"` Index int64 + CommentType string `yaml:"comment_type"` // see `commentStrings` in models/issues/comment.go PosterID int64 `yaml:"poster_id"` PosterName string `yaml:"poster_name"` PosterEmail string `yaml:"poster_email"` @@ -24,6 +25,7 @@ type Comment struct { Updated time.Time Content string Reactions []*Reaction + Meta map[string]interface{} `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct } // GetExternalName ExternalUserMigrated interface diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 23aa4ac2ca..f43c7378b8 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -454,15 +454,34 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { if comment.Updated.IsZero() { comment.Updated = comment.Created } - + if comment.CommentType == "" { + // if type field is missing, then assume a normal comment + comment.CommentType = issues_model.CommentTypeComment.String() + } cm := issues_model.Comment{ IssueID: issue.ID, - Type: issues_model.CommentTypeComment, + Type: issues_model.AsCommentType(comment.CommentType), Content: comment.Content, CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()), } + switch cm.Type { + case issues_model.CommentTypeAssignees: + cm.AssigneeID = comment.Meta["AssigneeID"].(int64) + if comment.Meta["RemovedAssigneeID"] != nil { + cm.RemovedAssignee = true + } + case issues_model.CommentTypeChangeTitle: + if comment.Meta["OldTitle"] != nil { + cm.OldTitle = fmt.Sprintf("%s", comment.Meta["OldTitle"]) + } + if comment.Meta["NewTitle"] != nil { + cm.NewTitle = fmt.Sprintf("%s", comment.Meta["NewTitle"]) + } + default: + } + if err := g.remapUser(comment, &cm); err != nil { return err }