Mirrors/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-09-22 20:05:48 +02:00
Code Issues Activity

fix migrations.go conflict

Browse Source
This commit is contained in:
DmitryFrolovTri
2022-11-24 17:13:26 +00:00
parent 62d3c054c8 fc7a2d5a95
commit ee31efc6b1
161 changed files with 3460 additions and 1899 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/forms/package_form.go
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package forms
import (
"net/http"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/web/middleware"
"gitea.com/go-chi/binding"
)
type PackageCleanupRuleForm struct {
ID int64
Enabled bool
Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"`
KeepCount int `binding:"In(0,1,5,10,25,50,100)"`
KeepPattern string `binding:"RegexPattern"`
RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`
RemovePattern string `binding:"RegexPattern"`
MatchFullName bool
Action string `binding:"Required;In(save,remove)"`
}
func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors {
ctx := context.GetContext(req)
return middleware.Validate(errs, ctx.Data, f, ctx.Locale)
}
services/forms/user_form.go
+4 -7
View File
@@ -91,13 +91,10 @@ func (f *InstallForm) Validate(req *http.Request, errs binding.Errors) binding.E
// RegisterForm form for registering
type RegisterForm struct {
UserName string `binding:"Required;Username;MaxSize(40)"`
Email string `binding:"Required;MaxSize(254)"`
Password string `binding:"MaxSize(255)"`
Retype string
GRecaptchaResponse string `form:"g-recaptcha-response"`
HcaptchaResponse string `form:"h-captcha-response"`
McaptchaResponse string `form:"m-captcha-response"`
UserName string `binding:"Required;Username;MaxSize(40)"`
Email string `binding:"Required;MaxSize(254)"`
Password string `binding:"MaxSize(255)"`
Retype string
}
// Validate validates the fields
services/forms/user_form_auth_openid.go
+2 -5
View File
@@ -27,11 +27,8 @@ func (f *SignInOpenIDForm) Validate(req *http.Request, errs binding.Errors) bind
// SignUpOpenIDForm form for signin up with OpenID
type SignUpOpenIDForm struct {
UserName string `binding:"Required;Username;MaxSize(40)"`
Email string `binding:"Required;Email;MaxSize(254)"`
GRecaptchaResponse string `form:"g-recaptcha-response"`
HcaptchaResponse string `form:"h-captcha-response"`
McaptchaResponse string `form:"m-captcha-response"`
UserName string `binding:"Required;Username;MaxSize(40)"`
Email string `binding:"Required;Email;MaxSize(254)"`
}
// Validate validates the fields
services/issue/issue.go
+1 -2
View File
@@ -18,7 +18,6 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
)
// NewIssue creates new issue with labels for repository.
@@ -201,7 +200,7 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i
for _, issue := range issues {
if issue.Ref != "" {
issueRefEndNames[issue.ID] = git.RefEndName(issue.Ref)
issueRefURLs[issue.ID] = git.RefURL(repoLink, util.PathEscapeSegments(issue.Ref))
issueRefURLs[issue.ID] = git.RefURL(repoLink, issue.Ref)
}
}
return issueRefEndNames, issueRefURLs
services/migrations/dump.go
+1 -1
View File
@@ -26,7 +26,7 @@ import (
"code.gitea.io/gitea/modules/structs"
"github.com/google/uuid"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
)
var _ base.Uploader = &RepositoryDumper{}
services/migrations/restore.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
base "code.gitea.io/gitea/modules/migration"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
)
// RepositoryRestorer implements an Downloader from the local directory
services/packages/container/cleanup.go
+20 -15
View File
@@ -6,13 +6,12 @@ package container
import (
"context"
"strings"
"time"
packages_model "code.gitea.io/gitea/models/packages"
container_model "code.gitea.io/gitea/models/packages/container"
user_model "code.gitea.io/gitea/models/user"
container_module "code.gitea.io/gitea/modules/packages/container"
"code.gitea.io/gitea/modules/packages/container/oci"
"code.gitea.io/gitea/modules/util"
)
@@ -82,24 +81,30 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e
return nil
}
// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
if err != nil {
return err
func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) {
// Always skip the "latest" tag
if pv.LowerVersion == "latest" {
return true, nil
}
newOwnerName = strings.ToLower(newOwnerName)
for _, p := range ps {
if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
return err
// Check if the version is a digest (or untagged)
if oci.Digest(pv.LowerVersion).Validate() {
// Check if there is another manifest referencing this version
has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID,
Properties: map[string]string{
container_module.PropertyManifestReference: pv.LowerVersion,
},
})
if err != nil {
return false, err
}
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
return err
// Skip it if the version is referenced
if has {
return true, nil
}
}
return nil
return false, nil
}
services/packages/container/common.go
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package container
import (
"context"
"strings"
packages_model "code.gitea.io/gitea/models/packages"
user_model "code.gitea.io/gitea/models/user"
container_module "code.gitea.io/gitea/modules/packages/container"
)
// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
if err != nil {
return err
}
newOwnerName = strings.ToLower(newOwnerName)
for _, p := range ps {
if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
return err
}
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
return err
}
}
return nil
}
services/packages/packages.go
+69 -2
View File
@@ -443,13 +443,80 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
}
// Cleanup removes expired package data
func Cleanup(unused context.Context, olderThan time.Duration) error {
ctx, committer, err := db.TxContext(db.DefaultContext)
func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
ctx, committer, err := db.TxContext(taskCtx)
if err != nil {
return err
}
defer committer.Close()
err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
select {
case <-taskCtx.Done():
return db.ErrCancelledf("While processing package cleanup rules")
default:
}
if err := pcr.CompiledPattern(); err != nil {
return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
}
olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
if err != nil {
return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
}
for _, p := range packages {
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID,
IsInternal: util.OptionalBoolFalse,
Sort: packages_model.SortCreatedDesc,
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
})
if err != nil {
return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
}
for _, pv := range pvs {
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
} else if skip {
log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
continue
}
toMatch := pv.LowerVersion
if pcr.MatchFullName {
toMatch = p.LowerName + "/" + pv.LowerVersion
}
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
continue
}
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
continue
}
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
continue
}
log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
}
}
}
return nil
})
if err != nil {
return err
}
if err := container_service.Cleanup(ctx, olderThan); err != nil {
return err
}
services/webhook/deliver.go
+56 -20
View File
@@ -23,6 +23,7 @@ import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
@@ -43,7 +44,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
return
}
// There was a panic whilst delivering a hook...
log.Error("PANIC whilst trying to deliver webhook[%d] to %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
log.Error("PANIC whilst trying to deliver webhook task[%d] to webhook %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
}()
t.IsDelivered = true
@@ -52,7 +53,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
switch w.HTTPMethod {
case "":
log.Info("HTTP Method for webhook %d empty, setting to POST as default", t.ID)
log.Info("HTTP Method for webhook %s empty, setting to POST as default", w.URL)
fallthrough
case http.MethodPost:
switch w.ContentType {
@@ -78,14 +79,14 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
case http.MethodGet:
u, err := url.Parse(w.URL)
if err != nil {
return err
return fmt.Errorf("unable to deliver webhook task[%d] as cannot parse webhook url %s: %w", t.ID, w.URL, err)
}
vals := u.Query()
vals["payload"] = []string{t.PayloadContent}
u.RawQuery = vals.Encode()
req, err = http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
return fmt.Errorf("unable to deliver webhook task[%d] as unable to create HTTP request for webhook url %s: %w", t.ID, w.URL, err)
}
case http.MethodPut:
switch w.Type {
@@ -97,13 +98,13 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
url := fmt.Sprintf("%s/%s", w.URL, url.PathEscape(txnID))
req, err = http.NewRequest("PUT", url, strings.NewReader(t.PayloadContent))
if err != nil {
return err
return fmt.Errorf("unable to deliver webhook task[%d] as cannot create matrix request for webhook url %s: %w", t.ID, w.URL, err)
}
default:
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
}
default:
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
}
var signatureSHA1 string
@@ -159,6 +160,20 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
Headers: map[string]string{},
}
// OK We're now ready to attempt to deliver the task - we must double check that it
// has not been delivered in the meantime
updated, err := webhook_model.MarkTaskDelivered(ctx, t)
if err != nil {
log.Error("MarkTaskDelivered[%d]: %v", t.ID, err)
return fmt.Errorf("unable to mark task[%d] delivered in the db: %w", t.ID, err)
}
if !updated {
// This webhook task has already been attempted to be delivered or is in the process of being delivered
log.Trace("Webhook Task[%d] already delivered", t.ID)
return nil
}
// All code from this point will update the hook task
defer func() {
t.Delivered = time.Now().UnixNano()
if t.IsSucceed {
@@ -190,13 +205,14 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
}
if !w.IsActive {
log.Trace("Webhook %s in Webhook Task[%d] is not active", w.URL, t.ID)
return nil
}
resp, err := webhookHTTPClient.Do(req.WithContext(ctx))
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
return err
return fmt.Errorf("unable to deliver webhook task[%d] in %s due to error in http client: %w", t.ID, w.URL, err)
}
defer resp.Body.Close()
@@ -210,7 +226,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
p, err := io.ReadAll(resp.Body)
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)
return err
return fmt.Errorf("unable to deliver webhook task[%d] in %s as unable to read response body: %w", t.ID, w.URL, err)
}
t.ResponseInfo.Body = string(p)
return nil
@@ -272,17 +288,37 @@ func Init() error {
}
go graceful.GetManager().RunWithShutdownFns(hookQueue.Run)
tasks, err := webhook_model.FindUndeliveredHookTasks(graceful.GetManager().HammerContext())
if err != nil {
log.Error("FindUndeliveredHookTasks failed: %v", err)
return err
}
for _, task := range tasks {
if err := enqueueHookTask(task); err != nil {
log.Error("enqueueHookTask failed: %v", err)
}
}
go graceful.GetManager().RunWithShutdownContext(populateWebhookSendingQueue)
return nil
}
func populateWebhookSendingQueue(ctx context.Context) {
ctx, _, finished := process.GetManager().AddContext(ctx, "Webhook: Populate sending queue")
defer finished()
lowerID := int64(0)
for {
taskIDs, err := webhook_model.FindUndeliveredHookTaskIDs(ctx, lowerID)
if err != nil {
log.Error("Unable to populate webhook queue as FindUndeliveredHookTaskIDs failed: %v", err)
return
}
if len(taskIDs) == 0 {
return
}
lowerID = taskIDs[len(taskIDs)-1]
for _, taskID := range taskIDs {
select {
case <-ctx.Done():
log.Warn("Shutdown before Webhook Sending queue finishing being populated")
return
default:
}
if err := enqueueHookTask(taskID); err != nil {
log.Error("Unable to push HookTask[%d] to the Webhook Sending queue: %v", taskID, err)
}
}
}
}
services/webhook/deliver_test.go
+9 -1
View File
@@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/models/unittest"
webhook_model "code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"github.com/stretchr/testify/assert"
)
@@ -67,8 +68,15 @@ func TestWebhookDeliverAuthorizationHeader(t *testing.T) {
err := hook.SetHeaderAuthorization("Bearer s3cr3t-t0ken")
assert.NoError(t, err)
assert.NoError(t, webhook_model.CreateWebhook(db.DefaultContext, hook))
db.GetEngine(db.DefaultContext).NoAutoTime().DB().Logger.ShowSQL(true)
hookTask := &webhook_model.HookTask{HookID: hook.ID, EventType: webhook_model.HookEventPush}
hookTask := &webhook_model.HookTask{HookID: hook.ID, EventType: webhook_model.HookEventPush, Payloader: &api.PushPayload{}}
hookTask, err = webhook_model.CreateHookTask(db.DefaultContext, hookTask)
assert.NoError(t, err)
if !assert.NotNil(t, hookTask) {
return
}
assert.NoError(t, Deliver(context.Background(), hookTask))
select {
services/webhook/webhook.go
+16 -9
View File
@@ -116,19 +116,26 @@ func handle(data ...queue.Data) []queue.Data {
for _, taskID := range data {
task, err := webhook_model.GetHookTaskByID(ctx, taskID.(int64))
if err != nil {
log.Error("GetHookTaskByID failed: %v", err)
} else {
if err := Deliver(ctx, task); err != nil {
log.Error("webhook.Deliver failed: %v", err)
}
log.Error("GetHookTaskByID[%d] failed: %v", taskID.(int64), err)
continue
}
if task.IsDelivered {
// Already delivered in the meantime
log.Trace("Task[%d] has already been delivered", task.ID)
continue
}
if err := Deliver(ctx, task); err != nil {
log.Error("Unable to deliver webhook task[%d]: %v", task.ID, err)
}
}
return nil
}
func enqueueHookTask(task *webhook_model.HookTask) error {
err := hookQueue.PushFunc(task.ID, nil)
func enqueueHookTask(taskID int64) error {
err := hookQueue.Push(taskID)
if err != nil && err != queue.ErrAlreadyInQueue {
return err
}
@@ -205,7 +212,7 @@ func PrepareWebhook(ctx context.Context, w *webhook_model.Webhook, event webhook
return fmt.Errorf("CreateHookTask: %w", err)
}
return enqueueHookTask(task)
return enqueueHookTask(task.ID)
}
// PrepareWebhooks adds new webhooks to task queue for given payload.
@@ -265,5 +272,5 @@ func ReplayHookTask(ctx context.Context, w *webhook_model.Webhook, uuid string)
return err
}
return enqueueHookTask(task)
return enqueueHookTask(task.ID)
}