Merge remote-tracking branch 'upstream/main' into limit-repo-size

This commit is contained in:
DmitryFrolovTri
2023-02-01 18:16:48 +00:00
178 changed files with 9427 additions and 538 deletions
+163
View File
@@ -0,0 +1,163 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"bufio"
"context"
"fmt"
"io"
"os"
"strings"
"time"
"code.gitea.io/gitea/models/dbfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
MaxLineSize = 64 * 1024
DBFSPrefix = "actions_log/"
timeFormat = "2006-01-02T15:04:05.0000000Z07:00"
defaultBufSize = MaxLineSize
)
func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runnerv1.LogRow) ([]int, error) {
name := DBFSPrefix + filename
f, err := dbfs.OpenFile(ctx, name, os.O_WRONLY|os.O_CREATE)
if err != nil {
return nil, fmt.Errorf("dbfs OpenFile %q: %w", name, err)
}
defer f.Close()
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return nil, fmt.Errorf("dbfs Seek %q: %w", name, err)
}
writer := bufio.NewWriterSize(f, defaultBufSize)
ns := make([]int, 0, len(rows))
for _, row := range rows {
n, err := writer.WriteString(FormatLog(row.Time.AsTime(), row.Content) + "\n")
if err != nil {
return nil, err
}
ns = append(ns, n)
}
if err := writer.Flush(); err != nil {
return nil, err
}
return ns, nil
}
func ReadLogs(ctx context.Context, inStorage bool, filename string, offset, limit int64) ([]*runnerv1.LogRow, error) {
f, err := openLogs(ctx, inStorage, filename)
if err != nil {
return nil, err
}
defer f.Close()
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return nil, fmt.Errorf("file seek: %w", err)
}
scanner := bufio.NewScanner(f)
maxLineSize := len(timeFormat) + MaxLineSize + 1
scanner.Buffer(make([]byte, maxLineSize), maxLineSize)
var rows []*runnerv1.LogRow
for scanner.Scan() && (int64(len(rows)) < limit || limit < 0) {
t, c, err := ParseLog(scanner.Text())
if err != nil {
return nil, fmt.Errorf("parse log %q: %w", scanner.Text(), err)
}
rows = append(rows, &runnerv1.LogRow{
Time: timestamppb.New(t),
Content: c,
})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("scan: %w", err)
}
return rows, nil
}
func TransferLogs(ctx context.Context, filename string) (func(), error) {
name := DBFSPrefix + filename
remove := func() {
if err := dbfs.Remove(ctx, name); err != nil {
log.Warn("dbfs remove %q: %v", name, err)
}
}
f, err := dbfs.Open(ctx, name)
if err != nil {
return nil, fmt.Errorf("dbfs open %q: %w", name, err)
}
defer f.Close()
if _, err := storage.Actions.Save(filename, f, -1); err != nil {
return nil, fmt.Errorf("storage save %q: %w", filename, err)
}
return remove, nil
}
func RemoveLogs(ctx context.Context, inStorage bool, filename string) error {
if !inStorage {
name := DBFSPrefix + filename
err := dbfs.Remove(ctx, name)
if err != nil {
return fmt.Errorf("dbfs remove %q: %w", name, err)
}
return nil
}
err := storage.Actions.Delete(filename)
if err != nil {
return fmt.Errorf("storage delete %q: %w", filename, err)
}
return nil
}
func openLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) {
if !inStorage {
name := DBFSPrefix + filename
f, err := dbfs.Open(ctx, name)
if err != nil {
return nil, fmt.Errorf("dbfs open %q: %w", name, err)
}
return f, nil
}
f, err := storage.Actions.Open(filename)
if err != nil {
return nil, fmt.Errorf("storage open %q: %w", filename, err)
}
return f, nil
}
func FormatLog(timestamp time.Time, content string) string {
// Content shouldn't contain new line, it will break log indexes, other control chars are safe.
content = strings.ReplaceAll(content, "\n", `\n`)
if len(content) > MaxLineSize {
content = content[:MaxLineSize]
}
return fmt.Sprintf("%s %s", timestamp.UTC().Format(timeFormat), content)
}
func ParseLog(in string) (time.Time, string, error) {
index := strings.IndexRune(in, ' ')
if index < 0 {
return time.Time{}, "", fmt.Errorf("invalid log: %q", in)
}
timestamp, err := time.Parse(timeFormat, in[:index])
if err != nil {
return time.Time{}, "", err
}
return timestamp, in[index+1:], nil
}
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
actions_model "code.gitea.io/gitea/models/actions"
)
const (
preStepName = "Set up job"
postStepName = "Complete job"
)
// FullSteps returns steps with "Set up job" and "Complete job"
func FullSteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep {
if len(task.Steps) == 0 {
return fullStepsOfEmptySteps(task)
}
firstStep := task.Steps[0]
var logIndex int64
preStep := &actions_model.ActionTaskStep{
Name: preStepName,
LogLength: task.LogLength,
Started: task.Started,
Status: actions_model.StatusRunning,
}
if firstStep.Status.HasRun() || firstStep.Status.IsRunning() {
preStep.LogLength = firstStep.LogIndex
preStep.Stopped = firstStep.Started
preStep.Status = actions_model.StatusSuccess
} else if task.Status.IsDone() {
preStep.Stopped = task.Stopped
preStep.Status = actions_model.StatusFailure
}
logIndex += preStep.LogLength
var lastHasRunStep *actions_model.ActionTaskStep
for _, step := range task.Steps {
if step.Status.HasRun() {
lastHasRunStep = step
}
logIndex += step.LogLength
}
if lastHasRunStep == nil {
lastHasRunStep = preStep
}
postStep := &actions_model.ActionTaskStep{
Name: postStepName,
Status: actions_model.StatusWaiting,
}
if task.Status.IsDone() {
postStep.LogIndex = logIndex
postStep.LogLength = task.LogLength - postStep.LogIndex
postStep.Status = task.Status
postStep.Started = lastHasRunStep.Stopped
postStep.Stopped = task.Stopped
}
ret := make([]*actions_model.ActionTaskStep, 0, len(task.Steps)+2)
ret = append(ret, preStep)
ret = append(ret, task.Steps...)
ret = append(ret, postStep)
return ret
}
func fullStepsOfEmptySteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep {
preStep := &actions_model.ActionTaskStep{
Name: preStepName,
LogLength: task.LogLength,
Started: task.Started,
Stopped: task.Stopped,
Status: actions_model.StatusRunning,
}
postStep := &actions_model.ActionTaskStep{
Name: postStepName,
LogIndex: task.LogLength,
Started: task.Stopped,
Stopped: task.Stopped,
Status: actions_model.StatusWaiting,
}
if task.Status.IsDone() {
preStep.Status = task.Status
if preStep.Status.IsSuccess() {
postStep.Status = actions_model.StatusSuccess
} else {
postStep.Status = actions_model.StatusCancelled
}
}
return []*actions_model.ActionTaskStep{
preStep,
postStep,
}
}
+112
View File
@@ -0,0 +1,112 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"testing"
actions_model "code.gitea.io/gitea/models/actions"
"github.com/stretchr/testify/assert"
)
func TestFullSteps(t *testing.T) {
tests := []struct {
name string
task *actions_model.ActionTask
want []*actions_model.ActionTaskStep
}{
{
name: "regular",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
},
Status: actions_model.StatusSuccess,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
},
},
{
name: "failed step",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020},
{Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090},
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
Status: actions_model.StatusFailure,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020},
{Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090},
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
},
},
{
name: "first step is running",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0},
},
Status: actions_model.StatusRunning,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusWaiting, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
},
{
name: "first step has canceled",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
Status: actions_model.StatusFailure,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusFailure, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100},
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100},
},
},
{
name: "empty steps",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{},
Status: actions_model.StatusSuccess,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100},
{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, FullSteps(tt.task), "FullSteps(%v)", tt.task)
})
}
}
+221
View File
@@ -0,0 +1,221 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"bytes"
"io"
"strings"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
api "code.gitea.io/gitea/modules/structs"
webhook_module "code.gitea.io/gitea/modules/webhook"
"github.com/gobwas/glob"
"github.com/nektos/act/pkg/jobparser"
"github.com/nektos/act/pkg/model"
)
func ListWorkflows(commit *git.Commit) (git.Entries, error) {
tree, err := commit.SubTree(".gitea/workflows")
if _, ok := err.(git.ErrNotExist); ok {
tree, err = commit.SubTree(".github/workflows")
}
if _, ok := err.(git.ErrNotExist); ok {
return nil, nil
}
if err != nil {
return nil, err
}
entries, err := tree.ListEntriesRecursiveFast()
if err != nil {
return nil, err
}
ret := make(git.Entries, 0, len(entries))
for _, entry := range entries {
if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") {
ret = append(ret, entry)
}
}
return ret, nil
}
func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) (map[string][]byte, error) {
entries, err := ListWorkflows(commit)
if err != nil {
return nil, err
}
workflows := make(map[string][]byte, len(entries))
for _, entry := range entries {
f, err := entry.Blob().DataAsync()
if err != nil {
return nil, err
}
content, err := io.ReadAll(f)
_ = f.Close()
if err != nil {
return nil, err
}
workflow, err := model.ReadWorkflow(bytes.NewReader(content))
if err != nil {
log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
continue
}
events, err := jobparser.ParseRawOn(&workflow.RawOn)
if err != nil {
log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
continue
}
for _, evt := range events {
if evt.Name != triggedEvent.Event() {
continue
}
if detectMatched(commit, triggedEvent, payload, evt) {
workflows[entry.Name()] = content
}
}
}
return workflows, nil
}
func detectMatched(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader, evt *jobparser.Event) bool {
if len(evt.Acts) == 0 {
return true
}
switch triggedEvent {
case webhook_module.HookEventCreate:
fallthrough
case webhook_module.HookEventDelete:
fallthrough
case webhook_module.HookEventFork:
log.Warn("unsupported event %q", triggedEvent.Event())
return false
case webhook_module.HookEventPush:
pushPayload := payload.(*api.PushPayload)
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts {
switch cond {
case "branches", "tags":
for _, val := range vals {
if glob.MustCompile(val, '/').Match(pushPayload.Ref) {
matchTimes++
break
}
}
case "paths":
filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before)
if err != nil {
log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err)
} else {
for _, val := range vals {
matched := false
for _, file := range filesChanged {
if glob.MustCompile(val, '/').Match(file) {
matched = true
break
}
}
if matched {
matchTimes++
break
}
}
}
default:
log.Warn("unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts)
case webhook_module.HookEventIssues:
fallthrough
case webhook_module.HookEventIssueAssign:
fallthrough
case webhook_module.HookEventIssueLabel:
fallthrough
case webhook_module.HookEventIssueMilestone:
fallthrough
case webhook_module.HookEventIssueComment:
fallthrough
case webhook_module.HookEventPullRequest:
prPayload := payload.(*api.PullRequestPayload)
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts {
switch cond {
case "types":
for _, val := range vals {
if glob.MustCompile(val, '/').Match(string(prPayload.Action)) {
matchTimes++
break
}
}
case "branches":
for _, val := range vals {
if glob.MustCompile(val, '/').Match(prPayload.PullRequest.Base.Ref) {
matchTimes++
break
}
}
case "paths":
filesChanged, err := commit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref)
if err != nil {
log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err)
} else {
for _, val := range vals {
matched := false
for _, file := range filesChanged {
if glob.MustCompile(val, '/').Match(file) {
matched = true
break
}
}
if matched {
matchTimes++
break
}
}
}
default:
log.Warn("unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts)
case webhook_module.HookEventPullRequestAssign:
fallthrough
case webhook_module.HookEventPullRequestLabel:
fallthrough
case webhook_module.HookEventPullRequestMilestone:
fallthrough
case webhook_module.HookEventPullRequestComment:
fallthrough
case webhook_module.HookEventPullRequestReviewApproved:
fallthrough
case webhook_module.HookEventPullRequestReviewRejected:
fallthrough
case webhook_module.HookEventPullRequestReviewComment:
fallthrough
case webhook_module.HookEventPullRequestSync:
fallthrough
case webhook_module.HookEventWiki:
fallthrough
case webhook_module.HookEventRepository:
fallthrough
case webhook_module.HookEventRelease:
fallthrough
case webhook_module.HookEventPackage:
fallthrough
default:
log.Warn("unsupported event %q", triggedEvent.Event())
}
return false
}
+1 -1
View File
@@ -28,7 +28,7 @@ func Init() {
Config: &webauthn.Config{
RPDisplayName: setting.AppName,
RPID: setting.Domain,
RPOrigin: appURL,
RPOrigins: []string{appURL},
AuthenticatorSelection: protocol.AuthenticatorSelection{
UserVerification: "discouraged",
},
+2 -2
View File
@@ -15,11 +15,11 @@ func TestInit(t *testing.T) {
setting.Domain = "domain"
setting.AppName = "AppName"
setting.AppURL = "https://domain/"
rpOrigin := "https://domain"
rpOrigin := []string{"https://domain"}
Init()
assert.Equal(t, setting.Domain, WebAuthn.Config.RPID)
assert.Equal(t, setting.AppName, WebAuthn.Config.RPDisplayName)
assert.Equal(t, rpOrigin, WebAuthn.Config.RPOrigin)
assert.Equal(t, rpOrigin, WebAuthn.Config.RPOrigins)
}
+2
View File
@@ -805,6 +805,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
ctx.Data["EnableOpenIDSignIn"] = setting.Service.EnableOpenIDSignIn
ctx.Data["DisableMigrations"] = setting.Repository.DisableMigrations
ctx.Data["DisableStars"] = setting.Repository.DisableStars
ctx.Data["EnableActions"] = setting.Actions.Enabled
ctx.Data["ManifestData"] = setting.ManifestData
@@ -812,6 +813,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
ctx.Data["UnitIssuesGlobalDisabled"] = unit.TypeIssues.UnitGlobalDisabled()
ctx.Data["UnitPullsGlobalDisabled"] = unit.TypePullRequests.UnitGlobalDisabled()
ctx.Data["UnitProjectsGlobalDisabled"] = unit.TypeProjects.UnitGlobalDisabled()
ctx.Data["UnitActionsGlobalDisabled"] = unit.TypeActions.UnitGlobalDisabled()
ctx.Data["locale"] = locale
ctx.Data["AllLangs"] = translation.AllLangs()
+1
View File
@@ -1043,6 +1043,7 @@ func UnitTypes() func(ctx *Context) {
ctx.Data["UnitTypeExternalTracker"] = unit_model.TypeExternalTracker
ctx.Data["UnitTypeProjects"] = unit_model.TypeProjects
ctx.Data["UnitTypePackages"] = unit_model.TypePackages
ctx.Data["UnitTypeActions"] = unit_model.TypeActions
}
}
+47
View File
@@ -0,0 +1,47 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hcaptcha
const (
ErrMissingInputSecret ErrorCode = "missing-input-secret"
ErrInvalidInputSecret ErrorCode = "invalid-input-secret"
ErrMissingInputResponse ErrorCode = "missing-input-response"
ErrInvalidInputResponse ErrorCode = "invalid-input-response"
ErrBadRequest ErrorCode = "bad-request"
ErrInvalidOrAlreadySeenResponse ErrorCode = "invalid-or-already-seen-response"
ErrNotUsingDummyPasscode ErrorCode = "not-using-dummy-passcode"
ErrSitekeySecretMismatch ErrorCode = "sitekey-secret-mismatch"
)
// ErrorCode is any possible error from hCaptcha
type ErrorCode string
// String fulfills the Stringer interface
func (err ErrorCode) String() string {
switch err {
case ErrMissingInputSecret:
return "Your secret key is missing."
case ErrInvalidInputSecret:
return "Your secret key is invalid or malformed."
case ErrMissingInputResponse:
return "The response parameter (verification token) is missing."
case ErrInvalidInputResponse:
return "The response parameter (verification token) is invalid or malformed."
case ErrBadRequest:
return "The request is invalid or malformed."
case ErrInvalidOrAlreadySeenResponse:
return "The response parameter has already been checked, or has another issue."
case ErrNotUsingDummyPasscode:
return "You have used a testing sitekey but have not used its matching secret."
case ErrSitekeySecretMismatch:
return "The sitekey is not registered with the provided secret."
default:
return ""
}
}
// Error fulfills the error interface
func (err ErrorCode) Error() string {
return err.String()
}
+111 -4
View File
@@ -5,20 +5,127 @@ package hcaptcha
import (
"context"
"io"
"net/http"
"net/url"
"strings"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/setting"
"go.jolheiser.com/hcaptcha"
)
const verifyURL = "https://hcaptcha.com/siteverify"
// Client is an hCaptcha client
type Client struct {
ctx context.Context
http *http.Client
secret string
}
// PostOptions are optional post form values
type PostOptions struct {
RemoteIP string
Sitekey string
}
// ClientOption is a func to modify a new Client
type ClientOption func(*Client)
// WithHTTP sets the http.Client of a Client
func WithHTTP(httpClient *http.Client) func(*Client) {
return func(hClient *Client) {
hClient.http = httpClient
}
}
// WithContext sets the context.Context of a Client
func WithContext(ctx context.Context) func(*Client) {
return func(hClient *Client) {
hClient.ctx = ctx
}
}
// New returns a new hCaptcha Client
func New(secret string, options ...ClientOption) (*Client, error) {
if strings.TrimSpace(secret) == "" {
return nil, ErrMissingInputSecret
}
client := &Client{
ctx: context.Background(),
http: http.DefaultClient,
secret: secret,
}
for _, opt := range options {
opt(client)
}
return client, nil
}
// Response is an hCaptcha response
type Response struct {
Success bool `json:"success"`
ChallengeTS string `json:"challenge_ts"`
Hostname string `json:"hostname"`
Credit bool `json:"credit,omitempty"`
ErrorCodes []ErrorCode `json:"error-codes"`
}
// Verify checks the response against the hCaptcha API
func (c *Client) Verify(token string, opts PostOptions) (*Response, error) {
if strings.TrimSpace(token) == "" {
return nil, ErrMissingInputResponse
}
post := url.Values{
"secret": []string{c.secret},
"response": []string{token},
}
if strings.TrimSpace(opts.RemoteIP) != "" {
post.Add("remoteip", opts.RemoteIP)
}
if strings.TrimSpace(opts.Sitekey) != "" {
post.Add("sitekey", opts.Sitekey)
}
// Basically a copy of http.PostForm, but with a context
req, err := http.NewRequestWithContext(c.ctx, http.MethodPost, verifyURL, strings.NewReader(post.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var response *Response
if err := json.Unmarshal(body, &response); err != nil {
return nil, err
}
return response, nil
}
// Verify calls hCaptcha API to verify token
func Verify(ctx context.Context, response string) (bool, error) {
client, err := hcaptcha.New(setting.Service.HcaptchaSecret, hcaptcha.WithContext(ctx))
client, err := New(setting.Service.HcaptchaSecret, WithContext(ctx))
if err != nil {
return false, err
}
resp, err := client.Verify(response, hcaptcha.PostOptions{
resp, err := client.Verify(response, PostOptions{
Sitekey: setting.Service.HcaptchaSitekey,
})
if err != nil {
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hcaptcha
import (
"net/http"
"os"
"strings"
"testing"
"time"
)
const (
dummySiteKey = "10000000-ffff-ffff-ffff-000000000001"
dummySecret = "0x0000000000000000000000000000000000000000"
dummyToken = "10000000-aaaa-bbbb-cccc-000000000001"
)
func TestMain(m *testing.M) {
os.Exit(m.Run())
}
func TestCaptcha(t *testing.T) {
tt := []struct {
Name string
Secret string
Token string
Error ErrorCode
}{
{
Name: "Success",
Secret: dummySecret,
Token: dummyToken,
},
{
Name: "Missing Secret",
Token: dummyToken,
Error: ErrMissingInputSecret,
},
{
Name: "Missing Token",
Secret: dummySecret,
Error: ErrMissingInputResponse,
},
{
Name: "Invalid Token",
Secret: dummySecret,
Token: "test",
Error: ErrInvalidInputResponse,
},
}
for _, tc := range tt {
t.Run(tc.Name, func(t *testing.T) {
client, err := New(tc.Secret, WithHTTP(&http.Client{
Timeout: time.Second * 5,
}))
if err != nil {
// The only error that can be returned from creating a client
if tc.Error == ErrMissingInputSecret && err == ErrMissingInputSecret {
return
}
t.Log(err)
t.FailNow()
}
resp, err := client.Verify(tc.Token, PostOptions{
Sitekey: dummySiteKey,
})
if err != nil {
// The only error that can be returned prior to the request
if tc.Error == ErrMissingInputResponse && err == ErrMissingInputResponse {
return
}
t.Log(err)
t.FailNow()
}
if tc.Error.String() != "" {
if resp.Success {
t.Log("Verification should fail.")
t.Fail()
}
if len(resp.ErrorCodes) == 0 {
t.Log("hCaptcha should have returned an error.")
t.Fail()
}
var hasErr bool
for _, err := range resp.ErrorCodes {
if strings.EqualFold(err.String(), tc.Error.String()) {
hasErr = true
break
}
}
if !hasErr {
t.Log("hCaptcha did not return the error being tested")
t.Fail()
}
} else if !resp.Success {
t.Log("Verification should succeed.")
t.Fail()
}
})
}
}
+30 -2
View File
@@ -164,6 +164,7 @@ var defaultProcessors = []processor{
linkProcessor,
mentionProcessor,
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emailAddressProcessor,
emojiProcessor,
@@ -190,6 +191,7 @@ var commitMessageProcessors = []processor{
linkProcessor,
mentionProcessor,
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emailAddressProcessor,
emojiProcessor,
@@ -221,6 +223,7 @@ var commitMessageSubjectProcessors = []processor{
linkProcessor,
mentionProcessor,
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emojiShortCodeProcessor,
emojiProcessor,
@@ -257,6 +260,7 @@ func RenderIssueTitle(
) (string, error) {
return renderProcessString(ctx, []processor{
issueIndexPatternProcessor,
commitCrossReferencePatternProcessor,
sha1CurrentPatternProcessor,
emojiShortCodeProcessor,
emojiProcessor,
@@ -354,12 +358,19 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output
}
func visitNode(ctx *RenderContext, procs, textProcs []processor, node *html.Node) {
// Add user-content- to IDs if they don't already have them
// Add user-content- to IDs and "#" links if they don't already have them
for idx, attr := range node.Attr {
if attr.Key == "id" && !(strings.HasPrefix(attr.Val, "user-content-") || blackfridayExtRegex.MatchString(attr.Val)) {
val := strings.TrimPrefix(attr.Val, "#")
notHasPrefix := !(strings.HasPrefix(val, "user-content-") || blackfridayExtRegex.MatchString(val))
if attr.Key == "id" && notHasPrefix {
node.Attr[idx].Val = "user-content-" + attr.Val
}
if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix {
node.Attr[idx].Val = "#user-content-" + val
}
if attr.Key == "class" && attr.Val == "emoji" {
textProcs = nil
}
@@ -907,6 +918,23 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
}
}
func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) {
next := node.NextSibling
for node != nil && node != next {
found, ref := references.FindRenderizableCommitCrossReference(node.Data)
if !found {
return
}
reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha)
link := createLink(util.URLJoin(setting.AppSubURL, ref.Owner, ref.Name, "commit", ref.CommitSha), reftext, "commit")
replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link)
node = node.NextSibling.NextSibling
}
}
// fullSha1PatternProcessor renders SHA containing URLs
func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) {
if ctx.Metas == nil {
+20
View File
@@ -10,6 +10,7 @@ import (
packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification/action"
"code.gitea.io/gitea/modules/notification/base"
"code.gitea.io/gitea/modules/notification/indexer"
@@ -106,6 +107,13 @@ func NotifyAutoMergePullRequest(ctx context.Context, doer *user_model.User, pr *
// NotifyNewPullRequest notifies new pull request to notifiers
func NotifyNewPullRequest(ctx context.Context, pr *issues_model.PullRequest, mentions []*user_model.User) {
if err := pr.LoadIssue(ctx); err != nil {
log.Error("%v", err)
return
}
if err := pr.Issue.LoadPoster(ctx); err != nil {
return
}
for _, notifier := range notifiers {
notifier.NotifyNewPullRequest(ctx, pr, mentions)
}
@@ -120,6 +128,10 @@ func NotifyPullRequestSynchronized(ctx context.Context, doer *user_model.User, p
// NotifyPullRequestReview notifies new pull request review
func NotifyPullRequestReview(ctx context.Context, pr *issues_model.PullRequest, review *issues_model.Review, comment *issues_model.Comment, mentions []*user_model.User) {
if err := review.LoadReviewer(ctx); err != nil {
log.Error("%v", err)
return
}
for _, notifier := range notifiers {
notifier.NotifyPullRequestReview(ctx, pr, review, comment, mentions)
}
@@ -127,6 +139,10 @@ func NotifyPullRequestReview(ctx context.Context, pr *issues_model.PullRequest,
// NotifyPullRequestCodeComment notifies new pull request code comment
func NotifyPullRequestCodeComment(ctx context.Context, pr *issues_model.PullRequest, comment *issues_model.Comment, mentions []*user_model.User) {
if err := comment.LoadPoster(ctx); err != nil {
log.Error("LoadPoster: %v", err)
return
}
for _, notifier := range notifiers {
notifier.NotifyPullRequestCodeComment(ctx, pr, comment, mentions)
}
@@ -169,6 +185,10 @@ func NotifyDeleteComment(ctx context.Context, doer *user_model.User, c *issues_m
// NotifyNewRelease notifies new release to notifiers
func NotifyNewRelease(ctx context.Context, rel *repo_model.Release) {
if err := rel.LoadAttributes(ctx); err != nil {
log.Error("LoadPublisher: %v", err)
return
}
for _, notifier := range notifiers {
notifier.NotifyNewRelease(ctx, rel)
}
+1 -2
View File
@@ -6,9 +6,8 @@ package password
import (
"context"
"code.gitea.io/gitea/modules/password/pwn"
"code.gitea.io/gitea/modules/setting"
"go.jolheiser.com/pwn"
)
// IsPwned checks whether a password has been pwned
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package pwn
import (
"context"
"crypto/sha1"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"code.gitea.io/gitea/modules/setting"
)
const passwordURL = "https://api.pwnedpasswords.com/range/"
// ErrEmptyPassword is an empty password error
var ErrEmptyPassword = errors.New("password cannot be empty")
// Client is a HaveIBeenPwned client
type Client struct {
ctx context.Context
http *http.Client
}
// New returns a new HaveIBeenPwned Client
func New(options ...ClientOption) *Client {
client := &Client{
ctx: context.Background(),
http: http.DefaultClient,
}
for _, opt := range options {
opt(client)
}
return client
}
// ClientOption is a way to modify a new Client
type ClientOption func(*Client)
// WithHTTP will set the http.Client of a Client
func WithHTTP(httpClient *http.Client) func(pwnClient *Client) {
return func(pwnClient *Client) {
pwnClient.http = httpClient
}
}
// WithContext will set the context.Context of a Client
func WithContext(ctx context.Context) func(pwnClient *Client) {
return func(pwnClient *Client) {
pwnClient.ctx = ctx
}
}
func newRequest(ctx context.Context, method, url string, body io.ReadCloser) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return nil, err
}
req.Header.Add("User-Agent", "Gitea "+setting.AppVer)
return req, nil
}
// CheckPassword returns the number of times a password has been compromised
// Adding padding will make requests more secure, however is also slower
// because artificial responses will be added to the response
// For more information, see https://www.troyhunt.com/enhancing-pwned-passwords-privacy-with-padding/
func (c *Client) CheckPassword(pw string, padding bool) (int, error) {
if strings.TrimSpace(pw) == "" {
return -1, ErrEmptyPassword
}
sha := sha1.New()
sha.Write([]byte(pw))
enc := hex.EncodeToString(sha.Sum(nil))
prefix, suffix := enc[:5], enc[5:]
req, err := newRequest(c.ctx, http.MethodGet, fmt.Sprintf("%s%s", passwordURL, prefix), nil)
if err != nil {
return -1, nil
}
if padding {
req.Header.Add("Add-Padding", "true")
}
resp, err := c.http.Do(req)
if err != nil {
return -1, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return -1, err
}
defer resp.Body.Close()
for _, pair := range strings.Split(string(body), "\n") {
parts := strings.Split(pair, ":")
if len(parts) != 2 {
continue
}
if strings.EqualFold(suffix, parts[0]) {
count, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
if err != nil {
return -1, err
}
return int(count), nil
}
}
return 0, nil
}
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package pwn
import (
"errors"
"math/rand"
"net/http"
"os"
"strings"
"testing"
"time"
)
var client = New(WithHTTP(&http.Client{
Timeout: time.Second * 2,
}))
func TestMain(m *testing.M) {
rand.Seed(time.Now().Unix())
os.Exit(m.Run())
}
func TestPassword(t *testing.T) {
// Check input error
_, err := client.CheckPassword("", false)
if err == nil {
t.Log("blank input should return an error")
t.Fail()
}
if !errors.Is(err, ErrEmptyPassword) {
t.Log("blank input should return ErrEmptyPassword")
t.Fail()
}
// Should fail
fail := "password1234"
count, err := client.CheckPassword(fail, false)
if err != nil {
t.Log(err)
t.Fail()
}
if count == 0 {
t.Logf("%s should fail as a password\n", fail)
t.Fail()
}
// Should fail (with padding)
failPad := "administrator"
count, err = client.CheckPassword(failPad, true)
if err != nil {
t.Log(err)
t.Fail()
}
if count == 0 {
t.Logf("%s should fail as a password\n", failPad)
t.Fail()
}
// Checking for a "good" password isn't going to be perfect, but we can give it a good try
// with hopefully minimal error. Try five times?
var good bool
var pw string
for idx := 0; idx <= 5; idx++ {
pw = testPassword()
count, err = client.CheckPassword(pw, false)
if err != nil {
t.Log(err)
t.Fail()
}
if count == 0 {
good = true
break
}
}
if !good {
t.Log("no generated passwords passed. there is a chance this is a fluke")
t.Fail()
}
// Again, but with padded responses
good = false
for idx := 0; idx <= 5; idx++ {
pw = testPassword()
count, err = client.CheckPassword(pw, true)
if err != nil {
t.Log(err)
t.Fail()
}
if count == 0 {
good = true
break
}
}
if !good {
t.Log("no generated passwords passed. there is a chance this is a fluke")
t.Fail()
}
}
// Credit to https://golangbyexample.com/generate-random-password-golang/
// DO NOT USE THIS FOR AN ACTUAL PASSWORD GENERATOR
var (
lowerCharSet = "abcdedfghijklmnopqrst"
upperCharSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
specialCharSet = "!@#$%&*"
numberSet = "0123456789"
allCharSet = lowerCharSet + upperCharSet + specialCharSet + numberSet
)
func testPassword() string {
var password strings.Builder
// Set special character
for i := 0; i < 5; i++ {
random := rand.Intn(len(specialCharSet))
password.WriteString(string(specialCharSet[random]))
}
// Set numeric
for i := 0; i < 5; i++ {
random := rand.Intn(len(numberSet))
password.WriteString(string(numberSet[random]))
}
// Set uppercase
for i := 0; i < 5; i++ {
random := rand.Intn(len(upperCharSet))
password.WriteString(string(upperCharSet[random]))
}
for i := 0; i < 5; i++ {
random := rand.Intn(len(allCharSet))
password.WriteString(string(allCharSet[random]))
}
inRune := []rune(password.String())
rand.Shuffle(len(inRune), func(i, j int) {
inRune[i], inRune[j] = inRune[j], inRune[i]
})
return string(inRune)
}
+1
View File
@@ -57,6 +57,7 @@ type HookOptions struct {
PullRequestID int64
DeployKeyID int64 // if the pusher is a DeployKey, then UserID is the repo's org user.
IsWiki bool
ActionPerm int
}
// SSHLogOption ssh log options
+21 -2
View File
@@ -35,9 +35,12 @@ var (
// issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$))`)
// crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
// e.g. gogits/gogs#12345
// e.g. org/repo#12345
crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`)
// spaceTrimmedPattern let's us find the trailing space
// crossReferenceCommitPattern matches a string that references a commit in a different repository
// e.g. go-gitea/gitea@d8a994ef, go-gitea/gitea@d8a994ef243349f321568f9e36d5c3f444b99cae (7-40 characters)
crossReferenceCommitPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+)/([0-9a-zA-Z-_\.]+)@([0-9a-f]{7,40})(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`)
// spaceTrimmedPattern let's find the trailing space
spaceTrimmedPattern = regexp.MustCompile(`(?:.*[0-9a-zA-Z-_])\s`)
// timeLogPattern matches string for time tracking
timeLogPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@([0-9]+([\.,][0-9]+)?(w|d|m|h))+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`)
@@ -92,6 +95,7 @@ type RenderizableReference struct {
Issue string
Owner string
Name string
CommitSha string
IsPull bool
RefLocation *RefSpan
Action XRefAction
@@ -350,6 +354,21 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende
}
}
// FindRenderizableCommitCrossReference returns the first unvalidated commit cross reference found in a string.
func FindRenderizableCommitCrossReference(content string) (bool, *RenderizableReference) {
m := crossReferenceCommitPattern.FindStringSubmatchIndex(content)
if len(m) < 8 {
return false, nil
}
return true, &RenderizableReference{
Owner: content[m[2]:m[3]],
Name: content[m[4]:m[5]],
CommitSha: content[m[6]:m[7]],
RefLocation: &RefSpan{Start: m[2], End: m[7]},
}
}
// FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string.
func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) {
match := pattern.FindStringSubmatchIndex(content)
+61
View File
@@ -303,6 +303,67 @@ func TestFindAllMentions(t *testing.T) {
}, res)
}
func TestFindRenderizableCommitCrossReference(t *testing.T) {
cases := []struct {
Input string
Expected *RenderizableReference
}{
{
Input: "",
Expected: nil,
},
{
Input: "test",
Expected: nil,
},
{
Input: "go-gitea/gitea@test",
Expected: nil,
},
{
Input: "go-gitea/gitea@ab1234",
Expected: nil,
},
{
Input: "go-gitea/gitea@abcd1234",
Expected: &RenderizableReference{
Owner: "go-gitea",
Name: "gitea",
CommitSha: "abcd1234",
RefLocation: &RefSpan{Start: 0, End: 23},
},
},
{
Input: "go-gitea/gitea@abcd1234abcd1234abcd1234abcd1234abcd1234",
Expected: &RenderizableReference{
Owner: "go-gitea",
Name: "gitea",
CommitSha: "abcd1234abcd1234abcd1234abcd1234abcd1234",
RefLocation: &RefSpan{Start: 0, End: 55},
},
},
{
Input: "go-gitea/gitea@abcd1234abcd1234abcd1234abcd1234abcd12340", // longer than 40 characters
Expected: nil,
},
{
Input: "test go-gitea/gitea@abcd1234 test",
Expected: &RenderizableReference{
Owner: "go-gitea",
Name: "gitea",
CommitSha: "abcd1234",
RefLocation: &RefSpan{Start: 5, End: 28},
},
},
}
for _, c := range cases {
found, ref := FindRenderizableCommitCrossReference(c.Input)
assert.Equal(t, ref != nil, found)
assert.Equal(t, c.Expected, ref)
}
}
func TestRegExp_mentionPattern(t *testing.T) {
trueTestCases := []struct {
pat string
+1
View File
@@ -27,6 +27,7 @@ const (
EnvPRID = "GITEA_PR_ID"
EnvIsInternal = "GITEA_INTERNAL_PUSH"
EnvAppURL = "GITEA_ROOT_URL"
EnvActionPerm = "GITEA_ACTION_PERM"
)
// InternalPushingEnvironment returns an os environment to switch off hooks on push
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"code.gitea.io/gitea/modules/log"
)
// Actions settings
var (
Actions = struct {
Storage // how the created logs should be stored
Enabled bool
DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
}{
Enabled: false,
DefaultActionsURL: "https://gitea.com",
}
)
func newActions() {
sec := Cfg.Section("actions")
if err := sec.MapTo(&Actions); err != nil {
log.Fatal("Failed to map Actions settings: %v", err)
}
Actions.Storage = getStorage("actions_log", "", nil)
}
+2
View File
@@ -1074,6 +1074,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) {
newPackages()
newActions()
if err = Cfg.Section("ui").MapTo(&UI); err != nil {
log.Fatal("Failed to map UI settings: %v", err)
} else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil {
+14
View File
@@ -125,6 +125,9 @@ var (
// Packages represents packages storage
Packages ObjectStorage = uninitializedStorage
// Actions represents actions storage
Actions ObjectStorage = uninitializedStorage
)
// Init init the stoarge
@@ -136,6 +139,7 @@ func Init() error {
initLFS,
initRepoArchives,
initPackages,
initActions,
} {
if err := f(); err != nil {
return err
@@ -204,3 +208,13 @@ func initPackages() (err error) {
Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage)
return err
}
func initActions() (err error) {
if !setting.Actions.Enabled {
Actions = discardStorage("Actions isn't enabled")
return nil
}
log.Info("Initialising Actions storage with type: %s", setting.Actions.Storage.Type)
Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage)
return err
}
+2
View File
@@ -18,6 +18,8 @@ const (
CommitStatusFailure CommitStatusState = "failure"
// CommitStatusWarning is for when the CommitStatus is Warning
CommitStatusWarning CommitStatusState = "warning"
// CommitStatusRunning is for when the CommitStatus is Running
CommitStatusRunning CommitStatusState = "running"
)
// NoBetterThan returns true if this State is no better than the given State
+3
View File
@@ -472,6 +472,9 @@ func NewFuncMap() []template.FuncMap {
curBranch,
)
},
"RefShortName": func(ref string) string {
return git.RefName(ref).ShortName()
},
}}
}