Merge remote-tracking branch 'upstream/main' into limit-repo-size

This commit is contained in:
DmitryFrolovTri
2023-12-02 08:38:29 +00:00
550 changed files with 3490 additions and 2005 deletions
+2 -1
View File
@@ -563,6 +563,7 @@ func RepoAssignment(ctx *Context) context.CancelFunc {
ctx.Data["CanWriteCode"] = ctx.Repo.CanWrite(unit_model.TypeCode)
ctx.Data["CanWriteIssues"] = ctx.Repo.CanWrite(unit_model.TypeIssues)
ctx.Data["CanWritePulls"] = ctx.Repo.CanWrite(unit_model.TypePullRequests)
ctx.Data["CanWriteActions"] = ctx.Repo.CanWrite(unit_model.TypeActions)
canSignedUserFork, err := repo_module.CanUserForkRepo(ctx, ctx.Doer, ctx.Repo.Repository)
if err != nil {
@@ -850,7 +851,7 @@ func getRefName(ctx *Base, repo *Repository, pathType RepoRefType) string {
return getRefNameFromPath(ctx, repo, path, func(s string) bool {
b, exist, err := git_model.FindRenamedBranch(ctx, repo.Repository.ID, s)
if err != nil {
log.Error("FindRenamedBranch", err)
log.Error("FindRenamedBranch: %v", err)
return false
}
+2 -2
View File
@@ -26,7 +26,7 @@ func handleDeleteOrphanedRepos(ctx context.Context, logger log.Logger, autofix b
// countOrphanedRepos count repository where user of owner_id do not exist
func countOrphanedRepos(ctx context.Context) (int64, error) {
return db.CountOrphanedObjects(ctx, "repository", "user", "repository.owner_id=user.id")
return db.CountOrphanedObjects(ctx, "repository", "user", "repository.owner_id=`user`.id")
}
// deleteOrphanedRepos delete repository where user of owner_id do not exist
@@ -43,7 +43,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) {
default:
var ids []int64
if err := e.Table("`repository`").
Join("LEFT", "`user`", "repository.owner_id=user.id").
Join("LEFT", "`user`", "repository.owner_id=`user`.id").
Where(builder.IsNull{"`user`.id"}).
Select("`repository`.id").Limit(batchSize).Find(&ids); err != nil {
return deleted, err
+25 -12
View File
@@ -11,6 +11,7 @@ import (
"io"
"os"
"regexp"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
@@ -18,8 +19,10 @@ import (
// BlamePart represents block of blame - continuous lines with one sha
type BlamePart struct {
Sha string
Lines []string
Sha string
Lines []string
PreviousSha string
PreviousPath string
}
// BlameReader returns part of file blame one by one
@@ -43,30 +46,38 @@ func (r *BlameReader) NextPart() (*BlamePart, error) {
var blamePart *BlamePart
if r.lastSha != nil {
blamePart = &BlamePart{*r.lastSha, make([]string, 0)}
blamePart = &BlamePart{
Sha: *r.lastSha,
Lines: make([]string, 0),
}
}
var line []byte
var lineBytes []byte
var isPrefix bool
var err error
for err != io.EOF {
line, isPrefix, err = r.bufferedReader.ReadLine()
lineBytes, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
if len(line) == 0 {
if len(lineBytes) == 0 {
// isPrefix will be false
continue
}
lines := shaLineRegex.FindSubmatch(line)
line := string(lineBytes)
lines := shaLineRegex.FindStringSubmatch(line)
if lines != nil {
sha1 := string(lines[1])
sha1 := lines[1]
if blamePart == nil {
blamePart = &BlamePart{sha1, make([]string, 0)}
blamePart = &BlamePart{
Sha: sha1,
Lines: make([]string, 0),
}
}
if blamePart.Sha != sha1 {
@@ -81,9 +92,11 @@ func (r *BlameReader) NextPart() (*BlamePart, error) {
return blamePart, nil
}
} else if line[0] == '\t' {
code := line[1:]
blamePart.Lines = append(blamePart.Lines, string(code))
blamePart.Lines = append(blamePart.Lines, line[1:])
} else if strings.HasPrefix(line, "previous ") {
parts := strings.SplitN(line[len("previous "):], " ", 2)
blamePart.PreviousSha = parts[0]
blamePart.PreviousPath = parts[1]
}
// need to munch to end of line...
+16 -12
View File
@@ -24,15 +24,17 @@ func TestReadingBlameOutput(t *testing.T) {
parts := []*BlamePart{
{
"72866af952e98d02a73003501836074b286a78f6",
[]string{
Sha: "72866af952e98d02a73003501836074b286a78f6",
Lines: []string{
"# test_repo",
"Test repository for testing migration from github to gitea",
},
},
{
"f32b0a9dfd09a60f616f29158f772cedd89942d2",
[]string{"", "Do not make any changes to this repo it is used for unit testing"},
Sha: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
Lines: []string{"", "Do not make any changes to this repo it is used for unit testing"},
PreviousSha: "72866af952e98d02a73003501836074b286a78f6",
PreviousPath: "README.md",
},
}
@@ -64,16 +66,18 @@ func TestReadingBlameOutput(t *testing.T) {
full := []*BlamePart{
{
"af7486bd54cfc39eea97207ca666aa69c9d6df93",
[]string{"line", "line"},
Sha: "af7486bd54cfc39eea97207ca666aa69c9d6df93",
Lines: []string{"line", "line"},
},
{
"45fb6cbc12f970b04eacd5cd4165edd11c8d7376",
[]string{"changed line"},
Sha: "45fb6cbc12f970b04eacd5cd4165edd11c8d7376",
Lines: []string{"changed line"},
PreviousSha: "af7486bd54cfc39eea97207ca666aa69c9d6df93",
PreviousPath: "blame.txt",
},
{
"af7486bd54cfc39eea97207ca666aa69c9d6df93",
[]string{"line", "line", ""},
Sha: "af7486bd54cfc39eea97207ca666aa69c9d6df93",
Lines: []string{"line", "line", ""},
},
}
@@ -89,8 +93,8 @@ func TestReadingBlameOutput(t *testing.T) {
Bypass: false,
Parts: []*BlamePart{
{
"af7486bd54cfc39eea97207ca666aa69c9d6df93",
[]string{"line", "line", "changed line", "line", "line", ""},
Sha: "af7486bd54cfc39eea97207ca666aa69c9d6df93",
Lines: []string{"line", "line", "changed line", "line", "line", ""},
},
},
},
+7
View File
@@ -7,6 +7,13 @@ import (
"context"
)
// Shutdown procedure:
// * cancel ShutdownContext: the registered context consumers have time to do their cleanup (they could use the hammer context)
// * cancel HammerContext: the all context consumers have limited time to do their cleanup (wait for a few seconds)
// * cancel TerminateContext: the registered context consumers have time to do their cleanup (but they shouldn't use shutdown/hammer context anymore)
// * cancel manager context
// If the shutdown is triggered again during the shutdown procedure, the hammer context will be canceled immediately to force to shut down.
// ShutdownContext returns a context.Context that is Done at shutdown
// Callers using this context should ensure that they are registered as a running server
// in order that they are waited for.
+8 -44
View File
@@ -39,10 +39,10 @@ type RunCanceler interface {
// and add a function to call manager.InformCleanup if it's not going to be used
const numberOfServersToCreate = 4
// Manager represents the graceful server manager interface
var manager *Manager
var initOnce = sync.Once{}
var (
manager *Manager
initOnce sync.Once
)
// GetManager returns the Manager
func GetManager() *Manager {
@@ -147,12 +147,12 @@ func (g *Manager) doShutdown() {
go g.doHammerTime(setting.GracefulHammerTime)
}
go func() {
g.WaitForServers()
g.runningServerWaitGroup.Wait()
// Mop up any remaining unclosed events.
g.doHammerTime(0)
<-time.After(1 * time.Second)
g.doTerminate()
g.WaitForTerminate()
g.terminateWaitGroup.Wait()
g.lock.Lock()
g.managerCtxCancel()
g.lock.Unlock()
@@ -199,26 +199,18 @@ func (g *Manager) IsChild() bool {
}
// IsShutdown returns a channel which will be closed at shutdown.
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
// The order of closure is shutdown, hammer (potentially), terminate
func (g *Manager) IsShutdown() <-chan struct{} {
return g.shutdownCtx.Done()
}
// IsHammer returns a channel which will be closed at hammer
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
// IsHammer returns a channel which will be closed at hammer.
// Servers running within the running server wait group should respond to IsHammer
// if not shutdown already
func (g *Manager) IsHammer() <-chan struct{} {
return g.hammerCtx.Done()
}
// IsTerminate returns a channel which will be closed at terminate
// The order of closure is IsShutdown, IsHammer (potentially), IsTerminate
// IsTerminate will only close once all running servers have stopped
func (g *Manager) IsTerminate() <-chan struct{} {
return g.terminateCtx.Done()
}
// ServerDone declares a running server done and subtracts one from the
// running server wait group. Users probably do not want to call this
// and should use one of the RunWithShutdown* functions
@@ -226,28 +218,7 @@ func (g *Manager) ServerDone() {
g.runningServerWaitGroup.Done()
}
// WaitForServers waits for all running servers to finish. Users should probably
// instead use AtTerminate or IsTerminate
func (g *Manager) WaitForServers() {
g.runningServerWaitGroup.Wait()
}
// WaitForTerminate waits for all terminating actions to finish.
// Only the main go-routine should use this
func (g *Manager) WaitForTerminate() {
g.terminateWaitGroup.Wait()
}
func (g *Manager) getState() state {
g.lock.RLock()
defer g.lock.RUnlock()
return g.state
}
func (g *Manager) setStateTransition(old, new state) bool {
if old != g.getState() {
return false
}
g.lock.Lock()
if g.state != old {
g.lock.Unlock()
@@ -258,13 +229,6 @@ func (g *Manager) setStateTransition(old, new state) bool {
return true
}
func (g *Manager) setState(st state) {
g.lock.Lock()
defer g.lock.Unlock()
g.state = st
}
// InformCleanup tells the cleanup wait group that we have either taken a listener or will not be taking a listener.
// At the moment the total number of servers (numberOfServersToCreate) are pre-defined as a const before global init,
// so this function MUST be called if a server is not used.
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package graceful
import (
"context"
"runtime/pprof"
"sync"
"time"
)
type systemdNotifyMsg string
const (
readyMsg systemdNotifyMsg = "READY=1"
stoppingMsg systemdNotifyMsg = "STOPPING=1"
reloadingMsg systemdNotifyMsg = "RELOADING=1"
watchdogMsg systemdNotifyMsg = "WATCHDOG=1"
)
func statusMsg(msg string) systemdNotifyMsg {
return systemdNotifyMsg("STATUS=" + msg)
}
// Manager manages the graceful shutdown process
type Manager struct {
ctx context.Context
isChild bool
forked bool
lock sync.RWMutex
state state
shutdownCtx context.Context
hammerCtx context.Context
terminateCtx context.Context
managerCtx context.Context
shutdownCtxCancel context.CancelFunc
hammerCtxCancel context.CancelFunc
terminateCtxCancel context.CancelFunc
managerCtxCancel context.CancelFunc
runningServerWaitGroup sync.WaitGroup
createServerWaitGroup sync.WaitGroup
terminateWaitGroup sync.WaitGroup
shutdownRequested chan struct{}
toRunAtShutdown []func()
toRunAtTerminate []func()
}
func newGracefulManager(ctx context.Context) *Manager {
manager := &Manager{ctx: ctx, shutdownRequested: make(chan struct{})}
manager.createServerWaitGroup.Add(numberOfServersToCreate)
manager.prepare(ctx)
manager.start()
return manager
}
func (g *Manager) prepare(ctx context.Context) {
g.terminateCtx, g.terminateCtxCancel = context.WithCancel(ctx)
g.shutdownCtx, g.shutdownCtxCancel = context.WithCancel(ctx)
g.hammerCtx, g.hammerCtxCancel = context.WithCancel(ctx)
g.managerCtx, g.managerCtxCancel = context.WithCancel(ctx)
g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "with-terminate"))
g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels("graceful-lifecycle", "with-shutdown"))
g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "with-hammer"))
g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "with-manager"))
if !g.setStateTransition(stateInit, stateRunning) {
panic("invalid graceful manager state: transition from init to running failed")
}
}
// DoImmediateHammer causes an immediate hammer
func (g *Manager) DoImmediateHammer() {
g.notify(statusMsg("Sending immediate hammer"))
g.doHammerTime(0 * time.Second)
}
// DoGracefulShutdown causes a graceful shutdown
func (g *Manager) DoGracefulShutdown() {
g.lock.Lock()
select {
case <-g.shutdownRequested:
default:
close(g.shutdownRequested)
}
forked := g.forked
g.lock.Unlock()
if !forked {
g.notify(stoppingMsg)
} else {
g.notify(statusMsg("Shutting down after fork"))
}
g.doShutdown()
}
// RegisterServer registers the running of a listening server, in the case of unix this means that the parent process can now die.
// Any call to RegisterServer must be matched by a call to ServerDone
func (g *Manager) RegisterServer() {
KillParent()
g.runningServerWaitGroup.Add(1)
}
+7 -93
View File
@@ -12,7 +12,6 @@ import (
"os/signal"
"runtime/pprof"
"strconv"
"sync"
"syscall"
"time"
@@ -22,51 +21,6 @@ import (
"code.gitea.io/gitea/modules/setting"
)
// Manager manages the graceful shutdown process
type Manager struct {
isChild bool
forked bool
lock *sync.RWMutex
state state
shutdownCtx context.Context
hammerCtx context.Context
terminateCtx context.Context
managerCtx context.Context
shutdownCtxCancel context.CancelFunc
hammerCtxCancel context.CancelFunc
terminateCtxCancel context.CancelFunc
managerCtxCancel context.CancelFunc
runningServerWaitGroup sync.WaitGroup
createServerWaitGroup sync.WaitGroup
terminateWaitGroup sync.WaitGroup
toRunAtShutdown []func()
toRunAtTerminate []func()
}
func newGracefulManager(ctx context.Context) *Manager {
manager := &Manager{
isChild: len(os.Getenv(listenFDsEnv)) > 0 && os.Getppid() > 1,
lock: &sync.RWMutex{},
}
manager.createServerWaitGroup.Add(numberOfServersToCreate)
manager.start(ctx)
return manager
}
type systemdNotifyMsg string
const (
readyMsg systemdNotifyMsg = "READY=1"
stoppingMsg systemdNotifyMsg = "STOPPING=1"
reloadingMsg systemdNotifyMsg = "RELOADING=1"
watchdogMsg systemdNotifyMsg = "WATCHDOG=1"
)
func statusMsg(msg string) systemdNotifyMsg {
return systemdNotifyMsg("STATUS=" + msg)
}
func pidMsg() systemdNotifyMsg {
return systemdNotifyMsg("MAINPID=" + strconv.Itoa(os.Getpid()))
}
@@ -89,25 +43,13 @@ func (g *Manager) notify(msg systemdNotifyMsg) {
}
}
func (g *Manager) start(ctx context.Context) {
// Make contexts
g.terminateCtx, g.terminateCtxCancel = context.WithCancel(ctx)
g.shutdownCtx, g.shutdownCtxCancel = context.WithCancel(ctx)
g.hammerCtx, g.hammerCtxCancel = context.WithCancel(ctx)
g.managerCtx, g.managerCtxCancel = context.WithCancel(ctx)
// Next add pprof labels to these contexts
g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "with-terminate"))
g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels("graceful-lifecycle", "with-shutdown"))
g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "with-hammer"))
g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "with-manager"))
func (g *Manager) start() {
// Now label this and all goroutines created by this goroutine with the graceful-lifecycle manager
pprof.SetGoroutineLabels(g.managerCtx)
defer pprof.SetGoroutineLabels(ctx)
defer pprof.SetGoroutineLabels(g.ctx)
g.isChild = len(os.Getenv(listenFDsEnv)) > 0 && os.Getppid() > 1
// Set the running state & handle signals
g.setState(stateRunning)
g.notify(statusMsg("Starting Gitea"))
g.notify(pidMsg())
go g.handleSignals(g.managerCtx)
@@ -116,11 +58,9 @@ func (g *Manager) start(ctx context.Context) {
startupDone := make(chan struct{})
go func() {
defer close(startupDone)
// Wait till we're done getting all of the listeners and then close
// the unused ones
// Wait till we're done getting all the listeners and then close the unused ones
g.createServerWaitGroup.Wait()
// Ignore the error here there's not much we can do with it
// They're logged in the CloseProvidedListeners function
// Ignore the error here there's not much we can do with it, they're logged in the CloseProvidedListeners function
_ = CloseProvidedListeners()
g.notify(readyMsg)
}()
@@ -131,7 +71,7 @@ func (g *Manager) start(ctx context.Context) {
return
case <-g.IsShutdown():
func() {
// When waitgroup counter goes negative it will panic - we don't care about this so we can just ignore it.
// When WaitGroup counter goes negative it will panic - we don't care about this so we can just ignore it.
defer func() {
_ = recover()
}()
@@ -253,29 +193,3 @@ func (g *Manager) DoGracefulRestart() {
g.doShutdown()
}
}
// DoImmediateHammer causes an immediate hammer
func (g *Manager) DoImmediateHammer() {
g.notify(statusMsg("Sending immediate hammer"))
g.doHammerTime(0 * time.Second)
}
// DoGracefulShutdown causes a graceful shutdown
func (g *Manager) DoGracefulShutdown() {
g.lock.Lock()
if !g.forked {
g.lock.Unlock()
g.notify(stoppingMsg)
} else {
g.lock.Unlock()
g.notify(statusMsg("Shutting down after fork"))
}
g.doShutdown()
}
// RegisterServer registers the running of a listening server, in the case of unix this means that the parent process can now die.
// Any call to RegisterServer must be matched by a call to ServerDone
func (g *Manager) RegisterServer() {
KillParent()
g.runningServerWaitGroup.Add(1)
}
+8 -77
View File
@@ -7,11 +7,9 @@
package graceful
import (
"context"
"os"
"runtime/pprof"
"strconv"
"sync"
"time"
"code.gitea.io/gitea/modules/log"
@@ -30,62 +28,11 @@ const (
acceptHammerCode = svc.Accepted(hammerCode)
)
// Manager manages the graceful shutdown process
type Manager struct {
ctx context.Context
isChild bool
lock *sync.RWMutex
state state
shutdownCtx context.Context
hammerCtx context.Context
terminateCtx context.Context
managerCtx context.Context
shutdownCtxCancel context.CancelFunc
hammerCtxCancel context.CancelFunc
terminateCtxCancel context.CancelFunc
managerCtxCancel context.CancelFunc
runningServerWaitGroup sync.WaitGroup
createServerWaitGroup sync.WaitGroup
terminateWaitGroup sync.WaitGroup
shutdownRequested chan struct{}
toRunAtShutdown []func()
toRunAtTerminate []func()
}
func newGracefulManager(ctx context.Context) *Manager {
manager := &Manager{
isChild: false,
lock: &sync.RWMutex{},
ctx: ctx,
}
manager.createServerWaitGroup.Add(numberOfServersToCreate)
manager.start()
return manager
}
func (g *Manager) start() {
// Make contexts
g.terminateCtx, g.terminateCtxCancel = context.WithCancel(g.ctx)
g.shutdownCtx, g.shutdownCtxCancel = context.WithCancel(g.ctx)
g.hammerCtx, g.hammerCtxCancel = context.WithCancel(g.ctx)
g.managerCtx, g.managerCtxCancel = context.WithCancel(g.ctx)
// Next add pprof labels to these contexts
g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "with-terminate"))
g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels("graceful-lifecycle", "with-shutdown"))
g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "with-hammer"))
g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "with-manager"))
// Now label this and all goroutines created by this goroutine with the graceful-lifecycle manager
pprof.SetGoroutineLabels(g.managerCtx)
defer pprof.SetGoroutineLabels(g.ctx)
// Make channels
g.shutdownRequested = make(chan struct{})
// Set the running state
g.setState(stateRunning)
if skip, _ := strconv.ParseBool(os.Getenv("SKIP_MINWINSVC")); skip {
log.Trace("Skipping SVC check as SKIP_MINWINSVC is set")
return
@@ -199,30 +146,6 @@ hammerLoop:
return false, 0
}
// DoImmediateHammer causes an immediate hammer
func (g *Manager) DoImmediateHammer() {
g.doHammerTime(0 * time.Second)
}
// DoGracefulShutdown causes a graceful shutdown
func (g *Manager) DoGracefulShutdown() {
g.lock.Lock()
select {
case <-g.shutdownRequested:
g.lock.Unlock()
default:
close(g.shutdownRequested)
g.lock.Unlock()
g.doShutdown()
}
}
// RegisterServer registers the running of a listening server.
// Any call to RegisterServer must be matched by a call to ServerDone
func (g *Manager) RegisterServer() {
g.runningServerWaitGroup.Add(1)
}
func (g *Manager) awaitServer(limit time.Duration) bool {
c := make(chan struct{})
go func() {
@@ -247,3 +170,11 @@ func (g *Manager) awaitServer(limit time.Duration) bool {
}
}
}
func (g *Manager) notify(msg systemdNotifyMsg) {
// Windows doesn't use systemd to notify
}
func KillParent() {
// Windows doesn't need to "kill parent" because there is no graceful restart
}
+2 -6
View File
@@ -150,12 +150,8 @@ func CloseProvidedListeners() error {
return returnableError
}
// DefaultGetListener obtains a listener for the local network address. The network must be
// a stream-oriented network: "tcp", "tcp4", "tcp6", "unix" or "unixpacket". It
// returns an provided net.Listener for the matching network and address, or
// creates a new one using net.Listen. This function can be replaced by changing the
// GetListener variable at the top of this file, for example to listen on an onion service using
// github.com/cretz/bine
// DefaultGetListener obtains a listener for the stream-oriented local network address:
// "tcp", "tcp4", "tcp6", "unix" or "unixpacket".
func DefaultGetListener(network, address string) (net.Listener, error) {
// Add a deferral to say that we've tried to grab a listener
defer GetManager().InformCleanup()
+1 -3
View File
@@ -10,9 +10,7 @@ package graceful
import "net"
// DefaultGetListener obtains a listener for the local network address.
// On windows this is basically just a shim around net.Listen. This function
// can be replaced by changing the GetListener variable at the top of this file,
// for example to listen on an onion service using github.com/cretz/bine
// On windows this is basically just a shim around net.Listen.
func DefaultGetListener(network, address string) (net.Listener, error) {
// Add a deferral to say that we've tried to grab a listener
defer GetManager().InformCleanup()
+3 -23
View File
@@ -20,31 +20,11 @@ import (
"code.gitea.io/gitea/modules/setting"
)
var (
// DefaultReadTimeOut default read timeout
DefaultReadTimeOut time.Duration
// DefaultWriteTimeOut default write timeout
DefaultWriteTimeOut time.Duration
// DefaultMaxHeaderBytes default max header bytes
DefaultMaxHeaderBytes int
// PerWriteWriteTimeout timeout for writes
PerWriteWriteTimeout = 30 * time.Second
// PerWriteWriteTimeoutKbTime is a timeout taking account of how much there is to be written
PerWriteWriteTimeoutKbTime = 10 * time.Second
)
// GetListener returns a listener from a GetListener function, which must have the
// signature: `func FunctioName(network, address string) (net.Listener, error)`.
// This determines the implementation of net.Listener which the server will use.`
// It is implemented in this way so that downstreams may specify the type of listener
// they want to provide Gitea on by default, such as with a hidden service or a p2p network
// No need to worry about "breaking" if there would be a refactoring for the Listeners. No compatibility-guarantee for this mechanism
// GetListener returns a net listener
// This determines the implementation of net.Listener which the server will use,
// so that downstreams could provide their own Listener, such as with a hidden service or a p2p network
var GetListener = DefaultGetListener
func init() {
DefaultMaxHeaderBytes = 0 // use http.DefaultMaxHeaderBytes - which currently is 1 << 20 (1MB)
}
// ServeFunction represents a listen.Accept loop
type ServeFunction = func(net.Listener) error
+2 -5
View File
@@ -13,11 +13,8 @@ import (
func newHTTPServer(network, address, name string, handler http.Handler) (*Server, ServeFunction) {
server := NewServer(network, address, name)
httpServer := http.Server{
ReadTimeout: DefaultReadTimeOut,
WriteTimeout: DefaultWriteTimeOut,
MaxHeaderBytes: DefaultMaxHeaderBytes,
Handler: handler,
BaseContext: func(net.Listener) context.Context { return GetManager().HammerContext() },
Handler: handler,
BaseContext: func(net.Listener) context.Context { return GetManager().HammerContext() },
}
server.OnShutdown = func() {
httpServer.SetKeepAlivesEnabled(false)
@@ -211,10 +211,11 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
skip, limit := indexer_internal.ParsePaginator(options.Paginator, maxTotalHits)
searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(options.Keyword, &meilisearch.SearchRequest{
Filter: query.Statement(),
Limit: int64(limit),
Offset: int64(skip),
Sort: sortBy,
Filter: query.Statement(),
Limit: int64(limit),
Offset: int64(skip),
Sort: sortBy,
MatchingStrategy: "all",
})
if err != nil {
return nil, err
+1 -1
View File
@@ -66,7 +66,7 @@ var (
// well as the HTML5 spec:
// http://spec.commonmark.org/0.28/#email-address
// https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail)
emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|\\.(\\s|$))")
emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))")
// blackfriday extensions create IDs like fn:user-content-footnote
blackfridayExtRegex = regexp.MustCompile(`[^:]*:user-content-`)
+12
View File
@@ -264,6 +264,18 @@ func TestRender_email(t *testing.T) {
"send email to info@gitea.co.uk.",
`<p>send email to <a href="mailto:info@gitea.co.uk" rel="nofollow">info@gitea.co.uk</a>.</p>`)
test(
`j.doe@example.com,
j.doe@example.com.
j.doe@example.com;
j.doe@example.com?
j.doe@example.com!`,
`<p><a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>,<br/>
<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>.<br/>
<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>;<br/>
<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>?<br/>
<a href="mailto:j.doe@example.com" rel="nofollow">j.doe@example.com</a>!</p>`)
// Test that should *not* be turned into email links
test(
"\"info@gitea.com\"",
+4 -4
View File
@@ -158,7 +158,7 @@ func (r *Writer) WriteRegularLink(l org.RegularLink) {
case "image":
if l.Description == nil {
imageSrc := getMediaURL(link)
fmt.Fprintf(r, `<img src="%s" alt="%s" title="%s" />`, imageSrc, link, link)
fmt.Fprintf(r, `<img src="%s" alt="%s" />`, imageSrc, link)
} else {
description := strings.TrimPrefix(org.String(l.Description...), "file:")
imageSrc := getMediaURL([]byte(description))
@@ -167,18 +167,18 @@ func (r *Writer) WriteRegularLink(l org.RegularLink) {
case "video":
if l.Description == nil {
imageSrc := getMediaURL(link)
fmt.Fprintf(r, `<video src="%s" title="%s">%s</video>`, imageSrc, link, link)
fmt.Fprintf(r, `<video src="%s">%s</video>`, imageSrc, link)
} else {
description := strings.TrimPrefix(org.String(l.Description...), "file:")
videoSrc := getMediaURL([]byte(description))
fmt.Fprintf(r, `<a href="%s"><video src="%s" title="%s"></video></a>`, link, videoSrc, videoSrc)
fmt.Fprintf(r, `<a href="%s"><video src="%s">%s</video></a>`, link, videoSrc, videoSrc)
}
default:
description := string(link)
if l.Description != nil {
description = r.WriteNodesAsString(l.Description...)
}
fmt.Fprintf(r, `<a href="%s" title="%s">%s</a>`, link, description, description)
fmt.Fprintf(r, `<a href="%s">%s</a>`, link, description)
}
}
+11 -7
View File
@@ -34,12 +34,12 @@ func TestRender_StandardLinks(t *testing.T) {
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
}
googleRendered := "<p><a href=\"https://google.com/\" title=\"https://google.com/\">https://google.com/</a></p>"
test("[[https://google.com/]]", googleRendered)
test("[[https://google.com/]]",
`<p><a href="https://google.com/">https://google.com/</a></p>`)
lnk := util.URLJoin(AppSubURL, "WikiPage")
test("[[WikiPage][WikiPage]]",
"<p><a href=\""+lnk+"\" title=\"WikiPage\">WikiPage</a></p>")
`<p><a href="`+lnk+`">WikiPage</a></p>`)
}
func TestRender_Media(t *testing.T) {
@@ -59,19 +59,23 @@ func TestRender_Media(t *testing.T) {
result := util.URLJoin(AppSubURL, url)
test("[[file:"+url+"]]",
"<p><img src=\""+result+"\" alt=\""+result+"\" title=\""+result+"\" /></p>")
`<p><img src="`+result+`" alt="`+result+`" /></p>`)
// With description.
test("[[https://example.com][https://example.com/example.svg]]",
`<p><a href="https://example.com"><img src="https://example.com/example.svg" alt="https://example.com/example.svg" /></a></p>`)
test("[[https://example.com][pre https://example.com/example.svg post]]",
`<p><a href="https://example.com">pre <img src="https://example.com/example.svg" alt="https://example.com/example.svg" /> post</a></p>`)
test("[[https://example.com][https://example.com/example.mp4]]",
`<p><a href="https://example.com"><video src="https://example.com/example.mp4" title="https://example.com/example.mp4"></video></a></p>`)
`<p><a href="https://example.com"><video src="https://example.com/example.mp4">https://example.com/example.mp4</video></a></p>`)
test("[[https://example.com][pre https://example.com/example.mp4 post]]",
`<p><a href="https://example.com">pre <video src="https://example.com/example.mp4">https://example.com/example.mp4</video> post</a></p>`)
// Without description.
test("[[https://example.com/example.svg]]",
`<p><img src="https://example.com/example.svg" alt="https://example.com/example.svg" title="https://example.com/example.svg" /></p>`)
`<p><img src="https://example.com/example.svg" alt="https://example.com/example.svg" /></p>`)
test("[[https://example.com/example.mp4]]",
`<p><video src="https://example.com/example.mp4" title="https://example.com/example.mp4">https://example.com/example.mp4</video></p>`)
`<p><video src="https://example.com/example.mp4">https://example.com/example.mp4</video></p>`)
}
func TestRender_Source(t *testing.T) {
+32 -3
View File
@@ -18,9 +18,10 @@ import (
// Sanitizer is a protection wrapper of *bluemonday.Policy which does not allow
// any modification to the underlying policies once it's been created.
type Sanitizer struct {
defaultPolicy *bluemonday.Policy
rendererPolicies map[string]*bluemonday.Policy
init sync.Once
defaultPolicy *bluemonday.Policy
descriptionPolicy *bluemonday.Policy
rendererPolicies map[string]*bluemonday.Policy
init sync.Once
}
var (
@@ -41,6 +42,7 @@ func NewSanitizer() {
func InitializeSanitizer() {
sanitizer.rendererPolicies = map[string]*bluemonday.Policy{}
sanitizer.defaultPolicy = createDefaultPolicy()
sanitizer.descriptionPolicy = createRepoDescriptionPolicy()
for name, renderer := range renderers {
sanitizerRules := renderer.SanitizerRules()
@@ -161,6 +163,27 @@ func createDefaultPolicy() *bluemonday.Policy {
return policy
}
// createRepoDescriptionPolicy returns a minimal more strict policy that is used for
// repository descriptions.
func createRepoDescriptionPolicy() *bluemonday.Policy {
policy := bluemonday.NewPolicy()
// Allow italics and bold.
policy.AllowElements("i", "b", "em", "strong")
// Allow code.
policy.AllowElements("code")
// Allow links
policy.AllowAttrs("href", "target", "rel").OnElements("a")
// Allow classes for emojis
policy.AllowAttrs("class").Matching(regexp.MustCompile(`^emoji$`)).OnElements("img", "span")
policy.AllowAttrs("aria-label").OnElements("span")
return policy
}
func addSanitizerRules(policy *bluemonday.Policy, rules []setting.MarkupSanitizerRule) {
for _, rule := range rules {
if rule.AllowDataURIImages {
@@ -176,6 +199,12 @@ func addSanitizerRules(policy *bluemonday.Policy, rules []setting.MarkupSanitize
}
}
// SanitizeDescription sanitizes the HTML generated for a repository description.
func SanitizeDescription(s string) string {
NewSanitizer()
return sanitizer.descriptionPolicy.Sanitize(s)
}
// Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist.
func Sanitize(s string) string {
NewSanitizer()
+22
View File
@@ -73,6 +73,28 @@ func Test_Sanitizer(t *testing.T) {
}
}
func TestDescriptionSanitizer(t *testing.T) {
NewSanitizer()
testCases := []string{
`<h1>Title</h1>`, `Title`,
`<img src='img.png' alt='image'>`, ``,
`<span class="emoji" aria-label="thumbs up">THUMBS UP</span>`, `<span class="emoji" aria-label="thumbs up">THUMBS UP</span>`,
`<span style="color: red">Hello World</span>`, `<span>Hello World</span>`,
`<br>`, ``,
`<a href="https://example.com" target="_blank" rel="noopener noreferrer">https://example.com</a>`, `<a href="https://example.com" target="_blank" rel="noopener noreferrer">https://example.com</a>`,
`<mark>Important!</mark>`, `Important!`,
`<details>Click me! <summary>Nothing to see here.</summary></details>`, `Click me! Nothing to see here.`,
`<input type="hidden">`, ``,
`<b>I</b> have a <i>strong</i> <strong>opinion</strong> about <em>this</em>.`, `<b>I</b> have a <i>strong</i> <strong>opinion</strong> about <em>this</em>.`,
`Provides alternative <code>wg(8)</code> tool`, `Provides alternative <code>wg(8)</code> tool`,
}
for i := 0; i < len(testCases); i += 2 {
assert.Equal(t, testCases[i+1], SanitizeDescription(testCases[i]))
}
}
func TestSanitizeNonEscape(t *testing.T) {
descStr := "<scrİpt>&lt;script&gt;alert(document.domain)&lt;/script&gt;</scrİpt>"
+12 -11
View File
@@ -160,24 +160,25 @@ const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | o
// getDirectorySize returns the disk consumption for a given path
func getDirectorySize(path string) (int64, error) {
var size int64
err := filepath.WalkDir(path, func(_ string, info os.DirEntry, err error) error {
if err != nil {
if os.IsNotExist(err) { // ignore the error because the file maybe deleted during traversing.
return nil
}
err := filepath.WalkDir(path, func(_ string, entry os.DirEntry, err error) error {
if os.IsNotExist(err) { // ignore the error because some files (like temp/lock file) may be deleted during traversing.
return nil
} else if err != nil {
return err
}
if info.IsDir() {
if entry.IsDir() {
return nil
}
f, err := info.Info()
if err != nil {
info, err := entry.Info()
if os.IsNotExist(err) { // ignore the error as above
return nil
} else if err != nil {
return err
}
if (f.Mode() & notRegularFileMode) == 0 {
size += f.Size()
if (info.Mode() & notRegularFileMode) == 0 {
size += info.Size()
}
return err
return nil
})
return size, err
}
+4 -2
View File
@@ -22,9 +22,11 @@ var (
ZombieTaskTimeout time.Duration `ini:"ZOMBIE_TASK_TIMEOUT"`
EndlessTaskTimeout time.Duration `ini:"ENDLESS_TASK_TIMEOUT"`
AbandonedJobTimeout time.Duration `ini:"ABANDONED_JOB_TIMEOUT"`
SkipWorkflowStrings []string `ìni:"SKIP_WORKFLOW_STRINGS"`
}{
Enabled: true,
DefaultActionsURL: defaultActionsURLGitHub,
Enabled: true,
DefaultActionsURL: defaultActionsURLGitHub,
SkipWorkflowStrings: []string{"[skip ci]", "[ci skip]", "[no ci]", "[skip actions]", "[actions skip]"},
}
)
+1 -1
View File
@@ -13,7 +13,7 @@ var Attachment = struct {
}{
Storage: &Storage{},
AllowedTypes: ".csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip",
MaxSize: 4,
MaxSize: 2048,
MaxFiles: 5,
Enabled: true,
}
+3 -2
View File
@@ -109,7 +109,7 @@ func DBConnStr() (string, error) {
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, Database.MysqlCharset, tls)
case "postgres":
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, paramSep, Database.SSLMode)
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode)
case "mssql":
host, port := ParseMSSQLHostPort(Database.Host)
connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd)
@@ -157,7 +157,8 @@ func parsePostgreSQLHostPort(info string) (host, port string) {
return host, port
}
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) {
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbsslMode string) (connStr string) {
dbName, dbParam, _ := strings.Cut(dbName, "?")
host, port := parsePostgreSQLHostPort(dbHost)
connURL := url.URL{
Scheme: "postgres",
+8 -7
View File
@@ -59,38 +59,39 @@ func Test_parsePostgreSQLHostPort(t *testing.T) {
func Test_getPostgreSQLConnectionString(t *testing.T) {
tests := []struct {
Host string
Port string
User string
Passwd string
Name string
Param string
SSLMode string
Output string
}{
{
Host: "/tmp/pg.sock",
Port: "4321",
User: "testuser",
Passwd: "space space !#$%^^%^```-=?=",
Name: "gitea",
Param: "",
SSLMode: "false",
Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/gitea?host=%2Ftmp%2Fpg.sock&sslmode=false",
},
{
Host: "localhost",
Port: "1234",
User: "pgsqlusername",
Passwd: "I love Gitea!",
Name: "gitea",
Param: "",
SSLMode: "true",
Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/gitea?sslmode=true",
},
{
Host: "localhost:1234",
User: "user",
Passwd: "pass",
Name: "gitea?param=1",
Output: "postgres://user:pass@localhost:1234/gitea?param=1&sslmode=",
},
}
for _, test := range tests {
connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.Param, test.SSLMode)
connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.SSLMode)
assert.Equal(t, test.Output, connStr)
}
}
+1 -1
View File
@@ -186,7 +186,7 @@ var (
Enabled: true,
TempPath: "data/tmp/uploads",
AllowedTypes: "",
FileMaxSize: 3,
FileMaxSize: 50,
MaxFiles: 5,
},
+1
View File
@@ -33,6 +33,7 @@ var UI = struct {
CustomEmojisMap map[string]string `ini:"-"`
SearchRepoDescription bool
OnlyShowRelevantRepos bool
ExploreDefaultSort string `ini:"EXPLORE_PAGING_DEFAULT_SORT"`
Notification struct {
MinTimeout time.Duration
+4
View File
@@ -136,6 +136,10 @@ func (r *Route) Get(pattern string, h ...any) {
r.Methods("GET", pattern, h...)
}
func (r *Route) Options(pattern string, h ...any) {
r.Methods("OPTIONS", pattern, h...)
}
// GetOptions delegate get and options method
func (r *Route) GetOptions(pattern string, h ...any) {
r.Methods("GET,OPTIONS", pattern, h...)