Merge remote-tracking branch 'upstream/main' into limit-repo-size

This commit is contained in:
DmitryFrolovTri
2023-05-23 06:04:57 +00:00
228 changed files with 6149 additions and 5925 deletions
+52
View File
@@ -33,6 +33,58 @@ type ConfigProvider interface {
Save() error
}
// ConfigSectionKey only searches the keys in the given section, but it is O(n).
// ini package has a special behavior: with "[sec] a=1" and an empty "[sec.sub]",
// then in "[sec.sub]", Key()/HasKey() can always see "a=1" because it always tries parent sections.
// It returns nil if the key doesn't exist.
func ConfigSectionKey(sec ConfigSection, key string) *ini.Key {
if sec == nil {
return nil
}
for _, k := range sec.Keys() {
if k.Name() == key {
return k
}
}
return nil
}
func ConfigSectionKeyString(sec ConfigSection, key string, def ...string) string {
k := ConfigSectionKey(sec, key)
if k != nil && k.String() != "" {
return k.String()
}
if len(def) > 0 {
return def[0]
}
return ""
}
// ConfigInheritedKey works like ini.Section.Key(), but it always returns a new key instance, it is O(n) because NewKey is O(n)
// and the returned key is safe to be used with "MustXxx", it doesn't change the parent's values.
// Otherwise, ini.Section.Key().MustXxx would pollute the parent section's keys.
// It never returns nil.
func ConfigInheritedKey(sec ConfigSection, key string) *ini.Key {
k := sec.Key(key)
if k != nil && k.String() != "" {
newKey, _ := sec.NewKey(k.Name(), k.String())
return newKey
}
newKey, _ := sec.NewKey(key, "")
return newKey
}
func ConfigInheritedKeyString(sec ConfigSection, key string, def ...string) string {
k := sec.Key(key)
if k != nil && k.String() != "" {
return k.String()
}
if len(def) > 0 {
return def[0]
}
return ""
}
type iniFileConfigProvider struct {
opts *Options
*ini.File
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestConfigProviderBehaviors(t *testing.T) {
t.Run("BuggyKeyOverwritten", func(t *testing.T) {
cfg, _ := NewConfigProviderFromData(`
[foo]
key =
`)
sec := cfg.Section("foo")
secSub := cfg.Section("foo.bar")
secSub.Key("key").MustString("1") // try to read a key from subsection
assert.Equal(t, "1", sec.Key("key").String()) // TODO: BUGGY! the key in [foo] is overwritten
})
t.Run("SubsectionSeeParentKeys", func(t *testing.T) {
cfg, _ := NewConfigProviderFromData(`
[foo]
key = 123
`)
secSub := cfg.Section("foo.bar.xxx")
assert.Equal(t, "123", secSub.Key("key").String())
})
}
func TestConfigProviderHelper(t *testing.T) {
cfg, _ := NewConfigProviderFromData(`
[foo]
empty =
key = 123
`)
sec := cfg.Section("foo")
secSub := cfg.Section("foo.bar")
// test empty key
assert.Equal(t, "def", ConfigSectionKeyString(sec, "empty", "def"))
assert.Equal(t, "xyz", ConfigSectionKeyString(secSub, "empty", "xyz"))
// test non-inherited key, only see the keys in current section
assert.NotNil(t, ConfigSectionKey(sec, "key"))
assert.Nil(t, ConfigSectionKey(secSub, "key"))
// test default behavior
assert.Equal(t, "123", ConfigSectionKeyString(sec, "key"))
assert.Equal(t, "", ConfigSectionKeyString(secSub, "key"))
assert.Equal(t, "def", ConfigSectionKeyString(secSub, "key", "def"))
assert.Equal(t, "123", ConfigInheritedKeyString(secSub, "key"))
// Workaround for ini package's BuggyKeyOverwritten behavior
assert.Equal(t, "", ConfigSectionKeyString(sec, "empty"))
assert.Equal(t, "", ConfigSectionKeyString(secSub, "empty"))
assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("def"))
assert.Equal(t, "def", ConfigInheritedKey(secSub, "empty").MustString("xyz"))
assert.Equal(t, "", ConfigSectionKeyString(sec, "empty"))
assert.Equal(t, "def", ConfigSectionKeyString(secSub, "empty"))
}
+1 -1
View File
@@ -92,7 +92,7 @@ func loadDBSetting(rootCfg ConfigProvider) {
Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0)
Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
Database.LogSQL = sec.Key("LOG_SQL").MustBool(true)
Database.LogSQL = sec.Key("LOG_SQL").MustBool(false)
Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10)
Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
Database.AutoMigration = sec.Key("AUTO_MIGRATION").MustBool(true)
+204 -337
View File
@@ -10,384 +10,251 @@ import (
"path"
"path/filepath"
"strings"
"sync"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
)
var (
filenameSuffix = ""
descriptionLock = sync.RWMutex{}
logDescriptions = make(map[string]*LogDescription)
)
type LogGlobalConfig struct {
RootPath string
// Log settings
var Log struct {
Mode string
Level log.Level
StacktraceLogLevel string
RootPath string
EnableSSHLog bool
EnableXORMLog bool
StacktraceLogLevel log.Level
BufferLen int
DisableRouterLog bool
EnableSSHLog bool
EnableAccessLog bool
AccessLogTemplate string
BufferLength int64
RequestIDHeaders []string
}
// GetLogDescriptions returns a race safe set of descriptions
func GetLogDescriptions() map[string]*LogDescription {
descriptionLock.RLock()
defer descriptionLock.RUnlock()
descs := make(map[string]*LogDescription, len(logDescriptions))
for k, v := range logDescriptions {
subLogDescriptions := make([]SubLogDescription, len(v.SubLogDescriptions))
copy(subLogDescriptions, v.SubLogDescriptions)
var Log LogGlobalConfig
descs[k] = &LogDescription{
Name: v.Name,
SubLogDescriptions: subLogDescriptions,
}
}
return descs
}
const accessLogTemplateDefault = `{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"`
// AddLogDescription adds a set of descriptions to the complete description
func AddLogDescription(key string, description *LogDescription) {
descriptionLock.Lock()
defer descriptionLock.Unlock()
logDescriptions[key] = description
}
// AddSubLogDescription adds a sub log description
func AddSubLogDescription(key string, subLogDescription SubLogDescription) bool {
descriptionLock.Lock()
defer descriptionLock.Unlock()
desc, ok := logDescriptions[key]
if !ok {
return false
}
for i, sub := range desc.SubLogDescriptions {
if sub.Name == subLogDescription.Name {
desc.SubLogDescriptions[i] = subLogDescription
return true
}
}
desc.SubLogDescriptions = append(desc.SubLogDescriptions, subLogDescription)
return true
}
// RemoveSubLogDescription removes a sub log description
func RemoveSubLogDescription(key, name string) bool {
descriptionLock.Lock()
defer descriptionLock.Unlock()
desc, ok := logDescriptions[key]
if !ok {
return false
}
for i, sub := range desc.SubLogDescriptions {
if sub.Name == name {
desc.SubLogDescriptions = append(desc.SubLogDescriptions[:i], desc.SubLogDescriptions[i+1:]...)
return true
}
}
return false
}
type defaultLogOptions struct {
levelName string // LogLevel
flags string
filename string // path.Join(LogRootPath, "gitea.log")
bufferLength int64
disableConsole bool
}
func newDefaultLogOptions() defaultLogOptions {
return defaultLogOptions{
levelName: Log.Level.String(),
flags: "stdflags",
filename: filepath.Join(Log.RootPath, "gitea.log"),
bufferLength: 10000,
disableConsole: false,
}
}
// SubLogDescription describes a sublogger
type SubLogDescription struct {
Name string
Provider string
Config string
}
// LogDescription describes a named logger
type LogDescription struct {
Name string
SubLogDescriptions []SubLogDescription
}
func getLogLevel(section ConfigSection, key string, defaultValue log.Level) log.Level {
value := section.Key(key).MustString(defaultValue.String())
return log.FromString(value)
}
func getStacktraceLogLevel(section ConfigSection, key, defaultValue string) string {
value := section.Key(key).MustString(defaultValue)
return log.FromString(value).String()
}
func loadLogFrom(rootCfg ConfigProvider) {
func loadLogGlobalFrom(rootCfg ConfigProvider) {
sec := rootCfg.Section("log")
Log.Level = getLogLevel(sec, "LEVEL", log.INFO)
Log.StacktraceLogLevel = getStacktraceLogLevel(sec, "STACKTRACE_LEVEL", "None")
Log.Level = log.LevelFromString(sec.Key("LEVEL").MustString(log.INFO.String()))
Log.StacktraceLogLevel = log.LevelFromString(sec.Key("STACKTRACE_LEVEL").MustString(log.NONE.String()))
Log.BufferLen = sec.Key("BUFFER_LEN").MustInt(10000)
Log.Mode = sec.Key("MODE").MustString("console")
Log.RootPath = sec.Key("ROOT_PATH").MustString(path.Join(AppWorkPath, "log"))
forcePathSeparator(Log.RootPath)
Log.BufferLength = sec.Key("BUFFER_LEN").MustInt64(10000)
if !filepath.IsAbs(Log.RootPath) {
Log.RootPath = filepath.Join(AppWorkPath, Log.RootPath)
}
Log.RootPath = util.FilePathJoinAbs(Log.RootPath)
Log.EnableSSHLog = sec.Key("ENABLE_SSH_LOG").MustBool(false)
Log.EnableAccessLog = sec.Key("ENABLE_ACCESS_LOG").MustBool(false)
Log.AccessLogTemplate = sec.Key("ACCESS_LOG_TEMPLATE").MustString(
`{{.Ctx.RemoteHost}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}" "{{.Ctx.Req.UserAgent}}"`,
)
Log.AccessLogTemplate = sec.Key("ACCESS_LOG_TEMPLATE").MustString(accessLogTemplateDefault)
Log.RequestIDHeaders = sec.Key("REQUEST_ID_HEADERS").Strings(",")
// the `MustString` updates the default value, and `log.ACCESS` is used by `generateNamedLogger("access")` later
_ = rootCfg.Section("log").Key("ACCESS").MustString("file")
sec.Key("ROUTER").MustString("console")
// Allow [log] DISABLE_ROUTER_LOG to override [server] DISABLE_ROUTER_LOG
Log.DisableRouterLog = sec.Key("DISABLE_ROUTER_LOG").MustBool(Log.DisableRouterLog)
Log.EnableXORMLog = rootCfg.Section("log").Key("ENABLE_XORM_LOG").MustBool(true)
}
func generateLogConfig(sec ConfigSection, name string, defaults defaultLogOptions) (mode, jsonConfig, levelName string) {
level := getLogLevel(sec, "LEVEL", Log.Level)
levelName = level.String()
stacktraceLevelName := getStacktraceLogLevel(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel)
stacktraceLevel := log.FromString(stacktraceLevelName)
mode = name
keys := sec.Keys()
logPath := defaults.filename
flags := log.FlagsFromString(defaults.flags)
expression := ""
prefix := ""
for _, key := range keys {
switch key.Name() {
case "MODE":
mode = key.MustString(name)
case "FILE_NAME":
logPath = key.MustString(defaults.filename)
forcePathSeparator(logPath)
if !filepath.IsAbs(logPath) {
logPath = path.Join(Log.RootPath, logPath)
}
case "FLAGS":
flags = log.FlagsFromString(key.MustString(defaults.flags))
case "EXPRESSION":
expression = key.MustString("")
case "PREFIX":
prefix = key.MustString("")
}
}
logConfig := map[string]interface{}{
"level": level.String(),
"expression": expression,
"prefix": prefix,
"flags": flags,
"stacktraceLevel": stacktraceLevel.String(),
}
// Generate log configuration.
switch mode {
case "console":
useStderr := sec.Key("STDERR").MustBool(false)
logConfig["stderr"] = useStderr
if useStderr {
logConfig["colorize"] = sec.Key("COLORIZE").MustBool(log.CanColorStderr)
} else {
logConfig["colorize"] = sec.Key("COLORIZE").MustBool(log.CanColorStdout)
}
case "file":
if err := os.MkdirAll(path.Dir(logPath), os.ModePerm); err != nil {
panic(err.Error())
}
logConfig["filename"] = logPath + filenameSuffix
logConfig["rotate"] = sec.Key("LOG_ROTATE").MustBool(true)
logConfig["maxsize"] = 1 << uint(sec.Key("MAX_SIZE_SHIFT").MustInt(28))
logConfig["daily"] = sec.Key("DAILY_ROTATE").MustBool(true)
logConfig["maxdays"] = sec.Key("MAX_DAYS").MustInt(7)
logConfig["compress"] = sec.Key("COMPRESS").MustBool(true)
logConfig["compressionLevel"] = sec.Key("COMPRESSION_LEVEL").MustInt(-1)
case "conn":
logConfig["reconnectOnMsg"] = sec.Key("RECONNECT_ON_MSG").MustBool()
logConfig["reconnect"] = sec.Key("RECONNECT").MustBool()
logConfig["net"] = sec.Key("PROTOCOL").In("tcp", []string{"tcp", "unix", "udp"})
logConfig["addr"] = sec.Key("ADDR").MustString(":7020")
case "smtp":
logConfig["username"] = sec.Key("USER").MustString("example@example.com")
logConfig["password"] = sec.Key("PASSWD").MustString("******")
logConfig["host"] = sec.Key("HOST").MustString("127.0.0.1:25")
sendTos := strings.Split(sec.Key("RECEIVERS").MustString(""), ",")
for i, address := range sendTos {
sendTos[i] = strings.TrimSpace(address)
}
logConfig["sendTos"] = sendTos
logConfig["subject"] = sec.Key("SUBJECT").MustString("Diagnostic message from Gitea")
}
logConfig["colorize"] = sec.Key("COLORIZE").MustBool(false)
byteConfig, err := json.Marshal(logConfig)
if err != nil {
log.Error("Failed to marshal log configuration: %v %v", logConfig, err)
return
}
jsonConfig = string(byteConfig)
return mode, jsonConfig, levelName
}
func generateNamedLogger(rootCfg ConfigProvider, key string, options defaultLogOptions) *LogDescription {
description := LogDescription{
Name: key,
}
sections := strings.Split(rootCfg.Section("log").Key(strings.ToUpper(key)).MustString(""), ",")
for i := 0; i < len(sections); i++ {
sections[i] = strings.TrimSpace(sections[i])
}
for _, name := range sections {
if len(name) == 0 || (name == "console" && options.disableConsole) {
continue
}
sec, err := rootCfg.GetSection("log." + name + "." + key)
if err != nil {
sec, _ = rootCfg.NewSection("log." + name + "." + key)
}
provider, config, levelName := generateLogConfig(sec, name, options)
if err := log.NewNamedLogger(key, options.bufferLength, name, provider, config); err != nil {
// Maybe panic here?
log.Error("Could not create new named logger: %v", err.Error())
}
description.SubLogDescriptions = append(description.SubLogDescriptions, SubLogDescription{
Name: name,
Provider: provider,
Config: config,
})
log.Info("%s Log: %s(%s:%s)", util.ToTitleCase(key), util.ToTitleCase(name), provider, levelName)
}
AddLogDescription(key, &description)
return &description
}
// initLogFrom initializes logging with settings from configuration provider
func initLogFrom(rootCfg ConfigProvider) {
func prepareLoggerConfig(rootCfg ConfigProvider) {
sec := rootCfg.Section("log")
options := newDefaultLogOptions()
options.bufferLength = Log.BufferLength
description := LogDescription{
Name: log.DEFAULT,
if !sec.HasKey("logger.default.MODE") {
sec.Key("logger.default.MODE").MustString(",")
}
sections := strings.Split(sec.Key("MODE").MustString("console"), ",")
useConsole := false
for _, name := range sections {
name = strings.TrimSpace(name)
if name == "" {
continue
}
if name == "console" {
useConsole = true
}
sec, err := rootCfg.GetSection("log." + name + ".default")
if err != nil {
sec, err = rootCfg.GetSection("log." + name)
if err != nil {
sec, _ = rootCfg.NewSection("log." + name)
}
}
provider, config, levelName := generateLogConfig(sec, name, options)
log.NewLogger(options.bufferLength, name, provider, config)
description.SubLogDescriptions = append(description.SubLogDescriptions, SubLogDescription{
Name: name,
Provider: provider,
Config: config,
})
log.Info("Gitea Log Mode: %s(%s:%s)", util.ToTitleCase(name), util.ToTitleCase(provider), levelName)
deprecatedSetting(rootCfg, "log", "ACCESS", "log", "logger.access.MODE", "1.21")
deprecatedSetting(rootCfg, "log", "ENABLE_ACCESS_LOG", "log", "logger.access.MODE", "1.21")
if val := sec.Key("ACCESS").String(); val != "" {
sec.Key("logger.access.MODE").MustString(val)
}
if sec.HasKey("ENABLE_ACCESS_LOG") && !sec.Key("ENABLE_ACCESS_LOG").MustBool() {
sec.Key("logger.access.MODE").SetValue("")
}
AddLogDescription(log.DEFAULT, &description)
if !useConsole {
log.Info("According to the configuration, subsequent logs will not be printed to the console")
if err := log.DelLogger("console"); err != nil {
log.Fatal("Cannot delete console logger: %v", err)
}
deprecatedSetting(rootCfg, "log", "ROUTER", "log", "logger.router.MODE", "1.21")
deprecatedSetting(rootCfg, "log", "DISABLE_ROUTER_LOG", "log", "logger.router.MODE", "1.21")
if val := sec.Key("ROUTER").String(); val != "" {
sec.Key("logger.router.MODE").MustString(val)
}
if !sec.HasKey("logger.router.MODE") {
sec.Key("logger.router.MODE").MustString(",") // use default logger
}
if sec.HasKey("DISABLE_ROUTER_LOG") && sec.Key("DISABLE_ROUTER_LOG").MustBool() {
sec.Key("logger.router.MODE").SetValue("")
}
// Finally redirect the default golog to here
golog.SetFlags(0)
golog.SetPrefix("")
golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
deprecatedSetting(rootCfg, "log", "XORM", "log", "logger.xorm.MODE", "1.21")
deprecatedSetting(rootCfg, "log", "ENABLE_XORM_LOG", "log", "logger.xorm.MODE", "1.21")
if val := sec.Key("XORM").String(); val != "" {
sec.Key("logger.xorm.MODE").MustString(val)
}
if !sec.HasKey("logger.xorm.MODE") {
sec.Key("logger.xorm.MODE").MustString(",") // use default logger
}
if sec.HasKey("ENABLE_XORM_LOG") && !sec.Key("ENABLE_XORM_LOG").MustBool() {
sec.Key("logger.xorm.MODE").SetValue("")
}
}
func LogPrepareFilenameForWriter(fileName, defaultFileName string) string {
if fileName == "" {
fileName = defaultFileName
}
if !filepath.IsAbs(fileName) {
fileName = filepath.Join(Log.RootPath, fileName)
} else {
fileName = filepath.Clean(fileName)
}
if err := os.MkdirAll(filepath.Dir(fileName), os.ModePerm); err != nil {
panic(fmt.Sprintf("unable to create directory for log %q: %v", fileName, err.Error()))
}
return fileName
}
func loadLogModeByName(rootCfg ConfigProvider, loggerName, modeName string) (writerName, writerType string, writerMode log.WriterMode, err error) {
sec := rootCfg.Section("log." + modeName)
writerMode = log.WriterMode{}
writerType = ConfigSectionKeyString(sec, "MODE")
if writerType == "" {
writerType = modeName
}
writerName = modeName
defaultFlags := "stdflags"
defaultFilaName := "gitea.log"
if loggerName == "access" {
// "access" logger is special, by default it doesn't have output flags, so it also needs a new writer name to avoid conflicting with other writers.
// so "access" logger's writer name is usually "file.access" or "console.access"
writerName += ".access"
defaultFlags = "none"
defaultFilaName = "access.log"
}
writerMode.Level = log.LevelFromString(ConfigInheritedKeyString(sec, "LEVEL", Log.Level.String()))
writerMode.StacktraceLevel = log.LevelFromString(ConfigInheritedKeyString(sec, "STACKTRACE_LEVEL", Log.StacktraceLogLevel.String()))
writerMode.Prefix = ConfigInheritedKeyString(sec, "PREFIX")
writerMode.Expression = ConfigInheritedKeyString(sec, "EXPRESSION")
writerMode.Flags = log.FlagsFromString(ConfigInheritedKeyString(sec, "FLAGS", defaultFlags))
switch writerType {
case "console":
useStderr := ConfigInheritedKey(sec, "STDERR").MustBool(false)
defaultCanColor := log.CanColorStdout
if useStderr {
defaultCanColor = log.CanColorStderr
}
writerOption := log.WriterConsoleOption{Stderr: useStderr}
writerMode.Colorize = ConfigInheritedKey(sec, "COLORIZE").MustBool(defaultCanColor)
writerMode.WriterOption = writerOption
case "file":
fileName := LogPrepareFilenameForWriter(ConfigInheritedKey(sec, "FILE_NAME").String(), defaultFilaName)
writerOption := log.WriterFileOption{}
writerOption.FileName = fileName + filenameSuffix // FIXME: the suffix doesn't seem right, see its related comments
writerOption.LogRotate = ConfigInheritedKey(sec, "LOG_ROTATE").MustBool(true)
writerOption.MaxSize = 1 << uint(ConfigInheritedKey(sec, "MAX_SIZE_SHIFT").MustInt(28))
writerOption.DailyRotate = ConfigInheritedKey(sec, "DAILY_ROTATE").MustBool(true)
writerOption.MaxDays = ConfigInheritedKey(sec, "MAX_DAYS").MustInt(7)
writerOption.Compress = ConfigInheritedKey(sec, "COMPRESS").MustBool(true)
writerOption.CompressionLevel = ConfigInheritedKey(sec, "COMPRESSION_LEVEL").MustInt(-1)
writerMode.WriterOption = writerOption
case "conn":
writerOption := log.WriterConnOption{}
writerOption.ReconnectOnMsg = ConfigInheritedKey(sec, "RECONNECT_ON_MSG").MustBool()
writerOption.Reconnect = ConfigInheritedKey(sec, "RECONNECT").MustBool()
writerOption.Protocol = ConfigInheritedKey(sec, "PROTOCOL").In("tcp", []string{"tcp", "unix", "udp"})
writerOption.Addr = ConfigInheritedKey(sec, "ADDR").MustString(":7020")
writerMode.WriterOption = writerOption
default:
if !log.HasEventWriter(writerType) {
return "", "", writerMode, fmt.Errorf("invalid log writer type (mode): %s", writerType)
}
}
return writerName, writerType, writerMode, nil
}
var filenameSuffix = ""
// RestartLogsWithPIDSuffix restarts the logs with a PID suffix on files
// FIXME: it seems not right, it breaks log rotating or log collectors
func RestartLogsWithPIDSuffix() {
filenameSuffix = fmt.Sprintf(".%d", os.Getpid())
InitLogs(false)
initAllLoggers() // when forking, before restarting, rename logger file and re-init all loggers
}
// InitLogs creates all the log services
func InitLogs(disableConsole bool) {
initLogFrom(CfgProvider)
func InitLoggersForTest() {
initAllLoggers()
}
if !Log.DisableRouterLog {
options := newDefaultLogOptions()
options.filename = filepath.Join(Log.RootPath, "router.log")
options.flags = "date,time" // For the router we don't want any prefixed flags
options.bufferLength = Log.BufferLength
generateNamedLogger(CfgProvider, "router", options)
// initAllLoggers creates all the log services
func initAllLoggers() {
initManagedLoggers(log.GetManager(), CfgProvider)
golog.SetFlags(0)
golog.SetPrefix("")
golog.SetOutput(log.LoggerToWriter(log.GetLogger(log.DEFAULT).Info))
}
func initManagedLoggers(manager *log.LoggerManager, cfg ConfigProvider) {
loadLogGlobalFrom(cfg)
prepareLoggerConfig(cfg)
initLoggerByName(manager, cfg, log.DEFAULT) // default
initLoggerByName(manager, cfg, "access")
initLoggerByName(manager, cfg, "router")
initLoggerByName(manager, cfg, "xorm")
}
func initLoggerByName(manager *log.LoggerManager, rootCfg ConfigProvider, loggerName string) {
sec := rootCfg.Section("log")
keyPrefix := "logger." + loggerName
disabled := sec.HasKey(keyPrefix+".MODE") && sec.Key(keyPrefix+".MODE").String() == ""
if disabled {
return
}
if Log.EnableAccessLog {
options := newDefaultLogOptions()
options.filename = filepath.Join(Log.RootPath, "access.log")
options.flags = "" // For the router we don't want any prefixed flags
options.bufferLength = Log.BufferLength
generateNamedLogger(CfgProvider, "access", options)
modeVal := sec.Key(keyPrefix + ".MODE").String()
if modeVal == "," {
modeVal = Log.Mode
}
initSQLLogFrom(CfgProvider, disableConsole)
}
// InitSQLLog initializes xorm logger setting
func InitSQLLog(disableConsole bool) {
initSQLLogFrom(CfgProvider, disableConsole)
}
func initSQLLogFrom(rootCfg ConfigProvider, disableConsole bool) {
if Log.EnableXORMLog {
options := newDefaultLogOptions()
options.filename = filepath.Join(Log.RootPath, "xorm.log")
options.bufferLength = Log.BufferLength
options.disableConsole = disableConsole
rootCfg.Section("log").Key("XORM").MustString(",")
generateNamedLogger(rootCfg, "xorm", options)
var eventWriters []log.EventWriter
modes := strings.Split(modeVal, ",")
for _, modeName := range modes {
modeName = strings.TrimSpace(modeName)
if modeName == "" {
continue
}
writerName, writerType, writerMode, err := loadLogModeByName(rootCfg, loggerName, modeName)
if err != nil {
log.FallbackErrorf("Failed to load writer mode %q for logger %s: %v", modeName, loggerName, err)
continue
}
if writerMode.BufferLen == 0 {
writerMode.BufferLen = Log.BufferLen
}
eventWriter := manager.GetSharedWriter(writerName)
if eventWriter == nil {
eventWriter, err = manager.NewSharedWriter(writerName, writerType, writerMode)
if err != nil {
log.FallbackErrorf("Failed to create event writer for logger %s: %v", loggerName, err)
continue
}
}
eventWriters = append(eventWriters, eventWriter)
}
manager.GetLogger(loggerName).RemoveAllWriters().AddWriters(eventWriters...)
}
func InitSQLLoggersForCli(level log.Level) {
log.SetConsoleLogger("xorm", "console", level)
}
func IsAccessLogEnabled() bool {
return log.IsLoggerEnabled("access")
}
func IsRouteLogEnabled() bool {
return log.IsLoggerEnabled("router")
}
+387
View File
@@ -0,0 +1,387 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"path/filepath"
"strings"
"testing"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func initLoggersByConfig(t *testing.T, config string) (*log.LoggerManager, func()) {
oldLogConfig := Log
Log = LogGlobalConfig{}
defer func() {
Log = oldLogConfig
}()
cfg, err := NewConfigProviderFromData(config)
assert.NoError(t, err)
manager := log.NewManager()
initManagedLoggers(manager, cfg)
return manager, manager.Close
}
func toJSON(v interface{}) string {
b, _ := json.MarshalIndent(v, "", "\t")
return string(b)
}
func TestLogConfigDefault(t *testing.T) {
manager, managerClose := initLoggersByConfig(t, ``)
defer managerClose()
writerDump := `
{
"console": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "info",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Stderr": false
},
"WriterType": "console"
}
}
`
dump := manager.GetLogger(log.DEFAULT).DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
dump = manager.GetLogger("access").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
dump = manager.GetLogger("router").DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
dump = manager.GetLogger("xorm").DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
}
func TestLogConfigDisable(t *testing.T) {
manager, managerClose := initLoggersByConfig(t, `
[log]
logger.router.MODE =
logger.xorm.MODE =
`)
defer managerClose()
writerDump := `
{
"console": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "info",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Stderr": false
},
"WriterType": "console"
}
}
`
dump := manager.GetLogger(log.DEFAULT).DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
dump = manager.GetLogger("access").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
dump = manager.GetLogger("router").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
dump = manager.GetLogger("xorm").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
}
func TestLogConfigLegacyDefault(t *testing.T) {
manager, managerClose := initLoggersByConfig(t, `
[log]
MODE = console
`)
defer managerClose()
writerDump := `
{
"console": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "info",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Stderr": false
},
"WriterType": "console"
}
}
`
dump := manager.GetLogger(log.DEFAULT).DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
dump = manager.GetLogger("access").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
dump = manager.GetLogger("router").DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
dump = manager.GetLogger("xorm").DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
}
func TestLogConfigLegacyMode(t *testing.T) {
tempDir := t.TempDir()
tempPath := func(file string) string {
return filepath.Join(tempDir, file)
}
manager, managerClose := initLoggersByConfig(t, `
[log]
ROOT_PATH = `+tempDir+`
MODE = file
ROUTER = file
ACCESS = file
`)
defer managerClose()
writerDump := `
{
"file": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "info",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Compress": true,
"CompressionLevel": -1,
"DailyRotate": true,
"FileName": "$FILENAME",
"LogRotate": true,
"MaxDays": 7,
"MaxSize": 268435456
},
"WriterType": "file"
}
}
`
writerDumpAccess := `
{
"file.access": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "none",
"Level": "info",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Compress": true,
"CompressionLevel": -1,
"DailyRotate": true,
"FileName": "$FILENAME",
"LogRotate": true,
"MaxDays": 7,
"MaxSize": 268435456
},
"WriterType": "file"
}
}
`
dump := manager.GetLogger(log.DEFAULT).DumpWriters()
require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("gitea.log")), toJSON(dump))
dump = manager.GetLogger("access").DumpWriters()
require.JSONEq(t, strings.ReplaceAll(writerDumpAccess, "$FILENAME", tempPath("access.log")), toJSON(dump))
dump = manager.GetLogger("router").DumpWriters()
require.JSONEq(t, strings.ReplaceAll(writerDump, "$FILENAME", tempPath("gitea.log")), toJSON(dump))
}
func TestLogConfigLegacyModeDisable(t *testing.T) {
manager, managerClose := initLoggersByConfig(t, `
[log]
ROUTER = file
ACCESS = file
DISABLE_ROUTER_LOG = true
ENABLE_ACCESS_LOG = false
`)
defer managerClose()
dump := manager.GetLogger("access").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
dump = manager.GetLogger("router").DumpWriters()
require.JSONEq(t, "{}", toJSON(dump))
}
func TestLogConfigNewConfig(t *testing.T) {
manager, managerClose := initLoggersByConfig(t, `
[log]
logger.access.MODE = console
logger.xorm.MODE = console, console-1
[log.console]
LEVEL = warn
[log.console-1]
MODE = console
LEVEL = error
STDERR = true
`)
defer managerClose()
writerDump := `
{
"console": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "warn",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Stderr": false
},
"WriterType": "console"
},
"console-1": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "error",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Stderr": true
},
"WriterType": "console"
}
}
`
writerDumpAccess := `
{
"console.access": {
"BufferLen": 10000,
"Colorize": false,
"Expression": "",
"Flags": "none",
"Level": "warn",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Stderr": false
},
"WriterType": "console"
}
}
`
dump := manager.GetLogger("xorm").DumpWriters()
require.JSONEq(t, writerDump, toJSON(dump))
dump = manager.GetLogger("access").DumpWriters()
require.JSONEq(t, writerDumpAccess, toJSON(dump))
}
func TestLogConfigModeFile(t *testing.T) {
tempDir := t.TempDir()
tempPath := func(file string) string {
return filepath.Join(tempDir, file)
}
manager, managerClose := initLoggersByConfig(t, `
[log]
ROOT_PATH = `+tempDir+`
BUFFER_LEN = 10
MODE = file, file1
[log.file1]
MODE = file
LEVEL = error
STACKTRACE_LEVEL = fatal
EXPRESSION = filter
FLAGS = medfile
PREFIX = "[Prefix] "
FILE_NAME = file-xxx.log
LOG_ROTATE = false
MAX_SIZE_SHIFT = 1
DAILY_ROTATE = false
MAX_DAYS = 90
COMPRESS = false
COMPRESSION_LEVEL = 4
`)
defer managerClose()
writerDump := `
{
"file": {
"BufferLen": 10,
"Colorize": false,
"Expression": "",
"Flags": "stdflags",
"Level": "info",
"Prefix": "",
"StacktraceLevel": "none",
"WriterOption": {
"Compress": true,
"CompressionLevel": -1,
"DailyRotate": true,
"FileName": "$FILENAME-0",
"LogRotate": true,
"MaxDays": 7,
"MaxSize": 268435456
},
"WriterType": "file"
},
"file1": {
"BufferLen": 10,
"Colorize": false,
"Expression": "filter",
"Flags": "medfile",
"Level": "error",
"Prefix": "[Prefix] ",
"StacktraceLevel": "fatal",
"WriterOption": {
"Compress": false,
"CompressionLevel": 4,
"DailyRotate": false,
"FileName": "$FILENAME-1",
"LogRotate": false,
"MaxDays": 90,
"MaxSize": 2
},
"WriterType": "file"
}
}
`
dump := manager.GetLogger(log.DEFAULT).DumpWriters()
expected := writerDump
expected = strings.ReplaceAll(expected, "$FILENAME-0", tempPath("gitea.log"))
expected = strings.ReplaceAll(expected, "$FILENAME-1", tempPath("file-xxx.log"))
require.JSONEq(t, expected, toJSON(dump))
}
+2
View File
@@ -31,6 +31,7 @@ var (
LimitSizeConan int64
LimitSizeConda int64
LimitSizeContainer int64
LimitSizeCran int64
LimitSizeDebian int64
LimitSizeGeneric int64
LimitSizeGo int64
@@ -78,6 +79,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) {
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA")
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
Packages.LimitSizeCran = mustBytes(sec, "LIMIT_SIZE_CRAN")
Packages.LimitSizeDebian = mustBytes(sec, "LIMIT_SIZE_DEBIAN")
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
Packages.LimitSizeGo = mustBytes(sec, "LIMIT_SIZE_GO")
-1
View File
@@ -307,7 +307,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) {
Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1)
Repository.DefaultBranch = sec.Key("DEFAULT_BRANCH").MustString(Repository.DefaultBranch)
RepoRootPath = sec.Key("ROOT").MustString(path.Join(AppDataPath, "gitea-repositories"))
forcePathSeparator(RepoRootPath)
if !filepath.IsAbs(RepoRootPath) {
RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath)
} else {
-1
View File
@@ -317,7 +317,6 @@ func loadServerFrom(rootCfg ConfigProvider) {
PortToRedirect = sec.Key("PORT_TO_REDIRECT").MustString("80")
RedirectorUseProxyProtocol = sec.Key("REDIRECTOR_USE_PROXY_PROTOCOL").MustBool(UseProxyProtocol)
OfflineMode = sec.Key("OFFLINE_MODE").MustBool()
Log.DisableRouterLog = sec.Key("DISABLE_ROUTER_LOG").MustBool()
if len(StaticRootPath) == 0 {
StaticRootPath = AppWorkPath
}
+23 -4
View File
@@ -10,6 +10,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/structs"
"github.com/gobwas/glob"
)
// enumerates all the types of captchas
@@ -33,8 +35,8 @@ var Service = struct {
ResetPwdCodeLives int
RegisterEmailConfirm bool
RegisterManualConfirm bool
EmailDomainWhitelist []string
EmailDomainBlocklist []string
EmailDomainAllowList []glob.Glob
EmailDomainBlockList []glob.Glob
DisableRegistration bool
AllowOnlyInternalRegistration bool
AllowOnlyExternalRegistration bool
@@ -114,6 +116,20 @@ func (a AllowedVisibility) ToVisibleTypeSlice() (result []structs.VisibleType) {
return result
}
func CompileEmailGlobList(sec ConfigSection, keys ...string) (globs []glob.Glob) {
for _, key := range keys {
list := sec.Key(key).Strings(",")
for _, s := range list {
if g, err := glob.Compile(s); err == nil {
globs = append(globs, g)
} else {
log.Error("Skip invalid email allow/block list expression %q: %v", s, err)
}
}
}
return globs
}
func loadServiceFrom(rootCfg ConfigProvider) {
sec := rootCfg.Section("service")
Service.ActiveCodeLives = sec.Key("ACTIVE_CODE_LIVE_MINUTES").MustInt(180)
@@ -130,8 +146,11 @@ func loadServiceFrom(rootCfg ConfigProvider) {
} else {
Service.RegisterManualConfirm = false
}
Service.EmailDomainWhitelist = sec.Key("EMAIL_DOMAIN_WHITELIST").Strings(",")
Service.EmailDomainBlocklist = sec.Key("EMAIL_DOMAIN_BLOCKLIST").Strings(",")
if sec.HasKey("EMAIL_DOMAIN_WHITELIST") {
deprecatedSetting(rootCfg, "service", "EMAIL_DOMAIN_WHITELIST", "service", "EMAIL_DOMAIN_ALLOWLIST", "1.21")
}
Service.EmailDomainAllowList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_WHITELIST", "EMAIL_DOMAIN_ALLOWLIST")
Service.EmailDomainBlockList = CompileEmailGlobList(sec, "EMAIL_DOMAIN_BLOCKLIST")
Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration))
Service.ShowMilestonesDashboardPage = sec.Key("SHOW_MILESTONES_DASHBOARD_PAGE").MustBool(true)
Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool()
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"testing"
"github.com/gobwas/glob"
"github.com/stretchr/testify/assert"
)
func TestLoadServices(t *testing.T) {
oldService := Service
defer func() {
Service = oldService
}()
cfg, err := NewConfigProviderFromData(`
[service]
EMAIL_DOMAIN_WHITELIST = d1, *.w
EMAIL_DOMAIN_ALLOWLIST = d2, *.a
EMAIL_DOMAIN_BLOCKLIST = d3, *.b
`)
assert.NoError(t, err)
loadServiceFrom(cfg)
match := func(globs []glob.Glob, s string) bool {
for _, g := range globs {
if g.Match(s) {
return true
}
}
return false
}
assert.True(t, match(Service.EmailDomainAllowList, "d1"))
assert.True(t, match(Service.EmailDomainAllowList, "foo.w"))
assert.True(t, match(Service.EmailDomainAllowList, "d2"))
assert.True(t, match(Service.EmailDomainAllowList, "foo.a"))
assert.False(t, match(Service.EmailDomainAllowList, "d3"))
assert.True(t, match(Service.EmailDomainBlockList, "d3"))
assert.True(t, match(Service.EmailDomainBlockList, "foo.b"))
assert.False(t, match(Service.EmailDomainBlockList, "d1"))
}
+5 -10
View File
@@ -115,7 +115,7 @@ func init() {
// We can rely on log.CanColorStdout being set properly because modules/log/console_windows.go comes before modules/setting/setting.go lexicographically
// By default set this logger at Info - we'll change it later, but we need to start with something.
log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "info", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout))
log.SetConsoleLogger(log.DEFAULT, "console", log.INFO)
var err error
if AppPath, err = getAppPath(); err != nil {
@@ -124,12 +124,6 @@ func init() {
AppWorkPath = getWorkPath(AppPath)
}
func forcePathSeparator(path string) {
if strings.Contains(path, "\\") {
log.Fatal("Do not use '\\' or '\\\\' in paths, instead, please use '/' in all places")
}
}
// IsRunUserMatchCurrentUser returns false if configured run user does not match
// actual user that runs the app. The first return value is the actual user name.
// This check is ignored under Windows since SSH remote login is not the main
@@ -218,9 +212,9 @@ func Init(opts *Options) {
// loadCommonSettingsFrom loads common configurations from a configuration provider.
func loadCommonSettingsFrom(cfg ConfigProvider) {
// WARNNING: don't change the sequence except you know what you are doing.
// WARNING: don't change the sequence except you know what you are doing.
loadRunModeFrom(cfg)
loadLogFrom(cfg)
loadLogGlobalFrom(cfg)
loadServerFrom(cfg)
loadSSHFrom(cfg)
@@ -282,10 +276,11 @@ func mustCurrentRunUserMatch(rootCfg ConfigProvider) {
// LoadSettings initializes the settings for normal start up
func LoadSettings() {
initAllLoggers()
loadDBSetting(CfgProvider)
loadServiceFrom(CfgProvider)
loadOAuth2ClientFrom(CfgProvider)
InitLogs(false)
loadCacheFrom(CfgProvider)
loadSessionFrom(CfgProvider)
loadCorsFrom(CfgProvider)