From 6d3c6741ec4ecdc59e2b57c8a9cfb2e019a7a9e7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 19 Apr 2025 19:49:57 -0700 Subject: [PATCH 1/3] Fix notify watch failure when the content is too long (#34233) Fix #28193 --------- Co-authored-by: wxiaoguang --- services/feed/feed.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/services/feed/feed.go b/services/feed/feed.go index 38a4e25308..1dbd2e0e26 100644 --- a/services/feed/feed.go +++ b/services/feed/feed.go @@ -6,7 +6,7 @@ package feed import ( "context" "fmt" - "strconv" + "strings" activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" @@ -14,15 +14,10 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) -func userFeedCacheKey(userID int64) string { - return fmt.Sprintf("user_feed_%d", userID) -} - func GetFeedsForDashboard(ctx context.Context, opts activities_model.GetFeedsOptions) (activities_model.ActionList, int, error) { opts.DontCount = opts.RequestedTeam == nil && opts.Date == "" results, cnt, err := activities_model.GetFeeds(ctx, opts) @@ -40,7 +35,18 @@ func GetFeeds(ctx context.Context, opts activities_model.GetFeedsOptions) (activ // * Organization action: UserID=100 (the repo's org), ActUserID=1 // * Watcher action: UserID=20 (a user who is watching a repo), ActUserID=1 func notifyWatchers(ctx context.Context, act *activities_model.Action, watchers []*repo_model.Watch, permCode, permIssue, permPR []bool) error { - // Add feed for actioner. + // MySQL has TEXT length limit 65535. + // Sometimes the content is "field1|field2|field3", sometimes the content is JSON (ActionMirrorSyncPush, ActionCommitRepo, ActionPushTag, etc...) + if left, right := util.EllipsisDisplayStringX(act.Content, 65535); right != "" { + if strings.HasPrefix(act.Content, `{"`) && strings.HasSuffix(act.Content, `}`) { + // FIXME: at the moment we can do nothing if the content is JSON and it is too long + act.Content = "{}" + } else { + act.Content = left + } + } + + // Add feed for actor. act.UserID = act.ActUserID if err := db.Insert(ctx, act); err != nil { return fmt.Errorf("insert new actioner: %w", err) @@ -76,24 +82,18 @@ func notifyWatchers(ctx context.Context, act *activities_model.Action, watchers if !permPR[i] { continue } + default: } if err := db.Insert(ctx, act); err != nil { return fmt.Errorf("insert new action: %w", err) } - - total, err := activities_model.CountUserFeeds(ctx, act.UserID) - if err != nil { - return fmt.Errorf("count user feeds: %w", err) - } - - _ = cache.GetCache().Put(userFeedCacheKey(act.UserID), strconv.FormatInt(total, 10), setting.CacheService.TTLSeconds()) } return nil } -// NotifyWatchersActions creates batch of actions for every watcher. +// NotifyWatchers creates batch of actions for every watcher. func NotifyWatchers(ctx context.Context, acts ...*activities_model.Action) error { return db.WithTx(ctx, func(ctx context.Context) error { if len(acts) == 0 { From af6be75adb99ff42215a945927c016aa5e40dab2 Mon Sep 17 00:00:00 2001 From: Dejan Kitic <137049545+dek5troza@users.noreply.github.com> Date: Sun, 20 Apr 2025 12:18:14 +0100 Subject: [PATCH 2/3] Valid email address should only start with alphanumeric (#28174) This fixes issue #27847 where regular expression allowed email address to start with special symbols. Valid email addresses should start with alphanumeric character, and as such will be rendered as email. Added test cases from the bug report to validate, such input will not be rendered anymore as email address. --------- Co-authored-by: wxiaoguang --- modules/markup/html.go | 3 ++- modules/markup/html_email.go | 14 +++++++++++++- modules/markup/html_test.go | 36 ++++++++++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 0e074cbcfa..7c3bd93699 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -71,7 +71,8 @@ var globalVars = sync.OnceValue(func() *globalVarsType { // it is still accepted by the CommonMark specification, as well as the HTML5 spec: // http://spec.commonmark.org/0.28/#email-address // https://html.spec.whatwg.org/multipage/input.html#e-mail-state-(type%3Demail) - v.emailRegex = regexp.MustCompile("(?:\\s|^|\\(|\\[)([a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)(?:\\s|$|\\)|\\]|;|,|\\?|!|\\.(\\s|$))") + // At the moment, we use stricter rule for rendering purpose: only allow the "name" part starting after the word boundary + v.emailRegex = regexp.MustCompile(`\b([-\w.!#$%&'*+/=?^{|}~]*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]{2,}(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+)\b`) // emojiShortCodeRegex find emoji by alias like :smile: v.emojiShortCodeRegex = regexp.MustCompile(`:[-+\w]+:`) diff --git a/modules/markup/html_email.go b/modules/markup/html_email.go index cbfae8b829..cf18e99d98 100644 --- a/modules/markup/html_email.go +++ b/modules/markup/html_email.go @@ -3,7 +3,11 @@ package markup -import "golang.org/x/net/html" +import ( + "strings" + + "golang.org/x/net/html" +) // emailAddressProcessor replaces raw email addresses with a mailto: link. func emailAddressProcessor(ctx *RenderContext, node *html.Node) { @@ -14,6 +18,14 @@ func emailAddressProcessor(ctx *RenderContext, node *html.Node) { return } + var nextByte byte + if len(node.Data) > m[3] { + nextByte = node.Data[m[3]] + } + if strings.IndexByte(":/", nextByte) != -1 { + // for cases: "git@gitea.com:owner/repo.git", "https://git@gitea.com/owner/repo.git" + return + } mail := node.Data[m[2]:m[3]] replaceContent(node, m[2], m[3], createLink(ctx, "mailto:"+mail, mail, "" /*mailto*/)) node = node.NextSibling.NextSibling diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index aab9fddd91..58f71bdd7b 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -225,10 +225,10 @@ func TestRender_email(t *testing.T) { test := func(input, expected string) { res, err := markup.RenderString(markup.NewTestRenderContext().WithRelativePath("a.md"), input) assert.NoError(t, err) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res), "input: %s", input) } - // Text that should be turned into email link + // Text that should be turned into email link test( "info@gitea.com", `

info@gitea.com

`) @@ -260,28 +260,48 @@ func TestRender_email(t *testing.T) { j.doe@example.com? j.doe@example.com!

`) + // match GitHub behavior + test("email@domain@domain.com", `

email@domain@domain.com

`) + + // match GitHub behavior + test(`"info@gitea.com"`, `

"info@gitea.com"

`) + // Test that should *not* be turned into email links - test( - "\"info@gitea.com\"", - `

"info@gitea.com"

`) test( "/home/gitea/mailstore/info@gitea/com", `

/home/gitea/mailstore/info@gitea/com

`) test( "git@try.gitea.io:go-gitea/gitea.git", `

git@try.gitea.io:go-gitea/gitea.git

`) + test( + "https://foo:bar@gitea.io", + `

https://foo:bar@gitea.io

`) test( "gitea@3", `

gitea@3

`) test( "gitea@gmail.c", `

gitea@gmail.c

`) - test( - "email@domain@domain.com", - `

email@domain@domain.com

`) test( "email@domain..com", `

email@domain..com

`) + + cases := []struct { + input, expected string + }{ + // match GitHub behavior + {"?a@d.zz", `

?a@d.zz

`}, + {"*a@d.zz", `

*a@d.zz

`}, + {"~a@d.zz", `

~a@d.zz

`}, + + // the following cases don't match GitHub behavior, but they are valid email addresses ... + // maybe we should reduce the candidate characters for the "name" part in the future + {"a*a@d.zz", `

a*a@d.zz

`}, + {"a~a@d.zz", `

a~a@d.zz

`}, + } + for _, c := range cases { + test(c.input, c.expected) + } } func TestRender_emoji(t *testing.T) { From d1a3bd68140adcc76ad6163c02a1fe00d59aecae Mon Sep 17 00:00:00 2001 From: Jannis Pohl <838818+jannispl@users.noreply.github.com> Date: Sun, 20 Apr 2025 13:43:48 +0200 Subject: [PATCH 3/3] Make ROOT_URL support using request Host header (#32564) Resolve #32554 --------- Co-authored-by: wxiaoguang --- custom/conf/app.example.ini | 37 ++++++++++++++-------------- modules/httplib/url.go | 11 ++++++--- modules/httplib/url_test.go | 20 +++++++++++++++ modules/setting/server.go | 43 ++++++++++++++++++++++++--------- routers/web/admin/admin_test.go | 1 + 5 files changed, 80 insertions(+), 32 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index c10de95953..53e25a8c3b 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -59,27 +59,16 @@ RUN_USER = ; git ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; -;; The protocol the server listens on. One of 'http', 'https', 'http+unix', 'fcgi' or 'fcgi+unix'. Defaults to 'http' -;; Note: Value must be lowercase. +;; The protocol the server listens on. One of "http", "https", "http+unix", "fcgi" or "fcgi+unix". ;PROTOCOL = http ;; -;; Expect PROXY protocol headers on connections -;USE_PROXY_PROTOCOL = false -;; -;; Use PROXY protocol in TLS Bridging mode -;PROXY_PROTOCOL_TLS_BRIDGING = false -;; -; Timeout to wait for PROXY protocol header (set to 0 to have no timeout) -;PROXY_PROTOCOL_HEADER_TIMEOUT=5s -;; -; Accept PROXY protocol headers with UNKNOWN type -;PROXY_PROTOCOL_ACCEPT_UNKNOWN=false -;; -;; Set the domain for the server +;; Set the domain for the server. +;; Most users should set it to the real website domain of their Gitea instance. ;DOMAIN = localhost ;; ;; The AppURL used by Gitea to generate absolute links, defaults to "{PROTOCOL}://{DOMAIN}:{HTTP_PORT}/". -;; Most users should set it to the real website URL of their Gitea instance. +;; Most users should set it to the real website URL of their Gitea instance when there is a reverse proxy. +;; When it is empty, Gitea will use HTTP "Host" header to generate ROOT_URL, and fall back to the default one if no "Host" header. ;ROOT_URL = ;; ;; For development purpose only. It makes Gitea handle sub-path ("/sub-path/owner/repo/...") directly when debugging without a reverse proxy. @@ -90,13 +79,25 @@ RUN_USER = ; git ;STATIC_URL_PREFIX = ;; ;; The address to listen on. Either a IPv4/IPv6 address or the path to a unix socket. -;; If PROTOCOL is set to `http+unix` or `fcgi+unix`, this should be the name of the Unix socket file to use. +;; If PROTOCOL is set to "http+unix" or "fcgi+unix", this should be the name of the Unix socket file to use. ;; Relative paths will be made absolute against the _`AppWorkPath`_. ;HTTP_ADDR = 0.0.0.0 ;; -;; The port to listen on. Leave empty when using a unix socket. +;; The port to listen on for "http" or "https" protocol. Leave empty when using a unix socket. ;HTTP_PORT = 3000 ;; +;; Expect PROXY protocol headers on connections +;USE_PROXY_PROTOCOL = false +;; +;; Use PROXY protocol in TLS Bridging mode +;PROXY_PROTOCOL_TLS_BRIDGING = false +;; +;; Timeout to wait for PROXY protocol header (set to 0 to have no timeout) +;PROXY_PROTOCOL_HEADER_TIMEOUT = 5s +;; +;; Accept PROXY protocol headers with UNKNOWN type +;PROXY_PROTOCOL_ACCEPT_UNKNOWN = false +;; ;; If REDIRECT_OTHER_PORT is true, and PROTOCOL is set to https an http server ;; will be started on PORT_TO_REDIRECT and it will redirect plain, non-secure http requests to the main ;; ROOT_URL. Defaults are false for REDIRECT_OTHER_PORT and 80 for diff --git a/modules/httplib/url.go b/modules/httplib/url.go index 5d5b64dc0c..dabc1f5f45 100644 --- a/modules/httplib/url.go +++ b/modules/httplib/url.go @@ -70,11 +70,16 @@ func GuessCurrentHostURL(ctx context.Context) string { // 1. The reverse proxy is configured correctly, it passes "X-Forwarded-Proto/Host" headers. Perfect, Gitea can handle it correctly. // 2. The reverse proxy is not configured correctly, doesn't pass "X-Forwarded-Proto/Host" headers, eg: only one "proxy_pass http://gitea:3000" in Nginx. // 3. There is no reverse proxy. - // Without an extra config option, Gitea is impossible to distinguish between case 2 and case 3, - // then case 2 would result in wrong guess like guessed AppURL becomes "http://gitea:3000/", which is not accessible by end users. - // So in the future maybe it should introduce a new config option, to let site admin decide how to guess the AppURL. + // Without more information, Gitea is impossible to distinguish between case 2 and case 3, then case 2 would result in + // wrong guess like guessed AppURL becomes "http://gitea:3000/" behind a "https" reverse proxy, which is not accessible by end users. + // So we introduced "UseHostHeader" option, it could be enabled by setting "ROOT_URL" to empty reqScheme := getRequestScheme(req) if reqScheme == "" { + // if no reverse proxy header, try to use "Host" header for absolute URL + if setting.UseHostHeader && req.Host != "" { + return util.Iif(req.TLS == nil, "http://", "https://") + req.Host + } + // fall back to default AppURL return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") } // X-Forwarded-Host has many problems: non-standard, not well-defined (X-Forwarded-Port or not), conflicts with Host header. diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go index d57653646b..0e198d7d73 100644 --- a/modules/httplib/url_test.go +++ b/modules/httplib/url_test.go @@ -5,6 +5,7 @@ package httplib import ( "context" + "crypto/tls" "net/http" "testing" @@ -39,6 +40,25 @@ func TestIsRelativeURL(t *testing.T) { } } +func TestGuessCurrentHostURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + defer test.MockVariableValue(&setting.UseHostHeader, false)() + + ctx := t.Context() + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(ctx, RequestContextKey, &http.Request{Host: "localhost:3000"}) + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(ctx)) + + defer test.MockVariableValue(&setting.UseHostHeader, true)() + ctx = context.WithValue(ctx, RequestContextKey, &http.Request{Host: "http-host:3000"}) + assert.Equal(t, "http://http-host:3000", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(ctx, RequestContextKey, &http.Request{Host: "http-host", TLS: &tls.ConnectionState{}}) + assert.Equal(t, "https://http-host", GuessCurrentHostURL(ctx)) +} + func TestMakeAbsoluteURL(t *testing.T) { defer test.MockVariableValue(&setting.Protocol, "http")() defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() diff --git a/modules/setting/server.go b/modules/setting/server.go index ca635c8abe..41b0ca8959 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -46,25 +46,37 @@ var ( // AppURL is the Application ROOT_URL. It always has a '/' suffix // It maps to ini:"ROOT_URL" AppURL string - // AppSubURL represents the sub-url mounting point for gitea. It is either "" or starts with '/' and ends without '/', such as '/{subpath}'. + + // AppSubURL represents the sub-url mounting point for gitea, parsed from "ROOT_URL" + // It is either "" or starts with '/' and ends without '/', such as '/{sub-path}'. // This value is empty if site does not have sub-url. AppSubURL string - // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", to make it easier to debug sub-path related problems without a reverse proxy. + + // UseSubURLPath makes Gitea handle requests with sub-path like "/sub-path/owner/repo/...", + // to make it easier to debug sub-path related problems without a reverse proxy. UseSubURLPath bool + + // UseHostHeader makes Gitea prefer to use the "Host" request header for construction of absolute URLs. + UseHostHeader bool + // AppDataPath is the default path for storing data. // It maps to ini:"APP_DATA_PATH" in [server] and defaults to AppWorkPath + "/data" AppDataPath string + // LocalURL is the url for locally running applications to contact Gitea. It always has a '/' suffix // It maps to ini:"LOCAL_ROOT_URL" in [server] LocalURL string - // AssetVersion holds a opaque value that is used for cache-busting assets + + // AssetVersion holds an opaque value that is used for cache-busting assets AssetVersion string - appTempPathInternal string // the temporary path for the app, it is only an internal variable, do not use it, always use AppDataTempDir + // appTempPathInternal is the temporary path for the app, it is only an internal variable + // DO NOT use it directly, always use AppDataTempDir + appTempPathInternal string Protocol Scheme - UseProxyProtocol bool // `ini:"USE_PROXY_PROTOCOL"` - ProxyProtocolTLSBridging bool //`ini:"PROXY_PROTOCOL_TLS_BRIDGING"` + UseProxyProtocol bool + ProxyProtocolTLSBridging bool ProxyProtocolHeaderTimeout time.Duration ProxyProtocolAcceptUnknown bool Domain string @@ -181,13 +193,14 @@ func loadServerFrom(rootCfg ConfigProvider) { EnableAcme = sec.Key("ENABLE_LETSENCRYPT").MustBool(false) } - Protocol = HTTP protocolCfg := sec.Key("PROTOCOL").String() if protocolCfg != "https" && EnableAcme { log.Fatal("ACME could only be used with HTTPS protocol") } switch protocolCfg { + case "", "http": + Protocol = HTTP case "https": Protocol = HTTPS if EnableAcme { @@ -243,7 +256,7 @@ func loadServerFrom(rootCfg ConfigProvider) { case "unix": log.Warn("unix PROTOCOL value is deprecated, please use http+unix") fallthrough - case "http+unix": + default: // "http+unix" Protocol = HTTPUnix } UnixSocketPermissionRaw := sec.Key("UNIX_SOCKET_PERMISSION").MustString("666") @@ -256,6 +269,8 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(HTTPAddr) { HTTPAddr = filepath.Join(AppWorkPath, HTTPAddr) } + default: + log.Fatal("Invalid PROTOCOL %q", Protocol) } UseProxyProtocol = sec.Key("USE_PROXY_PROTOCOL").MustBool(false) ProxyProtocolTLSBridging = sec.Key("PROXY_PROTOCOL_TLS_BRIDGING").MustBool(false) @@ -268,12 +283,16 @@ func loadServerFrom(rootCfg ConfigProvider) { PerWritePerKbTimeout = sec.Key("PER_WRITE_PER_KB_TIMEOUT").MustDuration(PerWritePerKbTimeout) defaultAppURL := string(Protocol) + "://" + Domain + ":" + HTTPPort - AppURL = sec.Key("ROOT_URL").MustString(defaultAppURL) + AppURL = sec.Key("ROOT_URL").String() + if AppURL == "" { + UseHostHeader = true + AppURL = defaultAppURL + } // Check validity of AppURL appURL, err := url.Parse(AppURL) if err != nil { - log.Fatal("Invalid ROOT_URL '%s': %s", AppURL, err) + log.Fatal("Invalid ROOT_URL %q: %s", AppURL, err) } // Remove default ports from AppURL. // (scheme-based URL normalization, RFC 3986 section 6.2.3) @@ -309,13 +328,15 @@ func loadServerFrom(rootCfg ConfigProvider) { defaultLocalURL = AppURL case FCGIUnix: defaultLocalURL = AppURL - default: + case HTTP, HTTPS: defaultLocalURL = string(Protocol) + "://" if HTTPAddr == "0.0.0.0" { defaultLocalURL += net.JoinHostPort("localhost", HTTPPort) + "/" } else { defaultLocalURL += net.JoinHostPort(HTTPAddr, HTTPPort) + "/" } + default: + log.Fatal("Invalid PROTOCOL %q", Protocol) } LocalURL = sec.Key("LOCAL_ROOT_URL").MustString(defaultLocalURL) LocalURL = strings.TrimRight(LocalURL, "/") + "/" diff --git a/routers/web/admin/admin_test.go b/routers/web/admin/admin_test.go index a568c7c5c8..04fad4663c 100644 --- a/routers/web/admin/admin_test.go +++ b/routers/web/admin/admin_test.go @@ -76,6 +76,7 @@ func TestShadowPassword(t *testing.T) { func TestSelfCheckPost(t *testing.T) { defer test.MockVariableValue(&setting.AppURL, "http://config/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + defer test.MockVariableValue(&setting.UseHostHeader, false)() ctx, resp := contexttest.MockContext(t, "GET http://host/sub/admin/self_check?location_origin=http://frontend") SelfCheckPost(ctx)