From be3a6d164757fc755691018d956ec89861833da0 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 31 Aug 2026 19:48:52 +0800 Subject: [PATCH] fix: charset lookup (#39187) --- custom/conf/app.example.ini | 2 +- modules/charset/charset.go | 22 +++++++++-- modules/charset/charset_test.go | 7 ++++ modules/setting/repository.go | 70 ++++++++++++++++----------------- modules/setting/setting.go | 2 +- services/gitdiff/gitdiff.go | 3 +- 6 files changed, 64 insertions(+), 42 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 86631be7007..c46fb6bbb65 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1029,7 +1029,7 @@ LEVEL = Info ;; If the charsets have equal confidence, tie-breaking will be done by order in this list ;; with charsets earlier in the list chosen in preference to those later. ;; Adding "defaults" will place the unused charsets at that position. -;DETECTED_CHARSETS_ORDER = UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE, ISO-8859, windows-1252, ISO-8859, windows-1250, ISO-8859, ISO-8859, ISO-8859, windows-1253, ISO-8859, windows-1255, ISO-8859, windows-1251, windows-1256, KOI8-R, ISO-8859, windows-1254, Shift_JIS, GB18030, EUC-JP, EUC-KR, Big5, ISO-2022, ISO-2022, ISO-2022, IBM424_rtl, IBM424_ltr, IBM420_rtl, IBM420_ltr +;DETECTED_CHARSETS_ORDER = UTF-8, UTF-16BE, UTF-16LE, UTF-32BE, UTF-32LE, ISO-8859, windows-1252, ISO-8859, windows-1250, ISO-8859, ISO-8859, ISO-8859, windows-1253, ISO-8859, windows-1255, ISO-8859, windows-1251, windows-1256, KOI8-R, ISO-8859, windows-1254, Shift_JIS, GB18030, EUC-JP, EUC-KR, Big5, ISO-2022, ISO-2022, ISO-2022 ;; ;; Default ANSI charset to override non-UTF-8 charsets to ;ANSI_CHARSET = diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 70f46142a52..49b741d4ee4 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -17,6 +17,8 @@ import ( "github.com/gogs/chardet" "golang.org/x/net/html/charset" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/unicode/utf32" "golang.org/x/text/transform" ) @@ -28,12 +30,26 @@ var globalVars = sync.OnceValue(func() (ret struct { invisibleRangeTable *unicode.RangeTable }, ) { - ret.utf8Bom = []byte{'\xef', '\xbb', '\xbf'} + ret.utf8Bom = []byte("\xef\xbb\xbf") ret.ambiguousTableMap = newAmbiguousTableMap() ret.invisibleRangeTable = newInvisibleRangeTable() return ret }) +func Lookup(label string) (e encoding.Encoding, name string) { + e, name = charset.Lookup(label) + if e != nil { + return e, name + } + switch { + case strings.EqualFold(label, "UTF-32BE"): + return utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM), "UTF-32BE" + case strings.EqualFold(label, "UTF-32LE"): + return utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM), "UTF-32LE" + } + return nil, "" +} + type ConvertOpts struct { KeepBOM bool ErrorReplacement []byte @@ -57,7 +73,7 @@ func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader { return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd) } - encoding, _ := charset.Lookup(charsetLabel) + encoding, _ := Lookup(charsetLabel) if encoding == nil { // unknown charset, don't do any processing return io.MultiReader(bytes.NewReader(buf[:n]), rd) @@ -86,7 +102,7 @@ func ToUTF8(content []byte, opts ConvertOpts) []byte { return maybeRemoveBOM(content, opts) } - encoding, _ := charset.Lookup(charsetLabel) + encoding, _ := Lookup(charsetLabel) if encoding == nil { setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel) if opts.ErrorReturnOrigin { diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 30b16e2d47d..ab1ef64e89f 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -245,3 +245,10 @@ func TestToUTF8WithFallbackReader(t *testing.T) { } } } + +func TestDefaultDetectedCharsetsOrder(t *testing.T) { + for _, charsetName := range setting.DefaultDetectedCharsetsOrder() { + e, _ := Lookup(charsetName) + assert.NotNil(t, e, "charset %s is not registered", charsetName) + } +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 82b6d93a0a1..d352b56de4a 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -128,41 +128,7 @@ var ( TrustedSSHKeys []string `ini:"TRUSTED_SSH_KEYS"` } `ini:"repository.signing"` }{ - DetectedCharsetsOrder: []string{ - "UTF-8", - "UTF-16BE", - "UTF-16LE", - "UTF-32BE", - "UTF-32LE", - "ISO-8859-1", - "windows-1252", - "ISO-8859-2", - "windows-1250", - "ISO-8859-5", - "ISO-8859-6", - "ISO-8859-7", - "windows-1253", - "ISO-8859-8-I", - "windows-1255", - "ISO-8859-8", - "windows-1251", - "windows-1256", - "KOI8-R", - "ISO-8859-9", - "windows-1254", - "Shift_JIS", - "GB18030", - "EUC-JP", - "EUC-KR", - "Big5", - "ISO-2022-JP", - "ISO-2022-KR", - "ISO-2022-CN", - "IBM424_rtl", - "IBM424_ltr", - "IBM420_rtl", - "IBM420_ltr", - }, + DetectedCharsetsOrder: DefaultDetectedCharsetsOrder(), DetectedCharsetScore: map[string]int{}, AnsiCharset: "", ForcePrivate: false, @@ -297,6 +263,40 @@ var ( ScriptType = "bash" ) +func DefaultDetectedCharsetsOrder() []string { + return []string{ + "UTF-8", + "UTF-16BE", + "UTF-16LE", + "UTF-32BE", + "UTF-32LE", + "ISO-8859-1", + "windows-1252", + "ISO-8859-2", + "windows-1250", + "ISO-8859-5", + "ISO-8859-6", + "ISO-8859-7", + "windows-1253", + "ISO-8859-8-I", + "windows-1255", + "ISO-8859-8", + "windows-1251", + "windows-1256", + "KOI8-R", + "ISO-8859-9", + "windows-1254", + "Shift_JIS", + "GB18030", + "EUC-JP", + "EUC-KR", + "Big5", + "ISO-2022-JP", + "ISO-2022-KR", + "ISO-2022-CN", + } +} + func loadRepositoryFrom(rootCfg ConfigProvider) { var err error // Determine and create root git repository path. diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a20c202828a..ccf43eaaed0 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -257,5 +257,5 @@ func PanicInDevOrTesting(msg string, a ...any) { if !IsProd || IsInTesting { panic(fmt.Sprintf(msg, a...)) } - log.Error(msg, a...) + log.ErrorWithSkip(1, msg, a...) } diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 04fdf46a5e5..1febdbaa938 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -42,7 +42,6 @@ import ( "github.com/alecthomas/chroma/v2" "github.com/sergi/go-diff/diffmatchpatch" - stdcharset "golang.org/x/net/html/charset" "golang.org/x/text/encoding" "golang.org/x/text/transform" ) @@ -947,7 +946,7 @@ parsingLoop: } charsetLabel, _ := charset.DetectEncoding(buffer.Bytes()) if charsetLabel != "UTF-8" { - charsetEncoding, _ := stdcharset.Lookup(charsetLabel) + charsetEncoding, _ := charset.Lookup(charsetLabel) if charsetEncoding != nil { diffLineTypeDecoders[lineType] = charsetEncoding.NewDecoder() }