Mirrors/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-09-24 02:05:27 +02:00
Code Issues Activity

Merge remote-tracking branch 'upstream/main' into limit-repo-size

Browse Source
This commit is contained in:
DmitryFrolovTri
2025-12-18 19:42:45 +00:00
parent 624fe24693 e0214ab841
commit 243c16d759
144 changed files with 3052 additions and 2074 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/charset/charset.go
+68 -86
View File
@@ -5,12 +5,10 @@ package charset
import (
"bytes"
"fmt"
"io"
"strings"
"unicode/utf8"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@@ -23,60 +21,39 @@ import (
var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
type ConvertOpts struct {
KeepBOM bool
KeepBOM bool
ErrorReplacement []byte
ErrorReturnOrigin bool
}
var ToUTF8WithFallbackReaderPrefetchSize = 16 * 1024
// ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
buf := make([]byte, 2048)
buf := make([]byte, ToUTF8WithFallbackReaderPrefetchSize)
n, err := util.ReadAtMost(rd, buf)
if err != nil {
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
}
charsetLabel, err := DetectEncoding(buf[:n])
if err != nil || charsetLabel == "UTF-8" {
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
}
encoding, _ := charset.Lookup(charsetLabel)
if encoding == nil {
// read error occurs, don't do any processing
return io.MultiReader(bytes.NewReader(buf[:n]), rd)
}
return transform.NewReader(
io.MultiReader(
bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
rd,
),
encoding.NewDecoder(),
)
}
// ToUTF8 converts content to UTF8 encoding
func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
charsetLabel, err := DetectEncoding(content)
if err != nil {
return "", err
} else if charsetLabel == "UTF-8" {
return string(MaybeRemoveBOM(content, opts)), nil
charsetLabel, _ := DetectEncoding(buf[:n])
if charsetLabel == "UTF-8" {
// is utf-8, try to remove BOM and read it as-is
return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd)
}
encoding, _ := charset.Lookup(charsetLabel)
if encoding == nil {
return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
// unknown charset, don't do any processing
return io.MultiReader(bytes.NewReader(buf[:n]), rd)
}
// If there is an error, we concatenate the nicely decoded part and the
// original left over. This way we won't lose much data.
result, n, err := transform.Bytes(encoding.NewDecoder(), content)
if err != nil {
result = append(result, content[n:]...)
}
result = MaybeRemoveBOM(result, opts)
return string(result), err
// convert from charset to utf-8
return transform.NewReader(
io.MultiReader(bytes.NewReader(buf[:n]), rd),
encoding.NewDecoder(),
)
}
// ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
@@ -85,73 +62,84 @@ func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
return bs
}
// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
charsetLabel, err := DetectEncoding(content)
if err != nil || charsetLabel == "UTF-8" {
return MaybeRemoveBOM(content, opts)
func ToUTF8DropErrors(content []byte) []byte {
return ToUTF8(content, ConvertOpts{ErrorReplacement: []byte{' '}})
}
func ToUTF8(content []byte, opts ConvertOpts) []byte {
charsetLabel, _ := DetectEncoding(content)
if charsetLabel == "UTF-8" {
return maybeRemoveBOM(content, opts)
}
encoding, _ := charset.Lookup(charsetLabel)
if encoding == nil {
setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel)
return content
}
// We ignore any non-decodable parts from the file.
// Some parts might be lost
var decoded []byte
decoder := encoding.NewDecoder()
idx := 0
for {
for idx < len(content) {
result, n, err := transform.Bytes(decoder, content[idx:])
decoded = append(decoded, result...)
if err == nil {
break
}
decoded = append(decoded, ' ')
idx = idx + n + 1
if idx >= len(content) {
break
if opts.ErrorReturnOrigin {
return content
}
if opts.ErrorReplacement == nil {
decoded = append(decoded, content[idx+n])
} else {
decoded = append(decoded, opts.ErrorReplacement...)
}
idx += n + 1
}
return MaybeRemoveBOM(decoded, opts)
return maybeRemoveBOM(decoded, opts)
}
// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
// maybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
func maybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
if opts.KeepBOM {
return content
}
if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
return content[3:]
}
return content
return bytes.TrimPrefix(content, UTF8BOM)
}
// DetectEncoding detect the encoding of content
func DetectEncoding(content []byte) (string, error) {
// it always returns a detected or guessed "encoding" string, no matter error happens or not
func DetectEncoding(content []byte) (encoding string, _ error) {
// First we check if the content represents valid utf8 content excepting a truncated character at the end.
// Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do
// instead we walk backwards from the end to trim off a the incomplete character
// instead we walk backwards from the end to trim off the incomplete character
toValidate := content
end := len(toValidate) - 1
if end < 0 {
// no-op
} else if toValidate[end]>>5 == 0b110 {
// Incomplete 1 byte extension e.g. © <c2><a9> which has been truncated to <c2>
toValidate = toValidate[:end]
} else if end > 0 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>4 == 0b1110 {
// Incomplete 2 byte extension e.g. ⛔ <e2><9b><94> which has been truncated to <e2><9b>
toValidate = toValidate[:end-1]
} else if end > 1 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>6 == 0b10 && toValidate[end-2]>>3 == 0b11110 {
// Incomplete 3 byte extension e.g. 💩 <f0><9f><92><a9> which has been truncated to <f0><9f><92>
toValidate = toValidate[:end-2]
// U+0000 U+007F 0yyyzzzz
// U+0080 U+07FF 110xxxyy 10yyzzzz
// U+0800 U+FFFF 1110wwww 10xxxxyy 10yyzzzz
// U+010000 U+10FFFF 11110uvv 10vvwwww 10xxxxyy 10yyzzzz
cnt := 0
for end >= 0 && cnt < 4 {
c := toValidate[end]
if c>>5 == 0b110 || c>>4 == 0b1110 || c>>3 == 0b11110 {
// a leading byte
toValidate = toValidate[:end]
break
} else if c>>6 == 0b10 {
// a continuation byte
end--
} else {
// not an utf-8 byte
break
}
cnt++
}
if utf8.Valid(toValidate) {
log.Debug("Detected encoding: utf-8 (fast)")
return "UTF-8", nil
}
@@ -160,7 +148,7 @@ func DetectEncoding(content []byte) (string, error) {
if len(content) < 1024 {
// Check if original content is valid
if _, err := textDetector.DetectBest(content); err != nil {
return "", err
return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err
}
times := 1024 / len(content)
detectContent = make([]byte, 0, times*len(content))
@@ -171,14 +159,10 @@ func DetectEncoding(content []byte) (string, error) {
detectContent = content
}
// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break
// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie-break
results, err := textDetector.DetectAll(detectContent)
if err != nil {
if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 {
log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
return setting.Repository.AnsiCharset, nil
}
return "", err
return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err
}
topConfidence := results[0].Confidence
@@ -201,11 +185,9 @@ func DetectEncoding(content []byte) (string, error) {
}
// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
if topResult.Charset != "UTF-8" && setting.Repository.AnsiCharset != "" {
return setting.Repository.AnsiCharset, err
}
log.Debug("Detected encoding: %s", topResult.Charset)
return topResult.Charset, err
return topResult.Charset, nil
}
modules/charset/charset_test.go
+67 -202
View File
@@ -4,108 +4,89 @@
package charset
import (
"bytes"
"io"
"os"
"strings"
"testing"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"github.com/stretchr/testify/assert"
)
func resetDefaultCharsetsOrder() {
defaultDetectedCharsetsOrder := make([]string, 0, len(setting.Repository.DetectedCharsetsOrder))
for _, charset := range setting.Repository.DetectedCharsetsOrder {
defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder, strings.ToLower(strings.TrimSpace(charset)))
}
func TestMain(m *testing.M) {
setting.Repository.DetectedCharsetScore = map[string]int{}
i := 0
for _, charset := range defaultDetectedCharsetsOrder {
canonicalCharset := strings.ToLower(strings.TrimSpace(charset))
if _, has := setting.Repository.DetectedCharsetScore[canonicalCharset]; !has {
setting.Repository.DetectedCharsetScore[canonicalCharset] = i
i++
}
for i, charset := range setting.Repository.DetectedCharsetsOrder {
setting.Repository.DetectedCharsetScore[strings.ToLower(charset)] = i
}
os.Exit(m.Run())
}
func TestMaybeRemoveBOM(t *testing.T) {
res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
res := maybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
res = maybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
}
func TestToUTF8(t *testing.T) {
resetDefaultCharsetsOrder()
// Note: golang compiler seems so behave differently depending on the current
// locale, so some conversions might behave differently. For that reason, we don't
// depend on particular conversions but in expected behaviors.
res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, "ABC", res)
res := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
assert.Equal(t, "ABC", string(res))
// "áéíóú"
res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
res = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// "áéíóú"
res, err = ToUTF8([]byte{
res = ToUTF8([]byte{
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
0xc3, 0xba,
}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
res, err = ToUTF8([]byte{
res = ToUTF8([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
}, ConvertOpts{})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
res, err = ToUTF8([]byte{
res = ToUTF8([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
}, ConvertOpts{})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
res, err = ToUTF8([]byte{
res = ToUTF8([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
}, ConvertOpts{})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
// Japanese (Shift-JIS)
// 日属秘ぞしちゅ。
res, err = ToUTF8([]byte{
res = ToUTF8([]byte{
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
0xBF, 0x82, 0xE3, 0x81, 0x42,
}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
},
[]byte(res))
}, res)
res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
res = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
}
func TestToUTF8WithFallback(t *testing.T) {
resetDefaultCharsetsOrder()
// "ABC"
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
@@ -152,54 +133,58 @@ func TestToUTF8WithFallback(t *testing.T) {
}
func TestToUTF8DropErrors(t *testing.T) {
resetDefaultCharsetsOrder()
// "ABC"
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
// "áéíóú"
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// UTF8 BOM + "áéíóú"
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// "Hola, así cómo ños"
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
assert.Equal(t, []byte{0x73}, res[len(res)-1:])
// "Hola, así cómo "
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Equal(t, minmatch, res[0:len(minmatch)])
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Equal(t, minmatch, res[0:len(minmatch)])
// Japanese (Shift-JIS)
// "日属秘ぞしちゅ。"
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
assert.Equal(t, []byte{
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
}, res)
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
}
func TestDetectEncoding(t *testing.T) {
resetDefaultCharsetsOrder()
testSuccess := func(b []byte, expected string) {
encoding, err := DetectEncoding(b)
assert.NoError(t, err)
assert.Equal(t, expected, encoding)
}
// invalid bytes
encoding, err := DetectEncoding([]byte{0xfa})
assert.Error(t, err)
assert.Equal(t, "UTF-8", encoding)
// utf-8
b := []byte("just some ascii")
testSuccess(b, "UTF-8")
@@ -214,169 +199,49 @@ func TestDetectEncoding(t *testing.T) {
// iso-8859-1: d<accented e>cor<newline>
b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
encoding, err := DetectEncoding(b)
encoding, err = DetectEncoding(b)
assert.NoError(t, err)
assert.Contains(t, encoding, "ISO-8859-1")
old := setting.Repository.AnsiCharset
setting.Repository.AnsiCharset = "placeholder"
defer func() {
setting.Repository.AnsiCharset = old
}()
testSuccess(b, "placeholder")
// invalid bytes
b = []byte{0xfa}
_, err = DetectEncoding(b)
assert.Error(t, err)
defer test.MockVariableValue(&setting.Repository.AnsiCharset, "MyEncoding")()
testSuccess(b, "MyEncoding")
}
func stringMustStartWith(t *testing.T, expected, value string) {
assert.Equal(t, expected, value[:len(expected)])
func stringMustStartWith(t *testing.T, expected string, value []byte) {
assert.Equal(t, expected, string(value[:len(expected)]))
}
func stringMustEndWith(t *testing.T, expected, value string) {
assert.Equal(t, expected, value[len(value)-len(expected):])
func stringMustEndWith(t *testing.T, expected string, value []byte) {
assert.Equal(t, expected, string(value[len(value)-len(expected):]))
}
func TestToUTF8WithFallbackReader(t *testing.T) {
resetDefaultCharsetsOrder()
test.MockVariableValue(&ToUTF8WithFallbackReaderPrefetchSize)
for testLen := range 2048 {
pattern := " test { () }\n"
input := ""
for len(input) < testLen {
input += pattern
}
input = input[:testLen]
input += "// Выключаем"
rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{})
block := "aá啊🤔"
runes := []rune(block)
assert.Len(t, string(runes[0]), 1)
assert.Len(t, string(runes[1]), 2)
assert.Len(t, string(runes[2]), 3)
assert.Len(t, string(runes[3]), 4)
content := strings.Repeat(block, 2)
for i := 1; i < len(content); i++ {
encoding, err := DetectEncoding([]byte(content[:i]))
assert.NoError(t, err)
assert.Equal(t, "UTF-8", encoding)
ToUTF8WithFallbackReaderPrefetchSize = i
rd := ToUTF8WithFallbackReader(strings.NewReader(content), ConvertOpts{})
r, _ := io.ReadAll(rd)
assert.Equalf(t, input, string(r), "testing string len=%d", testLen)
assert.Equal(t, content, string(r))
}
for _, r := range runes {
content = "abc abc " + string(r) + string(r) + string(r)
for i := 0; i < len(content); i++ {
encoding, err := DetectEncoding([]byte(content[:i]))
assert.NoError(t, err)
assert.Equal(t, "UTF-8", encoding)
}
}
truncatedOneByteExtension := failFastBytes
encoding, _ := DetectEncoding(truncatedOneByteExtension)
assert.Equal(t, "UTF-8", encoding)
truncatedTwoByteExtension := failFastBytes
truncatedTwoByteExtension[len(failFastBytes)-1] = 0x9b
truncatedTwoByteExtension[len(failFastBytes)-2] = 0xe2
encoding, _ = DetectEncoding(truncatedTwoByteExtension)
assert.Equal(t, "UTF-8", encoding)
truncatedThreeByteExtension := failFastBytes
truncatedThreeByteExtension[len(failFastBytes)-1] = 0x92
truncatedThreeByteExtension[len(failFastBytes)-2] = 0x9f
truncatedThreeByteExtension[len(failFastBytes)-3] = 0xf0
encoding, _ = DetectEncoding(truncatedThreeByteExtension)
assert.Equal(t, "UTF-8", encoding)
}
var failFastBytes = []byte{
0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x74, 0x6f,
0x6f, 0x6c, 0x73, 0x2e, 0x61, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x64, 0x65, 0x66, 0x73, 0x2e, 0x63, 0x6f, 0x6e,
0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4f, 0x73, 0x0a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67,
0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f,
0x74, 0x2e, 0x67, 0x72, 0x61, 0x64, 0x6c, 0x65, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x75, 0x6e, 0x2e, 0x42,
0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x0a, 0x0a, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20,
0x20, 0x20, 0x69, 0x64, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d,
0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x65,
0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65,
0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a,
0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d,
0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x2d, 0x64, 0x6f, 0x63, 0x73, 0x22, 0x29,
0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x64, 0x62,
0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65,
0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a,
0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x66,
0x73, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x6d, 0x71, 0x22, 0x29, 0x29, 0x0a, 0x0a,
0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22,
0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e,
0x2d, 0x61, 0x75, 0x74, 0x68, 0x2d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74,
0x65, 0x72, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63,
0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x68, 0x61, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c,
0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e,
0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x29, 0x0a,
0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28,
0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b,
0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74,
0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x77, 0x65, 0x62, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c,
0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69,
0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72,
0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x6f, 0x70,
0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f,
0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d,
0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x63, 0x74, 0x75, 0x61, 0x74, 0x6f, 0x72, 0x22, 0x29, 0x0a, 0x20,
0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f,
0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63,
0x6c, 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74,
0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x22, 0x29, 0x0a, 0x20, 0x20,
0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72,
0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61,
0x72, 0x74, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2d, 0x61, 0x6c, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20,
0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72,
0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61,
0x72, 0x74, 0x65, 0x72, 0x2d, 0x73, 0x6c, 0x65, 0x75, 0x74, 0x68, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d,
0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70,
0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x72, 0x65, 0x74, 0x72, 0x79, 0x3a,
0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20,
0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x63, 0x68, 0x2e, 0x71,
0x6f, 0x73, 0x2e, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x3a, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x2d, 0x63,
0x6c, 0x61, 0x73, 0x73, 0x69, 0x63, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d,
0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x69, 0x6f, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65,
0x74, 0x65, 0x72, 0x3a, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x2d, 0x72, 0x65, 0x67, 0x69, 0x73,
0x74, 0x72, 0x79, 0x2d, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20,
0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6b, 0x6f, 0x74,
0x6c, 0x69, 0x6e, 0x28, 0x22, 0x73, 0x74, 0x64, 0x6c, 0x69, 0x62, 0x22, 0x29, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20,
0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f,
0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64,
0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f,
0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74,
0x65, 0x73, 0x74, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a,
0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d,
0x74, 0x65, 0x73, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a,
0x61, 0x72, 0x20, 0x62, 0x79, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72,
0x69, 0x6e, 0x67, 0x28, 0x4a, 0x61, 0x72, 0x3a, 0x3a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20,
0x20, 0x20, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x69, 0x65, 0x72, 0x2e,
0x73, 0x65, 0x74, 0x28, 0x22, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x64, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20,
0x76, 0x61, 0x6c, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68,
0x20, 0x62, 0x79, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x67,
0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74,
0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65,
0x73, 0x28, 0x22, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x2d, 0x50, 0x61, 0x74, 0x68, 0x22, 0x20, 0x74, 0x6f, 0x20, 0x6f, 0x62,
0x6a, 0x65, 0x63, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70,
0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x20, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d,
0x20, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x2f, 0x2b, 0x22, 0x2e, 0x74, 0x6f, 0x52, 0x65, 0x67, 0x65, 0x78, 0x28, 0x29,
0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64,
0x65, 0x20, 0x66, 0x75, 0x6e, 0x20, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x3a, 0x20, 0x53, 0x74,
0x72, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70,
0x61, 0x74, 0x68, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x6f, 0x53, 0x74, 0x72, 0x69,
0x6e, 0x67, 0x28, 0x22, 0x20, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x74, 0x2e, 0x74, 0x6f, 0x55, 0x52, 0x49, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x55,
0x52, 0x4c, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c,
0x61, 0x63, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x22, 0x2f,
0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x74, 0x61, 0x73,
0x6b, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x3c, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x3e, 0x28, 0x22, 0x62,
0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x4f,
0x73, 0x2e, 0x69, 0x73, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x28, 0x4f, 0x73, 0x2e, 0x46, 0x41, 0x4d, 0x49, 0x4c, 0x59,
0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x28, 0x73,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x74, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x28, 0x22, 0x6d, 0x61, 0x69,
0x6e, 0x22, 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x20, 0x7b, 0x20, 0x69, 0x74, 0x2e, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x20,
0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a, 0x61, 0x72, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a,
0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0xd0,
}
modules/charset/escape.go
+5 -2
View File
@@ -20,14 +20,17 @@ import (
// RuneNBSP is the codepoint for NBSP
const RuneNBSP = 0xa0
// EscapeControlHTML escapes the unicode control sequences in a provided html document
// EscapeControlHTML escapes the Unicode control sequences in a provided html document
func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) {
if !setting.UI.AmbiguousUnicodeDetection {
return &EscapeStatus{}, html
}
sb := &strings.Builder{}
escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader
return escaped, template.HTML(sb.String())
}
// EscapeControlReader escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
// EscapeControlReader escapes the Unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
if !setting.UI.AmbiguousUnicodeDetection {
_, err = io.Copy(writer, reader)
modules/git/blame.go
-218
View File
@@ -1,218 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"bufio"
"bytes"
"context"
"io"
"os"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
// BlamePart represents block of blame - continuous lines with one sha
type BlamePart struct {
Sha string
Lines []string
PreviousSha string
PreviousPath string
}
// BlameReader returns part of file blame one by one
type BlameReader struct {
output io.WriteCloser
reader io.ReadCloser
bufferedReader *bufio.Reader
done chan error
lastSha *string
ignoreRevsFile string
objectFormat ObjectFormat
cleanupFuncs []func()
}
func (r *BlameReader) UsesIgnoreRevs() bool {
return r.ignoreRevsFile != ""
}
// NextPart returns next part of blame (sequential code lines with the same commit)
func (r *BlameReader) NextPart() (*BlamePart, error) {
var blamePart *BlamePart
if r.lastSha != nil {
blamePart = &BlamePart{
Sha: *r.lastSha,
Lines: make([]string, 0),
}
}
const previousHeader = "previous "
var lineBytes []byte
var isPrefix bool
var err error
for err != io.EOF {
lineBytes, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
if len(lineBytes) == 0 {
// isPrefix will be false
continue
}
var objectID string
objectFormatLength := r.objectFormat.FullLength()
if len(lineBytes) > objectFormatLength && lineBytes[objectFormatLength] == ' ' && r.objectFormat.IsValid(string(lineBytes[0:objectFormatLength])) {
objectID = string(lineBytes[0:objectFormatLength])
}
if len(objectID) > 0 {
if blamePart == nil {
blamePart = &BlamePart{
Sha: objectID,
Lines: make([]string, 0),
}
}
if blamePart.Sha != objectID {
r.lastSha = &objectID
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
return blamePart, nil
}
} else if lineBytes[0] == '\t' {
blamePart.Lines = append(blamePart.Lines, string(lineBytes[1:]))
} else if bytes.HasPrefix(lineBytes, []byte(previousHeader)) {
offset := len(previousHeader) // already includes a space
blamePart.PreviousSha = string(lineBytes[offset : offset+objectFormatLength])
offset += objectFormatLength + 1 // +1 for space
blamePart.PreviousPath = string(lineBytes[offset:])
}
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
}
r.lastSha = nil
return blamePart, nil
}
// Close BlameReader - don't run NextPart after invoking that
func (r *BlameReader) Close() error {
if r.bufferedReader == nil {
return nil
}
err := <-r.done
r.bufferedReader = nil
_ = r.reader.Close()
_ = r.output.Close()
for _, cleanup := range r.cleanupFuncs {
if cleanup != nil {
cleanup()
}
}
return err
}
// CreateBlameReader creates reader for given repository, commit and file
func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) {
var ignoreRevsFileName string
var ignoreRevsFileCleanup func()
defer func() {
if err != nil && ignoreRevsFileCleanup != nil {
ignoreRevsFileCleanup()
}
}()
cmd := gitcmd.NewCommand("blame", "--porcelain")
if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore {
ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit)
if err != nil && !IsErrNotExist(err) {
return nil, err
}
if ignoreRevsFileName != "" {
// Possible improvement: use --ignore-revs-file /dev/stdin on unix
// There is no equivalent on Windows. May be implemented if Gitea uses an external git backend.
cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName)
}
}
cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file)
done := make(chan error, 1)
reader, stdout, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
stderr := bytes.Buffer{}
// TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close"
err := cmd.WithDir(repoPath).
WithUseContextTimeout(true).
WithStdout(stdout).
WithStderr(&stderr).
Run(ctx)
done <- err
_ = stdout.Close()
if err != nil {
log.Error("Error running git blame (dir: %v): %v, stderr: %v", repoPath, err, stderr.String())
}
}()
bufferedReader := bufio.NewReader(reader)
return &BlameReader{
output: stdout,
reader: reader,
bufferedReader: bufferedReader,
done: done,
ignoreRevsFile: ignoreRevsFileName,
objectFormat: objectFormat,
cleanupFuncs: []func(){ignoreRevsFileCleanup},
}, nil
}
func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func(), error) {
entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
r, err := entry.Blob().DataAsync()
if err != nil {
return "", nil, err
}
defer r.Close()
f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
filename := f.Name()
_, err = io.Copy(f, r)
_ = f.Close()
if err != nil {
cleanup()
return "", nil, err
}
return filename, cleanup, nil
}
modules/git/commit.go
-8
View File
@@ -323,14 +323,6 @@ func GetFullCommitID(ctx context.Context, repoPath, shortID string) (string, err
return strings.TrimSpace(commitID), nil
}
// GetRepositoryDefaultPublicGPGKey returns the default public key for this commit
func (c *Commit) GetRepositoryDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, error) {
if c.repo == nil {
return nil, nil
}
return c.repo.GetDefaultPublicGPGKey(forceUpdate)
}
func IsStringLikelyCommitID(objFmt ObjectFormat, s string, minLength ...int) bool {
maxLen := 64 // sha256
if objFmt != nil {
modules/git/diff.go
+7 -2
View File
@@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
// RawDiffType type of a raw diff.
@@ -47,7 +48,9 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff
switch diffType {
case RawDiffNormal:
if len(startCommit) != 0 {
cmd.AddArguments("diff", "-M").AddDynamicArguments(startCommit, endCommit).AddDashesAndList(files...)
cmd.AddArguments("diff").
AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold).
AddDynamicArguments(startCommit, endCommit).AddDashesAndList(files...)
} else if commit.ParentCount() == 0 {
cmd.AddArguments("show").AddDynamicArguments(endCommit).AddDashesAndList(files...)
} else {
@@ -55,7 +58,9 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff
if err != nil {
return err
}
cmd.AddArguments("diff", "-M").AddDynamicArguments(c.ID.String(), endCommit).AddDashesAndList(files...)
cmd.AddArguments("diff").
AddOptionFormat("--find-renames=%s", setting.Git.DiffRenameSimilarityThreshold).
AddDynamicArguments(c.ID.String(), endCommit).AddDashesAndList(files...)
}
case RawDiffPatch:
if len(startCommit) != 0 {
modules/git/foreachref/format.go
+3 -3
View File
@@ -75,9 +75,9 @@ func (f Format) Parser(r io.Reader) *Parser {
// hexEscaped produces hex-escpaed characters from a string. For example, "\n\0"
// would turn into "%0a%00".
func (f Format) hexEscaped(delim []byte) string {
escaped := ""
var escaped strings.Builder
for i := range delim {
escaped += "%" + hex.EncodeToString([]byte{delim[i]})
escaped.WriteString("%" + hex.EncodeToString([]byte{delim[i]}))
}
return escaped
return escaped.String()
}
modules/git/gpg.go
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"context"
"fmt"
"os"
"strings"
"sync"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/process"
)
// GPGSettings represents the default GPG settings for this repository
type GPGSettings struct {
Sign bool
KeyID string
Email string
Name string
PublicKeyContent string
Format string
}
// LoadPublicKeyContent will load the key from gpg
func (gpgSettings *GPGSettings) LoadPublicKeyContent() error {
if gpgSettings.PublicKeyContent != "" {
return nil
}
if gpgSettings.Format == SigningKeyFormatSSH {
content, err := os.ReadFile(gpgSettings.KeyID)
if err != nil {
return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err)
}
gpgSettings.PublicKeyContent = string(content)
return nil
}
content, stderr, err := process.GetManager().Exec(
"gpg -a --export",
"gpg", "-a", "--export", gpgSettings.KeyID)
if err != nil {
return fmt.Errorf("unable to get default signing key: %s, %s, %w", gpgSettings.KeyID, stderr, err)
}
gpgSettings.PublicKeyContent = content
return nil
}
var (
loadPublicGPGKeyMutex sync.RWMutex
globalGPGSettings *GPGSettings
)
// GetDefaultPublicGPGKey will return and cache the default public GPG settings
func GetDefaultPublicGPGKey(ctx context.Context, forceUpdate bool) (*GPGSettings, error) {
if !forceUpdate {
loadPublicGPGKeyMutex.RLock()
if globalGPGSettings != nil {
defer loadPublicGPGKeyMutex.RUnlock()
return globalGPGSettings, nil
}
loadPublicGPGKeyMutex.RUnlock()
}
loadPublicGPGKeyMutex.Lock()
defer loadPublicGPGKeyMutex.Unlock()
if globalGPGSettings != nil && !forceUpdate {
return globalGPGSettings, nil
}
globalGPGSettings = &GPGSettings{
Sign: true,
}
value, _, _ := gitcmd.NewCommand("config", "--global", "--get", "commit.gpgsign").RunStdString(ctx)
sign, valid := ParseBool(strings.TrimSpace(value))
if !sign || !valid {
globalGPGSettings.Sign = false
return globalGPGSettings, nil
}
signingKey, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.signingkey").RunStdString(ctx)
globalGPGSettings.KeyID = strings.TrimSpace(signingKey)
format, _, _ := gitcmd.NewCommand("config", "--global", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(ctx)
globalGPGSettings.Format = strings.TrimSpace(format)
defaultEmail, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.email").RunStdString(ctx)
globalGPGSettings.Email = strings.TrimSpace(defaultEmail)
defaultName, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.name").RunStdString(ctx)
globalGPGSettings.Name = strings.TrimSpace(defaultName)
if err := globalGPGSettings.LoadPublicKeyContent(); err != nil {
return nil, err
}
return globalGPGSettings, nil
}
modules/git/key.go
+6 -6
View File
@@ -32,23 +32,23 @@ func (s *SigningKey) String() string {
}
// GetSigningKey returns the KeyID and git Signature for the repo
func GetSigningKey(ctx context.Context, repoPath string) (*SigningKey, *Signature) {
func GetSigningKey(ctx context.Context) (*SigningKey, *Signature) {
if setting.Repository.Signing.SigningKey == "none" {
return nil, nil
}
if setting.Repository.Signing.SigningKey == "default" || setting.Repository.Signing.SigningKey == "" {
// Can ignore the error here as it means that commit.gpgsign is not set
value, _, _ := gitcmd.NewCommand("config", "--get", "commit.gpgsign").WithDir(repoPath).RunStdString(ctx)
value, _, _ := gitcmd.NewCommand("config", "--global", "--get", "commit.gpgsign").RunStdString(ctx)
sign, valid := ParseBool(strings.TrimSpace(value))
if !sign || !valid {
return nil, nil
}
format, _, _ := gitcmd.NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").WithDir(repoPath).RunStdString(ctx)
signingKey, _, _ := gitcmd.NewCommand("config", "--get", "user.signingkey").WithDir(repoPath).RunStdString(ctx)
signingName, _, _ := gitcmd.NewCommand("config", "--get", "user.name").WithDir(repoPath).RunStdString(ctx)
signingEmail, _, _ := gitcmd.NewCommand("config", "--get", "user.email").WithDir(repoPath).RunStdString(ctx)
format, _, _ := gitcmd.NewCommand("config", "--global", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").RunStdString(ctx)
signingKey, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.signingkey").RunStdString(ctx)
signingName, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.name").RunStdString(ctx)
signingEmail, _, _ := gitcmd.NewCommand("config", "--global", "--get", "user.email").RunStdString(ctx)
if strings.TrimSpace(signingKey) == "" {
return nil, nil
modules/git/repo.go
-10
View File
@@ -20,16 +20,6 @@ import (
"code.gitea.io/gitea/modules/proxy"
)
// GPGSettings represents the default GPG settings for this repository
type GPGSettings struct {
Sign bool
KeyID string
Email string
Name string
PublicKeyContent string
Format string
}
const prettyLogFormat = `--pretty=format:%H`
func (repo *Repository) ShowPrettyFormatLogToList(ctx context.Context, revisionRange string) ([]*Commit, error) {
modules/git/repo_base_gogit.go
-1
View File
@@ -32,7 +32,6 @@ type Repository struct {
gogitRepo *gogit.Repository
gogitStorage *filesystem.Storage
gpgSettings *GPGSettings
Ctx context.Context
LastCommitCache *LastCommitCache
modules/git/repo_base_nogogit.go
-2
View File
@@ -23,8 +23,6 @@ type Repository struct {
tagCache *ObjectCache[*Tag]
gpgSettings *GPGSettings
batchInUse bool
batch *Batch
modules/git/repo_gpg.go
-71
View File
@@ -1,71 +0,0 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"fmt"
"os"
"strings"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/process"
)
// LoadPublicKeyContent will load the key from gpg
func (gpgSettings *GPGSettings) LoadPublicKeyContent() error {
if gpgSettings.Format == SigningKeyFormatSSH {
content, err := os.ReadFile(gpgSettings.KeyID)
if err != nil {
return fmt.Errorf("unable to read SSH public key file: %s, %w", gpgSettings.KeyID, err)
}
gpgSettings.PublicKeyContent = string(content)
return nil
}
content, stderr, err := process.GetManager().Exec(
"gpg -a --export",
"gpg", "-a", "--export", gpgSettings.KeyID)
if err != nil {
return fmt.Errorf("unable to get default signing key: %s, %s, %w", gpgSettings.KeyID, stderr, err)
}
gpgSettings.PublicKeyContent = content
return nil
}
// GetDefaultPublicGPGKey will return and cache the default public GPG settings for this repository
func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, error) {
if repo.gpgSettings != nil && !forceUpdate {
return repo.gpgSettings, nil
}
gpgSettings := &GPGSettings{
Sign: true,
}
value, _, _ := gitcmd.NewCommand("config", "--get", "commit.gpgsign").WithDir(repo.Path).RunStdString(repo.Ctx)
sign, valid := ParseBool(strings.TrimSpace(value))
if !sign || !valid {
gpgSettings.Sign = false
repo.gpgSettings = gpgSettings
return gpgSettings, nil
}
signingKey, _, _ := gitcmd.NewCommand("config", "--get", "user.signingkey").WithDir(repo.Path).RunStdString(repo.Ctx)
gpgSettings.KeyID = strings.TrimSpace(signingKey)
format, _, _ := gitcmd.NewCommand("config", "--default", SigningKeyFormatOpenPGP, "--get", "gpg.format").WithDir(repo.Path).RunStdString(repo.Ctx)
gpgSettings.Format = strings.TrimSpace(format)
defaultEmail, _, _ := gitcmd.NewCommand("config", "--get", "user.email").WithDir(repo.Path).RunStdString(repo.Ctx)
gpgSettings.Email = strings.TrimSpace(defaultEmail)
defaultName, _, _ := gitcmd.NewCommand("config", "--get", "user.name").WithDir(repo.Path).RunStdString(repo.Ctx)
gpgSettings.Name = strings.TrimSpace(defaultName)
if err := gpgSettings.LoadPublicKeyContent(); err != nil {
return nil, err
}
repo.gpgSettings = gpgSettings
return repo.gpgSettings, nil
}
modules/gitrepo/blame.go
+208
View File
@@ -4,9 +4,16 @@
package gitrepo
import (
"bufio"
"bytes"
"context"
"io"
"os"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
func LineBlame(ctx context.Context, repo Repository, revision, file string, line uint) (string, error) {
@@ -16,3 +23,204 @@ func LineBlame(ctx context.Context, repo Repository, revision, file string, line
AddOptionValues("-p", revision).
AddDashesAndList(file))
}
// BlamePart represents block of blame - continuous lines with one sha
type BlamePart struct {
Sha string
Lines []string
PreviousSha string
PreviousPath string
}
// BlameReader returns part of file blame one by one
type BlameReader struct {
output io.WriteCloser
reader io.ReadCloser
bufferedReader *bufio.Reader
done chan error
lastSha *string
ignoreRevsFile string
objectFormat git.ObjectFormat
cleanupFuncs []func()
}
func (r *BlameReader) UsesIgnoreRevs() bool {
return r.ignoreRevsFile != ""
}
// NextPart returns next part of blame (sequential code lines with the same commit)
func (r *BlameReader) NextPart() (*BlamePart, error) {
var blamePart *BlamePart
if r.lastSha != nil {
blamePart = &BlamePart{
Sha: *r.lastSha,
Lines: make([]string, 0),
}
}
const previousHeader = "previous "
var lineBytes []byte
var isPrefix bool
var err error
for err != io.EOF {
lineBytes, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
if len(lineBytes) == 0 {
// isPrefix will be false
continue
}
var objectID string
objectFormatLength := r.objectFormat.FullLength()
if len(lineBytes) > objectFormatLength && lineBytes[objectFormatLength] == ' ' && r.objectFormat.IsValid(string(lineBytes[0:objectFormatLength])) {
objectID = string(lineBytes[0:objectFormatLength])
}
if len(objectID) > 0 {
if blamePart == nil {
blamePart = &BlamePart{
Sha: objectID,
Lines: make([]string, 0),
}
}
if blamePart.Sha != objectID {
r.lastSha = &objectID
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
return blamePart, nil
}
} else if lineBytes[0] == '\t' {
blamePart.Lines = append(blamePart.Lines, string(lineBytes[1:]))
} else if bytes.HasPrefix(lineBytes, []byte(previousHeader)) {
offset := len(previousHeader) // already includes a space
blamePart.PreviousSha = string(lineBytes[offset : offset+objectFormatLength])
offset += objectFormatLength + 1 // +1 for space
blamePart.PreviousPath = string(lineBytes[offset:])
}
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
}
r.lastSha = nil
return blamePart, nil
}
// Close BlameReader - don't run NextPart after invoking that
func (r *BlameReader) Close() error {
if r.bufferedReader == nil {
return nil
}
err := <-r.done
r.bufferedReader = nil
_ = r.reader.Close()
_ = r.output.Close()
for _, cleanup := range r.cleanupFuncs {
if cleanup != nil {
cleanup()
}
}
return err
}
// CreateBlameReader creates reader for given repository, commit and file
func CreateBlameReader(ctx context.Context, objectFormat git.ObjectFormat, repo Repository, commit *git.Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) {
var ignoreRevsFileName string
var ignoreRevsFileCleanup func()
defer func() {
if err != nil && ignoreRevsFileCleanup != nil {
ignoreRevsFileCleanup()
}
}()
cmd := gitcmd.NewCommand("blame", "--porcelain")
if git.DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore {
ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit)
if err != nil && !git.IsErrNotExist(err) {
return nil, err
}
if ignoreRevsFileName != "" {
// Possible improvement: use --ignore-revs-file /dev/stdin on unix
// There is no equivalent on Windows. May be implemented if Gitea uses an external git backend.
cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName)
}
}
cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file)
done := make(chan error, 1)
reader, stdout, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
stderr := bytes.Buffer{}
// TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close"
err := RunCmd(ctx, repo, cmd.WithUseContextTimeout(true).
WithStdout(stdout).
WithStderr(&stderr),
)
done <- err
_ = stdout.Close()
if err != nil {
log.Error("Error running git blame (dir: %v): %v, stderr: %v", repoPath, err, stderr.String())
}
}()
bufferedReader := bufio.NewReader(reader)
return &BlameReader{
output: stdout,
reader: reader,
bufferedReader: bufferedReader,
done: done,
ignoreRevsFile: ignoreRevsFileName,
objectFormat: objectFormat,
cleanupFuncs: []func(){ignoreRevsFileCleanup},
}, nil
}
func tryCreateBlameIgnoreRevsFile(commit *git.Commit) (string, func(), error) {
entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
r, err := entry.Blob().DataAsync()
if err != nil {
return "", nil, err
}
defer r.Close()
f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
filename := f.Name()
_, err = io.Copy(f, r)
_ = f.Close()
if err != nil {
cleanup()
return "", nil, err
}
return filename, cleanup, nil
}
modules/git/blame_sha256_test.go → modules/gitrepo/blame_sha256_test.go
+9 -6
View File
@@ -1,12 +1,13 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
package gitrepo
import (
"context"
"testing"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
@@ -17,13 +18,14 @@ func TestReadingBlameOutputSha256(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
if isGogit {
if git.DefaultFeatures().UsingGogit {
t.Skip("Skipping test since gogit does not support sha256")
return
}
t.Run("Without .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls_sha256")
storage := &mockRepository{path: "repo5_pulls_sha256"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
@@ -47,7 +49,7 @@ func TestReadingBlameOutputSha256(t *testing.T) {
}
for _, bypass := range []bool{false, true} {
blameReader, err := CreateBlameReader(ctx, Sha256ObjectFormat, "./tests/repos/repo5_pulls_sha256", commit, "README.md", bypass)
blameReader, err := CreateBlameReader(ctx, git.Sha256ObjectFormat, storage, commit, "README.md", bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
@@ -68,7 +70,8 @@ func TestReadingBlameOutputSha256(t *testing.T) {
})
t.Run("With .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame_sha256")
storage := &mockRepository{path: "repo6_blame_sha256"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
@@ -131,7 +134,7 @@ func TestReadingBlameOutputSha256(t *testing.T) {
for _, c := range cases {
commit, err := repo.GetCommit(c.CommitID)
assert.NoError(t, err)
blameReader, err := CreateBlameReader(ctx, objectFormat, "./tests/repos/repo6_blame_sha256", commit, "blame.txt", c.Bypass)
blameReader, err := CreateBlameReader(ctx, objectFormat, storage, commit, "blame.txt", c.Bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
modules/git/blame_test.go → modules/gitrepo/blame_test.go
+8 -6
View File
@@ -1,12 +1,13 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
package gitrepo
import (
"context"
"testing"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
@@ -18,10 +19,10 @@ func TestReadingBlameOutput(t *testing.T) {
defer cancel()
t.Run("Without .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls")
storage := &mockRepository{path: "repo5_pulls"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
commit, err := repo.GetCommit("f32b0a9dfd09a60f616f29158f772cedd89942d2")
assert.NoError(t, err)
@@ -42,7 +43,7 @@ func TestReadingBlameOutput(t *testing.T) {
}
for _, bypass := range []bool{false, true} {
blameReader, err := CreateBlameReader(ctx, Sha1ObjectFormat, "./tests/repos/repo5_pulls", commit, "README.md", bypass)
blameReader, err := CreateBlameReader(ctx, git.Sha1ObjectFormat, storage, commit, "README.md", bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
@@ -63,7 +64,8 @@ func TestReadingBlameOutput(t *testing.T) {
})
t.Run("With .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame")
storage := &mockRepository{path: "repo6_blame"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
@@ -127,7 +129,7 @@ func TestReadingBlameOutput(t *testing.T) {
commit, err := repo.GetCommit(c.CommitID)
assert.NoError(t, err)
blameReader, err := CreateBlameReader(ctx, objectFormat, "./tests/repos/repo6_blame", commit, "blame.txt", c.Bypass)
blameReader, err := CreateBlameReader(ctx, objectFormat, storage, commit, "blame.txt", c.Bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
modules/gitrepo/gitrepo.go
+2 -2
View File
@@ -109,8 +109,8 @@ func IsRepoDirExist(ctx context.Context, repo Repository, relativeDirPath string
return util.IsDir(absoluteDirPath)
}
func RemoveRepoFile(ctx context.Context, repo Repository, relativeFilePath string) error {
absoluteFilePath := filepath.Join(repoPath(repo), relativeFilePath)
func RemoveRepoFileOrDir(ctx context.Context, repo Repository, relativeFileOrDirPath string) error {
absoluteFilePath := filepath.Join(repoPath(repo), relativeFileOrDirPath)
return util.Remove(absoluteFilePath)
}
modules/gitrepo/signing.go
+2 -2
View File
@@ -9,6 +9,6 @@ import (
"code.gitea.io/gitea/modules/git"
)
func GetSigningKey(ctx context.Context, repo Repository) (*git.SigningKey, *git.Signature) {
return git.GetSigningKey(ctx, repoPath(repo))
func GetSigningKey(ctx context.Context) (*git.SigningKey, *git.Signature) {
return git.GetSigningKey(ctx)
}
modules/highlight/highlight.go
+33 -1
View File
@@ -56,7 +56,39 @@ func NewContext() {
})
}
// Code returns a HTML version of code string with chroma syntax highlighting classes and the matched lexer name
// UnsafeSplitHighlightedLines splits highlighted code into lines preserving HTML tags
// It always includes '\n', '\n' can appear at the end of each line or in the middle of HTML tags
// The '\n' is necessary for copying code from web UI to preserve original code lines
// ATTENTION: It uses the unsafe conversion between string and []byte for performance reason
// DO NOT make any modification to the returned [][]byte slice items
func UnsafeSplitHighlightedLines(code template.HTML) (ret [][]byte) {
buf := util.UnsafeStringToBytes(string(code))
lineCount := bytes.Count(buf, []byte("\n")) + 1
ret = make([][]byte, 0, lineCount)
nlTagClose := []byte("\n</")
for {
pos := bytes.IndexByte(buf, '\n')
if pos == -1 {
if len(buf) > 0 {
ret = append(ret, buf)
}
return ret
}
// Chroma highlighting output sometimes have "</span>" right after \n, sometimes before.
// * "<span>text\n</span>"
// * "<span>text</span>\n"
if bytes.HasPrefix(buf[pos:], nlTagClose) {
pos1 := bytes.IndexByte(buf[pos:], '>')
if pos1 != -1 {
pos += pos1
}
}
ret = append(ret, buf[:pos+1])
buf = buf[pos+1:]
}
}
// Code returns an HTML version of code string with chroma syntax highlighting classes and the matched lexer name
func Code(fileName, language, code string) (output template.HTML, lexerName string) {
NewContext()
modules/highlight/highlight_test.go
+18
View File
@@ -181,3 +181,21 @@ c=2`),
})
}
}
func TestUnsafeSplitHighlightedLines(t *testing.T) {
ret := UnsafeSplitHighlightedLines("")
assert.Empty(t, ret)
ret = UnsafeSplitHighlightedLines("a")
assert.Len(t, ret, 1)
assert.Equal(t, "a", string(ret[0]))
ret = UnsafeSplitHighlightedLines("\n")
assert.Len(t, ret, 1)
assert.Equal(t, "\n", string(ret[0]))
ret = UnsafeSplitHighlightedLines("<span>a</span>\n<span>b\n</span>")
assert.Len(t, ret, 2)
assert.Equal(t, "<span>a</span>\n", string(ret[0]))
assert.Equal(t, "<span>b\n</span>", string(ret[1]))
}
modules/httplib/serve.go
+1 -6
View File
@@ -19,7 +19,6 @@ import (
charsetModule "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
@@ -109,11 +108,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byt
}
if isPlain {
charset, err := charsetModule.DetectEncoding(mineBuf)
if err != nil {
log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err)
charset = "utf-8"
}
charset, _ := charsetModule.DetectEncoding(mineBuf)
opts.ContentTypeCharset = strings.ToLower(charset)
}
modules/indexer/code/bleve/bleve.go
+1 -1
View File
@@ -203,7 +203,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro
RepoID: repo.ID,
CommitID: commitSha,
Filename: update.Filename,
Content: string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})),
Content: string(charset.ToUTF8DropErrors(fileContents)),
Language: analyze.GetCodeLanguage(update.Filename, fileContents),
UpdatedAt: time.Now().UTC(),
})
modules/indexer/code/elasticsearch/elasticsearch.go
+1 -1
View File
@@ -191,7 +191,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro
Doc(map[string]any{
"repo_id": repo.ID,
"filename": update.Filename,
"content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})),
"content": string(charset.ToUTF8DropErrors(fileContents)),
"commit_id": sha,
"language": analyze.GetCodeLanguage(update.Filename, fileContents),
"updated_at": timeutil.TimeStampNow(),
modules/indexer/issues/elasticsearch/elasticsearch_test.go
+5 -1
View File
@@ -30,7 +30,11 @@ func TestElasticsearchIndexer(t *testing.T) {
require.Eventually(t, func() bool {
resp, err := http.Get(url)
return err == nil && resp.StatusCode == http.StatusOK
if err != nil {
return false
}
defer resp.Body.Close()
return resp.StatusCode == http.StatusOK
}, time.Minute, time.Second, "Expected elasticsearch to be up")
indexer := NewIndexer(url, fmt.Sprintf("test_elasticsearch_indexer_%d", time.Now().Unix()))
modules/indexer/issues/meilisearch/meilisearch_test.go
+5 -1
View File
@@ -36,7 +36,11 @@ func TestMeilisearchIndexer(t *testing.T) {
require.Eventually(t, func() bool {
resp, err := http.Get(url)
return err == nil && resp.StatusCode == http.StatusOK
if err != nil {
return false
}
defer resp.Body.Close()
return resp.StatusCode == http.StatusOK
}, time.Minute, time.Second, "Expected meilisearch to be up")
indexer := NewIndexer(url, key, fmt.Sprintf("test_meilisearch_indexer_%d", time.Now().Unix()))
modules/migration/uploader.go
+1
View File
@@ -14,6 +14,7 @@ type Uploader interface {
CreateMilestones(ctx context.Context, milestones ...*Milestone) error
CreateReleases(ctx context.Context, releases ...*Release) error
SyncTags(ctx context.Context) error
SyncBranches(ctx context.Context) error
CreateLabels(ctx context.Context, labels ...*Label) error
CreateIssues(ctx context.Context, issues ...*Issue) error
CreateComments(ctx context.Context, comments ...*Comment) error
modules/packages/npm/creator.go
+23 -2
View File
@@ -62,7 +62,28 @@ type PackageMetadata struct {
Author User `json:"author"`
ReadmeFilename string `json:"readmeFilename,omitempty"`
Users map[string]bool `json:"users,omitempty"`
License string `json:"license,omitempty"`
License License `json:"license,omitempty"`
}
type License string
func (l *License) UnmarshalJSON(data []byte) error {
switch data[0] {
case '"':
var value string
if err := json.Unmarshal(data, &value); err != nil {
return err
}
*l = License(value)
case '{':
var values map[string]any
if err := json.Unmarshal(data, &values); err != nil {
return err
}
value, _ := values["type"].(string)
*l = License(value)
}
return nil
}
// PackageMetadataVersion documentation: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
@@ -74,7 +95,7 @@ type PackageMetadataVersion struct {
Description string `json:"description"`
Author User `json:"author"`
Homepage string `json:"homepage,omitempty"`
License string `json:"license,omitempty"`
License License `json:"license,omitempty"`
Repository Repository `json:"repository"`
Keywords []string `json:"keywords,omitempty"`
Dependencies map[string]string `json:"dependencies,omitempty"`
modules/packages/npm/creator_test.go
+27 -1
View File
@@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/modules/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParsePackage(t *testing.T) {
@@ -291,11 +292,36 @@ func TestParsePackage(t *testing.T) {
assert.Equal(t, packageDescription, p.Metadata.Readme)
assert.Equal(t, packageAuthor, p.Metadata.Author)
assert.Equal(t, packageBin, p.Metadata.Bin["bin"])
assert.Equal(t, "MIT", p.Metadata.License)
assert.Equal(t, "MIT", string(p.Metadata.License))
assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL)
assert.Contains(t, p.Metadata.Dependencies, "package")
assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"])
assert.Equal(t, repository.Type, p.Metadata.Repository.Type)
assert.Equal(t, repository.URL, p.Metadata.Repository.URL)
})
t.Run("ValidLicenseMap", func(t *testing.T) {
packageJSON := `{
"versions": {
"0.1.1": {
"name": "dev-null",
"version": "0.1.1",
"license": {
"type": "MIT"
},
"dist": {
"integrity": "sha256-"
}
}
},
"_attachments": {
"foo": {
"data": "AAAA"
}
}
}`
p, err := ParsePackage(strings.NewReader(packageJSON))
require.NoError(t, err)
require.Equal(t, "MIT", string(p.Metadata.License))
})
}
modules/packages/npm/metadata.go
+1 -1
View File
@@ -12,7 +12,7 @@ type Metadata struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Author string `json:"author,omitempty"`
License string `json:"license,omitempty"`
License License `json:"license,omitempty"`
ProjectURL string `json:"project_url,omitempty"`
Keywords []string `json:"keywords,omitempty"`
Dependencies map[string]string `json:"dependencies,omitempty"`
modules/setting/config.go
+4 -3
View File
@@ -4,6 +4,7 @@
package setting
import (
"strings"
"sync"
"code.gitea.io/gitea/modules/log"
@@ -23,11 +24,11 @@ type OpenWithEditorApp struct {
type OpenWithEditorAppsType []OpenWithEditorApp
func (t OpenWithEditorAppsType) ToTextareaString() string {
ret := ""
var ret strings.Builder
for _, app := range t {
ret += app.DisplayName + " = " + app.OpenURL + "\n"
ret.WriteString(app.DisplayName + " = " + app.OpenURL + "\n")
}
return ret
return ret.String()
}
func DefaultOpenWithEditorApps() OpenWithEditorAppsType {
modules/setting/git.go
+35 -27
View File
@@ -5,6 +5,7 @@ package setting
import (
"path/filepath"
"regexp"
"strings"
"time"
@@ -17,20 +18,21 @@ var Git = struct {
HomePath string
DisableDiffHighlight bool
MaxGitDiffLines int
MaxGitDiffLineCharacters int
MaxGitDiffFiles int
CommitsRangeSize int // CommitsRangeSize the default commits range size
BranchesRangeSize int // BranchesRangeSize the default branches range size
VerbosePush bool
VerbosePushDelay time.Duration
GCArgs []string `ini:"GC_ARGS" delim:" "`
EnableAutoGitWireProtocol bool
PullRequestPushMessage bool
LargeObjectThreshold int64
DisableCoreProtectNTFS bool
DisablePartialClone bool
Timeout struct {
MaxGitDiffLines int
MaxGitDiffLineCharacters int
MaxGitDiffFiles int
CommitsRangeSize int // CommitsRangeSize the default commits range size
BranchesRangeSize int // BranchesRangeSize the default branches range size
VerbosePush bool
VerbosePushDelay time.Duration
GCArgs []string `ini:"GC_ARGS" delim:" "`
EnableAutoGitWireProtocol bool
PullRequestPushMessage bool
LargeObjectThreshold int64
DisableCoreProtectNTFS bool
DisablePartialClone bool
DiffRenameSimilarityThreshold string
Timeout struct {
Default int
Migrate int
Mirror int
@@ -39,19 +41,20 @@ var Git = struct {
GC int `ini:"GC"`
} `ini:"git.timeout"`
}{
DisableDiffHighlight: false,
MaxGitDiffLines: 1000,
MaxGitDiffLineCharacters: 5000,
MaxGitDiffFiles: 100,
CommitsRangeSize: 50,
BranchesRangeSize: 20,
VerbosePush: true,
VerbosePushDelay: 5 * time.Second,
GCArgs: []string{},
EnableAutoGitWireProtocol: true,
PullRequestPushMessage: true,
LargeObjectThreshold: 1024 * 1024,
DisablePartialClone: false,
DisableDiffHighlight: false,
MaxGitDiffLines: 1000,
MaxGitDiffLineCharacters: 5000,
MaxGitDiffFiles: 100,
CommitsRangeSize: 50,
BranchesRangeSize: 20,
VerbosePush: true,
VerbosePushDelay: 5 * time.Second,
GCArgs: []string{},
EnableAutoGitWireProtocol: true,
PullRequestPushMessage: true,
LargeObjectThreshold: 1024 * 1024,
DisablePartialClone: false,
DiffRenameSimilarityThreshold: "50%",
Timeout: struct {
Default int
Migrate int
@@ -117,4 +120,9 @@ func loadGitFrom(rootCfg ConfigProvider) {
} else {
Git.HomePath = filepath.Clean(Git.HomePath)
}
// validate for a integer percentage between 0% and 100%
if !regexp.MustCompile(`^([0-9]|[1-9][0-9]|100)%$`).MatchString(Git.DiffRenameSimilarityThreshold) {
log.Fatal("Invalid git.DIFF_RENAME_SIMILARITY_THRESHOLD: %s", Git.DiffRenameSimilarityThreshold)
}
}
modules/setting/setting.go
+1
View File
@@ -240,4 +240,5 @@ func PanicInDevOrTesting(msg string, a ...any) {
if !IsProd || IsInTesting {
panic(fmt.Sprintf(msg, a...))
}
log.Error(msg, a...)
}
modules/templates/util_render.go
+5 -4
View File
@@ -249,17 +249,18 @@ func (ut *RenderUtils) MarkdownToHtml(input string) template.HTML { //nolint:rev
func (ut *RenderUtils) RenderLabels(labels []*issues_model.Label, repoLink string, issue *issues_model.Issue) template.HTML {
isPullRequest := issue != nil && issue.IsPull
baseLink := fmt.Sprintf("%s/%s", repoLink, util.Iif(isPullRequest, "pulls", "issues"))
htmlCode := `<span class="labels-list">`
var htmlCode strings.Builder
htmlCode.WriteString(`<span class="labels-list">`)
for _, label := range labels {
// Protect against nil value in labels - shouldn't happen but would cause a panic if so
if label == nil {
continue
}
link := fmt.Sprintf("%s?labels=%d", baseLink, label.ID)
htmlCode += string(ut.RenderLabelWithLink(label, template.URL(link)))
htmlCode.WriteString(string(ut.RenderLabelWithLink(label, template.URL(link))))
}
htmlCode += "</span>"
return template.HTML(htmlCode)
htmlCode.WriteString("</span>")
return template.HTML(htmlCode.String())
}
func (ut *RenderUtils) RenderThemeItem(info *webtheme.ThemeMetaInfo, iconSize int) template.HTML {