Merge remote-tracking branch 'upstream/main' into limit-repo-size

2026-09-19 23:22:57 +02:00 · 2024-09-07 15:21:12 -04:00
parent bf1870ddb2 d8f3498875
commit 748dbd012f
2043 changed files with 66095 additions and 81907 deletions
@@ -4,85 +4,67 @@
 package base

 import (
-	"math/big"
 	"unicode/utf8"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
 )

+func naturalSortGetRune(str string, pos int) (r rune, size int, has bool) {
+	if pos >= len(str) {
+		return 0, 0, false
+	}
+	r, size = utf8.DecodeRuneInString(str[pos:])
+	if r == utf8.RuneError {
+		r, size = rune(str[pos]), 1 // if invalid input, treat it as a single byte ascii
+	}
+	return r, size, true
+}
+
+func naturalSortAdvance(str string, pos int) (end int, isNumber bool) {
+	end = pos
+	for {
+		r, size, has := naturalSortGetRune(str, end)
+		if !has {
+			break
+		}
+		isCurRuneNum := '0' <= r && r <= '9'
+		if end == pos {
+			isNumber = isCurRuneNum
+			end += size
+		} else if isCurRuneNum == isNumber {
+			end += size
+		} else {
+			break
+		}
+	}
+	return end, isNumber
+}
+
 // NaturalSortLess compares two strings so that they could be sorted in natural order
 func NaturalSortLess(s1, s2 string) bool {
-	var i1, i2 int
-	for {
-		rune1, j1, end1 := getNextRune(s1, i1)
-		rune2, j2, end2 := getNextRune(s2, i2)
-		if end1 || end2 {
-			return end1 != end2 && end1
-		}
-		dec1 := isDecimal(rune1)
-		dec2 := isDecimal(rune2)
-		var less, equal bool
-		if dec1 && dec2 {
-			i1, i2, less, equal = compareByNumbers(s1, i1, s2, i2)
-		} else if !dec1 && !dec2 {
-			equal = rune1 == rune2
-			less = rune1 < rune2
-			i1 = j1
-			i2 = j2
+	// There is a bug in Golang's collate package: https://github.com/golang/go/issues/67997
+	// text/collate: CompareString(collate.Numeric) returns wrong result for "0.0" vs "1.0" #67997
+	// So we need to handle the number parts by ourselves
+	c := collate.New(language.English, collate.Numeric)
+	pos1, pos2 := 0, 0
+	for pos1 < len(s1) && pos2 < len(s2) {
+		end1, isNum1 := naturalSortAdvance(s1, pos1)
+		end2, isNum2 := naturalSortAdvance(s2, pos2)
+		part1, part2 := s1[pos1:end1], s2[pos2:end2]
+		if isNum1 && isNum2 {
+			if part1 != part2 {
+				if len(part1) != len(part2) {
+					return len(part1) < len(part2)
+				}
+				return part1 < part2
+			}
 		} else {
-			return rune1 < rune2
-		}
-		if !equal {
-			return less
-		}
-	}
-}
-
-func getNextRune(str string, pos int) (rune, int, bool) {
-	if pos < len(str) {
-		r, w := utf8.DecodeRuneInString(str[pos:])
-		// Fallback to ascii
-		if r == utf8.RuneError {
-			r = rune(str[pos])
-			w = 1
-		}
-		return r, pos + w, false
-	}
-	return 0, pos, true
-}
-
-func isDecimal(r rune) bool {
-	return '0' <= r && r <= '9'
-}
-
-func compareByNumbers(str1 string, pos1 int, str2 string, pos2 int) (i1, i2 int, less, equal bool) {
-	d1, d2 := true, true
-	var dec1, dec2 string
-	for d1 || d2 {
-		if d1 {
-			r, j, end := getNextRune(str1, pos1)
-			if !end && isDecimal(r) {
-				dec1 += string(r)
-				pos1 = j
-			} else {
-				d1 = false
-			}
-		}
-		if d2 {
-			r, j, end := getNextRune(str2, pos2)
-			if !end && isDecimal(r) {
-				dec2 += string(r)
-				pos2 = j
-			} else {
-				d2 = false
+			if cmp := c.CompareString(part1, part2); cmp != 0 {
+				return cmp < 0
 			}
 		}
+		pos1, pos2 = end1, end2
 	}
-	less, equal = compareBigNumbers(dec1, dec2)
-	return pos1, pos2, less, equal
-}
-
-func compareBigNumbers(dec1, dec2 string) (less, equal bool) {
-	d1, _ := big.NewInt(0).SetString(dec1, 10)
-	d2, _ := big.NewInt(0).SetString(dec2, 10)
-	cmp := d1.Cmp(d2)
-	return cmp < 0, cmp == 0
+	return len(s1) < len(s2)
 }
@@ -10,14 +10,36 @@ import (
 )

 func TestNaturalSortLess(t *testing.T) {
-	test := func(s1, s2 string, less bool) {
-		assert.Equal(t, less, NaturalSortLess(s1, s2))
+	testLess := func(s1, s2 string) {
+		assert.True(t, NaturalSortLess(s1, s2), "s1<s2 should be true: s1=%q, s2=%q", s1, s2)
+		assert.False(t, NaturalSortLess(s2, s1), "s2<s1 should be false: s1=%q, s2=%q", s1, s2)
 	}
-	test("v1.20.0", "v1.2.0", false)
-	test("v1.20.0", "v1.29.0", true)
-	test("v1.20.0", "v1.20.0", false)
-	test("abc", "bcd", true)
-	test("a-1-a", "a-1-b", true)
-	test("2", "12", true)
-	test("a", "ab", true)
+	testEqual := func(s1, s2 string) {
+		assert.False(t, NaturalSortLess(s1, s2), "s1<s2 should be false: s1=%q, s2=%q", s1, s2)
+		assert.False(t, NaturalSortLess(s2, s1), "s2<s1 should be false: s1=%q, s2=%q", s1, s2)
+	}
+
+	testEqual("", "")
+	testLess("", "a")
+	testLess("", "1")
+
+	testLess("v1.2", "v1.2.0")
+	testLess("v1.2.0", "v1.10.0")
+	testLess("v1.20.0", "v1.29.0")
+	testEqual("v1.20.0", "v1.20.0")
+
+	testLess("a", "A")
+	testLess("a", "B")
+	testLess("A", "b")
+	testLess("A", "ab")
+
+	testLess("abc", "bcd")
+	testLess("a-1-a", "a-1-b")
+	testLess("2", "12")
+
+	testLess("cafe", "café")
+	testLess("café", "caff")
+
+	testLess("A-2", "A-11")
+	testLess("0.txt", "1.txt")
 }
@@ -4,12 +4,15 @@
 package base

 import (
+	"crypto/hmac"
 	"crypto/sha1"
 	"crypto/sha256"
+	"crypto/subtle"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"hash"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -25,13 +28,6 @@ import (
 	"github.com/dustin/go-humanize"
 )

-// EncodeSha1 string to sha1 hex value.
-func EncodeSha1(str string) string {
-	h := sha1.New()
-	_, _ = h.Write([]byte(str))
-	return hex.EncodeToString(h.Sum(nil))
-}
-
 // EncodeSha256 string to sha256 hex value.
 func EncodeSha256(str string) string {
 	h := sha256.New()
@@ -52,73 +48,69 @@ func BasicAuthDecode(encoded string) (string, string, error) {
 		return "", "", err
 	}

-	auth := strings.SplitN(string(s), ":", 2)
-
-	if len(auth) != 2 {
-		return "", "", errors.New("invalid basic authentication")
+	if username, password, ok := strings.Cut(string(s), ":"); ok {
+		return username, password, nil
 	}
-
-	return auth[0], auth[1], nil
+	return "", "", errors.New("invalid basic authentication")
 }

 // VerifyTimeLimitCode verify time limit code
-func VerifyTimeLimitCode(data string, minutes int, code string) bool {
+func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) bool {
 	if len(code) <= 18 {
 		return false
 	}

-	// split code
-	start := code[:12]
-	lives := code[12:18]
-	if d, err := strconv.ParseInt(lives, 10, 0); err == nil {
-		minutes = int(d)
-	}
+	startTimeStr := code[:12]
+	aliveTimeStr := code[12:18]
+	aliveTime, _ := strconv.Atoi(aliveTimeStr) // no need to check err, if anything wrong, the following code check will fail soon

-	// right active code
-	retCode := CreateTimeLimitCode(data, minutes, start)
-	if retCode == code && minutes > 0 {
-		// check time is expired or not
-		before, _ := time.ParseInLocation("200601021504", start, time.Local)
-		now := time.Now()
-		if before.Add(time.Minute*time.Duration(minutes)).Unix() > now.Unix() {
-			return true
+	// check code
+	retCode := CreateTimeLimitCode(data, aliveTime, startTimeStr, nil)
+	if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 {
+		retCode = CreateTimeLimitCode(data, aliveTime, startTimeStr, sha1.New()) // TODO: this is only for the support of legacy codes, remove this in/after 1.23
+		if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 {
+			return false
 		}
 	}

-	return false
+	// check time is expired or not: startTime <= now && now < startTime + minutes
+	startTime, _ := time.ParseInLocation("200601021504", startTimeStr, time.Local)
+	return (startTime.Before(now) || startTime.Equal(now)) && now.Before(startTime.Add(time.Minute*time.Duration(minutes)))
 }

 // TimeLimitCodeLength default value for time limit code
 const TimeLimitCodeLength = 12 + 6 + 40

-// CreateTimeLimitCode create a time limit code
-// code format: 12 length date time string + 6 minutes string + 40 sha1 encoded string
-func CreateTimeLimitCode(data string, minutes int, startInf any) string {
-	format := "200601021504"
+// CreateTimeLimitCode create a time-limited code.
+// Format: 12 length date time string + 6 minutes string (not used) + 40 hash string, some other code depends on this fixed length
+// If h is nil, then use the default hmac hash.
+func CreateTimeLimitCode[T time.Time | string](data string, minutes int, startTimeGeneric T, h hash.Hash) string {
+	const format = "200601021504"

-	var start, end time.Time
-	var startStr, endStr string
-
-	if startInf == nil {
-		// Use now time create code
-		start = time.Now()
-		startStr = start.Format(format)
+	var start time.Time
+	var startTimeAny any = startTimeGeneric
+	if t, ok := startTimeAny.(time.Time); ok {
+		start = t
 	} else {
-		// use start string create code
-		startStr = startInf.(string)
-		start, _ = time.ParseInLocation(format, startStr, time.Local)
-		startStr = start.Format(format)
+		var err error
+		start, err = time.ParseInLocation(format, startTimeAny.(string), time.Local)
+		if err != nil {
+			return "" // return an invalid code because the "parse" failed
+		}
 	}
+	startStr := start.Format(format)
+	end := start.Add(time.Minute * time.Duration(minutes))

-	end = start.Add(time.Minute * time.Duration(minutes))
-	endStr = end.Format(format)
-
-	// create sha1 encode string
-	sh := sha1.New()
-	_, _ = sh.Write([]byte(fmt.Sprintf("%s%s%s%s%d", data, hex.EncodeToString(setting.GetGeneralTokenSigningSecret()), startStr, endStr, minutes)))
-	encoded := hex.EncodeToString(sh.Sum(nil))
+	if h == nil {
+		h = hmac.New(sha1.New, setting.GetGeneralTokenSigningSecret())
+	}
+	_, _ = fmt.Fprintf(h, "%s%s%s%s%d", data, hex.EncodeToString(setting.GetGeneralTokenSigningSecret()), startStr, end.Format(format), minutes)
+	encoded := hex.EncodeToString(h.Sum(nil))

 	code := fmt.Sprintf("%s%06d%s", startStr, minutes, encoded)
+	if len(code) != TimeLimitCodeLength {
+		panic("there is a hard requirement for the length of time-limited code") // it shouldn't happen
+	}
 	return code
 }

@@ -157,13 +149,16 @@ func TruncateString(str string, limit int) string {

 // StringsToInt64s converts a slice of string to a slice of int64.
 func StringsToInt64s(strs []string) ([]int64, error) {
-	ints := make([]int64, len(strs))
-	for i := range strs {
-		n, err := strconv.ParseInt(strs[i], 10, 64)
+	if strs == nil {
+		return nil, nil
+	}
+	ints := make([]int64, 0, len(strs))
+	for _, s := range strs {
+		n, err := strconv.ParseInt(s, 10, 64)
 		if err != nil {
-			return ints, err
+			return nil, err
 		}
-		ints[i] = n
+		ints = append(ints, n)
 	}
 	return ints, nil
 }
@@ -4,20 +4,18 @@
 package base

 import (
+	"crypto/sha1"
+	"fmt"
 	"os"
 	"testing"
 	"time"

+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+
 	"github.com/stretchr/testify/assert"
 )

-func TestEncodeSha1(t *testing.T) {
-	assert.Equal(t,
-		"8843d7f92416211de9ebb963ff4ce28125932878",
-		EncodeSha1("foobar"),
-	)
-}
-
 func TestEncodeSha256(t *testing.T) {
 	assert.Equal(t,
 		"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
@@ -43,46 +41,60 @@ func TestBasicAuthDecode(t *testing.T) {

 	_, _, err = BasicAuthDecode("invalid")
 	assert.Error(t, err)
+
+	_, _, err = BasicAuthDecode("YWxpY2U=") // "alice", no colon
+	assert.Error(t, err)
 }

 func TestVerifyTimeLimitCode(t *testing.T) {
-	tc := []struct {
-		data    string
-		minutes int
-		code    string
-		valid   bool
-	}{{
-		data:    "data",
-		minutes: 2,
-		code:    testCreateTimeLimitCode(t, "data", 2),
-		valid:   true,
-	}, {
-		data:    "abc123-ß",
-		minutes: 1,
-		code:    testCreateTimeLimitCode(t, "abc123-ß", 1),
-		valid:   true,
-	}, {
-		data:    "data",
-		minutes: 2,
-		code:    "2021012723240000005928251dac409d2c33a6eb82c63410aaad569bed",
-		valid:   false,
-	}}
-	for _, test := range tc {
-		actualValid := VerifyTimeLimitCode(test.data, test.minutes, test.code)
-		assert.Equal(t, test.valid, actualValid, "data: '%s' code: '%s' should be valid: %t", test.data, test.code, test.valid)
+	defer test.MockVariableValue(&setting.InstallLock, true)()
+	initGeneralSecret := func(secret string) {
+		setting.InstallLock = true
+		setting.CfgProvider, _ = setting.NewConfigProviderFromData(fmt.Sprintf(`
+[oauth2]
+JWT_SECRET = %s
+`, secret))
+		setting.LoadCommonSettings()
 	}
-}

-func testCreateTimeLimitCode(t *testing.T, data string, m int) string {
-	result0 := CreateTimeLimitCode(data, m, nil)
-	result1 := CreateTimeLimitCode(data, m, time.Now().Format("200601021504"))
-	result2 := CreateTimeLimitCode(data, m, time.Unix(time.Now().Unix()+int64(time.Minute)*int64(m), 0).Format("200601021504"))
+	initGeneralSecret("KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko")
+	now := time.Now()

-	assert.Equal(t, result0, result1)
-	assert.NotEqual(t, result0, result2)
+	t.Run("TestGenericParameter", func(t *testing.T) {
+		time2000 := time.Date(2000, 1, 2, 3, 4, 5, 0, time.Local)
+		assert.Equal(t, "2000010203040000026fa5221b2731b7cf80b1b506f5e39e38c115fee5", CreateTimeLimitCode("test-sha1", 2, time2000, sha1.New()))
+		assert.Equal(t, "2000010203040000026fa5221b2731b7cf80b1b506f5e39e38c115fee5", CreateTimeLimitCode("test-sha1", 2, "200001020304", sha1.New()))
+		assert.Equal(t, "2000010203040000024842227a2f87041ff82025199c0187410a9297bf", CreateTimeLimitCode("test-hmac", 2, time2000, nil))
+		assert.Equal(t, "2000010203040000024842227a2f87041ff82025199c0187410a9297bf", CreateTimeLimitCode("test-hmac", 2, "200001020304", nil))
+	})

-	assert.True(t, len(result0) != 0)
-	return result0
+	t.Run("TestInvalidCode", func(t *testing.T) {
+		assert.False(t, VerifyTimeLimitCode(now, "data", 2, ""))
+		assert.False(t, VerifyTimeLimitCode(now, "data", 2, "invalid code"))
+	})
+
+	t.Run("TestCreateAndVerify", func(t *testing.T) {
+		code := CreateTimeLimitCode("data", 2, now, nil)
+		assert.False(t, VerifyTimeLimitCode(now.Add(-time.Minute), "data", 2, code)) // not started yet
+		assert.True(t, VerifyTimeLimitCode(now, "data", 2, code))
+		assert.True(t, VerifyTimeLimitCode(now.Add(time.Minute), "data", 2, code))
+		assert.False(t, VerifyTimeLimitCode(now.Add(time.Minute), "DATA", 2, code))   // invalid data
+		assert.False(t, VerifyTimeLimitCode(now.Add(2*time.Minute), "data", 2, code)) // expired
+	})
+
+	t.Run("TestDifferentSecret", func(t *testing.T) {
+		// use another secret to ensure the code is invalid for different secret
+		verifyDataCode := func(c string) bool {
+			return VerifyTimeLimitCode(now, "data", 2, c)
+		}
+		code1 := CreateTimeLimitCode("data", 2, now, sha1.New())
+		code2 := CreateTimeLimitCode("data", 2, now, nil)
+		assert.True(t, verifyDataCode(code1))
+		assert.True(t, verifyDataCode(code2))
+		initGeneralSecret("000_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko")
+		assert.False(t, verifyDataCode(code1))
+		assert.False(t, verifyDataCode(code2))
+	})
 }

 func TestFileSize(t *testing.T) {
@@ -145,12 +157,13 @@ func TestStringsToInt64s(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, expected, result)
 	}
+	testSuccess(nil, nil)
 	testSuccess([]string{}, []int64{})
 	testSuccess([]string{"-1234"}, []int64{-1234})
-	testSuccess([]string{"1", "4", "16", "64", "256"},
-		[]int64{1, 4, 16, 64, 256})
+	testSuccess([]string{"1", "4", "16", "64", "256"}, []int64{1, 4, 16, 64, 256})

-	_, err := StringsToInt64s([]string{"-1", "a", "$"})
+	ints, err := StringsToInt64s([]string{"-1", "a"})
+	assert.Len(t, ints, 0)
 	assert.Error(t, err)
 }