Add build tools for code processing and testing

tools/code-batch-process.go

@@ -0,0 +1,273 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build ignore
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/build/codeformat"
+)
+
+// Windows has a limitation for command line arguments, the size can not exceed 32KB.
+// So we have to feed the files to some tools (like gofmt) batch by batch
+
+// We also introduce a `gitea-fmt` command, it does better import formatting than gofmt/goimports. `gitea-fmt` calls `gofmt` internally.
+
+var optionLogVerbose bool
+
+func logVerbose(msg string, args ...any) {
+	if optionLogVerbose {
+		log.Printf(msg, args...)
+	}
+}
+
+func passThroughCmd(cmd string, args []string) error {
+	foundCmd, err := exec.LookPath(cmd)
+	if err != nil {
+		log.Fatalf("can not find cmd: %s", cmd)
+	}
+	c := exec.Cmd{
+		Path:   foundCmd,
+		Args:   append([]string{cmd}, args...),
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+	return c.Run()
+}
+
+type fileCollector struct {
+	dirs            []string
+	includePatterns []*regexp.Regexp
+	excludePatterns []*regexp.Regexp
+	batchSize       int
+}
+
+func newFileCollector(fileFilter string, batchSize int) (*fileCollector, error) {
+	co := &fileCollector{batchSize: batchSize}
+	if fileFilter == "go-own" {
+		co.dirs = []string{
+			"build",
+			"cmd",
+			"contrib",
+			"tests",
+			"models",
+			"modules",
+			"routers",
+			"services",
+		}
+		co.includePatterns = append(co.includePatterns, regexp.MustCompile(`.*\.go$`))
+
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`.*\bbindata\.go$`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`\.pb\.go$`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/gitea-repositories-meta`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`tests/integration/migration-test`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`modules/git/tests`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/fixtures`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`models/migrations/fixtures`))
+		co.excludePatterns = append(co.excludePatterns, regexp.MustCompile(`services/gitdiff/testdata`))
+	}
+
+	if co.dirs == nil {
+		return nil, fmt.Errorf("unknown file-filter: %s", fileFilter)
+	}
+	return co, nil
+}
+
+func (fc *fileCollector) matchPatterns(path string, regexps []*regexp.Regexp) bool {
+	path = strings.ReplaceAll(path, "\\", "/")
+	for _, re := range regexps {
+		if re.MatchString(path) {
+			return true
+		}
+	}
+	return false
+}
+
+func (fc *fileCollector) collectFiles() (res [][]string, err error) {
+	var batch []string
+	for _, dir := range fc.dirs {
+		err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+			include := len(fc.includePatterns) == 0 || fc.matchPatterns(path, fc.includePatterns)
+			exclude := fc.matchPatterns(path, fc.excludePatterns)
+			process := include && !exclude
+			if !process {
+				if d.IsDir() {
+					if exclude {
+						logVerbose("exclude dir %s", path)
+						return filepath.SkipDir
+					}
+					// for a directory, if it is not excluded explicitly, we should walk into
+					return nil
+				}
+				// for a file, we skip it if it shouldn't be processed
+				logVerbose("skip process %s", path)
+				return nil
+			}
+			if d.IsDir() {
+				// skip dir, we don't add dirs to the file list now
+				return nil
+			}
+			if len(batch) >= fc.batchSize {
+				res = append(res, batch)
+				batch = nil
+			}
+			batch = append(batch, path)
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	res = append(res, batch)
+	return res, nil
+}
+
+// substArgFiles expands the {file-list} to a real file list for commands
+func substArgFiles(args, files []string) []string {
+	for i, s := range args {
+		if s == "{file-list}" {
+			newArgs := append(args[:i], files...)
+			newArgs = append(newArgs, args[i+1:]...)
+			return newArgs
+		}
+	}
+	return args
+}
+
+func exitWithCmdErrors(subCmd string, subArgs []string, cmdErrors []error) {
+	for _, err := range cmdErrors {
+		if err != nil {
+			if exitError, ok := err.(*exec.ExitError); ok {
+				exitCode := exitError.ExitCode()
+				log.Printf("run command failed (code=%d): %s %v", exitCode, subCmd, subArgs)
+				os.Exit(exitCode)
+			} else {
+				log.Fatalf("run command failed (err=%s) %s %v", err, subCmd, subArgs)
+			}
+		}
+	}
+}
+
+func parseArgs() (mainOptions map[string]string, subCmd string, subArgs []string) {
+	mainOptions = map[string]string{}
+	for i := 1; i < len(os.Args); i++ {
+		arg := os.Args[i]
+		if arg == "" {
+			break
+		}
+		if arg[0] == '-' {
+			arg = strings.TrimPrefix(arg, "-")
+			arg = strings.TrimPrefix(arg, "-")
+			fields := strings.SplitN(arg, "=", 2)
+			if len(fields) == 1 {
+				mainOptions[fields[0]] = "1"
+			} else {
+				mainOptions[fields[0]] = fields[1]
+			}
+		} else {
+			subCmd = arg
+			subArgs = os.Args[i+1:]
+			break
+		}
+	}
+	return mainOptions, subCmd, subArgs
+}
+
+func showUsage() {
+	fmt.Printf(`Usage: %[1]s [options] {command} [arguments]
+
+Options:
+  --verbose
+  --file-filter=go-own
+  --batch-size=100
+
+Commands:
+  %[1]s gofmt ...
+
+Arguments:
+  {file-list}     the file list
+
+Example:
+  %[1]s gofmt -s -d {file-list}
+
+`, "file-batch-exec")
+}
+
+func newFileCollectorFromMainOptions(mainOptions map[string]string) (fc *fileCollector, err error) {
+	fileFilter := mainOptions["file-filter"]
+	if fileFilter == "" {
+		fileFilter = "go-own"
+	}
+	batchSize, _ := strconv.Atoi(mainOptions["batch-size"])
+	if batchSize == 0 {
+		batchSize = 100
+	}
+
+	return newFileCollector(fileFilter, batchSize)
+}
+
+func giteaFormatGoImports(files []string, doWriteFile bool) error {
+	for _, file := range files {
+		if err := codeformat.FormatGoImports(file, doWriteFile); err != nil {
+			log.Printf("failed to format go imports: %s, err=%v", file, err)
+			return err
+		}
+	}
+	return nil
+}
+
+func main() {
+	mainOptions, subCmd, subArgs := parseArgs()
+	if subCmd == "" {
+		showUsage()
+		os.Exit(1)
+	}
+	optionLogVerbose = mainOptions["verbose"] != ""
+
+	fc, err := newFileCollectorFromMainOptions(mainOptions)
+	if err != nil {
+		log.Fatalf("can not create file collector: %s", err.Error())
+	}
+
+	fileBatches, err := fc.collectFiles()
+	if err != nil {
+		log.Fatalf("can not collect files: %s", err.Error())
+	}
+
+	processed := 0
+	var cmdErrors []error
+	for _, files := range fileBatches {
+		if len(files) == 0 {
+			break
+		}
+		substArgs := substArgFiles(subArgs, files)
+		logVerbose("batch cmd: %s %v", subCmd, substArgs)
+		switch subCmd {
+		case "gitea-fmt":
+			if slices.Contains(subArgs, "-d") {
+				log.Print("the -d option is not supported by gitea-fmt")
+			}
+			cmdErrors = append(cmdErrors, giteaFormatGoImports(files, slices.Contains(subArgs, "-w")))
+			cmdErrors = append(cmdErrors, passThroughCmd("gofmt", append([]string{"-w", "-r", "interface{} -> any"}, substArgs...)))
+			cmdErrors = append(cmdErrors, passThroughCmd("go", append([]string{"run", os.Getenv("GOFUMPT_PACKAGE"), "-extra"}, substArgs...)))
+		default:
+			log.Fatalf("unknown cmd: %s %v", subCmd, subArgs)
+		}
+		processed += len(files)
+	}
+
+	logVerbose("processed %d files", processed)
+	exitWithCmdErrors(subCmd, subArgs, cmdErrors)
+}

tools/gocovmerge.go

@@ -0,0 +1,118 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Copyright (c) 2015, Wade Simmons
+// SPDX-License-Identifier: MIT
+
+// gocovmerge takes the results from multiple `go test -coverprofile` runs and
+// merges them into one profile
+
+//go:build ignore
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sort"
+
+	"golang.org/x/tools/cover"
+)
+
+func mergeProfiles(p, merge *cover.Profile) {
+	if p.Mode != merge.Mode {
+		log.Fatalf("cannot merge profiles with different modes")
+	}
+	// Since the blocks are sorted, we can keep track of where the last block
+	// was inserted and only look at the blocks after that as targets for merge
+	startIndex := 0
+	for _, b := range merge.Blocks {
+		startIndex = mergeProfileBlock(p, b, startIndex)
+	}
+}
+
+func mergeProfileBlock(p *cover.Profile, pb cover.ProfileBlock, startIndex int) int {
+	sortFunc := func(i int) bool {
+		pi := p.Blocks[i+startIndex]
+		return pi.StartLine >= pb.StartLine && (pi.StartLine != pb.StartLine || pi.StartCol >= pb.StartCol)
+	}
+
+	i := 0
+	if sortFunc(i) != true {
+		i = sort.Search(len(p.Blocks)-startIndex, sortFunc)
+	}
+	i += startIndex
+	if i < len(p.Blocks) && p.Blocks[i].StartLine == pb.StartLine && p.Blocks[i].StartCol == pb.StartCol {
+		if p.Blocks[i].EndLine != pb.EndLine || p.Blocks[i].EndCol != pb.EndCol {
+			log.Fatalf("OVERLAP MERGE: %v %v %v", p.FileName, p.Blocks[i], pb)
+		}
+		switch p.Mode {
+		case "set":
+			p.Blocks[i].Count |= pb.Count
+		case "count", "atomic":
+			p.Blocks[i].Count += pb.Count
+		default:
+			log.Fatalf("unsupported covermode: '%s'", p.Mode)
+		}
+	} else {
+		if i > 0 {
+			pa := p.Blocks[i-1]
+			if pa.EndLine >= pb.EndLine && (pa.EndLine != pb.EndLine || pa.EndCol > pb.EndCol) {
+				log.Fatalf("OVERLAP BEFORE: %v %v %v", p.FileName, pa, pb)
+			}
+		}
+		if i < len(p.Blocks)-1 {
+			pa := p.Blocks[i+1]
+			if pa.StartLine <= pb.StartLine && (pa.StartLine != pb.StartLine || pa.StartCol < pb.StartCol) {
+				log.Fatalf("OVERLAP AFTER: %v %v %v", p.FileName, pa, pb)
+			}
+		}
+		p.Blocks = append(p.Blocks, cover.ProfileBlock{})
+		copy(p.Blocks[i+1:], p.Blocks[i:])
+		p.Blocks[i] = pb
+	}
+	return i + 1
+}
+
+func addProfile(profiles []*cover.Profile, p *cover.Profile) []*cover.Profile {
+	i := sort.Search(len(profiles), func(i int) bool { return profiles[i].FileName >= p.FileName })
+	if i < len(profiles) && profiles[i].FileName == p.FileName {
+		mergeProfiles(profiles[i], p)
+	} else {
+		profiles = append(profiles, nil)
+		copy(profiles[i+1:], profiles[i:])
+		profiles[i] = p
+	}
+	return profiles
+}
+
+func dumpProfiles(profiles []*cover.Profile, out io.Writer) {
+	if len(profiles) == 0 {
+		return
+	}
+	fmt.Fprintf(out, "mode: %s\n", profiles[0].Mode)
+	for _, p := range profiles {
+		for _, b := range p.Blocks {
+			fmt.Fprintf(out, "%s:%d.%d,%d.%d %d %d\n", p.FileName, b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt, b.Count)
+		}
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	var merged []*cover.Profile
+
+	for _, file := range flag.Args() {
+		profiles, err := cover.ParseProfiles(file)
+		if err != nil {
+			log.Fatalf("failed to parse profile '%s': %v", file, err)
+		}
+		for _, p := range profiles {
+			merged = addProfile(merged, p)
+		}
+	}
+
+	dumpProfiles(merged, os.Stdout)
+}

tools/test-echo.go

@@ -0,0 +1,20 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build ignore
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+func main() {
+	_, err := io.Copy(os.Stdout, os.Stdin)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v", err)
+		os.Exit(1)
+	}
+}