Mirrors/gitea
0
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-07-23 17:44:38 +02:00
Code Issues Activity

feat(repo): support file exclusion logic in .gitea/template in template generation (#38064)

Browse Source 
This PR adds support for file exclusion into `.gitea/template`

Docs: https://gitea.com/gitea/docs/pulls/470

### Usage

Template owners add a `.gitea/template` file to their template
repository:

```
# Files to include for variable expansion
*.go
text/*.txt
**/modules/*

# Files/directories to exclude from the generated repo
[exclude]
vendor/*
node_modules/
src/build/
```

Resolves #37202

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Paarth
2026-07-22 20:14:18 +00:00
committed by GitHub
co-authored by wxiaoguang
parent 1d698f12ce
commit fa3780268c
3 changed files with 226 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/test/file.go
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package test
import (
"os"
"path/filepath"
"gitea.dev/modules/util"
)
type FileHelper struct {
baseDir string
}
func NewFileHelper(baseDir string) *FileHelper {
return &FileHelper{baseDir: baseDir}
}
func (h *FileHelper) fullPath(path string) string {
return filepath.Join(h.baseDir, path)
}
func (h *FileHelper) AssertExists(t TestingT, path string, expected bool) {
t.Helper()
_, err := os.Stat(h.fullPath(path))
errIsNotExist := os.IsNotExist(err)
if expected != !errIsNotExist {
t.Fatalf("file %s existence expected %v", path, expected)
}
}
func (h *FileHelper) AssertFileExists(t TestingT, path string, expected *string) {
t.Helper()
if expected == nil {
h.AssertExists(t, path, false)
return
}
h.AssertFileContent(t, path, *expected)
}
func (h *FileHelper) AssertFileContent(t TestingT, path, expected string) {
t.Helper()
data, err := os.ReadFile(h.fullPath(path))
if err != nil {
t.Fatalf("failed to read file %s: %v", path, err)
}
if string(data) != expected {
t.Fatalf("file %s expected %q but got %q", path, expected, string(data))
}
}
func (h *FileHelper) AssertSymLink(t TestingT, path, expected string) {
t.Helper()
link, err := os.Readlink(h.fullPath(path))
if err != nil {
t.Fatalf("failed to read symlink %s: %v", path, err)
}
if link != h.fullPath(expected) {
t.Fatalf("symlink %s expected %q but got %q", path, expected, link)
}
}
func (h *FileHelper) WriteFile(t TestingT, path, content string, optMode ...os.FileMode) {
t.Helper()
err := os.WriteFile(h.fullPath(path), []byte(content), util.OptionalArg(optMode, 0o644))
if err != nil {
t.Fatalf("failed to write file %s: %v", path, err)
}
}
func (h *FileHelper) MkdirAll(t TestingT, path string, optMode ...os.FileMode) {
t.Helper()
err := os.MkdirAll(h.fullPath(path), util.OptionalArg(optMode, 0o755))
if err != nil {
t.Fatalf("failed to mkdir all %s: %v", path, err)
}
}
func (h *FileHelper) Symlink(t TestingT, oldName, newName string) {
t.Helper()
err := os.Symlink(h.fullPath(oldName), h.fullPath(newName))
if err != nil {
t.Fatalf("failed to symlink from %s to %s: %v", oldName, newName, err)
}
}
services/repository/generate.go
+55 -22
View File
@@ -100,37 +100,51 @@ func generateExpansion(ctx context.Context, src string, templateRepo, generateRe
})
}
// giteaTemplateFileMatcher holds information about a .gitea/template file
type giteaTemplateFileMatcher struct {
relPath string
globs []glob.Glob
relPath string
globsExpand []glob.Glob
globsExclude []glob.Glob
}
func newGiteaTemplateFileMatcher(relPath string, content []byte) *giteaTemplateFileMatcher {
gt := &giteaTemplateFileMatcher{relPath: relPath}
gt.globs = make([]glob.Glob, 0)
scanner := bufio.NewScanner(bytes.NewReader(content))
addGlob := func(globs *[]glob.Glob, pattern string) {
g, err := glob.Compile(pattern, '/')
if err != nil {
log.Debug("Invalid gitea template glob expression %q (skipped): %v", pattern, err)
return
}
*globs = append(*globs, g)
}
curGlobs := &gt.globsExpand
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
g, err := glob.Compile(line, '/')
if err != nil {
log.Debug("Invalid glob expression '%s' (skipped): %v", line, err)
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
switch line {
case "[expand]":
curGlobs = &gt.globsExpand
case "[exclude]":
curGlobs = &gt.globsExclude
default:
log.Debug("Invalid gitea template glob section %q", line)
}
continue
}
gt.globs = append(gt.globs, g)
addGlob(curGlobs, line)
}
return gt
}
func (gt *giteaTemplateFileMatcher) HasRules() bool {
return len(gt.globs) != 0
func (gt *giteaTemplateFileMatcher) hasAnyRules() bool {
return len(gt.globsExpand) != 0 || len(gt.globsExclude) != 0
}
func (gt *giteaTemplateFileMatcher) Match(s string) bool {
for _, g := range gt.globs {
func (gt *giteaTemplateFileMatcher) matchRules(globs []glob.Glob, s string) bool {
for _, g := range globs {
if g.Match(s) {
return true
}
@@ -164,14 +178,15 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t
return util.WriteRegularPathFile(tmpDir, substSubPath, []byte(generatedContent), 0o755, 0o644)
}
// processGiteaTemplateFile processes and removes the .gitea/template file, does variable expansion for template files
// and save the processed files to the filesystem. It returns a list of skipped files that are not regular paths.
// processGiteaTemplateFile processes and removes the .gitea/template file,
// does file exclusion and variable expansion for template files, and save the processed files to the filesystem.
// It returns a list of skipped files that are not regular paths.
func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, generateRepo *repo_model.Repository, fileMatcher *giteaTemplateFileMatcher) (skippedFiles []string, _ error) {
// Why not use "os.Root" here: symlink is unsafe even in the same root but "os.Root" can't help, it's more difficult to use "os.Root" to do the WalkDir.
if err := os.Remove(util.FilePathJoinAbs(tmpDir, fileMatcher.relPath)); err != nil {
return nil, fmt.Errorf("unable to remove .gitea/template: %w", err)
}
if !fileMatcher.HasRules() {
if !fileMatcher.hasAnyRules() {
return skippedFiles, nil // Avoid walking tree if there are no globs
}
@@ -179,17 +194,35 @@ func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo,
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
tmpDirSubPath, err := filepath.Rel(tmpDir, fullPath)
relPath, err := filepath.Rel(tmpDir, fullPath)
if err != nil {
return err
}
if fileMatcher.Match(filepath.ToSlash(tmpDirSubPath)) {
err := substGiteaTemplateFile(ctx, tmpDir, tmpDirSubPath, templateRepo, generateRepo)
if relPath == "." {
// WalkDir always visits the root "." first, don't process it (don't "exclude" to remove, or "expand" to subst)
return nil
}
treePath := filepath.ToSlash(relPath)
// try to "exclude" (remove) first
if fileMatcher.matchRules(fileMatcher.globsExclude, treePath) {
isDir := d.IsDir()
// if the target is a symlink, only the symlink is unlinked, so it is safe
if err := os.RemoveAll(fullPath); err != nil {
return err
}
return util.Iif(isDir, filepath.SkipDir, nil)
}
if d.IsDir() {
return nil
}
// try to expand variables for regular files
if fileMatcher.matchRules(fileMatcher.globsExpand, treePath) {
err := substGiteaTemplateFile(ctx, tmpDir, relPath, templateRepo, generateRepo)
if errors.Is(err, util.ErrNotRegularPathFile) {
skippedFiles = append(skippedFiles, tmpDirSubPath)
skippedFiles = append(skippedFiles, relPath)
} else if err != nil {
return err
}
services/repository/generate_test.go
+84 -51
View File
@@ -10,6 +10,8 @@ import (
"testing"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/test"
"gitea.dev/modules/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -27,29 +29,36 @@ text/*.txt
# All files in modules folders
**/modules/*
# Exclude some files
[exclude]
**/modules/*.tmp
`)
gt := newGiteaTemplateFileMatcher("", giteaTemplate)
assert.Len(t, gt.globs, 3)
assert.Len(t, gt.globsExpand, 3)
tt := []struct {
Path string
Match bool
Path string
Expand bool
Exclude bool
}{
{Path: "main.go", Match: true},
{Path: "sub/sub/foo.go", Match: true},
{Path: "main.go", Expand: true},
{Path: "sub/sub/foo.go", Expand: true},
{Path: "a.txt", Match: false},
{Path: "text/a.txt", Match: true},
{Path: "sub/text/a.txt", Match: false},
{Path: "text/a.json", Match: false},
{Path: "a.txt"},
{Path: "text/a.txt", Expand: true},
{Path: "sub/text/a.txt"},
{Path: "text/a.json"},
{Path: "a/b/c/modules/README.md", Match: true},
{Path: "a/b/c/modules/d/README.md", Match: false},
{Path: "a/b/c/modules/README.md", Expand: true},
{Path: "a/b/c/modules/README.md.tmp", Expand: true, Exclude: true},
{Path: "a/b/c/modules/d/README.md"},
}
for _, tc := range tt {
assert.Equal(t, tc.Match, gt.Match(tc.Path), "path: %s", tc.Path)
assert.Equal(t, tc.Expand, gt.matchRules(gt.globsExpand, tc.Path), "should expand: %s", tc.Path)
assert.Equal(t, tc.Exclude, gt.matchRules(gt.globsExclude, tc.Path), "should exclude: %s", tc.Path)
}
}
@@ -76,26 +85,7 @@ func TestFilePathSanitize(t *testing.T) {
func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
tmpDir := filepath.Join(t.TempDir(), "gitea-template-test")
assertFileContent := func(path, expected string) {
data, err := os.ReadFile(filepath.Join(tmpDir, path))
if expected == "" {
assert.ErrorIs(t, err, os.ErrNotExist)
return
}
require.NoError(t, err)
assert.Equal(t, expected, string(data), "file content mismatch for %s", path)
}
assertSymLink := func(path, expected string) {
link, err := os.Readlink(filepath.Join(tmpDir, path))
if expected == "" {
assert.ErrorIs(t, err, os.ErrNotExist)
return
}
require.NoError(t, err)
assert.Equal(t, expected, link, "symlink target mismatch for %s", path)
}
fh := test.NewFileHelper(tmpDir)
require.NoError(t, os.MkdirAll(tmpDir+"/.git", 0o755))
require.NoError(t, os.WriteFile(tmpDir+"/.git/config", []byte("git-config-dummy"), 0o644))
@@ -125,9 +115,10 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
// case-4
assertSubstTemplateName := func(normalContent, toLinkContent, fromLinkContent string) {
assertFileContent("subst-${TEMPLATE_NAME}-normal", normalContent)
assertFileContent("subst-${TEMPLATE_NAME}-to-link", toLinkContent)
assertFileContent("subst-${TEMPLATE_NAME}-from-link", fromLinkContent)
// empty as non-existing
fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-normal", util.Iif(normalContent == "", nil, new(normalContent)))
fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-to-link", util.Iif(toLinkContent == "", nil, new(toLinkContent)))
fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-from-link", util.Iif(fromLinkContent == "", nil, new(fromLinkContent)))
}
// case-5
@@ -155,7 +146,7 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
{
templateRepo := &repo_model.Repository{Name: "TemplateRepoName"}
generatedRepo := &repo_model.Repository{Name: "/../.gIt/name"}
assertFileContent(".git/config", "git-config-dummy")
fh.AssertFileContent(t, ".git/config", "git-config-dummy")
fileMatcher, _ := readGiteaTemplateFile(tmpDir)
skippedFiles, err := processGiteaTemplateFile(t.Context(), tmpDir, templateRepo, generatedRepo, fileMatcher)
require.NoError(t, err)
@@ -167,47 +158,47 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
"subst-${TEMPLATE_NAME}-to-link",
"subst-TemplateRepoName-to-link",
}, skippedFiles)
assertFileContent(".git/config", "")
assertFileContent(".gitea/template", "")
assertFileContent("include/foo/bar/test.txt", "include subdir TemplateRepoName")
fh.AssertExists(t, ".git/config", false)
fh.AssertExists(t, ".gitea/template", false)
fh.AssertFileContent(t, "include/foo/bar/test.txt", "include subdir TemplateRepoName")
}
// the lin target should never be modified, and since it is in a subdirectory, it is not affected by the template either
assertFileContent("sub/link-target", "link target content from ${TEMPLATE_NAME}")
fh.AssertFileContent(t, "sub/link-target", "link target content from ${TEMPLATE_NAME}")
// case-1
{
assertFileContent("no-such", "")
assertFileContent("normal", "normal content")
assertFileContent("template", "template from TemplateRepoName")
fh.AssertExists(t, "no-such", false)
fh.AssertFileContent(t, "normal", "normal content")
fh.AssertFileContent(t, "template", "template from TemplateRepoName")
}
// case-2
{
// symlink with templates should be preserved (not read or write)
assertSymLink("link", tmpDir+"/sub/link-target")
fh.AssertSymLink(t, "link", "sub/link-target")
}
// case-3
{
assertFileContent("subst-${REPO_NAME}", "")
assertFileContent("subst-/__/_gIt/name", "dummy subst repo name")
fh.AssertExists(t, "subst-${REPO_NAME}", false)
fh.AssertFileContent(t, "subst-/__/_gIt/name", "dummy subst repo name")
}
// case-4
{
// the paths with templates should have been removed, subst to a regular file, succeed, the link is preserved
assertSubstTemplateName("", "", "link target content from ${TEMPLATE_NAME}")
assertFileContent("subst-TemplateRepoName-normal", "dummy subst template name normal")
fh.AssertFileContent(t, "subst-TemplateRepoName-normal", "dummy subst template name normal")
// subst to a link, skip, and the target is unchanged
assertSymLink("subst-TemplateRepoName-to-link", tmpDir+"/sub/link-target")
fh.AssertSymLink(t, "subst-TemplateRepoName-to-link", "sub/link-target")
// subst from a link, skip, and the target is unchanged
assertSymLink("subst-${TEMPLATE_NAME}-from-link", tmpDir+"/sub/link-target")
fh.AssertSymLink(t, "subst-${TEMPLATE_NAME}-from-link", "sub/link-target")
}
// case-5
{
assertFileContent("real-dir/real-file", "origin content")
fh.AssertFileContent(t, "real-dir/real-file", "origin content")
}
}
@@ -234,7 +225,49 @@ func TestProcessGiteaTemplateFileRead(t *testing.T) {
require.Equal(t, "test-data-regular", string(content))
fm, err := readGiteaTemplateFile(tmpDir) // regular template file
require.NoError(t, err)
assert.Len(t, fm.globs, 1)
assert.Len(t, fm.globsExpand, 1)
}
func TestProcessGiteaTemplateFileExclusion(t *testing.T) {
tmpDir := t.TempDir()
fh := test.NewFileHelper(tmpDir)
require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, ".gitea"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".gitea", "template"), []byte(`
[expand]
*.md
*.txt
[exclude]
*.log
`), 0o644))
fh.WriteFile(t, "test.md", "from ${TEMPLATE_NAME}")
fh.WriteFile(t, "test.go", "package test")
fh.WriteFile(t, "test.log", "log-content")
fh.MkdirAll(t, "subdir")
fh.WriteFile(t, "subdir/foo", "bar")
fh.Symlink(t, "test.go", "symlink.go")
fh.Symlink(t, "test.log", "symlink.log")
fh.Symlink(t, "subdir", "symlink-dir.log")
templateRepo := &repo_model.Repository{Name: "MyTemplate"}
generatedRepo := &repo_model.Repository{Name: "MyRepo"}
fm, err := readGiteaTemplateFile(tmpDir)
require.NoError(t, err)
require.Len(t, fm.globsExpand, 2)
require.Len(t, fm.globsExclude, 1)
_, err = processGiteaTemplateFile(t.Context(), tmpDir, templateRepo, generatedRepo, fm)
require.NoError(t, err)
fh.AssertFileContent(t, "test.md", "from MyTemplate")
fh.AssertFileContent(t, "test.go", "package test")
fh.AssertExists(t, "test.log", false)
fh.AssertFileContent(t, "subdir/foo", "bar")
fh.AssertSymLink(t, "symlink.go", "test.go")
fh.AssertExists(t, "symlink.log", false)
fh.AssertExists(t, "symlink-dir.log", false)
}
func TestTransformers(t *testing.T) {