// Copyright 2019 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package repository import ( "bufio" "bytes" "context" "errors" "fmt" "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" "sync" "time" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/gitcmd" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/glob" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/huandu/xstrings" ) type transformer struct { Name string Transform func(string) string } type expansion struct { Name string Value string Transformers []transformer } var globalVars = sync.OnceValue(func() (ret struct { defaultTransformers []transformer fileNameSanitizeRegexp *regexp.Regexp }, ) { ret.defaultTransformers = []transformer{ {Name: "SNAKE", Transform: xstrings.ToSnakeCase}, {Name: "KEBAB", Transform: xstrings.ToKebabCase}, {Name: "CAMEL", Transform: xstrings.ToCamelCase}, {Name: "PASCAL", Transform: xstrings.ToPascalCase}, {Name: "LOWER", Transform: strings.ToLower}, {Name: "UPPER", Transform: strings.ToUpper}, {Name: "TITLE", Transform: util.ToTitleCase}, } // invalid filename contents, based on https://github.com/sindresorhus/filename-reserved-regex // "COM10" needs to be opened with UNC "\\.\COM10" on Windows, so itself is valid ret.fileNameSanitizeRegexp = regexp.MustCompile(`(?i)[<>:"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) return ret }) func generateExpansion(ctx context.Context, src string, templateRepo, generateRepo *repo_model.Repository) string { transformers := globalVars().defaultTransformers year, month, day := time.Now().Date() expansions := []expansion{ {Name: "YEAR", Value: strconv.Itoa(year), Transformers: nil}, {Name: "MONTH", Value: fmt.Sprintf("%02d", int(month)), Transformers: nil}, {Name: "MONTH_ENGLISH", Value: month.String(), Transformers: transformers}, {Name: "DAY", Value: fmt.Sprintf("%02d", day), Transformers: nil}, {Name: "REPO_NAME", Value: generateRepo.Name, Transformers: transformers}, {Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: transformers}, {Name: "REPO_DESCRIPTION", Value: generateRepo.Description, Transformers: nil}, {Name: "TEMPLATE_DESCRIPTION", Value: templateRepo.Description, Transformers: nil}, {Name: "REPO_OWNER", Value: generateRepo.OwnerName, Transformers: transformers}, {Name: "TEMPLATE_OWNER", Value: templateRepo.OwnerName, Transformers: transformers}, {Name: "REPO_LINK", Value: generateRepo.Link(), Transformers: nil}, {Name: "TEMPLATE_LINK", Value: templateRepo.Link(), Transformers: nil}, {Name: "REPO_HTTPS_URL", Value: generateRepo.CloneLinkGeneral(ctx).HTTPS, Transformers: nil}, {Name: "TEMPLATE_HTTPS_URL", Value: templateRepo.CloneLinkGeneral(ctx).HTTPS, Transformers: nil}, {Name: "REPO_SSH_URL", Value: generateRepo.CloneLinkGeneral(ctx).SSH, Transformers: nil}, {Name: "TEMPLATE_SSH_URL", Value: templateRepo.CloneLinkGeneral(ctx).SSH, Transformers: nil}, } expansionMap := make(map[string]string) for _, e := range expansions { expansionMap[e.Name] = e.Value for _, tr := range e.Transformers { expansionMap[fmt.Sprintf("%s_%s", e.Name, tr.Name)] = tr.Transform(e.Value) } } return os.Expand(src, func(key string) string { if val, ok := expansionMap[key]; ok { return val } return key }) } // giteaTemplateFileMatcher holds information about a .gitea/template file type giteaTemplateFileMatcher struct { relPath string globs []glob.Glob } func newGiteaTemplateFileMatcher(relPath string, content []byte) *giteaTemplateFileMatcher { gt := &giteaTemplateFileMatcher{relPath: relPath} gt.globs = make([]glob.Glob, 0) scanner := bufio.NewScanner(bytes.NewReader(content)) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } g, err := glob.Compile(line, '/') if err != nil { log.Debug("Invalid glob expression '%s' (skipped): %v", line, err) continue } gt.globs = append(gt.globs, g) } return gt } func (gt *giteaTemplateFileMatcher) HasRules() bool { return len(gt.globs) != 0 } func (gt *giteaTemplateFileMatcher) Match(s string) bool { for _, g := range gt.globs { if g.Match(s) { return true } } return false } func readGiteaTemplateFile(tmpDir string) (*giteaTemplateFileMatcher, error) { templateRelPath := filepath.Join(".gitea", "template") content, err := util.ReadRegularPathFile(tmpDir, templateRelPath, 1024*1024) if err != nil { return nil, util.Iif(errors.Is(err, util.ErrNotRegularPathFile), os.ErrNotExist, err) } return newGiteaTemplateFileMatcher(templateRelPath, content), nil } func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, templateRepo, generateRepo *repo_model.Repository) error { content, err := util.ReadRegularPathFile(tmpDir, tmpDirSubPath, 1024*1024) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } if err := os.Remove(util.FilePathJoinAbs(tmpDir, tmpDirSubPath)); err != nil { return err } generatedContent := generateExpansion(ctx, string(content), templateRepo, generateRepo) substSubPath := filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo)) return util.WriteRegularPathFile(tmpDir, substSubPath, []byte(generatedContent), 0o755, 0o644) } // processGiteaTemplateFile processes and removes the .gitea/template file, does variable expansion for template files // and save the processed files to the filesystem. It returns a list of skipped files that are not regular paths. func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, generateRepo *repo_model.Repository, fileMatcher *giteaTemplateFileMatcher) (skippedFiles []string, _ error) { // Why not use "os.Root" here: symlink is unsafe even in the same root but "os.Root" can't help, it's more difficult to use "os.Root" to do the WalkDir. if err := os.Remove(util.FilePathJoinAbs(tmpDir, fileMatcher.relPath)); err != nil { return nil, fmt.Errorf("unable to remove .gitea/template: %w", err) } if !fileMatcher.HasRules() { return skippedFiles, nil // Avoid walking tree if there are no globs } err := filepath.WalkDir(tmpDir, func(fullPath string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if d.IsDir() { return nil } tmpDirSubPath, err := filepath.Rel(tmpDir, fullPath) if err != nil { return err } if fileMatcher.Match(filepath.ToSlash(tmpDirSubPath)) { err := substGiteaTemplateFile(ctx, tmpDir, tmpDirSubPath, templateRepo, generateRepo) if errors.Is(err, util.ErrNotRegularPathFile) { skippedFiles = append(skippedFiles, tmpDirSubPath) } else if err != nil { return err } } return nil }) // end: WalkDir if err != nil { return nil, err } if err = util.RemoveAll(util.FilePathJoinAbs(tmpDir, ".git")); err != nil { return nil, err } return skippedFiles, nil } func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error { commitTimeStr := time.Now().Format(time.RFC3339) authorSig := repo.Owner.NewGitSig() // Because this may call hooks we should pass in the environment env := append(os.Environ(), "GIT_AUTHOR_NAME="+authorSig.Name, "GIT_AUTHOR_EMAIL="+authorSig.Email, "GIT_AUTHOR_DATE="+commitTimeStr, "GIT_COMMITTER_NAME="+authorSig.Name, "GIT_COMMITTER_EMAIL="+authorSig.Email, "GIT_COMMITTER_DATE="+commitTimeStr, ) // Clone to temporary path and do the init commit. templateRepoPath := templateRepo.RepoPath() if err := git.Clone(ctx, templateRepoPath, tmpDir, git.CloneRepoOptions{ Depth: 1, Branch: templateRepo.DefaultBranch, }); err != nil { return fmt.Errorf("git clone: %w", err) } // Get active submodules from the template submodules, err := git.GetTemplateSubmoduleCommits(ctx, tmpDir) if err != nil { return fmt.Errorf("GetTemplateSubmoduleCommits: %w", err) } if err = util.RemoveAll(filepath.Join(tmpDir, ".git")); err != nil { return fmt.Errorf("remove git dir: %w", err) } // Variable expansion fileMatcher, err := readGiteaTemplateFile(tmpDir) if err == nil { _, err = processGiteaTemplateFile(ctx, tmpDir, templateRepo, generateRepo, fileMatcher) if err != nil { return fmt.Errorf("processGiteaTemplateFile: %w", err) } } else if errors.Is(err, fs.ErrNotExist) { log.Debug("skip processing repo template files: no available .gitea/template") } else { return fmt.Errorf("readGiteaTemplateFile: %w", err) } if err = git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil { return err } if stdout, _, err := gitcmd.NewCommand("remote", "add", "origin").AddDynamicArguments(repo.RepoPath()). RunStdString(ctx, &gitcmd.RunOpts{Dir: tmpDir, Env: env}); err != nil { log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err) return fmt.Errorf("git remote add: %w", err) } if err = git.AddTemplateSubmoduleIndexes(ctx, tmpDir, submodules); err != nil { return fmt.Errorf("failed to add submodules: %v", err) } // set default branch based on whether it's specified in the newly generated repo or not defaultBranch := repo.DefaultBranch if strings.TrimSpace(defaultBranch) == "" { defaultBranch = templateRepo.DefaultBranch } return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch) } // GenerateGitContent generates git content from a template repository func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) (err error) { tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-" + generateRepo.Name) if err != nil { return fmt.Errorf("failed to create temp dir for repository %s: %w", generateRepo.FullName(), err) } defer cleanup() if err = generateRepoCommit(ctx, generateRepo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %w", err) } // re-fetch repo if generateRepo, err = repo_model.GetRepositoryByID(ctx, generateRepo.ID); err != nil { return fmt.Errorf("getRepositoryByID: %w", err) } // if there was no default branch supplied when generating the repo, use the default one from the template if strings.TrimSpace(generateRepo.DefaultBranch) == "" { generateRepo.DefaultBranch = templateRepo.DefaultBranch } if err = gitrepo.SetDefaultBranch(ctx, generateRepo, generateRepo.DefaultBranch); err != nil { return fmt.Errorf("setDefaultBranch: %w", err) } if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "default_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil { return fmt.Errorf("failed to update size for repository: %w", err) } if err := git_model.CopyLFS(ctx, generateRepo, templateRepo); err != nil { return fmt.Errorf("failed to copy LFS: %w", err) } return nil } // GenerateRepoOptions contains the template units to generate type GenerateRepoOptions struct { Name string DefaultBranch string Description string Private bool GitContent bool Topics bool GitHooks bool Webhooks bool Avatar bool IssueLabels bool ProtectedBranch bool } // IsValid checks whether at least one option is chosen for generation func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels || gro.ProtectedBranch // or other items as they are added } func filePathSanitize(s string) string { fields := strings.Split(filepath.ToSlash(s), "/") for i, field := range fields { field = strings.TrimSpace(strings.TrimSpace(globalVars().fileNameSanitizeRegexp.ReplaceAllString(field, "_"))) if strings.HasPrefix(field, "..") { field = "__" + field[2:] } if strings.EqualFold(field, ".git") { field = "_" + field[1:] } fields[i] = field } return filepath.Clean(filepath.FromSlash(strings.Trim(strings.Join(fields, "/"), "/"))) }