mirror of
https://github.com/go-gitea/gitea.git
synced 2026-06-18 20:10:04 +02:00
refactor(actions): defer dynamic matrix expansion to jobparser
Expand a needs-dependent matrix through the new jobparser.ExpandMatrixWithNeeds, which evaluates the raw matrix against the needs context via NewInterpeter the same way EvaluateConcurrency does. This replaces synthesizing a workflow with stub jobs and re-parsing it once the dependencies complete. Parse now emits a single placeholder per needs-dependent matrix and the whole matrix expands exactly once when the needs finish, instead of being split per static value at planning time and re-expanded per placeholder. This removes ExtractRawStrategies, HasMatrixWithNeeds and constructWorkflowWithNeeds. Side effects of deferring the whole matrix: a static+dynamic matrix no longer duplicates N times, needs-output detection no longer relies on string matching (so bracket notation is handled), and a deferred placeholder gets no commit status until it expands, avoiding an orphaned status from the name change. Co-Authored-By: Claude (Opus 4.8) <noreply@anthropic.com>
This commit is contained in:
silverwind
parent
2a33c56911
commit
d35b3e65d6
@ -92,7 +92,7 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
||||
}
|
||||
|
||||
// Clone + pre-evaluate only when the matrix has a ${{ }} expression; static matrices use
|
||||
// the origin job as-is. Unresolved expressions defer to ReEvaluateMatrixForJobWithNeeds.
|
||||
// the origin job as-is.
|
||||
evaluatedJob := originJob
|
||||
if originJob.Strategy != nil && rawMatrixHasExpression(&originJob.Strategy.RawMatrix) {
|
||||
jobCopy := *originJob
|
||||
@ -102,7 +102,21 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
||||
evaluatedJob = &jobCopy
|
||||
matrixEvaluator := NewExpressionEvaluator(NewInterpeter(id, evaluatedJob, nil, pc.gitContext, results, pc.vars, pc.inputs))
|
||||
if err := matrixEvaluator.EvaluateYamlNode(&evaluatedJob.Strategy.RawMatrix); err != nil {
|
||||
log.Debug("matrix evaluation deferred for job %s (unresolved expression): %v", id, err)
|
||||
// Matrix references needs.*.outputs.* (unavailable now). Emit one placeholder
|
||||
// keeping the raw strategy; the server expands it once the needs finish.
|
||||
if len(originJob.Needs()) > 0 {
|
||||
placeholder := job.Clone()
|
||||
if placeholder.Name == "" {
|
||||
placeholder.Name = id
|
||||
}
|
||||
swf := newSingleWorkflow(workflow)
|
||||
if err := swf.SetJob(id, placeholder); err != nil {
|
||||
return nil, fmt.Errorf("SetJob: %w", err)
|
||||
}
|
||||
ret = append(ret, swf)
|
||||
continue
|
||||
}
|
||||
log.Debug("matrix evaluation for job %s left unresolved (no needs): %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -111,27 +125,8 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
||||
return nil, fmt.Errorf("getMatrixes: %w", err)
|
||||
}
|
||||
for _, matrix := range matricxes {
|
||||
job := job.Clone()
|
||||
if job.Name == "" {
|
||||
job.Name = id
|
||||
}
|
||||
job.Strategy.RawMatrix = encodeMatrix(matrix)
|
||||
evaluator := NewExpressionEvaluator(NewInterpeter(id, evaluatedJob, matrix, pc.gitContext, results, pc.vars, pc.inputs))
|
||||
job.Name = nameWithMatrix(job.Name, matrix, evaluator)
|
||||
runsOn := evaluatedJob.RunsOn()
|
||||
for i, v := range runsOn {
|
||||
runsOn[i] = evaluator.Interpolate(v)
|
||||
}
|
||||
job.RawRunsOn = encodeRunsOn(runsOn)
|
||||
swf := &SingleWorkflow{
|
||||
Name: workflow.Name,
|
||||
RawOn: workflow.RawOn,
|
||||
Env: workflow.Env,
|
||||
Defaults: workflow.Defaults,
|
||||
RawPermissions: workflow.RawPermissions,
|
||||
RunName: workflow.RunName,
|
||||
}
|
||||
if err := swf.SetJob(id, job); err != nil {
|
||||
swf := newSingleWorkflow(workflow)
|
||||
if err := swf.SetJob(id, expandJobCombo(id, job, matrix, evaluatedJob, pc.gitContext, results, pc.vars, pc.inputs)); err != nil {
|
||||
return nil, fmt.Errorf("SetJob: %w", err)
|
||||
}
|
||||
ret = append(ret, swf)
|
||||
@ -140,6 +135,77 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// newSingleWorkflow returns a SingleWorkflow carrying w's global fields and no job.
|
||||
func newSingleWorkflow(w *SingleWorkflow) *SingleWorkflow {
|
||||
return &SingleWorkflow{
|
||||
Name: w.Name,
|
||||
RawOn: w.RawOn,
|
||||
Env: w.Env,
|
||||
Defaults: w.Defaults,
|
||||
RawPermissions: w.RawPermissions,
|
||||
RunName: w.RunName,
|
||||
}
|
||||
}
|
||||
|
||||
// expandJobCombo builds the Job for one matrix combination: it bakes the matrix into the strategy
|
||||
// and interpolates the name and runs-on. actJob drives the interpreter (it supplies the job's
|
||||
// needs and strategy contexts) and may differ from src (an act model.Job vs the source *Job).
|
||||
func expandJobCombo(jobID string, src *Job, matrix map[string]any, actJob *model.Job, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, inputs map[string]any) *Job {
|
||||
combo := src.Clone()
|
||||
if combo.Name == "" {
|
||||
combo.Name = jobID
|
||||
}
|
||||
combo.Strategy.RawMatrix = encodeMatrix(matrix)
|
||||
evaluator := NewExpressionEvaluator(NewInterpeter(jobID, actJob, matrix, gitCtx, results, vars, inputs))
|
||||
combo.Name = nameWithMatrix(combo.Name, matrix, evaluator)
|
||||
runsOn := combo.RunsOn()
|
||||
for i := range runsOn {
|
||||
runsOn[i] = evaluator.Interpolate(runsOn[i])
|
||||
}
|
||||
combo.RawRunsOn = encodeRunsOn(runsOn)
|
||||
return combo
|
||||
}
|
||||
|
||||
// RawMatrixHasExpression reports whether the job's matrix contains a ${{ }} expression.
|
||||
// With needs present, this marks a matrix whose expansion is deferred until they complete.
|
||||
func RawMatrixHasExpression(job *Job) bool {
|
||||
return rawMatrixHasExpression(&job.Strategy.RawMatrix)
|
||||
}
|
||||
|
||||
// ExpandMatrixWithNeeds expands job's matrix once its needs complete, returning one Job per
|
||||
// combination. Like EvaluateConcurrency it evaluates against the needs context via NewInterpeter,
|
||||
// with no workflow re-parse or stub jobs. job must carry its raw matrix and needs (RawNeeds).
|
||||
func ExpandMatrixWithNeeds(jobID string, job *Job, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, inputs map[string]any) ([]*Job, error) {
|
||||
var rawNeeds yaml.Node
|
||||
if err := rawNeeds.Encode(job.Needs()); err != nil {
|
||||
return nil, fmt.Errorf("encode needs: %w", err)
|
||||
}
|
||||
actJob := &model.Job{
|
||||
RawNeeds: rawNeeds,
|
||||
Strategy: &model.Strategy{
|
||||
FailFastString: job.Strategy.FailFastString,
|
||||
MaxParallelString: job.Strategy.MaxParallelString,
|
||||
RawMatrix: *deepCopyYamlNode(&job.Strategy.RawMatrix),
|
||||
},
|
||||
}
|
||||
|
||||
// Resolve fromJson(needs.*.outputs.*) and friends into concrete matrix values.
|
||||
if err := NewExpressionEvaluator(NewInterpeter(jobID, actJob, nil, gitCtx, results, vars, inputs)).
|
||||
EvaluateYamlNode(&actJob.Strategy.RawMatrix); err != nil {
|
||||
return nil, fmt.Errorf("evaluate matrix: %w", err)
|
||||
}
|
||||
matrixes, err := getMatrixes(actJob)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getMatrixes: %w", err)
|
||||
}
|
||||
|
||||
expanded := make([]*Job, 0, len(matrixes))
|
||||
for _, matrix := range matrixes {
|
||||
expanded = append(expanded, expandJobCombo(jobID, job, matrix, actJob, gitCtx, results, vars, inputs))
|
||||
}
|
||||
return expanded, nil
|
||||
}
|
||||
|
||||
func WithJobResults(results map[string]string) ParseOption {
|
||||
return func(c *parseContext) {
|
||||
c.jobResults = results
|
||||
|
||||
@ -533,3 +533,91 @@ jobs:
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseDefersDynamicMatrix(t *testing.T) {
|
||||
// A matrix referencing needs outputs is emitted as a single placeholder retaining the raw
|
||||
// expression, rather than being expanded or split per static value.
|
||||
workflowYAML := `
|
||||
name: test-defer
|
||||
on: push
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: echo setup
|
||||
|
||||
build:
|
||||
needs: setup
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
version: ${{ fromJson(needs.setup.outputs.versions) }}
|
||||
steps:
|
||||
- run: echo build
|
||||
`
|
||||
result, err := Parse([]byte(workflowYAML))
|
||||
require.NoError(t, err)
|
||||
|
||||
var buildJobs []*Job
|
||||
for _, w := range result {
|
||||
if id, j := w.Job(); id == "build" {
|
||||
buildJobs = append(buildJobs, j)
|
||||
}
|
||||
}
|
||||
require.Len(t, buildJobs, 1, "deferred matrix must yield exactly one placeholder")
|
||||
assert.True(t, RawMatrixHasExpression(buildJobs[0]), "placeholder must keep the raw matrix expression")
|
||||
}
|
||||
|
||||
func TestExpandMatrixWithNeeds(t *testing.T) {
|
||||
buildJob := func(t *testing.T, matrixYAML, runsOn string, needs []string) *Job {
|
||||
t.Helper()
|
||||
var strat Strategy
|
||||
require.NoError(t, yaml.Unmarshal([]byte(matrixYAML), &strat))
|
||||
job := &Job{Name: "build", Strategy: strat}
|
||||
require.NoError(t, job.RawRunsOn.Encode(runsOn))
|
||||
require.NoError(t, job.RawNeeds.Encode(needs))
|
||||
return job
|
||||
}
|
||||
|
||||
results := map[string]*JobResult{
|
||||
"setup": {Result: "success", Outputs: map[string]string{
|
||||
"versions": "[1.20, 1.21]",
|
||||
"platforms": `["linux", "darwin"]`,
|
||||
}},
|
||||
}
|
||||
|
||||
t.Run("single dynamic dimension", func(t *testing.T) {
|
||||
job := buildJob(t, "matrix:\n version: ${{ fromJson(needs.setup.outputs.versions) }}\n", "ubuntu-latest", []string{"setup"})
|
||||
got, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, results, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, 2)
|
||||
})
|
||||
|
||||
t.Run("static and dynamic expand to product once", func(t *testing.T) {
|
||||
// 2 static os * 2 dynamic versions = 4, expanded once (regression: not split then re-expanded).
|
||||
job := buildJob(t, "matrix:\n os: [ubuntu-latest, windows-latest]\n version: ${{ fromJson(needs.setup.outputs.versions) }}\n", "${{ matrix.os }}", []string{"setup"})
|
||||
got, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, results, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, 4)
|
||||
for _, combo := range got {
|
||||
runsOn := combo.RunsOn()
|
||||
require.Len(t, runsOn, 1)
|
||||
assert.Contains(t, []string{"ubuntu-latest", "windows-latest"}, runsOn[0], "runs-on must be interpolated from matrix.os")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple dynamic dimensions", func(t *testing.T) {
|
||||
job := buildJob(t, "matrix:\n version: ${{ fromJson(needs.setup.outputs.versions) }}\n platform: ${{ fromJson(needs.setup.outputs.platforms) }}\n", "ubuntu-latest", []string{"setup"})
|
||||
got, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, results, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, 4)
|
||||
})
|
||||
|
||||
t.Run("unresolved needs output errors", func(t *testing.T) {
|
||||
job := buildJob(t, "matrix:\n version: ${{ fromJson(needs.missing.outputs.versions) }}\n", "ubuntu-latest", []string{"missing"})
|
||||
_, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, map[string]*JobResult{}, nil, nil)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
@ -46,6 +46,11 @@ func CreateCommitStatusForRunJobs(ctx context.Context, run *actions_model.Action
|
||||
}
|
||||
|
||||
for _, job := range jobs {
|
||||
// A deferred-matrix placeholder's name changes when it expands, so a status created now
|
||||
// would be orphaned; the expanded combos get theirs at expansion time.
|
||||
if job.RawStrategy != "" && !job.IsMatrixEvaluated {
|
||||
continue
|
||||
}
|
||||
if err = createCommitStatus(ctx, run.Repo, event, commitID, run, job); err != nil {
|
||||
log.Error("Failed to create commit status for job %d: %v", job.ID, err)
|
||||
}
|
||||
|
||||
@ -7,10 +7,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
actions_model "gitea.dev/models/actions"
|
||||
"gitea.dev/models/db"
|
||||
@ -60,128 +56,9 @@ func checkTaskNeedsReady(ctx context.Context, job *actions_model.ActionRunJob) (
|
||||
return taskNeeds, true, nil
|
||||
}
|
||||
|
||||
// ExtractRawStrategies extracts strategy definitions from the raw workflow content.
|
||||
// Returns a map of jobID to strategy YAML for jobs that have matrix dependencies.
|
||||
func ExtractRawStrategies(content []byte) (map[string]string, error) {
|
||||
var workflowDef struct {
|
||||
Jobs map[string]struct {
|
||||
Strategy any `yaml:"strategy"`
|
||||
Needs any `yaml:"needs"`
|
||||
} `yaml:"jobs"`
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(content, &workflowDef); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
strategies := make(map[string]string)
|
||||
for jobID, jobDef := range workflowDef.Jobs {
|
||||
if jobDef.Strategy == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var needsList []string
|
||||
switch needs := jobDef.Needs.(type) {
|
||||
case string:
|
||||
needsList = append(needsList, needs)
|
||||
case []any:
|
||||
for _, need := range needs {
|
||||
if needStr, ok := need.(string); ok {
|
||||
needsList = append(needsList, needStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(needsList) > 0 {
|
||||
if strategyBytes, err := yaml.Marshal(jobDef.Strategy); err == nil {
|
||||
strategies[jobID] = string(strategyBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return strategies, nil
|
||||
}
|
||||
|
||||
// HasMatrixWithNeeds reports whether rawStrategy contains a matrix value whose
|
||||
// expression tree references needs.<id>.outputs.<key>.
|
||||
// It walks the parsed YAML tree to avoid false positives from values such as
|
||||
// "os: [needs.review-runner]" that merely contain the substring "needs.".
|
||||
func HasMatrixWithNeeds(rawStrategy string) bool {
|
||||
if rawStrategy == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal([]byte(rawStrategy), &root); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// The top-level document node wraps a single mapping node.
|
||||
doc := &root
|
||||
if doc.Kind == yaml.DocumentNode && len(doc.Content) == 1 {
|
||||
doc = doc.Content[0]
|
||||
}
|
||||
|
||||
// Find the "matrix" key inside the strategy mapping.
|
||||
var matrixNode *yaml.Node
|
||||
if doc.Kind == yaml.MappingNode {
|
||||
for i := 0; i+1 < len(doc.Content); i += 2 {
|
||||
if doc.Content[i].Value == "matrix" {
|
||||
matrixNode = doc.Content[i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if matrixNode == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return yamlNodeContainsNeedsOutputsExpr(matrixNode)
|
||||
}
|
||||
|
||||
// yamlNodeContainsNeedsOutputsExpr recursively inspects a yaml.Node and
|
||||
// returns true if any scalar value contains a GitHub Actions expression of
|
||||
// the form ${{ ... needs.<id>.outputs.<key> ... }}.
|
||||
func yamlNodeContainsNeedsOutputsExpr(node *yaml.Node) bool {
|
||||
if node == nil {
|
||||
return false
|
||||
}
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
return containsNeedsOutputsExpr(node.Value)
|
||||
}
|
||||
return slices.ContainsFunc(node.Content, yamlNodeContainsNeedsOutputsExpr)
|
||||
}
|
||||
|
||||
// containsNeedsOutputsExpr returns true when s contains an Actions expression
|
||||
// (${{ ... }}) that references needs.<id>.outputs.<key>.
|
||||
// A bare "needs." substring outside an expression block is not a match.
|
||||
func containsNeedsOutputsExpr(s string) bool {
|
||||
if !strings.Contains(s, "${{") {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(s); {
|
||||
start := strings.Index(s[i:], "${{")
|
||||
if start == -1 {
|
||||
break
|
||||
}
|
||||
start += i
|
||||
end := strings.Index(s[start:], "}}")
|
||||
if end == -1 {
|
||||
break
|
||||
}
|
||||
end += start
|
||||
expr := s[start : end+2]
|
||||
if strings.Contains(expr, "needs.") && strings.Contains(expr, ".outputs.") {
|
||||
return true
|
||||
}
|
||||
i = end + 2
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ReEvaluateMatrixForJobWithNeeds re-evaluates the matrix strategy of a job once all its
|
||||
// dependent jobs are done. It expands the matrix using job outputs and inserts the resulting
|
||||
// ActionRunJobs. The original placeholder job is marked as evaluated and skipped.
|
||||
// ReEvaluateMatrixForJobWithNeeds expands the matrix strategy of a job once all its dependent
|
||||
// jobs are done, using their outputs, and inserts the resulting ActionRunJobs. The original
|
||||
// placeholder job is reused as the first combination.
|
||||
// Returns nil, nil if the job is not ready yet or has nothing to do.
|
||||
func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.ActionRunJob, vars map[string]string) ([]*actions_model.ActionRunJob, error) {
|
||||
if job.IsMatrixEvaluated || job.RawStrategy == "" {
|
||||
@ -191,8 +68,8 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
||||
log.Debug("Starting matrix re-evaluation for job %d (JobID: %s)", job.ID, job.JobID)
|
||||
|
||||
// skipWithError marks the job as evaluated+skipped and wraps any secondary error.
|
||||
skipWithError := func(reason string, origErr error) ([]*actions_model.ActionRunJob, error) {
|
||||
if markErr := markMatrixAsEvaluatedAndSkip(ctx, job, reason); markErr != nil {
|
||||
skipWithError := func(origErr error) ([]*actions_model.ActionRunJob, error) {
|
||||
if markErr := markMatrixAsEvaluatedAndSkip(ctx, job, origErr.Error()); markErr != nil {
|
||||
return nil, fmt.Errorf("%w; additionally failed to mark as evaluated: %v", origErr, markErr)
|
||||
}
|
||||
return nil, origErr
|
||||
@ -201,7 +78,7 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
||||
taskNeeds, allDone, err := checkTaskNeedsReady(ctx, job)
|
||||
if err != nil {
|
||||
log.Error("Matrix re-evaluation error for job %d: check task needs: %v", job.ID, err)
|
||||
return skipWithError(fmt.Sprintf("task needs check failed: %v", err), fmt.Errorf("check task needs: %w", err))
|
||||
return skipWithError(fmt.Errorf("check task needs: %w", err))
|
||||
}
|
||||
if !allDone {
|
||||
return nil, nil
|
||||
@ -221,30 +98,36 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
||||
|
||||
giteaCtx := GenerateGiteaContext(ctx, job.Run, nil, job)
|
||||
|
||||
jobOutputs := make(map[string]map[string]string, len(taskNeeds))
|
||||
jobResults := make(map[string]string, len(taskNeeds))
|
||||
for jobID, need := range taskNeeds {
|
||||
jobOutputs[jobID] = need.Outputs
|
||||
jobResults[jobID] = need.Result.String()
|
||||
results := make(map[string]*jobparser.JobResult, len(taskNeeds))
|
||||
for needID, need := range taskNeeds {
|
||||
results[needID] = &jobparser.JobResult{Result: need.Result.String(), Outputs: need.Outputs}
|
||||
}
|
||||
|
||||
workflowYAML, err := constructWorkflowWithNeeds(job, taskNeeds)
|
||||
// Rebuild the job from its own payload + stored raw strategy and re-attach needs (erased from
|
||||
// the payload) so needs.*.outputs.* resolves. No synthetic workflow, no stub jobs, no re-parse.
|
||||
var baseSWF jobparser.SingleWorkflow
|
||||
if err := yaml.Unmarshal(job.WorkflowPayload, &baseSWF); err != nil {
|
||||
return skipWithError(fmt.Errorf("unmarshal payload: %w", err))
|
||||
}
|
||||
_, parsedJob := baseSWF.Job()
|
||||
if parsedJob == nil {
|
||||
return skipWithError(errors.New("payload contains no job"))
|
||||
}
|
||||
var rawStrategy jobparser.Strategy
|
||||
if err := yaml.Unmarshal([]byte(job.RawStrategy), &rawStrategy); err != nil {
|
||||
return skipWithError(fmt.Errorf("unmarshal raw strategy: %w", err))
|
||||
}
|
||||
parsedJob.Strategy = rawStrategy
|
||||
if err := parsedJob.RawNeeds.Encode(job.Needs); err != nil {
|
||||
return skipWithError(fmt.Errorf("encode needs: %w", err))
|
||||
}
|
||||
|
||||
expandedJobs, err := jobparser.ExpandMatrixWithNeeds(job.JobID, parsedJob, giteaCtx.ToGitHubContext(), results, vars, nil)
|
||||
if err != nil {
|
||||
return skipWithError(fmt.Sprintf("workflow construction failed: %v", err), fmt.Errorf("construct workflow: %w", err))
|
||||
return nil, markMatrixAsEvaluatedAndSkip(ctx, job, fmt.Sprintf("matrix expansion failed: %v", err))
|
||||
}
|
||||
|
||||
parsedJobs, err := jobparser.Parse(
|
||||
workflowYAML,
|
||||
jobparser.WithVars(vars),
|
||||
jobparser.WithGitContext(giteaCtx.ToGitHubContext()),
|
||||
jobparser.WithJobOutputs(jobOutputs),
|
||||
jobparser.WithJobResults(jobResults),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, markMatrixAsEvaluatedAndSkip(ctx, job, fmt.Sprintf("parse failed: %v", err))
|
||||
}
|
||||
|
||||
// One parsed workflow per combination; needs are kept on the model and erased from the
|
||||
// One workflow payload per combination; needs are kept on the model and erased from the
|
||||
// payload, as at initial planning time.
|
||||
type matrixCombo struct {
|
||||
name string
|
||||
@ -252,18 +135,15 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
||||
runsOn []string
|
||||
needs []string
|
||||
}
|
||||
var combos []matrixCombo
|
||||
for _, sw := range parsedJobs {
|
||||
id, jobDef := sw.Job()
|
||||
if jobDef == nil || id != job.JobID {
|
||||
continue
|
||||
combos := make([]matrixCombo, 0, len(expandedJobs))
|
||||
for _, expanded := range expandedJobs {
|
||||
combo := matrixCombo{name: expanded.Name, runsOn: expanded.RunsOn(), needs: expanded.Needs()}
|
||||
swf := baseSWF
|
||||
if err := swf.SetJob(job.JobID, expanded.EraseNeeds()); err != nil {
|
||||
return nil, fmt.Errorf("set expanded job %s: %w", job.JobID, err)
|
||||
}
|
||||
combo := matrixCombo{name: jobDef.Name, runsOn: jobDef.RunsOn(), needs: jobDef.Needs()}
|
||||
if err := sw.SetJob(id, jobDef.EraseNeeds()); err != nil {
|
||||
return nil, fmt.Errorf("erase needs for job %s: %w", id, err)
|
||||
}
|
||||
if combo.payload, err = sw.Marshal(); err != nil {
|
||||
return nil, fmt.Errorf("marshal expanded job %s: %w", id, err)
|
||||
if combo.payload, err = swf.Marshal(); err != nil {
|
||||
return nil, fmt.Errorf("marshal expanded job %s: %w", job.JobID, err)
|
||||
}
|
||||
combos = append(combos, combo)
|
||||
}
|
||||
@ -323,65 +203,3 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
||||
|
||||
return children, nil
|
||||
}
|
||||
|
||||
// constructWorkflowWithNeeds creates a workflow YAML that includes the target job
|
||||
// and stub definitions for its dependencies so the jobparser can resolve needs.*.outputs expressions.
|
||||
func constructWorkflowWithNeeds(job *actions_model.ActionRunJob, taskNeeds map[string]*TaskNeed) ([]byte, error) {
|
||||
var jobWorkflow map[string]any
|
||||
if err := yaml.Unmarshal(job.WorkflowPayload, &jobWorkflow); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal job workflow: %w", err)
|
||||
}
|
||||
|
||||
jobsSection, ok := jobWorkflow["jobs"].(map[string]any)
|
||||
if !ok {
|
||||
return nil, errors.New("invalid jobs section in workflow")
|
||||
}
|
||||
|
||||
newJobs := make(map[string]any)
|
||||
|
||||
for needJobID, taskNeed := range taskNeeds {
|
||||
stubJob := map[string]any{
|
||||
"runs-on": "ubuntu-latest",
|
||||
"outputs": taskNeed.Outputs,
|
||||
"steps": []any{},
|
||||
}
|
||||
newJobs[needJobID] = stubJob
|
||||
}
|
||||
|
||||
maps.Copy(newJobs, jobsSection)
|
||||
|
||||
// The WorkflowPayload may contain a normalised/wrapped matrix (e.g.
|
||||
// version: ["${{ fromJson(...) }}"]). Restore the original scalar expression
|
||||
// from RawStrategy so jobparser.Parse() can expand it correctly with job outputs.
|
||||
// Also drop the pre-baked "name" so jobparser regenerates it per matrix combination
|
||||
// (e.g. "build (1)", "build (2)", …) instead of "build (Array)".
|
||||
// Critically, re-add "needs" because EraseNeeds() removed them from WorkflowPayload:
|
||||
// without needs, NewInterpeter builds an empty Needs context and
|
||||
// "needs.generate.outputs.*" expressions can never be evaluated.
|
||||
if targetJobDef, ok := newJobs[job.JobID]; ok {
|
||||
if targetJobMap, ok := targetJobDef.(map[string]any); ok {
|
||||
delete(targetJobMap, "name")
|
||||
needsKeys := make([]string, 0, len(taskNeeds))
|
||||
for needJobID := range taskNeeds {
|
||||
needsKeys = append(needsKeys, needJobID)
|
||||
}
|
||||
sort.Strings(needsKeys)
|
||||
targetJobMap["needs"] = needsKeys
|
||||
if job.RawStrategy != "" {
|
||||
var rawStrategyMap map[string]any
|
||||
if err := yaml.Unmarshal([]byte(job.RawStrategy), &rawStrategyMap); err == nil {
|
||||
targetJobMap["strategy"] = rawStrategyMap
|
||||
}
|
||||
}
|
||||
newJobs[job.JobID] = targetJobMap
|
||||
}
|
||||
}
|
||||
|
||||
workflow := map[string]any{
|
||||
"name": "matrix-expansion",
|
||||
"on": "push",
|
||||
"jobs": newJobs,
|
||||
}
|
||||
|
||||
return yaml.Marshal(workflow)
|
||||
}
|
||||
|
||||
@ -1,171 +0,0 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package actions
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
actions_model "gitea.dev/models/actions"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestHasMatrixWithNeeds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
strategy string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "dynamic matrix referencing job output",
|
||||
strategy: `
|
||||
matrix:
|
||||
version: ${{ fromJson(needs.generate.outputs.matrix) }}
|
||||
`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "static matrix — no expression",
|
||||
strategy: `
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "value contains needs. but not inside expression",
|
||||
strategy: `
|
||||
matrix:
|
||||
os: [needs.review-runner]
|
||||
`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "needs. outside expression block",
|
||||
strategy: `
|
||||
matrix:
|
||||
runner: needs.something-but-no-braces
|
||||
`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "expression with needs but no .outputs.",
|
||||
strategy: `
|
||||
matrix:
|
||||
version: ${{ needs.job1 }}
|
||||
`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty strategy",
|
||||
strategy: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "strategy without matrix key",
|
||||
strategy: `
|
||||
fail-fast: false
|
||||
`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "multi-dimension dynamic matrix",
|
||||
strategy: `
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
version: ${{ fromJson(needs.setup.outputs.versions) }}
|
||||
`,
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, HasMatrixWithNeeds(tt.strategy))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConstructWorkflowWithNeeds(t *testing.T) {
|
||||
// Minimal WorkflowPayload with a strategy referencing a needs output.
|
||||
payload, err := yaml.Marshal(map[string]any{
|
||||
"jobs": map[string]any{
|
||||
"build": map[string]any{
|
||||
"runs-on": "ubuntu-latest",
|
||||
"strategy": map[string]any{
|
||||
"matrix": map[string]any{
|
||||
"version": `${{ fromJson(needs.setup.outputs.versions) }}`,
|
||||
},
|
||||
},
|
||||
"steps": []any{},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
job := &actions_model.ActionRunJob{
|
||||
JobID: "build",
|
||||
WorkflowPayload: payload,
|
||||
RawStrategy: `
|
||||
matrix:
|
||||
version: ${{ fromJson(needs.setup.outputs.versions) }}
|
||||
`,
|
||||
Needs: []string{"setup"},
|
||||
}
|
||||
needs := map[string]*TaskNeed{
|
||||
"setup": {
|
||||
Result: actions_model.StatusSuccess,
|
||||
Outputs: map[string]string{"versions": `["1.20","1.21"]`},
|
||||
},
|
||||
}
|
||||
|
||||
out, err := constructWorkflowWithNeeds(job, needs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// The resulting YAML should contain both the build job and a stub for setup.
|
||||
var wf map[string]any
|
||||
require.NoError(t, yaml.Unmarshal(out, &wf))
|
||||
|
||||
jobs, ok := wf["jobs"].(map[string]any)
|
||||
require.True(t, ok)
|
||||
assert.Contains(t, jobs, "build")
|
||||
assert.Contains(t, jobs, "setup", "stub for needs dependency must be present")
|
||||
|
||||
// build job needs must list the dependency
|
||||
buildJob, ok := jobs["build"].(map[string]any)
|
||||
require.True(t, ok)
|
||||
assert.Contains(t, buildJob, "needs")
|
||||
|
||||
// RawStrategy must be re-injected (not the pre-baked array form)
|
||||
strategy, ok := buildJob["strategy"].(map[string]any)
|
||||
require.True(t, ok)
|
||||
matrix, ok := strategy["matrix"].(map[string]any)
|
||||
require.True(t, ok)
|
||||
versionExpr, ok := matrix["version"].(string)
|
||||
require.True(t, ok)
|
||||
assert.Contains(t, versionExpr, "fromJson")
|
||||
}
|
||||
|
||||
func TestConstructWorkflowWithNeeds_InvalidPayload(t *testing.T) {
|
||||
job := &actions_model.ActionRunJob{
|
||||
JobID: "build",
|
||||
WorkflowPayload: []byte("not: valid: yaml: ["),
|
||||
}
|
||||
_, err := constructWorkflowWithNeeds(job, nil)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestConstructWorkflowWithNeeds_MissingJobsSection(t *testing.T) {
|
||||
payload, err := yaml.Marshal(map[string]any{"name": "test"})
|
||||
require.NoError(t, err)
|
||||
|
||||
job := &actions_model.ActionRunJob{
|
||||
JobID: "build",
|
||||
WorkflowPayload: payload,
|
||||
}
|
||||
_, err = constructWorkflowWithNeeds(job, nil)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@ -10,7 +10,6 @@ import (
|
||||
actions_model "gitea.dev/models/actions"
|
||||
"gitea.dev/models/db"
|
||||
"gitea.dev/modules/actions/jobparser"
|
||||
"gitea.dev/modules/log"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
act_model "gitea.com/gitea/runner/act/model"
|
||||
@ -129,15 +128,10 @@ func InsertRun(ctx context.Context, run *actions_model.ActionRun, content []byte
|
||||
runJobs := make([]*actions_model.ActionRunJob, 0, len(jobs))
|
||||
var hasWaitingJobs bool
|
||||
|
||||
rawStrategies, err := ExtractRawStrategies(content)
|
||||
if err != nil {
|
||||
log.Warn("Failed to extract raw strategies for run %d: %v", run.ID, err)
|
||||
rawStrategies = nil
|
||||
}
|
||||
|
||||
for i, v := range jobs {
|
||||
id, job := v.Job()
|
||||
needs := job.Needs()
|
||||
isDeferredMatrix := len(needs) > 0 && jobparser.RawMatrixHasExpression(job)
|
||||
if err := v.SetJob(id, job.EraseNeeds()); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -167,11 +161,14 @@ func InsertRun(ctx context.Context, run *actions_model.ActionRun, content []byte
|
||||
runJob.TokenPermissions = perms
|
||||
}
|
||||
|
||||
// Store the raw strategy for jobs whose matrix references needs outputs.
|
||||
// ReEvaluateMatrixForJobWithNeeds uses this to re-expand the matrix once
|
||||
// the dependency jobs have completed and their outputs are available.
|
||||
if rawStrategy, ok := rawStrategies[id]; ok && HasMatrixWithNeeds(rawStrategy) {
|
||||
runJob.RawStrategy = rawStrategy
|
||||
// Matrix references needs outputs: jobparser emitted a placeholder. Store the raw
|
||||
// strategy for ReEvaluateMatrixForJobWithNeeds to expand once the needs finish.
|
||||
if isDeferredMatrix {
|
||||
rawStrategy, err := yaml.Marshal(&job.Strategy)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal raw strategy for job %s: %w", id, err)
|
||||
}
|
||||
runJob.RawStrategy = string(rawStrategy)
|
||||
}
|
||||
|
||||
// check job concurrency
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user