mirror of
https://github.com/go-gitea/gitea.git
synced 2026-10-06 03:52:36 +02:00
refactor(actions): defer dynamic matrix expansion to jobparser
Expand a needs-dependent matrix through the new jobparser.ExpandMatrixWithNeeds, which evaluates the raw matrix against the needs context via NewInterpeter the same way EvaluateConcurrency does. This replaces synthesizing a workflow with stub jobs and re-parsing it once the dependencies complete. Parse now emits a single placeholder per needs-dependent matrix and the whole matrix expands exactly once when the needs finish, instead of being split per static value at planning time and re-expanded per placeholder. This removes ExtractRawStrategies, HasMatrixWithNeeds and constructWorkflowWithNeeds. Side effects of deferring the whole matrix: a static+dynamic matrix no longer duplicates N times, needs-output detection no longer relies on string matching (so bracket notation is handled), and a deferred placeholder gets no commit status until it expands, avoiding an orphaned status from the name change. Co-Authored-By: Claude (Opus 4.8) <noreply@anthropic.com>
This commit is contained in:
6 files changed
+230
-427
No files matched your search
@@ -92,7 +92,7 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Clone + pre-evaluate only when the matrix has a ${{ }} expression; static matrices use
|
// Clone + pre-evaluate only when the matrix has a ${{ }} expression; static matrices use
|
||||||
// the origin job as-is. Unresolved expressions defer to ReEvaluateMatrixForJobWithNeeds.
|
// the origin job as-is.
|
||||||
evaluatedJob := originJob
|
evaluatedJob := originJob
|
||||||
if originJob.Strategy != nil && rawMatrixHasExpression(&originJob.Strategy.RawMatrix) {
|
if originJob.Strategy != nil && rawMatrixHasExpression(&originJob.Strategy.RawMatrix) {
|
||||||
jobCopy := *originJob
|
jobCopy := *originJob
|
||||||
@@ -102,7 +102,21 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
|||||||
evaluatedJob = &jobCopy
|
evaluatedJob = &jobCopy
|
||||||
matrixEvaluator := NewExpressionEvaluator(NewInterpeter(id, evaluatedJob, nil, pc.gitContext, results, pc.vars, pc.inputs))
|
matrixEvaluator := NewExpressionEvaluator(NewInterpeter(id, evaluatedJob, nil, pc.gitContext, results, pc.vars, pc.inputs))
|
||||||
if err := matrixEvaluator.EvaluateYamlNode(&evaluatedJob.Strategy.RawMatrix); err != nil {
|
if err := matrixEvaluator.EvaluateYamlNode(&evaluatedJob.Strategy.RawMatrix); err != nil {
|
||||||
log.Debug("matrix evaluation deferred for job %s (unresolved expression): %v", id, err)
|
// Matrix references needs.*.outputs.* (unavailable now). Emit one placeholder
|
||||||
|
// keeping the raw strategy; the server expands it once the needs finish.
|
||||||
|
if len(originJob.Needs()) > 0 {
|
||||||
|
placeholder := job.Clone()
|
||||||
|
if placeholder.Name == "" {
|
||||||
|
placeholder.Name = id
|
||||||
|
}
|
||||||
|
swf := newSingleWorkflow(workflow)
|
||||||
|
if err := swf.SetJob(id, placeholder); err != nil {
|
||||||
|
return nil, fmt.Errorf("SetJob: %w", err)
|
||||||
|
}
|
||||||
|
ret = append(ret, swf)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Debug("matrix evaluation for job %s left unresolved (no needs): %v", id, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,27 +125,8 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
|||||||
return nil, fmt.Errorf("getMatrixes: %w", err)
|
return nil, fmt.Errorf("getMatrixes: %w", err)
|
||||||
}
|
}
|
||||||
for _, matrix := range matricxes {
|
for _, matrix := range matricxes {
|
||||||
job := job.Clone()
|
swf := newSingleWorkflow(workflow)
|
||||||
if job.Name == "" {
|
if err := swf.SetJob(id, expandJobCombo(id, job, matrix, evaluatedJob, pc.gitContext, results, pc.vars, pc.inputs)); err != nil {
|
||||||
job.Name = id
|
|
||||||
}
|
|
||||||
job.Strategy.RawMatrix = encodeMatrix(matrix)
|
|
||||||
evaluator := NewExpressionEvaluator(NewInterpeter(id, evaluatedJob, matrix, pc.gitContext, results, pc.vars, pc.inputs))
|
|
||||||
job.Name = nameWithMatrix(job.Name, matrix, evaluator)
|
|
||||||
runsOn := evaluatedJob.RunsOn()
|
|
||||||
for i, v := range runsOn {
|
|
||||||
runsOn[i] = evaluator.Interpolate(v)
|
|
||||||
}
|
|
||||||
job.RawRunsOn = encodeRunsOn(runsOn)
|
|
||||||
swf := &SingleWorkflow{
|
|
||||||
Name: workflow.Name,
|
|
||||||
RawOn: workflow.RawOn,
|
|
||||||
Env: workflow.Env,
|
|
||||||
Defaults: workflow.Defaults,
|
|
||||||
RawPermissions: workflow.RawPermissions,
|
|
||||||
RunName: workflow.RunName,
|
|
||||||
}
|
|
||||||
if err := swf.SetJob(id, job); err != nil {
|
|
||||||
return nil, fmt.Errorf("SetJob: %w", err)
|
return nil, fmt.Errorf("SetJob: %w", err)
|
||||||
}
|
}
|
||||||
ret = append(ret, swf)
|
ret = append(ret, swf)
|
||||||
@@ -140,6 +135,77 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
|
|||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newSingleWorkflow returns a SingleWorkflow carrying w's global fields and no job.
|
||||||
|
func newSingleWorkflow(w *SingleWorkflow) *SingleWorkflow {
|
||||||
|
return &SingleWorkflow{
|
||||||
|
Name: w.Name,
|
||||||
|
RawOn: w.RawOn,
|
||||||
|
Env: w.Env,
|
||||||
|
Defaults: w.Defaults,
|
||||||
|
RawPermissions: w.RawPermissions,
|
||||||
|
RunName: w.RunName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// expandJobCombo builds the Job for one matrix combination: it bakes the matrix into the strategy
|
||||||
|
// and interpolates the name and runs-on. actJob drives the interpreter (it supplies the job's
|
||||||
|
// needs and strategy contexts) and may differ from src (an act model.Job vs the source *Job).
|
||||||
|
func expandJobCombo(jobID string, src *Job, matrix map[string]any, actJob *model.Job, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, inputs map[string]any) *Job {
|
||||||
|
combo := src.Clone()
|
||||||
|
if combo.Name == "" {
|
||||||
|
combo.Name = jobID
|
||||||
|
}
|
||||||
|
combo.Strategy.RawMatrix = encodeMatrix(matrix)
|
||||||
|
evaluator := NewExpressionEvaluator(NewInterpeter(jobID, actJob, matrix, gitCtx, results, vars, inputs))
|
||||||
|
combo.Name = nameWithMatrix(combo.Name, matrix, evaluator)
|
||||||
|
runsOn := combo.RunsOn()
|
||||||
|
for i := range runsOn {
|
||||||
|
runsOn[i] = evaluator.Interpolate(runsOn[i])
|
||||||
|
}
|
||||||
|
combo.RawRunsOn = encodeRunsOn(runsOn)
|
||||||
|
return combo
|
||||||
|
}
|
||||||
|
|
||||||
|
// RawMatrixHasExpression reports whether the job's matrix contains a ${{ }} expression.
|
||||||
|
// With needs present, this marks a matrix whose expansion is deferred until they complete.
|
||||||
|
func RawMatrixHasExpression(job *Job) bool {
|
||||||
|
return rawMatrixHasExpression(&job.Strategy.RawMatrix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExpandMatrixWithNeeds expands job's matrix once its needs complete, returning one Job per
|
||||||
|
// combination. Like EvaluateConcurrency it evaluates against the needs context via NewInterpeter,
|
||||||
|
// with no workflow re-parse or stub jobs. job must carry its raw matrix and needs (RawNeeds).
|
||||||
|
func ExpandMatrixWithNeeds(jobID string, job *Job, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, inputs map[string]any) ([]*Job, error) {
|
||||||
|
var rawNeeds yaml.Node
|
||||||
|
if err := rawNeeds.Encode(job.Needs()); err != nil {
|
||||||
|
return nil, fmt.Errorf("encode needs: %w", err)
|
||||||
|
}
|
||||||
|
actJob := &model.Job{
|
||||||
|
RawNeeds: rawNeeds,
|
||||||
|
Strategy: &model.Strategy{
|
||||||
|
FailFastString: job.Strategy.FailFastString,
|
||||||
|
MaxParallelString: job.Strategy.MaxParallelString,
|
||||||
|
RawMatrix: *deepCopyYamlNode(&job.Strategy.RawMatrix),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve fromJson(needs.*.outputs.*) and friends into concrete matrix values.
|
||||||
|
if err := NewExpressionEvaluator(NewInterpeter(jobID, actJob, nil, gitCtx, results, vars, inputs)).
|
||||||
|
EvaluateYamlNode(&actJob.Strategy.RawMatrix); err != nil {
|
||||||
|
return nil, fmt.Errorf("evaluate matrix: %w", err)
|
||||||
|
}
|
||||||
|
matrixes, err := getMatrixes(actJob)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("getMatrixes: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expanded := make([]*Job, 0, len(matrixes))
|
||||||
|
for _, matrix := range matrixes {
|
||||||
|
expanded = append(expanded, expandJobCombo(jobID, job, matrix, actJob, gitCtx, results, vars, inputs))
|
||||||
|
}
|
||||||
|
return expanded, nil
|
||||||
|
}
|
||||||
|
|
||||||
func WithJobResults(results map[string]string) ParseOption {
|
func WithJobResults(results map[string]string) ParseOption {
|
||||||
return func(c *parseContext) {
|
return func(c *parseContext) {
|
||||||
c.jobResults = results
|
c.jobResults = results
|
||||||
|
|||||||
@@ -533,3 +533,91 @@ jobs:
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseDefersDynamicMatrix(t *testing.T) {
|
||||||
|
// A matrix referencing needs outputs is emitted as a single placeholder retaining the raw
|
||||||
|
// expression, rather than being expanded or split per static value.
|
||||||
|
workflowYAML := `
|
||||||
|
name: test-defer
|
||||||
|
on: push
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
setup:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- run: echo setup
|
||||||
|
|
||||||
|
build:
|
||||||
|
needs: setup
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
os: [ubuntu-latest, windows-latest]
|
||||||
|
version: ${{ fromJson(needs.setup.outputs.versions) }}
|
||||||
|
steps:
|
||||||
|
- run: echo build
|
||||||
|
`
|
||||||
|
result, err := Parse([]byte(workflowYAML))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var buildJobs []*Job
|
||||||
|
for _, w := range result {
|
||||||
|
if id, j := w.Job(); id == "build" {
|
||||||
|
buildJobs = append(buildJobs, j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.Len(t, buildJobs, 1, "deferred matrix must yield exactly one placeholder")
|
||||||
|
assert.True(t, RawMatrixHasExpression(buildJobs[0]), "placeholder must keep the raw matrix expression")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpandMatrixWithNeeds(t *testing.T) {
|
||||||
|
buildJob := func(t *testing.T, matrixYAML, runsOn string, needs []string) *Job {
|
||||||
|
t.Helper()
|
||||||
|
var strat Strategy
|
||||||
|
require.NoError(t, yaml.Unmarshal([]byte(matrixYAML), &strat))
|
||||||
|
job := &Job{Name: "build", Strategy: strat}
|
||||||
|
require.NoError(t, job.RawRunsOn.Encode(runsOn))
|
||||||
|
require.NoError(t, job.RawNeeds.Encode(needs))
|
||||||
|
return job
|
||||||
|
}
|
||||||
|
|
||||||
|
results := map[string]*JobResult{
|
||||||
|
"setup": {Result: "success", Outputs: map[string]string{
|
||||||
|
"versions": "[1.20, 1.21]",
|
||||||
|
"platforms": `["linux", "darwin"]`,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("single dynamic dimension", func(t *testing.T) {
|
||||||
|
job := buildJob(t, "matrix:\n version: ${{ fromJson(needs.setup.outputs.versions) }}\n", "ubuntu-latest", []string{"setup"})
|
||||||
|
got, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, results, nil, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, got, 2)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("static and dynamic expand to product once", func(t *testing.T) {
|
||||||
|
// 2 static os * 2 dynamic versions = 4, expanded once (regression: not split then re-expanded).
|
||||||
|
job := buildJob(t, "matrix:\n os: [ubuntu-latest, windows-latest]\n version: ${{ fromJson(needs.setup.outputs.versions) }}\n", "${{ matrix.os }}", []string{"setup"})
|
||||||
|
got, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, results, nil, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, got, 4)
|
||||||
|
for _, combo := range got {
|
||||||
|
runsOn := combo.RunsOn()
|
||||||
|
require.Len(t, runsOn, 1)
|
||||||
|
assert.Contains(t, []string{"ubuntu-latest", "windows-latest"}, runsOn[0], "runs-on must be interpolated from matrix.os")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("multiple dynamic dimensions", func(t *testing.T) {
|
||||||
|
job := buildJob(t, "matrix:\n version: ${{ fromJson(needs.setup.outputs.versions) }}\n platform: ${{ fromJson(needs.setup.outputs.platforms) }}\n", "ubuntu-latest", []string{"setup"})
|
||||||
|
got, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, results, nil, nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, got, 4)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unresolved needs output errors", func(t *testing.T) {
|
||||||
|
job := buildJob(t, "matrix:\n version: ${{ fromJson(needs.missing.outputs.versions) }}\n", "ubuntu-latest", []string{"missing"})
|
||||||
|
_, err := ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, map[string]*JobResult{}, nil, nil)
|
||||||
|
require.Error(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -46,6 +46,11 @@ func CreateCommitStatusForRunJobs(ctx context.Context, run *actions_model.Action
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, job := range jobs {
|
for _, job := range jobs {
|
||||||
|
// A deferred-matrix placeholder's name changes when it expands, so a status created now
|
||||||
|
// would be orphaned; the expanded combos get theirs at expansion time.
|
||||||
|
if job.RawStrategy != "" && !job.IsMatrixEvaluated {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if err = createCommitStatus(ctx, run.Repo, event, commitID, run, job); err != nil {
|
if err = createCommitStatus(ctx, run.Repo, event, commitID, run, job); err != nil {
|
||||||
log.Error("Failed to create commit status for job %d: %v", job.ID, err)
|
log.Error("Failed to create commit status for job %d: %v", job.ID, err)
|
||||||
}
|
}
|
||||||
|
|||||||
+39
-221
@@ -7,10 +7,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
actions_model "gitea.dev/models/actions"
|
actions_model "gitea.dev/models/actions"
|
||||||
"gitea.dev/models/db"
|
"gitea.dev/models/db"
|
||||||
@@ -60,128 +56,9 @@ func checkTaskNeedsReady(ctx context.Context, job *actions_model.ActionRunJob) (
|
|||||||
return taskNeeds, true, nil
|
return taskNeeds, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExtractRawStrategies extracts strategy definitions from the raw workflow content.
|
// ReEvaluateMatrixForJobWithNeeds expands the matrix strategy of a job once all its dependent
|
||||||
// Returns a map of jobID to strategy YAML for jobs that have matrix dependencies.
|
// jobs are done, using their outputs, and inserts the resulting ActionRunJobs. The original
|
||||||
func ExtractRawStrategies(content []byte) (map[string]string, error) {
|
// placeholder job is reused as the first combination.
|
||||||
var workflowDef struct {
|
|
||||||
Jobs map[string]struct {
|
|
||||||
Strategy any `yaml:"strategy"`
|
|
||||||
Needs any `yaml:"needs"`
|
|
||||||
} `yaml:"jobs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := yaml.Unmarshal(content, &workflowDef); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
strategies := make(map[string]string)
|
|
||||||
for jobID, jobDef := range workflowDef.Jobs {
|
|
||||||
if jobDef.Strategy == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var needsList []string
|
|
||||||
switch needs := jobDef.Needs.(type) {
|
|
||||||
case string:
|
|
||||||
needsList = append(needsList, needs)
|
|
||||||
case []any:
|
|
||||||
for _, need := range needs {
|
|
||||||
if needStr, ok := need.(string); ok {
|
|
||||||
needsList = append(needsList, needStr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(needsList) > 0 {
|
|
||||||
if strategyBytes, err := yaml.Marshal(jobDef.Strategy); err == nil {
|
|
||||||
strategies[jobID] = string(strategyBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return strategies, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// HasMatrixWithNeeds reports whether rawStrategy contains a matrix value whose
|
|
||||||
// expression tree references needs.<id>.outputs.<key>.
|
|
||||||
// It walks the parsed YAML tree to avoid false positives from values such as
|
|
||||||
// "os: [needs.review-runner]" that merely contain the substring "needs.".
|
|
||||||
func HasMatrixWithNeeds(rawStrategy string) bool {
|
|
||||||
if rawStrategy == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
var root yaml.Node
|
|
||||||
if err := yaml.Unmarshal([]byte(rawStrategy), &root); err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// The top-level document node wraps a single mapping node.
|
|
||||||
doc := &root
|
|
||||||
if doc.Kind == yaml.DocumentNode && len(doc.Content) == 1 {
|
|
||||||
doc = doc.Content[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find the "matrix" key inside the strategy mapping.
|
|
||||||
var matrixNode *yaml.Node
|
|
||||||
if doc.Kind == yaml.MappingNode {
|
|
||||||
for i := 0; i+1 < len(doc.Content); i += 2 {
|
|
||||||
if doc.Content[i].Value == "matrix" {
|
|
||||||
matrixNode = doc.Content[i+1]
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if matrixNode == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return yamlNodeContainsNeedsOutputsExpr(matrixNode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// yamlNodeContainsNeedsOutputsExpr recursively inspects a yaml.Node and
|
|
||||||
// returns true if any scalar value contains a GitHub Actions expression of
|
|
||||||
// the form ${{ ... needs.<id>.outputs.<key> ... }}.
|
|
||||||
func yamlNodeContainsNeedsOutputsExpr(node *yaml.Node) bool {
|
|
||||||
if node == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if node.Kind == yaml.ScalarNode {
|
|
||||||
return containsNeedsOutputsExpr(node.Value)
|
|
||||||
}
|
|
||||||
return slices.ContainsFunc(node.Content, yamlNodeContainsNeedsOutputsExpr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// containsNeedsOutputsExpr returns true when s contains an Actions expression
|
|
||||||
// (${{ ... }}) that references needs.<id>.outputs.<key>.
|
|
||||||
// A bare "needs." substring outside an expression block is not a match.
|
|
||||||
func containsNeedsOutputsExpr(s string) bool {
|
|
||||||
if !strings.Contains(s, "${{") {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for i := 0; i < len(s); {
|
|
||||||
start := strings.Index(s[i:], "${{")
|
|
||||||
if start == -1 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
start += i
|
|
||||||
end := strings.Index(s[start:], "}}")
|
|
||||||
if end == -1 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
end += start
|
|
||||||
expr := s[start : end+2]
|
|
||||||
if strings.Contains(expr, "needs.") && strings.Contains(expr, ".outputs.") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
i = end + 2
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReEvaluateMatrixForJobWithNeeds re-evaluates the matrix strategy of a job once all its
|
|
||||||
// dependent jobs are done. It expands the matrix using job outputs and inserts the resulting
|
|
||||||
// ActionRunJobs. The original placeholder job is marked as evaluated and skipped.
|
|
||||||
// Returns nil, nil if the job is not ready yet or has nothing to do.
|
// Returns nil, nil if the job is not ready yet or has nothing to do.
|
||||||
func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.ActionRunJob, vars map[string]string) ([]*actions_model.ActionRunJob, error) {
|
func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.ActionRunJob, vars map[string]string) ([]*actions_model.ActionRunJob, error) {
|
||||||
if job.IsMatrixEvaluated || job.RawStrategy == "" {
|
if job.IsMatrixEvaluated || job.RawStrategy == "" {
|
||||||
@@ -191,8 +68,8 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
|||||||
log.Debug("Starting matrix re-evaluation for job %d (JobID: %s)", job.ID, job.JobID)
|
log.Debug("Starting matrix re-evaluation for job %d (JobID: %s)", job.ID, job.JobID)
|
||||||
|
|
||||||
// skipWithError marks the job as evaluated+skipped and wraps any secondary error.
|
// skipWithError marks the job as evaluated+skipped and wraps any secondary error.
|
||||||
skipWithError := func(reason string, origErr error) ([]*actions_model.ActionRunJob, error) {
|
skipWithError := func(origErr error) ([]*actions_model.ActionRunJob, error) {
|
||||||
if markErr := markMatrixAsEvaluatedAndSkip(ctx, job, reason); markErr != nil {
|
if markErr := markMatrixAsEvaluatedAndSkip(ctx, job, origErr.Error()); markErr != nil {
|
||||||
return nil, fmt.Errorf("%w; additionally failed to mark as evaluated: %v", origErr, markErr)
|
return nil, fmt.Errorf("%w; additionally failed to mark as evaluated: %v", origErr, markErr)
|
||||||
}
|
}
|
||||||
return nil, origErr
|
return nil, origErr
|
||||||
@@ -201,7 +78,7 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
|||||||
taskNeeds, allDone, err := checkTaskNeedsReady(ctx, job)
|
taskNeeds, allDone, err := checkTaskNeedsReady(ctx, job)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("Matrix re-evaluation error for job %d: check task needs: %v", job.ID, err)
|
log.Error("Matrix re-evaluation error for job %d: check task needs: %v", job.ID, err)
|
||||||
return skipWithError(fmt.Sprintf("task needs check failed: %v", err), fmt.Errorf("check task needs: %w", err))
|
return skipWithError(fmt.Errorf("check task needs: %w", err))
|
||||||
}
|
}
|
||||||
if !allDone {
|
if !allDone {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -221,30 +98,36 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
|||||||
|
|
||||||
giteaCtx := GenerateGiteaContext(ctx, job.Run, nil, job)
|
giteaCtx := GenerateGiteaContext(ctx, job.Run, nil, job)
|
||||||
|
|
||||||
jobOutputs := make(map[string]map[string]string, len(taskNeeds))
|
results := make(map[string]*jobparser.JobResult, len(taskNeeds))
|
||||||
jobResults := make(map[string]string, len(taskNeeds))
|
for needID, need := range taskNeeds {
|
||||||
for jobID, need := range taskNeeds {
|
results[needID] = &jobparser.JobResult{Result: need.Result.String(), Outputs: need.Outputs}
|
||||||
jobOutputs[jobID] = need.Outputs
|
|
||||||
jobResults[jobID] = need.Result.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
workflowYAML, err := constructWorkflowWithNeeds(job, taskNeeds)
|
// Rebuild the job from its own payload + stored raw strategy and re-attach needs (erased from
|
||||||
|
// the payload) so needs.*.outputs.* resolves. No synthetic workflow, no stub jobs, no re-parse.
|
||||||
|
var baseSWF jobparser.SingleWorkflow
|
||||||
|
if err := yaml.Unmarshal(job.WorkflowPayload, &baseSWF); err != nil {
|
||||||
|
return skipWithError(fmt.Errorf("unmarshal payload: %w", err))
|
||||||
|
}
|
||||||
|
_, parsedJob := baseSWF.Job()
|
||||||
|
if parsedJob == nil {
|
||||||
|
return skipWithError(errors.New("payload contains no job"))
|
||||||
|
}
|
||||||
|
var rawStrategy jobparser.Strategy
|
||||||
|
if err := yaml.Unmarshal([]byte(job.RawStrategy), &rawStrategy); err != nil {
|
||||||
|
return skipWithError(fmt.Errorf("unmarshal raw strategy: %w", err))
|
||||||
|
}
|
||||||
|
parsedJob.Strategy = rawStrategy
|
||||||
|
if err := parsedJob.RawNeeds.Encode(job.Needs); err != nil {
|
||||||
|
return skipWithError(fmt.Errorf("encode needs: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
expandedJobs, err := jobparser.ExpandMatrixWithNeeds(job.JobID, parsedJob, giteaCtx.ToGitHubContext(), results, vars, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return skipWithError(fmt.Sprintf("workflow construction failed: %v", err), fmt.Errorf("construct workflow: %w", err))
|
return nil, markMatrixAsEvaluatedAndSkip(ctx, job, fmt.Sprintf("matrix expansion failed: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
parsedJobs, err := jobparser.Parse(
|
// One workflow payload per combination; needs are kept on the model and erased from the
|
||||||
workflowYAML,
|
|
||||||
jobparser.WithVars(vars),
|
|
||||||
jobparser.WithGitContext(giteaCtx.ToGitHubContext()),
|
|
||||||
jobparser.WithJobOutputs(jobOutputs),
|
|
||||||
jobparser.WithJobResults(jobResults),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, markMatrixAsEvaluatedAndSkip(ctx, job, fmt.Sprintf("parse failed: %v", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
// One parsed workflow per combination; needs are kept on the model and erased from the
|
|
||||||
// payload, as at initial planning time.
|
// payload, as at initial planning time.
|
||||||
type matrixCombo struct {
|
type matrixCombo struct {
|
||||||
name string
|
name string
|
||||||
@@ -252,18 +135,15 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
|||||||
runsOn []string
|
runsOn []string
|
||||||
needs []string
|
needs []string
|
||||||
}
|
}
|
||||||
var combos []matrixCombo
|
combos := make([]matrixCombo, 0, len(expandedJobs))
|
||||||
for _, sw := range parsedJobs {
|
for _, expanded := range expandedJobs {
|
||||||
id, jobDef := sw.Job()
|
combo := matrixCombo{name: expanded.Name, runsOn: expanded.RunsOn(), needs: expanded.Needs()}
|
||||||
if jobDef == nil || id != job.JobID {
|
swf := baseSWF
|
||||||
continue
|
if err := swf.SetJob(job.JobID, expanded.EraseNeeds()); err != nil {
|
||||||
|
return nil, fmt.Errorf("set expanded job %s: %w", job.JobID, err)
|
||||||
}
|
}
|
||||||
combo := matrixCombo{name: jobDef.Name, runsOn: jobDef.RunsOn(), needs: jobDef.Needs()}
|
if combo.payload, err = swf.Marshal(); err != nil {
|
||||||
if err := sw.SetJob(id, jobDef.EraseNeeds()); err != nil {
|
return nil, fmt.Errorf("marshal expanded job %s: %w", job.JobID, err)
|
||||||
return nil, fmt.Errorf("erase needs for job %s: %w", id, err)
|
|
||||||
}
|
|
||||||
if combo.payload, err = sw.Marshal(); err != nil {
|
|
||||||
return nil, fmt.Errorf("marshal expanded job %s: %w", id, err)
|
|
||||||
}
|
}
|
||||||
combos = append(combos, combo)
|
combos = append(combos, combo)
|
||||||
}
|
}
|
||||||
@@ -323,65 +203,3 @@ func ReEvaluateMatrixForJobWithNeeds(ctx context.Context, job *actions_model.Act
|
|||||||
|
|
||||||
return children, nil
|
return children, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// constructWorkflowWithNeeds creates a workflow YAML that includes the target job
|
|
||||||
// and stub definitions for its dependencies so the jobparser can resolve needs.*.outputs expressions.
|
|
||||||
func constructWorkflowWithNeeds(job *actions_model.ActionRunJob, taskNeeds map[string]*TaskNeed) ([]byte, error) {
|
|
||||||
var jobWorkflow map[string]any
|
|
||||||
if err := yaml.Unmarshal(job.WorkflowPayload, &jobWorkflow); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal job workflow: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
jobsSection, ok := jobWorkflow["jobs"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
return nil, errors.New("invalid jobs section in workflow")
|
|
||||||
}
|
|
||||||
|
|
||||||
newJobs := make(map[string]any)
|
|
||||||
|
|
||||||
for needJobID, taskNeed := range taskNeeds {
|
|
||||||
stubJob := map[string]any{
|
|
||||||
"runs-on": "ubuntu-latest",
|
|
||||||
"outputs": taskNeed.Outputs,
|
|
||||||
"steps": []any{},
|
|
||||||
}
|
|
||||||
newJobs[needJobID] = stubJob
|
|
||||||
}
|
|
||||||
|
|
||||||
maps.Copy(newJobs, jobsSection)
|
|
||||||
|
|
||||||
// The WorkflowPayload may contain a normalised/wrapped matrix (e.g.
|
|
||||||
// version: ["${{ fromJson(...) }}"]). Restore the original scalar expression
|
|
||||||
// from RawStrategy so jobparser.Parse() can expand it correctly with job outputs.
|
|
||||||
// Also drop the pre-baked "name" so jobparser regenerates it per matrix combination
|
|
||||||
// (e.g. "build (1)", "build (2)", …) instead of "build (Array)".
|
|
||||||
// Critically, re-add "needs" because EraseNeeds() removed them from WorkflowPayload:
|
|
||||||
// without needs, NewInterpeter builds an empty Needs context and
|
|
||||||
// "needs.generate.outputs.*" expressions can never be evaluated.
|
|
||||||
if targetJobDef, ok := newJobs[job.JobID]; ok {
|
|
||||||
if targetJobMap, ok := targetJobDef.(map[string]any); ok {
|
|
||||||
delete(targetJobMap, "name")
|
|
||||||
needsKeys := make([]string, 0, len(taskNeeds))
|
|
||||||
for needJobID := range taskNeeds {
|
|
||||||
needsKeys = append(needsKeys, needJobID)
|
|
||||||
}
|
|
||||||
sort.Strings(needsKeys)
|
|
||||||
targetJobMap["needs"] = needsKeys
|
|
||||||
if job.RawStrategy != "" {
|
|
||||||
var rawStrategyMap map[string]any
|
|
||||||
if err := yaml.Unmarshal([]byte(job.RawStrategy), &rawStrategyMap); err == nil {
|
|
||||||
targetJobMap["strategy"] = rawStrategyMap
|
|
||||||
}
|
|
||||||
}
|
|
||||||
newJobs[job.JobID] = targetJobMap
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
workflow := map[string]any{
|
|
||||||
"name": "matrix-expansion",
|
|
||||||
"on": "push",
|
|
||||||
"jobs": newJobs,
|
|
||||||
}
|
|
||||||
|
|
||||||
return yaml.Marshal(workflow)
|
|
||||||
}
|
|
||||||
@@ -1,171 +0,0 @@
|
|||||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
|
||||||
// SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
package actions
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
actions_model "gitea.dev/models/actions"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.yaml.in/yaml/v4"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHasMatrixWithNeeds(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
strategy string
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "dynamic matrix referencing job output",
|
|
||||||
strategy: `
|
|
||||||
matrix:
|
|
||||||
version: ${{ fromJson(needs.generate.outputs.matrix) }}
|
|
||||||
`,
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "static matrix — no expression",
|
|
||||||
strategy: `
|
|
||||||
matrix:
|
|
||||||
os: [ubuntu-latest, windows-latest]
|
|
||||||
`,
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "value contains needs. but not inside expression",
|
|
||||||
strategy: `
|
|
||||||
matrix:
|
|
||||||
os: [needs.review-runner]
|
|
||||||
`,
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "needs. outside expression block",
|
|
||||||
strategy: `
|
|
||||||
matrix:
|
|
||||||
runner: needs.something-but-no-braces
|
|
||||||
`,
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "expression with needs but no .outputs.",
|
|
||||||
strategy: `
|
|
||||||
matrix:
|
|
||||||
version: ${{ needs.job1 }}
|
|
||||||
`,
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty strategy",
|
|
||||||
strategy: "",
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "strategy without matrix key",
|
|
||||||
strategy: `
|
|
||||||
fail-fast: false
|
|
||||||
`,
|
|
||||||
want: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multi-dimension dynamic matrix",
|
|
||||||
strategy: `
|
|
||||||
matrix:
|
|
||||||
os: [ubuntu-latest, windows-latest]
|
|
||||||
version: ${{ fromJson(needs.setup.outputs.versions) }}
|
|
||||||
`,
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
assert.Equal(t, tt.want, HasMatrixWithNeeds(tt.strategy))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConstructWorkflowWithNeeds(t *testing.T) {
|
|
||||||
// Minimal WorkflowPayload with a strategy referencing a needs output.
|
|
||||||
payload, err := yaml.Marshal(map[string]any{
|
|
||||||
"jobs": map[string]any{
|
|
||||||
"build": map[string]any{
|
|
||||||
"runs-on": "ubuntu-latest",
|
|
||||||
"strategy": map[string]any{
|
|
||||||
"matrix": map[string]any{
|
|
||||||
"version": `${{ fromJson(needs.setup.outputs.versions) }}`,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"steps": []any{},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
job := &actions_model.ActionRunJob{
|
|
||||||
JobID: "build",
|
|
||||||
WorkflowPayload: payload,
|
|
||||||
RawStrategy: `
|
|
||||||
matrix:
|
|
||||||
version: ${{ fromJson(needs.setup.outputs.versions) }}
|
|
||||||
`,
|
|
||||||
Needs: []string{"setup"},
|
|
||||||
}
|
|
||||||
needs := map[string]*TaskNeed{
|
|
||||||
"setup": {
|
|
||||||
Result: actions_model.StatusSuccess,
|
|
||||||
Outputs: map[string]string{"versions": `["1.20","1.21"]`},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
out, err := constructWorkflowWithNeeds(job, needs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// The resulting YAML should contain both the build job and a stub for setup.
|
|
||||||
var wf map[string]any
|
|
||||||
require.NoError(t, yaml.Unmarshal(out, &wf))
|
|
||||||
|
|
||||||
jobs, ok := wf["jobs"].(map[string]any)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Contains(t, jobs, "build")
|
|
||||||
assert.Contains(t, jobs, "setup", "stub for needs dependency must be present")
|
|
||||||
|
|
||||||
// build job needs must list the dependency
|
|
||||||
buildJob, ok := jobs["build"].(map[string]any)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Contains(t, buildJob, "needs")
|
|
||||||
|
|
||||||
// RawStrategy must be re-injected (not the pre-baked array form)
|
|
||||||
strategy, ok := buildJob["strategy"].(map[string]any)
|
|
||||||
require.True(t, ok)
|
|
||||||
matrix, ok := strategy["matrix"].(map[string]any)
|
|
||||||
require.True(t, ok)
|
|
||||||
versionExpr, ok := matrix["version"].(string)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Contains(t, versionExpr, "fromJson")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConstructWorkflowWithNeeds_InvalidPayload(t *testing.T) {
|
|
||||||
job := &actions_model.ActionRunJob{
|
|
||||||
JobID: "build",
|
|
||||||
WorkflowPayload: []byte("not: valid: yaml: ["),
|
|
||||||
}
|
|
||||||
_, err := constructWorkflowWithNeeds(job, nil)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConstructWorkflowWithNeeds_MissingJobsSection(t *testing.T) {
|
|
||||||
payload, err := yaml.Marshal(map[string]any{"name": "test"})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
job := &actions_model.ActionRunJob{
|
|
||||||
JobID: "build",
|
|
||||||
WorkflowPayload: payload,
|
|
||||||
}
|
|
||||||
_, err = constructWorkflowWithNeeds(job, nil)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
+9
-12
@@ -10,7 +10,6 @@ import (
|
|||||||
actions_model "gitea.dev/models/actions"
|
actions_model "gitea.dev/models/actions"
|
||||||
"gitea.dev/models/db"
|
"gitea.dev/models/db"
|
||||||
"gitea.dev/modules/actions/jobparser"
|
"gitea.dev/modules/actions/jobparser"
|
||||||
"gitea.dev/modules/log"
|
|
||||||
"gitea.dev/modules/util"
|
"gitea.dev/modules/util"
|
||||||
|
|
||||||
act_model "gitea.com/gitea/runner/act/model"
|
act_model "gitea.com/gitea/runner/act/model"
|
||||||
@@ -129,15 +128,10 @@ func InsertRun(ctx context.Context, run *actions_model.ActionRun, content []byte
|
|||||||
runJobs := make([]*actions_model.ActionRunJob, 0, len(jobs))
|
runJobs := make([]*actions_model.ActionRunJob, 0, len(jobs))
|
||||||
var hasWaitingJobs bool
|
var hasWaitingJobs bool
|
||||||
|
|
||||||
rawStrategies, err := ExtractRawStrategies(content)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn("Failed to extract raw strategies for run %d: %v", run.ID, err)
|
|
||||||
rawStrategies = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, v := range jobs {
|
for i, v := range jobs {
|
||||||
id, job := v.Job()
|
id, job := v.Job()
|
||||||
needs := job.Needs()
|
needs := job.Needs()
|
||||||
|
isDeferredMatrix := len(needs) > 0 && jobparser.RawMatrixHasExpression(job)
|
||||||
if err := v.SetJob(id, job.EraseNeeds()); err != nil {
|
if err := v.SetJob(id, job.EraseNeeds()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -167,11 +161,14 @@ func InsertRun(ctx context.Context, run *actions_model.ActionRun, content []byte
|
|||||||
runJob.TokenPermissions = perms
|
runJob.TokenPermissions = perms
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store the raw strategy for jobs whose matrix references needs outputs.
|
// Matrix references needs outputs: jobparser emitted a placeholder. Store the raw
|
||||||
// ReEvaluateMatrixForJobWithNeeds uses this to re-expand the matrix once
|
// strategy for ReEvaluateMatrixForJobWithNeeds to expand once the needs finish.
|
||||||
// the dependency jobs have completed and their outputs are available.
|
if isDeferredMatrix {
|
||||||
if rawStrategy, ok := rawStrategies[id]; ok && HasMatrixWithNeeds(rawStrategy) {
|
rawStrategy, err := yaml.Marshal(&job.Strategy)
|
||||||
runJob.RawStrategy = rawStrategy
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal raw strategy for job %s: %w", id, err)
|
||||||
|
}
|
||||||
|
runJob.RawStrategy = string(rawStrategy)
|
||||||
}
|
}
|
||||||
|
|
||||||
// check job concurrency
|
// check job concurrency
|
||||||
|
|||||||
Reference in new issue
Block a user