feat: Add support for dynamic matrix evaluation in Gitea Actions workflows (#36564)

Adds dynamic matrix evaluation to Gitea Actions: a job's
`strategy.matrix` can be built from the outputs of the jobs it needs.

```yaml
jobs:
  generate:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set.outputs.result }}
    steps:
      - id: set
        run: echo "result=[1,2,3]" >> $GITHUB_OUTPUT

  build:
    needs: [generate]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.generate.outputs.matrix) }}
    steps:
      - run: echo "building ${{ matrix.version }}"
```

Such a matrix cannot be expanded at planning time, so the job is planned
as a single placeholder and expanded by the job emitter once its needs
finish. Each combination is then gated by `if:` and concurrency as
usual.

- A matrix that resolves to no combination fails the job, as on GitHub.
- Expansion is capped at `MaxJobNumPerRun`.
- Workflows without a needs-dependent matrix are unaffected.

Fixes https://github.com/go-gitea/gitea/issues/25179

---------

Signed-off-by: Pascal Zimmermann <pascal.zimmermann@theiotstudio.com>
Signed-off-by: ZPascal <pascal.zimmermann@theiotstudio.com>
Co-authored-by: Claude <claude-sonnet-4-5@anthropic.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Claude (Opus 4.8) <noreply@anthropic.com>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: Zettat123 <zettat123@gmail.com>

modules/actions/jobparser/jobparser.go

@@ -5,15 +5,61 @@ package jobparser
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 
 	"gitea.com/gitea/runner/act/exprparser"
 	"gitea.com/gitea/runner/act/model"
+	"github.com/rhysd/actionlint"
 	"go.yaml.in/yaml/v4"
 )
 
+// HasDeferredMatrix reports whether the job's matrix can only be expanded once its needs finish:
+// it reads the needs context and the job has needs to resolve that context against.
+// Parse emits such a job as a single placeholder rather than one job per combination, so every
+// caller that persists a job must agree with Parse on this condition.
+func HasDeferredMatrix(job *Job) bool {
+	return len(job.Needs()) > 0 && rawMatrixReadsNeeds(&job.Strategy.RawMatrix)
+}
+
+func rawMatrixReadsNeeds(node *yaml.Node) bool {
+	if node.Kind == yaml.ScalarNode {
+		return expressionReadsNeeds(node.Value)
+	}
+	return slices.ContainsFunc(node.Content, rawMatrixReadsNeeds)
+}
+
+// expressionReadsNeeds reports whether value holds a ${{ }} expression reading the needs context.
+// Every other context (github, vars, inputs, ...) is already available while planning, so deferring
+// those too would replace their combinations with one placeholder and change the commit status
+// contexts the run publishes, which a repository's required checks are configured against.
+func expressionReadsNeeds(value string) bool {
+	for rest := value; ; {
+		_, after, found := strings.Cut(rest, "${{")
+		if !found {
+			return false
+		}
+		rest = after
+		// The lexer ends the expression at its closing `}}`, so it can be handed the whole remainder.
+		expr, err := actionlint.NewExprParser().Parse(actionlint.NewExprLexer(rest))
+		if err != nil {
+			return true // unparseable here, let the expansion report it against the real values
+		}
+		readsNeeds := false
+		actionlint.VisitExprNode(expr, func(node, _ actionlint.ExprNode, entering bool) {
+			if variable, ok := node.(*actionlint.VariableNode); entering && ok && strings.EqualFold(variable.Name, "needs") {
+				readsNeeds = true
+			}
+		})
+		if readsNeeds {
+			return true
+		}
+	}
+}
+
 func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
 	origin, err := model.ReadWorkflow(bytes.NewReader(content))
 	if err != nil {
@@ -37,7 +83,7 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
 		results[id] = &JobResult{
 			Needs:   job.Needs(),
 			Result:  pc.jobResults[id],
-			Outputs: nil, // not supported yet
+			Outputs: nil, // resolved at expansion time, not at plan time
 		}
 	}
 
@@ -52,35 +98,34 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
 
 	for i, id := range ids {
 		job := jobs[i]
-		matricxes, err := getMatrixes(origin.GetJob(id))
-		if err != nil {
-			return nil, fmt.Errorf("getMatrixes: %w", err)
+		originJob := origin.GetJob(id)
+
+		if originJob == nil {
+			return nil, fmt.Errorf("job %s not found in origin workflow", id)
 		}
-		for _, matrix := range matricxes {
-			job := job.Clone()
-			if job.Name == "" {
-				job.Name = id
+
+		var combos []*Job
+		if HasDeferredMatrix(job) {
+			// The matrix reads values that do not exist yet (a needs output), so emit a single
+			// placeholder keeping it raw. Re-parsing that placeholder's payload yields it again,
+			// and the server expands it once the needs finish.
+			placeholder := job.Clone()
+			if placeholder.Name == "" {
+				placeholder.Name = id
 			}
-			job.Strategy.RawMatrix = encodeMatrix(matrix)
-			evaluator := NewExpressionEvaluator(NewInterpeter(id, origin.GetJob(id), matrix, pc.gitContext, results, pc.vars, pc.inputs))
-			job.Name = nameWithMatrix(job.Name, matrix, evaluator)
-			runsOn := origin.GetJob(id).RunsOn()
-			for i, v := range runsOn {
-				runsOn[i] = evaluator.Interpolate(v)
+			combos = []*Job{placeholder}
+		} else {
+			matricxes, err := getMatrixes(originJob)
+			if err != nil {
+				return nil, fmt.Errorf("getMatrixes: %w", err)
 			}
-			job.RawRunsOn = encodeRunsOn(runsOn)
-			if err := evaluator.EvaluateYamlNode(&job.RawContinueOnError); err != nil {
-				return nil, fmt.Errorf("evaluate continue-on-error for job %q: %w", id, err)
+			if combos, err = buildMatrixCombos(id, job, matricxes, originJob, pc.gitContext, results, pc.vars, pc.inputs); err != nil {
+				return nil, err
 			}
-			swf := &SingleWorkflow{
-				Name:           workflow.Name,
-				RawOn:          workflow.RawOn,
-				Env:            workflow.Env,
-				Defaults:       workflow.Defaults,
-				RawPermissions: workflow.RawPermissions,
-				RunName:        workflow.RunName,
-			}
-			if err := swf.SetJob(id, job); err != nil {
+		}
+		for _, combo := range combos {
+			swf := workflow.cloneHeader()
+			if err := swf.SetJob(id, combo); err != nil {
 				return nil, fmt.Errorf("SetJob: %w", err)
 			}
 			ret = append(ret, swf)
@@ -89,6 +134,121 @@ func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
 	return ret, nil
 }
 
+// cloneHeader returns a copy of w with its workflow-global fields but no jobs.
+func (w *SingleWorkflow) cloneHeader() *SingleWorkflow {
+	return &SingleWorkflow{
+		Name:           w.Name,
+		RawOn:          w.RawOn,
+		Env:            w.Env,
+		Defaults:       w.Defaults,
+		RawPermissions: w.RawPermissions,
+		RunName:        w.RunName,
+	}
+}
+
+// ExpandMatrixWithNeeds returns one Job per combination of job's matrix, resolved against the
+// completed needs in results. As for EvaluateConcurrency, results must also describe jobID itself,
+// which is where NewInterpeter reads the job's own needs from.
+// maxCombinations caps how many combinations may be built: the values come from a needs output at
+// runtime, so the cap has to be enforced before one Job is materialized per combination.
+func ExpandMatrixWithNeeds(jobID string, job *Job, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, inputs map[string]any, maxCombinations int) ([]*Job, error) {
+	actJob := &model.Job{Strategy: &model.Strategy{
+		FailFastString:    job.Strategy.FailFastString,
+		MaxParallelString: job.Strategy.MaxParallelString,
+		RawMatrix:         job.Strategy.RawMatrix,
+	}}
+
+	// Resolve fromJson(needs.*.outputs.*) and friends into concrete matrix values.
+	if err := NewExpressionEvaluator(NewInterpeter(jobID, actJob, nil, gitCtx, results, vars, inputs)).
+		EvaluateYamlNode(&actJob.Strategy.RawMatrix); err != nil {
+		return nil, fmt.Errorf("evaluate matrix: %w", err)
+	}
+	matrixes, err := getMatrixes(actJob)
+	if err != nil {
+		return nil, fmt.Errorf("getMatrixes: %w", err)
+	}
+	// act collapses a matrix that yields no combination (an empty vector or include, everything
+	// excluded, a whole-matrix expression that is not a mapping) into one empty combination, which
+	// would run the job once unparameterized. GitHub rejects such a matrix, so reject it too.
+	if len(matrixes) == 1 && len(matrixes[0]) == 0 {
+		return nil, errors.New("matrix must define at least one vector")
+	}
+	if len(matrixes) > maxCombinations {
+		return nil, fmt.Errorf("matrix expands to %d combinations, exceeding the limit of %d", len(matrixes), maxCombinations)
+	}
+	return buildMatrixCombos(jobID, job, matrixes, actJob, gitCtx, results, vars, inputs)
+}
+
+// matrixesOf is this package's only entry to act's GetMatrixes, so that every caller is covered by
+// the filter check below. A deferred placeholder is the first thing carrying a raw matrix this far,
+// and the emitter reads its `if:` before expanding it.
+// TODO: drop the check once gitea.com/gitea/runner validates the shape itself.
+func matrixesOf(job *model.Job) ([]map[string]any, error) {
+	if err := validateMatrixFilters(job); err != nil {
+		return nil, err
+	}
+	matrixes, err := job.GetMatrixes()
+	if err != nil {
+		return nil, fmt.Errorf("GetMatrixes: %w", err)
+	}
+	return matrixes, nil
+}
+
+// validateMatrixFilters rejects an `include`/`exclude` that is not a list of mappings. act asserts
+// that shape without checking, so anything else panics there; an unevaluated ${{ }} expression, which
+// is still a scalar, is the usual way to reach it.
+func validateMatrixFilters(job *model.Job) error {
+	if job.Strategy == nil || job.Strategy.RawMatrix.Kind != yaml.MappingNode {
+		return nil
+	}
+	content := job.Strategy.RawMatrix.Content
+	for i := 0; i+1 < len(content); i += 2 {
+		name, value := content[i].Value, content[i+1]
+		if name != "include" && name != "exclude" {
+			continue
+		}
+		entries := []*yaml.Node{value}
+		if value.Kind == yaml.SequenceNode {
+			entries = value.Content
+		}
+		for _, entry := range entries {
+			if entry.Kind == yaml.AliasNode {
+				entry = entry.Alias
+			}
+			if entry.Kind != yaml.MappingNode {
+				return fmt.Errorf("matrix %s must be a list of mappings", name)
+			}
+		}
+	}
+	return nil
+}
+
+// buildMatrixCombos builds one Job per matrix combination from src, baking the combination into the
+// strategy and interpolating the name, runs-on and continue-on-error with it.
+func buildMatrixCombos(jobID string, src *Job, matrixes []map[string]any, actJob *model.Job, gitCtx *model.GithubContext, results map[string]*JobResult, vars map[string]string, inputs map[string]any) ([]*Job, error) {
+	srcRunsOn := src.RunsOn()
+	combos := make([]*Job, 0, len(matrixes))
+	for _, matrix := range matrixes {
+		combo := src.Clone()
+		if combo.Name == "" {
+			combo.Name = jobID
+		}
+		combo.Strategy.RawMatrix = encodeMatrix(matrix)
+		evaluator := NewExpressionEvaluator(NewInterpeter(jobID, actJob, matrix, gitCtx, results, vars, inputs))
+		combo.Name = nameWithMatrix(combo.Name, matrix, evaluator)
+		runsOn := slices.Clone(srcRunsOn)
+		for i := range runsOn {
+			runsOn[i] = evaluator.Interpolate(runsOn[i])
+		}
+		combo.RawRunsOn = encodeRunsOn(runsOn)
+		if err := evaluator.EvaluateYamlNode(&combo.RawContinueOnError); err != nil {
+			return nil, fmt.Errorf("evaluate continue-on-error for job %q: %w", jobID, err)
+		}
+		combos = append(combos, combo)
+	}
+	return combos, nil
+}
+
 func WithGitContext(context *model.GithubContext) ParseOption {
 	return func(c *parseContext) {
 		c.gitContext = context
@@ -117,9 +277,9 @@ type parseContext struct {
 type ParseOption func(c *parseContext)
 
 func getMatrixes(job *model.Job) ([]map[string]any, error) {
-	ret, err := job.GetMatrixes()
+	ret, err := matrixesOf(job)
 	if err != nil {
-		return nil, fmt.Errorf("GetMatrixes: %w", err)
+		return nil, err
 	}
 	sort.Slice(ret, func(i, j int) bool {
 		return matrixName(ret[i]) < matrixName(ret[j])

modules/actions/jobparser/jobparser_test.go

@@ -4,9 +4,11 @@
 package jobparser
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
+	"gitea.com/gitea/runner/act/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.yaml.in/yaml/v4"
@@ -107,3 +109,150 @@ func TestParse(t *testing.T) {
 		})
 	}
 }
+
+func TestParseDefersDynamicMatrix(t *testing.T) {
+	// A matrix referencing needs outputs yields one placeholder keeping the raw expression, rather
+	// than one job per resolvable static value. Any other matrix expands at plan time as usual.
+	const workflow = `
+on: push
+jobs:
+  setup:
+    steps: [{run: echo}]
+  build:
+    %s
+    strategy:
+      matrix:
+        os: [a, b]
+        version: %s
+    steps: [{run: echo}]
+`
+	for _, tt := range []struct {
+		name     string
+		needs    string
+		version  string
+		deferred bool
+		want     int
+	}{
+		{"needs outputs", "needs: setup", "${{ fromJson(needs.setup.outputs.v) }}", true, 1},
+		{"static", "needs: setup", "[1, 2]", false, 4},
+		// Without needs there is nothing to resolve the expression from later, so deferring would
+		// strand the job as a single combination that never expands.
+		{"expression without needs", "", `["${{ github.sha }}"]`, false, 2},
+		// A context that is already available while planning must keep expanding there, otherwise
+		// such a workflow would silently lose the per-combination commit statuses it used to create.
+		{"expression over another context", "needs: setup", `["${{ github.sha }}"]`, false, 2},
+		// The needs context is looked up in the parsed expression, not in the raw text.
+		{"needs inside a string literal", "needs: setup", `["${{ format('needs.setup.outputs.v {0}', github.sha) }}"]`, false, 2},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := Parse(fmt.Appendf(nil, workflow, tt.needs, tt.version))
+			require.NoError(t, err)
+
+			var builds []*Job
+			for _, w := range result {
+				if id, job := w.Job(); id == "build" {
+					builds = append(builds, job)
+				}
+			}
+			require.Len(t, builds, tt.want)
+			assert.Equal(t, tt.deferred, HasDeferredMatrix(builds[0]))
+		})
+	}
+}
+
+func TestExpandMatrixWithNeeds(t *testing.T) {
+	// matrixYAML is the YAML value of the `matrix:` key, so a case can replace the whole node.
+	expandMax := func(t *testing.T, matrixYAML string, maxCombinations int) ([]*Job, error) {
+		t.Helper()
+		var strategy Strategy
+		require.NoError(t, yaml.Unmarshal([]byte("matrix:"+matrixYAML), &strategy))
+		job := &Job{Name: "build", Strategy: strategy}
+		require.NoError(t, job.RawRunsOn.Encode("${{ matrix.os || 'ubuntu-latest' }}"))
+		require.NoError(t, job.RawNeeds.Encode([]string{"setup"}))
+		// The results map must describe the job itself too, as findJobNeedsAndFillJobResults does.
+		return ExpandMatrixWithNeeds("build", job, &model.GithubContext{}, map[string]*JobResult{
+			"build": {Needs: []string{"setup"}},
+			"setup": {Result: "success", Outputs: map[string]string{
+				"versions": `["1.20", "1.21"]`,
+				"os":       `["linux", "darwin"]`,
+				"include":  `[{"os":"linux","fast":true},{"os":"windows","fast":false}]`,
+				"empty":    "[]",
+			}},
+		}, nil, nil, maxCombinations)
+	}
+	expand := func(t *testing.T, matrixYAML string) ([]*Job, error) {
+		t.Helper()
+		return expandMax(t, matrixYAML, 256)
+	}
+
+	t.Run("expands the product and interpolates runs-on", func(t *testing.T) {
+		got, err := expand(t, "\n  os: ${{ fromJson(needs.setup.outputs.os) }}\n  version: ${{ fromJson(needs.setup.outputs.versions) }}\n")
+		require.NoError(t, err)
+		names := make([]string, 0, len(got))
+		for _, combo := range got {
+			names = append(names, combo.Name)
+			assert.Contains(t, []string{"linux", "darwin"}, combo.RunsOn()[0])
+		}
+		// Dimensions are appended in key order, as GitHub names multi-dimension combinations.
+		assert.ElementsMatch(t, []string{
+			"build (linux, 1.20)", "build (linux, 1.21)", "build (darwin, 1.20)", "build (darwin, 1.21)",
+		}, names)
+	})
+
+	t.Run("static and dynamic dimensions expand together, once", func(t *testing.T) {
+		got, err := expand(t, "\n  os: [linux, darwin]\n  version: ${{ fromJson(needs.setup.outputs.versions) }}\n")
+		require.NoError(t, err)
+		assert.Len(t, got, 4)
+	})
+
+	t.Run("include-only matrix expands", func(t *testing.T) {
+		got, err := expand(t, "\n  include: ${{ fromJson(needs.setup.outputs.include) }}\n")
+		require.NoError(t, err)
+		assert.Len(t, got, 2)
+	})
+
+	// GitHub rejects a matrix that yields no combinations instead of running the job unparameterized.
+	for _, tt := range []struct{ name, matrix string }{
+		{"empty vector", "\n  version: ${{ fromJson(needs.setup.outputs.empty) }}\n"},
+		{"empty include", "\n  include: ${{ fromJson(needs.setup.outputs.empty) }}\n"},
+		{"whole matrix not a mapping", " ${{ fromJson(needs.setup.outputs.empty) }}\n"},
+	} {
+		t.Run(tt.name+" errors", func(t *testing.T) {
+			_, err := expand(t, tt.matrix)
+			require.ErrorContains(t, err, "matrix must define at least one vector")
+		})
+	}
+
+	t.Run("unresolved need errors", func(t *testing.T) {
+		_, err := expand(t, "\n  v: ${{ fromJson(needs.missing.outputs.v) }}\n")
+		require.ErrorContains(t, err, "evaluate matrix")
+	})
+
+	// The combination count comes from a runtime output, so it must be rejected before one Job per
+	// combination is built rather than after.
+	t.Run("too many combinations errors", func(t *testing.T) {
+		_, err := expandMax(t, "\n  version: ${{ fromJson(needs.setup.outputs.versions) }}\n", 1)
+		require.ErrorContains(t, err, "exceeding the limit of 1")
+	})
+}
+
+func TestRejectsUnevaluatedMatrixFilters(t *testing.T) {
+	// act dereferences include/exclude entries as mappings without checking, so an unevaluated
+	// expression panics there. Every entry point into act's matrix expansion must reject it: Parse
+	// sees one in a workflow file and in a deferred placeholder's payload, and the emitter reads a
+	// placeholder's `if:` while its matrix is still raw.
+	for _, filter := range []string{"include", "exclude"} {
+		t.Run(filter, func(t *testing.T) {
+			_, err := Parse(fmt.Appendf(nil,
+				"name: t\non: push\njobs:\n  build:\n    runs-on: ubuntu-latest\n    strategy:\n      matrix:\n        os: [a]\n        %s: ${{ fromJson(vars.MATRIX) }}\n    steps: [{run: echo}]\n", filter))
+			require.ErrorContains(t, err, "must be a list of mappings")
+
+			var strategy Strategy
+			require.NoError(t, yaml.Unmarshal(fmt.Appendf(nil, "matrix:\n  os: [a]\n  %s: ${{ fromJson(vars.MATRIX) }}\n", filter), &strategy))
+			job := &Job{Name: "build", Strategy: strategy}
+			require.NoError(t, job.If.Encode("${{ true }}"))
+			_, err = EvaluateJobIfExpression("build", job, map[string]any{}, map[string]*JobResult{"build": {}}, nil, nil)
+			require.ErrorContains(t, err, "must be a list of mappings")
+		})
+	}
+}

modules/actions/jobparser/model.go

@@ -269,7 +269,7 @@ func EvaluateConcurrency(rc *model.RawConcurrency, jobID string, job *Job, gitCt
 	}
 
 	matrix := make(map[string]any)
-	matrixes, err := actJob.GetMatrixes()
+	matrixes, err := matrixesOf(actJob)
 	if err != nil {
 		return "", false, err
 	}
@@ -510,7 +510,7 @@ func EvaluateJobIfExpression(jobID string, job *Job, gitCtx map[string]any, resu
 	// GetMatrixes always returns at least one element (an empty map for a job without a matrix),
 	// so only a non-empty combination should populate `matrix.*`, leaving it nil otherwise.
 	var matrix map[string]any
-	matrixes, err := actJob.GetMatrixes()
+	matrixes, err := matrixesOf(actJob)
 	if err != nil {
 		return false, err
 	}

modules/actions/jobparser/workflow_call.go

@@ -177,7 +177,7 @@ func EvaluateCallerWith(
 	}}
 
 	var matrix map[string]any
-	matrixes, err := actJob.GetMatrixes()
+	matrixes, err := matrixesOf(actJob)
 	if err != nil {
 		return nil, fmt.Errorf("get caller %q matrix: %w", jobID, err)
 	}