Add a doctor command to fix inconsistent run status (#35840) (#35845)

Backport #35840

#35783 fixes an actions rerun bug. Due to this bug, some runs may be
incorrectly marked as `StatusWaiting` even though all the jobs are in
done status. These runs cannot be run or cancelled. This PR adds a new
doctor command to fix the inconsistent run status.

```
gitea doctor check --run fix-actions-unfinished-run-status --fix
```

models/actions/run_test.go

@@ -30,6 +30,6 @@ func TestUpdateRepoRunsNumbers(t *testing.T) {
 	err = updateRepoRunsNumbers(t.Context(), repo)
 	assert.NoError(t, err)
 	repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 4})
-	assert.Equal(t, 4, repo.NumActionRuns)
+	assert.Equal(t, 5, repo.NumActionRuns)
 	assert.Equal(t, 3, repo.NumClosedActionRuns)
 }

models/fixtures/action_run.yml

@@ -139,3 +139,24 @@
   updated: 1683636626
   need_approval: 0
   approved_by: 0
+
+-
+  id: 796
+  title: "update actions"
+  repo_id: 4
+  owner_id: 1
+  workflow_id: "artifact.yaml"
+  index: 191
+  trigger_user_id: 1
+  ref: "refs/heads/master"
+  commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
+  event: "push"
+  trigger_event: "push"
+  is_fork_pull_request: 0
+  status: 5
+  started: 1683636528
+  stopped: 1683636626
+  created: 1683636108
+  updated: 1683636626
+  need_approval: 0
+  approved_by: 0

models/fixtures/action_run_job.yml

@@ -129,3 +129,18 @@
   status: 5
   started: 1683636528
   stopped: 1683636626
+
+-
+  id: 205
+  run_id: 796
+  repo_id: 4
+  owner_id: 1
+  commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
+  is_fork_pull_request: 0
+  name: job_2
+  attempt: 1
+  job_id: job_2
+  task_id: 55
+  status: 3
+  started: 1683636528
+  stopped: 1683636626

models/fixtures/action_task.yml

@@ -177,3 +177,24 @@
   log_length: 0
   log_size: 0
   log_expired: 0
+
+-
+  id: 55
+  job_id: 205
+  attempt: 1
+  runner_id: 1
+  status: 3 # 3 is the status code for "cancelled"
+  started: 1683636528
+  stopped: 1683636626
+  repo_id: 4
+  owner_id: 1
+  commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
+  is_fork_pull_request: 0
+  token_hash: 6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4240c64a69a2cc1508825121b7b8394e48e00b1bf3718b2aaaab
+  token_salt: eeeeeeee
+  token_last_eight: eeeeeeee
+  log_filename: artifact-test2/2f/47.log
+  log_in_storage: 1
+  log_length: 707
+  log_size: 90179
+  log_expired: 0

models/fixtures/repo_unit.yml

@@ -733,3 +733,10 @@
   type: 3
   config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}"
   created_unix: 946684810
+
+-
+  id: 111
+  repo_id: 4
+  type: 10
+  config: "{}"
+  created_unix: 946684810

services/doctor/actions.go

@@ -7,12 +7,17 @@ import (
 	"context"
 	"fmt"
 
+	actions_model "code.gitea.io/gitea/models/actions"
 	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	unit_model "code.gitea.io/gitea/models/unit"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/optional"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/timeutil"
 	repo_service "code.gitea.io/gitea/services/repository"
+
+	"xorm.io/builder"
 )
 
 func disableMirrorActionsUnit(ctx context.Context, logger log.Logger, autofix bool) error {
@@ -59,6 +64,95 @@ func disableMirrorActionsUnit(ctx context.Context, logger log.Logger, autofix bo
 	return nil
 }
 
+func fixUnfinishedRunStatus(ctx context.Context, logger log.Logger, autofix bool) error {
+	total := 0
+	inconsistent := 0
+	fixed := 0
+
+	cond := builder.In("status", []actions_model.Status{
+		actions_model.StatusWaiting,
+		actions_model.StatusRunning,
+		actions_model.StatusBlocked,
+	}).And(builder.Lt{"updated": timeutil.TimeStampNow().AddDuration(-setting.Actions.ZombieTaskTimeout)})
+
+	err := db.Iterate(
+		ctx,
+		cond,
+		func(ctx context.Context, run *actions_model.ActionRun) error {
+			total++
+
+			jobs, err := actions_model.GetRunJobsByRunID(ctx, run.ID)
+			if err != nil {
+				return fmt.Errorf("GetRunJobsByRunID: %w", err)
+			}
+			expected := actions_model.AggregateJobStatus(jobs)
+			if expected == run.Status {
+				return nil
+			}
+
+			inconsistent++
+			logger.Warn("Run %d (repo_id=%d, index=%d) has status %s, expected %s", run.ID, run.RepoID, run.Index, run.Status, expected)
+
+			if !autofix {
+				return nil
+			}
+
+			run.Started, run.Stopped = getRunTimestampsFromJobs(run, expected, jobs)
+			run.Status = expected
+
+			if err := actions_model.UpdateRun(ctx, run, "status", "started", "stopped"); err != nil {
+				return fmt.Errorf("UpdateRun: %w", err)
+			}
+			fixed++
+
+			return nil
+		},
+	)
+	if err != nil {
+		logger.Critical("Unable to iterate unfinished runs: %v", err)
+		return err
+	}
+
+	if inconsistent == 0 {
+		logger.Info("Checked %d unfinished runs; all statuses are consistent.", total)
+		return nil
+	}
+
+	if autofix {
+		logger.Info("Checked %d unfinished runs; fixed %d of %d runs.", total, fixed, inconsistent)
+	} else {
+		logger.Warn("Checked %d unfinished runs; found %d runs need to be fixed", total, inconsistent)
+	}
+
+	return nil
+}
+
+func getRunTimestampsFromJobs(run *actions_model.ActionRun, newStatus actions_model.Status, jobs actions_model.ActionJobList) (started, stopped timeutil.TimeStamp) {
+	started = run.Started
+	if (newStatus.IsRunning() || newStatus.IsDone()) && started.IsZero() {
+		var earliest timeutil.TimeStamp
+		for _, job := range jobs {
+			if job.Started > 0 && (earliest.IsZero() || job.Started < earliest) {
+				earliest = job.Started
+			}
+		}
+		started = earliest
+	}
+
+	stopped = run.Stopped
+	if newStatus.IsDone() && stopped.IsZero() {
+		var latest timeutil.TimeStamp
+		for _, job := range jobs {
+			if job.Stopped > latest {
+				latest = job.Stopped
+			}
+		}
+		stopped = latest
+	}
+
+	return started, stopped
+}
+
 func init() {
 	Register(&Check{
 		Title:     "Disable the actions unit for all mirrors",
@@ -67,4 +161,11 @@ func init() {
 		Run:       disableMirrorActionsUnit,
 		Priority:  9,
 	})
+	Register(&Check{
+		Title:     "Fix inconsistent status for unfinished actions runs",
+		Name:      "fix-actions-unfinished-run-status",
+		IsDefault: false,
+		Run:       fixUnfinishedRunStatus,
+		Priority:  9,
+	})
 }

services/doctor/actions_test.go

@@ -0,0 +1,24 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"testing"
+
+	actions_model "code.gitea.io/gitea/models/actions"
+	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_fixUnfinishedRunStatus(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	fixUnfinishedRunStatus(t.Context(), log.GetLogger(log.DEFAULT), true)
+
+	// check if the run is cancelled by id
+	run := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: 796})
+	assert.Equal(t, actions_model.StatusCancelled, run.Status)
+}