feat(ci): trigger, plan and run workflows, and block merges on checks

Add the CI use cases: ports.WorkflowTrigger.OnPush (called by
Pushes.Record after Refresh, and by a merge) only queues work for a
bounded worker; the worker reads .github/workflows at the pushed sha,
parses and filters them with ermite-actions (changed files from the diff,
300 and 1000 limits, tags, skip directives) and records invalid files as
failed runs without a job. The planner builds the job graph, evaluates
job conditions with status functions, expands matrices when a job is
ready, applies fail-fast and max-parallel and collects outputs for
needs. The runner protocol (registration, long-poll assignment with
poll ids and acknowledgement by the first heartbeat, step updates, log
append, watch, idempotent completion) and the owner side (runs, jobs,
log streaming, cancel, re-run, checks, runners) are implemented over the
CIStore and CILogStore ports, with a watchdog for acknowledgement,
heartbeat, cancel grace, timeouts, queue timeout and the 90-day sweep.
MergeRequests.merge now refuses failing or pending checks under
FailingChecksBlock. ERMITE_OWNER and ERMITE_CI_LOG_RETENTION are added
to the configuration.

commit: 29650181544575e1adecaa5c7c6907dc570c4ff5
author: brice · 9d ago · 2026-10-12 18:13 UTC
parent: f17c3d9
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

18 files · +3500 -92

M gen/ermite/storage/ci.pb.go +16 -4 generated file: collapsed

load diff

M internal/adapters/kvstore/ci.go +6 -6


@@ -26,7 +26,7 @@ import (
26	26		// run/<repo>/<id:020d> CIRun
27	27		// runsrc/<id:020d> CIRunSource
28	28		// job/<run:020d>/<attempt:010d>/<id> CIJob
29		-	// check/<repo>/<sha>/<job id> CICheck
	29	+	// check/<repo>/<sha>/<check id> CICheck
30	30		// runseq Sequence: last run id
31	31		// runnum/<repo>/<workflow path> Sequence: last run number
32	32		// runner/<id>, runnername/<name>, runnertoken/<hash>

@@ -83,8 +83,8 @@ func jobKey(runID int64, attempt int32, id string) []byte { return []byte(jobRef
83	83		func checksPrefix(repo domain.RepoName, sha string) string {
84	84		return checkPrefix + repo.String() + "/" + sha + "/"
85	85		}
86		-	func checkKey(repo domain.RepoName, sha, jobID string) []byte {
87		-	return []byte(checksPrefix(repo, sha) + jobID)
	86	+	func checkKey(repo domain.RepoName, sha, id string) []byte {
	87	+	return []byte(checksPrefix(repo, sha) + id)
88	88		}
89	89		func runNumKey(repo domain.RepoName, workflow string) []byte {
90	90		return []byte(runNumPrefix + repo.String() + "/" + workflow)

@@ -294,7 +294,7 @@ func encodeJob(j domain.CIJob) ([]byte, error) {
294	294		Labels: j.Labels, Needs: j.Needs, RunnerId: j.RunnerID, RunnerName: j.RunnerName,
295	295		Isolation: encodeIsolation(j.Isolation), Steps: steps,
296	296		CreatedAt: encodeTime(j.CreatedAt), StartedAt: encodeTime(j.StartedAt), CompletedAt: encodeTime(j.CompletedAt),
297		-	Index: int32(j.Index), Combo: int32(j.Combo), MatrixJson: j.MatrixJSON, ComboTotal: int32(j.ComboTotal), //nolint:gosec // bounded by the 256 job limit
	297	+	Index: int32(j.Index), Combo: int32(j.Combo), MatrixJson: j.MatrixJSON, ContextsJson: j.ContextsJSON, ComboTotal: int32(j.ComboTotal), //nolint:gosec // bounded by the 256 job limit
298	298		TimeoutMinutes: j.TimeoutMinutes, FailFast: j.FailFast, MaxParallel: int32(j.MaxParallel), //nolint:gosec // validated at planning
299	299		Outputs: j.Outputs, ReadyAt: encodeTime(j.ReadyAt), AssignedAt: encodeTime(j.AssignedAt),
300	300		AckedAt: encodeTime(j.AckedAt), AssignCancels: int32(j.AssignCancels), //nolint:gosec // at most 3

@@ -323,7 +323,7 @@ func decodeJob(item kv.Item) (domain.CIJob, error) {
323	323		Error: pb.GetError(), Labels: pb.GetLabels(), Needs: pb.GetNeeds(), RunnerID: pb.GetRunnerId(), RunnerName: pb.GetRunnerName(),
324	324		Isolation: decodeIsolation(pb.GetIsolation()),
325	325		CreatedAt: decodeTime(pb.GetCreatedAt()), StartedAt: decodeTime(pb.GetStartedAt()), CompletedAt: decodeTime(pb.GetCompletedAt()),
326		-	Index: int(pb.GetIndex()), Combo: int(pb.GetCombo()), Planned: pb.GetPlanned(), MatrixJSON: pb.GetMatrixJson(),
	326	+	Index: int(pb.GetIndex()), Combo: int(pb.GetCombo()), Planned: pb.GetPlanned(), MatrixJSON: pb.GetMatrixJson(), ContextsJSON: pb.GetContextsJson(),
327	327		ComboTotal: int(pb.GetComboTotal()), TimeoutMinutes: pb.GetTimeoutMinutes(), FailFast: pb.GetFailFast(),
328	328		MaxParallel: int(pb.GetMaxParallel()), Outputs: pb.GetOutputs(),
329	329		TokenValidUntil: decodeTime(pb.GetTokenValidUntil()), ReadyAt: decodeTime(pb.GetReadyAt()),

@@ -490,7 +490,7 @@ func (c CI) stageRecords(ctx context.Context, b kv.Batch, w ports.CIWrite) (gu
490	490		// Checks are derived from the jobs, written in the same batch and under
491	491		// the lock of their run: they are upserted without a precondition.
492	492		for _, ck := range w.Checks {
493		-	b.Put(checkKey(ck.Repo, ck.Sha, ck.JobID), encodeCheck(ck))
	493	+	b.Put(checkKey(ck.Repo, ck.Sha, ck.ID), encodeCheck(ck))
494	494		}
495	495		return guarded, nil
496	496		}


@@ -26,7 +26,7 @@ import (
26		// run/<repo>/<id:020d> CIRun	26		// run/<repo>/<id:020d> CIRun
27		// runsrc/<id:020d> CIRunSource	27		// runsrc/<id:020d> CIRunSource
28		// job/<run:020d>/<attempt:010d>/<id> CIJob	28		// job/<run:020d>/<attempt:010d>/<id> CIJob
29	-	// check/<repo>/<sha>/<job id> CICheck	29	+	// check/<repo>/<sha>/<check id> CICheck
30		// runseq Sequence: last run id	30		// runseq Sequence: last run id
31		// runnum/<repo>/<workflow path> Sequence: last run number	31		// runnum/<repo>/<workflow path> Sequence: last run number
32		// runner/<id>, runnername/<name>, runnertoken/<hash>	32		// runner/<id>, runnername/<name>, runnertoken/<hash>

@@ -83,8 +83,8 @@ func jobKey(runID int64, attempt int32, id string) []byte { return []byte(jobRef
83		func checksPrefix(repo domain.RepoName, sha string) string {	83		func checksPrefix(repo domain.RepoName, sha string) string {
84		return checkPrefix + repo.String() + "/" + sha + "/"	84		return checkPrefix + repo.String() + "/" + sha + "/"
85		}	85		}
86	-	func checkKey(repo domain.RepoName, sha, jobID string) []byte {	86	+	func checkKey(repo domain.RepoName, sha, id string) []byte {
87	-	return []byte(checksPrefix(repo, sha) + jobID)	87	+	return []byte(checksPrefix(repo, sha) + id)
88		}	88		}
89		func runNumKey(repo domain.RepoName, workflow string) []byte {	89		func runNumKey(repo domain.RepoName, workflow string) []byte {
90		return []byte(runNumPrefix + repo.String() + "/" + workflow)	90		return []byte(runNumPrefix + repo.String() + "/" + workflow)

@@ -294,7 +294,7 @@ func encodeJob(j domain.CIJob) ([]byte, error) {
294		Labels: j.Labels, Needs: j.Needs, RunnerId: j.RunnerID, RunnerName: j.RunnerName,	294		Labels: j.Labels, Needs: j.Needs, RunnerId: j.RunnerID, RunnerName: j.RunnerName,
295		Isolation: encodeIsolation(j.Isolation), Steps: steps,	295		Isolation: encodeIsolation(j.Isolation), Steps: steps,
296		CreatedAt: encodeTime(j.CreatedAt), StartedAt: encodeTime(j.StartedAt), CompletedAt: encodeTime(j.CompletedAt),	296		CreatedAt: encodeTime(j.CreatedAt), StartedAt: encodeTime(j.StartedAt), CompletedAt: encodeTime(j.CompletedAt),
297	-	Index: int32(j.Index), Combo: int32(j.Combo), MatrixJson: j.MatrixJSON, ComboTotal: int32(j.ComboTotal), //nolint:gosec // bounded by the 256 job limit	297	+	Index: int32(j.Index), Combo: int32(j.Combo), MatrixJson: j.MatrixJSON, ContextsJson: j.ContextsJSON, ComboTotal: int32(j.ComboTotal), //nolint:gosec // bounded by the 256 job limit
298		TimeoutMinutes: j.TimeoutMinutes, FailFast: j.FailFast, MaxParallel: int32(j.MaxParallel), //nolint:gosec // validated at planning	298		TimeoutMinutes: j.TimeoutMinutes, FailFast: j.FailFast, MaxParallel: int32(j.MaxParallel), //nolint:gosec // validated at planning
299		Outputs: j.Outputs, ReadyAt: encodeTime(j.ReadyAt), AssignedAt: encodeTime(j.AssignedAt),	299		Outputs: j.Outputs, ReadyAt: encodeTime(j.ReadyAt), AssignedAt: encodeTime(j.AssignedAt),
300		AckedAt: encodeTime(j.AckedAt), AssignCancels: int32(j.AssignCancels), //nolint:gosec // at most 3	300		AckedAt: encodeTime(j.AckedAt), AssignCancels: int32(j.AssignCancels), //nolint:gosec // at most 3

@@ -323,7 +323,7 @@ func decodeJob(item kv.Item) (domain.CIJob, error) {
323		Error: pb.GetError(), Labels: pb.GetLabels(), Needs: pb.GetNeeds(), RunnerID: pb.GetRunnerId(), RunnerName: pb.GetRunnerName(),	323		Error: pb.GetError(), Labels: pb.GetLabels(), Needs: pb.GetNeeds(), RunnerID: pb.GetRunnerId(), RunnerName: pb.GetRunnerName(),
324		Isolation: decodeIsolation(pb.GetIsolation()),	324		Isolation: decodeIsolation(pb.GetIsolation()),
325		CreatedAt: decodeTime(pb.GetCreatedAt()), StartedAt: decodeTime(pb.GetStartedAt()), CompletedAt: decodeTime(pb.GetCompletedAt()),	325		CreatedAt: decodeTime(pb.GetCreatedAt()), StartedAt: decodeTime(pb.GetStartedAt()), CompletedAt: decodeTime(pb.GetCompletedAt()),
326	-	Index: int(pb.GetIndex()), Combo: int(pb.GetCombo()), Planned: pb.GetPlanned(), MatrixJSON: pb.GetMatrixJson(),	326	+	Index: int(pb.GetIndex()), Combo: int(pb.GetCombo()), Planned: pb.GetPlanned(), MatrixJSON: pb.GetMatrixJson(), ContextsJSON: pb.GetContextsJson(),
327		ComboTotal: int(pb.GetComboTotal()), TimeoutMinutes: pb.GetTimeoutMinutes(), FailFast: pb.GetFailFast(),	327		ComboTotal: int(pb.GetComboTotal()), TimeoutMinutes: pb.GetTimeoutMinutes(), FailFast: pb.GetFailFast(),
328		MaxParallel: int(pb.GetMaxParallel()), Outputs: pb.GetOutputs(),	328		MaxParallel: int(pb.GetMaxParallel()), Outputs: pb.GetOutputs(),
329		TokenValidUntil: decodeTime(pb.GetTokenValidUntil()), ReadyAt: decodeTime(pb.GetReadyAt()),	329		TokenValidUntil: decodeTime(pb.GetTokenValidUntil()), ReadyAt: decodeTime(pb.GetReadyAt()),

@@ -490,7 +490,7 @@ func (c CI) stageRecords(ctx context.Context, b kv.Batch, w ports.CIWrite) (gu
490		// Checks are derived from the jobs, written in the same batch and under	490		// Checks are derived from the jobs, written in the same batch and under
491		// the lock of their run: they are upserted without a precondition.	491		// the lock of their run: they are upserted without a precondition.
492		for _, ck := range w.Checks {	492		for _, ck := range w.Checks {
493	-	b.Put(checkKey(ck.Repo, ck.Sha, ck.JobID), encodeCheck(ck))	493	+	b.Put(checkKey(ck.Repo, ck.Sha, ck.ID), encodeCheck(ck))
494		}	494		}
495		return guarded, nil	495		return guarded, nil
496		}	496		}

M internal/config/config.go +71 -0


@@ -14,6 +14,7 @@ import (
14	14		"path/filepath"
15	15		"strconv"
16	16		"strings"
	17	+	"time"
17	18		"unicode/utf8"
18	19		)
19	20

@@ -33,6 +34,11 @@ const (
33	34		// commits ermite creates (squash merges, ADR 0022).
34	35		EnvCommitterName = "ERMITE_COMMITTER_NAME"
35	36		EnvCommitterEmail = "ERMITE_COMMITTER_EMAIL"
	37	+	// EnvOwner is the owner login CI shows in the github context
	38	+	// ("<owner>/<repo>"); it follows the rules of repository names.
	39	+	EnvOwner = "ERMITE_OWNER"
	40	+	// EnvCILogRetention is how long CI runs, jobs, checks and logs are kept.
	41	+	EnvCILogRetention = "ERMITE_CI_LOG_RETENTION"
36	42		)
37	43
38	44		// Defaults.

@@ -44,6 +50,10 @@ const (
44	50		DefaultLogFormat = "json"
45	51		// DefaultCommitterName is the committer name of the commits ermite creates.
46	52		DefaultCommitterName = "ermite"
	53	+	// DefaultOwner is the owner login when ERMITE_OWNER is not set.
	54	+	DefaultOwner = "owner"
	55	+	// DefaultCIRetention is the retention of CI data.
	56	+	DefaultCIRetention = 90 * 24 * time.Hour
47	57		)
48	58
49	59		// Values of ERMITE_GIT_READER.

@@ -85,6 +95,10 @@ type Config struct {
85	95		// commits ermite creates. The email defaults to "ermite@<host of
86	96		// PublicURL>".
87	97		CommitterName, CommitterEmail string
	98	+	// Owner is the owner login of the github context of CI runs.
	99	+	Owner string
	100	+	// CIRetention is how long CI data is kept.
	101	+	CIRetention time.Duration
88	102		}
89	103
90	104		// Error lists every configuration problem found, in variable order.

@@ -218,12 +232,69 @@ func Load(get func(string) string) (Config, error) {
218	232		c.CommitterEmail = "ermite@" + u.Hostname()
219	233		}
220	234
	235	+	c.Owner = strings.TrimSpace(get(EnvOwner))
	236	+	if c.Owner == "" {
	237	+	c.Owner = DefaultOwner
	238	+	} else if err := validateOwner(c.Owner); err != nil {
	239	+	bad(EnvOwner, "%v", err)
	240	+	}
	241	+
	242	+	c.CIRetention = DefaultCIRetention
	243	+	if raw := strings.TrimSpace(get(EnvCILogRetention)); raw != "" {
	244	+	d, err := parseRetention(raw)
	245	+	if err != nil {
	246	+	bad(EnvCILogRetention, "%v", err)
	247	+	} else {
	248	+	c.CIRetention = d
	249	+	}
	250	+	}
	251	+
221	252		if len(problems) > 0 {
222	253		return Config{}, &Error{Problems: problems}
223	254		}
224	255		return c, nil
225	256		}
226	257
	258	+	// validateOwner applies the rules of repository names (lowercase letters,
	259	+	// digits, '.', '_' and '-', at most 100 characters, no leading or trailing
	260	+	// '.'), which config cannot import from the domain.
	261	+	func validateOwner(s string) error {
	262	+	if len(s) > 100 {
	263	+	return fmt.Errorf("must be at most 100 characters")
	264	+	}
	265	+	for _, r := range s {
	266	+	if (r < 'a' \|\| r > 'z') && (r < '0' \|\| r > '9') && r != '.' && r != '_' && r != '-' {
	267	+	return fmt.Errorf("must follow the rules of repository names: a-z, 0-9, '.', '_', '-' (got %q)", s)
	268	+	}
	269	+	}
	270	+	if strings.HasPrefix(s, ".") \|\| strings.HasSuffix(s, ".") \|\| strings.Contains(s, "..") {
	271	+	return fmt.Errorf("must not start or end with '.' nor contain '..' (got %q)", s)
	272	+	}
	273	+	return nil
	274	+	}
	275	+
	276	+	// parseRetention reads a duration in days ("90d") or in Go's syntax ("720h"),
	277	+	// at least one hour.
	278	+	func parseRetention(raw string) (time.Duration, error) {
	279	+	var d time.Duration
	280	+	if days, ok := strings.CutSuffix(raw, "d"); ok {
	281	+	n, err := strconv.Atoi(days)
	282	+	if err != nil \|\| n < 1 \|\| n > 36500 {
	283	+	return 0, fmt.Errorf("must be a number of days such as 90d (got %q)", raw)
	284	+	}
	285	+	d = time.Duration(n) * 24 * time.Hour
	286	+	} else {
	287	+	var err error
	288	+	if d, err = time.ParseDuration(raw); err != nil {
	289	+	return 0, fmt.Errorf("must be a duration such as 90d or 720h (got %q)", raw)
	290	+	}
	291	+	}
	292	+	if d < time.Hour {
	293	+	return 0, fmt.Errorf("must be at least one hour (got %q)", raw)
	294	+	}
	295	+	return d, nil
	296	+	}
	297	+
227	298		// validateIdentity accepts a name or an email address for a Git identity: valid
228	299		// UTF-8 without control characters or the angle brackets and line breaks that
229	300		// Git does not allow in an identity.


@@ -14,6 +14,7 @@ import (
14	"path/filepath"	14		"path/filepath"
15	"strconv"	15		"strconv"
16	"strings"	16		"strings"
		17	+	"time"
17	"unicode/utf8"	18		"unicode/utf8"
18	)	19		)
19		20

@@ -33,6 +34,11 @@ const (
33	// commits ermite creates (squash merges, ADR 0022).	34		// commits ermite creates (squash merges, ADR 0022).
34	EnvCommitterName = "ERMITE_COMMITTER_NAME"	35		EnvCommitterName = "ERMITE_COMMITTER_NAME"
35	EnvCommitterEmail = "ERMITE_COMMITTER_EMAIL"	36		EnvCommitterEmail = "ERMITE_COMMITTER_EMAIL"
		37	+	// EnvOwner is the owner login CI shows in the github context
		38	+	// ("<owner>/<repo>"); it follows the rules of repository names.
		39	+	EnvOwner = "ERMITE_OWNER"
		40	+	// EnvCILogRetention is how long CI runs, jobs, checks and logs are kept.
		41	+	EnvCILogRetention = "ERMITE_CI_LOG_RETENTION"
36	)	42		)
37		43
38	// Defaults.	44		// Defaults.

@@ -44,6 +50,10 @@ const (
44	DefaultLogFormat = "json"	50		DefaultLogFormat = "json"
45	// DefaultCommitterName is the committer name of the commits ermite creates.	51		// DefaultCommitterName is the committer name of the commits ermite creates.
46	DefaultCommitterName = "ermite"	52		DefaultCommitterName = "ermite"
		53	+	// DefaultOwner is the owner login when ERMITE_OWNER is not set.
		54	+	DefaultOwner = "owner"
		55	+	// DefaultCIRetention is the retention of CI data.
		56	+	DefaultCIRetention = 90 * 24 * time.Hour
47	)	57		)
48		58
49	// Values of ERMITE_GIT_READER.	59		// Values of ERMITE_GIT_READER.

@@ -85,6 +95,10 @@ type Config struct {
85	// commits ermite creates. The email defaults to "ermite@<host of	95		// commits ermite creates. The email defaults to "ermite@<host of
86	// PublicURL>".	96		// PublicURL>".
87	CommitterName, CommitterEmail string	97		CommitterName, CommitterEmail string
		98	+	// Owner is the owner login of the github context of CI runs.
		99	+	Owner string
		100	+	// CIRetention is how long CI data is kept.
		101	+	CIRetention time.Duration
88	}	102		}
89		103
90	// Error lists every configuration problem found, in variable order.	104		// Error lists every configuration problem found, in variable order.

@@ -218,12 +232,69 @@ func Load(get func(string) string) (Config, error) {
218	c.CommitterEmail = "ermite@" + u.Hostname()	232		c.CommitterEmail = "ermite@" + u.Hostname()
219	}	233		}
220		234
		235	+	c.Owner = strings.TrimSpace(get(EnvOwner))
		236	+	if c.Owner == "" {
		237	+	c.Owner = DefaultOwner
		238	+	} else if err := validateOwner(c.Owner); err != nil {
		239	+	bad(EnvOwner, "%v", err)
		240	+	}
		241	+
		242	+	c.CIRetention = DefaultCIRetention
		243	+	if raw := strings.TrimSpace(get(EnvCILogRetention)); raw != "" {
		244	+	d, err := parseRetention(raw)
		245	+	if err != nil {
		246	+	bad(EnvCILogRetention, "%v", err)
		247	+	} else {
		248	+	c.CIRetention = d
		249	+	}
		250	+	}
		251	+
221	if len(problems) > 0 {	252		if len(problems) > 0 {
222	return Config{}, &Error{Problems: problems}	253		return Config{}, &Error{Problems: problems}
223	}	254		}
224	return c, nil	255		return c, nil
225	}	256		}
226		257
		258	+	// validateOwner applies the rules of repository names (lowercase letters,
		259	+	// digits, '.', '_' and '-', at most 100 characters, no leading or trailing
		260	+	// '.'), which config cannot import from the domain.
		261	+	func validateOwner(s string) error {
		262	+	if len(s) > 100 {
		263	+	return fmt.Errorf("must be at most 100 characters")
		264	+	}
		265	+	for _, r := range s {
		266	+	if (r < 'a' \|\| r > 'z') && (r < '0' \|\| r > '9') && r != '.' && r != '_' && r != '-' {
		267	+	return fmt.Errorf("must follow the rules of repository names: a-z, 0-9, '.', '_', '-' (got %q)", s)
		268	+	}
		269	+	}
		270	+	if strings.HasPrefix(s, ".") \|\| strings.HasSuffix(s, ".") \|\| strings.Contains(s, "..") {
		271	+	return fmt.Errorf("must not start or end with '.' nor contain '..' (got %q)", s)
		272	+	}
		273	+	return nil
		274	+	}
		275	+
		276	+	// parseRetention reads a duration in days ("90d") or in Go's syntax ("720h"),
		277	+	// at least one hour.
		278	+	func parseRetention(raw string) (time.Duration, error) {
		279	+	var d time.Duration
		280	+	if days, ok := strings.CutSuffix(raw, "d"); ok {
		281	+	n, err := strconv.Atoi(days)
		282	+	if err != nil \|\| n < 1 \|\| n > 36500 {
		283	+	return 0, fmt.Errorf("must be a number of days such as 90d (got %q)", raw)
		284	+	}
		285	+	d = time.Duration(n) * 24 * time.Hour
		286	+	} else {
		287	+	var err error
		288	+	if d, err = time.ParseDuration(raw); err != nil {
		289	+	return 0, fmt.Errorf("must be a duration such as 90d or 720h (got %q)", raw)
		290	+	}
		291	+	}
		292	+	if d < time.Hour {
		293	+	return 0, fmt.Errorf("must be at least one hour (got %q)", raw)
		294	+	}
		295	+	return d, nil
		296	+	}
		297	+
227	// validateIdentity accepts a name or an email address for a Git identity: valid	298		// validateIdentity accepts a name or an email address for a Git identity: valid
228	// UTF-8 without control characters or the angle brackets and line breaks that	299		// UTF-8 without control characters or the angle brackets and line breaks that
229	// Git does not allow in an identity.	300		// Git does not allow in an identity.

M internal/config/config_test.go +46 -0


@@ -8,6 +8,7 @@ import (
8	8		"net/netip"
9	9		"strings"
10	10		"testing"
	11	+	"time"
11	12
12	13		"github.com/stretchr/testify/assert"
13	14		"github.com/stretchr/testify/require"

@@ -208,3 +209,48 @@ func TestLoadGitReader(t *testing.T) {
208	209		assert.Equal(t, want, c.GitReader, in)
209	210		}
210	211		}
	212	+
	213	+	func TestOwnerAndCIRetention(t *testing.T) {
	214	+	base := map[string]string{
	215	+	config.EnvDataDir: "/data", config.EnvPublicURL: "https://forge.example.org",
	216	+	config.EnvAdminToken: "0123456789abcdef0123456789abcdef",
	217	+	}
	218	+	load := func(extra map[string]string) (config.Config, error) {
	219	+	env := map[string]string{}
	220	+	for k, v := range base {
	221	+	env[k] = v
	222	+	}
	223	+	for k, v := range extra {
	224	+	env[k] = v
	225	+	}
	226	+	return config.Load(func(k string) string { return env[k] })
	227	+	}
	228	+
	229	+	c, err := load(nil)
	230	+	if err != nil {
	231	+	t.Fatal(err)
	232	+	}
	233	+	if c.Owner != "owner" \|\| c.CIRetention != 9024time.Hour {
	234	+	t.Fatalf("defaults: owner %q, retention %v", c.Owner, c.CIRetention)
	235	+	}
	236	+
	237	+	c, err = load(map[string]string{config.EnvOwner: "bnema", config.EnvCILogRetention: "30d"})
	238	+	if err != nil {
	239	+	t.Fatal(err)
	240	+	}
	241	+	if c.Owner != "bnema" \|\| c.CIRetention != 3024time.Hour {
	242	+	t.Fatalf("set: owner %q, retention %v", c.Owner, c.CIRetention)
	243	+	}
	244	+	if c, err = load(map[string]string{config.EnvCILogRetention: "48h"}); err != nil \|\| c.CIRetention != 48*time.Hour {
	245	+	t.Fatalf("go duration: %v, %v", c.CIRetention, err)
	246	+	}
	247	+
	248	+	for _, extra := range []map[string]string{
	249	+	{config.EnvOwner: "Bad Name"}, {config.EnvOwner: ".dot"},
	250	+	{config.EnvCILogRetention: "soon"}, {config.EnvCILogRetention: "0d"}, {config.EnvCILogRetention: "10m"},
	251	+	} {
	252	+	if _, err := load(extra); err == nil {
	253	+	t.Errorf("%v: want an error", extra)
	254	+	}
	255	+	}
	256	+	}


@@ -8,6 +8,7 @@ import (
8	"net/netip"	8		"net/netip"
9	"strings"	9		"strings"
10	"testing"	10		"testing"
		11	+	"time"
11		12
12	"github.com/stretchr/testify/assert"	13		"github.com/stretchr/testify/assert"
13	"github.com/stretchr/testify/require"	14		"github.com/stretchr/testify/require"

@@ -208,3 +209,48 @@ func TestLoadGitReader(t *testing.T) {
208	assert.Equal(t, want, c.GitReader, in)	209		assert.Equal(t, want, c.GitReader, in)
209	}	210		}
210	}	211		}
		212	+
		213	+	func TestOwnerAndCIRetention(t *testing.T) {
		214	+	base := map[string]string{
		215	+	config.EnvDataDir: "/data", config.EnvPublicURL: "https://forge.example.org",
		216	+	config.EnvAdminToken: "0123456789abcdef0123456789abcdef",
		217	+	}
		218	+	load := func(extra map[string]string) (config.Config, error) {
		219	+	env := map[string]string{}
		220	+	for k, v := range base {
		221	+	env[k] = v
		222	+	}
		223	+	for k, v := range extra {
		224	+	env[k] = v
		225	+	}
		226	+	return config.Load(func(k string) string { return env[k] })
		227	+	}
		228	+
		229	+	c, err := load(nil)
		230	+	if err != nil {
		231	+	t.Fatal(err)
		232	+	}
		233	+	if c.Owner != "owner" \|\| c.CIRetention != 9024time.Hour {
		234	+	t.Fatalf("defaults: owner %q, retention %v", c.Owner, c.CIRetention)
		235	+	}
		236	+
		237	+	c, err = load(map[string]string{config.EnvOwner: "bnema", config.EnvCILogRetention: "30d"})
		238	+	if err != nil {
		239	+	t.Fatal(err)
		240	+	}
		241	+	if c.Owner != "bnema" \|\| c.CIRetention != 3024time.Hour {
		242	+	t.Fatalf("set: owner %q, retention %v", c.Owner, c.CIRetention)
		243	+	}
		244	+	if c, err = load(map[string]string{config.EnvCILogRetention: "48h"}); err != nil \|\| c.CIRetention != 48*time.Hour {
		245	+	t.Fatalf("go duration: %v, %v", c.CIRetention, err)
		246	+	}
		247	+
		248	+	for _, extra := range []map[string]string{
		249	+	{config.EnvOwner: "Bad Name"}, {config.EnvOwner: ".dot"},
		250	+	{config.EnvCILogRetention: "soon"}, {config.EnvCILogRetention: "0d"}, {config.EnvCILogRetention: "10m"},
		251	+	} {
		252	+	if _, err := load(extra); err == nil {
		253	+	t.Errorf("%v: want an error", extra)
		254	+	}
		255	+	}
		256	+	}

M internal/domain/ci.go +41 -0


@@ -237,6 +237,9 @@ type CIJob struct {
237	237		// A job that is not planned has not been expanded yet.
238	238		Planned bool
239	239		MatrixJSON string
	240	+	// ContextsJSON is the JSON object of contexts given to the runner, computed
	241	+	// when the job is planned.
	242	+	ContextsJSON string
240	243		// ComboTotal is the number of instances of the job's matrix.
241	244		ComboTotal int
242	245		// TimeoutMinutes is the timeout of the job, at least 1.

@@ -546,3 +549,41 @@ type JobUpdated struct {
546	549
547	550		// EventKind implements Event.
548	551		func (JobUpdated) EventKind() string { return "ci.job_updated" }
	552	+
	553	+	// CheckState is the combined state of the checks of a commit.
	554	+	type CheckState string
	555	+
	556	+	// Combined check states.
	557	+	const (
	558	+	// CheckStateNone: the commit has no check.
	559	+	CheckStateNone CheckState = ""
	560	+	// CheckStatePending: a check is queued or in progress, none has failed.
	561	+	CheckStatePending CheckState = "pending"
	562	+	// CheckStateFailure: a check failed (or needs an action), whatever the others.
	563	+	CheckStateFailure CheckState = "failure"
	564	+	// CheckStateSuccess: every check passed.
	565	+	CheckStateSuccess CheckState = "success"
	566	+	)
	567	+
	568	+	// CombineChecks reduces the checks of a commit to one state, the one the
	569	+	// merge rule uses too: a failed check makes the commit failing even while
	570	+	// others are pending; else a pending check makes it pending; else it is a
	571	+	// success.
	572	+	func CombineChecks(checks []CICheck) CheckState {
	573	+	if len(checks) == 0 {
	574	+	return CheckStateNone
	575	+	}
	576	+	pending := false
	577	+	for _, c := range checks {
	578	+	switch {
	579	+	case c.Status != CICompleted:
	580	+	pending = true
	581	+	case !c.Conclusion.PassesMerge():
	582	+	return CheckStateFailure
	583	+	}
	584	+	}
	585	+	if pending {
	586	+	return CheckStatePending
	587	+	}
	588	+	return CheckStateSuccess
	589	+	}


@@ -237,6 +237,9 @@ type CIJob struct {
237	// A job that is not planned has not been expanded yet.	237		// A job that is not planned has not been expanded yet.
238	Planned bool	238		Planned bool
239	MatrixJSON string	239		MatrixJSON string
		240	+	// ContextsJSON is the JSON object of contexts given to the runner, computed
		241	+	// when the job is planned.
		242	+	ContextsJSON string
240	// ComboTotal is the number of instances of the job's matrix.	243		// ComboTotal is the number of instances of the job's matrix.
241	ComboTotal int	244		ComboTotal int
242	// TimeoutMinutes is the timeout of the job, at least 1.	245		// TimeoutMinutes is the timeout of the job, at least 1.

@@ -546,3 +549,41 @@ type JobUpdated struct {
546		549
547	// EventKind implements Event.	550		// EventKind implements Event.
548	func (JobUpdated) EventKind() string { return "ci.job_updated" }	551		func (JobUpdated) EventKind() string { return "ci.job_updated" }
		552	+
		553	+	// CheckState is the combined state of the checks of a commit.
		554	+	type CheckState string
		555	+
		556	+	// Combined check states.
		557	+	const (
		558	+	// CheckStateNone: the commit has no check.
		559	+	CheckStateNone CheckState = ""
		560	+	// CheckStatePending: a check is queued or in progress, none has failed.
		561	+	CheckStatePending CheckState = "pending"
		562	+	// CheckStateFailure: a check failed (or needs an action), whatever the others.
		563	+	CheckStateFailure CheckState = "failure"
		564	+	// CheckStateSuccess: every check passed.
		565	+	CheckStateSuccess CheckState = "success"
		566	+	)
		567	+
		568	+	// CombineChecks reduces the checks of a commit to one state, the one the
		569	+	// merge rule uses too: a failed check makes the commit failing even while
		570	+	// others are pending; else a pending check makes it pending; else it is a
		571	+	// success.
		572	+	func CombineChecks(checks []CICheck) CheckState {
		573	+	if len(checks) == 0 {
		574	+	return CheckStateNone
		575	+	}
		576	+	pending := false
		577	+	for _, c := range checks {
		578	+	switch {
		579	+	case c.Status != CICompleted:
		580	+	pending = true
		581	+	case !c.Conclusion.PassesMerge():
		582	+	return CheckStateFailure
		583	+	}
		584	+	}
		585	+	if pending {
		586	+	return CheckStatePending
		587	+	}
		588	+	return CheckStateSuccess
		589	+	}

M internal/ports/ci.go +38 -13


@@ -92,9 +92,20 @@ type CIStore interface {
92	92		RemoveRunner(ctx context.Context, id string) (domain.CIRunner, error)
93	93		}
94	94
95		-	// CIChecksReader is the read side of the checks, for the merge rule.
	95	+	// CIRunCleaner is what repository deletion needs from CI: the runs of the
	96	+	// repository before its records go, and the removal of their logs after.
	97	+	// *usecase.CI implements it.
	98	+	type CIRunCleaner interface {
	99	+	RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error)
	100	+	RemoveLogs(ctx context.Context, runIDs []int64) error
	101	+	}
	102	+
	103	+	// CIChecksReader is the read side of the checks, for the merge rule and the
	104	+	// pages that show them. *usecase.CI implements it.
96	105		type CIChecksReader interface {
97		-	ListChecks(ctx context.Context, repo domain.RepoName, sha string, limit int) (checks []domain.CICheck, more bool, err error)
	106	+	// CommitChecks returns the checks of a commit, at most 1,000, without
	107	+	// access control (the caller has checked it).
	108	+	CommitChecks(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error)
98	109		}
99	110
100	111		// CILogRef names the log of one execution of a job.

@@ -262,7 +273,7 @@ type JobCancel struct {
262	273		// RunnerService is what runners call. The secret is the bearer secret of the
263	274		// call: the runner token for FetchJob, the job token for the other calls.
264	275		// A bad secret is domain.ErrUnauthenticated; a job token of another job than
265		-	// the one named is domain.ErrForbidden. *usecase.CIRunner implements it.
	276	+	// the one named is domain.ErrForbidden. *usecase.CI implements it.
266	277		type RunnerService interface {
267	278		Register(ctx context.Context, p domain.Principal, params RegisterRunnerParams) (RegisteredRunner, error)
268	279		FetchJob(ctx context.Context, secret string, params FetchJobParams) (FetchJobResult, error)

@@ -273,12 +284,6 @@ type RunnerService interface {
273	284		CompleteJob(ctx context.Context, secret string, params CompleteJobParams) (CompleteJobResult, error)
274	285		}
275	286
276		-	// RunStep is a step with its place in the log.
277		-	type RunStep struct {
278		-	domain.CIStep
279		-	LogEnd int64
280		-	}
281		-
282	287		// JobView is a job as owners see it.
283	288		type JobView struct {
284	289		domain.CIJob

@@ -288,7 +293,7 @@ type JobView struct {
288	293		RunnerGone bool
289	294		}
290	295
291		-	// Steps returns the steps with their log ranges.
	296	+	// StepRanges returns where the lines of every step are in the log.
292	297		func (v JobView) StepRanges() []domain.StepRange { return domain.StepRanges(v.Steps, v.Lines) }
293	298
294	299		// ListRunsParams select a page of runs.

@@ -322,9 +327,29 @@ type JobLogLine struct {
322	327		domain.CILogLine
323	328		}
324	329
	330	+	// GetJobLogParams select the log to stream.
	331	+	type GetJobLogParams struct {
	332	+	JobID string
	333	+	// Attempt is the attempt of the job, 0 for the latest.
	334	+	Attempt int32
	335	+	Offset int64
	336	+	Follow bool
	337	+	// Client identifies where the call comes from, for the limit of streams
	338	+	// per client; empty counts against the total only.
	339	+	Client string
	340	+	}
	341	+
	342	+	// CheckView is a check with what owners need to read it.
	343	+	type CheckView struct {
	344	+	domain.CICheck
	345	+	// WaitingForRunner is set for a queued check whose job no registered
	346	+	// runner can take (no runner has all its labels).
	347	+	WaitingForRunner bool
	348	+	}
	349	+
325	350		// ChecksResult are the checks of a commit.
326	351		type ChecksResult struct {
327		-	Checks []domain.CICheck
	352	+	Checks []CheckView
328	353		Truncated bool
329	354		}
330	355

@@ -341,12 +366,12 @@ type RunnerView struct {
341	366		Busy bool
342	367		}
343	368
344		-	// RunService is the owner side of CI. *usecase.CIRuns implements it.
	369	+	// RunService is the owner side of CI. *usecase.CI implements it.
345	370		type RunService interface {
346	371		ListRuns(ctx context.Context, p domain.Principal, repo domain.RepoName, params ListRunsParams) (RunPage, error)
347	372		GetRun(ctx context.Context, p domain.Principal, repo domain.RepoName, runID int64, attempt int32) (RunDetail, error)
348	373		GetJob(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32) (JobView, error)
349		-	GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(LogBatch) error) error
	374	+	GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, params GetJobLogParams, send func(LogBatch) error) error
350	375		CancelRun(ctx context.Context, p domain.Principal, repo domain.RepoName, runID int64) (domain.CIRun, error)
351	376		RerunRun(ctx context.Context, p domain.Principal, repo domain.RepoName, runID int64, failedOnly bool) (domain.CIRun, error)
352	377		ListChecks(ctx context.Context, p domain.Principal, repo domain.RepoName, sha string) (ChecksResult, error)

 	RemoveRunner(ctx context.Context, id string) (domain.CIRunner, error)
 }
-// CIChecksReader is the read side of the checks, for the merge rule.
+// CIRunCleaner is what repository deletion needs from CI: the runs of the
+// repository before its records go, and the removal of their logs after.
+// *usecase.CI implements it.
+type CIRunCleaner interface {
+	RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error)
+	RemoveLogs(ctx context.Context, runIDs []int64) error
+}
+// CIChecksReader is the read side of the checks, for the merge rule and the
+// pages that show them. *usecase.CI implements it.
 type CIChecksReader interface {
-	ListChecks(ctx context.Context, repo domain.RepoName, sha string, limit int) (checks []domain.CICheck, more bool, err error)
+	// CommitChecks returns the checks of a commit, at most 1,000, without
+	// access control (the caller has checked it).
+	CommitChecks(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error)
 }
 // CILogRef names the log of one execution of a job.
 // RunnerService is what runners call. The secret is the bearer secret of the
 // call: the runner token for FetchJob, the job token for the other calls.
 // A bad secret is domain.ErrUnauthenticated; a job token of another job than
-// the one named is domain.ErrForbidden. *usecase.CIRunner implements it.
+// the one named is domain.ErrForbidden. *usecase.CI implements it.
 type RunnerService interface {
 	Register(ctx context.Context, p domain.Principal, params RegisterRunnerParams) (RegisteredRunner, error)
 	FetchJob(ctx context.Context, secret string, params FetchJobParams) (FetchJobResult, error)
 	CompleteJob(ctx context.Context, secret string, params CompleteJobParams) (CompleteJobResult, error)
 }
-// RunStep is a step with its place in the log.
-type RunStep struct {
-	domain.CIStep
-	LogEnd int64
-}
 // JobView is a job as owners see it.
 type JobView struct {
 	domain.CIJob
 	RunnerGone bool
 }
-// Steps returns the steps with their log ranges.
+// StepRanges returns where the lines of every step are in the log.
 func (v JobView) StepRanges() []domain.StepRange { return domain.StepRanges(v.Steps, v.Lines) }
 // ListRunsParams select a page of runs.
 	domain.CILogLine
 }
+// GetJobLogParams select the log to stream.
+type GetJobLogParams struct {
+	JobID string
+	// Attempt is the attempt of the job, 0 for the latest.
+	Attempt int32
+	Offset  int64
+	Follow  bool
+	// Client identifies where the call comes from, for the limit of streams
+	// per client; empty counts against the total only.
+	Client string
+}
+// CheckView is a check with what owners need to read it.
+type CheckView struct {
+	domain.CICheck
+	// WaitingForRunner is set for a queued check whose job no registered
+	// runner can take (no runner has all its labels).
+	WaitingForRunner bool
+}
 // ChecksResult are the checks of a commit.
 type ChecksResult struct {
-	Checks    []domain.CICheck
+	Checks    []CheckView
 	Truncated bool
 }
 	Busy     bool
 }
-// RunService is the owner side of CI. *usecase.CIRuns implements it.
+// RunService is the owner side of CI. *usecase.CI implements it.
 type RunService interface {
 	ListRuns(ctx context.Context, p domain.Principal, repo domain.RepoName, params ListRunsParams) (RunPage, error)
 	GetRun(ctx context.Context, p domain.Principal, repo domain.RepoName, runID int64, attempt int32) (RunDetail, error)
 	GetJob(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32) (JobView, error)
-	GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(LogBatch) error) error
+	GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, params GetJobLogParams, send func(LogBatch) error) error
 	CancelRun(ctx context.Context, p domain.Principal, repo domain.RepoName, runID int64) (domain.CIRun, error)
 	RerunRun(ctx context.Context, p domain.Principal, repo domain.RepoName, runID int64, failedOnly bool) (domain.CIRun, error)
 	ListChecks(ctx context.Context, p domain.Principal, repo domain.RepoName, sha string) (ChecksResult, error)

M internal/ports/mocks/CIChecksReader.go +22 -34


@@ -47,55 +47,48 @@ func (_m CIChecksReader) EXPECT() CIChecksReader_Expecter {
47	47		return &CIChecksReader_Expecter{mock: &_m.Mock}
48	48		}
49	49
50		-	// ListChecks provides a mock function for the type CIChecksReader
51		-	func (_mock *CIChecksReader) ListChecks(ctx context.Context, repo domain.RepoName, sha string, limit int) ([]domain.CICheck, bool, error) {
52		-	ret := _mock.Called(ctx, repo, sha, limit)
	50	+	// CommitChecks provides a mock function for the type CIChecksReader
	51	+	func (_mock *CIChecksReader) CommitChecks(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error) {
	52	+	ret := _mock.Called(ctx, repo, sha)
53	53
54	54		if len(ret) == 0 {
55		-	panic("no return value specified for ListChecks")
	55	+	panic("no return value specified for CommitChecks")
56	56		}
57	57
58	58		var r0 []domain.CICheck
59		-	var r1 bool
60		-	var r2 error
61		-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string, int) ([]domain.CICheck, bool, error)); ok {
62		-	return returnFunc(ctx, repo, sha, limit)
	59	+	var r1 error
	60	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string) ([]domain.CICheck, error)); ok {
	61	+	return returnFunc(ctx, repo, sha)
63	62		}
64		-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string, int) []domain.CICheck); ok {
65		-	r0 = returnFunc(ctx, repo, sha, limit)
	63	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string) []domain.CICheck); ok {
	64	+	r0 = returnFunc(ctx, repo, sha)
66	65		} else {
67	66		if ret.Get(0) != nil {
68	67		r0 = ret.Get(0).([]domain.CICheck)
69	68		}
70	69		}
71		-	if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RepoName, string, int) bool); ok {
72		-	r1 = returnFunc(ctx, repo, sha, limit)
	70	+	if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RepoName, string) error); ok {
	71	+	r1 = returnFunc(ctx, repo, sha)
73	72		} else {
74		-	r1 = ret.Get(1).(bool)
	73	+	r1 = ret.Error(1)
75	74		}
76		-	if returnFunc, ok := ret.Get(2).(func(context.Context, domain.RepoName, string, int) error); ok {
77		-	r2 = returnFunc(ctx, repo, sha, limit)
78		-	} else {
79		-	r2 = ret.Error(2)
80		-	}
81		-	return r0, r1, r2
	75	+	return r0, r1
82	76		}
83	77
84		-	// CIChecksReader_ListChecks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListChecks'
85		-	type CIChecksReader_ListChecks_Call struct {
	78	+	// CIChecksReader_CommitChecks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CommitChecks'
	79	+	type CIChecksReader_CommitChecks_Call struct {
86	80		*mock.Call
87	81		}
88	82
89		-	// ListChecks is a helper method to define mock.On call
	83	+	// CommitChecks is a helper method to define mock.On call
90	84		// - ctx context.Context
91	85		// - repo domain.RepoName
92	86		// - sha string
93		-	// - limit int
94		-	func (_e CIChecksReader_Expecter) ListChecks(ctx any, repo any, sha any, limit any) CIChecksReader_ListChecks_Call {
95		-	return &CIChecksReader_ListChecks_Call{Call: _e.mock.On("ListChecks", ctx, repo, sha, limit)}
	87	+	func (_e CIChecksReader_Expecter) CommitChecks(ctx any, repo any, sha any) CIChecksReader_CommitChecks_Call {
	88	+	return &CIChecksReader_CommitChecks_Call{Call: _e.mock.On("CommitChecks", ctx, repo, sha)}
96	89		}
97	90
98		-	func (_c CIChecksReader_ListChecks_Call) Run(run func(ctx context.Context, repo domain.RepoName, sha string, limit int)) CIChecksReader_ListChecks_Call {
	91	+	func (_c CIChecksReader_CommitChecks_Call) Run(run func(ctx context.Context, repo domain.RepoName, sha string)) CIChecksReader_CommitChecks_Call {
99	92		_c.Call.Run(func(args mock.Arguments) {
100	93		var arg0 context.Context
101	94		if args[0] != nil {

@@ -109,26 +102,21 @@ func (_c *CIChecksReader_ListChecks_Call) Run(run func(ctx context.Context, repo
109	102		if args[2] != nil {
110	103		arg2 = args[2].(string)
111	104		}
112		-	var arg3 int
113		-	if args[3] != nil {
114		-	arg3 = args[3].(int)
115		-	}
116	105		run(
117	106		arg0,
118	107		arg1,
119	108		arg2,
120		-	arg3,
121	109		)
122	110		})
123	111		return _c
124	112		}
125	113
126		-	func (_c CIChecksReader_ListChecks_Call) Return(checks []domain.CICheck, more bool, err error) CIChecksReader_ListChecks_Call {
127		-	_c.Call.Return(checks, more, err)
	114	+	func (_c CIChecksReader_CommitChecks_Call) Return(cIChecks []domain.CICheck, err error) CIChecksReader_CommitChecks_Call {
	115	+	_c.Call.Return(cIChecks, err)
128	116		return _c
129	117		}
130	118
131		-	func (_c CIChecksReader_ListChecks_Call) RunAndReturn(run func(ctx context.Context, repo domain.RepoName, sha string, limit int) ([]domain.CICheck, bool, error)) CIChecksReader_ListChecks_Call {
	119	+	func (_c CIChecksReader_CommitChecks_Call) RunAndReturn(run func(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error)) CIChecksReader_CommitChecks_Call {
132	120		_c.Call.Return(run)
133	121		return _c
134	122		}

 	return &CIChecksReader_Expecter{mock: &_m.Mock}
 }
-// ListChecks provides a mock function for the type CIChecksReader
+// CommitChecks provides a mock function for the type CIChecksReader
-func (_mock *CIChecksReader) ListChecks(ctx context.Context, repo domain.RepoName, sha string, limit int) ([]domain.CICheck, bool, error) {
+func (_mock *CIChecksReader) CommitChecks(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error) {
-	ret := _mock.Called(ctx, repo, sha, limit)
+	ret := _mock.Called(ctx, repo, sha)
 	if len(ret) == 0 {
-		panic("no return value specified for ListChecks")
+		panic("no return value specified for CommitChecks")
 	}
 	var r0 []domain.CICheck
-	var r1 bool
+	var r1 error
-	var r2 error
+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string) ([]domain.CICheck, error)); ok {
-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string, int) ([]domain.CICheck, bool, error)); ok {
+		return returnFunc(ctx, repo, sha)
-		return returnFunc(ctx, repo, sha, limit)
 	}
-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string, int) []domain.CICheck); ok {
+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName, string) []domain.CICheck); ok {
-		r0 = returnFunc(ctx, repo, sha, limit)
+		r0 = returnFunc(ctx, repo, sha)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).([]domain.CICheck)
 		}
 	}
-	if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RepoName, string, int) bool); ok {
+	if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RepoName, string) error); ok {
-		r1 = returnFunc(ctx, repo, sha, limit)
+		r1 = returnFunc(ctx, repo, sha)
 	} else {
-		r1 = ret.Get(1).(bool)
+		r1 = ret.Error(1)
 	}
-	if returnFunc, ok := ret.Get(2).(func(context.Context, domain.RepoName, string, int) error); ok {
+	return r0, r1
-		r2 = returnFunc(ctx, repo, sha, limit)
-	} else {
-		r2 = ret.Error(2)
-	}
-	return r0, r1, r2
 }
-// CIChecksReader_ListChecks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListChecks'
+// CIChecksReader_CommitChecks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CommitChecks'
-type CIChecksReader_ListChecks_Call struct {
+type CIChecksReader_CommitChecks_Call struct {
 	*mock.Call
 }
-// ListChecks is a helper method to define mock.On call
+// CommitChecks is a helper method to define mock.On call
 //   - ctx context.Context
 //   - repo domain.RepoName
 //   - sha string
-//   - limit int
+func (_e *CIChecksReader_Expecter) CommitChecks(ctx any, repo any, sha any) *CIChecksReader_CommitChecks_Call {
-func (_e *CIChecksReader_Expecter) ListChecks(ctx any, repo any, sha any, limit any) *CIChecksReader_ListChecks_Call {
+	return &CIChecksReader_CommitChecks_Call{Call: _e.mock.On("CommitChecks", ctx, repo, sha)}
-	return &CIChecksReader_ListChecks_Call{Call: _e.mock.On("ListChecks", ctx, repo, sha, limit)}
 }
-func (_c *CIChecksReader_ListChecks_Call) Run(run func(ctx context.Context, repo domain.RepoName, sha string, limit int)) *CIChecksReader_ListChecks_Call {
+func (_c *CIChecksReader_CommitChecks_Call) Run(run func(ctx context.Context, repo domain.RepoName, sha string)) *CIChecksReader_CommitChecks_Call {
 	_c.Call.Run(func(args mock.Arguments) {
 		var arg0 context.Context
 		if args[0] != nil {
 		if args[2] != nil {
 			arg2 = args[2].(string)
 		}
-		var arg3 int
-		if args[3] != nil {
-			arg3 = args[3].(int)
-		}
 		run(
 			arg0,
 			arg1,
 			arg2,
-			arg3,
 		)
 	})
 	return _c
 }
-func (_c *CIChecksReader_ListChecks_Call) Return(checks []domain.CICheck, more bool, err error) *CIChecksReader_ListChecks_Call {
+func (_c *CIChecksReader_CommitChecks_Call) Return(cIChecks []domain.CICheck, err error) *CIChecksReader_CommitChecks_Call {
-	_c.Call.Return(checks, more, err)
+	_c.Call.Return(cIChecks, err)
 	return _c
 }
-func (_c *CIChecksReader_ListChecks_Call) RunAndReturn(run func(ctx context.Context, repo domain.RepoName, sha string, limit int) ([]domain.CICheck, bool, error)) *CIChecksReader_ListChecks_Call {
+func (_c *CIChecksReader_CommitChecks_Call) RunAndReturn(run func(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error)) *CIChecksReader_CommitChecks_Call {
 	_c.Call.Return(run)
 	return _c
 }

A internal/ports/mocks/CIRunCleaner.go +173 -0

@@ -0,0 +1,173 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package mocks
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	"bnema.dev/ermite/internal/domain"
	11	+	mock "github.com/stretchr/testify/mock"
	12	+	)
	13	+
	14	+	// NewCIRunCleaner creates a new instance of CIRunCleaner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	15	+	// The first argument is typically a *testing.T value.
	16	+	func NewCIRunCleaner(t interface {
	17	+	mock.TestingT
	18	+	Cleanup(func())
	19	+	}) *CIRunCleaner {
	20	+	if helper, ok := t.(interface{ Helper() }); ok {
	21	+	helper.Helper()
	22	+	}
	23	+
	24	+	mock := &CIRunCleaner{}
	25	+	mock.Mock.Test(t)
	26	+
	27	+	t.Cleanup(func() {
	28	+	if helper, ok := t.(interface{ Helper() }); ok {
	29	+	helper.Helper()
	30	+	}
	31	+	mock.AssertExpectations(t)
	32	+	})
	33	+
	34	+	return mock
	35	+	}
	36	+
	37	+	// CIRunCleaner is an autogenerated mock type for the CIRunCleaner type
	38	+	type CIRunCleaner struct {
	39	+	mock.Mock
	40	+	}
	41	+
	42	+	type CIRunCleaner_Expecter struct {
	43	+	mock *mock.Mock
	44	+	}
	45	+
	46	+	func (_m CIRunCleaner) EXPECT() CIRunCleaner_Expecter {
	47	+	return &CIRunCleaner_Expecter{mock: &_m.Mock}
	48	+	}
	49	+
	50	+	// RemoveLogs provides a mock function for the type CIRunCleaner
	51	+	func (_mock *CIRunCleaner) RemoveLogs(ctx context.Context, runIDs []int64) error {
	52	+	ret := _mock.Called(ctx, runIDs)
	53	+
	54	+	if len(ret) == 0 {
	55	+	panic("no return value specified for RemoveLogs")
	56	+	}
	57	+
	58	+	var r0 error
	59	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []int64) error); ok {
	60	+	r0 = returnFunc(ctx, runIDs)
	61	+	} else {
	62	+	r0 = ret.Error(0)
	63	+	}
	64	+	return r0
	65	+	}
	66	+
	67	+	// CIRunCleaner_RemoveLogs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveLogs'
	68	+	type CIRunCleaner_RemoveLogs_Call struct {
	69	+	*mock.Call
	70	+	}
	71	+
	72	+	// RemoveLogs is a helper method to define mock.On call
	73	+	// - ctx context.Context
	74	+	// - runIDs []int64
	75	+	func (_e CIRunCleaner_Expecter) RemoveLogs(ctx any, runIDs any) CIRunCleaner_RemoveLogs_Call {
	76	+	return &CIRunCleaner_RemoveLogs_Call{Call: _e.mock.On("RemoveLogs", ctx, runIDs)}
	77	+	}
	78	+
	79	+	func (_c CIRunCleaner_RemoveLogs_Call) Run(run func(ctx context.Context, runIDs []int64)) CIRunCleaner_RemoveLogs_Call {
	80	+	_c.Call.Run(func(args mock.Arguments) {
	81	+	var arg0 context.Context
	82	+	if args[0] != nil {
	83	+	arg0 = args[0].(context.Context)
	84	+	}
	85	+	var arg1 []int64
	86	+	if args[1] != nil {
	87	+	arg1 = args[1].([]int64)
	88	+	}
	89	+	run(
	90	+	arg0,
	91	+	arg1,
	92	+	)
	93	+	})
	94	+	return _c
	95	+	}
	96	+
	97	+	func (_c CIRunCleaner_RemoveLogs_Call) Return(err error) CIRunCleaner_RemoveLogs_Call {
	98	+	_c.Call.Return(err)
	99	+	return _c
	100	+	}
	101	+
	102	+	func (_c CIRunCleaner_RemoveLogs_Call) RunAndReturn(run func(ctx context.Context, runIDs []int64) error) CIRunCleaner_RemoveLogs_Call {
	103	+	_c.Call.Return(run)
	104	+	return _c
	105	+	}
	106	+
	107	+	// RunIDs provides a mock function for the type CIRunCleaner
	108	+	func (_mock *CIRunCleaner) RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error) {
	109	+	ret := _mock.Called(ctx, repo)
	110	+
	111	+	if len(ret) == 0 {
	112	+	panic("no return value specified for RunIDs")
	113	+	}
	114	+
	115	+	var r0 []int64
	116	+	var r1 error
	117	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName) ([]int64, error)); ok {
	118	+	return returnFunc(ctx, repo)
	119	+	}
	120	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName) []int64); ok {
	121	+	r0 = returnFunc(ctx, repo)
	122	+	} else {
	123	+	if ret.Get(0) != nil {
	124	+	r0 = ret.Get(0).([]int64)
	125	+	}
	126	+	}
	127	+	if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RepoName) error); ok {
	128	+	r1 = returnFunc(ctx, repo)
	129	+	} else {
	130	+	r1 = ret.Error(1)
	131	+	}
	132	+	return r0, r1
	133	+	}
	134	+
	135	+	// CIRunCleaner_RunIDs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunIDs'
	136	+	type CIRunCleaner_RunIDs_Call struct {
	137	+	*mock.Call
	138	+	}
	139	+
	140	+	// RunIDs is a helper method to define mock.On call
	141	+	// - ctx context.Context
	142	+	// - repo domain.RepoName
	143	+	func (_e CIRunCleaner_Expecter) RunIDs(ctx any, repo any) CIRunCleaner_RunIDs_Call {
	144	+	return &CIRunCleaner_RunIDs_Call{Call: _e.mock.On("RunIDs", ctx, repo)}
	145	+	}
	146	+
	147	+	func (_c CIRunCleaner_RunIDs_Call) Run(run func(ctx context.Context, repo domain.RepoName)) CIRunCleaner_RunIDs_Call {
	148	+	_c.Call.Run(func(args mock.Arguments) {
	149	+	var arg0 context.Context
	150	+	if args[0] != nil {
	151	+	arg0 = args[0].(context.Context)
	152	+	}
	153	+	var arg1 domain.RepoName
	154	+	if args[1] != nil {
	155	+	arg1 = args[1].(domain.RepoName)
	156	+	}
	157	+	run(
	158	+	arg0,
	159	+	arg1,
	160	+	)
	161	+	})
	162	+	return _c
	163	+	}
	164	+
	165	+	func (_c CIRunCleaner_RunIDs_Call) Return(int64s []int64, err error) CIRunCleaner_RunIDs_Call {
	166	+	_c.Call.Return(int64s, err)
	167	+	return _c
	168	+	}
	169	+
	170	+	func (_c CIRunCleaner_RunIDs_Call) RunAndReturn(run func(ctx context.Context, repo domain.RepoName) ([]int64, error)) CIRunCleaner_RunIDs_Call {
	171	+	_c.Call.Return(run)
	172	+	return _c
	173	+	}

@@ -0,0 +1,173 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package mocks
	6	+
	7	+	import (
	8	+	"context"
	9	+
	10	+	"bnema.dev/ermite/internal/domain"
	11	+	mock "github.com/stretchr/testify/mock"
	12	+	)
	13	+
	14	+	// NewCIRunCleaner creates a new instance of CIRunCleaner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	15	+	// The first argument is typically a *testing.T value.
	16	+	func NewCIRunCleaner(t interface {
	17	+	mock.TestingT
	18	+	Cleanup(func())
	19	+	}) *CIRunCleaner {
	20	+	if helper, ok := t.(interface{ Helper() }); ok {
	21	+	helper.Helper()
	22	+	}
	23	+
	24	+	mock := &CIRunCleaner{}
	25	+	mock.Mock.Test(t)
	26	+
	27	+	t.Cleanup(func() {
	28	+	if helper, ok := t.(interface{ Helper() }); ok {
	29	+	helper.Helper()
	30	+	}
	31	+	mock.AssertExpectations(t)
	32	+	})
	33	+
	34	+	return mock
	35	+	}
	36	+
	37	+	// CIRunCleaner is an autogenerated mock type for the CIRunCleaner type
	38	+	type CIRunCleaner struct {
	39	+	mock.Mock
	40	+	}
	41	+
	42	+	type CIRunCleaner_Expecter struct {
	43	+	mock *mock.Mock
	44	+	}
	45	+
	46	+	func (_m CIRunCleaner) EXPECT() CIRunCleaner_Expecter {
	47	+	return &CIRunCleaner_Expecter{mock: &_m.Mock}
	48	+	}
	49	+
	50	+	// RemoveLogs provides a mock function for the type CIRunCleaner
	51	+	func (_mock *CIRunCleaner) RemoveLogs(ctx context.Context, runIDs []int64) error {
	52	+	ret := _mock.Called(ctx, runIDs)
	53	+
	54	+	if len(ret) == 0 {
	55	+	panic("no return value specified for RemoveLogs")
	56	+	}
	57	+
	58	+	var r0 error
	59	+	if returnFunc, ok := ret.Get(0).(func(context.Context, []int64) error); ok {
	60	+	r0 = returnFunc(ctx, runIDs)
	61	+	} else {
	62	+	r0 = ret.Error(0)
	63	+	}
	64	+	return r0
	65	+	}
	66	+
	67	+	// CIRunCleaner_RemoveLogs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveLogs'
	68	+	type CIRunCleaner_RemoveLogs_Call struct {
	69	+	*mock.Call
	70	+	}
	71	+
	72	+	// RemoveLogs is a helper method to define mock.On call
	73	+	// - ctx context.Context
	74	+	// - runIDs []int64
	75	+	func (_e CIRunCleaner_Expecter) RemoveLogs(ctx any, runIDs any) CIRunCleaner_RemoveLogs_Call {
	76	+	return &CIRunCleaner_RemoveLogs_Call{Call: _e.mock.On("RemoveLogs", ctx, runIDs)}
	77	+	}
	78	+
	79	+	func (_c CIRunCleaner_RemoveLogs_Call) Run(run func(ctx context.Context, runIDs []int64)) CIRunCleaner_RemoveLogs_Call {
	80	+	_c.Call.Run(func(args mock.Arguments) {
	81	+	var arg0 context.Context
	82	+	if args[0] != nil {
	83	+	arg0 = args[0].(context.Context)
	84	+	}
	85	+	var arg1 []int64
	86	+	if args[1] != nil {
	87	+	arg1 = args[1].([]int64)
	88	+	}
	89	+	run(
	90	+	arg0,
	91	+	arg1,
	92	+	)
	93	+	})
	94	+	return _c
	95	+	}
	96	+
	97	+	func (_c CIRunCleaner_RemoveLogs_Call) Return(err error) CIRunCleaner_RemoveLogs_Call {
	98	+	_c.Call.Return(err)
	99	+	return _c
	100	+	}
	101	+
	102	+	func (_c CIRunCleaner_RemoveLogs_Call) RunAndReturn(run func(ctx context.Context, runIDs []int64) error) CIRunCleaner_RemoveLogs_Call {
	103	+	_c.Call.Return(run)
	104	+	return _c
	105	+	}
	106	+
	107	+	// RunIDs provides a mock function for the type CIRunCleaner
	108	+	func (_mock *CIRunCleaner) RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error) {
	109	+	ret := _mock.Called(ctx, repo)
	110	+
	111	+	if len(ret) == 0 {
	112	+	panic("no return value specified for RunIDs")
	113	+	}
	114	+
	115	+	var r0 []int64
	116	+	var r1 error
	117	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName) ([]int64, error)); ok {
	118	+	return returnFunc(ctx, repo)
	119	+	}
	120	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.RepoName) []int64); ok {
	121	+	r0 = returnFunc(ctx, repo)
	122	+	} else {
	123	+	if ret.Get(0) != nil {
	124	+	r0 = ret.Get(0).([]int64)
	125	+	}
	126	+	}
	127	+	if returnFunc, ok := ret.Get(1).(func(context.Context, domain.RepoName) error); ok {
	128	+	r1 = returnFunc(ctx, repo)
	129	+	} else {
	130	+	r1 = ret.Error(1)
	131	+	}
	132	+	return r0, r1
	133	+	}
	134	+
	135	+	// CIRunCleaner_RunIDs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunIDs'
	136	+	type CIRunCleaner_RunIDs_Call struct {
	137	+	*mock.Call
	138	+	}
	139	+
	140	+	// RunIDs is a helper method to define mock.On call
	141	+	// - ctx context.Context
	142	+	// - repo domain.RepoName
	143	+	func (_e CIRunCleaner_Expecter) RunIDs(ctx any, repo any) CIRunCleaner_RunIDs_Call {
	144	+	return &CIRunCleaner_RunIDs_Call{Call: _e.mock.On("RunIDs", ctx, repo)}
	145	+	}
	146	+
	147	+	func (_c CIRunCleaner_RunIDs_Call) Run(run func(ctx context.Context, repo domain.RepoName)) CIRunCleaner_RunIDs_Call {
	148	+	_c.Call.Run(func(args mock.Arguments) {
	149	+	var arg0 context.Context
	150	+	if args[0] != nil {
	151	+	arg0 = args[0].(context.Context)
	152	+	}
	153	+	var arg1 domain.RepoName
	154	+	if args[1] != nil {
	155	+	arg1 = args[1].(domain.RepoName)
	156	+	}
	157	+	run(
	158	+	arg0,
	159	+	arg1,
	160	+	)
	161	+	})
	162	+	return _c
	163	+	}
	164	+
	165	+	func (_c CIRunCleaner_RunIDs_Call) Return(int64s []int64, err error) CIRunCleaner_RunIDs_Call {
	166	+	_c.Call.Return(int64s, err)
	167	+	return _c
	168	+	}
	169	+
	170	+	func (_c CIRunCleaner_RunIDs_Call) RunAndReturn(run func(ctx context.Context, repo domain.RepoName) ([]int64, error)) CIRunCleaner_RunIDs_Call {
	171	+	_c.Call.Return(run)
	172	+	return _c
	173	+	}

M internal/ports/mocks/RunService.go +13 -31


@@ -211,16 +211,16 @@ func (_c *RunService_GetJob_Call) RunAndReturn(run func(ctx context.Context, p d
211	211		}
212	212
213	213		// GetJobLog provides a mock function for the type RunService
214		-	func (_mock *RunService) GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(ports.LogBatch) error) error {
215		-	ret := _mock.Called(ctx, p, repo, jobID, attempt, offset, follow, send)
	214	+	func (_mock *RunService) GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error) error {
	215	+	ret := _mock.Called(ctx, p, repo, params, send)
216	216
217	217		if len(ret) == 0 {
218	218		panic("no return value specified for GetJobLog")
219	219		}
220	220
221	221		var r0 error
222		-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.Principal, domain.RepoName, string, int32, int64, bool, func(ports.LogBatch) error) error); ok {
223		-	r0 = returnFunc(ctx, p, repo, jobID, attempt, offset, follow, send)
	222	+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.Principal, domain.RepoName, ports.GetJobLogParams, func(ports.LogBatch) error) error); ok {
	223	+	r0 = returnFunc(ctx, p, repo, params, send)
224	224		} else {
225	225		r0 = ret.Error(0)
226	226		}

@@ -236,16 +236,13 @@ type RunService_GetJobLog_Call struct {
236	236		// - ctx context.Context
237	237		// - p domain.Principal
238	238		// - repo domain.RepoName
239		-	// - jobID string
240		-	// - attempt int32
241		-	// - offset int64
242		-	// - follow bool
	239	+	// - params ports.GetJobLogParams
243	240		// - send func(ports.LogBatch) error
244		-	func (_e RunService_Expecter) GetJobLog(ctx any, p any, repo any, jobID any, attempt any, offset any, follow any, send any) RunService_GetJobLog_Call {
245		-	return &RunService_GetJobLog_Call{Call: _e.mock.On("GetJobLog", ctx, p, repo, jobID, attempt, offset, follow, send)}
	241	+	func (_e RunService_Expecter) GetJobLog(ctx any, p any, repo any, params any, send any) RunService_GetJobLog_Call {
	242	+	return &RunService_GetJobLog_Call{Call: _e.mock.On("GetJobLog", ctx, p, repo, params, send)}
246	243		}
247	244
248		-	func (_c RunService_GetJobLog_Call) Run(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(ports.LogBatch) error)) RunService_GetJobLog_Call {
	245	+	func (_c RunService_GetJobLog_Call) Run(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error)) RunService_GetJobLog_Call {
249	246		_c.Call.Run(func(args mock.Arguments) {
250	247		var arg0 context.Context
251	248		if args[0] != nil {

@@ -259,25 +256,13 @@ func (_c *RunService_GetJobLog_Call) Run(run func(ctx context.Context, p domain.
259	256		if args[2] != nil {
260	257		arg2 = args[2].(domain.RepoName)
261	258		}
262		-	var arg3 string
	259	+	var arg3 ports.GetJobLogParams
263	260		if args[3] != nil {
264		-	arg3 = args[3].(string)
	261	+	arg3 = args[3].(ports.GetJobLogParams)
265	262		}
266		-	var arg4 int32
	263	+	var arg4 func(ports.LogBatch) error
267	264		if args[4] != nil {
268		-	arg4 = args[4].(int32)
269		-	}
270		-	var arg5 int64
271		-	if args[5] != nil {
272		-	arg5 = args[5].(int64)
273		-	}
274		-	var arg6 bool
275		-	if args[6] != nil {
276		-	arg6 = args[6].(bool)
277		-	}
278		-	var arg7 func(ports.LogBatch) error
279		-	if args[7] != nil {
280		-	arg7 = args[7].(func(ports.LogBatch) error)
	265	+	arg4 = args[4].(func(ports.LogBatch) error)
281	266		}
282	267		run(
283	268		arg0,

@@ -285,9 +270,6 @@ func (_c *RunService_GetJobLog_Call) Run(run func(ctx context.Context, p domain.
285	270		arg2,
286	271		arg3,
287	272		arg4,
288		-	arg5,
289		-	arg6,
290		-	arg7,
291	273		)
292	274		})
293	275		return _c

@@ -298,7 +280,7 @@ func (_c RunService_GetJobLog_Call) Return(err error) RunService_GetJobLog_Cal
298	280		return _c
299	281		}
300	282
301		-	func (_c RunService_GetJobLog_Call) RunAndReturn(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(ports.LogBatch) error) error) RunService_GetJobLog_Call {
	283	+	func (_c RunService_GetJobLog_Call) RunAndReturn(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error) error) RunService_GetJobLog_Call {
302	284		_c.Call.Return(run)
303	285		return _c
304	286		}

 }
 // GetJobLog provides a mock function for the type RunService
-func (_mock *RunService) GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(ports.LogBatch) error) error {
+func (_mock *RunService) GetJobLog(ctx context.Context, p domain.Principal, repo domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error) error {
-	ret := _mock.Called(ctx, p, repo, jobID, attempt, offset, follow, send)
+	ret := _mock.Called(ctx, p, repo, params, send)
 	if len(ret) == 0 {
 		panic("no return value specified for GetJobLog")
 	}
 	var r0 error
-	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.Principal, domain.RepoName, string, int32, int64, bool, func(ports.LogBatch) error) error); ok {
+	if returnFunc, ok := ret.Get(0).(func(context.Context, domain.Principal, domain.RepoName, ports.GetJobLogParams, func(ports.LogBatch) error) error); ok {
-		r0 = returnFunc(ctx, p, repo, jobID, attempt, offset, follow, send)
+		r0 = returnFunc(ctx, p, repo, params, send)
 	} else {
 		r0 = ret.Error(0)
 	}
 //   - ctx context.Context
 //   - p domain.Principal
 //   - repo domain.RepoName
-//   - jobID string
+//   - params ports.GetJobLogParams
-//   - attempt int32
-//   - offset int64
-//   - follow bool
 //   - send func(ports.LogBatch) error
-func (_e *RunService_Expecter) GetJobLog(ctx any, p any, repo any, jobID any, attempt any, offset any, follow any, send any) *RunService_GetJobLog_Call {
+func (_e *RunService_Expecter) GetJobLog(ctx any, p any, repo any, params any, send any) *RunService_GetJobLog_Call {
-	return &RunService_GetJobLog_Call{Call: _e.mock.On("GetJobLog", ctx, p, repo, jobID, attempt, offset, follow, send)}
+	return &RunService_GetJobLog_Call{Call: _e.mock.On("GetJobLog", ctx, p, repo, params, send)}
 }
-func (_c *RunService_GetJobLog_Call) Run(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(ports.LogBatch) error)) *RunService_GetJobLog_Call {
+func (_c *RunService_GetJobLog_Call) Run(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error)) *RunService_GetJobLog_Call {
 	_c.Call.Run(func(args mock.Arguments) {
 		var arg0 context.Context
 		if args[0] != nil {
 		if args[2] != nil {
 			arg2 = args[2].(domain.RepoName)
 		}
-		var arg3 string
+		var arg3 ports.GetJobLogParams
 		if args[3] != nil {
-			arg3 = args[3].(string)
+			arg3 = args[3].(ports.GetJobLogParams)
 		}
-		var arg4 int32
+		var arg4 func(ports.LogBatch) error
 		if args[4] != nil {
-			arg4 = args[4].(int32)
+			arg4 = args[4].(func(ports.LogBatch) error)
-		}
-		var arg5 int64
-		if args[5] != nil {
-			arg5 = args[5].(int64)
-		}
-		var arg6 bool
-		if args[6] != nil {
-			arg6 = args[6].(bool)
-		}
-		var arg7 func(ports.LogBatch) error
-		if args[7] != nil {
-			arg7 = args[7].(func(ports.LogBatch) error)
 		}
 		run(
 			arg0,
 			arg2,
 			arg3,
 			arg4,
-			arg5,
-			arg6,
-			arg7,
 		)
 	})
 	return _c
 	return _c
 }
-func (_c *RunService_GetJobLog_Call) RunAndReturn(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, jobID string, attempt int32, offset int64, follow bool, send func(ports.LogBatch) error) error) *RunService_GetJobLog_Call {
+func (_c *RunService_GetJobLog_Call) RunAndReturn(run func(ctx context.Context, p domain.Principal, repo domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error) error) *RunService_GetJobLog_Call {
 	_c.Call.Return(run)
 	return _c
 }

A internal/usecase/ci.go +529 -0

@@ -0,0 +1,529 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"crypto/rand"
	8	+	"encoding/base64"
	9	+	"encoding/hex"
	10	+	"fmt"
	11	+	"slices"
	12	+	"strings"
	13	+	"sync"
	14	+	"time"
	15	+
	16	+	"github.com/bnema/zerowrap"
	17	+
	18	+	"bnema.dev/ermite/internal/domain"
	19	+	"bnema.dev/ermite/internal/ports"
	20	+	)
	21	+
	22	+	// Defaults of the CI engine.
	23	+	const (
	24	+	// DefaultCIPollTimeout is how long FetchJob holds a call: the contract's 50 s.
	25	+	DefaultCIPollTimeout = 50 * time.Second
	26	+	// DefaultCITick is how often the watchdog looks at the jobs in flight.
	27	+	DefaultCITick = time.Second
	28	+	// DefaultCIKeepalive is how often WatchJob sends a keepalive.
	29	+	DefaultCIKeepalive = 30 * time.Second
	30	+	// DefaultCITriggerQueue is the size of the queue of pushes to plan.
	31	+	DefaultCITriggerQueue = 256
	32	+	// MaxCIChecks is the most checks ListChecks returns.
	33	+	MaxCIChecks = 1000
	34	+	// MaxCILogStreams and MaxCILogStreamsPerClient bound the concurrent log
	35	+	// streams (GetJobLog).
	36	+	MaxCILogStreams = 64
	37	+	MaxCILogStreamsPerClient = 8
	38	+	// maxCIWorkflowBytes is the largest workflow file (the parser's limit).
	39	+	maxCIWorkflowBytes = 1 << 20
	40	+	// maxCIWorkflowFiles bounds the workflow files read per push.
	41	+	maxCIWorkflowFiles = 256
	42	+	)
	43	+
	44	+	// CIConfig holds what the CI use cases need. Store, Logs, Meta, Repos, Reader,
	45	+	// Git, Diff, Bus and Clock are required.
	46	+	type CIConfig struct {
	47	+	Store ports.CIStore
	48	+	Logs ports.CILogStore
	49	+	Meta ports.RepoMetaStore
	50	+	Repos ports.GitRepos
	51	+	Reader ports.RepoReader
	52	+	Git ports.MergeGit
	53	+	Diff ports.CIGit
	54	+	Bus ports.EventBus
	55	+	Clock ports.Clock
	56	+	// Auth re-validates the token of a log stream; nil skips it.
	57	+	Auth ports.Revalidator
	58	+	// Owner is the owner login of the github context.
	59	+	Owner string
	60	+	// PublicURL is the base URL of the instance (github.server_url).
	61	+	PublicURL string
	62	+	// Retention is how long runs, jobs, checks and logs are kept.
	63	+	Retention time.Duration
	64	+
	65	+	// The durations below default to the contract's; tests shorten them.
	66	+	PollTimeout time.Duration
	67	+	Tick time.Duration
	68	+	Keepalive time.Duration
	69	+	AckTimeout time.Duration
	70	+	HeartbeatDeadline time.Duration
	71	+	CancelGrace time.Duration
	72	+	TimeoutSlack time.Duration
	73	+	QueueTimeout time.Duration
	74	+	TriggerQueue int
	75	+	}
	76	+
	77	+	func (c CIConfig) withDefaults() CIConfig {
	78	+	def := func(d *time.Duration, v time.Duration) {
	79	+	if *d <= 0 {
	80	+	*d = v
	81	+	}
	82	+	}
	83	+	def(&c.PollTimeout, DefaultCIPollTimeout)
	84	+	def(&c.Tick, DefaultCITick)
	85	+	def(&c.Keepalive, DefaultCIKeepalive)
	86	+	def(&c.AckTimeout, domain.CIAckTimeout)
	87	+	def(&c.HeartbeatDeadline, domain.CIHeartbeatDeadline)
	88	+	def(&c.CancelGrace, domain.CICancelGrace)
	89	+	def(&c.TimeoutSlack, domain.CITimeoutSlack)
	90	+	def(&c.QueueTimeout, domain.CIQueueTimeout)
	91	+	def(&c.Retention, domain.CIRetentionDays24time.Hour)
	92	+	if c.TriggerQueue <= 0 {
	93	+	c.TriggerQueue = DefaultCITriggerQueue
	94	+	}
	95	+	if c.Owner == "" {
	96	+	c.Owner = "owner"
	97	+	}
	98	+	return c
	99	+	}
	100	+
	101	+	// CI is the CI use cases: the workflow trigger, the planner, the assignment of
	102	+	// jobs to runners, the runner protocol and the owner side (ADR 0025).
	103	+	//
	104	+	// Every change of a run goes through its lock (withRun): the records of a run
	105	+	// are read, changed and written again by one goroutine at a time, so that two
	106	+	// calls (a runner's update and an owner's cancel) never lose one another's
	107	+	// write. Assignment takes the assignment lock before the lock of the run.
	108	+	type CI struct {
	109	+	cfg CIConfig
	110	+	boot time.Time
	111	+	queue chan ciTrigger
	112	+
	113	+	stop context.CancelFunc
	114	+	ctx context.Context
	115	+	wg sync.WaitGroup
	116	+
	117	+	locks sync.Map // run id -> *sync.Mutex
	118	+	assign sync.Mutex
	119	+	jobs notifier // woken when a job changes: key is the job reference
	120	+	pending notifier // woken when a job may be ready or a runner was removed
	121	+	flows *wfCache
	122	+
	123	+	mu sync.Mutex
	124	+	heartbeats map[string]time.Time // job reference -> last heartbeat
	125	+	seen map[string]time.Time // runner id -> last seen
	126	+	polls map[string]pollRecord
	127	+	streams int
	128	+	perClient map[string]int
	129	+	closed bool
	130	+	}
	131	+
	132	+	var (
	133	+	_ ports.RunnerService = (*CI)(nil)
	134	+	_ ports.RunService = (*CI)(nil)
	135	+	_ ports.WorkflowTrigger = (*CI)(nil)
	136	+	_ ports.CIChecksReader = (*CI)(nil)
	137	+	)
	138	+
	139	+	// pollRecord remembers the assignment a poll made, for the retry of a lost
	140	+	// response (FetchJobRequest.poll_id). The token is kept in memory only.
	141	+	type pollRecord struct {
	142	+	pollID string
	143	+	ref string
	144	+	token string
	145	+	hash domain.TokenHash
	146	+	}
	147	+
	148	+	// NewCI builds the CI use cases. ctx carries the logger; its cancellation is
	149	+	// not followed: call Start to run the workers and Close to stop them.
	150	+	func NewCI(ctx context.Context, cfg CIConfig) *CI {
	151	+	cfg = cfg.withDefaults()
	152	+	run, stop := context.WithCancel(context.WithoutCancel(ctx))
	153	+	return &CI{
	154	+	cfg: cfg, boot: cfg.Clock.Now(), queue: make(chan ciTrigger, cfg.TriggerQueue),
	155	+	ctx: run, stop: stop, flows: newWfCache(64),
	156	+	heartbeats: map[string]time.Time{}, seen: map[string]time.Time{}, polls: map[string]pollRecord{},
	157	+	perClient: map[string]int{},
	158	+	}
	159	+	}
	160	+
	161	+	// Start runs the trigger worker and the watchdog, and plans again the runs
	162	+	// that were open when the server stopped.
	163	+	func (c *CI) Start() {
	164	+	c.wg.Add(2)
	165	+	go c.triggerLoop()
	166	+	go c.watchdog()
	167	+	c.wg.Add(1)
	168	+	go func() {
	169	+	defer c.wg.Done()
	170	+	c.resume()
	171	+	}()
	172	+	}
	173	+
	174	+	// Close stops the workers and waits for them.
	175	+	func (c *CI) Close(_ context.Context) {
	176	+	c.mu.Lock()
	177	+	c.closed = true
	178	+	c.mu.Unlock()
	179	+	c.stop()
	180	+	c.wg.Wait()
	181	+	}
	182	+
	183	+	func (c *CI) now() time.Time { return c.cfg.Clock.Now().UTC() }
	184	+
	185	+	// --- identifiers and secrets ---
	186	+
	187	+	func newToken(prefix string) string {
	188	+	var raw [32]byte
	189	+	_, _ = rand.Read(raw[:])
	190	+	return prefix + base64.RawURLEncoding.EncodeToString(raw[:])
	191	+	}
	192	+
	193	+	func newRunnerID() string {
	194	+	var raw [8]byte
	195	+	_, _ = rand.Read(raw[:])
	196	+	return hex.EncodeToString(raw[:])
	197	+	}
	198	+
	199	+	func jobRef(j domain.CIJob) string { return fmt.Sprintf("%d/%d/%s", j.RunID, j.RunAttempt, j.ID) }
	200	+
	201	+	// --- locks and notifications ---
	202	+
	203	+	// notifier wakes the goroutines waiting on a key. A channel exists only while
	204	+	// somebody waits on it.
	205	+	type notifier struct {
	206	+	mu sync.Mutex
	207	+	ch map[string]chan struct{}
	208	+	}
	209	+
	210	+	// wait returns a channel closed by the next notify of key.
	211	+	func (n *notifier) wait(key string) <-chan struct{} {
	212	+	n.mu.Lock()
	213	+	defer n.mu.Unlock()
	214	+	if n.ch == nil {
	215	+	n.ch = map[string]chan struct{}{}
	216	+	}
	217	+	c, ok := n.ch[key]
	218	+	if !ok {
	219	+	c = make(chan struct{})
	220	+	n.ch[key] = c
	221	+	}
	222	+	return c
	223	+	}
	224	+
	225	+	func (n *notifier) notify(key string) {
	226	+	n.mu.Lock()
	227	+	defer n.mu.Unlock()
	228	+	if c, ok := n.ch[key]; ok {
	229	+	close(c)
	230	+	delete(n.ch, key)
	231	+	}
	232	+	}
	233	+
	234	+	const queueKey = "q"
	235	+
	236	+	func (c *CI) wakeQueue() { c.pending.notify(queueKey) }
	237	+
	238	+	func (c CI) runLock(runID int64) sync.Mutex {
	239	+	m, _ := c.locks.LoadOrStore(runID, &sync.Mutex{})
	240	+	return m.(*sync.Mutex)
	241	+	}
	242	+
	243	+	// --- run state ---
	244	+
	245	+	// runState is a run being changed under its lock: the run, the jobs of one of
	246	+	// its attempts, and what must be written. flush writes everything in one batch
	247	+	// and publishes the events.
	248	+	type runState struct {
	249	+	c *CI
	250	+	run domain.CIRun
	251	+	attempt int32
	252	+	jobs []domain.CIJob
	253	+	src *domain.CIRunSource
	254	+
	255	+	runDirty bool
	256	+	dirty map[string]bool
	257	+	// chk marks jobs whose check must be written even when the job itself did
	258	+	// not change (a rerun moves every check to the new attempt).
	259	+	checks map[string]bool
	260	+	}
	261	+
	262	+	// withRun locks the run, loads it with the jobs of attempt (0 for the latest)
	263	+	// and calls fn. The run is found by id alone through its source.
	264	+	func (c CI) withRun(ctx context.Context, runID int64, attempt int32, fn func(rs runState) error) error {
	265	+	lock := c.runLock(runID)
	266	+	lock.Lock()
	267	+	defer lock.Unlock()
	268	+	rs, err := c.loadRun(ctx, runID, attempt)
	269	+	if err != nil {
	270	+	return err
	271	+	}
	272	+	return fn(rs)
	273	+	}
	274	+
	275	+	func (c CI) loadRun(ctx context.Context, runID int64, attempt int32) (runState, error) {
	276	+	src, err := c.cfg.Store.GetRunSource(ctx, runID)
	277	+	if err != nil {
	278	+	return nil, err
	279	+	}
	280	+	run, err := c.cfg.Store.GetRun(ctx, src.Repo, runID)
	281	+	if err != nil {
	282	+	return nil, err
	283	+	}
	284	+	if attempt <= 0 {
	285	+	attempt = run.Attempt
	286	+	}
	287	+	jobs, err := c.cfg.Store.ListJobs(ctx, runID, attempt)
	288	+	if err != nil {
	289	+	return nil, err
	290	+	}
	291	+	return &runState{c: c, run: run, attempt: attempt, jobs: jobs, src: &src, dirty: map[string]bool{}, checks: map[string]bool{}}, nil
	292	+	}
	293	+
	294	+	// latest reports whether the state holds the latest attempt of the run.
	295	+	func (rs *runState) latest() bool { return rs.attempt == rs.run.Attempt }
	296	+
	297	+	func (rs runState) find(id string) domain.CIJob {
	298	+	for i := range rs.jobs {
	299	+	if rs.jobs[i].ID == id {
	300	+	return &rs.jobs[i]
	301	+	}
	302	+	}
	303	+	return nil
	304	+	}
	305	+
	306	+	func (rs runState) touch(j domain.CIJob) { rs.dirty[j.ID] = true }
	307	+
	308	+	// checkOf builds the check of a job.
	309	+	func (rs *runState) checkOf(j domain.CIJob) domain.CICheck {
	310	+	return domain.CICheck{
	311	+	ID: j.ID, Repo: rs.run.Repo, Sha: rs.run.Sha, Name: j.Name, Status: j.Status(), Conclusion: j.Conclusion,
	312	+	RunID: rs.run.ID, JobID: j.ID, Attempt: rs.run.Attempt, WorkflowName: rs.run.WorkflowName,
	313	+	CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, CompletedAt: j.CompletedAt,
	314	+	}
	315	+	}
	316	+
	317	+	// flush recomputes the status of the run, writes the changed jobs, their
	318	+	// checks and the run in one batch, and publishes RunUpdated and JobUpdated.
	319	+	func (rs *runState) flush(ctx context.Context) error {
	320	+	c := rs.c
	321	+	now := c.now()
	322	+	var w ports.CIWrite
	323	+	var changedJobs []int
	324	+	for i := range rs.jobs {
	325	+	if rs.dirty[rs.jobs[i].ID] {
	326	+	w.Jobs = append(w.Jobs, rs.jobs[i])
	327	+	changedJobs = append(changedJobs, i)
	328	+	}
	329	+	}
	330	+	if rs.latest() {
	331	+	for i := range rs.jobs {
	332	+	if rs.dirty[rs.jobs[i].ID] \|\| rs.checks[rs.jobs[i].ID] {
	333	+	w.Checks = append(w.Checks, rs.checkOf(rs.jobs[i]))
	334	+	}
	335	+	}
	336	+	if next := rs.recompute(now); next != rs.run {
	337	+	rs.run, rs.runDirty = next, true
	338	+	}
	339	+	}
	340	+	if rs.runDirty {
	341	+	w.Runs = []domain.CIRun{rs.run}
	342	+	}
	343	+	if len(w.Jobs) == 0 && len(w.Runs) == 0 && len(w.Checks) == 0 {
	344	+	return nil
	345	+	}
	346	+	v, err := c.cfg.Store.Apply(ctx, w)
	347	+	if err != nil {
	348	+	return fmt.Errorf("write run %d: %w", rs.run.ID, err)
	349	+	}
	350	+	for _, i := range changedJobs {
	351	+	rs.jobs[i].Version = v
	352	+	}
	353	+	if rs.runDirty {
	354	+	rs.run.Version = v
	355	+	}
	356	+	changedRun := rs.runDirty
	357	+	rs.dirty, rs.checks, rs.runDirty = map[string]bool{}, map[string]bool{}, false
	358	+
	359	+	repo, err := c.cfg.Meta.Get(ctx, rs.run.Repo)
	360	+	if err != nil {
	361	+	for _, i := range changedJobs {
	362	+	c.jobs.notify(jobRef(rs.jobs[i]))
	363	+	}
	364	+	return nil // the repository is gone: nobody to tell
	365	+	}
	366	+	for _, i := range changedJobs {
	367	+	j := rs.jobs[i]
	368	+	lines, _ := c.cfg.Logs.Count(ctx, ports.CILogRef{RunID: j.RunID, JobID: j.ID, Attempt: j.Attempt})
	369	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, Lines: lines, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	370	+	c.jobs.notify(jobRef(j))
	371	+	if j.Phase == domain.CIPhaseReady {
	372	+	c.wakeQueue()
	373	+	}
	374	+	}
	375	+	if changedRun {
	376	+	c.cfg.Bus.Publish(ctx, domain.RunUpdated{Run: rs.run, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	377	+	}
	378	+	return nil
	379	+	}
	380	+
	381	+	// recompute returns the run with the status, the conclusion and the
	382	+	// timestamps its jobs imply.
	383	+	func (rs *runState) recompute(now time.Time) domain.CIRun {
	384	+	run := rs.run
	385	+	if run.Status == domain.CICompleted && len(rs.jobs) == 0 {
	386	+	return run // an invalid workflow file: failed from the start
	387	+	}
	388	+	allEnded := len(rs.jobs) > 0
	389	+	started := time.Time{}
	390	+	var failed, cancelled bool
	391	+	for _, j := range rs.jobs {
	392	+	if !j.Ended() {
	393	+	allEnded = false
	394	+	}
	395	+	// A job kept from an earlier attempt did not start in this one.
	396	+	if j.Attempt == rs.attempt && !j.StartedAt.IsZero() && (started.IsZero() \|\| j.StartedAt.Before(started)) {
	397	+	started = j.StartedAt
	398	+	}
	399	+	switch {
	400	+	case j.Conclusion.IsFailure():
	401	+	failed = true
	402	+	case j.Conclusion == domain.CICancelled:
	403	+	cancelled = true
	404	+	}
	405	+	}
	406	+	if run.StartedAt.IsZero() && !started.IsZero() {
	407	+	run.StartedAt = started
	408	+	}
	409	+	switch {
	410	+	case allEnded:
	411	+	run.Status, run.CompletedAt = domain.CICompleted, now
	412	+	switch {
	413	+	case run.CancelRequested:
	414	+	run.Conclusion = domain.CICancelled
	415	+	case failed:
	416	+	run.Conclusion = domain.CIFailure
	417	+	case cancelled:
	418	+	run.Conclusion = domain.CICancelled
	419	+	default:
	420	+	run.Conclusion = domain.CISuccess
	421	+	}
	422	+	if rs.run.Status == domain.CICompleted {
	423	+	run.CompletedAt = rs.run.CompletedAt
	424	+	}
	425	+	case !run.StartedAt.IsZero():
	426	+	run.Status, run.Conclusion, run.CompletedAt = domain.CIInProgress, domain.CINoConclusion, time.Time{}
	427	+	default:
	428	+	run.Status, run.Conclusion, run.CompletedAt = domain.CIQueued, domain.CINoConclusion, time.Time{}
	429	+	}
	430	+	return run
	431	+	}
	432	+
	433	+	// --- access ---
	434	+
	435	+	// readableRepo looks the repository up and applies the read rule.
	436	+	func (c *CI) readableRepo(ctx context.Context, p domain.Principal, name domain.RepoName) (domain.Repo, error) {
	437	+	repo, err := lookupRepo(ctx, c.cfg.Meta, name)
	438	+	if err != nil {
	439	+	return domain.Repo{}, err
	440	+	}
	441	+	if !canRead(p, repo) {
	442	+	return domain.Repo{}, domain.ErrNotFound
	443	+	}
	444	+	return repo, nil
	445	+	}
	446	+
	447	+	// writableRepo is the access rule of cancelling and re-running: a principal
	448	+	// whose token holds ci:write on the repository sees the repository for this
	449	+	// action even without repo:read, as for merge requests.
	450	+	func (c *CI) writableRepo(ctx context.Context, p domain.Principal, name domain.RepoName) (domain.Repo, error) {
	451	+	repo, err := lookupRepo(ctx, c.cfg.Meta, name)
	452	+	if err != nil {
	453	+	return domain.Repo{}, err
	454	+	}
	455	+	if !canRead(p, repo) && !p.Allows(domain.ScopeCIWrite, name) {
	456	+	return domain.Repo{}, domain.ErrNotFound
	457	+	}
	458	+	if err := scopeAccess(p, repo, domain.ScopeCIWrite); err != nil {
	459	+	return domain.Repo{}, err
	460	+	}
	461	+	return repo, nil
	462	+	}
	463	+
	464	+	// canManageRunners is the access rule of runners: runner:register on a token
	465	+	// that is not restricted to repositories (a runner serves every repository),
	466	+	// or root.
	467	+	func canManageRunners(p domain.Principal) error {
	468	+	switch {
	469	+	case p.IsAnonymous():
	470	+	return domain.ErrUnauthenticated
	471	+	case !p.HasScope(domain.ScopeRunnerRegister):
	472	+	return fmt.Errorf("%w: this needs the %s scope", domain.ErrForbidden, domain.ScopeRunnerRegister)
	473	+	case len(p.Repos()) > 0:
	474	+	return fmt.Errorf("%w: a runner serves every repository: the token must not be restricted to some", domain.ErrForbidden)
	475	+	}
	476	+	return nil
	477	+	}
	478	+
	479	+	// --- small helpers ---
	480	+
	481	+	func truncateRunes(s string, n int) string {
	482	+	if len(s) <= n {
	483	+	return s
	484	+	}
	485	+	r := []rune(s)
	486	+	if len(r) <= n {
	487	+	return s
	488	+	}
	489	+	return string(r[:n])
	490	+	}
	491	+
	492	+	// cutText cuts s to n bytes at a character boundary.
	493	+	func cutText(s string, n int) string {
	494	+	if len(s) <= n {
	495	+	return s
	496	+	}
	497	+	for n > 0 && n < len(s) && s[n]&0xC0 == 0x80 {
	498	+	n--
	499	+	}
	500	+	return s[:n]
	501	+	}
	502	+
	503	+	func sortedKeys[V any](m map[string]V) []string {
	504	+	keys := make([]string, 0, len(m))
	505	+	for k := range m {
	506	+	keys = append(keys, k)
	507	+	}
	508	+	slices.Sort(keys)
	509	+	return keys
	510	+	}
	511	+
	512	+	// ctxLog returns the logger of the context, addressable so that the level
	513	+	// methods can be chained on it.
	514	+	func ctxLog(ctx context.Context) *zerowrap.Logger {
	515	+	l := zerowrap.FromCtx(ctx)
	516	+	return &l
	517	+	}
	518	+
	519	+	// logRun returns the logger of the context with the fields of the run.
	520	+	func logRun(ctx context.Context, run domain.CIRun) zerowrap.Logger {
	521	+	return zerowrap.FromCtx(ctx).WithFields(map[string]any{"repo": run.Repo.String(), "run_id": run.ID})
	522	+	}
	523	+
	524	+	func isHex40(s string) bool {
	525	+	if len(s) != 40 {
	526	+	return false
	527	+	}
	528	+	return strings.Trim(s, "0123456789abcdef") == ""
	529	+	}

@@ -0,0 +1,529 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"crypto/rand"
	8	+	"encoding/base64"
	9	+	"encoding/hex"
	10	+	"fmt"
	11	+	"slices"
	12	+	"strings"
	13	+	"sync"
	14	+	"time"
	15	+
	16	+	"github.com/bnema/zerowrap"
	17	+
	18	+	"bnema.dev/ermite/internal/domain"
	19	+	"bnema.dev/ermite/internal/ports"
	20	+	)
	21	+
	22	+	// Defaults of the CI engine.
	23	+	const (
	24	+	// DefaultCIPollTimeout is how long FetchJob holds a call: the contract's 50 s.
	25	+	DefaultCIPollTimeout = 50 * time.Second
	26	+	// DefaultCITick is how often the watchdog looks at the jobs in flight.
	27	+	DefaultCITick = time.Second
	28	+	// DefaultCIKeepalive is how often WatchJob sends a keepalive.
	29	+	DefaultCIKeepalive = 30 * time.Second
	30	+	// DefaultCITriggerQueue is the size of the queue of pushes to plan.
	31	+	DefaultCITriggerQueue = 256
	32	+	// MaxCIChecks is the most checks ListChecks returns.
	33	+	MaxCIChecks = 1000
	34	+	// MaxCILogStreams and MaxCILogStreamsPerClient bound the concurrent log
	35	+	// streams (GetJobLog).
	36	+	MaxCILogStreams = 64
	37	+	MaxCILogStreamsPerClient = 8
	38	+	// maxCIWorkflowBytes is the largest workflow file (the parser's limit).
	39	+	maxCIWorkflowBytes = 1 << 20
	40	+	// maxCIWorkflowFiles bounds the workflow files read per push.
	41	+	maxCIWorkflowFiles = 256
	42	+	)
	43	+
	44	+	// CIConfig holds what the CI use cases need. Store, Logs, Meta, Repos, Reader,
	45	+	// Git, Diff, Bus and Clock are required.
	46	+	type CIConfig struct {
	47	+	Store ports.CIStore
	48	+	Logs ports.CILogStore
	49	+	Meta ports.RepoMetaStore
	50	+	Repos ports.GitRepos
	51	+	Reader ports.RepoReader
	52	+	Git ports.MergeGit
	53	+	Diff ports.CIGit
	54	+	Bus ports.EventBus
	55	+	Clock ports.Clock
	56	+	// Auth re-validates the token of a log stream; nil skips it.
	57	+	Auth ports.Revalidator
	58	+	// Owner is the owner login of the github context.
	59	+	Owner string
	60	+	// PublicURL is the base URL of the instance (github.server_url).
	61	+	PublicURL string
	62	+	// Retention is how long runs, jobs, checks and logs are kept.
	63	+	Retention time.Duration
	64	+
	65	+	// The durations below default to the contract's; tests shorten them.
	66	+	PollTimeout time.Duration
	67	+	Tick time.Duration
	68	+	Keepalive time.Duration
	69	+	AckTimeout time.Duration
	70	+	HeartbeatDeadline time.Duration
	71	+	CancelGrace time.Duration
	72	+	TimeoutSlack time.Duration
	73	+	QueueTimeout time.Duration
	74	+	TriggerQueue int
	75	+	}
	76	+
	77	+	func (c CIConfig) withDefaults() CIConfig {
	78	+	def := func(d *time.Duration, v time.Duration) {
	79	+	if *d <= 0 {
	80	+	*d = v
	81	+	}
	82	+	}
	83	+	def(&c.PollTimeout, DefaultCIPollTimeout)
	84	+	def(&c.Tick, DefaultCITick)
	85	+	def(&c.Keepalive, DefaultCIKeepalive)
	86	+	def(&c.AckTimeout, domain.CIAckTimeout)
	87	+	def(&c.HeartbeatDeadline, domain.CIHeartbeatDeadline)
	88	+	def(&c.CancelGrace, domain.CICancelGrace)
	89	+	def(&c.TimeoutSlack, domain.CITimeoutSlack)
	90	+	def(&c.QueueTimeout, domain.CIQueueTimeout)
	91	+	def(&c.Retention, domain.CIRetentionDays24time.Hour)
	92	+	if c.TriggerQueue <= 0 {
	93	+	c.TriggerQueue = DefaultCITriggerQueue
	94	+	}
	95	+	if c.Owner == "" {
	96	+	c.Owner = "owner"
	97	+	}
	98	+	return c
	99	+	}
	100	+
	101	+	// CI is the CI use cases: the workflow trigger, the planner, the assignment of
	102	+	// jobs to runners, the runner protocol and the owner side (ADR 0025).
	103	+	//
	104	+	// Every change of a run goes through its lock (withRun): the records of a run
	105	+	// are read, changed and written again by one goroutine at a time, so that two
	106	+	// calls (a runner's update and an owner's cancel) never lose one another's
	107	+	// write. Assignment takes the assignment lock before the lock of the run.
	108	+	type CI struct {
	109	+	cfg CIConfig
	110	+	boot time.Time
	111	+	queue chan ciTrigger
	112	+
	113	+	stop context.CancelFunc
	114	+	ctx context.Context
	115	+	wg sync.WaitGroup
	116	+
	117	+	locks sync.Map // run id -> *sync.Mutex
	118	+	assign sync.Mutex
	119	+	jobs notifier // woken when a job changes: key is the job reference
	120	+	pending notifier // woken when a job may be ready or a runner was removed
	121	+	flows *wfCache
	122	+
	123	+	mu sync.Mutex
	124	+	heartbeats map[string]time.Time // job reference -> last heartbeat
	125	+	seen map[string]time.Time // runner id -> last seen
	126	+	polls map[string]pollRecord
	127	+	streams int
	128	+	perClient map[string]int
	129	+	closed bool
	130	+	}
	131	+
	132	+	var (
	133	+	_ ports.RunnerService = (*CI)(nil)
	134	+	_ ports.RunService = (*CI)(nil)
	135	+	_ ports.WorkflowTrigger = (*CI)(nil)
	136	+	_ ports.CIChecksReader = (*CI)(nil)
	137	+	)
	138	+
	139	+	// pollRecord remembers the assignment a poll made, for the retry of a lost
	140	+	// response (FetchJobRequest.poll_id). The token is kept in memory only.
	141	+	type pollRecord struct {
	142	+	pollID string
	143	+	ref string
	144	+	token string
	145	+	hash domain.TokenHash
	146	+	}
	147	+
	148	+	// NewCI builds the CI use cases. ctx carries the logger; its cancellation is
	149	+	// not followed: call Start to run the workers and Close to stop them.
	150	+	func NewCI(ctx context.Context, cfg CIConfig) *CI {
	151	+	cfg = cfg.withDefaults()
	152	+	run, stop := context.WithCancel(context.WithoutCancel(ctx))
	153	+	return &CI{
	154	+	cfg: cfg, boot: cfg.Clock.Now(), queue: make(chan ciTrigger, cfg.TriggerQueue),
	155	+	ctx: run, stop: stop, flows: newWfCache(64),
	156	+	heartbeats: map[string]time.Time{}, seen: map[string]time.Time{}, polls: map[string]pollRecord{},
	157	+	perClient: map[string]int{},
	158	+	}
	159	+	}
	160	+
	161	+	// Start runs the trigger worker and the watchdog, and plans again the runs
	162	+	// that were open when the server stopped.
	163	+	func (c *CI) Start() {
	164	+	c.wg.Add(2)
	165	+	go c.triggerLoop()
	166	+	go c.watchdog()
	167	+	c.wg.Add(1)
	168	+	go func() {
	169	+	defer c.wg.Done()
	170	+	c.resume()
	171	+	}()
	172	+	}
	173	+
	174	+	// Close stops the workers and waits for them.
	175	+	func (c *CI) Close(_ context.Context) {
	176	+	c.mu.Lock()
	177	+	c.closed = true
	178	+	c.mu.Unlock()
	179	+	c.stop()
	180	+	c.wg.Wait()
	181	+	}
	182	+
	183	+	func (c *CI) now() time.Time { return c.cfg.Clock.Now().UTC() }
	184	+
	185	+	// --- identifiers and secrets ---
	186	+
	187	+	func newToken(prefix string) string {
	188	+	var raw [32]byte
	189	+	_, _ = rand.Read(raw[:])
	190	+	return prefix + base64.RawURLEncoding.EncodeToString(raw[:])
	191	+	}
	192	+
	193	+	func newRunnerID() string {
	194	+	var raw [8]byte
	195	+	_, _ = rand.Read(raw[:])
	196	+	return hex.EncodeToString(raw[:])
	197	+	}
	198	+
	199	+	func jobRef(j domain.CIJob) string { return fmt.Sprintf("%d/%d/%s", j.RunID, j.RunAttempt, j.ID) }
	200	+
	201	+	// --- locks and notifications ---
	202	+
	203	+	// notifier wakes the goroutines waiting on a key. A channel exists only while
	204	+	// somebody waits on it.
	205	+	type notifier struct {
	206	+	mu sync.Mutex
	207	+	ch map[string]chan struct{}
	208	+	}
	209	+
	210	+	// wait returns a channel closed by the next notify of key.
	211	+	func (n *notifier) wait(key string) <-chan struct{} {
	212	+	n.mu.Lock()
	213	+	defer n.mu.Unlock()
	214	+	if n.ch == nil {
	215	+	n.ch = map[string]chan struct{}{}
	216	+	}
	217	+	c, ok := n.ch[key]
	218	+	if !ok {
	219	+	c = make(chan struct{})
	220	+	n.ch[key] = c
	221	+	}
	222	+	return c
	223	+	}
	224	+
	225	+	func (n *notifier) notify(key string) {
	226	+	n.mu.Lock()
	227	+	defer n.mu.Unlock()
	228	+	if c, ok := n.ch[key]; ok {
	229	+	close(c)
	230	+	delete(n.ch, key)
	231	+	}
	232	+	}
	233	+
	234	+	const queueKey = "q"
	235	+
	236	+	func (c *CI) wakeQueue() { c.pending.notify(queueKey) }
	237	+
	238	+	func (c CI) runLock(runID int64) sync.Mutex {
	239	+	m, _ := c.locks.LoadOrStore(runID, &sync.Mutex{})
	240	+	return m.(*sync.Mutex)
	241	+	}
	242	+
	243	+	// --- run state ---
	244	+
	245	+	// runState is a run being changed under its lock: the run, the jobs of one of
	246	+	// its attempts, and what must be written. flush writes everything in one batch
	247	+	// and publishes the events.
	248	+	type runState struct {
	249	+	c *CI
	250	+	run domain.CIRun
	251	+	attempt int32
	252	+	jobs []domain.CIJob
	253	+	src *domain.CIRunSource
	254	+
	255	+	runDirty bool
	256	+	dirty map[string]bool
	257	+	// chk marks jobs whose check must be written even when the job itself did
	258	+	// not change (a rerun moves every check to the new attempt).
	259	+	checks map[string]bool
	260	+	}
	261	+
	262	+	// withRun locks the run, loads it with the jobs of attempt (0 for the latest)
	263	+	// and calls fn. The run is found by id alone through its source.
	264	+	func (c CI) withRun(ctx context.Context, runID int64, attempt int32, fn func(rs runState) error) error {
	265	+	lock := c.runLock(runID)
	266	+	lock.Lock()
	267	+	defer lock.Unlock()
	268	+	rs, err := c.loadRun(ctx, runID, attempt)
	269	+	if err != nil {
	270	+	return err
	271	+	}
	272	+	return fn(rs)
	273	+	}
	274	+
	275	+	func (c CI) loadRun(ctx context.Context, runID int64, attempt int32) (runState, error) {
	276	+	src, err := c.cfg.Store.GetRunSource(ctx, runID)
	277	+	if err != nil {
	278	+	return nil, err
	279	+	}
	280	+	run, err := c.cfg.Store.GetRun(ctx, src.Repo, runID)
	281	+	if err != nil {
	282	+	return nil, err
	283	+	}
	284	+	if attempt <= 0 {
	285	+	attempt = run.Attempt
	286	+	}
	287	+	jobs, err := c.cfg.Store.ListJobs(ctx, runID, attempt)
	288	+	if err != nil {
	289	+	return nil, err
	290	+	}
	291	+	return &runState{c: c, run: run, attempt: attempt, jobs: jobs, src: &src, dirty: map[string]bool{}, checks: map[string]bool{}}, nil
	292	+	}
	293	+
	294	+	// latest reports whether the state holds the latest attempt of the run.
	295	+	func (rs *runState) latest() bool { return rs.attempt == rs.run.Attempt }
	296	+
	297	+	func (rs runState) find(id string) domain.CIJob {
	298	+	for i := range rs.jobs {
	299	+	if rs.jobs[i].ID == id {
	300	+	return &rs.jobs[i]
	301	+	}
	302	+	}
	303	+	return nil
	304	+	}
	305	+
	306	+	func (rs runState) touch(j domain.CIJob) { rs.dirty[j.ID] = true }
	307	+
	308	+	// checkOf builds the check of a job.
	309	+	func (rs *runState) checkOf(j domain.CIJob) domain.CICheck {
	310	+	return domain.CICheck{
	311	+	ID: j.ID, Repo: rs.run.Repo, Sha: rs.run.Sha, Name: j.Name, Status: j.Status(), Conclusion: j.Conclusion,
	312	+	RunID: rs.run.ID, JobID: j.ID, Attempt: rs.run.Attempt, WorkflowName: rs.run.WorkflowName,
	313	+	CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, CompletedAt: j.CompletedAt,
	314	+	}
	315	+	}
	316	+
	317	+	// flush recomputes the status of the run, writes the changed jobs, their
	318	+	// checks and the run in one batch, and publishes RunUpdated and JobUpdated.
	319	+	func (rs *runState) flush(ctx context.Context) error {
	320	+	c := rs.c
	321	+	now := c.now()
	322	+	var w ports.CIWrite
	323	+	var changedJobs []int
	324	+	for i := range rs.jobs {
	325	+	if rs.dirty[rs.jobs[i].ID] {
	326	+	w.Jobs = append(w.Jobs, rs.jobs[i])
	327	+	changedJobs = append(changedJobs, i)
	328	+	}
	329	+	}
	330	+	if rs.latest() {
	331	+	for i := range rs.jobs {
	332	+	if rs.dirty[rs.jobs[i].ID] \|\| rs.checks[rs.jobs[i].ID] {
	333	+	w.Checks = append(w.Checks, rs.checkOf(rs.jobs[i]))
	334	+	}
	335	+	}
	336	+	if next := rs.recompute(now); next != rs.run {
	337	+	rs.run, rs.runDirty = next, true
	338	+	}
	339	+	}
	340	+	if rs.runDirty {
	341	+	w.Runs = []domain.CIRun{rs.run}
	342	+	}
	343	+	if len(w.Jobs) == 0 && len(w.Runs) == 0 && len(w.Checks) == 0 {
	344	+	return nil
	345	+	}
	346	+	v, err := c.cfg.Store.Apply(ctx, w)
	347	+	if err != nil {
	348	+	return fmt.Errorf("write run %d: %w", rs.run.ID, err)
	349	+	}
	350	+	for _, i := range changedJobs {
	351	+	rs.jobs[i].Version = v
	352	+	}
	353	+	if rs.runDirty {
	354	+	rs.run.Version = v
	355	+	}
	356	+	changedRun := rs.runDirty
	357	+	rs.dirty, rs.checks, rs.runDirty = map[string]bool{}, map[string]bool{}, false
	358	+
	359	+	repo, err := c.cfg.Meta.Get(ctx, rs.run.Repo)
	360	+	if err != nil {
	361	+	for _, i := range changedJobs {
	362	+	c.jobs.notify(jobRef(rs.jobs[i]))
	363	+	}
	364	+	return nil // the repository is gone: nobody to tell
	365	+	}
	366	+	for _, i := range changedJobs {
	367	+	j := rs.jobs[i]
	368	+	lines, _ := c.cfg.Logs.Count(ctx, ports.CILogRef{RunID: j.RunID, JobID: j.ID, Attempt: j.Attempt})
	369	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, Lines: lines, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	370	+	c.jobs.notify(jobRef(j))
	371	+	if j.Phase == domain.CIPhaseReady {
	372	+	c.wakeQueue()
	373	+	}
	374	+	}
	375	+	if changedRun {
	376	+	c.cfg.Bus.Publish(ctx, domain.RunUpdated{Run: rs.run, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	377	+	}
	378	+	return nil
	379	+	}
	380	+
	381	+	// recompute returns the run with the status, the conclusion and the
	382	+	// timestamps its jobs imply.
	383	+	func (rs *runState) recompute(now time.Time) domain.CIRun {
	384	+	run := rs.run
	385	+	if run.Status == domain.CICompleted && len(rs.jobs) == 0 {
	386	+	return run // an invalid workflow file: failed from the start
	387	+	}
	388	+	allEnded := len(rs.jobs) > 0
	389	+	started := time.Time{}
	390	+	var failed, cancelled bool
	391	+	for _, j := range rs.jobs {
	392	+	if !j.Ended() {
	393	+	allEnded = false
	394	+	}
	395	+	// A job kept from an earlier attempt did not start in this one.
	396	+	if j.Attempt == rs.attempt && !j.StartedAt.IsZero() && (started.IsZero() \|\| j.StartedAt.Before(started)) {
	397	+	started = j.StartedAt
	398	+	}
	399	+	switch {
	400	+	case j.Conclusion.IsFailure():
	401	+	failed = true
	402	+	case j.Conclusion == domain.CICancelled:
	403	+	cancelled = true
	404	+	}
	405	+	}
	406	+	if run.StartedAt.IsZero() && !started.IsZero() {
	407	+	run.StartedAt = started
	408	+	}
	409	+	switch {
	410	+	case allEnded:
	411	+	run.Status, run.CompletedAt = domain.CICompleted, now
	412	+	switch {
	413	+	case run.CancelRequested:
	414	+	run.Conclusion = domain.CICancelled
	415	+	case failed:
	416	+	run.Conclusion = domain.CIFailure
	417	+	case cancelled:
	418	+	run.Conclusion = domain.CICancelled
	419	+	default:
	420	+	run.Conclusion = domain.CISuccess
	421	+	}
	422	+	if rs.run.Status == domain.CICompleted {
	423	+	run.CompletedAt = rs.run.CompletedAt
	424	+	}
	425	+	case !run.StartedAt.IsZero():
	426	+	run.Status, run.Conclusion, run.CompletedAt = domain.CIInProgress, domain.CINoConclusion, time.Time{}
	427	+	default:
	428	+	run.Status, run.Conclusion, run.CompletedAt = domain.CIQueued, domain.CINoConclusion, time.Time{}
	429	+	}
	430	+	return run
	431	+	}
	432	+
	433	+	// --- access ---
	434	+
	435	+	// readableRepo looks the repository up and applies the read rule.
	436	+	func (c *CI) readableRepo(ctx context.Context, p domain.Principal, name domain.RepoName) (domain.Repo, error) {
	437	+	repo, err := lookupRepo(ctx, c.cfg.Meta, name)
	438	+	if err != nil {
	439	+	return domain.Repo{}, err
	440	+	}
	441	+	if !canRead(p, repo) {
	442	+	return domain.Repo{}, domain.ErrNotFound
	443	+	}
	444	+	return repo, nil
	445	+	}
	446	+
	447	+	// writableRepo is the access rule of cancelling and re-running: a principal
	448	+	// whose token holds ci:write on the repository sees the repository for this
	449	+	// action even without repo:read, as for merge requests.
	450	+	func (c *CI) writableRepo(ctx context.Context, p domain.Principal, name domain.RepoName) (domain.Repo, error) {
	451	+	repo, err := lookupRepo(ctx, c.cfg.Meta, name)
	452	+	if err != nil {
	453	+	return domain.Repo{}, err
	454	+	}
	455	+	if !canRead(p, repo) && !p.Allows(domain.ScopeCIWrite, name) {
	456	+	return domain.Repo{}, domain.ErrNotFound
	457	+	}
	458	+	if err := scopeAccess(p, repo, domain.ScopeCIWrite); err != nil {
	459	+	return domain.Repo{}, err
	460	+	}
	461	+	return repo, nil
	462	+	}
	463	+
	464	+	// canManageRunners is the access rule of runners: runner:register on a token
	465	+	// that is not restricted to repositories (a runner serves every repository),
	466	+	// or root.
	467	+	func canManageRunners(p domain.Principal) error {
	468	+	switch {
	469	+	case p.IsAnonymous():
	470	+	return domain.ErrUnauthenticated
	471	+	case !p.HasScope(domain.ScopeRunnerRegister):
	472	+	return fmt.Errorf("%w: this needs the %s scope", domain.ErrForbidden, domain.ScopeRunnerRegister)
	473	+	case len(p.Repos()) > 0:
	474	+	return fmt.Errorf("%w: a runner serves every repository: the token must not be restricted to some", domain.ErrForbidden)
	475	+	}
	476	+	return nil
	477	+	}
	478	+
	479	+	// --- small helpers ---
	480	+
	481	+	func truncateRunes(s string, n int) string {
	482	+	if len(s) <= n {
	483	+	return s
	484	+	}
	485	+	r := []rune(s)
	486	+	if len(r) <= n {
	487	+	return s
	488	+	}
	489	+	return string(r[:n])
	490	+	}
	491	+
	492	+	// cutText cuts s to n bytes at a character boundary.
	493	+	func cutText(s string, n int) string {
	494	+	if len(s) <= n {
	495	+	return s
	496	+	}
	497	+	for n > 0 && n < len(s) && s[n]&0xC0 == 0x80 {
	498	+	n--
	499	+	}
	500	+	return s[:n]
	501	+	}
	502	+
	503	+	func sortedKeys[V any](m map[string]V) []string {
	504	+	keys := make([]string, 0, len(m))
	505	+	for k := range m {
	506	+	keys = append(keys, k)
	507	+	}
	508	+	slices.Sort(keys)
	509	+	return keys
	510	+	}
	511	+
	512	+	// ctxLog returns the logger of the context, addressable so that the level
	513	+	// methods can be chained on it.
	514	+	func ctxLog(ctx context.Context) *zerowrap.Logger {
	515	+	l := zerowrap.FromCtx(ctx)
	516	+	return &l
	517	+	}
	518	+
	519	+	// logRun returns the logger of the context with the fields of the run.
	520	+	func logRun(ctx context.Context, run domain.CIRun) zerowrap.Logger {
	521	+	return zerowrap.FromCtx(ctx).WithFields(map[string]any{"repo": run.Repo.String(), "run_id": run.ID})
	522	+	}
	523	+
	524	+	func isHex40(s string) bool {
	525	+	if len(s) != 40 {
	526	+	return false
	527	+	}
	528	+	return strings.Trim(s, "0123456789abcdef") == ""
	529	+	}

A internal/usecase/ci_plan.go +489 -0

@@ -0,0 +1,489 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"encoding/json"
	9	+	"fmt"
	10	+	"math"
	11	+	"slices"
	12	+	"strconv"
	13	+	"strings"
	14	+	"sync"
	15	+
	16	+	"bnema.dev/ermite-actions/expr"
	17	+	"bnema.dev/ermite-actions/template"
	18	+	"bnema.dev/ermite-actions/workflow"
	19	+	"bnema.dev/ermite/internal/domain"
	20	+	)
	21	+
	22	+	// maxJobError bounds JobError text (4 KiB, run.proto).
	23	+	const maxJobError = 4 << 10
	24	+
	25	+	// wfCache keeps the parsed workflows of the runs in flight, so that planning a
	26	+	// job does not parse the file again.
	27	+	type wfCache struct {
	28	+	mu sync.Mutex
	29	+	max int
	30	+	m map[int64]*workflow.Workflow
	31	+	}
	32	+
	33	+	func newWfCache(max int) wfCache { return &wfCache{max: max, m: map[int64]workflow.Workflow{}} }
	34	+
	35	+	func (w wfCache) get(runID int64, path, yaml string) (workflow.Workflow, error) {
	36	+	w.mu.Lock()
	37	+	if wf, ok := w.m[runID]; ok {
	38	+	w.mu.Unlock()
	39	+	return wf, nil
	40	+	}
	41	+	w.mu.Unlock()
	42	+	wf, errs := workflow.Parse(path, []byte(yaml))
	43	+	if len(errs) > 0 {
	44	+	return nil, fmt.Errorf("the workflow file no longer parses: %s", errs[0].Error())
	45	+	}
	46	+	w.mu.Lock()
	47	+	defer w.mu.Unlock()
	48	+	if len(w.m) >= w.max {
	49	+	clear(w.m)
	50	+	}
	51	+	w.m[runID] = wf
	52	+	return wf, nil
	53	+	}
	54	+
	55	+	// unsupported names the feature of a job that CI-1 does not run, or "". Such
	56	+	// a job fails with "not supported yet: <feature>" instead of running wrong.
	57	+	// Relax it as the runner learns the features (ADR 0025, CI-2 and CI-3).
	58	+	func unsupported(j *workflow.Job) string {
	59	+	switch {
	60	+	case j.IsReusable():
	61	+	return "reusable workflows (uses at the job level)"
	62	+	case j.Container != nil:
	63	+	return "container"
	64	+	case j.Services != nil:
	65	+	return "services"
	66	+	case j.Environment != nil:
	67	+	return "environment"
	68	+	case j.Concurrency != nil:
	69	+	return "concurrency"
	70	+	case present(j.ContinueOnError):
	71	+	return "continue-on-error"
	72	+	}
	73	+	for _, s := range j.Steps {
	74	+	switch {
	75	+	case s.Uses != nil:
	76	+	return "uses: " + s.Uses.Value
	77	+	case present(s.ContinueOnError):
	78	+	return "continue-on-error on a step"
	79	+	case present(s.TimeoutMinutes):
	80	+	return "timeout-minutes on a step"
	81	+	}
	82	+	}
	83	+	return ""
	84	+	}
	85	+
	86	+	// present reports whether an optional key is given and is not a literal
	87	+	// false (continue-on-error: false is the default).
	88	+	func present(t template.Token) bool {
	89	+	if t == nil {
	90	+	return false
	91	+	}
	92	+	if b, ok := t.(*template.Boolean); ok && !b.Value {
	93	+	return false
	94	+	}
	95	+	return true
	96	+	}
	97	+
	98	+	// instances returns the indexes of the records of the job key, in order.
	99	+	func (rs *runState) instances(key string) []int {
	100	+	var out []int
	101	+	for i, j := range rs.jobs {
	102	+	if j.Key == key {
	103	+	out = append(out, i)
	104	+	}
	105	+	}
	106	+	return out
	107	+	}
	108	+
	109	+	func (rs *runState) sortJobs() {
	110	+	slices.SortStableFunc(rs.jobs, func(a, b domain.CIJob) int {
	111	+	if a.Index != b.Index {
	112	+	return a.Index - b.Index
	113	+	}
	114	+	return a.Combo - b.Combo
	115	+	})
	116	+	}
	117	+
	118	+	// result is the aggregate conclusion of the instances of a job as the needs
	119	+	// context shows it: success, failure, cancelled or skipped.
	120	+	func result(insts []domain.CIJob) string {
	121	+	var failed, cancelled, ok bool
	122	+	for _, j := range insts {
	123	+	switch j.Conclusion {
	124	+	case domain.CIFailure, domain.CITimedOut, domain.CIActionRequired:
	125	+	failed = true
	126	+	case domain.CICancelled:
	127	+	cancelled = true
	128	+	case domain.CISkipped:
	129	+	default:
	130	+	ok = true
	131	+	}
	132	+	}
	133	+	switch {
	134	+	case failed:
	135	+	return "failure"
	136	+	case cancelled:
	137	+	return "cancelled"
	138	+	case ok:
	139	+	return "success"
	140	+	}
	141	+	return "skipped"
	142	+	}
	143	+
	144	+	// needsOf computes the needs context and the status functions of a job from
	145	+	// the jobs it needs, and whether they all have ended.
	146	+	func (rs runState) needsOf(j workflow.Job) (needs *expr.Dictionary, st workflow.StatusFuncs, ready bool) {
	147	+	needs = expr.NewDictionary()
	148	+	st = workflow.StatusFuncs{Success: true, Cancelled: rs.run.CancelRequested}
	149	+	for _, n := range j.Needs {
	150	+	var insts []domain.CIJob
	151	+	for _, i := range rs.instances(n) {
	152	+	if !rs.jobs[i].Ended() {
	153	+	return nil, st, false
	154	+	}
	155	+	insts = append(insts, rs.jobs[i])
	156	+	}
	157	+	if len(insts) == 0 {
	158	+	return nil, st, false
	159	+	}
	160	+	res := result(insts)
	161	+	switch res {
	162	+	case "failure":
	163	+	st.Failure, st.Success = true, false
	164	+	case "cancelled":
	165	+	st.Cancelled, st.Success = true, false
	166	+	case "skipped":
	167	+	st.Success = false
	168	+	}
	169	+	outs := expr.NewDictionary()
	170	+	for _, in := range insts {
	171	+	for _, k := range sortedKeys(in.Outputs) {
	172	+	// Instances of a matrix share names: the last non-empty value wins.
	173	+	if v := in.Outputs[k]; v != "" \|\| !hasKey(outs, k) {
	174	+	setKey(outs, k, expr.String(v))
	175	+	}
	176	+	}
	177	+	}
	178	+	d := expr.NewDictionary()
	179	+	d.Add("result", expr.String(res))
	180	+	d.Add("outputs", outs)
	181	+	needs.Add(n, d)
	182	+	}
	183	+	return needs, st, true
	184	+	}
	185	+
	186	+	func hasKey(d *expr.Dictionary, k string) bool { _, ok := d.Get(k); return ok }
	187	+
	188	+	// setKey sets a key of a dictionary, replacing an existing value.
	189	+	func setKey(d *expr.Dictionary, k string, v expr.Value) {
	190	+	if !d.Add(k, v) {
	191	+	pairs := d.Pairs()
	192	+	d = expr.NewDictionary()
	193	+	for _, p := range pairs {
	194	+	if strings.EqualFold(p.Key, k) {
	195	+	d.Add(p.Key, v)
	196	+	} else {
	197	+	d.Add(p.Key, p.Value)
	198	+	}
	199	+	}
	200	+	}
	201	+	}
	202	+
	203	+	// githubValue is the github context of the run with the keys of the job.
	204	+	func (rs runState) githubValue(key string, attempt int32) (expr.Dictionary, error) {
	205	+	v, err := expr.FromJSON([]byte(rs.src.GithubJSON))
	206	+	if err != nil {
	207	+	return nil, fmt.Errorf("the stored github context is not valid: %w", err)
	208	+	}
	209	+	d, ok := v.(*expr.Dictionary)
	210	+	if !ok {
	211	+	return nil, fmt.Errorf("the stored github context is not an object")
	212	+	}
	213	+	d.Add("job", expr.String(key))
	214	+	d.Add("run_attempt", expr.String(strconv.Itoa(int(attempt))))
	215	+	return d, nil
	216	+	}
	217	+
	218	+	// trim cuts a message for a job error.
	219	+	func trim(s string) string { return cutText(s, maxJobError) }
	220	+
	221	+	// end ends a job with a conclusion and an error text, completes its steps like
	222	+	// the server does, and applies fail-fast to its siblings.
	223	+	func (rs runState) end(j domain.CIJob, conclusion domain.CIConclusion, msg string) {
	224	+	now := rs.c.now()
	225	+	cancelish := conclusion == domain.CICancelled \|\| conclusion == domain.CITimedOut
	226	+	for i := range j.Steps {
	227	+	s := &j.Steps[i]
	228	+	if s.State == domain.CIStepCompleted {
	229	+	continue
	230	+	}
	231	+	s.Conclusion = domain.CICancelled
	232	+	if s.State == domain.CIStepPending && !cancelish {
	233	+	s.Conclusion = domain.CISkipped
	234	+	}
	235	+	s.State, s.CompletedAt = domain.CIStepCompleted, now
	236	+	}
	237	+	j.Phase, j.Conclusion, j.CompletedAt = domain.CIPhaseEnded, conclusion, now
	238	+	if msg != "" {
	239	+	j.Error = trim(msg)
	240	+	}
	241	+	if j.HasToken() {
	242	+	j.TokenValidUntil = now.Add(domain.CITokenGrace)
	243	+	}
	244	+	rs.touch(j)
	245	+	rs.c.mu.Lock()
	246	+	delete(rs.c.heartbeats, jobRef(*j))
	247	+	rs.c.mu.Unlock()
	248	+
	249	+	if j.FailFast && conclusion.IsFailure() {
	250	+	for i := range rs.jobs {
	251	+	s := &rs.jobs[i]
	252	+	if s.Key != j.Key \|\| s.ID == j.ID \|\| s.Ended() {
	253	+	continue
	254	+	}
	255	+	if s.Active() {
	256	+	rs.requestCancel(s, domain.CICancelFailFast, "Canceled because another job of the matrix failed (fail-fast)")
	257	+	} else {
	258	+	rs.end(s, domain.CICancelled, "")
	259	+	}
	260	+	}
	261	+	}
	262	+	}
	263	+
	264	+	// requestCancel asks the runner of a job to stop, once: the first reason wins.
	265	+	func (rs runState) requestCancel(j domain.CIJob, reason domain.CICancelReason, msg string) {
	266	+	if j.CancelReason != domain.CICancelNone \|\| j.Ended() {
	267	+	return
	268	+	}
	269	+	j.CancelReason, j.CancelMessage, j.CancelRequestedAt = reason, msg, rs.c.now()
	270	+	rs.touch(j)
	271	+	}
	272	+
	273	+	// cancelAll cancels what is left of the run: jobs that no runner has are over
	274	+	// at once, the ones a runner has get a cancel request.
	275	+	func (rs *runState) cancelAll(msg string) {
	276	+	for i := range rs.jobs {
	277	+	j := &rs.jobs[i]
	278	+	switch {
	279	+	case j.Ended():
	280	+	case j.Active():
	281	+	rs.requestCancel(j, domain.CICancelUser, msg)
	282	+	default:
	283	+	rs.end(j, domain.CICancelled, "")
	284	+	}
	285	+	}
	286	+	}
	287	+
	288	+	// advance plans the jobs that became ready and promotes queued matrix
	289	+	// instances, until nothing changes.
	290	+	func (rs *runState) advance(ctx context.Context) error {
	291	+	if !rs.latest() {
	292	+	return nil
	293	+	}
	294	+	if rs.run.CancelRequested {
	295	+	rs.cancelAll("Canceled by the owner")
	296	+	return nil
	297	+	}
	298	+	wf, err := rs.c.flows.get(rs.run.ID, rs.run.WorkflowPath, rs.src.WorkflowYAML)
	299	+	if err != nil {
	300	+	for i := range rs.jobs {
	301	+	if !rs.jobs[i].Ended() && !rs.jobs[i].Active() {
	302	+	rs.end(&rs.jobs[i], domain.CIFailure, err.Error())
	303	+	}
	304	+	}
	305	+	return nil
	306	+	}
	307	+	for changed := true; changed; {
	308	+	changed = false
	309	+	for _, wj := range wf.Jobs {
	310	+	insts := rs.instances(wj.ID)
	311	+	if len(insts) != 1 {
	312	+	continue
	313	+	}
	314	+	rec := rs.jobs[insts[0]]
	315	+	if rec.Planned \|\| rec.Ended() \|\| rec.Phase != domain.CIPhaseWaiting {
	316	+	continue
	317	+	}
	318	+	needs, st, ready := rs.needsOf(wj)
	319	+	if !ready {
	320	+	continue
	321	+	}
	322	+	rs.planJob(ctx, wj, insts[0], needs, st)
	323	+	changed = true
	324	+	}
	325	+	}
	326	+	rs.promote()
	327	+	return nil
	328	+	}
	329	+
	330	+	// promote makes planned instances ready while max-parallel allows it.
	331	+	func (rs *runState) promote() {
	332	+	now := rs.c.now()
	333	+	counts := map[string]int{}
	334	+	for _, j := range rs.jobs {
	335	+	if j.Phase == domain.CIPhaseReady \|\| j.Active() {
	336	+	counts[j.Key]++
	337	+	}
	338	+	}
	339	+	for i := range rs.jobs {
	340	+	j := &rs.jobs[i]
	341	+	if !j.Planned \|\| j.Phase != domain.CIPhaseWaiting {
	342	+	continue
	343	+	}
	344	+	if j.MaxParallel > 0 && counts[j.Key] >= j.MaxParallel {
	345	+	continue
	346	+	}
	347	+	j.Phase, j.ReadyAt = domain.CIPhaseReady, now
	348	+	counts[j.Key]++
	349	+	rs.touch(j)
	350	+	}
	351	+	}
	352	+
	353	+	// plan evaluates a job whose needs have ended: its condition, its strategy and
	354	+	// the parts of every instance the server decides (name, labels, timeout,
	355	+	// contexts). The record at idx becomes the first instance.
	356	+	func (rs runState) planJob(ctx context.Context, wj workflow.Job, idx int, needs *expr.Dictionary, st workflow.StatusFuncs) {
	357	+	rec := rs.jobs[idx]
	358	+	fail := func(msg string) { rs.end(&rs.jobs[idx], domain.CIFailure, msg) }
	359	+
	360	+	gh, err := rs.githubValue(wj.ID, rs.run.Attempt)
	361	+	if err != nil {
	362	+	fail(err.Error())
	363	+	return
	364	+	}
	365	+	env := template.Env{Contexts: map[string]expr.Value{
	366	+	"github": gh, "needs": needs, "inputs": expr.NewDictionary(), "vars": expr.NewDictionary(),
	367	+	}}
	368	+	run, err := wj.EvaluateIf(ctx, env, st)
	369	+	switch {
	370	+	case err != nil:
	371	+	fail(err.Error())
	372	+	return
	373	+	case !run:
	374	+	rs.jobs[idx].Name = rs.skippedName(ctx, wj, env)
	375	+	rs.end(&rs.jobs[idx], domain.CISkipped, "")
	376	+	return
	377	+	}
	378	+	if feature := unsupported(wj); feature != "" {
	379	+	fail("not supported yet: " + feature)
	380	+	return
	381	+	}
	382	+	plan, err := wj.EvaluateStrategy(ctx, env)
	383	+	if err != nil {
	384	+	fail(err.Error())
	385	+	return
	386	+	}
	387	+
	388	+	insts := make([]domain.CIJob, 0, len(plan.Combos))
	389	+	for i := range plan.Combos {
	390	+	menv, err := plan.MatrixEnv(env, i)
	391	+	if err != nil {
	392	+	fail(err.Error())
	393	+	return
	394	+	}
	395	+	inst, err := rs.instance(ctx, wj, plan, rec, i, menv, needs)
	396	+	if err != nil {
	397	+	fail(err.Error())
	398	+	return
	399	+	}
	400	+	insts = append(insts, inst)
	401	+	}
	402	+	rs.jobs[idx] = insts[0]
	403	+	rs.touch(&rs.jobs[idx])
	404	+	for _, inst := range insts[1:] {
	405	+	rs.jobs = append(rs.jobs, inst)
	406	+	rs.dirty[inst.ID] = true
	407	+	}
	408	+	rs.sortJobs()
	409	+	}
	410	+
	411	+	// skippedName is the best name for a job that did not run: its evaluated name
	412	+	// when that does not read the matrix, else its key.
	413	+	func (rs runState) skippedName(ctx context.Context, wj workflow.Job, env template.Env) string {
	414	+	if wj.HasName {
	415	+	if name, err := template.EvaluateString(ctx, wj.Name, env); err == nil && name != "" {
	416	+	return name
	417	+	}
	418	+	}
	419	+	return wj.ID
	420	+	}
	421	+
	422	+	// instance builds the planned record of combination i of a job.
	423	+	func (rs runState) instance(ctx context.Context, wj workflow.Job, plan workflow.Strategy, base domain.CIJob, i int, env template.Env, needs expr.Dictionary) (domain.CIJob, error) {
	424	+	j := base
	425	+	j.Combo = i
	426	+	j.ID = domain.NewJobID(rs.run.ID, base.Index, i)
	427	+	if i > 0 {
	428	+	j.Version = 0
	429	+	}
	430	+	name, err := wj.DisplayName(ctx, env, plan, i)
	431	+	if err != nil {
	432	+	return j, err
	433	+	}
	434	+	labels, _, err := wj.EvaluateRunsOn(ctx, env)
	435	+	if err != nil {
	436	+	return j, err
	437	+	}
	438	+	if len(labels) == 0 {
	439	+	return j, fmt.Errorf("job '%s' has no runs-on labels", wj.ID)
	440	+	}
	441	+	timeout := int32(domain.CIDefaultTimeoutMinutes)
	442	+	if wj.TimeoutMinutes != nil {
	443	+	n, err := template.EvaluateNumber(ctx, wj.TimeoutMinutes, env)
	444	+	if err != nil {
	445	+	return j, err
	446	+	}
	447	+	timeout = int32(min(max(math.Round(n), 1), 525600)) //nolint:gosec // clamped
	448	+	}
	449	+	gh, err := rs.githubValue(wj.ID, rs.run.Attempt)
	450	+	if err != nil {
	451	+	return j, err
	452	+	}
	453	+	matrix := plan.Combos[i].Values
	454	+	contexts, err := contextsJSON(gh, needs, plan.Context(i), matrix)
	455	+	if err != nil {
	456	+	return j, err
	457	+	}
	458	+	j.Name, j.Labels, j.TimeoutMinutes = name, labels, timeout
	459	+	j.MatrixJSON = compactJSON(expr.ToJSON(matrix))
	460	+	j.ContextsJSON, j.ComboTotal = contexts, len(plan.Combos)
	461	+	j.FailFast, j.MaxParallel = plan.FailFast, plan.MaxParallel
	462	+	j.Planned, j.Phase = true, domain.CIPhaseWaiting
	463	+	return j, nil
	464	+	}
	465	+
	466	+	func compactJSON(s string) string {
	467	+	var b bytes.Buffer
	468	+	if err := json.Compact(&b, []byte(s)); err != nil {
	469	+	return "{}"
	470	+	}
	471	+	return b.String()
	472	+	}
	473	+
	474	+	// contextsJSON is the JSON object of the contexts the server computes for a
	475	+	// job (Job.contexts in runner.proto).
	476	+	func contextsJSON(github, needs expr.Dictionary, strategy, matrix expr.Dictionary) (string, error) {
	477	+	out := map[string]json.RawMessage{}
	478	+	for k, v := range map[string]expr.Value{
	479	+	"github": github, "needs": needs, "strategy": strategy, "matrix": matrix,
	480	+	"inputs": expr.NewDictionary(), "vars": expr.NewDictionary(),
	481	+	} {
	482	+	out[k] = json.RawMessage(compactJSON(expr.ToJSON(v)))
	483	+	}
	484	+	b, err := json.Marshal(out)
	485	+	if err != nil {
	486	+	return "", err
	487	+	}
	488	+	return string(b), nil
	489	+	}

@@ -0,0 +1,489 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"encoding/json"
	9	+	"fmt"
	10	+	"math"
	11	+	"slices"
	12	+	"strconv"
	13	+	"strings"
	14	+	"sync"
	15	+
	16	+	"bnema.dev/ermite-actions/expr"
	17	+	"bnema.dev/ermite-actions/template"
	18	+	"bnema.dev/ermite-actions/workflow"
	19	+	"bnema.dev/ermite/internal/domain"
	20	+	)
	21	+
	22	+	// maxJobError bounds JobError text (4 KiB, run.proto).
	23	+	const maxJobError = 4 << 10
	24	+
	25	+	// wfCache keeps the parsed workflows of the runs in flight, so that planning a
	26	+	// job does not parse the file again.
	27	+	type wfCache struct {
	28	+	mu sync.Mutex
	29	+	max int
	30	+	m map[int64]*workflow.Workflow
	31	+	}
	32	+
	33	+	func newWfCache(max int) wfCache { return &wfCache{max: max, m: map[int64]workflow.Workflow{}} }
	34	+
	35	+	func (w wfCache) get(runID int64, path, yaml string) (workflow.Workflow, error) {
	36	+	w.mu.Lock()
	37	+	if wf, ok := w.m[runID]; ok {
	38	+	w.mu.Unlock()
	39	+	return wf, nil
	40	+	}
	41	+	w.mu.Unlock()
	42	+	wf, errs := workflow.Parse(path, []byte(yaml))
	43	+	if len(errs) > 0 {
	44	+	return nil, fmt.Errorf("the workflow file no longer parses: %s", errs[0].Error())
	45	+	}
	46	+	w.mu.Lock()
	47	+	defer w.mu.Unlock()
	48	+	if len(w.m) >= w.max {
	49	+	clear(w.m)
	50	+	}
	51	+	w.m[runID] = wf
	52	+	return wf, nil
	53	+	}
	54	+
	55	+	// unsupported names the feature of a job that CI-1 does not run, or "". Such
	56	+	// a job fails with "not supported yet: <feature>" instead of running wrong.
	57	+	// Relax it as the runner learns the features (ADR 0025, CI-2 and CI-3).
	58	+	func unsupported(j *workflow.Job) string {
	59	+	switch {
	60	+	case j.IsReusable():
	61	+	return "reusable workflows (uses at the job level)"
	62	+	case j.Container != nil:
	63	+	return "container"
	64	+	case j.Services != nil:
	65	+	return "services"
	66	+	case j.Environment != nil:
	67	+	return "environment"
	68	+	case j.Concurrency != nil:
	69	+	return "concurrency"
	70	+	case present(j.ContinueOnError):
	71	+	return "continue-on-error"
	72	+	}
	73	+	for _, s := range j.Steps {
	74	+	switch {
	75	+	case s.Uses != nil:
	76	+	return "uses: " + s.Uses.Value
	77	+	case present(s.ContinueOnError):
	78	+	return "continue-on-error on a step"
	79	+	case present(s.TimeoutMinutes):
	80	+	return "timeout-minutes on a step"
	81	+	}
	82	+	}
	83	+	return ""
	84	+	}
	85	+
	86	+	// present reports whether an optional key is given and is not a literal
	87	+	// false (continue-on-error: false is the default).
	88	+	func present(t template.Token) bool {
	89	+	if t == nil {
	90	+	return false
	91	+	}
	92	+	if b, ok := t.(*template.Boolean); ok && !b.Value {
	93	+	return false
	94	+	}
	95	+	return true
	96	+	}
	97	+
	98	+	// instances returns the indexes of the records of the job key, in order.
	99	+	func (rs *runState) instances(key string) []int {
	100	+	var out []int
	101	+	for i, j := range rs.jobs {
	102	+	if j.Key == key {
	103	+	out = append(out, i)
	104	+	}
	105	+	}
	106	+	return out
	107	+	}
	108	+
	109	+	func (rs *runState) sortJobs() {
	110	+	slices.SortStableFunc(rs.jobs, func(a, b domain.CIJob) int {
	111	+	if a.Index != b.Index {
	112	+	return a.Index - b.Index
	113	+	}
	114	+	return a.Combo - b.Combo
	115	+	})
	116	+	}
	117	+
	118	+	// result is the aggregate conclusion of the instances of a job as the needs
	119	+	// context shows it: success, failure, cancelled or skipped.
	120	+	func result(insts []domain.CIJob) string {
	121	+	var failed, cancelled, ok bool
	122	+	for _, j := range insts {
	123	+	switch j.Conclusion {
	124	+	case domain.CIFailure, domain.CITimedOut, domain.CIActionRequired:
	125	+	failed = true
	126	+	case domain.CICancelled:
	127	+	cancelled = true
	128	+	case domain.CISkipped:
	129	+	default:
	130	+	ok = true
	131	+	}
	132	+	}
	133	+	switch {
	134	+	case failed:
	135	+	return "failure"
	136	+	case cancelled:
	137	+	return "cancelled"
	138	+	case ok:
	139	+	return "success"
	140	+	}
	141	+	return "skipped"
	142	+	}
	143	+
	144	+	// needsOf computes the needs context and the status functions of a job from
	145	+	// the jobs it needs, and whether they all have ended.
	146	+	func (rs runState) needsOf(j workflow.Job) (needs *expr.Dictionary, st workflow.StatusFuncs, ready bool) {
	147	+	needs = expr.NewDictionary()
	148	+	st = workflow.StatusFuncs{Success: true, Cancelled: rs.run.CancelRequested}
	149	+	for _, n := range j.Needs {
	150	+	var insts []domain.CIJob
	151	+	for _, i := range rs.instances(n) {
	152	+	if !rs.jobs[i].Ended() {
	153	+	return nil, st, false
	154	+	}
	155	+	insts = append(insts, rs.jobs[i])
	156	+	}
	157	+	if len(insts) == 0 {
	158	+	return nil, st, false
	159	+	}
	160	+	res := result(insts)
	161	+	switch res {
	162	+	case "failure":
	163	+	st.Failure, st.Success = true, false
	164	+	case "cancelled":
	165	+	st.Cancelled, st.Success = true, false
	166	+	case "skipped":
	167	+	st.Success = false
	168	+	}
	169	+	outs := expr.NewDictionary()
	170	+	for _, in := range insts {
	171	+	for _, k := range sortedKeys(in.Outputs) {
	172	+	// Instances of a matrix share names: the last non-empty value wins.
	173	+	if v := in.Outputs[k]; v != "" \|\| !hasKey(outs, k) {
	174	+	setKey(outs, k, expr.String(v))
	175	+	}
	176	+	}
	177	+	}
	178	+	d := expr.NewDictionary()
	179	+	d.Add("result", expr.String(res))
	180	+	d.Add("outputs", outs)
	181	+	needs.Add(n, d)
	182	+	}
	183	+	return needs, st, true
	184	+	}
	185	+
	186	+	func hasKey(d *expr.Dictionary, k string) bool { _, ok := d.Get(k); return ok }
	187	+
	188	+	// setKey sets a key of a dictionary, replacing an existing value.
	189	+	func setKey(d *expr.Dictionary, k string, v expr.Value) {
	190	+	if !d.Add(k, v) {
	191	+	pairs := d.Pairs()
	192	+	d = expr.NewDictionary()
	193	+	for _, p := range pairs {
	194	+	if strings.EqualFold(p.Key, k) {
	195	+	d.Add(p.Key, v)
	196	+	} else {
	197	+	d.Add(p.Key, p.Value)
	198	+	}
	199	+	}
	200	+	}
	201	+	}
	202	+
	203	+	// githubValue is the github context of the run with the keys of the job.
	204	+	func (rs runState) githubValue(key string, attempt int32) (expr.Dictionary, error) {
	205	+	v, err := expr.FromJSON([]byte(rs.src.GithubJSON))
	206	+	if err != nil {
	207	+	return nil, fmt.Errorf("the stored github context is not valid: %w", err)
	208	+	}
	209	+	d, ok := v.(*expr.Dictionary)
	210	+	if !ok {
	211	+	return nil, fmt.Errorf("the stored github context is not an object")
	212	+	}
	213	+	d.Add("job", expr.String(key))
	214	+	d.Add("run_attempt", expr.String(strconv.Itoa(int(attempt))))
	215	+	return d, nil
	216	+	}
	217	+
	218	+	// trim cuts a message for a job error.
	219	+	func trim(s string) string { return cutText(s, maxJobError) }
	220	+
	221	+	// end ends a job with a conclusion and an error text, completes its steps like
	222	+	// the server does, and applies fail-fast to its siblings.
	223	+	func (rs runState) end(j domain.CIJob, conclusion domain.CIConclusion, msg string) {
	224	+	now := rs.c.now()
	225	+	cancelish := conclusion == domain.CICancelled \|\| conclusion == domain.CITimedOut
	226	+	for i := range j.Steps {
	227	+	s := &j.Steps[i]
	228	+	if s.State == domain.CIStepCompleted {
	229	+	continue
	230	+	}
	231	+	s.Conclusion = domain.CICancelled
	232	+	if s.State == domain.CIStepPending && !cancelish {
	233	+	s.Conclusion = domain.CISkipped
	234	+	}
	235	+	s.State, s.CompletedAt = domain.CIStepCompleted, now
	236	+	}
	237	+	j.Phase, j.Conclusion, j.CompletedAt = domain.CIPhaseEnded, conclusion, now
	238	+	if msg != "" {
	239	+	j.Error = trim(msg)
	240	+	}
	241	+	if j.HasToken() {
	242	+	j.TokenValidUntil = now.Add(domain.CITokenGrace)
	243	+	}
	244	+	rs.touch(j)
	245	+	rs.c.mu.Lock()
	246	+	delete(rs.c.heartbeats, jobRef(*j))
	247	+	rs.c.mu.Unlock()
	248	+
	249	+	if j.FailFast && conclusion.IsFailure() {
	250	+	for i := range rs.jobs {
	251	+	s := &rs.jobs[i]
	252	+	if s.Key != j.Key \|\| s.ID == j.ID \|\| s.Ended() {
	253	+	continue
	254	+	}
	255	+	if s.Active() {
	256	+	rs.requestCancel(s, domain.CICancelFailFast, "Canceled because another job of the matrix failed (fail-fast)")
	257	+	} else {
	258	+	rs.end(s, domain.CICancelled, "")
	259	+	}
	260	+	}
	261	+	}
	262	+	}
	263	+
	264	+	// requestCancel asks the runner of a job to stop, once: the first reason wins.
	265	+	func (rs runState) requestCancel(j domain.CIJob, reason domain.CICancelReason, msg string) {
	266	+	if j.CancelReason != domain.CICancelNone \|\| j.Ended() {
	267	+	return
	268	+	}
	269	+	j.CancelReason, j.CancelMessage, j.CancelRequestedAt = reason, msg, rs.c.now()
	270	+	rs.touch(j)
	271	+	}
	272	+
	273	+	// cancelAll cancels what is left of the run: jobs that no runner has are over
	274	+	// at once, the ones a runner has get a cancel request.
	275	+	func (rs *runState) cancelAll(msg string) {
	276	+	for i := range rs.jobs {
	277	+	j := &rs.jobs[i]
	278	+	switch {
	279	+	case j.Ended():
	280	+	case j.Active():
	281	+	rs.requestCancel(j, domain.CICancelUser, msg)
	282	+	default:
	283	+	rs.end(j, domain.CICancelled, "")
	284	+	}
	285	+	}
	286	+	}
	287	+
	288	+	// advance plans the jobs that became ready and promotes queued matrix
	289	+	// instances, until nothing changes.
	290	+	func (rs *runState) advance(ctx context.Context) error {
	291	+	if !rs.latest() {
	292	+	return nil
	293	+	}
	294	+	if rs.run.CancelRequested {
	295	+	rs.cancelAll("Canceled by the owner")
	296	+	return nil
	297	+	}
	298	+	wf, err := rs.c.flows.get(rs.run.ID, rs.run.WorkflowPath, rs.src.WorkflowYAML)
	299	+	if err != nil {
	300	+	for i := range rs.jobs {
	301	+	if !rs.jobs[i].Ended() && !rs.jobs[i].Active() {
	302	+	rs.end(&rs.jobs[i], domain.CIFailure, err.Error())
	303	+	}
	304	+	}
	305	+	return nil
	306	+	}
	307	+	for changed := true; changed; {
	308	+	changed = false
	309	+	for _, wj := range wf.Jobs {
	310	+	insts := rs.instances(wj.ID)
	311	+	if len(insts) != 1 {
	312	+	continue
	313	+	}
	314	+	rec := rs.jobs[insts[0]]
	315	+	if rec.Planned \|\| rec.Ended() \|\| rec.Phase != domain.CIPhaseWaiting {
	316	+	continue
	317	+	}
	318	+	needs, st, ready := rs.needsOf(wj)
	319	+	if !ready {
	320	+	continue
	321	+	}
	322	+	rs.planJob(ctx, wj, insts[0], needs, st)
	323	+	changed = true
	324	+	}
	325	+	}
	326	+	rs.promote()
	327	+	return nil
	328	+	}
	329	+
	330	+	// promote makes planned instances ready while max-parallel allows it.
	331	+	func (rs *runState) promote() {
	332	+	now := rs.c.now()
	333	+	counts := map[string]int{}
	334	+	for _, j := range rs.jobs {
	335	+	if j.Phase == domain.CIPhaseReady \|\| j.Active() {
	336	+	counts[j.Key]++
	337	+	}
	338	+	}
	339	+	for i := range rs.jobs {
	340	+	j := &rs.jobs[i]
	341	+	if !j.Planned \|\| j.Phase != domain.CIPhaseWaiting {
	342	+	continue
	343	+	}
	344	+	if j.MaxParallel > 0 && counts[j.Key] >= j.MaxParallel {
	345	+	continue
	346	+	}
	347	+	j.Phase, j.ReadyAt = domain.CIPhaseReady, now
	348	+	counts[j.Key]++
	349	+	rs.touch(j)
	350	+	}
	351	+	}
	352	+
	353	+	// plan evaluates a job whose needs have ended: its condition, its strategy and
	354	+	// the parts of every instance the server decides (name, labels, timeout,
	355	+	// contexts). The record at idx becomes the first instance.
	356	+	func (rs runState) planJob(ctx context.Context, wj workflow.Job, idx int, needs *expr.Dictionary, st workflow.StatusFuncs) {
	357	+	rec := rs.jobs[idx]
	358	+	fail := func(msg string) { rs.end(&rs.jobs[idx], domain.CIFailure, msg) }
	359	+
	360	+	gh, err := rs.githubValue(wj.ID, rs.run.Attempt)
	361	+	if err != nil {
	362	+	fail(err.Error())
	363	+	return
	364	+	}
	365	+	env := template.Env{Contexts: map[string]expr.Value{
	366	+	"github": gh, "needs": needs, "inputs": expr.NewDictionary(), "vars": expr.NewDictionary(),
	367	+	}}
	368	+	run, err := wj.EvaluateIf(ctx, env, st)
	369	+	switch {
	370	+	case err != nil:
	371	+	fail(err.Error())
	372	+	return
	373	+	case !run:
	374	+	rs.jobs[idx].Name = rs.skippedName(ctx, wj, env)
	375	+	rs.end(&rs.jobs[idx], domain.CISkipped, "")
	376	+	return
	377	+	}
	378	+	if feature := unsupported(wj); feature != "" {
	379	+	fail("not supported yet: " + feature)
	380	+	return
	381	+	}
	382	+	plan, err := wj.EvaluateStrategy(ctx, env)
	383	+	if err != nil {
	384	+	fail(err.Error())
	385	+	return
	386	+	}
	387	+
	388	+	insts := make([]domain.CIJob, 0, len(plan.Combos))
	389	+	for i := range plan.Combos {
	390	+	menv, err := plan.MatrixEnv(env, i)
	391	+	if err != nil {
	392	+	fail(err.Error())
	393	+	return
	394	+	}
	395	+	inst, err := rs.instance(ctx, wj, plan, rec, i, menv, needs)
	396	+	if err != nil {
	397	+	fail(err.Error())
	398	+	return
	399	+	}
	400	+	insts = append(insts, inst)
	401	+	}
	402	+	rs.jobs[idx] = insts[0]
	403	+	rs.touch(&rs.jobs[idx])
	404	+	for _, inst := range insts[1:] {
	405	+	rs.jobs = append(rs.jobs, inst)
	406	+	rs.dirty[inst.ID] = true
	407	+	}
	408	+	rs.sortJobs()
	409	+	}
	410	+
	411	+	// skippedName is the best name for a job that did not run: its evaluated name
	412	+	// when that does not read the matrix, else its key.
	413	+	func (rs runState) skippedName(ctx context.Context, wj workflow.Job, env template.Env) string {
	414	+	if wj.HasName {
	415	+	if name, err := template.EvaluateString(ctx, wj.Name, env); err == nil && name != "" {
	416	+	return name
	417	+	}
	418	+	}
	419	+	return wj.ID
	420	+	}
	421	+
	422	+	// instance builds the planned record of combination i of a job.
	423	+	func (rs runState) instance(ctx context.Context, wj workflow.Job, plan workflow.Strategy, base domain.CIJob, i int, env template.Env, needs expr.Dictionary) (domain.CIJob, error) {
	424	+	j := base
	425	+	j.Combo = i
	426	+	j.ID = domain.NewJobID(rs.run.ID, base.Index, i)
	427	+	if i > 0 {
	428	+	j.Version = 0
	429	+	}
	430	+	name, err := wj.DisplayName(ctx, env, plan, i)
	431	+	if err != nil {
	432	+	return j, err
	433	+	}
	434	+	labels, _, err := wj.EvaluateRunsOn(ctx, env)
	435	+	if err != nil {
	436	+	return j, err
	437	+	}
	438	+	if len(labels) == 0 {
	439	+	return j, fmt.Errorf("job '%s' has no runs-on labels", wj.ID)
	440	+	}
	441	+	timeout := int32(domain.CIDefaultTimeoutMinutes)
	442	+	if wj.TimeoutMinutes != nil {
	443	+	n, err := template.EvaluateNumber(ctx, wj.TimeoutMinutes, env)
	444	+	if err != nil {
	445	+	return j, err
	446	+	}
	447	+	timeout = int32(min(max(math.Round(n), 1), 525600)) //nolint:gosec // clamped
	448	+	}
	449	+	gh, err := rs.githubValue(wj.ID, rs.run.Attempt)
	450	+	if err != nil {
	451	+	return j, err
	452	+	}
	453	+	matrix := plan.Combos[i].Values
	454	+	contexts, err := contextsJSON(gh, needs, plan.Context(i), matrix)
	455	+	if err != nil {
	456	+	return j, err
	457	+	}
	458	+	j.Name, j.Labels, j.TimeoutMinutes = name, labels, timeout
	459	+	j.MatrixJSON = compactJSON(expr.ToJSON(matrix))
	460	+	j.ContextsJSON, j.ComboTotal = contexts, len(plan.Combos)
	461	+	j.FailFast, j.MaxParallel = plan.FailFast, plan.MaxParallel
	462	+	j.Planned, j.Phase = true, domain.CIPhaseWaiting
	463	+	return j, nil
	464	+	}
	465	+
	466	+	func compactJSON(s string) string {
	467	+	var b bytes.Buffer
	468	+	if err := json.Compact(&b, []byte(s)); err != nil {
	469	+	return "{}"
	470	+	}
	471	+	return b.String()
	472	+	}
	473	+
	474	+	// contextsJSON is the JSON object of the contexts the server computes for a
	475	+	// job (Job.contexts in runner.proto).
	476	+	func contextsJSON(github, needs expr.Dictionary, strategy, matrix expr.Dictionary) (string, error) {
	477	+	out := map[string]json.RawMessage{}
	478	+	for k, v := range map[string]expr.Value{
	479	+	"github": github, "needs": needs, "strategy": strategy, "matrix": matrix,
	480	+	"inputs": expr.NewDictionary(), "vars": expr.NewDictionary(),
	481	+	} {
	482	+	out[k] = json.RawMessage(compactJSON(expr.ToJSON(v)))
	483	+	}
	484	+	b, err := json.Marshal(out)
	485	+	if err != nil {
	486	+	return "", err
	487	+	}
	488	+	return string(b), nil
	489	+	}

A internal/usecase/ci_runner.go +730 -0

@@ -0,0 +1,730 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"slices"
	10	+	"strings"
	11	+	"time"
	12	+	"unicode/utf8"
	13	+
	14	+	"github.com/bnema/zerowrap"
	15	+
	16	+	"bnema.dev/ermite/internal/domain"
	17	+	"bnema.dev/ermite/internal/ports"
	18	+	)
	19	+
	20	+	// maxVersionLen bounds RegisterRunner.version.
	21	+	const maxVersionLen = 64
	22	+
	23	+	// --- registration ---
	24	+
	25	+	// Register adds a runner and returns its token, shown once. The principal
	26	+	// needs runner:register on a token that is not restricted to repositories, or
	27	+	// to be root. A name in use is domain.ErrAlreadyExists unless Replace is set,
	28	+	// which removes the old runner (its token stops working at once).
	29	+	func (c *CI) Register(ctx context.Context, p domain.Principal, params ports.RegisterRunnerParams) (ports.RegisteredRunner, error) {
	30	+	if err := canManageRunners(p); err != nil {
	31	+	return ports.RegisteredRunner{}, err
	32	+	}
	33	+	if err := domain.ParseRunnerName(params.Name); err != nil {
	34	+	return ports.RegisteredRunner{}, err
	35	+	}
	36	+	labels, err := domain.ParseRunnerLabels(params.Labels)
	37	+	if err != nil {
	38	+	return ports.RegisteredRunner{}, err
	39	+	}
	40	+	var v domain.Violations
	41	+	if len(params.Version) > maxVersionLen \|\| !utf8.ValidString(params.Version) {
	42	+	v.Add("version", "is at most 64 characters")
	43	+	}
	44	+	switch params.Isolation {
	45	+	case domain.CIIsolationVM, domain.CIIsolationContainerSysbox, domain.CIIsolationContainerPrivileged:
	46	+	default:
	47	+	v.Add("isolation", "is required")
	48	+	}
	49	+	if params.Arch != "x64" && params.Arch != "arm64" {
	50	+	v.Add("arch", `must be "x64" or "arm64"`)
	51	+	}
	52	+	if params.OS != "linux" {
	53	+	v.Add("os", `must be "linux"`)
	54	+	}
	55	+	if err := v.Err(); err != nil {
	56	+	return ports.RegisteredRunner{}, err
	57	+	}
	58	+
	59	+	token := newToken(domain.RunnerTokenPrefix)
	60	+	r := domain.CIRunner{
	61	+	ID: newRunnerID(), Name: params.Name, Labels: domain.RunnerLabels(labels, params.OS, params.Arch),
	62	+	Isolation: params.Isolation, Version: params.Version, Arch: params.Arch, OS: params.OS,
	63	+	RegisteredAt: c.now(), TokenHash: domain.HashTokenSecret(token),
	64	+	}
	65	+	removed, err := c.cfg.Store.CreateRunner(ctx, r, params.Replace)
	66	+	if err != nil {
	67	+	return ports.RegisteredRunner{}, err
	68	+	}
	69	+	log := zerowrap.FromCtx(ctx)
	70	+	log.Info().Str("runner_id", r.ID).Str("runner", r.Name).Str("isolation", string(r.Isolation)).
	71	+	Int("replaced", len(removed)).Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("runner registered")
	72	+	c.forgetRunners(removed)
	73	+	return ports.RegisteredRunner{Runner: r, Token: token}, nil
	74	+	}
	75	+
	76	+	// forgetRunners drops what is kept in memory for removed runners and ends
	77	+	// their polls in progress.
	78	+	func (c *CI) forgetRunners(rs []domain.CIRunner) {
	79	+	c.mu.Lock()
	80	+	for _, r := range rs {
	81	+	delete(c.polls, r.ID)
	82	+	delete(c.seen, r.ID)
	83	+	}
	84	+	c.mu.Unlock()
	85	+	c.wakeQueue()
	86	+	}
	87	+
	88	+	// runnerFor authenticates a runner token.
	89	+	func (c *CI) runnerFor(ctx context.Context, secret string) (domain.CIRunner, error) {
	90	+	if !domain.IsRunnerToken(secret) {
	91	+	return domain.CIRunner{}, domain.ErrUnauthenticated
	92	+	}
	93	+	r, err := c.cfg.Store.FindRunnerByToken(ctx, domain.HashTokenSecret(secret))
	94	+	if errors.Is(err, domain.ErrNotFound) {
	95	+	return domain.CIRunner{}, domain.ErrUnauthenticated
	96	+	}
	97	+	return r, err
	98	+	}
	99	+
	100	+	func (c *CI) markSeen(runnerID string) {
	101	+	c.mu.Lock()
	102	+	c.seen[runnerID] = c.now()
	103	+	c.mu.Unlock()
	104	+	}
	105	+
	106	+	// --- FetchJob ---
	107	+
	108	+	var errNoJob = errors.New("no job")
	109	+
	110	+	// FetchJob holds the call until a queued job matches the runner, or the poll
	111	+	// time passes. A job is given to exactly one runner: assignments are made one
	112	+	// at a time under a lock and recorded before the answer is sent.
	113	+	func (c *CI) FetchJob(ctx context.Context, secret string, params ports.FetchJobParams) (ports.FetchJobResult, error) {
	114	+	runner, err := c.runnerFor(ctx, secret)
	115	+	if err != nil {
	116	+	return ports.FetchJobResult{}, err
	117	+	}
	118	+	offered, err := offeredLabels(runner, params.Labels)
	119	+	if err != nil {
	120	+	return ports.FetchJobResult{}, err
	121	+	}
	122	+	if params.PollID != "" {
	123	+	if err := checkPollID(params.PollID); err != nil {
	124	+	return ports.FetchJobResult{}, err
	125	+	}
	126	+	if j, ok := c.repeatPoll(ctx, runner, params.PollID); ok {
	127	+	return ports.FetchJobResult{Job: j}, nil
	128	+	}
	129	+	}
	130	+	c.markSeen(runner.ID)
	131	+	deadline := time.NewTimer(c.cfg.PollTimeout)
	132	+	defer deadline.Stop()
	133	+	for {
	134	+	wake := c.pending.wait(queueKey) // before looking: a job made meanwhile wakes us
	135	+	job, err := c.assignTo(ctx, runner, offered, params.PollID)
	136	+	switch {
	137	+	case err == nil:
	138	+	c.markSeen(runner.ID)
	139	+	return ports.FetchJobResult{Job: job}, nil
	140	+	case !errors.Is(err, errNoJob):
	141	+	return ports.FetchJobResult{}, err
	142	+	}
	143	+	select {
	144	+	case <-wake:
	145	+	case <-deadline.C:
	146	+	c.markSeen(runner.ID)
	147	+	return ports.FetchJobResult{}, nil
	148	+	case <-ctx.Done():
	149	+	return ports.FetchJobResult{}, ctx.Err()
	150	+	case <-c.ctx.Done():
	151	+	return ports.FetchJobResult{RetryAfter: time.Second}, nil
	152	+	}
	153	+	// A removed runner's poll ends with unauthenticated.
	154	+	if _, err := c.cfg.Store.GetRunner(ctx, runner.ID); errors.Is(err, domain.ErrNotFound) {
	155	+	return ports.FetchJobResult{}, domain.ErrUnauthenticated
	156	+	}
	157	+	}
	158	+	}
	159	+
	160	+	func checkPollID(id string) error {
	161	+	if len(id) > 64 {
	162	+	return fmt.Errorf("%w: poll_id is at most 64 characters", domain.ErrInvalid)
	163	+	}
	164	+	for _, r := range id {
	165	+	if (r < 'a' \|\| r > 'z') && (r < 'A' \|\| r > 'Z') && (r < '0' \|\| r > '9') && r != '_' && r != '-' {
	166	+	return fmt.Errorf("%w: poll_id has letters, digits, '_' and '-' only", domain.ErrInvalid)
	167	+	}
	168	+	}
	169	+	return nil
	170	+	}
	171	+
	172	+	// offeredLabels are the labels of one poll: all those of the runner, or the
	173	+	// subset the call names.
	174	+	func offeredLabels(r domain.CIRunner, asked []string) ([]string, error) {
	175	+	if len(asked) == 0 {
	176	+	return r.Labels, nil
	177	+	}
	178	+	var out []string
	179	+	for _, l := range asked {
	180	+	l = strings.ToLower(l)
	181	+	if !slices.Contains(r.Labels, l) {
	182	+	return nil, fmt.Errorf("%w: label %q is not a label of the runner", domain.ErrInvalid, l)
	183	+	}
	184	+	out = append(out, l)
	185	+	}
	186	+	return out, nil
	187	+	}
	188	+
	189	+	// repeatPoll answers a retried poll with the assignment the first call made,
	190	+	// as long as it has not been acknowledged or cancelled.
	191	+	func (c CI) repeatPoll(ctx context.Context, runner domain.CIRunner, pollID string) (ports.RunnerJob, bool) {
	192	+	c.mu.Lock()
	193	+	rec, ok := c.polls[runner.ID]
	194	+	c.mu.Unlock()
	195	+	if !ok \|\| rec.pollID != pollID {
	196	+	return nil, false
	197	+	}
	198	+	var rid int64
	199	+	var ra int32
	200	+	var id string
	201	+	if _, err := fmt.Sscanf(rec.ref, "%d/%d/%s", &rid, &ra, &id); err != nil {
	202	+	return nil, false
	203	+	}
	204	+	var out *ports.RunnerJob
	205	+	err := c.withRun(ctx, rid, ra, func(rs *runState) error {
	206	+	j := rs.find(id)
	207	+	if j == nil \|\| j.Phase != domain.CIPhaseAssigned \|\| j.TokenHash != rec.hash \|\| j.RunnerID != runner.ID {
	208	+	return errNoJob
	209	+	}
	210	+	out = c.runnerJob(rs, *j, rec.token)
	211	+	return nil
	212	+	})
	213	+	return out, err == nil && out != nil
	214	+	}
	215	+
	216	+	// runnerJob is the job as the runner gets it.
	217	+	func (c CI) runnerJob(rs runState, j domain.CIJob, token string) *ports.RunnerJob {
	218	+	return &ports.RunnerJob{
	219	+	Job: j, Token: token, Repo: rs.run.Repo, Sha: rs.run.Sha, Ref: rs.run.Ref,
	220	+	WorkflowPath: rs.run.WorkflowPath, WorkflowYAML: rs.src.WorkflowYAML,
	221	+	}
	222	+	}
	223	+
	224	+	// assignTo gives the oldest queued job that matches the labels to the runner,
	225	+	// or errNoJob. The assignment lock makes the choice and the write one step.
	226	+	func (c CI) assignTo(ctx context.Context, runner domain.CIRunner, offered []string, pollID string) (ports.RunnerJob, error) {
	227	+	c.assign.Lock()
	228	+	defer c.assign.Unlock()
	229	+	ready, err := c.cfg.Store.ListReadyJobs(ctx, 0)
	230	+	if err != nil {
	231	+	return nil, err
	232	+	}
	233	+	for _, cand := range ready {
	234	+	if !domain.LabelsMatch(cand.Labels, offered) {
	235	+	continue
	236	+	}
	237	+	var out *ports.RunnerJob
	238	+	err := c.withRun(ctx, cand.RunID, cand.RunAttempt, func(rs *runState) error {
	239	+	j := rs.find(cand.ID)
	240	+	if j == nil \|\| j.Phase != domain.CIPhaseReady \|\| rs.run.CancelRequested {
	241	+	return errNoJob
	242	+	}
	243	+	token := newToken(domain.JobTokenPrefix)
	244	+	j.Phase, j.AssignedAt = domain.CIPhaseAssigned, c.now()
	245	+	j.TokenHash, j.TokenValidUntil = domain.HashTokenSecret(token), time.Time{}
	246	+	j.RunnerID, j.RunnerName, j.Isolation = runner.ID, runner.Name, runner.Isolation
	247	+	rs.touch(j)
	248	+	if err := rs.flush(ctx); err != nil {
	249	+	return err
	250	+	}
	251	+	out = c.runnerJob(rs, *j, token)
	252	+	if pollID != "" {
	253	+	c.mu.Lock()
	254	+	c.polls[runner.ID] = pollRecord{pollID: pollID, ref: jobRef(*j), token: token, hash: j.TokenHash}
	255	+	c.mu.Unlock()
	256	+	}
	257	+	return nil
	258	+	})
	259	+	switch {
	260	+	case err == nil:
	261	+	log := zerowrap.FromCtx(ctx)
	262	+	log.Info().Str("runner", runner.Name).Str("job_id", out.Job.ID).Int64("run_id", out.Job.RunID).Msg("job assigned")
	263	+	return out, nil
	264	+	case errors.Is(err, errNoJob), errors.Is(err, domain.ErrNotFound), errors.Is(err, domain.ErrConflict):
	265	+	continue // taken or gone meanwhile
	266	+	default:
	267	+	return nil, err
	268	+	}
	269	+	}
	270	+	return nil, errNoJob
	271	+	}
	272	+
	273	+	// --- the calls of a job token ---
	274	+
	275	+	// jobFor authenticates a job token and checks that it is the one of jobID.
	276	+	func (c *CI) jobFor(ctx context.Context, secret, jobID string) (domain.CIJob, error) {
	277	+	if !domain.IsJobToken(secret) {
	278	+	return domain.CIJob{}, domain.ErrUnauthenticated
	279	+	}
	280	+	j, err := c.cfg.Store.FindJobByToken(ctx, domain.HashTokenSecret(secret), c.now())
	281	+	if errors.Is(err, domain.ErrNotFound) {
	282	+	return domain.CIJob{}, domain.ErrUnauthenticated
	283	+	}
	284	+	if err != nil {
	285	+	return domain.CIJob{}, err
	286	+	}
	287	+	if j.ID != jobID {
	288	+	return domain.CIJob{}, fmt.Errorf("%w: this token is the token of another job", domain.ErrForbidden)
	289	+	}
	290	+	return j, nil
	291	+	}
	292	+
	293	+	func endedError(j domain.CIJob) error {
	294	+	return &domain.JobEndedError{Conclusion: j.Conclusion, Message: j.Error}
	295	+	}
	296	+
	297	+	// lockedJob runs fn with the lock of the run and the job read again.
	298	+	func (c CI) lockedJob(ctx context.Context, j domain.CIJob, fn func(cur domain.CIJob) error) error {
	299	+	lock := c.runLock(j.RunID)
	300	+	lock.Lock()
	301	+	defer lock.Unlock()
	302	+	cur, err := c.cfg.Store.GetJob(ctx, j.RunID, j.RunAttempt, j.ID)
	303	+	if err != nil {
	304	+	return err
	305	+	}
	306	+	return fn(&cur)
	307	+	}
	308	+
	309	+	// saveJob writes one job (its steps changed) and publishes JobUpdated. The
	310	+	// status of the run does not depend on steps.
	311	+	func (c CI) saveJob(ctx context.Context, j domain.CIJob) error {
	312	+	v, err := c.cfg.Store.Apply(ctx, ports.CIWrite{Jobs: []domain.CIJob{*j}})
	313	+	if err != nil {
	314	+	return fmt.Errorf("write job %s: %w", j.ID, err)
	315	+	}
	316	+	j.Version = v
	317	+	if repo, err := c.cfg.Meta.Get(ctx, j.Repo); err == nil {
	318	+	lines, _ := c.cfg.Logs.Count(ctx, logRef(*j))
	319	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: *j, Lines: lines, At: c.now(), Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	320	+	}
	321	+	c.jobs.notify(jobRef(*j))
	322	+	return nil
	323	+	}
	324	+
	325	+	func logRef(j domain.CIJob) ports.CILogRef {
	326	+	return ports.CILogRef{RunID: j.RunID, JobID: j.ID, Attempt: j.Attempt}
	327	+	}
	328	+
	329	+	// UpdateJob applies the steps of the request whole or not at all.
	330	+	func (c *CI) UpdateJob(ctx context.Context, secret, jobID string, steps []ports.StepUpdate) error {
	331	+	j, err := c.jobFor(ctx, secret, jobID)
	332	+	if err != nil {
	333	+	return err
	334	+	}
	335	+	if j.Ended() {
	336	+	return endedError(j)
	337	+	}
	338	+	if err := checkStepUpdates(steps); err != nil {
	339	+	return err
	340	+	}
	341	+	return c.lockedJob(ctx, j, func(cur *domain.CIJob) error {
	342	+	if cur.Ended() {
	343	+	return endedError(*cur)
	344	+	}
	345	+	next, changed, err := applySteps(cur.Steps, steps)
	346	+	if err != nil \|\| !changed {
	347	+	return err
	348	+	}
	349	+	cur.Steps = next
	350	+	return c.saveJob(ctx, cur)
	351	+	})
	352	+	}
	353	+
	354	+	// checkStepUpdates validates the shape of the steps of a request.
	355	+	func checkStepUpdates(steps []ports.StepUpdate) error {
	356	+	if len(steps) > domain.CIMaxSteps {
	357	+	return fmt.Errorf("%w: at most %d steps", domain.ErrInvalid, domain.CIMaxSteps)
	358	+	}
	359	+	for _, s := range steps {
	360	+	if s.Number < 1 \|\| s.Number > domain.CIMaxSteps {
	361	+	return fmt.Errorf("%w: step number %d is not between 1 and %d", domain.ErrInvalid, s.Number, domain.CIMaxSteps)
	362	+	}
	363	+	switch s.State {
	364	+	case domain.CIStepPending:
	365	+	if s.Conclusion != domain.CINoConclusion {
	366	+	return fmt.Errorf("%w: step %d is not completed: it has no conclusion", domain.ErrInvalid, s.Number)
	367	+	}
	368	+	case domain.CIStepInProgress:
	369	+	if s.Conclusion != domain.CINoConclusion \|\| s.StartedAt.IsZero() {
	370	+	return fmt.Errorf("%w: step %d is in progress: started_at is required, conclusion is not allowed", domain.ErrInvalid, s.Number)
	371	+	}
	372	+	case domain.CIStepCompleted:
	373	+	switch s.Conclusion {
	374	+	case domain.CISuccess, domain.CIFailure, domain.CICancelled, domain.CISkipped:
	375	+	default:
	376	+	return fmt.Errorf("%w: step %d is completed: its conclusion must be success, failure, cancelled or skipped", domain.ErrInvalid, s.Number)
	377	+	}
	378	+	if s.StartedAt.IsZero() && s.Conclusion != domain.CISkipped {
	379	+	return fmt.Errorf("%w: step %d: started_at is required", domain.ErrInvalid, s.Number)
	380	+	}
	381	+	if s.CompletedAt.IsZero() {
	382	+	return fmt.Errorf("%w: step %d is completed: completed_at is required", domain.ErrInvalid, s.Number)
	383	+	}
	384	+	default:
	385	+	return fmt.Errorf("%w: step %d has no state", domain.ErrInvalid, s.Number)
	386	+	}
	387	+	}
	388	+	return nil
	389	+	}
	390	+
	391	+	// applySteps returns the steps after the updates, and whether anything
	392	+	// changed. The input is not modified.
	393	+	func applySteps(have []domain.CIStep, ups []ports.StepUpdate) ([]domain.CIStep, bool, error) {
	394	+	out := slices.Clone(have)
	395	+	changed := false
	396	+	for _, u := range ups {
	397	+	name := truncateRunes(u.Name, domain.CIMaxStepName)
	398	+	i := slices.IndexFunc(out, func(s domain.CIStep) bool { return s.Number == u.Number })
	399	+	if i < 0 {
	400	+	out = append(out, domain.CIStep{
	401	+	Number: u.Number, Name: name, State: u.State, Conclusion: u.Conclusion, StartedAt: u.StartedAt, CompletedAt: u.CompletedAt,
	402	+	})
	403	+	changed = true
	404	+	continue
	405	+	}
	406	+	cur := out[i]
	407	+	next := cur
	408	+	next.Name, next.State, next.Conclusion = name, u.State, u.Conclusion
	409	+	next.StartedAt, next.CompletedAt = u.StartedAt, u.CompletedAt
	410	+	if cur.Name != "" && name == "" {
	411	+	next.Name = cur.Name // an empty name never erases one
	412	+	}
	413	+	switch {
	414	+	case cur.State == domain.CIStepCompleted:
	415	+	same := next.State == cur.State && next.Conclusion == cur.Conclusion && next.Name == cur.Name &&
	416	+	next.StartedAt.Equal(cur.StartedAt) && next.CompletedAt.Equal(cur.CompletedAt)
	417	+	if !same {
	418	+	return nil, false, fmt.Errorf("%w: step %d is completed: it cannot change", domain.ErrPreconditionFailed, u.Number)
	419	+	}
	420	+	continue
	421	+	case cur.State == domain.CIStepInProgress && next.State == domain.CIStepPending:
	422	+	return nil, false, fmt.Errorf("%w: step %d cannot go from in progress back to pending", domain.ErrPreconditionFailed, u.Number)
	423	+	}
	424	+	if !stepsEqual(cur, next) {
	425	+	out[i], changed = next, true
	426	+	}
	427	+	}
	428	+	slices.SortFunc(out, func(a, b domain.CIStep) int { return int(a.Number) - int(b.Number) })
	429	+	return out, changed, nil
	430	+	}
	431	+
	432	+	func stepsEqual(a, b domain.CIStep) bool {
	433	+	return a.Number == b.Number && a.Name == b.Name && a.State == b.State && a.Conclusion == b.Conclusion &&
	434	+	a.StartedAt.Equal(b.StartedAt) && a.CompletedAt.Equal(b.CompletedAt) && a.LogStart == b.LogStart && a.HasLines == b.HasLines
	435	+	}
	436	+
	437	+	// lastLineStep returns the highest step number that has lines, 0 for none.
	438	+	func lastLineStep(steps []domain.CIStep) int32 {
	439	+	var last int32
	440	+	for _, s := range steps {
	441	+	if s.HasLines {
	442	+	last = max(last, s.Number)
	443	+	}
	444	+	}
	445	+	return last
	446	+	}
	447	+
	448	+	// checkLines validates the shape of the lines of an AppendLog request.
	449	+	func checkLines(step int32, offset int64, lines []domain.CILogLine) error {
	450	+	if step < 1 \|\| step > domain.CIMaxSteps {
	451	+	return fmt.Errorf("%w: step %d is not between 1 and %d", domain.ErrInvalid, step, domain.CIMaxSteps)
	452	+	}
	453	+	if offset < 0 {
	454	+	return fmt.Errorf("%w: negative offset", domain.ErrInvalid)
	455	+	}
	456	+	if len(lines) > domain.CIMaxLogLines {
	457	+	return fmt.Errorf("%w: at most %d lines per request", domain.ErrInvalid, domain.CIMaxLogLines)
	458	+	}
	459	+	total := 0
	460	+	for i, l := range lines {
	461	+	if strings.ContainsRune(l.Text, '\n') \|\| !utf8.ValidString(l.Text) {
	462	+	return fmt.Errorf("%w: line %d has a line feed or is not valid UTF-8", domain.ErrInvalid, i)
	463	+	}
	464	+	if l.Time.IsZero() {
	465	+	return fmt.Errorf("%w: line %d has no time", domain.ErrInvalid, i)
	466	+	}
	467	+	if l.Continued && offset == 0 && i == 0 {
	468	+	return fmt.Errorf("%w: the first line of a job cannot be continued", domain.ErrInvalid)
	469	+	}
	470	+	total += len(l.Text)
	471	+	}
	472	+	if total > domain.CIMaxLogChunkBytes {
	473	+	return fmt.Errorf("%w: at most %d bytes of text per request", domain.ErrInvalid, domain.CIMaxLogChunkBytes)
	474	+	}
	475	+	return nil
	476	+	}
	477	+
	478	+	// AppendLog appends lines of a step to the log of the job; see runner.proto
	479	+	// for the rules of offsets. The metadata store is written only when a step
	480	+	// gets its first line.
	481	+	func (c *CI) AppendLog(ctx context.Context, secret, jobID string, step int32, offset int64, lines []domain.CILogLine) (int64, error) {
	482	+	j, err := c.jobFor(ctx, secret, jobID)
	483	+	if err != nil {
	484	+	return 0, err
	485	+	}
	486	+	if j.Ended() {
	487	+	return 0, endedError(j)
	488	+	}
	489	+	if err := checkLines(step, offset, lines); err != nil {
	490	+	return 0, err
	491	+	}
	492	+	var next int64
	493	+	limitHit := false
	494	+	err = c.lockedJob(ctx, j, func(cur *domain.CIJob) error {
	495	+	if cur.Ended() {
	496	+	return endedError(*cur)
	497	+	}
	498	+	ref := logRef(*cur)
	499	+	count, err := c.cfg.Logs.Count(ctx, ref)
	500	+	if err != nil {
	501	+	return err
	502	+	}
	503	+	if offset > count {
	504	+	return &ports.ErrCILogGap{Count: count}
	505	+	}
	506	+	skip := count - offset
	507	+	if skip >= int64(len(lines)) {
	508	+	next = count // everything is stored already
	509	+	return nil
	510	+	}
	511	+	last := lastLineStep(cur.Steps)
	512	+	if step < last {
	513	+	return fmt.Errorf("%w: step %d is below step %d, which already has lines", domain.ErrPreconditionFailed, step, last)
	514	+	}
	515	+	if first := lines[skip]; first.Continued && count > 0 && step != last {
	516	+	return fmt.Errorf("%w: a continued line must be in the step of the line before it", domain.ErrPreconditionFailed)
	517	+	}
	518	+	n, added, err := c.cfg.Logs.Append(ctx, ref, offset, lines, domain.CIMaxLogBytes)
	519	+	if errors.Is(err, ports.ErrCILogLimit) {
	520	+	limitHit = true
	521	+	return nil
	522	+	}
	523	+	if err != nil {
	524	+	return err
	525	+	}
	526	+	next = n
	527	+	if added == 0 {
	528	+	return nil
	529	+	}
	530	+	return c.startStepLines(ctx, cur, step, count, lines[skip].Time)
	531	+	})
	532	+	if err != nil {
	533	+	return 0, err
	534	+	}
	535	+	if limitHit {
	536	+	return 0, c.stopForLogLimit(ctx, j)
	537	+	}
	538	+	return next, nil
	539	+	}
	540	+
	541	+	// startStepLines records where the lines of a step begin, creating the step
	542	+	// when UpdateJob did not announce it.
	543	+	func (c CI) startStepLines(ctx context.Context, cur domain.CIJob, step int32, firstLine int64, at time.Time) error {
	544	+	i := slices.IndexFunc(cur.Steps, func(s domain.CIStep) bool { return s.Number == step })
	545	+	switch {
	546	+	case i < 0:
	547	+	cur.Steps = append(cur.Steps, domain.CIStep{
	548	+	Number: step, State: domain.CIStepInProgress, StartedAt: at, HasLines: true, LogStart: firstLine,
	549	+	})
	550	+	slices.SortFunc(cur.Steps, func(a, b domain.CIStep) int { return int(a.Number) - int(b.Number) })
	551	+	case cur.Steps[i].HasLines:
	552	+	return nil
	553	+	default:
	554	+	cur.Steps[i].HasLines, cur.Steps[i].LogStart = true, firstLine
	555	+	}
	556	+	return c.saveJob(ctx, cur)
	557	+	}
	558	+
	559	+	// stopForLogLimit ends a job whose log reached its limit: a last line says
	560	+	// so, the job fails, and the call answers like any call after the end.
	561	+	func (c *CI) stopForLogLimit(ctx context.Context, j domain.CIJob) error {
	562	+	var ended *domain.JobEndedError
	563	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	564	+	cur := rs.find(j.ID)
	565	+	if cur == nil \|\| cur.Ended() {
	566	+	if cur != nil {
	567	+	ended = &domain.JobEndedError{Conclusion: cur.Conclusion, Message: cur.Error}
	568	+	}
	569	+	return nil
	570	+	}
	571	+	_, _ = c.cfg.Logs.AppendForced(ctx, logRef(*cur), []domain.CILogLine{{Time: c.now(), Text: "##[error]" + domain.CILogLimitMessage}})
	572	+	rs.end(cur, domain.CIFailure, domain.CILogLimitMessage)
	573	+	ended = &domain.JobEndedError{Conclusion: domain.CIFailure, Message: domain.CILogLimitMessage}
	574	+	if err := rs.advance(ctx); err != nil {
	575	+	return err
	576	+	}
	577	+	return rs.flush(ctx)
	578	+	})
	579	+	if err != nil {
	580	+	return err
	581	+	}
	582	+	return ended
	583	+	}
	584	+
	585	+	// Heartbeat tells the server that the runner is alive. The first one
	586	+	// acknowledges the assignment; the others only move the deadline.
	587	+	func (c *CI) Heartbeat(ctx context.Context, secret, jobID string) error {
	588	+	j, err := c.jobFor(ctx, secret, jobID)
	589	+	if err != nil {
	590	+	return err
	591	+	}
	592	+	if j.Ended() {
	593	+	return endedError(j)
	594	+	}
	595	+	now := c.now()
	596	+	if j.Phase == domain.CIPhaseAssigned {
	597	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	598	+	cur := rs.find(j.ID)
	599	+	switch {
	600	+	case cur == nil:
	601	+	return domain.ErrUnauthenticated
	602	+	case cur.Ended():
	603	+	return endedError(*cur)
	604	+	case cur.TokenHash != j.TokenHash:
	605	+	return domain.ErrUnauthenticated // the assignment was cancelled and given again
	606	+	case cur.Phase != domain.CIPhaseAssigned:
	607	+	return nil // acknowledged by a concurrent call
	608	+	}
	609	+	cur.Phase, cur.AckedAt, cur.StartedAt, cur.AssignCancels = domain.CIPhaseRunning, now, now, 0
	610	+	rs.touch(cur)
	611	+	return rs.flush(ctx)
	612	+	})
	613	+	if err != nil {
	614	+	return err
	615	+	}
	616	+	}
	617	+	c.mu.Lock()
	618	+	c.heartbeats[jobRef(j)] = now
	619	+	c.seen[j.RunnerID] = now
	620	+	c.mu.Unlock()
	621	+	return nil
	622	+	}
	623	+
	624	+	// WatchJob streams what the runner must react to: the cancel request (the
	625	+	// first message when one is already pending), keepalives, and the end.
	626	+	func (c *CI) WatchJob(ctx context.Context, secret, jobID string, send func(ports.JobSignal) error) error {
	627	+	j, err := c.jobFor(ctx, secret, jobID)
	628	+	if err != nil {
	629	+	return err
	630	+	}
	631	+	if j.Ended() {
	632	+	return endedError(j)
	633	+	}
	634	+	ref := jobRef(j)
	635	+	keepalive := time.NewTicker(c.cfg.Keepalive)
	636	+	defer keepalive.Stop()
	637	+	cancelSent := false
	638	+	for {
	639	+	wake := c.jobs.wait(ref)
	640	+	cur, err := c.cfg.Store.GetJob(ctx, j.RunID, j.RunAttempt, j.ID)
	641	+	if err != nil {
	642	+	return err
	643	+	}
	644	+	if cur.TokenHash != j.TokenHash && !cur.Ended() {
	645	+	return domain.ErrUnauthenticated // given again
	646	+	}
	647	+	if cur.Ended() {
	648	+	return send(ports.JobSignal{Ended: &domain.JobEndedError{Conclusion: cur.Conclusion, Message: cur.Error}})
	649	+	}
	650	+	if cur.CancelReason != domain.CICancelNone && !cancelSent {
	651	+	cancelSent = true
	652	+	if err := send(ports.JobSignal{Cancel: &ports.JobCancel{
	653	+	Reason: cur.CancelReason, Message: cur.CancelMessage, RequestedAt: cur.CancelRequestedAt,
	654	+	}}); err != nil {
	655	+	return err
	656	+	}
	657	+	}
	658	+	select {
	659	+	case <-wake:
	660	+	case <-keepalive.C:
	661	+	if err := send(ports.JobSignal{Keepalive: true}); err != nil {
	662	+	return err
	663	+	}
	664	+	case <-ctx.Done():
	665	+	return nil
	666	+	case <-c.ctx.Done():
	667	+	return nil
	668	+	}
	669	+	}
	670	+	}
	671	+
	672	+	// CompleteJob ends a job, once. A later call returns the stored result.
	673	+	func (c *CI) CompleteJob(ctx context.Context, secret string, params ports.CompleteJobParams) (ports.CompleteJobResult, error) {
	674	+	j, err := c.jobFor(ctx, secret, params.JobID)
	675	+	if err != nil {
	676	+	return ports.CompleteJobResult{}, err
	677	+	}
	678	+	if j.Ended() {
	679	+	return ports.CompleteJobResult{Conclusion: j.Conclusion, Error: j.Error}, nil
	680	+	}
	681	+	switch params.Conclusion {
	682	+	case domain.CISuccess, domain.CIFailure, domain.CICancelled, domain.CITimedOut:
	683	+	default:
	684	+	return ports.CompleteJobResult{}, fmt.Errorf("%w: a runner reports success, failure, cancelled or timed_out", domain.ErrInvalid)
	685	+	}
	686	+	if params.RunnerShutdown && params.Conclusion != domain.CICancelled {
	687	+	return ports.CompleteJobResult{}, fmt.Errorf("%w: runner_shutdown goes with cancelled", domain.ErrInvalid)
	688	+	}
	689	+	if err := domain.CheckOutputs(params.Outputs); err != nil {
	690	+	return ports.CompleteJobResult{}, err
	691	+	}
	692	+	if err := checkStepUpdates(params.Steps); err != nil {
	693	+	return ports.CompleteJobResult{}, err
	694	+	}
	695	+	var res ports.CompleteJobResult
	696	+	err = c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	697	+	cur := rs.find(j.ID)
	698	+	if cur == nil {
	699	+	return domain.ErrNotFound
	700	+	}
	701	+	if cur.Ended() { // ended by the server since
	702	+	res = ports.CompleteJobResult{Conclusion: cur.Conclusion, Error: cur.Error}
	703	+	return nil
	704	+	}
	705	+	steps, _, err := applySteps(cur.Steps, params.Steps)
	706	+	if err != nil {
	707	+	return err
	708	+	}
	709	+	cur.Steps = steps
	710	+	cur.Outputs = params.Outputs
	711	+	msg := ""
	712	+	if params.RunnerShutdown {
	713	+	msg = domain.CIShutdownMessage
	714	+	}
	715	+	rs.end(cur, params.Conclusion, msg)
	716	+	if err := rs.advance(ctx); err != nil {
	717	+	return err
	718	+	}
	719	+	res = ports.CompleteJobResult{Conclusion: cur.Conclusion, Error: cur.Error, Applied: true}
	720	+	return rs.flush(ctx)
	721	+	})
	722	+	if err != nil {
	723	+	return ports.CompleteJobResult{}, err
	724	+	}
	725	+	if res.Applied {
	726	+	log := zerowrap.FromCtx(ctx)
	727	+	log.Info().Str("job_id", j.ID).Int64("run_id", j.RunID).Str("conclusion", string(res.Conclusion)).Msg("job completed")
	728	+	}
	729	+	return res, nil
	730	+	}

@@ -0,0 +1,730 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"fmt"
	9	+	"slices"
	10	+	"strings"
	11	+	"time"
	12	+	"unicode/utf8"
	13	+
	14	+	"github.com/bnema/zerowrap"
	15	+
	16	+	"bnema.dev/ermite/internal/domain"
	17	+	"bnema.dev/ermite/internal/ports"
	18	+	)
	19	+
	20	+	// maxVersionLen bounds RegisterRunner.version.
	21	+	const maxVersionLen = 64
	22	+
	23	+	// --- registration ---
	24	+
	25	+	// Register adds a runner and returns its token, shown once. The principal
	26	+	// needs runner:register on a token that is not restricted to repositories, or
	27	+	// to be root. A name in use is domain.ErrAlreadyExists unless Replace is set,
	28	+	// which removes the old runner (its token stops working at once).
	29	+	func (c *CI) Register(ctx context.Context, p domain.Principal, params ports.RegisterRunnerParams) (ports.RegisteredRunner, error) {
	30	+	if err := canManageRunners(p); err != nil {
	31	+	return ports.RegisteredRunner{}, err
	32	+	}
	33	+	if err := domain.ParseRunnerName(params.Name); err != nil {
	34	+	return ports.RegisteredRunner{}, err
	35	+	}
	36	+	labels, err := domain.ParseRunnerLabels(params.Labels)
	37	+	if err != nil {
	38	+	return ports.RegisteredRunner{}, err
	39	+	}
	40	+	var v domain.Violations
	41	+	if len(params.Version) > maxVersionLen \|\| !utf8.ValidString(params.Version) {
	42	+	v.Add("version", "is at most 64 characters")
	43	+	}
	44	+	switch params.Isolation {
	45	+	case domain.CIIsolationVM, domain.CIIsolationContainerSysbox, domain.CIIsolationContainerPrivileged:
	46	+	default:
	47	+	v.Add("isolation", "is required")
	48	+	}
	49	+	if params.Arch != "x64" && params.Arch != "arm64" {
	50	+	v.Add("arch", `must be "x64" or "arm64"`)
	51	+	}
	52	+	if params.OS != "linux" {
	53	+	v.Add("os", `must be "linux"`)
	54	+	}
	55	+	if err := v.Err(); err != nil {
	56	+	return ports.RegisteredRunner{}, err
	57	+	}
	58	+
	59	+	token := newToken(domain.RunnerTokenPrefix)
	60	+	r := domain.CIRunner{
	61	+	ID: newRunnerID(), Name: params.Name, Labels: domain.RunnerLabels(labels, params.OS, params.Arch),
	62	+	Isolation: params.Isolation, Version: params.Version, Arch: params.Arch, OS: params.OS,
	63	+	RegisteredAt: c.now(), TokenHash: domain.HashTokenSecret(token),
	64	+	}
	65	+	removed, err := c.cfg.Store.CreateRunner(ctx, r, params.Replace)
	66	+	if err != nil {
	67	+	return ports.RegisteredRunner{}, err
	68	+	}
	69	+	log := zerowrap.FromCtx(ctx)
	70	+	log.Info().Str("runner_id", r.ID).Str("runner", r.Name).Str("isolation", string(r.Isolation)).
	71	+	Int("replaced", len(removed)).Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("runner registered")
	72	+	c.forgetRunners(removed)
	73	+	return ports.RegisteredRunner{Runner: r, Token: token}, nil
	74	+	}
	75	+
	76	+	// forgetRunners drops what is kept in memory for removed runners and ends
	77	+	// their polls in progress.
	78	+	func (c *CI) forgetRunners(rs []domain.CIRunner) {
	79	+	c.mu.Lock()
	80	+	for _, r := range rs {
	81	+	delete(c.polls, r.ID)
	82	+	delete(c.seen, r.ID)
	83	+	}
	84	+	c.mu.Unlock()
	85	+	c.wakeQueue()
	86	+	}
	87	+
	88	+	// runnerFor authenticates a runner token.
	89	+	func (c *CI) runnerFor(ctx context.Context, secret string) (domain.CIRunner, error) {
	90	+	if !domain.IsRunnerToken(secret) {
	91	+	return domain.CIRunner{}, domain.ErrUnauthenticated
	92	+	}
	93	+	r, err := c.cfg.Store.FindRunnerByToken(ctx, domain.HashTokenSecret(secret))
	94	+	if errors.Is(err, domain.ErrNotFound) {
	95	+	return domain.CIRunner{}, domain.ErrUnauthenticated
	96	+	}
	97	+	return r, err
	98	+	}
	99	+
	100	+	func (c *CI) markSeen(runnerID string) {
	101	+	c.mu.Lock()
	102	+	c.seen[runnerID] = c.now()
	103	+	c.mu.Unlock()
	104	+	}
	105	+
	106	+	// --- FetchJob ---
	107	+
	108	+	var errNoJob = errors.New("no job")
	109	+
	110	+	// FetchJob holds the call until a queued job matches the runner, or the poll
	111	+	// time passes. A job is given to exactly one runner: assignments are made one
	112	+	// at a time under a lock and recorded before the answer is sent.
	113	+	func (c *CI) FetchJob(ctx context.Context, secret string, params ports.FetchJobParams) (ports.FetchJobResult, error) {
	114	+	runner, err := c.runnerFor(ctx, secret)
	115	+	if err != nil {
	116	+	return ports.FetchJobResult{}, err
	117	+	}
	118	+	offered, err := offeredLabels(runner, params.Labels)
	119	+	if err != nil {
	120	+	return ports.FetchJobResult{}, err
	121	+	}
	122	+	if params.PollID != "" {
	123	+	if err := checkPollID(params.PollID); err != nil {
	124	+	return ports.FetchJobResult{}, err
	125	+	}
	126	+	if j, ok := c.repeatPoll(ctx, runner, params.PollID); ok {
	127	+	return ports.FetchJobResult{Job: j}, nil
	128	+	}
	129	+	}
	130	+	c.markSeen(runner.ID)
	131	+	deadline := time.NewTimer(c.cfg.PollTimeout)
	132	+	defer deadline.Stop()
	133	+	for {
	134	+	wake := c.pending.wait(queueKey) // before looking: a job made meanwhile wakes us
	135	+	job, err := c.assignTo(ctx, runner, offered, params.PollID)
	136	+	switch {
	137	+	case err == nil:
	138	+	c.markSeen(runner.ID)
	139	+	return ports.FetchJobResult{Job: job}, nil
	140	+	case !errors.Is(err, errNoJob):
	141	+	return ports.FetchJobResult{}, err
	142	+	}
	143	+	select {
	144	+	case <-wake:
	145	+	case <-deadline.C:
	146	+	c.markSeen(runner.ID)
	147	+	return ports.FetchJobResult{}, nil
	148	+	case <-ctx.Done():
	149	+	return ports.FetchJobResult{}, ctx.Err()
	150	+	case <-c.ctx.Done():
	151	+	return ports.FetchJobResult{RetryAfter: time.Second}, nil
	152	+	}
	153	+	// A removed runner's poll ends with unauthenticated.
	154	+	if _, err := c.cfg.Store.GetRunner(ctx, runner.ID); errors.Is(err, domain.ErrNotFound) {
	155	+	return ports.FetchJobResult{}, domain.ErrUnauthenticated
	156	+	}
	157	+	}
	158	+	}
	159	+
	160	+	func checkPollID(id string) error {
	161	+	if len(id) > 64 {
	162	+	return fmt.Errorf("%w: poll_id is at most 64 characters", domain.ErrInvalid)
	163	+	}
	164	+	for _, r := range id {
	165	+	if (r < 'a' \|\| r > 'z') && (r < 'A' \|\| r > 'Z') && (r < '0' \|\| r > '9') && r != '_' && r != '-' {
	166	+	return fmt.Errorf("%w: poll_id has letters, digits, '_' and '-' only", domain.ErrInvalid)
	167	+	}
	168	+	}
	169	+	return nil
	170	+	}
	171	+
	172	+	// offeredLabels are the labels of one poll: all those of the runner, or the
	173	+	// subset the call names.
	174	+	func offeredLabels(r domain.CIRunner, asked []string) ([]string, error) {
	175	+	if len(asked) == 0 {
	176	+	return r.Labels, nil
	177	+	}
	178	+	var out []string
	179	+	for _, l := range asked {
	180	+	l = strings.ToLower(l)
	181	+	if !slices.Contains(r.Labels, l) {
	182	+	return nil, fmt.Errorf("%w: label %q is not a label of the runner", domain.ErrInvalid, l)
	183	+	}
	184	+	out = append(out, l)
	185	+	}
	186	+	return out, nil
	187	+	}
	188	+
	189	+	// repeatPoll answers a retried poll with the assignment the first call made,
	190	+	// as long as it has not been acknowledged or cancelled.
	191	+	func (c CI) repeatPoll(ctx context.Context, runner domain.CIRunner, pollID string) (ports.RunnerJob, bool) {
	192	+	c.mu.Lock()
	193	+	rec, ok := c.polls[runner.ID]
	194	+	c.mu.Unlock()
	195	+	if !ok \|\| rec.pollID != pollID {
	196	+	return nil, false
	197	+	}
	198	+	var rid int64
	199	+	var ra int32
	200	+	var id string
	201	+	if _, err := fmt.Sscanf(rec.ref, "%d/%d/%s", &rid, &ra, &id); err != nil {
	202	+	return nil, false
	203	+	}
	204	+	var out *ports.RunnerJob
	205	+	err := c.withRun(ctx, rid, ra, func(rs *runState) error {
	206	+	j := rs.find(id)
	207	+	if j == nil \|\| j.Phase != domain.CIPhaseAssigned \|\| j.TokenHash != rec.hash \|\| j.RunnerID != runner.ID {
	208	+	return errNoJob
	209	+	}
	210	+	out = c.runnerJob(rs, *j, rec.token)
	211	+	return nil
	212	+	})
	213	+	return out, err == nil && out != nil
	214	+	}
	215	+
	216	+	// runnerJob is the job as the runner gets it.
	217	+	func (c CI) runnerJob(rs runState, j domain.CIJob, token string) *ports.RunnerJob {
	218	+	return &ports.RunnerJob{
	219	+	Job: j, Token: token, Repo: rs.run.Repo, Sha: rs.run.Sha, Ref: rs.run.Ref,
	220	+	WorkflowPath: rs.run.WorkflowPath, WorkflowYAML: rs.src.WorkflowYAML,
	221	+	}
	222	+	}
	223	+
	224	+	// assignTo gives the oldest queued job that matches the labels to the runner,
	225	+	// or errNoJob. The assignment lock makes the choice and the write one step.
	226	+	func (c CI) assignTo(ctx context.Context, runner domain.CIRunner, offered []string, pollID string) (ports.RunnerJob, error) {
	227	+	c.assign.Lock()
	228	+	defer c.assign.Unlock()
	229	+	ready, err := c.cfg.Store.ListReadyJobs(ctx, 0)
	230	+	if err != nil {
	231	+	return nil, err
	232	+	}
	233	+	for _, cand := range ready {
	234	+	if !domain.LabelsMatch(cand.Labels, offered) {
	235	+	continue
	236	+	}
	237	+	var out *ports.RunnerJob
	238	+	err := c.withRun(ctx, cand.RunID, cand.RunAttempt, func(rs *runState) error {
	239	+	j := rs.find(cand.ID)
	240	+	if j == nil \|\| j.Phase != domain.CIPhaseReady \|\| rs.run.CancelRequested {
	241	+	return errNoJob
	242	+	}
	243	+	token := newToken(domain.JobTokenPrefix)
	244	+	j.Phase, j.AssignedAt = domain.CIPhaseAssigned, c.now()
	245	+	j.TokenHash, j.TokenValidUntil = domain.HashTokenSecret(token), time.Time{}
	246	+	j.RunnerID, j.RunnerName, j.Isolation = runner.ID, runner.Name, runner.Isolation
	247	+	rs.touch(j)
	248	+	if err := rs.flush(ctx); err != nil {
	249	+	return err
	250	+	}
	251	+	out = c.runnerJob(rs, *j, token)
	252	+	if pollID != "" {
	253	+	c.mu.Lock()
	254	+	c.polls[runner.ID] = pollRecord{pollID: pollID, ref: jobRef(*j), token: token, hash: j.TokenHash}
	255	+	c.mu.Unlock()
	256	+	}
	257	+	return nil
	258	+	})
	259	+	switch {
	260	+	case err == nil:
	261	+	log := zerowrap.FromCtx(ctx)
	262	+	log.Info().Str("runner", runner.Name).Str("job_id", out.Job.ID).Int64("run_id", out.Job.RunID).Msg("job assigned")
	263	+	return out, nil
	264	+	case errors.Is(err, errNoJob), errors.Is(err, domain.ErrNotFound), errors.Is(err, domain.ErrConflict):
	265	+	continue // taken or gone meanwhile
	266	+	default:
	267	+	return nil, err
	268	+	}
	269	+	}
	270	+	return nil, errNoJob
	271	+	}
	272	+
	273	+	// --- the calls of a job token ---
	274	+
	275	+	// jobFor authenticates a job token and checks that it is the one of jobID.
	276	+	func (c *CI) jobFor(ctx context.Context, secret, jobID string) (domain.CIJob, error) {
	277	+	if !domain.IsJobToken(secret) {
	278	+	return domain.CIJob{}, domain.ErrUnauthenticated
	279	+	}
	280	+	j, err := c.cfg.Store.FindJobByToken(ctx, domain.HashTokenSecret(secret), c.now())
	281	+	if errors.Is(err, domain.ErrNotFound) {
	282	+	return domain.CIJob{}, domain.ErrUnauthenticated
	283	+	}
	284	+	if err != nil {
	285	+	return domain.CIJob{}, err
	286	+	}
	287	+	if j.ID != jobID {
	288	+	return domain.CIJob{}, fmt.Errorf("%w: this token is the token of another job", domain.ErrForbidden)
	289	+	}
	290	+	return j, nil
	291	+	}
	292	+
	293	+	func endedError(j domain.CIJob) error {
	294	+	return &domain.JobEndedError{Conclusion: j.Conclusion, Message: j.Error}
	295	+	}
	296	+
	297	+	// lockedJob runs fn with the lock of the run and the job read again.
	298	+	func (c CI) lockedJob(ctx context.Context, j domain.CIJob, fn func(cur domain.CIJob) error) error {
	299	+	lock := c.runLock(j.RunID)
	300	+	lock.Lock()
	301	+	defer lock.Unlock()
	302	+	cur, err := c.cfg.Store.GetJob(ctx, j.RunID, j.RunAttempt, j.ID)
	303	+	if err != nil {
	304	+	return err
	305	+	}
	306	+	return fn(&cur)
	307	+	}
	308	+
	309	+	// saveJob writes one job (its steps changed) and publishes JobUpdated. The
	310	+	// status of the run does not depend on steps.
	311	+	func (c CI) saveJob(ctx context.Context, j domain.CIJob) error {
	312	+	v, err := c.cfg.Store.Apply(ctx, ports.CIWrite{Jobs: []domain.CIJob{*j}})
	313	+	if err != nil {
	314	+	return fmt.Errorf("write job %s: %w", j.ID, err)
	315	+	}
	316	+	j.Version = v
	317	+	if repo, err := c.cfg.Meta.Get(ctx, j.Repo); err == nil {
	318	+	lines, _ := c.cfg.Logs.Count(ctx, logRef(*j))
	319	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: *j, Lines: lines, At: c.now(), Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	320	+	}
	321	+	c.jobs.notify(jobRef(*j))
	322	+	return nil
	323	+	}
	324	+
	325	+	func logRef(j domain.CIJob) ports.CILogRef {
	326	+	return ports.CILogRef{RunID: j.RunID, JobID: j.ID, Attempt: j.Attempt}
	327	+	}
	328	+
	329	+	// UpdateJob applies the steps of the request whole or not at all.
	330	+	func (c *CI) UpdateJob(ctx context.Context, secret, jobID string, steps []ports.StepUpdate) error {
	331	+	j, err := c.jobFor(ctx, secret, jobID)
	332	+	if err != nil {
	333	+	return err
	334	+	}
	335	+	if j.Ended() {
	336	+	return endedError(j)
	337	+	}
	338	+	if err := checkStepUpdates(steps); err != nil {
	339	+	return err
	340	+	}
	341	+	return c.lockedJob(ctx, j, func(cur *domain.CIJob) error {
	342	+	if cur.Ended() {
	343	+	return endedError(*cur)
	344	+	}
	345	+	next, changed, err := applySteps(cur.Steps, steps)
	346	+	if err != nil \|\| !changed {
	347	+	return err
	348	+	}
	349	+	cur.Steps = next
	350	+	return c.saveJob(ctx, cur)
	351	+	})
	352	+	}
	353	+
	354	+	// checkStepUpdates validates the shape of the steps of a request.
	355	+	func checkStepUpdates(steps []ports.StepUpdate) error {
	356	+	if len(steps) > domain.CIMaxSteps {
	357	+	return fmt.Errorf("%w: at most %d steps", domain.ErrInvalid, domain.CIMaxSteps)
	358	+	}
	359	+	for _, s := range steps {
	360	+	if s.Number < 1 \|\| s.Number > domain.CIMaxSteps {
	361	+	return fmt.Errorf("%w: step number %d is not between 1 and %d", domain.ErrInvalid, s.Number, domain.CIMaxSteps)
	362	+	}
	363	+	switch s.State {
	364	+	case domain.CIStepPending:
	365	+	if s.Conclusion != domain.CINoConclusion {
	366	+	return fmt.Errorf("%w: step %d is not completed: it has no conclusion", domain.ErrInvalid, s.Number)
	367	+	}
	368	+	case domain.CIStepInProgress:
	369	+	if s.Conclusion != domain.CINoConclusion \|\| s.StartedAt.IsZero() {
	370	+	return fmt.Errorf("%w: step %d is in progress: started_at is required, conclusion is not allowed", domain.ErrInvalid, s.Number)
	371	+	}
	372	+	case domain.CIStepCompleted:
	373	+	switch s.Conclusion {
	374	+	case domain.CISuccess, domain.CIFailure, domain.CICancelled, domain.CISkipped:
	375	+	default:
	376	+	return fmt.Errorf("%w: step %d is completed: its conclusion must be success, failure, cancelled or skipped", domain.ErrInvalid, s.Number)
	377	+	}
	378	+	if s.StartedAt.IsZero() && s.Conclusion != domain.CISkipped {
	379	+	return fmt.Errorf("%w: step %d: started_at is required", domain.ErrInvalid, s.Number)
	380	+	}
	381	+	if s.CompletedAt.IsZero() {
	382	+	return fmt.Errorf("%w: step %d is completed: completed_at is required", domain.ErrInvalid, s.Number)
	383	+	}
	384	+	default:
	385	+	return fmt.Errorf("%w: step %d has no state", domain.ErrInvalid, s.Number)
	386	+	}
	387	+	}
	388	+	return nil
	389	+	}
	390	+
	391	+	// applySteps returns the steps after the updates, and whether anything
	392	+	// changed. The input is not modified.
	393	+	func applySteps(have []domain.CIStep, ups []ports.StepUpdate) ([]domain.CIStep, bool, error) {
	394	+	out := slices.Clone(have)
	395	+	changed := false
	396	+	for _, u := range ups {
	397	+	name := truncateRunes(u.Name, domain.CIMaxStepName)
	398	+	i := slices.IndexFunc(out, func(s domain.CIStep) bool { return s.Number == u.Number })
	399	+	if i < 0 {
	400	+	out = append(out, domain.CIStep{
	401	+	Number: u.Number, Name: name, State: u.State, Conclusion: u.Conclusion, StartedAt: u.StartedAt, CompletedAt: u.CompletedAt,
	402	+	})
	403	+	changed = true
	404	+	continue
	405	+	}
	406	+	cur := out[i]
	407	+	next := cur
	408	+	next.Name, next.State, next.Conclusion = name, u.State, u.Conclusion
	409	+	next.StartedAt, next.CompletedAt = u.StartedAt, u.CompletedAt
	410	+	if cur.Name != "" && name == "" {
	411	+	next.Name = cur.Name // an empty name never erases one
	412	+	}
	413	+	switch {
	414	+	case cur.State == domain.CIStepCompleted:
	415	+	same := next.State == cur.State && next.Conclusion == cur.Conclusion && next.Name == cur.Name &&
	416	+	next.StartedAt.Equal(cur.StartedAt) && next.CompletedAt.Equal(cur.CompletedAt)
	417	+	if !same {
	418	+	return nil, false, fmt.Errorf("%w: step %d is completed: it cannot change", domain.ErrPreconditionFailed, u.Number)
	419	+	}
	420	+	continue
	421	+	case cur.State == domain.CIStepInProgress && next.State == domain.CIStepPending:
	422	+	return nil, false, fmt.Errorf("%w: step %d cannot go from in progress back to pending", domain.ErrPreconditionFailed, u.Number)
	423	+	}
	424	+	if !stepsEqual(cur, next) {
	425	+	out[i], changed = next, true
	426	+	}
	427	+	}
	428	+	slices.SortFunc(out, func(a, b domain.CIStep) int { return int(a.Number) - int(b.Number) })
	429	+	return out, changed, nil
	430	+	}
	431	+
	432	+	func stepsEqual(a, b domain.CIStep) bool {
	433	+	return a.Number == b.Number && a.Name == b.Name && a.State == b.State && a.Conclusion == b.Conclusion &&
	434	+	a.StartedAt.Equal(b.StartedAt) && a.CompletedAt.Equal(b.CompletedAt) && a.LogStart == b.LogStart && a.HasLines == b.HasLines
	435	+	}
	436	+
	437	+	// lastLineStep returns the highest step number that has lines, 0 for none.
	438	+	func lastLineStep(steps []domain.CIStep) int32 {
	439	+	var last int32
	440	+	for _, s := range steps {
	441	+	if s.HasLines {
	442	+	last = max(last, s.Number)
	443	+	}
	444	+	}
	445	+	return last
	446	+	}
	447	+
	448	+	// checkLines validates the shape of the lines of an AppendLog request.
	449	+	func checkLines(step int32, offset int64, lines []domain.CILogLine) error {
	450	+	if step < 1 \|\| step > domain.CIMaxSteps {
	451	+	return fmt.Errorf("%w: step %d is not between 1 and %d", domain.ErrInvalid, step, domain.CIMaxSteps)
	452	+	}
	453	+	if offset < 0 {
	454	+	return fmt.Errorf("%w: negative offset", domain.ErrInvalid)
	455	+	}
	456	+	if len(lines) > domain.CIMaxLogLines {
	457	+	return fmt.Errorf("%w: at most %d lines per request", domain.ErrInvalid, domain.CIMaxLogLines)
	458	+	}
	459	+	total := 0
	460	+	for i, l := range lines {
	461	+	if strings.ContainsRune(l.Text, '\n') \|\| !utf8.ValidString(l.Text) {
	462	+	return fmt.Errorf("%w: line %d has a line feed or is not valid UTF-8", domain.ErrInvalid, i)
	463	+	}
	464	+	if l.Time.IsZero() {
	465	+	return fmt.Errorf("%w: line %d has no time", domain.ErrInvalid, i)
	466	+	}
	467	+	if l.Continued && offset == 0 && i == 0 {
	468	+	return fmt.Errorf("%w: the first line of a job cannot be continued", domain.ErrInvalid)
	469	+	}
	470	+	total += len(l.Text)
	471	+	}
	472	+	if total > domain.CIMaxLogChunkBytes {
	473	+	return fmt.Errorf("%w: at most %d bytes of text per request", domain.ErrInvalid, domain.CIMaxLogChunkBytes)
	474	+	}
	475	+	return nil
	476	+	}
	477	+
	478	+	// AppendLog appends lines of a step to the log of the job; see runner.proto
	479	+	// for the rules of offsets. The metadata store is written only when a step
	480	+	// gets its first line.
	481	+	func (c *CI) AppendLog(ctx context.Context, secret, jobID string, step int32, offset int64, lines []domain.CILogLine) (int64, error) {
	482	+	j, err := c.jobFor(ctx, secret, jobID)
	483	+	if err != nil {
	484	+	return 0, err
	485	+	}
	486	+	if j.Ended() {
	487	+	return 0, endedError(j)
	488	+	}
	489	+	if err := checkLines(step, offset, lines); err != nil {
	490	+	return 0, err
	491	+	}
	492	+	var next int64
	493	+	limitHit := false
	494	+	err = c.lockedJob(ctx, j, func(cur *domain.CIJob) error {
	495	+	if cur.Ended() {
	496	+	return endedError(*cur)
	497	+	}
	498	+	ref := logRef(*cur)
	499	+	count, err := c.cfg.Logs.Count(ctx, ref)
	500	+	if err != nil {
	501	+	return err
	502	+	}
	503	+	if offset > count {
	504	+	return &ports.ErrCILogGap{Count: count}
	505	+	}
	506	+	skip := count - offset
	507	+	if skip >= int64(len(lines)) {
	508	+	next = count // everything is stored already
	509	+	return nil
	510	+	}
	511	+	last := lastLineStep(cur.Steps)
	512	+	if step < last {
	513	+	return fmt.Errorf("%w: step %d is below step %d, which already has lines", domain.ErrPreconditionFailed, step, last)
	514	+	}
	515	+	if first := lines[skip]; first.Continued && count > 0 && step != last {
	516	+	return fmt.Errorf("%w: a continued line must be in the step of the line before it", domain.ErrPreconditionFailed)
	517	+	}
	518	+	n, added, err := c.cfg.Logs.Append(ctx, ref, offset, lines, domain.CIMaxLogBytes)
	519	+	if errors.Is(err, ports.ErrCILogLimit) {
	520	+	limitHit = true
	521	+	return nil
	522	+	}
	523	+	if err != nil {
	524	+	return err
	525	+	}
	526	+	next = n
	527	+	if added == 0 {
	528	+	return nil
	529	+	}
	530	+	return c.startStepLines(ctx, cur, step, count, lines[skip].Time)
	531	+	})
	532	+	if err != nil {
	533	+	return 0, err
	534	+	}
	535	+	if limitHit {
	536	+	return 0, c.stopForLogLimit(ctx, j)
	537	+	}
	538	+	return next, nil
	539	+	}
	540	+
	541	+	// startStepLines records where the lines of a step begin, creating the step
	542	+	// when UpdateJob did not announce it.
	543	+	func (c CI) startStepLines(ctx context.Context, cur domain.CIJob, step int32, firstLine int64, at time.Time) error {
	544	+	i := slices.IndexFunc(cur.Steps, func(s domain.CIStep) bool { return s.Number == step })
	545	+	switch {
	546	+	case i < 0:
	547	+	cur.Steps = append(cur.Steps, domain.CIStep{
	548	+	Number: step, State: domain.CIStepInProgress, StartedAt: at, HasLines: true, LogStart: firstLine,
	549	+	})
	550	+	slices.SortFunc(cur.Steps, func(a, b domain.CIStep) int { return int(a.Number) - int(b.Number) })
	551	+	case cur.Steps[i].HasLines:
	552	+	return nil
	553	+	default:
	554	+	cur.Steps[i].HasLines, cur.Steps[i].LogStart = true, firstLine
	555	+	}
	556	+	return c.saveJob(ctx, cur)
	557	+	}
	558	+
	559	+	// stopForLogLimit ends a job whose log reached its limit: a last line says
	560	+	// so, the job fails, and the call answers like any call after the end.
	561	+	func (c *CI) stopForLogLimit(ctx context.Context, j domain.CIJob) error {
	562	+	var ended *domain.JobEndedError
	563	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	564	+	cur := rs.find(j.ID)
	565	+	if cur == nil \|\| cur.Ended() {
	566	+	if cur != nil {
	567	+	ended = &domain.JobEndedError{Conclusion: cur.Conclusion, Message: cur.Error}
	568	+	}
	569	+	return nil
	570	+	}
	571	+	_, _ = c.cfg.Logs.AppendForced(ctx, logRef(*cur), []domain.CILogLine{{Time: c.now(), Text: "##[error]" + domain.CILogLimitMessage}})
	572	+	rs.end(cur, domain.CIFailure, domain.CILogLimitMessage)
	573	+	ended = &domain.JobEndedError{Conclusion: domain.CIFailure, Message: domain.CILogLimitMessage}
	574	+	if err := rs.advance(ctx); err != nil {
	575	+	return err
	576	+	}
	577	+	return rs.flush(ctx)
	578	+	})
	579	+	if err != nil {
	580	+	return err
	581	+	}
	582	+	return ended
	583	+	}
	584	+
	585	+	// Heartbeat tells the server that the runner is alive. The first one
	586	+	// acknowledges the assignment; the others only move the deadline.
	587	+	func (c *CI) Heartbeat(ctx context.Context, secret, jobID string) error {
	588	+	j, err := c.jobFor(ctx, secret, jobID)
	589	+	if err != nil {
	590	+	return err
	591	+	}
	592	+	if j.Ended() {
	593	+	return endedError(j)
	594	+	}
	595	+	now := c.now()
	596	+	if j.Phase == domain.CIPhaseAssigned {
	597	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	598	+	cur := rs.find(j.ID)
	599	+	switch {
	600	+	case cur == nil:
	601	+	return domain.ErrUnauthenticated
	602	+	case cur.Ended():
	603	+	return endedError(*cur)
	604	+	case cur.TokenHash != j.TokenHash:
	605	+	return domain.ErrUnauthenticated // the assignment was cancelled and given again
	606	+	case cur.Phase != domain.CIPhaseAssigned:
	607	+	return nil // acknowledged by a concurrent call
	608	+	}
	609	+	cur.Phase, cur.AckedAt, cur.StartedAt, cur.AssignCancels = domain.CIPhaseRunning, now, now, 0
	610	+	rs.touch(cur)
	611	+	return rs.flush(ctx)
	612	+	})
	613	+	if err != nil {
	614	+	return err
	615	+	}
	616	+	}
	617	+	c.mu.Lock()
	618	+	c.heartbeats[jobRef(j)] = now
	619	+	c.seen[j.RunnerID] = now
	620	+	c.mu.Unlock()
	621	+	return nil
	622	+	}
	623	+
	624	+	// WatchJob streams what the runner must react to: the cancel request (the
	625	+	// first message when one is already pending), keepalives, and the end.
	626	+	func (c *CI) WatchJob(ctx context.Context, secret, jobID string, send func(ports.JobSignal) error) error {
	627	+	j, err := c.jobFor(ctx, secret, jobID)
	628	+	if err != nil {
	629	+	return err
	630	+	}
	631	+	if j.Ended() {
	632	+	return endedError(j)
	633	+	}
	634	+	ref := jobRef(j)
	635	+	keepalive := time.NewTicker(c.cfg.Keepalive)
	636	+	defer keepalive.Stop()
	637	+	cancelSent := false
	638	+	for {
	639	+	wake := c.jobs.wait(ref)
	640	+	cur, err := c.cfg.Store.GetJob(ctx, j.RunID, j.RunAttempt, j.ID)
	641	+	if err != nil {
	642	+	return err
	643	+	}
	644	+	if cur.TokenHash != j.TokenHash && !cur.Ended() {
	645	+	return domain.ErrUnauthenticated // given again
	646	+	}
	647	+	if cur.Ended() {
	648	+	return send(ports.JobSignal{Ended: &domain.JobEndedError{Conclusion: cur.Conclusion, Message: cur.Error}})
	649	+	}
	650	+	if cur.CancelReason != domain.CICancelNone && !cancelSent {
	651	+	cancelSent = true
	652	+	if err := send(ports.JobSignal{Cancel: &ports.JobCancel{
	653	+	Reason: cur.CancelReason, Message: cur.CancelMessage, RequestedAt: cur.CancelRequestedAt,
	654	+	}}); err != nil {
	655	+	return err
	656	+	}
	657	+	}
	658	+	select {
	659	+	case <-wake:
	660	+	case <-keepalive.C:
	661	+	if err := send(ports.JobSignal{Keepalive: true}); err != nil {
	662	+	return err
	663	+	}
	664	+	case <-ctx.Done():
	665	+	return nil
	666	+	case <-c.ctx.Done():
	667	+	return nil
	668	+	}
	669	+	}
	670	+	}
	671	+
	672	+	// CompleteJob ends a job, once. A later call returns the stored result.
	673	+	func (c *CI) CompleteJob(ctx context.Context, secret string, params ports.CompleteJobParams) (ports.CompleteJobResult, error) {
	674	+	j, err := c.jobFor(ctx, secret, params.JobID)
	675	+	if err != nil {
	676	+	return ports.CompleteJobResult{}, err
	677	+	}
	678	+	if j.Ended() {
	679	+	return ports.CompleteJobResult{Conclusion: j.Conclusion, Error: j.Error}, nil
	680	+	}
	681	+	switch params.Conclusion {
	682	+	case domain.CISuccess, domain.CIFailure, domain.CICancelled, domain.CITimedOut:
	683	+	default:
	684	+	return ports.CompleteJobResult{}, fmt.Errorf("%w: a runner reports success, failure, cancelled or timed_out", domain.ErrInvalid)
	685	+	}
	686	+	if params.RunnerShutdown && params.Conclusion != domain.CICancelled {
	687	+	return ports.CompleteJobResult{}, fmt.Errorf("%w: runner_shutdown goes with cancelled", domain.ErrInvalid)
	688	+	}
	689	+	if err := domain.CheckOutputs(params.Outputs); err != nil {
	690	+	return ports.CompleteJobResult{}, err
	691	+	}
	692	+	if err := checkStepUpdates(params.Steps); err != nil {
	693	+	return ports.CompleteJobResult{}, err
	694	+	}
	695	+	var res ports.CompleteJobResult
	696	+	err = c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	697	+	cur := rs.find(j.ID)
	698	+	if cur == nil {
	699	+	return domain.ErrNotFound
	700	+	}
	701	+	if cur.Ended() { // ended by the server since
	702	+	res = ports.CompleteJobResult{Conclusion: cur.Conclusion, Error: cur.Error}
	703	+	return nil
	704	+	}
	705	+	steps, _, err := applySteps(cur.Steps, params.Steps)
	706	+	if err != nil {
	707	+	return err
	708	+	}
	709	+	cur.Steps = steps
	710	+	cur.Outputs = params.Outputs
	711	+	msg := ""
	712	+	if params.RunnerShutdown {
	713	+	msg = domain.CIShutdownMessage
	714	+	}
	715	+	rs.end(cur, params.Conclusion, msg)
	716	+	if err := rs.advance(ctx); err != nil {
	717	+	return err
	718	+	}
	719	+	res = ports.CompleteJobResult{Conclusion: cur.Conclusion, Error: cur.Error, Applied: true}
	720	+	return rs.flush(ctx)
	721	+	})
	722	+	if err != nil {
	723	+	return ports.CompleteJobResult{}, err
	724	+	}
	725	+	if res.Applied {
	726	+	log := zerowrap.FromCtx(ctx)
	727	+	log.Info().Str("job_id", j.ID).Int64("run_id", j.RunID).Str("conclusion", string(res.Conclusion)).Msg("job completed")
	728	+	}
	729	+	return res, nil
	730	+	}

A internal/usecase/ci_runs.go +531 -0

@@ -0,0 +1,531 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/base64"
	8	+	"errors"
	9	+	"fmt"
	10	+	"slices"
	11	+	"strconv"
	12	+	"strings"
	13	+	"time"
	14	+
	15	+	"bnema.dev/ermite/internal/domain"
	16	+	"bnema.dev/ermite/internal/ports"
	17	+	)
	18	+
	19	+	const runTokenPrefix = "ru1:"
	20	+
	21	+	// --- runs and jobs ---
	22	+
	23	+	// ListRuns returns a page of the runs of a repository, newest first. The
	24	+	// principal needs the access of reading the repository.
	25	+	func (c *CI) ListRuns(ctx context.Context, p domain.Principal, name domain.RepoName, params ports.ListRunsParams) (ports.RunPage, error) {
	26	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	27	+	return ports.RunPage{}, err
	28	+	}
	29	+	n, err := pageSize(params.PageSize)
	30	+	if err != nil {
	31	+	return ports.RunPage{}, err
	32	+	}
	33	+	var before int64
	34	+	if params.PageToken != "" {
	35	+	raw, ok := decodeToken(params.PageToken, runTokenPrefix)
	36	+	id, perr := strconv.ParseInt(raw, 10, 64)
	37	+	if !ok \|\| perr != nil \|\| id < 1 {
	38	+	return ports.RunPage{}, fmt.Errorf("%w: runs", domain.ErrInvalidPageToken)
	39	+	}
	40	+	before = id
	41	+	}
	42	+	runs, err := c.cfg.Store.ListRuns(ctx, name, before, n+1)
	43	+	if err != nil {
	44	+	return ports.RunPage{}, err
	45	+	}
	46	+	page := ports.RunPage{Runs: runs}
	47	+	if len(runs) > n {
	48	+	page.Runs = runs[:n]
	49	+	page.NextPageToken = base64.RawURLEncoding.EncodeToString([]byte(runTokenPrefix + strconv.FormatInt(runs[n-1].ID, 10)))
	50	+	}
	51	+	return page, nil
	52	+	}
	53	+
	54	+	// runOf finds a run of the repository; a run of another repository is not
	55	+	// found.
	56	+	func (c *CI) runOf(ctx context.Context, name domain.RepoName, runID int64) (domain.CIRun, error) {
	57	+	run, err := c.cfg.Store.GetRun(ctx, name, runID)
	58	+	if err != nil {
	59	+	return domain.CIRun{}, err
	60	+	}
	61	+	return run, nil
	62	+	}
	63	+
	64	+	// view adds the live line count to a job.
	65	+	func (c *CI) view(ctx context.Context, j domain.CIJob) ports.JobView {
	66	+	v := ports.JobView{CIJob: j}
	67	+	if j.Phase != domain.CIPhaseWaiting && j.Phase != domain.CIPhaseReady {
	68	+	v.Lines, _ = c.cfg.Logs.Count(ctx, logRef(j))
	69	+	}
	70	+	if j.RunnerID != "" {
	71	+	if _, err := c.cfg.Store.GetRunner(ctx, j.RunnerID); errors.Is(err, domain.ErrNotFound) {
	72	+	v.RunnerGone = true
	73	+	}
	74	+	}
	75	+	return v
	76	+	}
	77	+
	78	+	// GetRun returns a run with the jobs of an attempt (0 for the latest).
	79	+	func (c *CI) GetRun(ctx context.Context, p domain.Principal, name domain.RepoName, runID int64, attempt int32) (ports.RunDetail, error) {
	80	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	81	+	return ports.RunDetail{}, err
	82	+	}
	83	+	run, err := c.runOf(ctx, name, runID)
	84	+	if err != nil {
	85	+	return ports.RunDetail{}, err
	86	+	}
	87	+	latest := run.Attempt
	88	+	if attempt < 0 \|\| attempt > latest {
	89	+	return ports.RunDetail{}, domain.ErrNotFound
	90	+	}
	91	+	if attempt == 0 {
	92	+	attempt = latest
	93	+	}
	94	+	jobs, err := c.cfg.Store.ListJobs(ctx, runID, attempt)
	95	+	if err != nil {
	96	+	return ports.RunDetail{}, err
	97	+	}
	98	+	if attempt != latest {
	99	+	run = c.runAtAttempt(run, jobs, attempt)
	100	+	}
	101	+	d := ports.RunDetail{Run: run, LatestAttempt: latest}
	102	+	for _, j := range jobs {
	103	+	d.Jobs = append(d.Jobs, c.view(ctx, j))
	104	+	}
	105	+	return d, nil
	106	+	}
	107	+
	108	+	// runAtAttempt describes an earlier attempt of a run from its jobs: the run
	109	+	// record holds the latest attempt only.
	110	+	func (c *CI) runAtAttempt(run domain.CIRun, jobs []domain.CIJob, attempt int32) domain.CIRun {
	111	+	rs := &runState{c: c, run: run, jobs: jobs, attempt: attempt}
	112	+	r := rs.recompute(c.now())
	113	+	r.Attempt, r.Version = attempt, run.Version
	114	+	r.CancelRequested = false
	115	+	return r
	116	+	}
	117	+
	118	+	// jobOf finds a job of the repository by its id and attempt (0 for the
	119	+	// latest). The attempt is the one of the run it was kept by, or executed in.
	120	+	func (c *CI) jobOf(ctx context.Context, name domain.RepoName, jobID string, attempt int32) (domain.CIRun, domain.CIJob, error) {
	121	+	runID, _, _, ok := domain.ParseJobID(jobID)
	122	+	if !ok {
	123	+	return domain.CIRun{}, domain.CIJob{}, domain.ErrNotFound
	124	+	}
	125	+	run, err := c.runOf(ctx, name, runID)
	126	+	if err != nil {
	127	+	return domain.CIRun{}, domain.CIJob{}, err
	128	+	}
	129	+	if attempt < 0 \|\| attempt > run.Attempt {
	130	+	return domain.CIRun{}, domain.CIJob{}, domain.ErrNotFound
	131	+	}
	132	+	if attempt == 0 {
	133	+	attempt = run.Attempt
	134	+	}
	135	+	j, err := c.cfg.Store.GetJob(ctx, runID, attempt, jobID)
	136	+	if err != nil {
	137	+	return domain.CIRun{}, domain.CIJob{}, err
	138	+	}
	139	+	return run, j, nil
	140	+	}
	141	+
	142	+	// GetJob returns one job with its steps.
	143	+	func (c *CI) GetJob(ctx context.Context, p domain.Principal, name domain.RepoName, jobID string, attempt int32) (ports.JobView, error) {
	144	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	145	+	return ports.JobView{}, err
	146	+	}
	147	+	_, j, err := c.jobOf(ctx, name, jobID, attempt)
	148	+	if err != nil {
	149	+	return ports.JobView{}, err
	150	+	}
	151	+	return c.view(ctx, j), nil
	152	+	}
	153	+
	154	+	// --- log streaming ---
	155	+
	156	+	// acquireStream counts a log stream against the limits.
	157	+	func (c *CI) acquireStream(client string) (release func(), err error) {
	158	+	c.mu.Lock()
	159	+	defer c.mu.Unlock()
	160	+	if c.streams >= MaxCILogStreams \|\| (client != "" && c.perClient[client] >= MaxCILogStreamsPerClient) {
	161	+	return nil, fmt.Errorf("%w: too many log streams", domain.ErrResourceExhausted)
	162	+	}
	163	+	c.streams++
	164	+	if client != "" {
	165	+	c.perClient[client]++
	166	+	}
	167	+	return func() {
	168	+	c.mu.Lock()
	169	+	defer c.mu.Unlock()
	170	+	c.streams--
	171	+	if client != "" {
	172	+	if c.perClient[client]--; c.perClient[client] <= 0 {
	173	+	delete(c.perClient, client)
	174	+	}
	175	+	}
	176	+	}, nil
	177	+	}
	178	+
	179	+	// GetJobLog streams the lines of a job log from an offset, in batches of at
	180	+	// most 1,000 lines and 64 KiB. With Follow it goes on while the job runs, and
	181	+	// ends once the job completed and every line was sent.
	182	+	func (c *CI) GetJobLog(ctx context.Context, p domain.Principal, name domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error) error {
	183	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	184	+	return err
	185	+	}
	186	+	if params.Offset < 0 {
	187	+	return fmt.Errorf("%w: negative offset", domain.ErrInvalid)
	188	+	}
	189	+	run, j, err := c.jobOf(ctx, name, params.JobID, params.Attempt)
	190	+	if err != nil {
	191	+	return err
	192	+	}
	193	+	release, err := c.acquireStream(params.Client)
	194	+	if err != nil {
	195	+	return err
	196	+	}
	197	+	defer release()
	198	+	ref := logRef(j)
	199	+	jref := jobRef(j)
	200	+	offset := params.Offset
	201	+	checked := time.Now()
	202	+	for {
	203	+	wakeJob, wakeLog := c.jobs.wait(jref), c.cfg.Logs.Changed(ref)
	204	+	cur, err := c.cfg.Store.GetJob(ctx, j.RunID, j.RunAttempt, j.ID)
	205	+	if err != nil {
	206	+	return err
	207	+	}
	208	+	done := cur.Ended()
	209	+	lines, err := c.cfg.Logs.Read(ctx, ref, offset, domain.CIMaxLogLines, domain.CIMaxLogChunkBytes)
	210	+	switch {
	211	+	case errors.Is(err, domain.ErrNotFound):
	212	+	// No line yet. Without follow, or for a job that ended without
	213	+	// lines, there is no log.
	214	+	if !params.Follow \|\| done {
	215	+	if done && params.Offset == 0 && params.Follow {
	216	+	return nil
	217	+	}
	218	+	return domain.ErrNotFound
	219	+	}
	220	+	case errors.Is(err, domain.ErrOutOfRange):
	221	+	if !params.Follow \|\| done {
	222	+	return fmt.Errorf("%w: the log has fewer than %d lines", domain.ErrOutOfRange, offset)
	223	+	}
	224	+	return fmt.Errorf("%w: the log has fewer than %d lines", domain.ErrOutOfRange, offset)
	225	+	case err != nil:
	226	+	return err
	227	+	}
	228	+	if len(lines) > 0 {
	229	+	batch := ports.LogBatch{Offset: offset, Lines: c.withSteps(ctx, cur, offset, lines)}
	230	+	if err := send(batch); err != nil {
	231	+	return err
	232	+	}
	233	+	offset += int64(len(lines))
	234	+	continue // more may be waiting
	235	+	}
	236	+	if !params.Follow \|\| done {
	237	+	return nil
	238	+	}
	239	+	if p.TokenID() != "" && c.cfg.Auth != nil && time.Since(checked) > DefaultRevalidateEvery {
	240	+	if err := c.cfg.Auth.Revalidate(ctx, p); err != nil {
	241	+	return err
	242	+	}
	243	+	checked = time.Now()
	244	+	}
	245	+	_ = run
	246	+	select {
	247	+	case <-wakeJob:
	248	+	case <-wakeLog:
	249	+	case <-time.After(DefaultRevalidateEvery):
	250	+	case <-ctx.Done():
	251	+	return nil
	252	+	case <-c.ctx.Done():
	253	+	return nil
	254	+	}
	255	+	}
	256	+	}
	257	+
	258	+	// withSteps tags the lines of a batch with their step from the step ranges.
	259	+	func (c *CI) withSteps(_ context.Context, j domain.CIJob, offset int64, lines []domain.CILogLine) []ports.JobLogLine {
	260	+	out := make([]ports.JobLogLine, len(lines))
	261	+	for i, l := range lines {
	262	+	out[i] = ports.JobLogLine{Step: domain.StepOfLine(j.Steps, offset+int64(i)), CILogLine: l}
	263	+	}
	264	+	return out
	265	+	}
	266	+
	267	+	// --- cancel and rerun ---
	268	+
	269	+	// CancelRun cancels the latest attempt of a run. The principal needs ci:write
	270	+	// on the repository. Jobs not given to a runner end at once; the others get a
	271	+	// cancel request, and the run completes when they have ended.
	272	+	func (c *CI) CancelRun(ctx context.Context, p domain.Principal, name domain.RepoName, runID int64) (domain.CIRun, error) {
	273	+	if _, err := c.writableRepo(ctx, p, name); err != nil {
	274	+	return domain.CIRun{}, err
	275	+	}
	276	+	if _, err := c.runOf(ctx, name, runID); err != nil {
	277	+	return domain.CIRun{}, err
	278	+	}
	279	+	var out domain.CIRun
	280	+	err := c.withRun(ctx, runID, 0, func(rs *runState) error {
	281	+	if rs.run.Status == domain.CICompleted {
	282	+	return fmt.Errorf("%w: run %d has completed", domain.ErrPreconditionFailed, runID)
	283	+	}
	284	+	if !rs.run.CancelRequested {
	285	+	rs.run.CancelRequested, rs.run.CancelRequestedAt, rs.runDirty = true, c.now(), true
	286	+	rs.cancelAll("Canceled by " + authorOf(p))
	287	+	}
	288	+	if err := rs.flush(ctx); err != nil {
	289	+	return err
	290	+	}
	291	+	out = rs.run
	292	+	return nil
	293	+	})
	294	+	if err != nil {
	295	+	return domain.CIRun{}, err
	296	+	}
	297	+	ctxLog(ctx).Info().Str("repo", name.String()).Int64("run_id", runID).Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("ci run cancelled")
	298	+	return out, nil
	299	+	}
	300	+
	301	+	// RerunRun starts a new attempt of a completed run, from the stored workflow
	302	+	// file: all the jobs, or with failedOnly the jobs that did not succeed and
	303	+	// those that need them; the others are kept with their result, steps and log.
	304	+	func (c *CI) RerunRun(ctx context.Context, p domain.Principal, name domain.RepoName, runID int64, failedOnly bool) (domain.CIRun, error) {
	305	+	repo, err := c.writableRepo(ctx, p, name)
	306	+	if err != nil {
	307	+	return domain.CIRun{}, err
	308	+	}
	309	+	if _, err := c.runOf(ctx, name, runID); err != nil {
	310	+	return domain.CIRun{}, err
	311	+	}
	312	+	err = c.withRun(ctx, runID, 0, func(rs *runState) error {
	313	+	if rs.run.Status != domain.CICompleted {
	314	+	return fmt.Errorf("%w: run %d has not completed", domain.ErrPreconditionFailed, runID)
	315	+	}
	316	+	if len(rs.jobs) == 0 {
	317	+	return fmt.Errorf("%w: run %d has no job to run again", domain.ErrPreconditionFailed, runID)
	318	+	}
	319	+	redo, err := rerunSet(rs, failedOnly)
	320	+	if err != nil {
	321	+	return err
	322	+	}
	323	+	next := rs.run.Attempt + 1
	324	+	now := c.now()
	325	+	w := ports.CIWrite{}
	326	+	var fresh []domain.CIJob
	327	+	for _, j := range rs.jobs {
	328	+	if !redo[j.Key] {
	329	+	kept := j // kept with its own attempt, steps and log
	330	+	kept.RunAttempt, kept.Version = next, 0
	331	+	w.Jobs = append(w.Jobs, kept)
	332	+	fresh = append(fresh, kept)
	333	+	continue
	334	+	}
	335	+	if j.Combo > 0 {
	336	+	continue // the matrix is expanded again when the job is planned
	337	+	}
	338	+	n := domain.CIJob{
	339	+	ID: domain.NewJobID(runID, j.Index, 0), RunID: runID, Attempt: next, RunAttempt: next, Repo: rs.run.Repo,
	340	+	Key: j.Key, Name: j.Key, Phase: domain.CIPhaseWaiting, Needs: slices.Clone(j.Needs), Index: j.Index, CreatedAt: now,
	341	+	}
	342	+	w.Jobs = append(w.Jobs, n)
	343	+	fresh = append(fresh, n)
	344	+	}
	345	+	run := rs.run
	346	+	run.Attempt, run.Status, run.Conclusion = next, domain.CIQueued, domain.CINoConclusion
	347	+	run.StartedAt, run.CompletedAt, run.CreatedAt = time.Time{}, time.Time{}, now
	348	+	run.CancelRequested, run.CancelRequestedAt = false, time.Time{}
	349	+	w.Runs = []domain.CIRun{run}
	350	+	rs2 := &runState{c: c, run: run, attempt: next, jobs: fresh, src: rs.src, dirty: map[string]bool{}, checks: map[string]bool{}}
	351	+	for _, j := range fresh {
	352	+	w.Checks = append(w.Checks, rs2.checkOf(j))
	353	+	}
	354	+	v, err := c.cfg.Store.Apply(ctx, w)
	355	+	if err != nil {
	356	+	return err
	357	+	}
	358	+	run.Version = v
	359	+	c.flows.drop(runID)
	360	+	c.publishRun(ctx, repo, run)
	361	+	for _, j := range w.Jobs {
	362	+	j.Version = v
	363	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	364	+	}
	365	+	return nil
	366	+	})
	367	+	if err != nil {
	368	+	return domain.CIRun{}, err
	369	+	}
	370	+	// Plan the new attempt: the kept jobs count as the results of their needs.
	371	+	if err := c.advanceRun(ctx, runID); err != nil {
	372	+	return domain.CIRun{}, err
	373	+	}
	374	+	run, err := c.runOf(ctx, name, runID)
	375	+	if err != nil {
	376	+	return domain.CIRun{}, err
	377	+	}
	378	+	ctxLog(ctx).Info().Str("repo", name.String()).Int64("run_id", runID).Int32("attempt", run.Attempt).Bool("failed_only", failedOnly).
	379	+	Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("ci run started again")
	380	+	return run, nil
	381	+	}
	382	+
	383	+	// rerunSet returns the keys of the jobs to run again: all of them, or the ones
	384	+	// that did not succeed and those that need them, directly or not.
	385	+	func rerunSet(rs *runState, failedOnly bool) (map[string]bool, error) {
	386	+	redo := map[string]bool{}
	387	+	if !failedOnly {
	388	+	for _, j := range rs.jobs {
	389	+	redo[j.Key] = true
	390	+	}
	391	+	return redo, nil
	392	+	}
	393	+	for _, j := range rs.jobs {
	394	+	switch j.Conclusion {
	395	+	case domain.CIFailure, domain.CICancelled, domain.CITimedOut, domain.CIActionRequired:
	396	+	redo[j.Key] = true
	397	+	}
	398	+	}
	399	+	if len(redo) == 0 {
	400	+	return nil, fmt.Errorf("%w: no job of run %d failed", domain.ErrPreconditionFailed, rs.run.ID)
	401	+	}
	402	+	for changed := true; changed; {
	403	+	changed = false
	404	+	for _, j := range rs.jobs {
	405	+	if redo[j.Key] {
	406	+	continue
	407	+	}
	408	+	if slices.ContainsFunc(j.Needs, func(n string) bool { return redo[n] }) {
	409	+	redo[j.Key], changed = true, true
	410	+	}
	411	+	}
	412	+	}
	413	+	return redo, nil
	414	+	}
	415	+
	416	+	func (w *wfCache) drop(runID int64) {
	417	+	w.mu.Lock()
	418	+	delete(w.m, runID)
	419	+	w.mu.Unlock()
	420	+	}
	421	+
	422	+	// --- checks ---
	423	+
	424	+	// CommitChecks returns the checks of a commit, without access control.
	425	+	func (c *CI) CommitChecks(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error) {
	426	+	checks, _, err := c.cfg.Store.ListChecks(ctx, repo, sha, MaxCIChecks)
	427	+	return checks, err
	428	+	}
	429	+
	430	+	// ListChecks returns the checks of a commit: the one of each job of the latest
	431	+	// attempt of each run on it. A commit without a run has none.
	432	+	func (c *CI) ListChecks(ctx context.Context, p domain.Principal, name domain.RepoName, sha string) (ports.ChecksResult, error) {
	433	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	434	+	return ports.ChecksResult{}, err
	435	+	}
	436	+	if !isHex40(sha) {
	437	+	return ports.ChecksResult{}, fmt.Errorf("%w: a commit is 40 lowercase hexadecimal characters", domain.ErrInvalidObjectID)
	438	+	}
	439	+	checks, more, err := c.cfg.Store.ListChecks(ctx, name, sha, MaxCIChecks)
	440	+	if err != nil {
	441	+	return ports.ChecksResult{}, err
	442	+	}
	443	+	res := ports.ChecksResult{Truncated: more}
	444	+	var runners []domain.CIRunner
	445	+	for _, ck := range checks {
	446	+	v := ports.CheckView{CICheck: ck}
	447	+	if ck.Status == domain.CIQueued {
	448	+	if runners == nil {
	449	+	runners, _ = c.cfg.Store.ListRunners(ctx)
	450	+	}
	451	+	v.WaitingForRunner = c.noRunnerFor(ctx, ck, runners)
	452	+	}
	453	+	res.Checks = append(res.Checks, v)
	454	+	}
	455	+	return res, nil
	456	+	}
	457	+
	458	+	// noRunnerFor reports whether no registered runner can take a queued check's
	459	+	// job. A job that waits for its needs is not waiting for a runner.
	460	+	func (c *CI) noRunnerFor(ctx context.Context, ck domain.CICheck, runners []domain.CIRunner) bool {
	461	+	if ck.JobID == "" {
	462	+	return false
	463	+	}
	464	+	j, err := c.cfg.Store.GetJob(ctx, ck.RunID, ck.Attempt, ck.JobID)
	465	+	if err != nil \|\| j.Phase != domain.CIPhaseReady {
	466	+	return false
	467	+	}
	468	+	return !slices.ContainsFunc(runners, func(r domain.CIRunner) bool { return domain.LabelsMatch(j.Labels, r.Labels) })
	469	+	}
	470	+
	471	+	// --- runners ---
	472	+
	473	+	// ListRunners returns a page of the runners ordered by name. The principal
	474	+	// needs runner:register on a token that is not restricted to repositories.
	475	+	func (c *CI) ListRunners(ctx context.Context, p domain.Principal, size int, token string) (ports.RunnerPage, error) {
	476	+	if err := canManageRunners(p); err != nil {
	477	+	return ports.RunnerPage{}, err
	478	+	}
	479	+	n, err := pageSize(size)
	480	+	if err != nil {
	481	+	return ports.RunnerPage{}, err
	482	+	}
	483	+	after := ""
	484	+	if token != "" {
	485	+	raw, ok := decodeToken(token, "rn1:")
	486	+	if !ok \|\| raw == "" {
	487	+	return ports.RunnerPage{}, fmt.Errorf("%w: runners", domain.ErrInvalidPageToken)
	488	+	}
	489	+	after = raw
	490	+	}
	491	+	all, err := c.cfg.Store.ListRunners(ctx)
	492	+	if err != nil {
	493	+	return ports.RunnerPage{}, err
	494	+	}
	495	+	busy := map[string]bool{}
	496	+	if active, err := c.cfg.Store.ListActiveJobs(ctx); err == nil {
	497	+	for _, j := range active {
	498	+	if j.Phase == domain.CIPhaseRunning {
	499	+	busy[j.RunnerID] = true
	500	+	}
	501	+	}
	502	+	}
	503	+	var page ports.RunnerPage
	504	+	var rows []domain.CIRunner
	505	+	rows, page.NextPageToken = pageAfter(all, n,
	506	+	func(r domain.CIRunner) bool { return after == "" \|\| strings.ToLower(r.Name) > after },
	507	+	func(r domain.CIRunner) string {
	508	+	return base64.RawURLEncoding.EncodeToString([]byte("rn1:" + strings.ToLower(r.Name)))
	509	+	})
	510	+	c.mu.Lock()
	511	+	defer c.mu.Unlock()
	512	+	for _, r := range rows {
	513	+	page.Runners = append(page.Runners, ports.RunnerView{CIRunner: r, LastSeen: c.seen[r.ID], Busy: busy[r.ID]})
	514	+	}
	515	+	return page, nil
	516	+	}
	517	+
	518	+	// RemoveRunner removes a runner: its token stops working at once and a poll of
	519	+	// it in progress ends with unauthenticated. A job it runs is not affected.
	520	+	func (c *CI) RemoveRunner(ctx context.Context, p domain.Principal, id string) error {
	521	+	if err := canManageRunners(p); err != nil {
	522	+	return err
	523	+	}
	524	+	r, err := c.cfg.Store.RemoveRunner(ctx, id)
	525	+	if err != nil {
	526	+	return err
	527	+	}
	528	+	c.forgetRunners([]domain.CIRunner{r})
	529	+	ctxLog(ctx).Info().Str("runner_id", r.ID).Str("runner", r.Name).Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("runner removed")
	530	+	return nil
	531	+	}

@@ -0,0 +1,531 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/base64"
	8	+	"errors"
	9	+	"fmt"
	10	+	"slices"
	11	+	"strconv"
	12	+	"strings"
	13	+	"time"
	14	+
	15	+	"bnema.dev/ermite/internal/domain"
	16	+	"bnema.dev/ermite/internal/ports"
	17	+	)
	18	+
	19	+	const runTokenPrefix = "ru1:"
	20	+
	21	+	// --- runs and jobs ---
	22	+
	23	+	// ListRuns returns a page of the runs of a repository, newest first. The
	24	+	// principal needs the access of reading the repository.
	25	+	func (c *CI) ListRuns(ctx context.Context, p domain.Principal, name domain.RepoName, params ports.ListRunsParams) (ports.RunPage, error) {
	26	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	27	+	return ports.RunPage{}, err
	28	+	}
	29	+	n, err := pageSize(params.PageSize)
	30	+	if err != nil {
	31	+	return ports.RunPage{}, err
	32	+	}
	33	+	var before int64
	34	+	if params.PageToken != "" {
	35	+	raw, ok := decodeToken(params.PageToken, runTokenPrefix)
	36	+	id, perr := strconv.ParseInt(raw, 10, 64)
	37	+	if !ok \|\| perr != nil \|\| id < 1 {
	38	+	return ports.RunPage{}, fmt.Errorf("%w: runs", domain.ErrInvalidPageToken)
	39	+	}
	40	+	before = id
	41	+	}
	42	+	runs, err := c.cfg.Store.ListRuns(ctx, name, before, n+1)
	43	+	if err != nil {
	44	+	return ports.RunPage{}, err
	45	+	}
	46	+	page := ports.RunPage{Runs: runs}
	47	+	if len(runs) > n {
	48	+	page.Runs = runs[:n]
	49	+	page.NextPageToken = base64.RawURLEncoding.EncodeToString([]byte(runTokenPrefix + strconv.FormatInt(runs[n-1].ID, 10)))
	50	+	}
	51	+	return page, nil
	52	+	}
	53	+
	54	+	// runOf finds a run of the repository; a run of another repository is not
	55	+	// found.
	56	+	func (c *CI) runOf(ctx context.Context, name domain.RepoName, runID int64) (domain.CIRun, error) {
	57	+	run, err := c.cfg.Store.GetRun(ctx, name, runID)
	58	+	if err != nil {
	59	+	return domain.CIRun{}, err
	60	+	}
	61	+	return run, nil
	62	+	}
	63	+
	64	+	// view adds the live line count to a job.
	65	+	func (c *CI) view(ctx context.Context, j domain.CIJob) ports.JobView {
	66	+	v := ports.JobView{CIJob: j}
	67	+	if j.Phase != domain.CIPhaseWaiting && j.Phase != domain.CIPhaseReady {
	68	+	v.Lines, _ = c.cfg.Logs.Count(ctx, logRef(j))
	69	+	}
	70	+	if j.RunnerID != "" {
	71	+	if _, err := c.cfg.Store.GetRunner(ctx, j.RunnerID); errors.Is(err, domain.ErrNotFound) {
	72	+	v.RunnerGone = true
	73	+	}
	74	+	}
	75	+	return v
	76	+	}
	77	+
	78	+	// GetRun returns a run with the jobs of an attempt (0 for the latest).
	79	+	func (c *CI) GetRun(ctx context.Context, p domain.Principal, name domain.RepoName, runID int64, attempt int32) (ports.RunDetail, error) {
	80	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	81	+	return ports.RunDetail{}, err
	82	+	}
	83	+	run, err := c.runOf(ctx, name, runID)
	84	+	if err != nil {
	85	+	return ports.RunDetail{}, err
	86	+	}
	87	+	latest := run.Attempt
	88	+	if attempt < 0 \|\| attempt > latest {
	89	+	return ports.RunDetail{}, domain.ErrNotFound
	90	+	}
	91	+	if attempt == 0 {
	92	+	attempt = latest
	93	+	}
	94	+	jobs, err := c.cfg.Store.ListJobs(ctx, runID, attempt)
	95	+	if err != nil {
	96	+	return ports.RunDetail{}, err
	97	+	}
	98	+	if attempt != latest {
	99	+	run = c.runAtAttempt(run, jobs, attempt)
	100	+	}
	101	+	d := ports.RunDetail{Run: run, LatestAttempt: latest}
	102	+	for _, j := range jobs {
	103	+	d.Jobs = append(d.Jobs, c.view(ctx, j))
	104	+	}
	105	+	return d, nil
	106	+	}
	107	+
	108	+	// runAtAttempt describes an earlier attempt of a run from its jobs: the run
	109	+	// record holds the latest attempt only.
	110	+	func (c *CI) runAtAttempt(run domain.CIRun, jobs []domain.CIJob, attempt int32) domain.CIRun {
	111	+	rs := &runState{c: c, run: run, jobs: jobs, attempt: attempt}
	112	+	r := rs.recompute(c.now())
	113	+	r.Attempt, r.Version = attempt, run.Version
	114	+	r.CancelRequested = false
	115	+	return r
	116	+	}
	117	+
	118	+	// jobOf finds a job of the repository by its id and attempt (0 for the
	119	+	// latest). The attempt is the one of the run it was kept by, or executed in.
	120	+	func (c *CI) jobOf(ctx context.Context, name domain.RepoName, jobID string, attempt int32) (domain.CIRun, domain.CIJob, error) {
	121	+	runID, _, _, ok := domain.ParseJobID(jobID)
	122	+	if !ok {
	123	+	return domain.CIRun{}, domain.CIJob{}, domain.ErrNotFound
	124	+	}
	125	+	run, err := c.runOf(ctx, name, runID)
	126	+	if err != nil {
	127	+	return domain.CIRun{}, domain.CIJob{}, err
	128	+	}
	129	+	if attempt < 0 \|\| attempt > run.Attempt {
	130	+	return domain.CIRun{}, domain.CIJob{}, domain.ErrNotFound
	131	+	}
	132	+	if attempt == 0 {
	133	+	attempt = run.Attempt
	134	+	}
	135	+	j, err := c.cfg.Store.GetJob(ctx, runID, attempt, jobID)
	136	+	if err != nil {
	137	+	return domain.CIRun{}, domain.CIJob{}, err
	138	+	}
	139	+	return run, j, nil
	140	+	}
	141	+
	142	+	// GetJob returns one job with its steps.
	143	+	func (c *CI) GetJob(ctx context.Context, p domain.Principal, name domain.RepoName, jobID string, attempt int32) (ports.JobView, error) {
	144	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	145	+	return ports.JobView{}, err
	146	+	}
	147	+	_, j, err := c.jobOf(ctx, name, jobID, attempt)
	148	+	if err != nil {
	149	+	return ports.JobView{}, err
	150	+	}
	151	+	return c.view(ctx, j), nil
	152	+	}
	153	+
	154	+	// --- log streaming ---
	155	+
	156	+	// acquireStream counts a log stream against the limits.
	157	+	func (c *CI) acquireStream(client string) (release func(), err error) {
	158	+	c.mu.Lock()
	159	+	defer c.mu.Unlock()
	160	+	if c.streams >= MaxCILogStreams \|\| (client != "" && c.perClient[client] >= MaxCILogStreamsPerClient) {
	161	+	return nil, fmt.Errorf("%w: too many log streams", domain.ErrResourceExhausted)
	162	+	}
	163	+	c.streams++
	164	+	if client != "" {
	165	+	c.perClient[client]++
	166	+	}
	167	+	return func() {
	168	+	c.mu.Lock()
	169	+	defer c.mu.Unlock()
	170	+	c.streams--
	171	+	if client != "" {
	172	+	if c.perClient[client]--; c.perClient[client] <= 0 {
	173	+	delete(c.perClient, client)
	174	+	}
	175	+	}
	176	+	}, nil
	177	+	}
	178	+
	179	+	// GetJobLog streams the lines of a job log from an offset, in batches of at
	180	+	// most 1,000 lines and 64 KiB. With Follow it goes on while the job runs, and
	181	+	// ends once the job completed and every line was sent.
	182	+	func (c *CI) GetJobLog(ctx context.Context, p domain.Principal, name domain.RepoName, params ports.GetJobLogParams, send func(ports.LogBatch) error) error {
	183	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	184	+	return err
	185	+	}
	186	+	if params.Offset < 0 {
	187	+	return fmt.Errorf("%w: negative offset", domain.ErrInvalid)
	188	+	}
	189	+	run, j, err := c.jobOf(ctx, name, params.JobID, params.Attempt)
	190	+	if err != nil {
	191	+	return err
	192	+	}
	193	+	release, err := c.acquireStream(params.Client)
	194	+	if err != nil {
	195	+	return err
	196	+	}
	197	+	defer release()
	198	+	ref := logRef(j)
	199	+	jref := jobRef(j)
	200	+	offset := params.Offset
	201	+	checked := time.Now()
	202	+	for {
	203	+	wakeJob, wakeLog := c.jobs.wait(jref), c.cfg.Logs.Changed(ref)
	204	+	cur, err := c.cfg.Store.GetJob(ctx, j.RunID, j.RunAttempt, j.ID)
	205	+	if err != nil {
	206	+	return err
	207	+	}
	208	+	done := cur.Ended()
	209	+	lines, err := c.cfg.Logs.Read(ctx, ref, offset, domain.CIMaxLogLines, domain.CIMaxLogChunkBytes)
	210	+	switch {
	211	+	case errors.Is(err, domain.ErrNotFound):
	212	+	// No line yet. Without follow, or for a job that ended without
	213	+	// lines, there is no log.
	214	+	if !params.Follow \|\| done {
	215	+	if done && params.Offset == 0 && params.Follow {
	216	+	return nil
	217	+	}
	218	+	return domain.ErrNotFound
	219	+	}
	220	+	case errors.Is(err, domain.ErrOutOfRange):
	221	+	if !params.Follow \|\| done {
	222	+	return fmt.Errorf("%w: the log has fewer than %d lines", domain.ErrOutOfRange, offset)
	223	+	}
	224	+	return fmt.Errorf("%w: the log has fewer than %d lines", domain.ErrOutOfRange, offset)
	225	+	case err != nil:
	226	+	return err
	227	+	}
	228	+	if len(lines) > 0 {
	229	+	batch := ports.LogBatch{Offset: offset, Lines: c.withSteps(ctx, cur, offset, lines)}
	230	+	if err := send(batch); err != nil {
	231	+	return err
	232	+	}
	233	+	offset += int64(len(lines))
	234	+	continue // more may be waiting
	235	+	}
	236	+	if !params.Follow \|\| done {
	237	+	return nil
	238	+	}
	239	+	if p.TokenID() != "" && c.cfg.Auth != nil && time.Since(checked) > DefaultRevalidateEvery {
	240	+	if err := c.cfg.Auth.Revalidate(ctx, p); err != nil {
	241	+	return err
	242	+	}
	243	+	checked = time.Now()
	244	+	}
	245	+	_ = run
	246	+	select {
	247	+	case <-wakeJob:
	248	+	case <-wakeLog:
	249	+	case <-time.After(DefaultRevalidateEvery):
	250	+	case <-ctx.Done():
	251	+	return nil
	252	+	case <-c.ctx.Done():
	253	+	return nil
	254	+	}
	255	+	}
	256	+	}
	257	+
	258	+	// withSteps tags the lines of a batch with their step from the step ranges.
	259	+	func (c *CI) withSteps(_ context.Context, j domain.CIJob, offset int64, lines []domain.CILogLine) []ports.JobLogLine {
	260	+	out := make([]ports.JobLogLine, len(lines))
	261	+	for i, l := range lines {
	262	+	out[i] = ports.JobLogLine{Step: domain.StepOfLine(j.Steps, offset+int64(i)), CILogLine: l}
	263	+	}
	264	+	return out
	265	+	}
	266	+
	267	+	// --- cancel and rerun ---
	268	+
	269	+	// CancelRun cancels the latest attempt of a run. The principal needs ci:write
	270	+	// on the repository. Jobs not given to a runner end at once; the others get a
	271	+	// cancel request, and the run completes when they have ended.
	272	+	func (c *CI) CancelRun(ctx context.Context, p domain.Principal, name domain.RepoName, runID int64) (domain.CIRun, error) {
	273	+	if _, err := c.writableRepo(ctx, p, name); err != nil {
	274	+	return domain.CIRun{}, err
	275	+	}
	276	+	if _, err := c.runOf(ctx, name, runID); err != nil {
	277	+	return domain.CIRun{}, err
	278	+	}
	279	+	var out domain.CIRun
	280	+	err := c.withRun(ctx, runID, 0, func(rs *runState) error {
	281	+	if rs.run.Status == domain.CICompleted {
	282	+	return fmt.Errorf("%w: run %d has completed", domain.ErrPreconditionFailed, runID)
	283	+	}
	284	+	if !rs.run.CancelRequested {
	285	+	rs.run.CancelRequested, rs.run.CancelRequestedAt, rs.runDirty = true, c.now(), true
	286	+	rs.cancelAll("Canceled by " + authorOf(p))
	287	+	}
	288	+	if err := rs.flush(ctx); err != nil {
	289	+	return err
	290	+	}
	291	+	out = rs.run
	292	+	return nil
	293	+	})
	294	+	if err != nil {
	295	+	return domain.CIRun{}, err
	296	+	}
	297	+	ctxLog(ctx).Info().Str("repo", name.String()).Int64("run_id", runID).Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("ci run cancelled")
	298	+	return out, nil
	299	+	}
	300	+
	301	+	// RerunRun starts a new attempt of a completed run, from the stored workflow
	302	+	// file: all the jobs, or with failedOnly the jobs that did not succeed and
	303	+	// those that need them; the others are kept with their result, steps and log.
	304	+	func (c *CI) RerunRun(ctx context.Context, p domain.Principal, name domain.RepoName, runID int64, failedOnly bool) (domain.CIRun, error) {
	305	+	repo, err := c.writableRepo(ctx, p, name)
	306	+	if err != nil {
	307	+	return domain.CIRun{}, err
	308	+	}
	309	+	if _, err := c.runOf(ctx, name, runID); err != nil {
	310	+	return domain.CIRun{}, err
	311	+	}
	312	+	err = c.withRun(ctx, runID, 0, func(rs *runState) error {
	313	+	if rs.run.Status != domain.CICompleted {
	314	+	return fmt.Errorf("%w: run %d has not completed", domain.ErrPreconditionFailed, runID)
	315	+	}
	316	+	if len(rs.jobs) == 0 {
	317	+	return fmt.Errorf("%w: run %d has no job to run again", domain.ErrPreconditionFailed, runID)
	318	+	}
	319	+	redo, err := rerunSet(rs, failedOnly)
	320	+	if err != nil {
	321	+	return err
	322	+	}
	323	+	next := rs.run.Attempt + 1
	324	+	now := c.now()
	325	+	w := ports.CIWrite{}
	326	+	var fresh []domain.CIJob
	327	+	for _, j := range rs.jobs {
	328	+	if !redo[j.Key] {
	329	+	kept := j // kept with its own attempt, steps and log
	330	+	kept.RunAttempt, kept.Version = next, 0
	331	+	w.Jobs = append(w.Jobs, kept)
	332	+	fresh = append(fresh, kept)
	333	+	continue
	334	+	}
	335	+	if j.Combo > 0 {
	336	+	continue // the matrix is expanded again when the job is planned
	337	+	}
	338	+	n := domain.CIJob{
	339	+	ID: domain.NewJobID(runID, j.Index, 0), RunID: runID, Attempt: next, RunAttempt: next, Repo: rs.run.Repo,
	340	+	Key: j.Key, Name: j.Key, Phase: domain.CIPhaseWaiting, Needs: slices.Clone(j.Needs), Index: j.Index, CreatedAt: now,
	341	+	}
	342	+	w.Jobs = append(w.Jobs, n)
	343	+	fresh = append(fresh, n)
	344	+	}
	345	+	run := rs.run
	346	+	run.Attempt, run.Status, run.Conclusion = next, domain.CIQueued, domain.CINoConclusion
	347	+	run.StartedAt, run.CompletedAt, run.CreatedAt = time.Time{}, time.Time{}, now
	348	+	run.CancelRequested, run.CancelRequestedAt = false, time.Time{}
	349	+	w.Runs = []domain.CIRun{run}
	350	+	rs2 := &runState{c: c, run: run, attempt: next, jobs: fresh, src: rs.src, dirty: map[string]bool{}, checks: map[string]bool{}}
	351	+	for _, j := range fresh {
	352	+	w.Checks = append(w.Checks, rs2.checkOf(j))
	353	+	}
	354	+	v, err := c.cfg.Store.Apply(ctx, w)
	355	+	if err != nil {
	356	+	return err
	357	+	}
	358	+	run.Version = v
	359	+	c.flows.drop(runID)
	360	+	c.publishRun(ctx, repo, run)
	361	+	for _, j := range w.Jobs {
	362	+	j.Version = v
	363	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	364	+	}
	365	+	return nil
	366	+	})
	367	+	if err != nil {
	368	+	return domain.CIRun{}, err
	369	+	}
	370	+	// Plan the new attempt: the kept jobs count as the results of their needs.
	371	+	if err := c.advanceRun(ctx, runID); err != nil {
	372	+	return domain.CIRun{}, err
	373	+	}
	374	+	run, err := c.runOf(ctx, name, runID)
	375	+	if err != nil {
	376	+	return domain.CIRun{}, err
	377	+	}
	378	+	ctxLog(ctx).Info().Str("repo", name.String()).Int64("run_id", runID).Int32("attempt", run.Attempt).Bool("failed_only", failedOnly).
	379	+	Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("ci run started again")
	380	+	return run, nil
	381	+	}
	382	+
	383	+	// rerunSet returns the keys of the jobs to run again: all of them, or the ones
	384	+	// that did not succeed and those that need them, directly or not.
	385	+	func rerunSet(rs *runState, failedOnly bool) (map[string]bool, error) {
	386	+	redo := map[string]bool{}
	387	+	if !failedOnly {
	388	+	for _, j := range rs.jobs {
	389	+	redo[j.Key] = true
	390	+	}
	391	+	return redo, nil
	392	+	}
	393	+	for _, j := range rs.jobs {
	394	+	switch j.Conclusion {
	395	+	case domain.CIFailure, domain.CICancelled, domain.CITimedOut, domain.CIActionRequired:
	396	+	redo[j.Key] = true
	397	+	}
	398	+	}
	399	+	if len(redo) == 0 {
	400	+	return nil, fmt.Errorf("%w: no job of run %d failed", domain.ErrPreconditionFailed, rs.run.ID)
	401	+	}
	402	+	for changed := true; changed; {
	403	+	changed = false
	404	+	for _, j := range rs.jobs {
	405	+	if redo[j.Key] {
	406	+	continue
	407	+	}
	408	+	if slices.ContainsFunc(j.Needs, func(n string) bool { return redo[n] }) {
	409	+	redo[j.Key], changed = true, true
	410	+	}
	411	+	}
	412	+	}
	413	+	return redo, nil
	414	+	}
	415	+
	416	+	func (w *wfCache) drop(runID int64) {
	417	+	w.mu.Lock()
	418	+	delete(w.m, runID)
	419	+	w.mu.Unlock()
	420	+	}
	421	+
	422	+	// --- checks ---
	423	+
	424	+	// CommitChecks returns the checks of a commit, without access control.
	425	+	func (c *CI) CommitChecks(ctx context.Context, repo domain.RepoName, sha string) ([]domain.CICheck, error) {
	426	+	checks, _, err := c.cfg.Store.ListChecks(ctx, repo, sha, MaxCIChecks)
	427	+	return checks, err
	428	+	}
	429	+
	430	+	// ListChecks returns the checks of a commit: the one of each job of the latest
	431	+	// attempt of each run on it. A commit without a run has none.
	432	+	func (c *CI) ListChecks(ctx context.Context, p domain.Principal, name domain.RepoName, sha string) (ports.ChecksResult, error) {
	433	+	if _, err := c.readableRepo(ctx, p, name); err != nil {
	434	+	return ports.ChecksResult{}, err
	435	+	}
	436	+	if !isHex40(sha) {
	437	+	return ports.ChecksResult{}, fmt.Errorf("%w: a commit is 40 lowercase hexadecimal characters", domain.ErrInvalidObjectID)
	438	+	}
	439	+	checks, more, err := c.cfg.Store.ListChecks(ctx, name, sha, MaxCIChecks)
	440	+	if err != nil {
	441	+	return ports.ChecksResult{}, err
	442	+	}
	443	+	res := ports.ChecksResult{Truncated: more}
	444	+	var runners []domain.CIRunner
	445	+	for _, ck := range checks {
	446	+	v := ports.CheckView{CICheck: ck}
	447	+	if ck.Status == domain.CIQueued {
	448	+	if runners == nil {
	449	+	runners, _ = c.cfg.Store.ListRunners(ctx)
	450	+	}
	451	+	v.WaitingForRunner = c.noRunnerFor(ctx, ck, runners)
	452	+	}
	453	+	res.Checks = append(res.Checks, v)
	454	+	}
	455	+	return res, nil
	456	+	}
	457	+
	458	+	// noRunnerFor reports whether no registered runner can take a queued check's
	459	+	// job. A job that waits for its needs is not waiting for a runner.
	460	+	func (c *CI) noRunnerFor(ctx context.Context, ck domain.CICheck, runners []domain.CIRunner) bool {
	461	+	if ck.JobID == "" {
	462	+	return false
	463	+	}
	464	+	j, err := c.cfg.Store.GetJob(ctx, ck.RunID, ck.Attempt, ck.JobID)
	465	+	if err != nil \|\| j.Phase != domain.CIPhaseReady {
	466	+	return false
	467	+	}
	468	+	return !slices.ContainsFunc(runners, func(r domain.CIRunner) bool { return domain.LabelsMatch(j.Labels, r.Labels) })
	469	+	}
	470	+
	471	+	// --- runners ---
	472	+
	473	+	// ListRunners returns a page of the runners ordered by name. The principal
	474	+	// needs runner:register on a token that is not restricted to repositories.
	475	+	func (c *CI) ListRunners(ctx context.Context, p domain.Principal, size int, token string) (ports.RunnerPage, error) {
	476	+	if err := canManageRunners(p); err != nil {
	477	+	return ports.RunnerPage{}, err
	478	+	}
	479	+	n, err := pageSize(size)
	480	+	if err != nil {
	481	+	return ports.RunnerPage{}, err
	482	+	}
	483	+	after := ""
	484	+	if token != "" {
	485	+	raw, ok := decodeToken(token, "rn1:")
	486	+	if !ok \|\| raw == "" {
	487	+	return ports.RunnerPage{}, fmt.Errorf("%w: runners", domain.ErrInvalidPageToken)
	488	+	}
	489	+	after = raw
	490	+	}
	491	+	all, err := c.cfg.Store.ListRunners(ctx)
	492	+	if err != nil {
	493	+	return ports.RunnerPage{}, err
	494	+	}
	495	+	busy := map[string]bool{}
	496	+	if active, err := c.cfg.Store.ListActiveJobs(ctx); err == nil {
	497	+	for _, j := range active {
	498	+	if j.Phase == domain.CIPhaseRunning {
	499	+	busy[j.RunnerID] = true
	500	+	}
	501	+	}
	502	+	}
	503	+	var page ports.RunnerPage
	504	+	var rows []domain.CIRunner
	505	+	rows, page.NextPageToken = pageAfter(all, n,
	506	+	func(r domain.CIRunner) bool { return after == "" \|\| strings.ToLower(r.Name) > after },
	507	+	func(r domain.CIRunner) string {
	508	+	return base64.RawURLEncoding.EncodeToString([]byte("rn1:" + strings.ToLower(r.Name)))
	509	+	})
	510	+	c.mu.Lock()
	511	+	defer c.mu.Unlock()
	512	+	for _, r := range rows {
	513	+	page.Runners = append(page.Runners, ports.RunnerView{CIRunner: r, LastSeen: c.seen[r.ID], Busy: busy[r.ID]})
	514	+	}
	515	+	return page, nil
	516	+	}
	517	+
	518	+	// RemoveRunner removes a runner: its token stops working at once and a poll of
	519	+	// it in progress ends with unauthenticated. A job it runs is not affected.
	520	+	func (c *CI) RemoveRunner(ctx context.Context, p domain.Principal, id string) error {
	521	+	if err := canManageRunners(p); err != nil {
	522	+	return err
	523	+	}
	524	+	r, err := c.cfg.Store.RemoveRunner(ctx, id)
	525	+	if err != nil {
	526	+	return err
	527	+	}
	528	+	c.forgetRunners([]domain.CIRunner{r})
	529	+	ctxLog(ctx).Info().Str("runner_id", r.ID).Str("runner", r.Name).Str("principal", p.Label()).Str("token_id", p.TokenID()).Msg("runner removed")
	530	+	return nil
	531	+	}

A internal/usecase/ci_trigger.go +457 -0

@@ -0,0 +1,457 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/json"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"path"
	12	+	"runtime/debug"
	13	+	"slices"
	14	+	"strconv"
	15	+	"strings"
	16	+	"time"
	17	+
	18	+	"github.com/bnema/zerowrap"
	19	+
	20	+	"bnema.dev/ermite-actions/workflow"
	21	+	"bnema.dev/ermite/internal/domain"
	22	+	"bnema.dev/ermite/internal/ports"
	23	+	)
	24	+
	25	+	const (
	26	+	workflowDir = ".github/workflows"
	27	+	// maxRunError bounds Run.error (8 KiB, run.proto).
	28	+	maxRunError = 8 << 10
	29	+	// maxPushCommits is the number of commits of the push event payload.
	30	+	maxPushCommits = 20
	31	+	)
	32	+
	33	+	// ciTrigger is a push waiting to be planned.
	34	+	type ciTrigger struct {
	35	+	repo domain.RepoName
	36	+	updates []domain.RefUpdate
	37	+	actor string
	38	+	at time.Time
	39	+	log zerowrap.Logger
	40	+	}
	41	+
	42	+	// OnPush queues the refs of a push or a merge for planning and returns at once;
	43	+	// a worker plans them. A full queue drops the trigger, logged at error: a push
	44	+	// is never slowed down or failed by CI.
	45	+	func (c *CI) OnPush(ctx context.Context, repo domain.RepoName, updates []domain.RefUpdate, pusher domain.Principal) {
	46	+	var kept []domain.RefUpdate
	47	+	for _, u := range updates {
	48	+	if !u.IsDelete() && (strings.HasPrefix(u.Ref, "refs/heads/") \|\| strings.HasPrefix(u.Ref, "refs/tags/")) {
	49	+	kept = append(kept, u)
	50	+	}
	51	+	}
	52	+	if len(kept) == 0 {
	53	+	return
	54	+	}
	55	+	t := ciTrigger{repo: repo, updates: kept, actor: authorOf(pusher), at: c.now(), log: zerowrap.FromCtx(ctx)}
	56	+	c.mu.Lock()
	57	+	closed := c.closed
	58	+	c.mu.Unlock()
	59	+	if closed {
	60	+	return
	61	+	}
	62	+	select {
	63	+	case c.queue <- t:
	64	+	default:
	65	+	t.log.Error().Str("repo", repo.String()).Int("refs", len(kept)).Msg("ci trigger queue is full: the workflows of this push are not run")
	66	+	}
	67	+	}
	68	+
	69	+	func (c *CI) triggerLoop() {
	70	+	defer c.wg.Done()
	71	+	for {
	72	+	select {
	73	+	case <-c.ctx.Done():
	74	+	return
	75	+	case t := <-c.queue:
	76	+	c.planPush(t)
	77	+	}
	78	+	}
	79	+	}
	80	+
	81	+	// planPush plans one push; a panic or an error is logged and never escapes.
	82	+	func (c *CI) planPush(t ciTrigger) {
	83	+	ctx := zerowrap.WithCtx(c.ctx, t.log)
	84	+	defer func() {
	85	+	if r := recover(); r != nil {
	86	+	t.log.Error().Interface("panic", r).Bytes("stack", debug.Stack()).Str("repo", t.repo.String()).Msg("ci trigger panicked")
	87	+	}
	88	+	}()
	89	+	if err := c.handlePush(ctx, t); err != nil && ctx.Err() == nil {
	90	+	t.log.Error().Err(err).Str("repo", t.repo.String()).Msg("ci trigger failed")
	91	+	}
	92	+	}
	93	+
	94	+	// handlePush starts the workflows that match each ref of the push.
	95	+	func (c *CI) handlePush(ctx context.Context, t ciTrigger) error {
	96	+	tags := 0
	97	+	for _, u := range t.updates {
	98	+	if strings.HasPrefix(u.Ref, "refs/tags/") {
	99	+	tags++
	100	+	}
	101	+	}
	102	+	var errs []error
	103	+	for _, u := range t.updates {
	104	+	if err := c.handleRef(ctx, t, u, tags); err != nil {
	105	+	errs = append(errs, fmt.Errorf("%s: %w", u.Ref, err))
	106	+	}
	107	+	}
	108	+	return errors.Join(errs...)
	109	+	}
	110	+
	111	+	// pushFacts is what is known of one pushed ref.
	112	+	type pushFacts struct {
	113	+	update domain.RefUpdate
	114	+	commit domain.ObjectID // the commit the ref points to, tags followed
	115	+	head domain.Commit
	116	+	commits []domain.Commit // oldest first, at most maxPushCommits (branches only)
	117	+	forced bool
	118	+	created bool
	119	+	files []string
	120	+	// filesTruncated: the list of changed files is not available.
	121	+	filesTruncated bool
	122	+	tagCount int
	123	+	isTag bool
	124	+	}
	125	+
	126	+	func (c *CI) handleRef(ctx context.Context, t ciTrigger, u domain.RefUpdate, tagCount int) error {
	127	+	isTag := strings.HasPrefix(u.Ref, "refs/tags/")
	128	+	if isTag && tagCount > workflow.MaxTagsPerPush {
	129	+	return nil // GitHub creates no event for the tags of such a push
	130	+	}
	131	+	repo, err := c.cfg.Meta.Get(ctx, t.repo)
	132	+	if err != nil {
	133	+	return nil // deleted meanwhile
	134	+	}
	135	+	facts, err := c.collect(ctx, t.repo, u, isTag, tagCount)
	136	+	if err != nil {
	137	+	return err
	138	+	}
	139	+	files, err := c.workflowFiles(ctx, t.repo, facts.commit)
	140	+	if err != nil \|\| len(files) == 0 {
	141	+	return err
	142	+	}
	143	+	skip := workflow.SkipCI(facts.head.Message)
	144	+	var errs []error
	145	+	for _, f := range files {
	146	+	wf, perrs := workflow.Parse(f.path, f.content)
	147	+	switch {
	148	+	case f.problem != "":
	149	+	errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, f.problem))
	150	+	case len(perrs) > 0:
	151	+	var lines []string
	152	+	for _, e := range perrs {
	153	+	lines = append(lines, e.Error())
	154	+	}
	155	+	errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, strings.Join(lines, "\n")))
	156	+	case skip \|\| !wf.MatchPush(workflow.PushInfo{
	157	+	Ref: u.Ref, Created: facts.created, ChangedFiles: facts.files, FilesTruncated: facts.filesTruncated, TagCount: tagCount,
	158	+	}):
	159	+	default:
	160	+	errs = append(errs, c.startRun(ctx, repo, t, facts, f.path, string(f.content), wf))
	161	+	}
	162	+	}
	163	+	return errors.Join(errs...)
	164	+	}
	165	+
	166	+	// collect reads what the payload and the filters need from Git.
	167	+	func (c *CI) collect(ctx context.Context, repo domain.RepoName, u domain.RefUpdate, isTag bool, tagCount int) (pushFacts, error) {
	168	+	newID, err := domain.ParseObjectID(u.New)
	169	+	if err != nil {
	170	+	return pushFacts{}, err
	171	+	}
	172	+	f := pushFacts{update: u, created: u.IsCreate(), isTag: isTag, tagCount: tagCount}
	173	+	res, err := c.cfg.Reader.ResolveRevision(ctx, repo, domain.RevisionOf(newID))
	174	+	if err != nil {
	175	+	return f, fmt.Errorf("resolve %s: %w", u.New, err)
	176	+	}
	177	+	f.commit = res.CommitID
	178	+	if isTag {
	179	+	cs, _, err := c.cfg.Git.ListRange(ctx, repo, domain.ObjectID{}, f.commit, 1)
	180	+	if err != nil \|\| len(cs) == 0 {
	181	+	return f, fmt.Errorf("read the commit of the tag: %w", err)
	182	+	}
	183	+	f.head = cs[0]
	184	+	return f, nil
	185	+	}
	186	+
	187	+	var exclude domain.ObjectID
	188	+	if !f.created {
	189	+	if old, err := domain.ParseObjectID(u.Old); err == nil {
	190	+	exclude = old
	191	+	if ok, err := c.cfg.Git.IsAncestor(ctx, repo, old, f.commit); err == nil && !ok {
	192	+	f.forced = true
	193	+	}
	194	+	}
	195	+	}
	196	+	base := exclude
	197	+	if f.created {
	198	+	base = c.defaultBase(ctx, repo, strings.TrimPrefix(u.Ref, "refs/heads/"), f.commit)
	199	+	exclude = base
	200	+	}
	201	+	cs, more, err := c.cfg.Git.ListRange(ctx, repo, exclude, f.commit, ports.MaxCommitLimit)
	202	+	if err != nil \|\| len(cs) == 0 {
	203	+	return f, fmt.Errorf("read the commits of the push: %w", err)
	204	+	}
	205	+	f.head = cs[0]
	206	+	n := min(len(cs), maxPushCommits)
	207	+	f.commits = slices.Clone(cs[:n])
	208	+	slices.Reverse(f.commits)
	209	+	if more \|\| base == f.commit {
	210	+	// More than 1,000 commits: the diff is not looked at (it runs).
	211	+	f.filesTruncated = more
	212	+	}
	213	+	if !more && base != f.commit {
	214	+	paths, _, err := c.cfg.Diff.ChangedPaths(ctx, repo, base, f.commit, workflow.MaxFilesEvaluated)
	215	+	if err != nil {
	216	+	f.filesTruncated = true // diff unavailable: runs
	217	+	} else {
	218	+	f.files = paths
	219	+	}
	220	+	}
	221	+	return f, nil
	222	+	}
	223	+
	224	+	// defaultBase is the merge base of a new branch with the default branch, the
	225	+	// zero id when there is none (every file of the branch counts).
	226	+	func (c *CI) defaultBase(ctx context.Context, repo domain.RepoName, branch string, head domain.ObjectID) domain.ObjectID {
	227	+	def, exists, err := c.cfg.Repos.DefaultBranch(ctx, repo)
	228	+	if err != nil \|\| !exists \|\| def == branch {
	229	+	return domain.ObjectID{}
	230	+	}
	231	+	tip, err := c.cfg.Git.BranchHead(ctx, repo, def)
	232	+	if err != nil {
	233	+	return domain.ObjectID{}
	234	+	}
	235	+	base, err := c.cfg.Git.MergeBase(ctx, repo, tip, head)
	236	+	if err != nil {
	237	+	return domain.ObjectID{}
	238	+	}
	239	+	return base
	240	+	}
	241	+
	242	+	type workflowFile struct {
	243	+	path string
	244	+	content []byte
	245	+	// problem is set when the file cannot even be parsed (too large).
	246	+	problem string
	247	+	}
	248	+
	249	+	// workflowFiles reads .github/workflows/.yml\|.yaml at the commit.
	250	+	func (c *CI) workflowFiles(ctx context.Context, repo domain.RepoName, commit domain.ObjectID) ([]workflowFile, error) {
	251	+	dir, err := domain.ParseTreePath(workflowDir)
	252	+	if err != nil {
	253	+	return nil, err
	254	+	}
	255	+	tree, err := c.cfg.Reader.GetTree(ctx, repo, domain.RevisionOf(commit), dir, false)
	256	+	if errors.Is(err, domain.ErrNotFound) \|\| errors.Is(err, domain.ErrNotDirectory) {
	257	+	return nil, nil
	258	+	}
	259	+	if err != nil {
	260	+	return nil, fmt.Errorf("list %s: %w", workflowDir, err)
	261	+	}
	262	+	var out []workflowFile
	263	+	for _, e := range tree.Entries {
	264	+	ext := strings.ToLower(path.Ext(e.Name))
	265	+	if (e.Kind != domain.EntryFile && e.Kind != domain.EntryExecutable) \|\| (ext != ".yml" && ext != ".yaml") {
	266	+	continue
	267	+	}
	268	+	if len(out) == maxCIWorkflowFiles {
	269	+	break
	270	+	}
	271	+	wf := workflowFile{path: e.Path}
	272	+	if wf.path == "" {
	273	+	wf.path = workflowDir + "/" + e.Name
	274	+	}
	275	+	if e.Size > maxCIWorkflowBytes {
	276	+	wf.problem = fmt.Sprintf("%s: the workflow file is larger than 1 MiB", wf.path)
	277	+	out = append(out, wf)
	278	+	continue
	279	+	}
	280	+	tp, err := domain.ParseTreePath(wf.path)
	281	+	if err != nil {
	282	+	continue
	283	+	}
	284	+	_, rc, err := c.cfg.Reader.OpenBlob(ctx, repo, domain.RevisionOf(commit), tp)
	285	+	if err != nil {
	286	+	return nil, fmt.Errorf("read %s: %w", wf.path, err)
	287	+	}
	288	+	wf.content, err = io.ReadAll(io.LimitReader(rc, maxCIWorkflowBytes+1))
	289	+	_ = rc.Close()
	290	+	if err != nil {
	291	+	return nil, fmt.Errorf("read %s: %w", wf.path, err)
	292	+	}
	293	+	if len(wf.content) > maxCIWorkflowBytes {
	294	+	wf.content, wf.problem = nil, fmt.Sprintf("%s: the workflow file is larger than 1 MiB", wf.path)
	295	+	}
	296	+	out = append(out, wf)
	297	+	}
	298	+	return out, nil
	299	+	}
	300	+
	301	+	// --- the run ---
	302	+
	303	+	func firstLine(s string) string {
	304	+	line, _, _ := strings.Cut(s, "\n")
	305	+	return truncateRunes(strings.TrimSpace(line), 256)
	306	+	}
	307	+
	308	+	// startInvalid records the failed run of an invalid workflow file: no job, the
	309	+	// errors as Run.error, like GitHub's "Invalid workflow file".
	310	+	func (c *CI) startInvalid(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath, problem string) error {
	311	+	now := c.now()
	312	+	draft, err := c.cfg.Store.CreateRun(ctx, repo.Name, wfPath, func(id, number int64) ports.CIRunDraft {
	313	+	run := domain.CIRun{
	314	+	Repo: repo.Name, ID: id, Number: number, Attempt: 1, WorkflowPath: wfPath, WorkflowName: wfPath,
	315	+	Title: firstLine(f.head.Message), Event: "push", Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor,
	316	+	Status: domain.CICompleted, Conclusion: domain.CIFailure, Error: cutText(problem, maxRunError),
	317	+	CreatedAt: now, StartedAt: now, CompletedAt: now,
	318	+	}
	319	+	return ports.CIRunDraft{
	320	+	Run: run, Source: domain.CIRunSource{Repo: repo.Name, WorkflowYAML: "", GithubJSON: "{}"},
	321	+	Checks: []domain.CICheck{{
	322	+	ID: "run-" + strconv.FormatInt(id, 10), Repo: repo.Name, Sha: run.Sha, Name: wfPath, Status: domain.CICompleted,
	323	+	Conclusion: domain.CIFailure, RunID: id, Attempt: 1, WorkflowName: wfPath, CreatedAt: now, StartedAt: now, CompletedAt: now,
	324	+	}},
	325	+	}
	326	+	})
	327	+	if errors.Is(err, domain.ErrNotFound) {
	328	+	return nil
	329	+	}
	330	+	if err != nil {
	331	+	return err
	332	+	}
	333	+	c.publishRun(ctx, repo, draft.Run)
	334	+	log := logRun(ctx, draft.Run)
	335	+	log.Info().Str("workflow", wfPath).Msg("ci run failed: invalid workflow file")
	336	+	return nil
	337	+	}
	338	+
	339	+	func (c *CI) publishRun(ctx context.Context, repo domain.Repo, run domain.CIRun) {
	340	+	c.cfg.Bus.Publish(ctx, domain.RunUpdated{Run: run, At: c.now(), Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	341	+	}
	342	+
	343	+	// startRun creates the run of a matching workflow with one waiting record per
	344	+	// job of the file, then plans the jobs that can start.
	345	+	func (c CI) startRun(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath, yaml string, wf workflow.Workflow) error {
	346	+	now := c.now()
	347	+	name := wf.Name
	348	+	if name == "" {
	349	+	name = wfPath
	350	+	}
	351	+	draft, err := c.cfg.Store.CreateRun(ctx, repo.Name, wfPath, func(id, number int64) ports.CIRunDraft {
	352	+	run := domain.CIRun{
	353	+	Repo: repo.Name, ID: id, Number: number, Attempt: 1, WorkflowPath: wfPath, WorkflowName: name,
	354	+	Title: firstLine(f.head.Message), Event: "push", Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor,
	355	+	Status: domain.CIQueued, CreatedAt: now,
	356	+	}
	357	+	d := ports.CIRunDraft{Run: run, Source: domain.CIRunSource{
	358	+	Repo: repo.Name, WorkflowYAML: yaml, GithubJSON: c.githubJSON(repo, run, f, wf),
	359	+	}}
	360	+	for i, j := range wf.Jobs {
	361	+	job := domain.CIJob{
	362	+	ID: domain.NewJobID(id, i, 0), RunID: id, Attempt: 1, RunAttempt: 1, Repo: repo.Name, Key: j.ID, Name: j.ID,
	363	+	Phase: domain.CIPhaseWaiting, Needs: slices.Clone(j.Needs), Index: i, CreatedAt: now,
	364	+	}
	365	+	d.Jobs = append(d.Jobs, job)
	366	+	d.Checks = append(d.Checks, domain.CICheck{
	367	+	ID: job.ID, Repo: repo.Name, Sha: run.Sha, Name: job.Name, Status: domain.CIQueued, RunID: id, JobID: job.ID,
	368	+	Attempt: 1, WorkflowName: name, CreatedAt: now,
	369	+	})
	370	+	}
	371	+	return d
	372	+	})
	373	+	if errors.Is(err, domain.ErrNotFound) {
	374	+	return nil
	375	+	}
	376	+	if err != nil {
	377	+	return err
	378	+	}
	379	+	c.publishRun(ctx, repo, draft.Run)
	380	+	for _, j := range draft.Jobs {
	381	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	382	+	}
	383	+	log := logRun(ctx, draft.Run)
	384	+	log.Info().Str("workflow", wfPath).Str("ref", f.update.Ref).Int("jobs", len(draft.Jobs)).Msg("ci run created")
	385	+	return c.advanceRun(ctx, draft.Run.ID)
	386	+	}
	387	+
	388	+	// --- the github context ---
	389	+
	390	+	type obj = map[string]any
	391	+
	392	+	func person(name, email string) obj { return obj{"name": name, "email": email} }
	393	+
	394	+	func (c *CI) commitJSON(repo domain.Repo, cm domain.Commit) obj {
	395	+	ts := cm.Committer.Time.UTC().Format(time.RFC3339)
	396	+	return obj{
	397	+	"id": cm.ID.String(), "tree_id": "", "distinct": true, "message": cm.Message, "timestamp": ts,
	398	+	"url": c.cfg.PublicURL + "/" + repo.Name.String() + "/commit/" + cm.ID.String(),
	399	+	"author": obj{"name": cm.Author.Name, "email": cm.Author.Email, "username": ""},
	400	+	"committer": obj{"name": cm.Committer.Name, "email": cm.Committer.Email, "username": ""},
	401	+	"added": []string{}, "removed": []string{}, "modified": []string{},
	402	+	}
	403	+	}
	404	+
	405	+	// githubJSON builds the "github" context of the run as GitHub does for a push
	406	+	// (the job-dependent keys are added when a job is planned). Values that
	407	+	// GitHub's documentation does not settle are marked for the recordings.
	408	+	//
	409	+	// TODO(verify with recording): the exact types of run_id, run_number,
	410	+	// run_attempt (strings) and retention_days (a number), the empty tree_id and
	411	+	// the empty added/removed/modified lists of the push commits.
	412	+	func (c CI) githubJSON(repo domain.Repo, run domain.CIRun, f pushFacts, wf workflow.Workflow) string {
	413	+	full := c.cfg.Owner + "/" + repo.Name.String()
	414	+	refName, refType := strings.TrimPrefix(run.Ref, "refs/heads/"), "branch"
	415	+	if f.isTag {
	416	+	refName, refType = strings.TrimPrefix(run.Ref, "refs/tags/"), "tag"
	417	+	}
	418	+	repoURL := c.cfg.PublicURL + "/" + repo.Name.String()
	419	+	before := f.update.Old
	420	+	if f.created {
	421	+	before = domain.ZeroOID
	422	+	}
	423	+	compare := repoURL + "/compare/" + before + "..." + f.update.New
	424	+	if f.created {
	425	+	compare = repoURL + "/commit/" + f.update.New
	426	+	}
	427	+	commits := make([]any, 0, len(f.commits))
	428	+	for _, cm := range f.commits {
	429	+	commits = append(commits, c.commitJSON(repo, cm))
	430	+	}
	431	+	event := obj{
	432	+	"ref": run.Ref, "before": before, "after": f.update.New, "created": f.created, "deleted": false, "forced": f.forced,
	433	+	"base_ref": nil, "compare": compare, "commits": commits, "head_commit": c.commitJSON(repo, f.head),
	434	+	"pusher": person(run.Actor, ""),
	435	+	"repository": obj{
	436	+	"name": repo.Name.String(), "full_name": full, "private": repo.Visibility == domain.VisibilityPrivate,
	437	+	"owner": obj{"login": c.cfg.Owner, "name": c.cfg.Owner}, "html_url": repoURL, "clone_url": repoURL + ".git",
	438	+	"default_branch": "", "url": repoURL,
	439	+	},
	440	+	"sender": obj{"login": run.Actor, "type": "User"},
	441	+	}
	442	+	wfRef := full + "/" + run.WorkflowPath + "@" + run.Ref
	443	+	gh := obj{
	444	+	"event_name": "push", "event": event, "ref": run.Ref, "ref_name": refName, "ref_type": refType, "ref_protected": false,
	445	+	"sha": run.Sha, "repository": full, "repository_owner": c.cfg.Owner, "repositoryUrl": repoURL + ".git",
	446	+	"actor": run.Actor, "triggering_actor": run.Actor, "workflow": run.WorkflowName, "workflow_ref": wfRef,
	447	+	"workflow_sha": run.Sha, "run_id": strconv.FormatInt(run.ID, 10), "run_number": strconv.FormatInt(run.Number, 10),
	448	+	"server_url": c.cfg.PublicURL, "api_url": c.cfg.PublicURL + "/-/github/api/v3",
	449	+	"graphql_url": c.cfg.PublicURL + "/-/github/api/graphql", "retention_days": domain.CIRetentionDays,
	450	+	"head_ref": "", "base_ref": "",
	451	+	}
	452	+	b, err := json.Marshal(gh)
	453	+	if err != nil {
	454	+	return "{}"
	455	+	}
	456	+	return string(b)
	457	+	}

@@ -0,0 +1,457 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"encoding/json"
	8	+	"errors"
	9	+	"fmt"
	10	+	"io"
	11	+	"path"
	12	+	"runtime/debug"
	13	+	"slices"
	14	+	"strconv"
	15	+	"strings"
	16	+	"time"
	17	+
	18	+	"github.com/bnema/zerowrap"
	19	+
	20	+	"bnema.dev/ermite-actions/workflow"
	21	+	"bnema.dev/ermite/internal/domain"
	22	+	"bnema.dev/ermite/internal/ports"
	23	+	)
	24	+
	25	+	const (
	26	+	workflowDir = ".github/workflows"
	27	+	// maxRunError bounds Run.error (8 KiB, run.proto).
	28	+	maxRunError = 8 << 10
	29	+	// maxPushCommits is the number of commits of the push event payload.
	30	+	maxPushCommits = 20
	31	+	)
	32	+
	33	+	// ciTrigger is a push waiting to be planned.
	34	+	type ciTrigger struct {
	35	+	repo domain.RepoName
	36	+	updates []domain.RefUpdate
	37	+	actor string
	38	+	at time.Time
	39	+	log zerowrap.Logger
	40	+	}
	41	+
	42	+	// OnPush queues the refs of a push or a merge for planning and returns at once;
	43	+	// a worker plans them. A full queue drops the trigger, logged at error: a push
	44	+	// is never slowed down or failed by CI.
	45	+	func (c *CI) OnPush(ctx context.Context, repo domain.RepoName, updates []domain.RefUpdate, pusher domain.Principal) {
	46	+	var kept []domain.RefUpdate
	47	+	for _, u := range updates {
	48	+	if !u.IsDelete() && (strings.HasPrefix(u.Ref, "refs/heads/") \|\| strings.HasPrefix(u.Ref, "refs/tags/")) {
	49	+	kept = append(kept, u)
	50	+	}
	51	+	}
	52	+	if len(kept) == 0 {
	53	+	return
	54	+	}
	55	+	t := ciTrigger{repo: repo, updates: kept, actor: authorOf(pusher), at: c.now(), log: zerowrap.FromCtx(ctx)}
	56	+	c.mu.Lock()
	57	+	closed := c.closed
	58	+	c.mu.Unlock()
	59	+	if closed {
	60	+	return
	61	+	}
	62	+	select {
	63	+	case c.queue <- t:
	64	+	default:
	65	+	t.log.Error().Str("repo", repo.String()).Int("refs", len(kept)).Msg("ci trigger queue is full: the workflows of this push are not run")
	66	+	}
	67	+	}
	68	+
	69	+	func (c *CI) triggerLoop() {
	70	+	defer c.wg.Done()
	71	+	for {
	72	+	select {
	73	+	case <-c.ctx.Done():
	74	+	return
	75	+	case t := <-c.queue:
	76	+	c.planPush(t)
	77	+	}
	78	+	}
	79	+	}
	80	+
	81	+	// planPush plans one push; a panic or an error is logged and never escapes.
	82	+	func (c *CI) planPush(t ciTrigger) {
	83	+	ctx := zerowrap.WithCtx(c.ctx, t.log)
	84	+	defer func() {
	85	+	if r := recover(); r != nil {
	86	+	t.log.Error().Interface("panic", r).Bytes("stack", debug.Stack()).Str("repo", t.repo.String()).Msg("ci trigger panicked")
	87	+	}
	88	+	}()
	89	+	if err := c.handlePush(ctx, t); err != nil && ctx.Err() == nil {
	90	+	t.log.Error().Err(err).Str("repo", t.repo.String()).Msg("ci trigger failed")
	91	+	}
	92	+	}
	93	+
	94	+	// handlePush starts the workflows that match each ref of the push.
	95	+	func (c *CI) handlePush(ctx context.Context, t ciTrigger) error {
	96	+	tags := 0
	97	+	for _, u := range t.updates {
	98	+	if strings.HasPrefix(u.Ref, "refs/tags/") {
	99	+	tags++
	100	+	}
	101	+	}
	102	+	var errs []error
	103	+	for _, u := range t.updates {
	104	+	if err := c.handleRef(ctx, t, u, tags); err != nil {
	105	+	errs = append(errs, fmt.Errorf("%s: %w", u.Ref, err))
	106	+	}
	107	+	}
	108	+	return errors.Join(errs...)
	109	+	}
	110	+
	111	+	// pushFacts is what is known of one pushed ref.
	112	+	type pushFacts struct {
	113	+	update domain.RefUpdate
	114	+	commit domain.ObjectID // the commit the ref points to, tags followed
	115	+	head domain.Commit
	116	+	commits []domain.Commit // oldest first, at most maxPushCommits (branches only)
	117	+	forced bool
	118	+	created bool
	119	+	files []string
	120	+	// filesTruncated: the list of changed files is not available.
	121	+	filesTruncated bool
	122	+	tagCount int
	123	+	isTag bool
	124	+	}
	125	+
	126	+	func (c *CI) handleRef(ctx context.Context, t ciTrigger, u domain.RefUpdate, tagCount int) error {
	127	+	isTag := strings.HasPrefix(u.Ref, "refs/tags/")
	128	+	if isTag && tagCount > workflow.MaxTagsPerPush {
	129	+	return nil // GitHub creates no event for the tags of such a push
	130	+	}
	131	+	repo, err := c.cfg.Meta.Get(ctx, t.repo)
	132	+	if err != nil {
	133	+	return nil // deleted meanwhile
	134	+	}
	135	+	facts, err := c.collect(ctx, t.repo, u, isTag, tagCount)
	136	+	if err != nil {
	137	+	return err
	138	+	}
	139	+	files, err := c.workflowFiles(ctx, t.repo, facts.commit)
	140	+	if err != nil \|\| len(files) == 0 {
	141	+	return err
	142	+	}
	143	+	skip := workflow.SkipCI(facts.head.Message)
	144	+	var errs []error
	145	+	for _, f := range files {
	146	+	wf, perrs := workflow.Parse(f.path, f.content)
	147	+	switch {
	148	+	case f.problem != "":
	149	+	errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, f.problem))
	150	+	case len(perrs) > 0:
	151	+	var lines []string
	152	+	for _, e := range perrs {
	153	+	lines = append(lines, e.Error())
	154	+	}
	155	+	errs = append(errs, c.startInvalid(ctx, repo, t, facts, f.path, strings.Join(lines, "\n")))
	156	+	case skip \|\| !wf.MatchPush(workflow.PushInfo{
	157	+	Ref: u.Ref, Created: facts.created, ChangedFiles: facts.files, FilesTruncated: facts.filesTruncated, TagCount: tagCount,
	158	+	}):
	159	+	default:
	160	+	errs = append(errs, c.startRun(ctx, repo, t, facts, f.path, string(f.content), wf))
	161	+	}
	162	+	}
	163	+	return errors.Join(errs...)
	164	+	}
	165	+
	166	+	// collect reads what the payload and the filters need from Git.
	167	+	func (c *CI) collect(ctx context.Context, repo domain.RepoName, u domain.RefUpdate, isTag bool, tagCount int) (pushFacts, error) {
	168	+	newID, err := domain.ParseObjectID(u.New)
	169	+	if err != nil {
	170	+	return pushFacts{}, err
	171	+	}
	172	+	f := pushFacts{update: u, created: u.IsCreate(), isTag: isTag, tagCount: tagCount}
	173	+	res, err := c.cfg.Reader.ResolveRevision(ctx, repo, domain.RevisionOf(newID))
	174	+	if err != nil {
	175	+	return f, fmt.Errorf("resolve %s: %w", u.New, err)
	176	+	}
	177	+	f.commit = res.CommitID
	178	+	if isTag {
	179	+	cs, _, err := c.cfg.Git.ListRange(ctx, repo, domain.ObjectID{}, f.commit, 1)
	180	+	if err != nil \|\| len(cs) == 0 {
	181	+	return f, fmt.Errorf("read the commit of the tag: %w", err)
	182	+	}
	183	+	f.head = cs[0]
	184	+	return f, nil
	185	+	}
	186	+
	187	+	var exclude domain.ObjectID
	188	+	if !f.created {
	189	+	if old, err := domain.ParseObjectID(u.Old); err == nil {
	190	+	exclude = old
	191	+	if ok, err := c.cfg.Git.IsAncestor(ctx, repo, old, f.commit); err == nil && !ok {
	192	+	f.forced = true
	193	+	}
	194	+	}
	195	+	}
	196	+	base := exclude
	197	+	if f.created {
	198	+	base = c.defaultBase(ctx, repo, strings.TrimPrefix(u.Ref, "refs/heads/"), f.commit)
	199	+	exclude = base
	200	+	}
	201	+	cs, more, err := c.cfg.Git.ListRange(ctx, repo, exclude, f.commit, ports.MaxCommitLimit)
	202	+	if err != nil \|\| len(cs) == 0 {
	203	+	return f, fmt.Errorf("read the commits of the push: %w", err)
	204	+	}
	205	+	f.head = cs[0]
	206	+	n := min(len(cs), maxPushCommits)
	207	+	f.commits = slices.Clone(cs[:n])
	208	+	slices.Reverse(f.commits)
	209	+	if more \|\| base == f.commit {
	210	+	// More than 1,000 commits: the diff is not looked at (it runs).
	211	+	f.filesTruncated = more
	212	+	}
	213	+	if !more && base != f.commit {
	214	+	paths, _, err := c.cfg.Diff.ChangedPaths(ctx, repo, base, f.commit, workflow.MaxFilesEvaluated)
	215	+	if err != nil {
	216	+	f.filesTruncated = true // diff unavailable: runs
	217	+	} else {
	218	+	f.files = paths
	219	+	}
	220	+	}
	221	+	return f, nil
	222	+	}
	223	+
	224	+	// defaultBase is the merge base of a new branch with the default branch, the
	225	+	// zero id when there is none (every file of the branch counts).
	226	+	func (c *CI) defaultBase(ctx context.Context, repo domain.RepoName, branch string, head domain.ObjectID) domain.ObjectID {
	227	+	def, exists, err := c.cfg.Repos.DefaultBranch(ctx, repo)
	228	+	if err != nil \|\| !exists \|\| def == branch {
	229	+	return domain.ObjectID{}
	230	+	}
	231	+	tip, err := c.cfg.Git.BranchHead(ctx, repo, def)
	232	+	if err != nil {
	233	+	return domain.ObjectID{}
	234	+	}
	235	+	base, err := c.cfg.Git.MergeBase(ctx, repo, tip, head)
	236	+	if err != nil {
	237	+	return domain.ObjectID{}
	238	+	}
	239	+	return base
	240	+	}
	241	+
	242	+	type workflowFile struct {
	243	+	path string
	244	+	content []byte
	245	+	// problem is set when the file cannot even be parsed (too large).
	246	+	problem string
	247	+	}
	248	+
	249	+	// workflowFiles reads .github/workflows/.yml\|.yaml at the commit.
	250	+	func (c *CI) workflowFiles(ctx context.Context, repo domain.RepoName, commit domain.ObjectID) ([]workflowFile, error) {
	251	+	dir, err := domain.ParseTreePath(workflowDir)
	252	+	if err != nil {
	253	+	return nil, err
	254	+	}
	255	+	tree, err := c.cfg.Reader.GetTree(ctx, repo, domain.RevisionOf(commit), dir, false)
	256	+	if errors.Is(err, domain.ErrNotFound) \|\| errors.Is(err, domain.ErrNotDirectory) {
	257	+	return nil, nil
	258	+	}
	259	+	if err != nil {
	260	+	return nil, fmt.Errorf("list %s: %w", workflowDir, err)
	261	+	}
	262	+	var out []workflowFile
	263	+	for _, e := range tree.Entries {
	264	+	ext := strings.ToLower(path.Ext(e.Name))
	265	+	if (e.Kind != domain.EntryFile && e.Kind != domain.EntryExecutable) \|\| (ext != ".yml" && ext != ".yaml") {
	266	+	continue
	267	+	}
	268	+	if len(out) == maxCIWorkflowFiles {
	269	+	break
	270	+	}
	271	+	wf := workflowFile{path: e.Path}
	272	+	if wf.path == "" {
	273	+	wf.path = workflowDir + "/" + e.Name
	274	+	}
	275	+	if e.Size > maxCIWorkflowBytes {
	276	+	wf.problem = fmt.Sprintf("%s: the workflow file is larger than 1 MiB", wf.path)
	277	+	out = append(out, wf)
	278	+	continue
	279	+	}
	280	+	tp, err := domain.ParseTreePath(wf.path)
	281	+	if err != nil {
	282	+	continue
	283	+	}
	284	+	_, rc, err := c.cfg.Reader.OpenBlob(ctx, repo, domain.RevisionOf(commit), tp)
	285	+	if err != nil {
	286	+	return nil, fmt.Errorf("read %s: %w", wf.path, err)
	287	+	}
	288	+	wf.content, err = io.ReadAll(io.LimitReader(rc, maxCIWorkflowBytes+1))
	289	+	_ = rc.Close()
	290	+	if err != nil {
	291	+	return nil, fmt.Errorf("read %s: %w", wf.path, err)
	292	+	}
	293	+	if len(wf.content) > maxCIWorkflowBytes {
	294	+	wf.content, wf.problem = nil, fmt.Sprintf("%s: the workflow file is larger than 1 MiB", wf.path)
	295	+	}
	296	+	out = append(out, wf)
	297	+	}
	298	+	return out, nil
	299	+	}
	300	+
	301	+	// --- the run ---
	302	+
	303	+	func firstLine(s string) string {
	304	+	line, _, _ := strings.Cut(s, "\n")
	305	+	return truncateRunes(strings.TrimSpace(line), 256)
	306	+	}
	307	+
	308	+	// startInvalid records the failed run of an invalid workflow file: no job, the
	309	+	// errors as Run.error, like GitHub's "Invalid workflow file".
	310	+	func (c *CI) startInvalid(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath, problem string) error {
	311	+	now := c.now()
	312	+	draft, err := c.cfg.Store.CreateRun(ctx, repo.Name, wfPath, func(id, number int64) ports.CIRunDraft {
	313	+	run := domain.CIRun{
	314	+	Repo: repo.Name, ID: id, Number: number, Attempt: 1, WorkflowPath: wfPath, WorkflowName: wfPath,
	315	+	Title: firstLine(f.head.Message), Event: "push", Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor,
	316	+	Status: domain.CICompleted, Conclusion: domain.CIFailure, Error: cutText(problem, maxRunError),
	317	+	CreatedAt: now, StartedAt: now, CompletedAt: now,
	318	+	}
	319	+	return ports.CIRunDraft{
	320	+	Run: run, Source: domain.CIRunSource{Repo: repo.Name, WorkflowYAML: "", GithubJSON: "{}"},
	321	+	Checks: []domain.CICheck{{
	322	+	ID: "run-" + strconv.FormatInt(id, 10), Repo: repo.Name, Sha: run.Sha, Name: wfPath, Status: domain.CICompleted,
	323	+	Conclusion: domain.CIFailure, RunID: id, Attempt: 1, WorkflowName: wfPath, CreatedAt: now, StartedAt: now, CompletedAt: now,
	324	+	}},
	325	+	}
	326	+	})
	327	+	if errors.Is(err, domain.ErrNotFound) {
	328	+	return nil
	329	+	}
	330	+	if err != nil {
	331	+	return err
	332	+	}
	333	+	c.publishRun(ctx, repo, draft.Run)
	334	+	log := logRun(ctx, draft.Run)
	335	+	log.Info().Str("workflow", wfPath).Msg("ci run failed: invalid workflow file")
	336	+	return nil
	337	+	}
	338	+
	339	+	func (c *CI) publishRun(ctx context.Context, repo domain.Repo, run domain.CIRun) {
	340	+	c.cfg.Bus.Publish(ctx, domain.RunUpdated{Run: run, At: c.now(), Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	341	+	}
	342	+
	343	+	// startRun creates the run of a matching workflow with one waiting record per
	344	+	// job of the file, then plans the jobs that can start.
	345	+	func (c CI) startRun(ctx context.Context, repo domain.Repo, t ciTrigger, f pushFacts, wfPath, yaml string, wf workflow.Workflow) error {
	346	+	now := c.now()
	347	+	name := wf.Name
	348	+	if name == "" {
	349	+	name = wfPath
	350	+	}
	351	+	draft, err := c.cfg.Store.CreateRun(ctx, repo.Name, wfPath, func(id, number int64) ports.CIRunDraft {
	352	+	run := domain.CIRun{
	353	+	Repo: repo.Name, ID: id, Number: number, Attempt: 1, WorkflowPath: wfPath, WorkflowName: name,
	354	+	Title: firstLine(f.head.Message), Event: "push", Ref: f.update.Ref, Sha: f.commit.String(), Actor: t.actor,
	355	+	Status: domain.CIQueued, CreatedAt: now,
	356	+	}
	357	+	d := ports.CIRunDraft{Run: run, Source: domain.CIRunSource{
	358	+	Repo: repo.Name, WorkflowYAML: yaml, GithubJSON: c.githubJSON(repo, run, f, wf),
	359	+	}}
	360	+	for i, j := range wf.Jobs {
	361	+	job := domain.CIJob{
	362	+	ID: domain.NewJobID(id, i, 0), RunID: id, Attempt: 1, RunAttempt: 1, Repo: repo.Name, Key: j.ID, Name: j.ID,
	363	+	Phase: domain.CIPhaseWaiting, Needs: slices.Clone(j.Needs), Index: i, CreatedAt: now,
	364	+	}
	365	+	d.Jobs = append(d.Jobs, job)
	366	+	d.Checks = append(d.Checks, domain.CICheck{
	367	+	ID: job.ID, Repo: repo.Name, Sha: run.Sha, Name: job.Name, Status: domain.CIQueued, RunID: id, JobID: job.ID,
	368	+	Attempt: 1, WorkflowName: name, CreatedAt: now,
	369	+	})
	370	+	}
	371	+	return d
	372	+	})
	373	+	if errors.Is(err, domain.ErrNotFound) {
	374	+	return nil
	375	+	}
	376	+	if err != nil {
	377	+	return err
	378	+	}
	379	+	c.publishRun(ctx, repo, draft.Run)
	380	+	for _, j := range draft.Jobs {
	381	+	c.cfg.Bus.Publish(ctx, domain.JobUpdated{Job: j, At: now, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt})
	382	+	}
	383	+	log := logRun(ctx, draft.Run)
	384	+	log.Info().Str("workflow", wfPath).Str("ref", f.update.Ref).Int("jobs", len(draft.Jobs)).Msg("ci run created")
	385	+	return c.advanceRun(ctx, draft.Run.ID)
	386	+	}
	387	+
	388	+	// --- the github context ---
	389	+
	390	+	type obj = map[string]any
	391	+
	392	+	func person(name, email string) obj { return obj{"name": name, "email": email} }
	393	+
	394	+	func (c *CI) commitJSON(repo domain.Repo, cm domain.Commit) obj {
	395	+	ts := cm.Committer.Time.UTC().Format(time.RFC3339)
	396	+	return obj{
	397	+	"id": cm.ID.String(), "tree_id": "", "distinct": true, "message": cm.Message, "timestamp": ts,
	398	+	"url": c.cfg.PublicURL + "/" + repo.Name.String() + "/commit/" + cm.ID.String(),
	399	+	"author": obj{"name": cm.Author.Name, "email": cm.Author.Email, "username": ""},
	400	+	"committer": obj{"name": cm.Committer.Name, "email": cm.Committer.Email, "username": ""},
	401	+	"added": []string{}, "removed": []string{}, "modified": []string{},
	402	+	}
	403	+	}
	404	+
	405	+	// githubJSON builds the "github" context of the run as GitHub does for a push
	406	+	// (the job-dependent keys are added when a job is planned). Values that
	407	+	// GitHub's documentation does not settle are marked for the recordings.
	408	+	//
	409	+	// TODO(verify with recording): the exact types of run_id, run_number,
	410	+	// run_attempt (strings) and retention_days (a number), the empty tree_id and
	411	+	// the empty added/removed/modified lists of the push commits.
	412	+	func (c CI) githubJSON(repo domain.Repo, run domain.CIRun, f pushFacts, wf workflow.Workflow) string {
	413	+	full := c.cfg.Owner + "/" + repo.Name.String()
	414	+	refName, refType := strings.TrimPrefix(run.Ref, "refs/heads/"), "branch"
	415	+	if f.isTag {
	416	+	refName, refType = strings.TrimPrefix(run.Ref, "refs/tags/"), "tag"
	417	+	}
	418	+	repoURL := c.cfg.PublicURL + "/" + repo.Name.String()
	419	+	before := f.update.Old
	420	+	if f.created {
	421	+	before = domain.ZeroOID
	422	+	}
	423	+	compare := repoURL + "/compare/" + before + "..." + f.update.New
	424	+	if f.created {
	425	+	compare = repoURL + "/commit/" + f.update.New
	426	+	}
	427	+	commits := make([]any, 0, len(f.commits))
	428	+	for _, cm := range f.commits {
	429	+	commits = append(commits, c.commitJSON(repo, cm))
	430	+	}
	431	+	event := obj{
	432	+	"ref": run.Ref, "before": before, "after": f.update.New, "created": f.created, "deleted": false, "forced": f.forced,
	433	+	"base_ref": nil, "compare": compare, "commits": commits, "head_commit": c.commitJSON(repo, f.head),
	434	+	"pusher": person(run.Actor, ""),
	435	+	"repository": obj{
	436	+	"name": repo.Name.String(), "full_name": full, "private": repo.Visibility == domain.VisibilityPrivate,
	437	+	"owner": obj{"login": c.cfg.Owner, "name": c.cfg.Owner}, "html_url": repoURL, "clone_url": repoURL + ".git",
	438	+	"default_branch": "", "url": repoURL,
	439	+	},
	440	+	"sender": obj{"login": run.Actor, "type": "User"},
	441	+	}
	442	+	wfRef := full + "/" + run.WorkflowPath + "@" + run.Ref
	443	+	gh := obj{
	444	+	"event_name": "push", "event": event, "ref": run.Ref, "ref_name": refName, "ref_type": refType, "ref_protected": false,
	445	+	"sha": run.Sha, "repository": full, "repository_owner": c.cfg.Owner, "repositoryUrl": repoURL + ".git",
	446	+	"actor": run.Actor, "triggering_actor": run.Actor, "workflow": run.WorkflowName, "workflow_ref": wfRef,
	447	+	"workflow_sha": run.Sha, "run_id": strconv.FormatInt(run.ID, 10), "run_number": strconv.FormatInt(run.Number, 10),
	448	+	"server_url": c.cfg.PublicURL, "api_url": c.cfg.PublicURL + "/-/github/api/v3",
	449	+	"graphql_url": c.cfg.PublicURL + "/-/github/api/graphql", "retention_days": domain.CIRetentionDays,
	450	+	"head_ref": "", "base_ref": "",
	451	+	}
	452	+	b, err := json.Marshal(gh)
	453	+	if err != nil {
	454	+	return "{}"
	455	+	}
	456	+	return string(b)
	457	+	}

A internal/usecase/ci_watch.go +271 -0

@@ -0,0 +1,271 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"runtime/debug"
	9	+	"strconv"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+
	14	+	"bnema.dev/ermite/internal/domain"
	15	+	)
	16	+
	17	+	const sweepEvery = 24 * time.Hour
	18	+
	19	+	// advanceRun plans what can start in the latest attempt of a run.
	20	+	func (c *CI) advanceRun(ctx context.Context, runID int64) error {
	21	+	return c.withRun(ctx, runID, 0, func(rs *runState) error {
	22	+	if err := rs.advance(ctx); err != nil {
	23	+	return err
	24	+	}
	25	+	return rs.flush(ctx)
	26	+	})
	27	+	}
	28	+
	29	+	// resume plans again the runs that were open when the server stopped: jobs
	30	+	// whose needs ended while it was down are planned now. Running jobs go on:
	31	+	// their tokens are in the store, and the heartbeat deadline counts from the
	32	+	// start of this process.
	33	+	func (c *CI) resume() {
	34	+	ctx := c.ctx
	35	+	runs, err := c.cfg.Store.ListOpenRuns(ctx)
	36	+	if err != nil {
	37	+	ctxLog(ctx).Error().Err(err).Msg("ci: list the open runs")
	38	+	return
	39	+	}
	40	+	for _, r := range runs {
	41	+	if err := c.advanceRun(ctx, r.ID); err != nil && !errors.Is(err, domain.ErrNotFound) && ctx.Err() == nil {
	42	+	ctxLog(ctx).Error().Err(err).Int64("run_id", r.ID).Msg("ci: plan an open run again")
	43	+	}
	44	+	}
	45	+	c.wakeQueue()
	46	+	}
	47	+
	48	+	// watchdog looks at the jobs in flight every tick, and sweeps the old runs
	49	+	// once a day.
	50	+	func (c *CI) watchdog() {
	51	+	defer c.wg.Done()
	52	+	tick := time.NewTicker(c.cfg.Tick)
	53	+	defer tick.Stop()
	54	+	lastSweep := time.Time{}
	55	+	for {
	56	+	select {
	57	+	case <-c.ctx.Done():
	58	+	return
	59	+	case <-tick.C:
	60	+	}
	61	+	c.guard("watchdog", func() {
	62	+	c.checkActive(c.ctx)
	63	+	c.checkQueue(c.ctx)
	64	+	if now := c.now(); now.Sub(lastSweep) >= sweepEvery {
	65	+	lastSweep = now
	66	+	c.sweep(c.ctx)
	67	+	}
	68	+	})
	69	+	}
	70	+	}
	71	+
	72	+	// guard runs fn and logs a panic instead of letting it end the worker.
	73	+	func (c *CI) guard(what string, fn func()) {
	74	+	defer func() {
	75	+	if r := recover(); r != nil {
	76	+	ctxLog(c.ctx).Error().Interface("panic", r).Bytes("stack", debug.Stack()).Str("worker", what).Msg("ci worker panicked")
	77	+	}
	78	+	}()
	79	+	fn()
	80	+	}
	81	+
	82	+	// checkActive applies the server's deadlines to the jobs given to runners:
	83	+	// acknowledgement, heartbeat, time budget, and the grace after a cancel.
	84	+	func (c *CI) checkActive(ctx context.Context) {
	85	+	jobs, err := c.cfg.Store.ListActiveJobs(ctx)
	86	+	if err != nil {
	87	+	ctxLog(ctx).Error().Err(err).Msg("ci: list the active jobs")
	88	+	return
	89	+	}
	90	+	now := c.now()
	91	+	for _, j := range jobs {
	92	+	if !c.due(j, now) {
	93	+	continue
	94	+	}
	95	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	96	+	cur := rs.find(j.ID)
	97	+	if cur == nil \|\| !cur.Active() {
	98	+	return nil
	99	+	}
	100	+	if !c.enforce(ctx, rs, cur, c.now()) {
	101	+	return nil
	102	+	}
	103	+	if err := rs.advance(ctx); err != nil {
	104	+	return err
	105	+	}
	106	+	return rs.flush(ctx)
	107	+	})
	108	+	if err != nil && !errors.Is(err, domain.ErrNotFound) && ctx.Err() == nil {
	109	+	ctxLog(ctx).Error().Err(err).Str("job_id", j.ID).Msg("ci: enforce a deadline")
	110	+	}
	111	+	}
	112	+	}
	113	+
	114	+	// lastBeat is the time of the last sign of life of an active job: the later
	115	+	// of its last heartbeat, its acknowledgement and the start of the server.
	116	+	func (c *CI) lastBeat(j domain.CIJob) time.Time {
	117	+	c.mu.Lock()
	118	+	beat := c.heartbeats[jobRef(j)]
	119	+	c.mu.Unlock()
	120	+	for _, t := range []time.Time{j.AckedAt, c.boot} {
	121	+	if t.After(beat) {
	122	+	beat = t
	123	+	}
	124	+	}
	125	+	return beat
	126	+	}
	127	+
	128	+	// due is a cheap test, on the listed copy, of whether a deadline may have
	129	+	// passed.
	130	+	func (c *CI) due(j domain.CIJob, now time.Time) bool {
	131	+	switch {
	132	+	case j.Phase == domain.CIPhaseAssigned:
	133	+	return now.Sub(j.AssignedAt) > c.cfg.AckTimeout
	134	+	case j.CancelReason != domain.CICancelNone:
	135	+	return now.Sub(j.CancelRequestedAt) > c.cfg.CancelGrace
	136	+	}
	137	+	return now.Sub(c.lastBeat(j)) > c.cfg.HeartbeatDeadline \|\| now.After(c.deadline(j))
	138	+	}
	139	+
	140	+	// deadline is when the job's time budget ends for the server: timeout-minutes
	141	+	// from the assignment, plus the slack that lets the runner's own timeout win.
	142	+	func (c *CI) deadline(j domain.CIJob) time.Time {
	143	+	return j.AssignedAt.Add(time.Duration(j.TimeoutMinutes) * time.Minute).Add(c.cfg.TimeoutSlack)
	144	+	}
	145	+
	146	+	// enforce applies the deadlines to a job read under the lock of its run and
	147	+	// reports whether it changed.
	148	+	func (c CI) enforce(ctx context.Context, rs runState, j *domain.CIJob, now time.Time) bool {
	149	+	switch {
	150	+	case j.Phase == domain.CIPhaseAssigned && now.Sub(j.AssignedAt) > c.cfg.AckTimeout:
	151	+	c.cancelAssignment(ctx, rs, j)
	152	+	return true
	153	+	case j.CancelReason != domain.CICancelNone && now.Sub(j.CancelRequestedAt) > c.cfg.CancelGrace:
	154	+	conclusion := domain.CICancelled
	155	+	if j.CancelReason == domain.CICancelTimeout {
	156	+	conclusion = domain.CITimedOut
	157	+	}
	158	+	rs.end(j, conclusion, "The runner "+j.RunnerName+" did not answer the cancellation request in time; the job was ended by the server.")
	159	+	return true
	160	+	case j.Phase == domain.CIPhaseRunning && now.Sub(c.lastBeat(*j)) > c.cfg.HeartbeatDeadline:
	161	+	rs.end(j, domain.CIFailure, domain.CILostMessage(j.RunnerName))
	162	+	return true
	163	+	case j.CancelReason == domain.CICancelNone && now.After(c.deadline(*j)):
	164	+	rs.requestCancel(j, domain.CICancelTimeout, "The job exceeded its timeout of "+itoa(int64(j.TimeoutMinutes))+" minutes")
	165	+	return true
	166	+	}
	167	+	return false
	168	+	}
	169	+
	170	+	func itoa(n int64) string { return strconv.FormatInt(n, 10) }
	171	+
	172	+	// cancelAssignment takes a job back from a runner that did not acknowledge
	173	+	// it: the token is revoked, what the assignment stored is dropped, and the job
	174	+	// is queued again with a fresh token at its next assignment. Three in a row
	175	+	// fail the job.
	176	+	func (c CI) cancelAssignment(ctx context.Context, rs runState, j *domain.CIJob) {
	177	+	ctxLog(ctx).Warn().Str("job_id", j.ID).Str("runner", j.RunnerName).Msg("job assignment not acknowledged: cancelled")
	178	+	_ = c.cfg.Logs.Reset(ctx, logRef(*j))
	179	+	j.AssignCancels++
	180	+	j.TokenHash, j.TokenValidUntil = domain.TokenHash{}, time.Time{}
	181	+	j.RunnerID, j.RunnerName, j.Isolation = "", "", domain.CIIsolationNone
	182	+	j.Steps, j.Outputs, j.AssignedAt = nil, nil, time.Time{}
	183	+	c.mu.Lock()
	184	+	for id, p := range c.polls {
	185	+	if p.ref == jobRef(*j) {
	186	+	delete(c.polls, id)
	187	+	}
	188	+	}
	189	+	c.mu.Unlock()
	190	+	if j.AssignCancels >= domain.CIMaxAssignCancels {
	191	+	rs.end(j, domain.CIFailure, "The job was given to a runner "+itoa(int64(domain.CIMaxAssignCancels))+" times and none of them acknowledged it.")
	192	+	return
	193	+	}
	194	+	j.Phase, j.ReadyAt = domain.CIPhaseReady, c.now()
	195	+	rs.touch(j)
	196	+	}
	197	+
	198	+	// checkQueue fails the jobs that no runner took within the queue timeout.
	199	+	func (c *CI) checkQueue(ctx context.Context) {
	200	+	ready, err := c.cfg.Store.ListReadyJobs(ctx, 0)
	201	+	if err != nil {
	202	+	return
	203	+	}
	204	+	now := c.now()
	205	+	for _, j := range ready {
	206	+	if now.Sub(j.ReadyAt) <= c.cfg.QueueTimeout {
	207	+	continue
	208	+	}
	209	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	210	+	cur := rs.find(j.ID)
	211	+	if cur == nil \|\| cur.Phase != domain.CIPhaseReady \|\| c.now().Sub(cur.ReadyAt) <= c.cfg.QueueTimeout {
	212	+	return nil
	213	+	}
	214	+	rs.end(cur, domain.CIFailure, domain.CIQueueTimeoutMessage)
	215	+	if err := rs.advance(ctx); err != nil {
	216	+	return err
	217	+	}
	218	+	return rs.flush(ctx)
	219	+	})
	220	+	if err != nil && !errors.Is(err, domain.ErrNotFound) && ctx.Err() == nil {
	221	+	ctxLog(ctx).Error().Err(err).Str("job_id", j.ID).Msg("ci: end a job nobody took")
	222	+	}
	223	+	}
	224	+	}
	225	+
	226	+	// sweep removes the runs older than the retention, and the logs of runs that
	227	+	// no longer exist (a repository deleted while the server was down).
	228	+	func (c *CI) sweep(ctx context.Context) {
	229	+	log := zerowrap.FromCtx(ctx)
	230	+	ids, err := c.cfg.Store.Sweep(ctx, c.now().Add(-c.cfg.Retention))
	231	+	if err != nil {
	232	+	log.Error().Err(err).Msg("ci: sweep the old runs")
	233	+	return
	234	+	}
	235	+	if len(ids) > 0 {
	236	+	if err := c.cfg.Logs.RemoveRuns(ctx, ids); err != nil {
	237	+	log.Error().Err(err).Msg("ci: remove the logs of the old runs")
	238	+	}
	239	+	for _, id := range ids {
	240	+	c.locks.Delete(id)
	241	+	}
	242	+	log.Info().Int("runs", len(ids)).Msg("ci: old runs removed")
	243	+	}
	244	+	onDisk, err := c.cfg.Logs.RunIDs(ctx)
	245	+	if err != nil {
	246	+	return
	247	+	}
	248	+	var orphans []int64
	249	+	for _, id := range onDisk {
	250	+	if _, err := c.cfg.Store.GetRunSource(ctx, id); errors.Is(err, domain.ErrNotFound) {
	251	+	orphans = append(orphans, id)
	252	+	}
	253	+	}
	254	+	if len(orphans) > 0 {
	255	+	_ = c.cfg.Logs.RemoveRuns(ctx, orphans)
	256	+	}
	257	+	}
	258	+
	259	+	// RunIDs lists the runs of a repository, for the cleanup that follows its
	260	+	// deletion.
	261	+	func (c *CI) RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error) {
	262	+	return c.cfg.Store.RunIDs(ctx, repo)
	263	+	}
	264	+
	265	+	// RemoveLogs deletes the logs of runs of a repository that was deleted.
	266	+	func (c *CI) RemoveLogs(ctx context.Context, runIDs []int64) error {
	267	+	for _, id := range runIDs {
	268	+	c.locks.Delete(id)
	269	+	}
	270	+	return c.cfg.Logs.RemoveRuns(ctx, runIDs)
	271	+	}

@@ -0,0 +1,271 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"context"
	7	+	"errors"
	8	+	"runtime/debug"
	9	+	"strconv"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+
	14	+	"bnema.dev/ermite/internal/domain"
	15	+	)
	16	+
	17	+	const sweepEvery = 24 * time.Hour
	18	+
	19	+	// advanceRun plans what can start in the latest attempt of a run.
	20	+	func (c *CI) advanceRun(ctx context.Context, runID int64) error {
	21	+	return c.withRun(ctx, runID, 0, func(rs *runState) error {
	22	+	if err := rs.advance(ctx); err != nil {
	23	+	return err
	24	+	}
	25	+	return rs.flush(ctx)
	26	+	})
	27	+	}
	28	+
	29	+	// resume plans again the runs that were open when the server stopped: jobs
	30	+	// whose needs ended while it was down are planned now. Running jobs go on:
	31	+	// their tokens are in the store, and the heartbeat deadline counts from the
	32	+	// start of this process.
	33	+	func (c *CI) resume() {
	34	+	ctx := c.ctx
	35	+	runs, err := c.cfg.Store.ListOpenRuns(ctx)
	36	+	if err != nil {
	37	+	ctxLog(ctx).Error().Err(err).Msg("ci: list the open runs")
	38	+	return
	39	+	}
	40	+	for _, r := range runs {
	41	+	if err := c.advanceRun(ctx, r.ID); err != nil && !errors.Is(err, domain.ErrNotFound) && ctx.Err() == nil {
	42	+	ctxLog(ctx).Error().Err(err).Int64("run_id", r.ID).Msg("ci: plan an open run again")
	43	+	}
	44	+	}
	45	+	c.wakeQueue()
	46	+	}
	47	+
	48	+	// watchdog looks at the jobs in flight every tick, and sweeps the old runs
	49	+	// once a day.
	50	+	func (c *CI) watchdog() {
	51	+	defer c.wg.Done()
	52	+	tick := time.NewTicker(c.cfg.Tick)
	53	+	defer tick.Stop()
	54	+	lastSweep := time.Time{}
	55	+	for {
	56	+	select {
	57	+	case <-c.ctx.Done():
	58	+	return
	59	+	case <-tick.C:
	60	+	}
	61	+	c.guard("watchdog", func() {
	62	+	c.checkActive(c.ctx)
	63	+	c.checkQueue(c.ctx)
	64	+	if now := c.now(); now.Sub(lastSweep) >= sweepEvery {
	65	+	lastSweep = now
	66	+	c.sweep(c.ctx)
	67	+	}
	68	+	})
	69	+	}
	70	+	}
	71	+
	72	+	// guard runs fn and logs a panic instead of letting it end the worker.
	73	+	func (c *CI) guard(what string, fn func()) {
	74	+	defer func() {
	75	+	if r := recover(); r != nil {
	76	+	ctxLog(c.ctx).Error().Interface("panic", r).Bytes("stack", debug.Stack()).Str("worker", what).Msg("ci worker panicked")
	77	+	}
	78	+	}()
	79	+	fn()
	80	+	}
	81	+
	82	+	// checkActive applies the server's deadlines to the jobs given to runners:
	83	+	// acknowledgement, heartbeat, time budget, and the grace after a cancel.
	84	+	func (c *CI) checkActive(ctx context.Context) {
	85	+	jobs, err := c.cfg.Store.ListActiveJobs(ctx)
	86	+	if err != nil {
	87	+	ctxLog(ctx).Error().Err(err).Msg("ci: list the active jobs")
	88	+	return
	89	+	}
	90	+	now := c.now()
	91	+	for _, j := range jobs {
	92	+	if !c.due(j, now) {
	93	+	continue
	94	+	}
	95	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	96	+	cur := rs.find(j.ID)
	97	+	if cur == nil \|\| !cur.Active() {
	98	+	return nil
	99	+	}
	100	+	if !c.enforce(ctx, rs, cur, c.now()) {
	101	+	return nil
	102	+	}
	103	+	if err := rs.advance(ctx); err != nil {
	104	+	return err
	105	+	}
	106	+	return rs.flush(ctx)
	107	+	})
	108	+	if err != nil && !errors.Is(err, domain.ErrNotFound) && ctx.Err() == nil {
	109	+	ctxLog(ctx).Error().Err(err).Str("job_id", j.ID).Msg("ci: enforce a deadline")
	110	+	}
	111	+	}
	112	+	}
	113	+
	114	+	// lastBeat is the time of the last sign of life of an active job: the later
	115	+	// of its last heartbeat, its acknowledgement and the start of the server.
	116	+	func (c *CI) lastBeat(j domain.CIJob) time.Time {
	117	+	c.mu.Lock()
	118	+	beat := c.heartbeats[jobRef(j)]
	119	+	c.mu.Unlock()
	120	+	for _, t := range []time.Time{j.AckedAt, c.boot} {
	121	+	if t.After(beat) {
	122	+	beat = t
	123	+	}
	124	+	}
	125	+	return beat
	126	+	}
	127	+
	128	+	// due is a cheap test, on the listed copy, of whether a deadline may have
	129	+	// passed.
	130	+	func (c *CI) due(j domain.CIJob, now time.Time) bool {
	131	+	switch {
	132	+	case j.Phase == domain.CIPhaseAssigned:
	133	+	return now.Sub(j.AssignedAt) > c.cfg.AckTimeout
	134	+	case j.CancelReason != domain.CICancelNone:
	135	+	return now.Sub(j.CancelRequestedAt) > c.cfg.CancelGrace
	136	+	}
	137	+	return now.Sub(c.lastBeat(j)) > c.cfg.HeartbeatDeadline \|\| now.After(c.deadline(j))
	138	+	}
	139	+
	140	+	// deadline is when the job's time budget ends for the server: timeout-minutes
	141	+	// from the assignment, plus the slack that lets the runner's own timeout win.
	142	+	func (c *CI) deadline(j domain.CIJob) time.Time {
	143	+	return j.AssignedAt.Add(time.Duration(j.TimeoutMinutes) * time.Minute).Add(c.cfg.TimeoutSlack)
	144	+	}
	145	+
	146	+	// enforce applies the deadlines to a job read under the lock of its run and
	147	+	// reports whether it changed.
	148	+	func (c CI) enforce(ctx context.Context, rs runState, j *domain.CIJob, now time.Time) bool {
	149	+	switch {
	150	+	case j.Phase == domain.CIPhaseAssigned && now.Sub(j.AssignedAt) > c.cfg.AckTimeout:
	151	+	c.cancelAssignment(ctx, rs, j)
	152	+	return true
	153	+	case j.CancelReason != domain.CICancelNone && now.Sub(j.CancelRequestedAt) > c.cfg.CancelGrace:
	154	+	conclusion := domain.CICancelled
	155	+	if j.CancelReason == domain.CICancelTimeout {
	156	+	conclusion = domain.CITimedOut
	157	+	}
	158	+	rs.end(j, conclusion, "The runner "+j.RunnerName+" did not answer the cancellation request in time; the job was ended by the server.")
	159	+	return true
	160	+	case j.Phase == domain.CIPhaseRunning && now.Sub(c.lastBeat(*j)) > c.cfg.HeartbeatDeadline:
	161	+	rs.end(j, domain.CIFailure, domain.CILostMessage(j.RunnerName))
	162	+	return true
	163	+	case j.CancelReason == domain.CICancelNone && now.After(c.deadline(*j)):
	164	+	rs.requestCancel(j, domain.CICancelTimeout, "The job exceeded its timeout of "+itoa(int64(j.TimeoutMinutes))+" minutes")
	165	+	return true
	166	+	}
	167	+	return false
	168	+	}
	169	+
	170	+	func itoa(n int64) string { return strconv.FormatInt(n, 10) }
	171	+
	172	+	// cancelAssignment takes a job back from a runner that did not acknowledge
	173	+	// it: the token is revoked, what the assignment stored is dropped, and the job
	174	+	// is queued again with a fresh token at its next assignment. Three in a row
	175	+	// fail the job.
	176	+	func (c CI) cancelAssignment(ctx context.Context, rs runState, j *domain.CIJob) {
	177	+	ctxLog(ctx).Warn().Str("job_id", j.ID).Str("runner", j.RunnerName).Msg("job assignment not acknowledged: cancelled")
	178	+	_ = c.cfg.Logs.Reset(ctx, logRef(*j))
	179	+	j.AssignCancels++
	180	+	j.TokenHash, j.TokenValidUntil = domain.TokenHash{}, time.Time{}
	181	+	j.RunnerID, j.RunnerName, j.Isolation = "", "", domain.CIIsolationNone
	182	+	j.Steps, j.Outputs, j.AssignedAt = nil, nil, time.Time{}
	183	+	c.mu.Lock()
	184	+	for id, p := range c.polls {
	185	+	if p.ref == jobRef(*j) {
	186	+	delete(c.polls, id)
	187	+	}
	188	+	}
	189	+	c.mu.Unlock()
	190	+	if j.AssignCancels >= domain.CIMaxAssignCancels {
	191	+	rs.end(j, domain.CIFailure, "The job was given to a runner "+itoa(int64(domain.CIMaxAssignCancels))+" times and none of them acknowledged it.")
	192	+	return
	193	+	}
	194	+	j.Phase, j.ReadyAt = domain.CIPhaseReady, c.now()
	195	+	rs.touch(j)
	196	+	}
	197	+
	198	+	// checkQueue fails the jobs that no runner took within the queue timeout.
	199	+	func (c *CI) checkQueue(ctx context.Context) {
	200	+	ready, err := c.cfg.Store.ListReadyJobs(ctx, 0)
	201	+	if err != nil {
	202	+	return
	203	+	}
	204	+	now := c.now()
	205	+	for _, j := range ready {
	206	+	if now.Sub(j.ReadyAt) <= c.cfg.QueueTimeout {
	207	+	continue
	208	+	}
	209	+	err := c.withRun(ctx, j.RunID, j.RunAttempt, func(rs *runState) error {
	210	+	cur := rs.find(j.ID)
	211	+	if cur == nil \|\| cur.Phase != domain.CIPhaseReady \|\| c.now().Sub(cur.ReadyAt) <= c.cfg.QueueTimeout {
	212	+	return nil
	213	+	}
	214	+	rs.end(cur, domain.CIFailure, domain.CIQueueTimeoutMessage)
	215	+	if err := rs.advance(ctx); err != nil {
	216	+	return err
	217	+	}
	218	+	return rs.flush(ctx)
	219	+	})
	220	+	if err != nil && !errors.Is(err, domain.ErrNotFound) && ctx.Err() == nil {
	221	+	ctxLog(ctx).Error().Err(err).Str("job_id", j.ID).Msg("ci: end a job nobody took")
	222	+	}
	223	+	}
	224	+	}
	225	+
	226	+	// sweep removes the runs older than the retention, and the logs of runs that
	227	+	// no longer exist (a repository deleted while the server was down).
	228	+	func (c *CI) sweep(ctx context.Context) {
	229	+	log := zerowrap.FromCtx(ctx)
	230	+	ids, err := c.cfg.Store.Sweep(ctx, c.now().Add(-c.cfg.Retention))
	231	+	if err != nil {
	232	+	log.Error().Err(err).Msg("ci: sweep the old runs")
	233	+	return
	234	+	}
	235	+	if len(ids) > 0 {
	236	+	if err := c.cfg.Logs.RemoveRuns(ctx, ids); err != nil {
	237	+	log.Error().Err(err).Msg("ci: remove the logs of the old runs")
	238	+	}
	239	+	for _, id := range ids {
	240	+	c.locks.Delete(id)
	241	+	}
	242	+	log.Info().Int("runs", len(ids)).Msg("ci: old runs removed")
	243	+	}
	244	+	onDisk, err := c.cfg.Logs.RunIDs(ctx)
	245	+	if err != nil {
	246	+	return
	247	+	}
	248	+	var orphans []int64
	249	+	for _, id := range onDisk {
	250	+	if _, err := c.cfg.Store.GetRunSource(ctx, id); errors.Is(err, domain.ErrNotFound) {
	251	+	orphans = append(orphans, id)
	252	+	}
	253	+	}
	254	+	if len(orphans) > 0 {
	255	+	_ = c.cfg.Logs.RemoveRuns(ctx, orphans)
	256	+	}
	257	+	}
	258	+
	259	+	// RunIDs lists the runs of a repository, for the cleanup that follows its
	260	+	// deletion.
	261	+	func (c *CI) RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error) {
	262	+	return c.cfg.Store.RunIDs(ctx, repo)
	263	+	}
	264	+
	265	+	// RemoveLogs deletes the logs of runs of a repository that was deleted.
	266	+	func (c *CI) RemoveLogs(ctx context.Context, runIDs []int64) error {
	267	+	for _, id := range runIDs {
	268	+	c.locks.Delete(id)
	269	+	}
	270	+	return c.cfg.Logs.RemoveRuns(ctx, runIDs)
	271	+	}

M internal/usecase/mergerequests.go +48 -1


@@ -8,6 +8,7 @@ import (
8	8		"errors"
9	9		"fmt"
10	10		"strconv"
	11	+	"strings"
11	12		"sync"
12	13
13	14		"github.com/bnema/zerowrap"

@@ -48,6 +49,8 @@ type MergeRequests struct {
48	49		bus ports.EventBus
49	50		clock ports.Clock
50	51		maint ports.MaintenanceScheduler
	52	+	ci ports.WorkflowTrigger // nil: no CI
	53	+	checks ports.CIChecksReader // nil: no CI, so no check blocks a merge
51	54
52	55		mu sync.Mutex
53	56		locks map[domain.RepoName]chan struct{}

@@ -71,13 +74,18 @@ type MergeRequestsConfig struct {
71	74		Clock ports.Clock
72	75		// Maint queues the maintenance that follows a merge, as it does a push.
73	76		Maint ports.MaintenanceScheduler
	77	+	// CI starts the workflows a merge triggers (GitHub runs "push" workflows
	78	+	// on a merge) and Checks gives the checks the merge rule reads. Both are
	79	+	// optional: without them there is no CI.
	80	+	CI ports.WorkflowTrigger
	81	+	Checks ports.CIChecksReader
74	82		}
75	83
76	84		// NewMergeRequests builds the merge request use cases.
77	85		func NewMergeRequests(cfg MergeRequestsConfig) *MergeRequests {
78	86		return &MergeRequests{
79	87		meta: cfg.Meta, store: cfg.Store, repos: cfg.Repos, git: cfg.Git, signer: cfg.Signer,
80		-	committer: cfg.Committer, bus: cfg.Bus, clock: cfg.Clock, maint: cfg.Maint,
	88	+	committer: cfg.Committer, bus: cfg.Bus, clock: cfg.Clock, maint: cfg.Maint, ci: cfg.CI, checks: cfg.Checks,
81	89		locks: make(map[domain.RepoName]chan struct{}),
82	90		}
83	91		}

@@ -753,6 +761,11 @@ func (m *MergeRequests) merge(ctx context.Context, p domain.Principal, name doma
753	761		case !ok:
754	762		return domain.MergeRequest{}, false, fmt.Errorf("%w: %q is not an ancestor of %q: rebase it on %q and push again", domain.ErrPreconditionFailed, mr.Target, mr.Source, mr.Target)
755	763		}
	764	+	if settings.FailingChecksBlock {
	765	+	if err := m.checksAllow(ctx, name, head); err != nil {
	766	+	return domain.MergeRequest{}, false, err
	767	+	}
	768	+	}
756	769		if params.DeleteSource {
757	770		if err := m.checkDeletable(ctx, name, mr); err != nil {
758	771		return domain.MergeRequest{}, false, err

@@ -792,9 +805,43 @@ func (m *MergeRequests) merge(ctx context.Context, p domain.Principal, name doma
792	805		m.bus.Publish(ctx, domain.PushEvent{
793	806		Repo: name, At: at, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt, Updates: updates,
794	807		})
	808	+	if m.ci != nil {
	809	+	// GitHub runs the "push" workflows of a merge, with the owner as pusher.
	810	+	m.ci.OnPush(ctx, name, updates[:1], domain.Owner())
	811	+	}
795	812		return mr, deleted, nil
796	813		}
797	814
	815	+	// checksAllow is the merge rule of FailingChecksBlock: every check of the
	816	+	// head must have ended as success, skipped or neutral. A check still queued or
	817	+	// running blocks too (the merge could land code that then fails), and a head
	818	+	// without any check merges.
	819	+	func (m *MergeRequests) checksAllow(ctx context.Context, name domain.RepoName, head domain.ObjectID) error {
	820	+	if m.checks == nil {
	821	+	return nil
	822	+	}
	823	+	checks, err := m.checks.CommitChecks(ctx, name, head.String())
	824	+	if err != nil {
	825	+	return fmt.Errorf("read the checks of %s: %w", head, err)
	826	+	}
	827	+	var failing, pending []string
	828	+	for _, c := range checks {
	829	+	switch {
	830	+	case c.Status != domain.CICompleted:
	831	+	pending = append(pending, c.Name)
	832	+	case !c.Conclusion.PassesMerge():
	833	+	failing = append(failing, c.Name)
	834	+	}
	835	+	}
	836	+	switch {
	837	+	case len(failing) > 0:
	838	+	return fmt.Errorf("%w: failing checks block the merge: %s (turn off failing_checks_block to merge anyway)", domain.ErrPreconditionFailed, strings.Join(failing, ", "))
	839	+	case len(pending) > 0:
	840	+	return fmt.Errorf("%w: checks have not finished: %s", domain.ErrPreconditionFailed, strings.Join(pending, ", "))
	841	+	}
	842	+	return nil
	843	+	}
	844	+
798	845		// checkDeletable refuses to delete the source branch of mr when it is the
799	846		// default branch or the source or target of another open request. It runs
800	847		// under the merge lock, which Create also holds while it checks its branches


@@ -8,6 +8,7 @@ import (
8		"errors"	8		"errors"
9		"fmt"	9		"fmt"
10		"strconv"	10		"strconv"
			11	+	"strings"
11		"sync"	12		"sync"
12			13
13		"github.com/bnema/zerowrap"	14		"github.com/bnema/zerowrap"

@@ -48,6 +49,8 @@ type MergeRequests struct {
48		bus ports.EventBus	49		bus ports.EventBus
49		clock ports.Clock	50		clock ports.Clock
50		maint ports.MaintenanceScheduler	51		maint ports.MaintenanceScheduler
			52	+	ci ports.WorkflowTrigger // nil: no CI
			53	+	checks ports.CIChecksReader // nil: no CI, so no check blocks a merge
51			54
52		mu sync.Mutex	55		mu sync.Mutex
53		locks map[domain.RepoName]chan struct{}	56		locks map[domain.RepoName]chan struct{}

@@ -71,13 +74,18 @@ type MergeRequestsConfig struct {
71		Clock ports.Clock	74		Clock ports.Clock
72		// Maint queues the maintenance that follows a merge, as it does a push.	75		// Maint queues the maintenance that follows a merge, as it does a push.
73		Maint ports.MaintenanceScheduler	76		Maint ports.MaintenanceScheduler
			77	+	// CI starts the workflows a merge triggers (GitHub runs "push" workflows
			78	+	// on a merge) and Checks gives the checks the merge rule reads. Both are
			79	+	// optional: without them there is no CI.
			80	+	CI ports.WorkflowTrigger
			81	+	Checks ports.CIChecksReader
74		}	82		}
75			83
76		// NewMergeRequests builds the merge request use cases.	84		// NewMergeRequests builds the merge request use cases.
77		func NewMergeRequests(cfg MergeRequestsConfig) *MergeRequests {	85		func NewMergeRequests(cfg MergeRequestsConfig) *MergeRequests {
78		return &MergeRequests{	86		return &MergeRequests{
79		meta: cfg.Meta, store: cfg.Store, repos: cfg.Repos, git: cfg.Git, signer: cfg.Signer,	87		meta: cfg.Meta, store: cfg.Store, repos: cfg.Repos, git: cfg.Git, signer: cfg.Signer,
80	-	committer: cfg.Committer, bus: cfg.Bus, clock: cfg.Clock, maint: cfg.Maint,	88	+	committer: cfg.Committer, bus: cfg.Bus, clock: cfg.Clock, maint: cfg.Maint, ci: cfg.CI, checks: cfg.Checks,
81		locks: make(map[domain.RepoName]chan struct{}),	89		locks: make(map[domain.RepoName]chan struct{}),
82		}	90		}
83		}	91		}

@@ -753,6 +761,11 @@ func (m *MergeRequests) merge(ctx context.Context, p domain.Principal, name doma
753		case !ok:	761		case !ok:
754		return domain.MergeRequest{}, false, fmt.Errorf("%w: %q is not an ancestor of %q: rebase it on %q and push again", domain.ErrPreconditionFailed, mr.Target, mr.Source, mr.Target)	762		return domain.MergeRequest{}, false, fmt.Errorf("%w: %q is not an ancestor of %q: rebase it on %q and push again", domain.ErrPreconditionFailed, mr.Target, mr.Source, mr.Target)
755		}	763		}
			764	+	if settings.FailingChecksBlock {
			765	+	if err := m.checksAllow(ctx, name, head); err != nil {
			766	+	return domain.MergeRequest{}, false, err
			767	+	}
			768	+	}
756		if params.DeleteSource {	769		if params.DeleteSource {
757		if err := m.checkDeletable(ctx, name, mr); err != nil {	770		if err := m.checkDeletable(ctx, name, mr); err != nil {
758		return domain.MergeRequest{}, false, err	771		return domain.MergeRequest{}, false, err

@@ -792,9 +805,43 @@ func (m *MergeRequests) merge(ctx context.Context, p domain.Principal, name doma
792		m.bus.Publish(ctx, domain.PushEvent{	805		m.bus.Publish(ctx, domain.PushEvent{
793		Repo: name, At: at, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt, Updates: updates,	806		Repo: name, At: at, Visibility: repo.Visibility, CreatedAt: repo.CreatedAt, Updates: updates,
794		})	807		})
			808	+	if m.ci != nil {
			809	+	// GitHub runs the "push" workflows of a merge, with the owner as pusher.
			810	+	m.ci.OnPush(ctx, name, updates[:1], domain.Owner())
			811	+	}
795		return mr, deleted, nil	812		return mr, deleted, nil
796		}	813		}
797			814
			815	+	// checksAllow is the merge rule of FailingChecksBlock: every check of the
			816	+	// head must have ended as success, skipped or neutral. A check still queued or
			817	+	// running blocks too (the merge could land code that then fails), and a head
			818	+	// without any check merges.
			819	+	func (m *MergeRequests) checksAllow(ctx context.Context, name domain.RepoName, head domain.ObjectID) error {
			820	+	if m.checks == nil {
			821	+	return nil
			822	+	}
			823	+	checks, err := m.checks.CommitChecks(ctx, name, head.String())
			824	+	if err != nil {
			825	+	return fmt.Errorf("read the checks of %s: %w", head, err)
			826	+	}
			827	+	var failing, pending []string
			828	+	for _, c := range checks {
			829	+	switch {
			830	+	case c.Status != domain.CICompleted:
			831	+	pending = append(pending, c.Name)
			832	+	case !c.Conclusion.PassesMerge():
			833	+	failing = append(failing, c.Name)
			834	+	}
			835	+	}
			836	+	switch {
			837	+	case len(failing) > 0:
			838	+	return fmt.Errorf("%w: failing checks block the merge: %s (turn off failing_checks_block to merge anyway)", domain.ErrPreconditionFailed, strings.Join(failing, ", "))
			839	+	case len(pending) > 0:
			840	+	return fmt.Errorf("%w: checks have not finished: %s", domain.ErrPreconditionFailed, strings.Join(pending, ", "))
			841	+	}
			842	+	return nil
			843	+	}
			844	+
798		// checkDeletable refuses to delete the source branch of mr when it is the	845		// checkDeletable refuses to delete the source branch of mr when it is the
799		// default branch or the source or target of another open request. It runs	846		// default branch or the source or target of another open request. It runs
800		// under the merge lock, which Create also holds while it checks its branches	847		// under the merge lock, which Create also holds while it checks its branches

M internal/usecase/pushes.go +14 -2


@@ -27,7 +27,8 @@ type Pushes struct {
27	27		clock ports.Clock
28	28		maint ports.MaintenanceScheduler
29	29		auth ports.Revalidator
30		-	fresh ports.RepoFreshener // nil: the reader needs no telling
	30	+	fresh ports.RepoFreshener // nil: the reader needs no telling
	31	+	ci ports.WorkflowTrigger // nil: no CI
31	32		}
32	33
33	34		var (

@@ -49,6 +50,13 @@ func (u Pushes) WithFreshener(f ports.RepoFreshener) Pushes {
49	50		return u
50	51		}
51	52
	53	+	// WithWorkflowTrigger makes Record start the workflows that match the push.
	54	+	// Call it before the first push; it returns u.
	55	+	func (u Pushes) WithWorkflowTrigger(t ports.WorkflowTrigger) Pushes {
	56	+	u.ci = t
	57	+	return u
	58	+	}
	59	+
52	60		// Open decides whether the principal may push to the repository and, with
53	61		// create, creates it when it does not exist yet (private, published as
54	62		// RepoCreated by Repos.Create). create is for the first request of a push only.

@@ -121,7 +129,8 @@ func (u *Pushes) Check(ctx context.Context, p domain.Principal, name domain.Repo
121	129
122	130		// Record is the post-receive step of an accepted push: it sets pushed_at,
123	131		// repairs HEAD when it points to a missing branch, tells the reader that the
124		-	// repository changed, publishes a PushEvent and schedules maintenance. Every
	132	+	// repository changed, queues the workflows the push triggers, publishes a
	133	+	// PushEvent and schedules maintenance. Every
125	134		// step runs even when an earlier one fails; the failures are returned together
126	135		// and the push itself is not affected.
127	136		func (u *Pushes) Record(ctx context.Context, p domain.Principal, name domain.RepoName, updates []domain.RefUpdate) error {

@@ -139,6 +148,9 @@ func (u *Pushes) Record(ctx context.Context, p domain.Principal, name domain.Rep
139	148		if u.fresh != nil {
140	149		u.fresh.Refresh(ctx, name)
141	150		}
	151	+	if u.ci != nil {
	152	+	u.ci.OnPush(ctx, name, updates, p) // queues only: never slows or fails the push
	153	+	}
142	154		u.bus.Publish(ctx, domain.PushEvent{
143	155		Repo: name, Updates: append([]domain.RefUpdate(nil), updates...), At: now,
144	156		Visibility: repo.Visibility, CreatedAt: repo.CreatedAt,

 	clock ports.Clock
 	maint ports.MaintenanceScheduler
 	auth  ports.Revalidator
-	fresh ports.RepoFreshener // nil: the reader needs no telling
+	fresh ports.RepoFreshener   // nil: the reader needs no telling
+	ci    ports.WorkflowTrigger // nil: no CI
 }
 var (
 	return u
 }
+// WithWorkflowTrigger makes Record start the workflows that match the push.
+// Call it before the first push; it returns u.
+func (u *Pushes) WithWorkflowTrigger(t ports.WorkflowTrigger) *Pushes {
+	u.ci = t
+	return u
+}
 // Open decides whether the principal may push to the repository and, with
 // create, creates it when it does not exist yet (private, published as
 // RepoCreated by Repos.Create). create is for the first request of a push only.
 // Record is the post-receive step of an accepted push: it sets pushed_at,
 // repairs HEAD when it points to a missing branch, tells the reader that the
-// repository changed, publishes a PushEvent and schedules maintenance. Every
+// repository changed, queues the workflows the push triggers, publishes a
+// PushEvent and schedules maintenance. Every
 // step runs even when an earlier one fails; the failures are returned together
 // and the push itself is not affected.
 func (u *Pushes) Record(ctx context.Context, p domain.Principal, name domain.RepoName, updates []domain.RefUpdate) error {
 	if u.fresh != nil {
 		u.fresh.Refresh(ctx, name)
 	}
+	if u.ci != nil {
+		u.ci.OnPush(ctx, name, updates, p) // queues only: never slows or fails the push
+	}
 	u.bus.Publish(ctx, domain.PushEvent{
 		Repo: name, Updates: append([]domain.RefUpdate(nil), updates...), At: now,
 		Visibility: repo.Visibility, CreatedAt: repo.CreatedAt,

M proto/ermite/storage/ci.proto +5 -1


@@ -161,9 +161,13 @@ message CIJob {
161	161		// re-run of the failed jobs is copied with the run attempt of the re-run and
162	162		// keeps its own attempt.
163	163		int32 run_attempt = 38;
	164	+	// The contexts JSON given to the runner, computed when the job is planned
	165	+	// (needs outputs are final then).
	166	+	string contexts_json = 39;
164	167		}
165	168
166		-	// CICheck is stored under "check/<repo>/<sha>/<job id>".
	169	+	// CICheck is stored under "check/<repo>/<sha>/<id>": the id of its job, or
	170	+	// "run-<run id>" for a run that failed without a job (invalid workflow file).
167	171		message CICheck {
168	172		string id = 1;
169	173		string repo = 2;

   // re-run of the failed jobs is copied with the run attempt of the re-run and
   // keeps its own attempt.
   int32 run_attempt = 38;
+  // The contexts JSON given to the runner, computed when the job is planned
+  // (needs outputs are final then).
+  string contexts_json = 39;
 }
-// CICheck is stored under "check/<repo>/<sha>/<job id>".
+// CICheck is stored under "check/<repo>/<sha>/<id>": the id of its job, or
+// "run-<run id>" for a run that failed without a job (invalid workflow file).
 message CICheck {
   string id = 1;
   string repo = 2;