1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package kvstore
4
5import (
6 "context"
7 "errors"
8 "fmt"
9 "slices"
10 "strconv"
11 "strings"
12 "time"
13
14 kv "bnema.dev/ermite-kv"
15 "google.golang.org/protobuf/proto"
16
17 storagepb "bnema.dev/ermite/gen/ermite/storage"
18 "bnema.dev/ermite/internal/domain"
19 "bnema.dev/ermite/internal/ports"
20)
21
22// Key layout of CI. Repository names hold no '/', so a prefix ending in "/"
23// never matches another repository; ids are zero-padded so that keys sort
24// numerically. Job records are keyed by run, attempt of the run and job id.
25//
26// run/<repo>/<id:020d> CIRun
27// runsrc/<id:020d> CIRunSource
28// job/<run:020d>/<attempt:010d>/<id> CIJob
29// check/<repo>/<sha>/<check id> CICheck
30// runseq Sequence: last run id
31// runnum/<repo>/<workflow path> Sequence: last run number
32// runner/<id>, runnername/<name>, runnertoken/<hash>
33//
34// Indexes, derived from the job and run records by every write:
35//
36// ciopen/<id:020d> run not completed (value: repo)
37// ciready/<ready_at:020d>/<job key> job waiting for a runner
38// ciactive/<job key> job assigned or running
39// jobtoken/<hash> job token (value: job key)
40// concur/<repo>/<key(group)> concurrency group (ci_concurrency.go)
41//
42// The job repository tokens and the action resolutions are in ci_token.go, the
43// annotations in ci_annotation.go.
44const (
45 runPrefix = "run/"
46 runSrcPrefix = "runsrc/"
47 jobPrefix = "job/"
48 checkPrefix = "check/"
49 runSeqKey = "runseq"
50 runNumPrefix = "runnum/"
51 runnerPrefix = "runner/"
52 runnerName = "runnername/"
53 runnerTokPref = "runnertoken/"
54 openPrefix = "ciopen/"
55 readyPrefix = "ciready/"
56 activePrefix = "ciactive/"
57 jobTokPrefix = "jobtoken/"
58 idFormat = "%020d"
59)
60
61// CI is the ermite-kv backed ports.CIStore. It shares the database of the
62// Store it comes from.
63type CI struct {
64 db *kv.DB
65}
66
67var _ ports.CIStore = (*CI)(nil)
68
69// CI returns the CI store over the same database. It is valid until the Store
70// is closed.
71func (s *Store) CI() *CI { return &CI{db: s.db} }
72
73func runKey(repo domain.RepoName, id int64) []byte {
74 return []byte(runPrefix + repo.String() + "/" + fmt.Sprintf(idFormat, id))
75}
76func runsPrefix(repo domain.RepoName) string { return runPrefix + repo.String() + "/" }
77func runSrcKey(id int64) []byte { return []byte(runSrcPrefix + fmt.Sprintf(idFormat, id)) }
78func openKey(id int64) []byte { return []byte(openPrefix + fmt.Sprintf(idFormat, id)) }
79func jobsPrefix(runID int64) string { return jobPrefix + fmt.Sprintf(idFormat, runID) + "/" }
80func jobAttemptPrefix(runID int64, attempt int32) string {
81 return jobsPrefix(runID) + fmt.Sprintf("%010d", attempt) + "/"
82}
83func jobRef(runID int64, attempt int32, id string) string {
84 return jobAttemptPrefix(runID, attempt) + id
85}
86func jobKey(runID int64, attempt int32, id string) []byte { return []byte(jobRef(runID, attempt, id)) }
87func checksPrefix(repo domain.RepoName, sha string) string {
88 return checkPrefix + repo.String() + "/" + sha + "/"
89}
90func checkKey(repo domain.RepoName, sha, id string) []byte {
91 return []byte(checksPrefix(repo, sha) + id)
92}
93func runNumKey(repo domain.RepoName, workflow string) []byte {
94 return []byte(runNumPrefix + repo.String() + "/" + workflow)
95}
96func readyKey(at time.Time, ref string) []byte {
97 return []byte(readyPrefix + fmt.Sprintf(idFormat, at.UnixNano()) + "/" + ref)
98}
99func activeKey(ref string) []byte { return []byte(activePrefix + ref) }
100func jobTokenKey(h domain.TokenHash) []byte {
101 return []byte(jobTokPrefix + h.String())
102}
103
104// --- encoding ---
105
106func encodeStatus(s domain.CIStatus) storagepb.CIStatus {
107 switch s {
108 case domain.CIQueued:
109 return storagepb.CIStatus_CI_STATUS_QUEUED
110 case domain.CIInProgress:
111 return storagepb.CIStatus_CI_STATUS_IN_PROGRESS
112 case domain.CICompleted:
113 return storagepb.CIStatus_CI_STATUS_COMPLETED
114 case domain.CIPending:
115 return storagepb.CIStatus_CI_STATUS_PENDING
116 }
117 return storagepb.CIStatus_CI_STATUS_UNSPECIFIED
118}
119
120func decodeStatus(s storagepb.CIStatus) domain.CIStatus {
121 switch s {
122 case storagepb.CIStatus_CI_STATUS_IN_PROGRESS:
123 return domain.CIInProgress
124 case storagepb.CIStatus_CI_STATUS_COMPLETED:
125 return domain.CICompleted
126 case storagepb.CIStatus_CI_STATUS_PENDING:
127 return domain.CIPending
128 }
129 return domain.CIQueued
130}
131
132var conclusions = []struct {
133 d domain.CIConclusion
134 p storagepb.CIConclusion
135}{
136 {domain.CISuccess, storagepb.CIConclusion_CI_CONCLUSION_SUCCESS},
137 {domain.CIFailure, storagepb.CIConclusion_CI_CONCLUSION_FAILURE},
138 {domain.CICancelled, storagepb.CIConclusion_CI_CONCLUSION_CANCELLED},
139 {domain.CISkipped, storagepb.CIConclusion_CI_CONCLUSION_SKIPPED},
140 {domain.CITimedOut, storagepb.CIConclusion_CI_CONCLUSION_TIMED_OUT},
141 {domain.CINeutral, storagepb.CIConclusion_CI_CONCLUSION_NEUTRAL},
142 {domain.CIActionRequired, storagepb.CIConclusion_CI_CONCLUSION_ACTION_REQUIRED},
143}
144
145func encodeConclusion(c domain.CIConclusion) storagepb.CIConclusion {
146 for _, e := range conclusions {
147 if e.d == c {
148 return e.p
149 }
150 }
151 return storagepb.CIConclusion_CI_CONCLUSION_UNSPECIFIED
152}
153
154func decodeConclusion(c storagepb.CIConclusion) domain.CIConclusion {
155 for _, e := range conclusions {
156 if e.p == c {
157 return e.d
158 }
159 }
160 return domain.CINoConclusion
161}
162
163var phases = []struct {
164 d domain.CIJobPhase
165 p storagepb.CIJobPhase
166}{
167 {domain.CIPhaseWaiting, storagepb.CIJobPhase_CI_JOB_PHASE_WAITING},
168 {domain.CIPhaseReady, storagepb.CIJobPhase_CI_JOB_PHASE_READY},
169 {domain.CIPhaseAssigned, storagepb.CIJobPhase_CI_JOB_PHASE_ASSIGNED},
170 {domain.CIPhaseRunning, storagepb.CIJobPhase_CI_JOB_PHASE_RUNNING},
171 {domain.CIPhaseEnded, storagepb.CIJobPhase_CI_JOB_PHASE_ENDED},
172}
173
174func encodePhase(p domain.CIJobPhase) (storagepb.CIJobPhase, error) {
175 for _, e := range phases {
176 if e.d == p {
177 return e.p, nil
178 }
179 }
180 return 0, fmt.Errorf("invalid job phase %q", p)
181}
182
183func decodePhase(p storagepb.CIJobPhase) domain.CIJobPhase {
184 for _, e := range phases {
185 if e.p == p {
186 return e.d
187 }
188 }
189 return domain.CIPhaseWaiting
190}
191
192var isolations = []struct {
193 d domain.CIIsolation
194 p storagepb.CIIsolation
195}{
196 {domain.CIIsolationVM, storagepb.CIIsolation_CI_ISOLATION_VM},
197 {domain.CIIsolationContainerSysbox, storagepb.CIIsolation_CI_ISOLATION_CONTAINER_SYSBOX},
198 {domain.CIIsolationContainerPrivileged, storagepb.CIIsolation_CI_ISOLATION_CONTAINER_PRIVILEGED},
199}
200
201func encodeIsolation(i domain.CIIsolation) storagepb.CIIsolation {
202 for _, e := range isolations {
203 if e.d == i {
204 return e.p
205 }
206 }
207 return storagepb.CIIsolation_CI_ISOLATION_UNSPECIFIED
208}
209
210func decodeIsolation(i storagepb.CIIsolation) domain.CIIsolation {
211 for _, e := range isolations {
212 if e.p == i {
213 return e.d
214 }
215 }
216 return domain.CIIsolationNone
217}
218
219var stepStates = []struct {
220 d domain.CIStepState
221 p storagepb.CIStepState
222}{
223 {domain.CIStepPending, storagepb.CIStepState_CI_STEP_STATE_PENDING},
224 {domain.CIStepInProgress, storagepb.CIStepState_CI_STEP_STATE_IN_PROGRESS},
225 {domain.CIStepCompleted, storagepb.CIStepState_CI_STEP_STATE_COMPLETED},
226}
227
228func encodeStepState(s domain.CIStepState) storagepb.CIStepState {
229 for _, e := range stepStates {
230 if e.d == s {
231 return e.p
232 }
233 }
234 return storagepb.CIStepState_CI_STEP_STATE_UNSPECIFIED
235}
236
237func decodeStepState(s storagepb.CIStepState) domain.CIStepState {
238 for _, e := range stepStates {
239 if e.p == s {
240 return e.d
241 }
242 }
243 return domain.CIStepPending
244}
245
246var cancelReasons = []struct {
247 d domain.CICancelReason
248 p storagepb.CICancelReason
249}{
250 {domain.CICancelUser, storagepb.CICancelReason_CI_CANCEL_REASON_USER},
251 {domain.CICancelFailFast, storagepb.CICancelReason_CI_CANCEL_REASON_FAIL_FAST},
252 {domain.CICancelTimeout, storagepb.CICancelReason_CI_CANCEL_REASON_TIMEOUT},
253}
254
255func encodeCancelReason(r domain.CICancelReason) storagepb.CICancelReason {
256 for _, e := range cancelReasons {
257 if e.d == r {
258 return e.p
259 }
260 }
261 return storagepb.CICancelReason_CI_CANCEL_REASON_UNSPECIFIED
262}
263
264func decodeCancelReason(r storagepb.CICancelReason) domain.CICancelReason {
265 for _, e := range cancelReasons {
266 if e.p == r {
267 return e.d
268 }
269 }
270 return domain.CICancelNone
271}
272
273func encodeRun(r domain.CIRun) ([]byte, error) {
274 if r.Repo.IsZero() {
275 return nil, fmt.Errorf("%w: empty name", domain.ErrInvalidName)
276 }
277 return mustMarshal(&storagepb.CIRun{
278 Repo: r.Repo.String(), Id: r.ID, Number: r.Number, Attempt: r.Attempt,
279 WorkflowPath: r.WorkflowPath, WorkflowName: r.WorkflowName, Name: r.Name, Title: r.Title,
280 Event: r.Event, Ref: r.Ref, Sha: r.Sha, Actor: r.Actor,
281 Status: encodeStatus(r.Status), Conclusion: encodeConclusion(r.Conclusion), Error: r.Error,
282 CreatedAt: encodeTime(r.CreatedAt), StartedAt: encodeTime(r.StartedAt), CompletedAt: encodeTime(r.CompletedAt),
283 CancelRequested: r.CancelRequested, CancelRequestedAt: encodeTime(r.CancelRequestedAt), Trusted: r.Trusted,
284 Concurrency: encodeConcurrency(r.Concurrency),
285 }), nil
286}
287
288func decodeRun(item kv.Item) (domain.CIRun, error) {
289 var pb storagepb.CIRun
290 if err := proto.Unmarshal(item.Value, &pb); err != nil {
291 return domain.CIRun{}, fmt.Errorf("decode run: %w", err)
292 }
293 repo, err := domain.ParseRepoName(pb.GetRepo())
294 if err != nil {
295 return domain.CIRun{}, fmt.Errorf("decode run: %w", err)
296 }
297 name := pb.GetName()
298 if name == "" {
299 name = pb.GetWorkflowName() // stored before the field existed
300 }
301 return domain.CIRun{
302 Repo: repo, ID: pb.GetId(), Number: pb.GetNumber(), Attempt: pb.GetAttempt(),
303 WorkflowPath: pb.GetWorkflowPath(), WorkflowName: pb.GetWorkflowName(), Name: name, Title: pb.GetTitle(),
304 Event: pb.GetEvent(), Ref: pb.GetRef(), Sha: pb.GetSha(), Actor: pb.GetActor(),
305 Status: decodeStatus(pb.GetStatus()), Conclusion: decodeConclusion(pb.GetConclusion()), Error: pb.GetError(),
306 CreatedAt: decodeTime(pb.GetCreatedAt()), StartedAt: decodeTime(pb.GetStartedAt()), CompletedAt: decodeTime(pb.GetCompletedAt()),
307 CancelRequested: pb.GetCancelRequested(), CancelRequestedAt: decodeTime(pb.GetCancelRequestedAt()),
308 Trusted: pb.GetTrusted(), Concurrency: decodeConcurrency(pb.GetConcurrency()), Version: item.Version,
309 }, nil
310}
311
312func encodeJob(j domain.CIJob) ([]byte, error) {
313 if j.Repo.IsZero() {
314 return nil, fmt.Errorf("%w: empty name", domain.ErrInvalidName)
315 }
316 phase, err := encodePhase(j.Phase)
317 if err != nil {
318 return nil, err
319 }
320 steps := make([]*storagepb.CIStep, 0, len(j.Steps))
321 for _, s := range j.Steps {
322 steps = append(steps, &storagepb.CIStep{
323 Number: s.Number, Name: s.Name, State: encodeStepState(s.State), Conclusion: encodeConclusion(s.Conclusion),
324 StartedAt: encodeTime(s.StartedAt), CompletedAt: encodeTime(s.CompletedAt),
325 LogStart: s.LogStart, HasLines: s.HasLines, Outcome: encodeConclusion(s.Outcome),
326 })
327 }
328 pb := &storagepb.CIJob{
329 Id: j.ID, RunId: j.RunID, Attempt: j.Attempt, RunAttempt: j.RunAttempt, Repo: j.Repo.String(),
330 Key: j.Key, Name: j.Name, Phase: phase, Conclusion: encodeConclusion(j.Conclusion), Error: j.Error,
331 Labels: j.Labels, Needs: j.Needs, RunnerId: j.RunnerID, RunnerName: j.RunnerName,
332 Isolation: encodeIsolation(j.Isolation), Steps: steps,
333 CreatedAt: encodeTime(j.CreatedAt), StartedAt: encodeTime(j.StartedAt), CompletedAt: encodeTime(j.CompletedAt),
334 Index: int32(j.Index), Combo: int32(j.Combo), MatrixJson: j.MatrixJSON, ContextsJson: j.ContextsJSON, ComboTotal: int32(j.ComboTotal), //nolint:gosec // bounded by the 256 job limit
335 TimeoutMinutes: j.TimeoutMinutes, FailFast: j.FailFast, ContinueOnError: j.ContinueOnError, MaxParallel: int32(j.MaxParallel), //nolint:gosec // validated at planning
336 Outputs: j.Outputs, ReadyAt: encodeTime(j.ReadyAt), AssignedAt: encodeTime(j.AssignedAt),
337 AckedAt: encodeTime(j.AckedAt), AssignCancels: int32(j.AssignCancels), //nolint:gosec // at most 3
338 TokenValidUntil: encodeTime(j.TokenValidUntil), CancelReason: encodeCancelReason(j.CancelReason), CancelMessage: j.CancelMessage,
339 CancelRequestedAt: encodeTime(j.CancelRequestedAt), Planned: j.Planned, Environment: j.Environment, Permissions: j.Permissions,
340 Concurrency: encodeConcurrency(j.Concurrency),
341 }
342 if j.HasToken() {
343 pb.TokenHash = j.TokenHash[:]
344 }
345 return mustMarshal(pb), nil
346}
347
348//hexcheck:ignore suspicious-business-logic-in-adapter maps the stored record to the domain job field by field: no rule is decided here
349func decodeJob(item kv.Item) (domain.CIJob, error) {
350 var pb storagepb.CIJob
351 if err := proto.Unmarshal(item.Value, &pb); err != nil {
352 return domain.CIJob{}, fmt.Errorf("decode job: %w", err)
353 }
354 repo, err := domain.ParseRepoName(pb.GetRepo())
355 if err != nil {
356 return domain.CIJob{}, fmt.Errorf("decode job: %w", err)
357 }
358 j := domain.CIJob{
359 ID: pb.GetId(), RunID: pb.GetRunId(), Attempt: pb.GetAttempt(), RunAttempt: pb.GetRunAttempt(), Repo: repo,
360 Key: pb.GetKey(), Name: pb.GetName(), Phase: decodePhase(pb.GetPhase()), Conclusion: decodeConclusion(pb.GetConclusion()),
361 Error: pb.GetError(), Labels: pb.GetLabels(), Needs: pb.GetNeeds(), RunnerID: pb.GetRunnerId(), RunnerName: pb.GetRunnerName(),
362 Isolation: decodeIsolation(pb.GetIsolation()),
363 CreatedAt: decodeTime(pb.GetCreatedAt()), StartedAt: decodeTime(pb.GetStartedAt()), CompletedAt: decodeTime(pb.GetCompletedAt()),
364 Index: int(pb.GetIndex()), Combo: int(pb.GetCombo()), Planned: pb.GetPlanned(), MatrixJSON: pb.GetMatrixJson(), ContextsJSON: pb.GetContextsJson(),
365 ComboTotal: int(pb.GetComboTotal()), TimeoutMinutes: pb.GetTimeoutMinutes(), FailFast: pb.GetFailFast(), ContinueOnError: pb.GetContinueOnError(),
366 MaxParallel: int(pb.GetMaxParallel()), Outputs: pb.GetOutputs(),
367 TokenValidUntil: decodeTime(pb.GetTokenValidUntil()), ReadyAt: decodeTime(pb.GetReadyAt()),
368 AssignedAt: decodeTime(pb.GetAssignedAt()), AckedAt: decodeTime(pb.GetAckedAt()), AssignCancels: int(pb.GetAssignCancels()),
369 CancelMessage: pb.GetCancelMessage(), CancelRequestedAt: decodeTime(pb.GetCancelRequestedAt()),
370 Environment: pb.GetEnvironment(), Permissions: pb.GetPermissions(), Concurrency: decodeConcurrency(pb.GetConcurrency()), Version: item.Version,
371 }
372 j.CancelReason = decodeCancelReason(pb.GetCancelReason())
373 copy(j.TokenHash[:], pb.GetTokenHash())
374 for _, s := range pb.GetSteps() {
375 j.Steps = append(j.Steps, domain.CIStep{
376 Number: s.GetNumber(), Name: s.GetName(), State: decodeStepState(s.GetState()), Conclusion: decodeConclusion(s.GetConclusion()),
377 StartedAt: decodeTime(s.GetStartedAt()), CompletedAt: decodeTime(s.GetCompletedAt()),
378 LogStart: s.GetLogStart(), HasLines: s.GetHasLines(), Outcome: decodeConclusion(s.GetOutcome()),
379 })
380 }
381 return j, nil
382}
383
384func encodeCheck(c domain.CICheck) []byte {
385 return mustMarshal(&storagepb.CICheck{
386 Id: c.ID, Repo: c.Repo.String(), Sha: c.Sha, Name: c.Name, Status: encodeStatus(c.Status),
387 Conclusion: encodeConclusion(c.Conclusion), RunId: c.RunID, JobId: c.JobID, Attempt: c.Attempt,
388 WorkflowName: c.WorkflowName, CreatedAt: encodeTime(c.CreatedAt), StartedAt: encodeTime(c.StartedAt),
389 CompletedAt: encodeTime(c.CompletedAt),
390 })
391}
392
393func decodeCheck(item kv.Item) (domain.CICheck, error) {
394 var pb storagepb.CICheck
395 if err := proto.Unmarshal(item.Value, &pb); err != nil {
396 return domain.CICheck{}, fmt.Errorf("decode check: %w", err)
397 }
398 repo, err := domain.ParseRepoName(pb.GetRepo())
399 if err != nil {
400 return domain.CICheck{}, fmt.Errorf("decode check: %w", err)
401 }
402 return domain.CICheck{
403 ID: pb.GetId(), Repo: repo, Sha: pb.GetSha(), Name: pb.GetName(), Status: decodeStatus(pb.GetStatus()),
404 Conclusion: decodeConclusion(pb.GetConclusion()), RunID: pb.GetRunId(), JobID: pb.GetJobId(), Attempt: pb.GetAttempt(),
405 WorkflowName: pb.GetWorkflowName(), CreatedAt: decodeTime(pb.GetCreatedAt()), StartedAt: decodeTime(pb.GetStartedAt()),
406 CompletedAt: decodeTime(pb.GetCompletedAt()), Version: item.Version,
407 }, nil
408}
409
410func encodeRunner(r domain.CIRunner) []byte {
411 return mustMarshal(&storagepb.CIRunner{
412 Id: r.ID, Name: r.Name, Labels: r.Labels, Isolation: encodeIsolation(r.Isolation), Version: r.Version,
413 Arch: r.Arch, Os: r.OS, RegisteredAt: encodeTime(r.RegisteredAt), TokenHash: r.TokenHash[:],
414 })
415}
416
417func decodeRunner(val []byte) (domain.CIRunner, error) {
418 var pb storagepb.CIRunner
419 if err := proto.Unmarshal(val, &pb); err != nil {
420 return domain.CIRunner{}, fmt.Errorf("decode runner: %w", err)
421 }
422 r := domain.CIRunner{
423 ID: pb.GetId(), Name: pb.GetName(), Labels: pb.GetLabels(), Isolation: decodeIsolation(pb.GetIsolation()),
424 Version: pb.GetVersion(), Arch: pb.GetArch(), OS: pb.GetOs(), RegisteredAt: decodeTime(pb.GetRegisteredAt()),
425 }
426 copy(r.TokenHash[:], pb.GetTokenHash())
427 return r, nil
428}
429
430// --- writes ---
431
432// stage adds the preconditions and writes of one record to the batch.
433func stage(b *kv.Batch, key, val []byte, version uint64) {
434 if version == 0 {
435 b.RequireAbsent(key)
436 } else {
437 b.RequireVersion(key, version)
438 }
439 b.Put(key, val)
440}
441
442// runIndexes stages the open-run index of the run.
443func runIndexes(b *kv.Batch, r domain.CIRun) {
444 if r.Status == domain.CICompleted {
445 b.Delete(openKey(r.ID))
446 return
447 }
448 b.Put(openKey(r.ID), []byte(r.Repo.String()))
449}
450
451// jobIndexes stages the ready, active and token indexes of the job. old is
452// the job as stored before, if any: its token index goes when the token
453// changed.
454func (c *CI) jobIndexes(b *kv.Batch, j domain.CIJob, old *domain.CIJob) {
455 ref := jobRef(j.RunID, j.RunAttempt, j.ID)
456 if j.Phase == domain.CIPhaseReady {
457 b.Put(readyKey(j.ReadyAt, ref), nil)
458 } else {
459 b.Delete(readyKey(j.ReadyAt, ref))
460 }
461 if old != nil && old.Phase == domain.CIPhaseReady && (j.Phase != domain.CIPhaseReady || !old.ReadyAt.Equal(j.ReadyAt)) {
462 b.Delete(readyKey(old.ReadyAt, ref))
463 }
464 if j.Active() {
465 b.Put(activeKey(ref), nil)
466 } else {
467 b.Delete(activeKey(ref))
468 }
469 if old != nil && old.HasToken() && old.TokenHash != j.TokenHash {
470 b.Delete(jobTokenKey(old.TokenHash))
471 }
472 if j.HasToken() {
473 b.Put(jobTokenKey(j.TokenHash), []byte(ref))
474 }
475}
476
477// guardRepo is guardRepoKey on the database of the CI store, for the writes that
478// create keys under a repository.
479func (c *CI) guardRepo(ctx context.Context, b *kv.Batch, repo domain.RepoName) error {
480 return guardRepoKey(ctx, c.db, b, repo)
481}
482
483// stageRecords stages the writes of w. guarded collects the repositories that
484// need guarding.
485func (c *CI) stageRecords(ctx context.Context, b *kv.Batch, w ports.CIWrite) (guarded map[domain.RepoName]bool, err error) {
486 guarded = map[domain.RepoName]bool{}
487 for _, r := range w.Runs {
488 val, err := encodeRun(r)
489 if err != nil {
490 return nil, err
491 }
492 stage(b, runKey(r.Repo, r.ID), val, r.Version)
493 runIndexes(b, r)
494 if r.Version == 0 {
495 guarded[r.Repo] = true
496 }
497 }
498 for _, j := range w.Jobs {
499 val, err := encodeJob(j)
500 if err != nil {
501 return nil, err
502 }
503 key := jobKey(j.RunID, j.RunAttempt, j.ID)
504 var old *domain.CIJob
505 if j.Version != 0 {
506 item, ok := c.db.Get(ctx, key)
507 if !ok {
508 return nil, domain.ErrConflict
509 }
510 if o, err := decodeJob(item); err == nil {
511 old = &o
512 }
513 }
514 stage(b, key, val, j.Version)
515 c.jobIndexes(b, j, old)
516 if j.Version == 0 {
517 guarded[j.Repo] = true
518 }
519 }
520 c.stageRepoTokens(ctx, b, w, guarded)
521 c.stageAnnotations(ctx, b, w, guarded)
522 // Checks are derived from the jobs, written in the same batch and under
523 // the lock of their run: they are upserted without a precondition.
524 for _, ck := range w.Checks {
525 b.Put(checkKey(ck.Repo, ck.Sha, ck.ID), encodeCheck(ck))
526 }
527 return guarded, nil
528}
529
530// Apply writes the records in one batch.
531//
532//hexcheck:ignore suspicious-business-logic-in-adapter stages records and their derived indexes under version preconditions: a storage concern, no policy
533func (c *CI) Apply(ctx context.Context, w ports.CIWrite) (uint64, error) {
534 var b kv.Batch
535 guarded, err := c.stageRecords(ctx, &b, w)
536 if err != nil {
537 return 0, err
538 }
539 for repo := range guarded {
540 if err := c.guardRepo(ctx, &b, repo); err != nil {
541 return 0, err
542 }
543 }
544 v, err := c.db.Apply(ctx, &b)
545 if errors.Is(err, kv.ErrConflict) {
546 return 0, domain.ErrConflict
547 }
548 if err != nil {
549 return 0, fmt.Errorf("write CI records: %w", err)
550 }
551 return v, nil
552}
553
554func (c *CI) readSeq(ctx context.Context, key []byte) (last int64, item kv.Item, ok bool, err error) {
555 item, ok = c.db.Get(ctx, key)
556 if !ok {
557 return 0, kv.Item{}, false, nil
558 }
559 var seq storagepb.Sequence
560 if err := proto.Unmarshal(item.Value, &seq); err != nil {
561 return 0, kv.Item{}, false, fmt.Errorf("decode sequence %s: %w", key, err)
562 }
563 return seq.GetLast(), item, true, nil
564}
565
566func stageSeq(b *kv.Batch, key []byte, item kv.Item, ok bool, last int64) {
567 if ok {
568 b.RequireVersion(key, item.Version)
569 } else {
570 b.RequireAbsent(key)
571 }
572 b.Put(key, mustMarshal(&storagepb.Sequence{Last: last}))
573}
574
575const createRunTries = 20
576
577// CreateRun takes the next run id and number and stores the draft.
578//
579//hexcheck:ignore suspicious-business-logic-in-adapter takes the next numbers under version preconditions: a storage concern, no policy
580func (c *CI) CreateRun(ctx context.Context, repo domain.RepoName, workflowPath string, build func(id, number int64) ports.CIRunDraft) (ports.CIRunDraft, error) {
581 for range createRunTries {
582 lastID, idItem, idOK, err := c.readSeq(ctx, []byte(runSeqKey))
583 if err != nil {
584 return ports.CIRunDraft{}, err
585 }
586 numKey := runNumKey(repo, workflowPath)
587 lastNum, numItem, numOK, err := c.readSeq(ctx, numKey)
588 if err != nil {
589 return ports.CIRunDraft{}, err
590 }
591 draft := build(lastID+1, lastNum+1)
592 var b kv.Batch
593 stageSeq(&b, []byte(runSeqKey), idItem, idOK, lastID+1)
594 stageSeq(&b, numKey, numItem, numOK, lastNum+1)
595 w := ports.CIWrite{Runs: []domain.CIRun{draft.Run}, Jobs: draft.Jobs, Checks: draft.Checks}
596 guarded, err := c.stageRecords(ctx, &b, w)
597 if err != nil {
598 return ports.CIRunDraft{}, err
599 }
600 src := mustMarshal(&storagepb.CIRunSource{
601 RunId: draft.Run.ID, Repo: repo.String(), WorkflowYaml: draft.Source.WorkflowYAML, GithubJson: draft.Source.GithubJSON,
602 PusherKind: string(draft.Source.Pusher.Kind), PusherTokenId: draft.Source.Pusher.TokenID,
603 })
604 b.RequireAbsent(runSrcKey(draft.Run.ID))
605 b.Put(runSrcKey(draft.Run.ID), src)
606 for r := range guarded {
607 if err := c.guardRepo(ctx, &b, r); err != nil {
608 return ports.CIRunDraft{}, err
609 }
610 }
611 v, err := c.db.Apply(ctx, &b)
612 if errors.Is(err, kv.ErrConflict) {
613 continue
614 }
615 if err != nil {
616 return ports.CIRunDraft{}, fmt.Errorf("create run of %q: %w", repo, err)
617 }
618 draft.Run.Version = v
619 for i := range draft.Jobs {
620 draft.Jobs[i].Version = v
621 }
622 for i := range draft.Checks {
623 draft.Checks[i].Version = v
624 }
625 return draft, nil
626 }
627 return ports.CIRunDraft{}, fmt.Errorf("create run of %q: %w", repo, domain.ErrConflict)
628}
629
630// --- reads ---
631
632// GetRun reads run/<repo>/<id>.
633func (c *CI) GetRun(ctx context.Context, repo domain.RepoName, id int64) (domain.CIRun, error) {
634 item, ok := c.db.Get(ctx, runKey(repo, id))
635 if !ok || id < 1 {
636 return domain.CIRun{}, domain.ErrNotFound
637 }
638 return decodeRun(item)
639}
640
641// GetRunSource reads runsrc/<id>.
642func (c *CI) GetRunSource(ctx context.Context, runID int64) (domain.CIRunSource, error) {
643 item, ok := c.db.Get(ctx, runSrcKey(runID))
644 if !ok || runID < 1 {
645 return domain.CIRunSource{}, domain.ErrNotFound
646 }
647 var pb storagepb.CIRunSource
648 if err := proto.Unmarshal(item.Value, &pb); err != nil {
649 return domain.CIRunSource{}, fmt.Errorf("decode run source: %w", err)
650 }
651 repo, err := domain.ParseRepoName(pb.GetRepo())
652 if err != nil {
653 return domain.CIRunSource{}, fmt.Errorf("decode run source: %w", err)
654 }
655 return domain.CIRunSource{
656 RunID: pb.GetRunId(), Repo: repo, WorkflowYAML: pb.GetWorkflowYaml(), GithubJSON: pb.GetGithubJson(),
657 Pusher: domain.CIPusher{Kind: domain.CIPusherKind(pb.GetPusherKind()), TokenID: pb.GetPusherTokenId()},
658 }, nil
659}
660
661// ListRuns returns the runs of the repository, newest first.
662//
663//hexcheck:ignore suspicious-business-logic-in-adapter port method name only: reads a key range and reverses it
664func (c *CI) ListRuns(ctx context.Context, repo domain.RepoName, before int64, limit int) ([]domain.CIRun, error) {
665 start := []byte(runsPrefix(repo))
666 end := []byte(runPrefix + repo.String() + "0") // '/' + 1
667 if before > 0 {
668 end = runKey(repo, before)
669 }
670 // The snapshot iterates in key order only: keep the last limit items in a
671 // ring and decode only those, so the cost of a page does not grow with the
672 // decoded history of the repository.
673 type entry struct {
674 key []byte
675 item kv.Item
676 }
677 var ring []entry
678 next := 0
679 for key, item := range c.db.Snapshot(ctx).Range(ctx, start, end) {
680 if limit <= 0 || len(ring) < limit {
681 ring = append(ring, entry{key, item})
682 continue
683 }
684 ring[next] = entry{key, item}
685 next = (next + 1) % limit
686 }
687 ordered := append(ring[next:len(ring):len(ring)], ring[:next]...)
688 all := make([]domain.CIRun, 0, len(ordered))
689 for i := len(ordered) - 1; i >= 0; i-- {
690 r, err := decodeRun(ordered[i].item)
691 if err != nil {
692 return nil, fmt.Errorf("decode %s: %w", ordered[i].key, err)
693 }
694 all = append(all, r)
695 }
696 return all, nil
697}
698
699// RunIDs returns the ids of the runs of the repository.
700func (c *CI) RunIDs(ctx context.Context, repo domain.RepoName) ([]int64, error) {
701 var ids []int64
702 prefix := runsPrefix(repo)
703 for key := range c.db.Snapshot(ctx).Prefix(ctx, []byte(prefix)) {
704 id, err := strconv.ParseInt(strings.TrimPrefix(string(key), prefix), 10, 64)
705 if err != nil {
706 return nil, fmt.Errorf("decode %s: %w", key, err)
707 }
708 ids = append(ids, id)
709 }
710 return ids, nil
711}
712
713// ListOpenRuns returns the runs that have not completed.
714func (c *CI) ListOpenRuns(ctx context.Context) ([]domain.CIRun, error) {
715 snap := c.db.Snapshot(ctx)
716 var out []domain.CIRun
717 for key, item := range snap.Prefix(ctx, []byte(openPrefix)) {
718 id, err := strconv.ParseInt(strings.TrimPrefix(string(key), openPrefix), 10, 64)
719 if err != nil {
720 return nil, fmt.Errorf("decode %s: %w", key, err)
721 }
722 repo, err := domain.ParseRepoName(string(item.Value))
723 if err != nil {
724 return nil, err
725 }
726 it, ok := snap.Get(ctx, runKey(repo, id))
727 if !ok {
728 continue
729 }
730 r, err := decodeRun(it)
731 if err != nil {
732 return nil, err
733 }
734 out = append(out, r)
735 }
736 return out, nil
737}
738
739// GetJob reads one job record.
740func (c *CI) GetJob(ctx context.Context, runID int64, runAttempt int32, jobID string) (domain.CIJob, error) {
741 item, ok := c.db.Get(ctx, jobKey(runID, runAttempt, jobID))
742 if !ok {
743 return domain.CIJob{}, domain.ErrNotFound
744 }
745 return decodeJob(item)
746}
747
748// ListJobs returns the jobs of a run at an attempt, ordered by index then
749// combination.
750func (c *CI) ListJobs(ctx context.Context, runID int64, runAttempt int32) ([]domain.CIJob, error) {
751 var out []domain.CIJob
752 for key, item := range c.db.Snapshot(ctx).Prefix(ctx, []byte(jobAttemptPrefix(runID, runAttempt))) {
753 j, err := decodeJob(item)
754 if err != nil {
755 return nil, fmt.Errorf("decode %s: %w", key, err)
756 }
757 out = append(out, j)
758 }
759 slices.SortFunc(out, func(a, b domain.CIJob) int {
760 if a.Index != b.Index {
761 return a.Index - b.Index
762 }
763 return a.Combo - b.Combo
764 })
765 return out, nil
766}
767
768func (c *CI) jobByRef(snap *kv.Snapshot, ctx context.Context, ref string) (domain.CIJob, error) {
769 item, ok := snap.Get(ctx, []byte(ref))
770 if !ok {
771 return domain.CIJob{}, domain.ErrNotFound
772 }
773 return decodeJob(item)
774}
775
776// FindJobByToken resolves the token index, then reads the job.
777func (c *CI) FindJobByToken(ctx context.Context, hash domain.TokenHash, now time.Time) (domain.CIJob, error) {
778 snap := c.db.Snapshot(ctx)
779 idx, ok := snap.Get(ctx, jobTokenKey(hash))
780 if !ok {
781 return domain.CIJob{}, domain.ErrNotFound
782 }
783 j, err := c.jobByRef(snap, ctx, string(idx.Value))
784 if err != nil {
785 return domain.CIJob{}, err
786 }
787 if j.TokenHash != hash || (!j.TokenValidUntil.IsZero() && !now.Before(j.TokenValidUntil)) {
788 return domain.CIJob{}, domain.ErrNotFound
789 }
790 return j, nil
791}
792
793// ListReadyJobs returns the jobs waiting for a runner, oldest first.
794func (c *CI) ListReadyJobs(ctx context.Context, limit int) ([]domain.CIJob, error) {
795 snap := c.db.Snapshot(ctx)
796 var out []domain.CIJob
797 for key := range snap.Prefix(ctx, []byte(readyPrefix)) {
798 ref := string(key)[len(readyPrefix)+len(fmt.Sprintf(idFormat, 0))+1:]
799 j, err := c.jobByRef(snap, ctx, ref)
800 if errors.Is(err, domain.ErrNotFound) {
801 continue
802 }
803 if err != nil {
804 return nil, err
805 }
806 out = append(out, j)
807 if limit > 0 && len(out) >= limit {
808 break
809 }
810 }
811 return out, nil
812}
813
814// ListActiveJobs returns the jobs given to a runner and not over.
815func (c *CI) ListActiveJobs(ctx context.Context) ([]domain.CIJob, error) {
816 snap := c.db.Snapshot(ctx)
817 var out []domain.CIJob
818 for key := range snap.Prefix(ctx, []byte(activePrefix)) {
819 j, err := c.jobByRef(snap, ctx, strings.TrimPrefix(string(key), activePrefix))
820 if errors.Is(err, domain.ErrNotFound) {
821 continue
822 }
823 if err != nil {
824 return nil, err
825 }
826 out = append(out, j)
827 }
828 return out, nil
829}
830
831// ListChecks returns the checks of a commit ordered by creation time then
832// name.
833func (c *CI) ListChecks(ctx context.Context, repo domain.RepoName, sha string, limit int) ([]domain.CICheck, bool, error) {
834 var out []domain.CICheck
835 for key, item := range c.db.Snapshot(ctx).Prefix(ctx, []byte(checksPrefix(repo, sha))) {
836 ck, err := decodeCheck(item)
837 if err != nil {
838 return nil, false, fmt.Errorf("decode %s: %w", key, err)
839 }
840 out = append(out, ck)
841 }
842 slices.SortStableFunc(out, func(a, b domain.CICheck) int {
843 if d := a.CreatedAt.Compare(b.CreatedAt); d != 0 {
844 return d
845 }
846 return strings.Compare(a.Name, b.Name)
847 })
848 more := limit > 0 && len(out) > limit
849 if more {
850 out = out[:limit]
851 }
852 return out, more, nil
853}
854
855// --- retention and deletion ---
856
857// runKeys lists every key of a run except its record: source, jobs, indexes.
858func (c *CI) runKeys(ctx context.Context, snap *kv.Snapshot, repo domain.RepoName, runID int64) [][]byte {
859 keys := [][]byte{runKey(repo, runID), runSrcKey(runID), openKey(runID)}
860 keys = append(keys, repoTokenKeys(ctx, snap, runID)...)
861 keys = append(keys, annotationKeys(ctx, snap, runID)...)
862 for key, item := range snap.Prefix(ctx, []byte(jobsPrefix(runID))) {
863 keys = append(keys, slices.Clone(key))
864 if j, err := decodeJob(item); err == nil {
865 ref := jobRef(j.RunID, j.RunAttempt, j.ID)
866 keys = append(keys, readyKey(j.ReadyAt, ref), activeKey(ref))
867 if j.HasToken() {
868 keys = append(keys, jobTokenKey(j.TokenHash))
869 }
870 }
871 }
872 return keys
873}
874
875// ciKeys lists every key of the snapshot that belongs to the repository's CI
876// runs, checks and numbers.
877func ciKeys(ctx context.Context, snap *kv.Snapshot, repo domain.RepoName) [][]byte {
878 c := &CI{}
879 var keys [][]byte
880 for key, item := range snap.Prefix(ctx, []byte(runsPrefix(repo))) {
881 if r, err := decodeRun(item); err == nil {
882 keys = append(keys, c.runKeys(ctx, snap, repo, r.ID)...)
883 }
884 keys = append(keys, slices.Clone(key))
885 }
886 keys = append(keys, repoReaderKeys(ctx, snap, repo)...)
887 for _, p := range []string{checkPrefix + repo.String() + "/", runNumPrefix + repo.String() + "/", concurPrefix + repo.String() + "/", concurMarkPrefixAll + repo.String() + "/"} {
888 for k := range snap.Prefix(ctx, []byte(p)) {
889 keys = append(keys, slices.Clone(k))
890 }
891 }
892 return keys
893}
894
895const sweepChunk = 500
896
897// Sweep removes the runs completed before the time.
898//
899//hexcheck:ignore suspicious-business-logic-in-adapter chunked delete of expired records: a storage concern, the retention is decided by the caller
900func (c *CI) Sweep(ctx context.Context, before time.Time) ([]int64, error) {
901 snap := c.db.Snapshot(ctx)
902 var ids []int64
903 var keys [][]byte
904 for _, rp := range []string{runPrefix} {
905 for _, item := range snap.Prefix(ctx, []byte(rp)) {
906 r, err := decodeRun(item)
907 if err != nil {
908 return nil, err
909 }
910 if r.Status != domain.CICompleted || r.CompletedAt.IsZero() || !r.CompletedAt.Before(before) {
911 continue
912 }
913 ids = append(ids, r.ID)
914 keys = append(keys, c.runKeys(ctx, snap, r.Repo, r.ID)...)
915 for ck := range snap.Prefix(ctx, []byte(checkPrefix+r.Repo.String()+"/"+r.Sha+"/")) {
916 // A check belongs to the run only while it is the latest one
917 // for its job: leave it when another run replaced it.
918 if it, ok := snap.Get(ctx, ck); ok {
919 if chk, err := decodeCheck(it); err == nil && chk.RunID == r.ID {
920 keys = append(keys, slices.Clone(ck))
921 }
922 }
923 }
924 }
925 }
926 for len(keys) > 0 {
927 n := min(sweepChunk, len(keys))
928 var b kv.Batch
929 for _, k := range keys[:n] {
930 b.Delete(k)
931 }
932 if _, err := c.db.Apply(ctx, &b); err != nil {
933 return nil, fmt.Errorf("sweep CI runs: %w", err)
934 }
935 keys = keys[n:]
936 }
937 return ids, nil
938}
939
940// --- runners ---
941
942func runnerKey(id string) []byte { return []byte(runnerPrefix + id) }
943func runnerNameKey(n string) []byte { return []byte(runnerName + strings.ToLower(n)) }
944func runnerTokenKey(h domain.TokenHash) []byte { return []byte(runnerTokPref + h.String()) }
945
946// CreateRunner stores the runner, its name index and its token index.
947//
948//hexcheck:ignore suspicious-business-logic-in-adapter replaces a runner of the same name in one batch: a storage concern, the rule is decided by the caller
949func (c *CI) CreateRunner(ctx context.Context, r domain.CIRunner, replace bool) ([]domain.CIRunner, error) {
950 for range createRunTries {
951 var b kv.Batch
952 var removed []domain.CIRunner
953 if idx, ok := c.db.Get(ctx, runnerNameKey(r.Name)); ok {
954 if !replace {
955 return nil, domain.ErrAlreadyExists
956 }
957 old, err := c.GetRunner(ctx, string(idx.Value))
958 if err != nil && !errors.Is(err, domain.ErrNotFound) {
959 return nil, err
960 }
961 b.RequireVersion(runnerNameKey(r.Name), idx.Version)
962 if err == nil {
963 removed = append(removed, old)
964 b.Delete(runnerKey(old.ID))
965 b.Delete(runnerTokenKey(old.TokenHash))
966 }
967 } else {
968 b.RequireAbsent(runnerNameKey(r.Name))
969 }
970 b.RequireAbsent(runnerKey(r.ID))
971 b.RequireAbsent(runnerTokenKey(r.TokenHash))
972 b.Put(runnerKey(r.ID), encodeRunner(r))
973 b.Put(runnerNameKey(r.Name), []byte(r.ID))
974 b.Put(runnerTokenKey(r.TokenHash), []byte(r.ID))
975 _, err := c.db.Apply(ctx, &b)
976 if errors.Is(err, kv.ErrConflict) {
977 if !replace {
978 if _, ok := c.db.Get(ctx, runnerNameKey(r.Name)); ok {
979 return nil, domain.ErrAlreadyExists
980 }
981 }
982 continue
983 }
984 if err != nil {
985 return nil, fmt.Errorf("create runner %q: %w", r.Name, err)
986 }
987 return removed, nil
988 }
989 return nil, fmt.Errorf("create runner %q: %w", r.Name, domain.ErrConflict)
990}
991
992// GetRunner reads runner/<id>.
993func (c *CI) GetRunner(ctx context.Context, id string) (domain.CIRunner, error) {
994 item, ok := c.db.Get(ctx, runnerKey(id))
995 if !ok || id == "" || strings.Contains(id, "/") {
996 return domain.CIRunner{}, domain.ErrNotFound
997 }
998 return decodeRunner(item.Value)
999}
1000
1001// FindRunnerByToken resolves the token index, then reads the runner.
1002func (c *CI) FindRunnerByToken(ctx context.Context, hash domain.TokenHash) (domain.CIRunner, error) {
1003 snap := c.db.Snapshot(ctx)
1004 idx, ok := snap.Get(ctx, runnerTokenKey(hash))
1005 if !ok {
1006 return domain.CIRunner{}, domain.ErrNotFound
1007 }
1008 item, ok := snap.Get(ctx, runnerKey(string(idx.Value)))
1009 if !ok {
1010 return domain.CIRunner{}, domain.ErrNotFound
1011 }
1012 r, err := decodeRunner(item.Value)
1013 if err != nil {
1014 return domain.CIRunner{}, err
1015 }
1016 if r.TokenHash != hash {
1017 return domain.CIRunner{}, fmt.Errorf("runner %q: token index does not match the stored hash", r.ID)
1018 }
1019 return r, nil
1020}
1021
1022// ListRunners returns every runner ordered by name.
1023func (c *CI) ListRunners(ctx context.Context) ([]domain.CIRunner, error) {
1024 var out []domain.CIRunner
1025 for key, item := range c.db.Snapshot(ctx).Prefix(ctx, []byte(runnerPrefix)) {
1026 r, err := decodeRunner(item.Value)
1027 if err != nil {
1028 return nil, fmt.Errorf("decode %s: %w", key, err)
1029 }
1030 out = append(out, r)
1031 }
1032 slices.SortFunc(out, func(a, b domain.CIRunner) int {
1033 return strings.Compare(strings.ToLower(a.Name), strings.ToLower(b.Name))
1034 })
1035 return out, nil
1036}
1037
1038// RemoveRunner removes the runner, its name and its token.
1039func (c *CI) RemoveRunner(ctx context.Context, id string) (domain.CIRunner, error) {
1040 for range createRunTries {
1041 item, ok := c.db.Get(ctx, runnerKey(id))
1042 if !ok || id == "" || strings.Contains(id, "/") {
1043 return domain.CIRunner{}, domain.ErrNotFound
1044 }
1045 r, err := decodeRunner(item.Value)
1046 if err != nil {
1047 return domain.CIRunner{}, err
1048 }
1049 var b kv.Batch
1050 b.RequireVersion(runnerKey(id), item.Version)
1051 b.Delete(runnerKey(id))
1052 b.Delete(runnerNameKey(r.Name))
1053 b.Delete(runnerTokenKey(r.TokenHash))
1054 _, err = c.db.Apply(ctx, &b)
1055 if errors.Is(err, kv.ErrConflict) {
1056 continue
1057 }
1058 if err != nil {
1059 return domain.CIRunner{}, fmt.Errorf("remove runner %q: %w", id, err)
1060 }
1061 return r, nil
1062 }
1063 return domain.CIRunner{}, fmt.Errorf("remove runner %q: %w", id, domain.ErrConflict)
1064}