1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package app
4
5import (
6 "context"
7 "encoding/base64"
8 "errors"
9 "fmt"
10 "net/http/httptest"
11 "strings"
12 "sync/atomic"
13 "testing"
14
15 "connectrpc.com/connect"
16 "github.com/stretchr/testify/assert"
17 "github.com/stretchr/testify/require"
18 "google.golang.org/protobuf/encoding/protojson"
19 "google.golang.org/protobuf/proto"
20
21 pb "bnema.dev/ermite-api/gen/ermite"
22 "bnema.dev/ermite/internal/config"
23 "bnema.dev/ermite/internal/domain"
24 "bnema.dev/ermite/internal/ports"
25 "bnema.dev/ermite/internal/usecase"
26)
27
28var secretsKeyA = base64.StdEncoding.EncodeToString([]byte("0123456789abcdef0123456789abcdef"))
29var secretsKeyB = base64.StdEncoding.EncodeToString([]byte("fedcba9876543210fedcba9876543210"))
30
31// newSecretsHarness is a CI harness whose server has ERMITE_SECRETS_KEY.
32func newSecretsHarness(t *testing.T, key string) *ciHarness {
33 t.Helper()
34 o := harnessOpts{}
35 if key != "" {
36 o.env = map[string]string{config.EnvSecretsKey: key}
37 }
38 h := newHarnessWith(t, o, nil)
39 c := &ciHarness{harness: h, root: h.api(adminToken), repo: "proj"}
40 _, err := h.forge.Repos.Create(h.ctx, domain.Root(), ports.CreateRepoParams{Name: h.name("proj"), Visibility: domain.VisibilityPrivate})
41 require.NoError(t, err)
42 c.wt = h.workTree("main")
43 return c
44}
45
46func instanceScope() *pb.SettingsScope {
47 return &pb.SettingsScope{Where: &pb.SettingsScope_Instance{Instance: true}}
48}
49func repoScope(r string) *pb.SettingsScope {
50 return &pb.SettingsScope{Where: &pb.SettingsScope_Repo{Repo: r}}
51}
52func envScope(r, e string) *pb.SettingsScope {
53 return &pb.SettingsScope{Where: &pb.SettingsScope_Environment{Environment: &pb.EnvironmentRef{Repo: r, Name: e}}}
54}
55
56func (c *ciHarness) setSecret(sc *pb.SettingsScope, name, value string) {
57 c.t.Helper()
58 _, err := c.root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: sc, Name: name, Value: value}))
59 require.NoError(c.t, err)
60}
61
62func (c *ciHarness) setVar(sc *pb.SettingsScope, name, value string) {
63 c.t.Helper()
64 _, err := c.root.vars.SetVariable(c.ctx, connect.NewRequest(&pb.SetVariableRequest{Scope: sc, Name: name, Value: value}))
65 require.NoError(c.t, err)
66}
67
68const settingsWorkflow = `name: s
69on:
70 push:
71 branches: ['**']
72env:
73 FROM_WORKFLOW: ${{ secrets.WF_SECRET }}
74jobs:
75 plain:
76 runs-on: ubuntu-latest
77 steps:
78 - run: echo ${{ vars.GREETING }}
79 uses-one:
80 runs-on: ubuntu-latest
81 steps:
82 - run: echo "$TOKEN"
83 env:
84 TOKEN: ${{ secrets.api_token }}
85 OTHER: ${{ secrets['Bracket_One'] }}
86 uses-all:
87 runs-on: ubuntu-latest
88 steps:
89 - run: echo ${{ toJSON(secrets) }}
90 deploy:
91 runs-on: ubuntu-latest
92 environment: Production
93 steps:
94 - run: echo ${{ secrets.ENV_ONLY }} ${{ vars.GREETING }}
95`
96
97// jobsByKey fetches jobs from the runner until it holds one per key.
98func (r *fakeRunner) jobsByKey(want int) map[string]*pb.Job {
99 r.c.t.Helper()
100 out := map[string]*pb.Job{}
101 for len(out) < want {
102 j := r.await()
103 out[j.GetJobKey()] = j
104 }
105 return out
106}
107
108func secretNames(j *pb.Job) []string {
109 var out []string
110 for k := range j.GetSecrets() {
111 out = append(out, k)
112 }
113 return out
114}
115
116// AC1/AC2: what a job receives. The owner's run gets the secrets its job
117// refers to and no others, variables merged with environment over repository
118// over instance, and the environment is created on first use.
119func TestCIJobReceivesOnlyTheSecretsItRefersTo(t *testing.T) {
120 c := newSecretsHarness(t, secretsKeyA)
121 runner := c.registerRunner("alpha", "ubuntu-latest")
122
123 c.setSecret(instanceScope(), "api_token", "instance-token")
124 c.setSecret(repoScope("proj"), "API_TOKEN", "repo-token")
125 c.setSecret(repoScope("proj"), "bracket_one", "bracket-value")
126 c.setSecret(repoScope("proj"), "WF_SECRET", "workflow-level-secret")
127 c.setSecret(repoScope("proj"), "UNUSED", "never-sent")
128 c.setSecret(instanceScope(), "ENV_ONLY", "instance-env-only")
129 c.setVar(instanceScope(), "GREETING", "from-instance")
130 c.setVar(instanceScope(), "ONLY_INSTANCE", "i")
131 c.setVar(repoScope("proj"), "GREETING", "from-repo")
132
133 c.push("main", map[string]string{".github/workflows/s.yml": settingsWorkflow}, "")
134 jobs := runner.jobsByKey(4)
135
136 // A job that refers to no secret gets none, and the workflow-level env
137 // counts for every job.
138 assert.ElementsMatch(t, []string{"WF_SECRET"}, secretNames(jobs["plain"]))
139 assert.Equal(t, "workflow-level-secret", jobs["plain"].GetSecrets()["WF_SECRET"])
140
141 // secrets.NAME and secrets['NAME'], case-insensitive, repository over instance.
142 one := jobs["uses-one"].GetSecrets()
143 assert.ElementsMatch(t, []string{"API_TOKEN", "BRACKET_ONE", "WF_SECRET"}, secretNames(jobs["uses-one"]))
144 assert.Equal(t, "repo-token", one["API_TOKEN"], "repository over instance")
145 assert.Equal(t, "bracket-value", one["BRACKET_ONE"])
146 assert.NotContains(t, one, "UNUSED")
147
148 // toJSON(secrets) is any other use: every secret of the scopes.
149 assert.ElementsMatch(t, []string{"API_TOKEN", "BRACKET_ONE", "WF_SECRET", "UNUSED", "ENV_ONLY"}, secretNames(jobs["uses-all"]))
150
151 // The serialized Job holds only the values that were sent: the unused
152 // secret is nowhere in the bytes, and no job but the one that reads
153 // everything holds it.
154 for key, j := range jobs {
155 raw, err := proto.Marshal(j)
156 require.NoError(t, err)
157 if key == "uses-all" {
158 assert.Contains(t, string(raw), "never-sent")
159 continue
160 }
161 assert.NotContains(t, string(raw), "never-sent", key)
162 assert.NotContains(t, string(raw), "instance-token", key)
163 }
164
165 // Variables are merged, and the same values are in the vars context.
166 for key, j := range jobs {
167 want := "from-repo"
168 if key == "deploy" {
169 assert.Equal(t, "Production", j.GetEnvironment())
170 } else {
171 assert.Empty(t, j.GetEnvironment(), key)
172 }
173 assert.Equal(t, want, j.GetVars()["GREETING"], key)
174 assert.Equal(t, "i", j.GetVars()["ONLY_INSTANCE"], key)
175 assert.Contains(t, j.GetContexts(), `"GREETING":"from-repo"`, key)
176 assert.True(t, j.GetTrusted(), key)
177 }
178
179 // The environment was created by its first use, and a job that names one
180 // reads the secrets and variables set in it afterwards.
181 envs, err := c.root.env.ListEnvironments(c.ctx, connect.NewRequest(&pb.ListEnvironmentsRequest{Repo: "proj"}))
182 require.NoError(t, err)
183 require.Len(t, envs.Msg.GetEnvironments(), 1)
184 assert.Equal(t, "Production", envs.Msg.GetEnvironments()[0].GetName())
185}
186
187func TestCIEnvironmentOverridesVariablesAndSecrets(t *testing.T) {
188 c := newSecretsHarness(t, secretsKeyA)
189 runner := c.registerRunner("alpha", "ubuntu-latest")
190 _, err := c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "production"}))
191 require.NoError(t, err)
192
193 c.setSecret(instanceScope(), "ENV_ONLY", "instance")
194 c.setSecret(repoScope("proj"), "ENV_ONLY", "repo")
195 c.setSecret(envScope("proj", "PRODUCTION"), "ENV_ONLY", "environment")
196 c.setVar(instanceScope(), "GREETING", "instance")
197 c.setVar(repoScope("proj"), "GREETING", "repo")
198 c.setVar(envScope("proj", "Production"), "GREETING", "environment")
199
200 c.push("main", map[string]string{".github/workflows/s.yml": settingsWorkflow}, "")
201 jobs := runner.jobsByKey(4)
202 deploy := jobs["deploy"]
203 assert.Equal(t, "environment", deploy.GetSecrets()["ENV_ONLY"], "environment over repository over instance")
204 assert.Equal(t, "environment", deploy.GetVars()["GREETING"])
205 assert.Equal(t, "repo", jobs["plain"].GetVars()["GREETING"], "a job without the environment does not see its variables")
206 assert.NotContains(t, jobs["plain"].GetSecrets(), "ENV_ONLY")
207 assert.Contains(t, deploy.GetContexts(), `"GREETING":"environment"`)
208 envs, err := c.root.env.ListEnvironments(c.ctx, connect.NewRequest(&pb.ListEnvironmentsRequest{Repo: "proj"}))
209 require.NoError(t, err)
210 assert.Len(t, envs.Msg.GetEnvironments(), 1, "an existing environment is not created again, whatever the case")
211}
212
213// D1 / AC1: the run of a token limited to branch prefixes is not trusted: no
214// secret, and an owner who runs it again does not unlock them. An owner's run
215// that an agent runs again loses them too.
216func TestCIUntrustedRunsGetNoSecretsAndRerunsNeverRaiseTrust(t *testing.T) {
217 c := newSecretsHarness(t, secretsKeyA)
218 runner := c.registerRunner("alpha", "ubuntu-latest")
219 c.setSecret(repoScope("proj"), "API_TOKEN", "repo-token")
220 c.setSecret(repoScope("proj"), "WF_SECRET", "wf")
221 c.setVar(repoScope("proj"), "GREETING", "hello")
222
223 agent, _ := c.branchToken(scopes(domain.ScopeRepoRead, domain.ScopeRepoWrite, domain.ScopeCIWrite), []string{"agent/x/"}, "proj")
224 agentAPI := c.api(agent)
225
226 // The owner's push: trusted.
227 c.push("main", map[string]string{".github/workflows/s.yml": settingsWorkflow}, "")
228 for _, j := range runner.jobsByKey(4) {
229 assert.True(t, j.GetTrusted())
230 assert.NotEmpty(t, j.GetSecrets(), j.GetJobKey())
231 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil)
232 }
233 c.waitFor("main done", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
234 assert.True(t, c.runs()[0].GetTrusted(), "Run.trusted")
235 mainRun := c.runs()[0]
236
237 // The agent's push: not trusted. It still gets the variables.
238 c.pushAs(agent, "agent/x/a", map[string]string{"f.txt": "x"})
239 c.waitFor("the agent run", func() bool { return len(c.runs()) == 2 })
240 var agentRun *pb.Run
241 for _, r := range c.runs() {
242 if r.GetRef() == "refs/heads/agent/x/a" {
243 agentRun = r
244 }
245 }
246 require.NotNil(t, agentRun)
247 assert.False(t, agentRun.GetTrusted())
248 for _, j := range runner.jobsByKey(4) {
249 assert.False(t, j.GetTrusted(), j.GetJobKey())
250 assert.Empty(t, j.GetSecrets(), "no secret for an agent's run: "+j.GetJobKey())
251 raw, err := proto.Marshal(j)
252 require.NoError(t, err)
253 assert.NotContains(t, string(raw), "repo-token", "untrusted: no secret value anywhere in the job")
254 assert.Equal(t, "hello", j.GetVars()["GREETING"], "variables are not secret")
255 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil)
256 }
257 c.waitFor("agent run done", func() bool { return c.getRun(agentRun.GetId()).GetRun().GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
258
259 // The owner re-runs the agent's run: still not trusted.
260 _, err := c.root.run.RerunRun(c.ctx, connect.NewRequest(&pb.RerunRunRequest{Repo: "proj", RunId: agentRun.GetId()}))
261 require.NoError(t, err)
262 assert.False(t, c.getRun(agentRun.GetId()).GetRun().GetTrusted(), "a re-run never raises trust")
263 for _, j := range runner.jobsByKey(4) {
264 assert.Empty(t, j.GetSecrets(), "the owner re-running an agent's run does not unlock its secrets: "+j.GetJobKey())
265 assert.False(t, j.GetTrusted())
266 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil)
267 }
268
269 // An agent re-running the owner's run on its branch: only the branches under
270 // its prefix are allowed, so a re-run of main is refused, and the trusted
271 // flag of main is untouched.
272 _, err = agentAPI.run.RerunRun(c.ctx, connect.NewRequest(&pb.RerunRunRequest{Repo: "proj", RunId: mainRun.GetId()}))
273 assert.Equal(t, connect.CodePermissionDenied, connect.CodeOf(err))
274 assert.True(t, c.getRun(mainRun.GetId()).GetRun().GetTrusted())
275
276 // The owner re-running their own run keeps it.
277 _, err = c.root.run.RerunRun(c.ctx, connect.NewRequest(&pb.RerunRunRequest{Repo: "proj", RunId: mainRun.GetId()}))
278 require.NoError(t, err)
279 assert.True(t, c.getRun(mainRun.GetId()).GetRun().GetTrusted())
280 for _, j := range runner.jobsByKey(4) {
281 assert.NotEmpty(t, j.GetSecrets(), j.GetJobKey())
282 runner.runJob(j, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil)
283 }
284}
285
286// Without ERMITE_SECRETS_KEY secrets cannot be set (failed_precondition) and
287// jobs get none; variables and environments work.
288func TestCIWithoutASecretsKey(t *testing.T) {
289 c := newSecretsHarness(t, "")
290 runner := c.registerRunner("alpha", "ubuntu-latest")
291 _, err := c.root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "A", Value: "v"}))
292 assert.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err))
293 c.setVar(repoScope("proj"), "GREETING", "still works")
294 _, err = c.root.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: repoScope("proj")}))
295 require.NoError(t, err, "names can be listed")
296
297 c.push("main", map[string]string{".github/workflows/s.yml": settingsWorkflow}, "")
298 for _, j := range runner.jobsByKey(4) {
299 assert.Empty(t, j.GetSecrets(), j.GetJobKey())
300 assert.Equal(t, "still works", j.GetVars()["GREETING"])
301 }
302}
303
304// The server refuses to start with another key than the one the secrets were
305// sealed with, and starts again with the right one or after they are gone.
306func TestAnotherSecretsKeyRefusesToStart(t *testing.T) {
307 c := newSecretsHarness(t, secretsKeyA)
308 c.setSecret(repoScope("proj"), "A", "value")
309
310 cfg := c.forge.cfg
311 c.srv.Close()
312 c.forge.Close(c.ctx)
313
314 other, err := base64.StdEncoding.DecodeString(secretsKeyB)
315 require.NoError(t, err)
316 bad := cfg
317 bad.SecretsKey = other
318 _, err = newForge(c.ctx, bad, options{hookExe: binary(t)})
319 require.ErrorIs(t, err, domain.ErrSecretsKeyMismatch)
320 assert.NotContains(t, err.Error(), secretsKeyB)
321 assert.NotContains(t, err.Error(), secretsKeyA)
322
323 f, err := newForge(c.ctx, cfg, options{hookExe: binary(t)})
324 require.NoError(t, err, "the right key starts")
325 srv := httptest.NewServer(f.server.Handler())
326 t.Cleanup(func() { srv.Close(); f.Close(context.WithoutCancel(c.ctx)) })
327 list, err := newClients(srv.URL+"/-/api", adminToken).secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: repoScope("proj")}))
328 require.NoError(t, err)
329 assert.Len(t, list.Msg.GetSecrets(), 1)
330}
331
332// Secret values never appear in a response of the settings services, in the
333// ermite log, or in the run record.
334func TestCISecretValuesAreNeverReturnedOrLogged(t *testing.T) {
335 c := newSecretsHarness(t, secretsKeyA)
336 runner := c.registerRunner("alpha", "ubuntu-latest")
337 const value = "SUPER-SECRET-VALUE-0123456789"
338 c.setSecret(instanceScope(), "I", value+"-i")
339 c.setSecret(repoScope("proj"), "R", value+"-r")
340 _, err := c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "prod"}))
341 require.NoError(t, err)
342 c.setSecret(envScope("proj", "prod"), "E", value+"-e")
343
344 var seen []proto.Message
345 set, err := c.root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "r", Value: value + "-r2"}))
346 require.NoError(t, err)
347 seen = append(seen, set.Msg)
348 for _, sc := range []*pb.SettingsScope{instanceScope(), repoScope("proj"), envScope("proj", "PROD")} {
349 list, err := c.root.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: sc}))
350 require.NoError(t, err)
351 require.NotEmpty(t, list.Msg.GetSecrets())
352 seen = append(seen, list.Msg)
353 for _, s := range list.Msg.GetSecrets() {
354 assert.Equal(t, strings.ToUpper(s.GetName()), s.GetName(), "names are stored in upper case")
355 assert.NotNil(t, s.GetUpdatedAt())
356 }
357 }
358 for _, m := range seen {
359 assert.NotContains(t, protojson.Format(m), value)
360 }
361
362 // An invalid request echoes no value either.
363 _, err = c.root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "GITHUB_X", Value: value}))
364 require.Error(t, err)
365 assert.NotContains(t, err.Error(), value)
366
367 c.push("main", map[string]string{".github/workflows/s.yml": "on: push\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo ${{ secrets.R }}\n"}, "")
368 job := runner.await()
369 assert.Equal(t, value+"-r2", job.GetSecrets()["R"])
370 runner.runJob(job, pb.CiConclusion_CI_CONCLUSION_SUCCESS, nil)
371 c.waitFor("done", func() bool { return len(c.runs()) == 1 && c.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
372
373 // Nothing the owner can read about the run holds the value.
374 run := c.getRun(c.runs()[0].GetId())
375 assert.NotContains(t, protojson.Format(run), value)
376 src, err := c.forge.store.CI().GetRunSource(c.ctx, c.runs()[0].GetId())
377 require.NoError(t, err)
378 assert.NotContains(t, src.GithubJSON, value)
379 assert.NotContains(t, src.WorkflowYAML, value)
380 jobs, err := c.forge.store.CI().ListJobs(c.ctx, c.runs()[0].GetId(), 1)
381 require.NoError(t, err)
382 for _, j := range jobs {
383 assert.NotContains(t, j.ContextsJSON, value)
384 }
385 assert.NotContains(t, c.logs.String(), value, "the ermite log")
386}
387
388// Branch-limited tokens never manage secrets, whatever their scopes (ADR 0027);
389// a token needs secrets:write on the repository, and the instance scope needs
390// one that is not limited to repositories.
391func TestSettingsServicesAccessRules(t *testing.T) {
392 c := newSecretsHarness(t, secretsKeyA)
393 _, err := c.forge.Repos.Create(c.ctx, domain.Root(), ports.CreateRepoParams{Name: c.name("other"), Visibility: domain.VisibilityPublic})
394 require.NoError(t, err)
395
396 limited, _ := c.branchToken(scopes(domain.ScopeSecretsWrite, domain.ScopeRepoRead), []string{"agent/"})
397 reader := c.token(scopes(domain.ScopeRepoRead), "proj")
398 writer := c.token(scopes(domain.ScopeSecretsWrite, domain.ScopeRepoRead), "proj")
399 writerAll := c.token(scopes(domain.ScopeSecretsWrite))
400
401 code := func(secret string, sc *pb.SettingsScope) (set, list, vset, vget, del connect.Code) {
402 api := c.api(secret)
403 codeOf := func(err error) connect.Code {
404 if err == nil {
405 return 0 // ok
406 }
407 return connect.CodeOf(err)
408 }
409 _, err := api.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: sc, Name: "A", Value: "v"}))
410 set = codeOf(err)
411 _, err = api.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: sc}))
412 list = codeOf(err)
413 _, err = api.vars.SetVariable(c.ctx, connect.NewRequest(&pb.SetVariableRequest{Scope: sc, Name: "A", Value: "v"}))
414 vset = codeOf(err)
415 _, err = api.vars.GetVariable(c.ctx, connect.NewRequest(&pb.GetVariableRequest{Scope: sc, Name: "A"}))
416 vget = codeOf(err)
417 _, err = api.secret.DeleteSecret(c.ctx, connect.NewRequest(&pb.DeleteSecretRequest{Scope: sc, Name: "ZZ"}))
418 del = codeOf(err)
419 return
420 }
421 allOf := func(want connect.Code, got ...connect.Code) {
422 t.Helper()
423 for i, g := range got {
424 assert.Equal(t, want, g, "call %d", i)
425 }
426 }
427
428 // A branch-limited token with secrets:write is refused everywhere.
429 for _, sc := range []*pb.SettingsScope{repoScope("proj"), instanceScope(), envScope("proj", "x")} {
430 a, b, d, e, f := code(limited, sc)
431 allOf(connect.CodePermissionDenied, a, b, d, e, f)
432 }
433 _, err = c.api(limited).env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "x"}))
434 assert.Equal(t, connect.CodePermissionDenied, connect.CodeOf(err))
435
436 // Without secrets:write, a repository the token reads: permission denied.
437 a, b, d, e, f := code(reader, repoScope("proj"))
438 allOf(connect.CodePermissionDenied, a, b, d, e, f)
439 // Anonymous.
440 a, b, d, e, f = code("", repoScope("proj"))
441 allOf(connect.CodeUnauthenticated, a, b, d, e, f)
442 // With it: allowed (the delete of a missing secret is not found).
443 a, b, d, e, f = code(writer, repoScope("proj"))
444 assert.Equal(t, []connect.Code{0, 0, 0, 0, connect.CodeNotFound}, []connect.Code{a, b, d, e, f})
445 // The writer is limited to one repository: not the instance, not another one.
446 a, b, d, e, f = code(writer, instanceScope())
447 allOf(connect.CodePermissionDenied, a, b, d, e, f)
448 a, b, d, e, f = code(writer, repoScope("other"))
449 allOf(connect.CodePermissionDenied, a, b, d, e, f) // can read it (public), may not write there
450 // Not restricted to repositories: the instance scope works.
451 a, b, d, e, f = code(writerAll, instanceScope())
452 assert.Equal(t, []connect.Code{0, 0, 0, 0, connect.CodeNotFound}, []connect.Code{a, b, d, e, f})
453
454 // A repository the caller cannot see is not found.
455 _, err = c.forge.Repos.Create(c.ctx, domain.Root(), ports.CreateRepoParams{Name: c.name("hidden"), Visibility: domain.VisibilityPrivate})
456 require.NoError(t, err)
457 a, b, d, e, f = code(writer, repoScope("hidden"))
458 allOf(connect.CodeNotFound, a, b, d, e, f)
459 a, _, _, _, _ = code(writer, repoScope("ghost"))
460 assert.Equal(t, connect.CodeNotFound, a)
461
462 // Validation.
463 root := c.root
464 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "1BAD", Value: "v"}))
465 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
466 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "github_token", Value: "v"}))
467 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
468 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "OK", Value: ""}))
469 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), "an empty secret")
470 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "OK", Value: strings.Repeat("x", 49153)}))
471 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), "over 48 KiB")
472 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: repoScope("proj"), Name: "OK", Value: strings.Repeat("x", 49152)}))
473 assert.NoError(t, err, "exactly 48 KiB")
474 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Name: "OK", Value: "v"}))
475 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), "no scope")
476 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: &pb.SettingsScope{Where: &pb.SettingsScope_Instance{}}, Name: "OK", Value: "v"}))
477 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), "instance: false")
478 _, err = root.vars.SetVariable(c.ctx, connect.NewRequest(&pb.SetVariableRequest{Scope: repoScope("proj"), Name: "EMPTY", Value: ""}))
479 assert.NoError(t, err, "an empty variable is allowed")
480 _, err = root.secret.SetSecret(c.ctx, connect.NewRequest(&pb.SetSecretRequest{Scope: envScope("proj", "missing"), Name: "OK", Value: "v"}))
481 assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err), "an environment that does not exist")
482 _, err = root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "bad\nname"}))
483 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
484 _, err = root.env.DeleteEnvironment(c.ctx, connect.NewRequest(&pb.DeleteEnvironmentRequest{Repo: "proj", Name: "missing"}))
485 assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err))
486}
487
488func TestSettingsServicesPagingAndCrud(t *testing.T) {
489 c := newSecretsHarness(t, secretsKeyA)
490 for _, n := range []string{"c", "a", "b"} {
491 c.setSecret(repoScope("proj"), n, "v")
492 c.setVar(repoScope("proj"), n, "val-"+n)
493 }
494 var names []string
495 token := ""
496 for range 5 {
497 res, err := c.root.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: repoScope("proj"), PageSize: 2, PageToken: token}))
498 require.NoError(t, err)
499 for _, s := range res.Msg.GetSecrets() {
500 names = append(names, s.GetName())
501 }
502 if token = res.Msg.GetNextPageToken(); token == "" {
503 break
504 }
505 }
506 assert.Equal(t, []string{"A", "B", "C"}, names)
507
508 v, err := c.root.vars.GetVariable(c.ctx, connect.NewRequest(&pb.GetVariableRequest{Scope: repoScope("proj"), Name: "b"}))
509 require.NoError(t, err)
510 assert.Equal(t, "val-b", v.Msg.GetVariable().GetValue())
511 vl, err := c.root.vars.ListVariables(c.ctx, connect.NewRequest(&pb.ListVariablesRequest{Scope: repoScope("proj")}))
512 require.NoError(t, err)
513 require.Len(t, vl.Msg.GetVariables(), 3)
514 assert.Equal(t, "A", vl.Msg.GetVariables()[0].GetName())
515 _, err = c.root.vars.DeleteVariable(c.ctx, connect.NewRequest(&pb.DeleteVariableRequest{Scope: repoScope("proj"), Name: "A"}))
516 require.NoError(t, err)
517 _, err = c.root.vars.GetVariable(c.ctx, connect.NewRequest(&pb.GetVariableRequest{Scope: repoScope("proj"), Name: "a"}))
518 assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err))
519 _, err = c.root.secret.DeleteSecret(c.ctx, connect.NewRequest(&pb.DeleteSecretRequest{Scope: repoScope("proj"), Name: "b"}))
520 require.NoError(t, err)
521 _, err = c.root.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: repoScope("proj"), PageToken: "garbage"}))
522 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
523
524 // Environments: create twice (any case), list, delete with their content.
525 e1, err := c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "Staging"}))
526 require.NoError(t, err)
527 e2, err := c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "STAGING"}))
528 require.NoError(t, err)
529 assert.Equal(t, "Staging", e2.Msg.GetEnvironment().GetName(), "the case it was given is kept")
530 assert.Equal(t, e1.Msg.GetEnvironment().GetCreatedAt().AsTime(), e2.Msg.GetEnvironment().GetCreatedAt().AsTime())
531 c.setSecret(envScope("proj", "staging"), "S", "v")
532 _, err = c.root.env.DeleteEnvironment(c.ctx, connect.NewRequest(&pb.DeleteEnvironmentRequest{Repo: "proj", Name: "staging"}))
533 require.NoError(t, err)
534 _, err = c.root.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: envScope("proj", "Staging")}))
535 assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err))
536
537 // Deleting the repository removes its scopes.
538 require.NoError(t, c.forge.Repos.Delete(c.ctx, domain.Root(), c.name("proj")))
539 _, err = c.forge.Repos.Create(c.ctx, domain.Root(), ports.CreateRepoParams{Name: c.name("proj"), Visibility: domain.VisibilityPrivate})
540 require.NoError(t, err)
541 res, err := c.root.secret.ListSecrets(c.ctx, connect.NewRequest(&pb.ListSecretsRequest{Scope: repoScope("proj")}))
542 require.NoError(t, err)
543 assert.Empty(t, res.Msg.GetSecrets())
544 vl, err = c.root.vars.ListVariables(c.ctx, connect.NewRequest(&pb.ListVariablesRequest{Scope: repoScope("proj")}))
545 require.NoError(t, err)
546 assert.Empty(t, vl.Msg.GetVariables())
547}
548
549// environment: as a mapping (the url is ignored) and as an expression of the
550// matrix; an untrusted run does not add environments to the repository.
551func TestCIEnvironmentForms(t *testing.T) {
552 c := newSecretsHarness(t, secretsKeyA)
553 runner := c.registerRunner("alpha", "ubuntu-latest")
554 _, err := c.root.vars.SetVariable(c.ctx, connect.NewRequest(&pb.SetVariableRequest{Scope: envScope("proj", "alpha"), Name: "V", Value: "x"}))
555 require.Equal(t, connect.CodeNotFound, connect.CodeOf(err), "not an environment yet")
556 _, err = c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: "Alpha"}))
557 require.NoError(t, err)
558 c.setVar(envScope("proj", "alpha"), "V", "from-alpha")
559
560 c.push("main", map[string]string{".github/workflows/e.yml": `on: push
561jobs:
562 mapping:
563 runs-on: ubuntu-latest
564 environment:
565 name: alpha
566 url: https://example.test
567 steps:
568 - run: echo
569 matrix:
570 runs-on: ubuntu-latest
571 strategy:
572 matrix:
573 env: [Alpha, beta]
574 environment: ${{ matrix.env }}
575 steps:
576 - run: echo
577`}, "")
578 jobs := map[string]*pb.Job{}
579 for len(jobs) < 3 {
580 j := runner.await()
581 key := j.GetJobKey() + "/" + j.GetEnvironment()
582 jobs[key] = j
583 }
584 assert.Equal(t, "from-alpha", jobs["mapping/alpha"].GetVars()["V"])
585 assert.Equal(t, "from-alpha", jobs["matrix/Alpha"].GetVars()["V"])
586 assert.NotContains(t, jobs["matrix/beta"].GetVars(), "V")
587
588 names := func() []string {
589 res, err := c.root.env.ListEnvironments(c.ctx, connect.NewRequest(&pb.ListEnvironmentsRequest{Repo: "proj"}))
590 require.NoError(t, err)
591 var out []string
592 for _, e := range res.Msg.GetEnvironments() {
593 out = append(out, e.GetName())
594 }
595 return out
596 }
597 assert.Equal(t, []string{"Alpha", "beta"}, names(), "beta was created on first use; alpha already existed")
598
599 // An agent's run names a new environment: it is not created.
600 agent, _ := c.branchToken(scopes(domain.ScopeRepoRead, domain.ScopeRepoWrite), []string{"agent/x/"}, "proj")
601 c.pushAs(agent, "agent/x/a", map[string]string{".github/workflows/e2.yml": "on: push\njobs:\n j:\n runs-on: ubuntu-latest\n environment: agent-made\n steps:\n - run: echo\n"})
602 var seen *pb.Job
603 for seen == nil || seen.GetEnvironment() != "agent-made" {
604 seen = runner.await()
605 }
606 assert.False(t, seen.GetTrusted())
607 assert.Equal(t, []string{"Alpha", "beta"}, names())
608}
609
610// flakySettings fails the next n resolutions, then works again.
611type flakySettings struct {
612 inner ports.CIJobSettings
613 fail atomic.Int32
614 // err is what a failure returns; nil is a plain error.
615 err error
616}
617
618func (f *flakySettings) Resolve(ctx context.Context, req ports.JobSettingsRequest) (ports.JobSettings, error) {
619 if f.fail.Add(-1) >= 0 {
620 if f.err != nil {
621 return ports.JobSettings{}, f.err
622 }
623 return ports.JobSettings{}, errors.New("injected failure")
624 }
625 f.fail.Store(0)
626 return f.inner.Resolve(ctx, req)
627}
628
629func newFlakyHarness(t *testing.T) (*ciHarness, *flakySettings) {
630 t.Helper()
631 flaky := &flakySettings{}
632 h := newHarnessWith(t, harnessOpts{env: map[string]string{config.EnvSecretsKey: secretsKeyA}}, func(o *options) {
633 o.ci = func(cfg *usecase.CIConfig) { flaky.inner = cfg.Settings; cfg.Settings = flaky }
634 })
635 c := &ciHarness{harness: h, root: h.api(adminToken), repo: "proj"}
636 _, err := h.forge.Repos.Create(h.ctx, domain.Root(), ports.CreateRepoParams{Name: h.name("proj"), Visibility: domain.VisibilityPrivate})
637 require.NoError(t, err)
638 c.wt = h.workTree("main")
639 return c, flaky
640}
641
642// jobRecord reads the stored record of the only job.
643func (c *ciHarness) jobRecord(runID int64) domain.CIJob {
644 c.t.Helper()
645 jobs, err := c.forge.store.CI().ListJobs(c.ctx, runID, 1)
646 require.NoError(c.t, err)
647 require.Len(c.t, jobs, 1)
648 return jobs[0]
649}
650
651const oneSecretJob = "on: push\njobs:\n j:\n runs-on: ubuntu-latest\n steps:\n - run: echo ${{ secrets.A }}\n"
652
653// A settings resolver that fails at the assignment leaves the job ready: no
654// write, no token; the runner's retry gets it. The same holds for the repeat of
655// a poll, which reads the settings again.
656func TestCIAssignmentSurvivesASettingsFailure(t *testing.T) {
657 c, flaky := newFlakyHarness(t)
658 runner := c.registerRunner("alpha", "ubuntu-latest")
659 c.setSecret(repoScope("proj"), "A", "value")
660 c.push("main", map[string]string{".github/workflows/s.yml": oneSecretJob}, "")
661 c.waitFor("a ready job", func() bool {
662 ready, err := c.forge.store.CI().ListReadyJobs(c.ctx, 0)
663 return err == nil && len(ready) == 1
664 })
665 runID := c.runs()[0].GetId()
666 before := c.jobRecord(runID)
667 require.Equal(t, domain.CIPhaseReady, before.Phase)
668
669 flaky.fail.Store(1)
670 _, err := runner.fetch("")
671 require.Error(t, err, "the poll fails instead of losing the job")
672 after := c.jobRecord(runID)
673 assert.Equal(t, domain.CIPhaseReady, after.Phase)
674 assert.Equal(t, before.Version, after.Version, "no assignment write")
675 assert.False(t, after.HasToken(), "no job token")
676 ready, err := c.forge.store.CI().ListReadyJobs(c.ctx, 0)
677 require.NoError(t, err)
678 assert.Len(t, ready, 1, "still in the queue")
679 assert.Zero(t, c.repoTokens(), "no repository token issued")
680
681 // The retry works.
682 job, err := runner.fetch("poll-1")
683 require.NoError(t, err)
684 require.NotNil(t, job)
685 assert.Equal(t, "value", job.GetSecrets()["A"])
686 assigned := c.jobRecord(runID)
687 assert.Equal(t, domain.CIPhaseAssigned, assigned.Phase)
688 assert.Equal(t, 1, c.repoTokens())
689
690 // A repeat of the poll reads the settings again: a failure is an error, the
691 // assignment stays, and the next repeat returns the very same job.
692 flaky.fail.Store(1)
693 _, err = runner.fetch("poll-1")
694 require.Error(t, err)
695 same := c.jobRecord(runID)
696 assert.Equal(t, assigned.Version, same.Version)
697 assert.Equal(t, assigned.TokenHash, same.TokenHash)
698 assert.Equal(t, 1, c.repoTokens())
699 again, err := runner.fetch("poll-1")
700 require.NoError(t, err)
701 require.NotNil(t, again)
702 assert.Equal(t, job.GetJobToken(), again.GetJobToken())
703 assert.Equal(t, job.GetGithubToken(), again.GetGithubToken())
704 assert.Equal(t, "value", again.GetSecrets()["A"])
705}
706
707// repoTokens counts the job repository tokens stored for the repository.
708func (c *ciHarness) repoTokens() int {
709 c.t.Helper()
710 n, err := c.forge.store.CI().CountRepoTokens(c.ctx, c.name("proj"))
711 require.NoError(c.t, err)
712 return n
713}
714
715const varsEverywhereWorkflow = `name: v
716run-name: run ${{ vars.RUN }}
717on: push
718jobs:
719 j:
720 if: ${{ vars.GATE == 'on' }}
721 runs-on: ${{ vars.LABEL }}
722 strategy:
723 matrix:
724 n: ${{ fromJSON(vars.LEGS) }}
725 environment: ${{ vars.ENVNAME }}
726 steps:
727 - run: echo ${{ vars.LATE }}
728`
729
730// Variables are read where the workflow reads them: in run-name, a job's if,
731// strategy, runs-on and environment name, when the job is planned; and again
732// when the job is assigned, without planning it again.
733func TestCIVarsInPlanningAndAtAssignment(t *testing.T) {
734 c := newSecretsHarness(t, secretsKeyA)
735 runner := c.registerRunner("alpha", "ubuntu-latest")
736 for k, v := range map[string]string{"RUN": "first", "GATE": "on", "LABEL": "ubuntu-latest", "LEGS": "[1, 2]", "ENVNAME": "Prod", "LATE": "before"} {
737 c.setVar(repoScope("proj"), k, v)
738 }
739 c.push("main", map[string]string{".github/workflows/v.yml": varsEverywhereWorkflow}, "")
740 c.waitFor("two ready legs", func() bool {
741 ready, err := c.forge.store.CI().ListReadyJobs(c.ctx, 0)
742 return err == nil && len(ready) == 2
743 })
744 assert.Equal(t, "run first", c.runs()[0].GetName(), "run-name")
745
746 // Change variables after the jobs are ready and before they are fetched.
747 c.setVar(repoScope("proj"), "LATE", "after")
748 c.setVar(repoScope("proj"), "GATE", "off")
749 c.setVar(repoScope("proj"), "LABEL", "gpu")
750 c.setVar(repoScope("proj"), "LEGS", "[1, 2, 3, 4]")
751 c.setVar(repoScope("proj"), "RUN", "second")
752
753 got := map[string]*pb.Job{}
754 for len(got) < 2 {
755 j := runner.await()
756 got[j.GetJobId()] = j
757 }
758 assert.Len(t, got, 2, "the strategy is not planned again: two legs, not four")
759 for _, j := range got {
760 assert.Equal(t, "after", j.GetVars()["LATE"], "the assignment sees the new value")
761 assert.Contains(t, j.GetContexts(), `"LATE":"after"`)
762 assert.Contains(t, j.GetContexts(), `"GATE":"off"`, "all the variables are current")
763 assert.Equal(t, []string{"ubuntu-latest"}, j.GetLabels(), "runs-on is the planned one")
764 assert.Equal(t, "Prod", j.GetEnvironment())
765 }
766 assert.Equal(t, "run first", c.runs()[0].GetName(), "the run is not renamed")
767 // The environment named by a variable is created on first use, at assignment.
768 envs, err := c.root.env.ListEnvironments(c.ctx, connect.NewRequest(&pb.ListEnvironmentsRequest{Repo: "proj"}))
769 require.NoError(t, err)
770 require.Len(t, envs.Msg.GetEnvironments(), 1)
771 assert.Equal(t, "Prod", envs.Msg.GetEnvironments()[0].GetName(), "environment name")
772}
773
774// One rule for environment names: what the API accepts is what a workflow can
775// name, with the same spelling; a name with surrounding whitespace is refused by
776// the API and fails the job with a clear error instead of being trimmed.
777func TestCIEnvironmentNameRoundTrip(t *testing.T) {
778 c := newSecretsHarness(t, secretsKeyA)
779 runner := c.registerRunner("alpha", "ubuntu-latest")
780 const name = "Prod EU/2"
781 _, err := c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: name}))
782 require.NoError(t, err)
783 c.setVar(envScope("proj", name), "WHERE", "eu")
784 for _, bad := range []string{" Prod", "Prod ", "\tProd", "Prod\u00a0"} {
785 _, err := c.root.env.CreateEnvironment(c.ctx, connect.NewRequest(&pb.CreateEnvironmentRequest{Repo: "proj", Name: bad}))
786 assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err), "%q", bad)
787 }
788
789 c.push("main", map[string]string{
790 ".github/workflows/ok.yml": "on: push\njobs:\n ok:\n runs-on: ubuntu-latest\n environment: '" + name + "'\n steps:\n - run: echo\n",
791 ".github/workflows/bad.yml": "on: push\njobs:\n bad:\n runs-on: ubuntu-latest\n environment: ' Prod EU/2 '\n steps:\n - run: echo\n",
792 }, "")
793 job := runner.await()
794 assert.Equal(t, "ok", job.GetJobKey())
795 assert.Equal(t, name, job.GetEnvironment())
796 assert.Equal(t, "eu", job.GetVars()["WHERE"], "the job reads the environment the API created")
797
798 c.waitFor("the bad job fails", func() bool {
799 for _, r := range c.runs() {
800 if r.GetWorkflowPath() == ".github/workflows/bad.yml" && r.GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED {
801 return true
802 }
803 }
804 return false
805 })
806 for _, r := range c.runs() {
807 if r.GetWorkflowPath() != ".github/workflows/bad.yml" {
808 continue
809 }
810 j := c.getRun(r.GetId()).GetJobs()[0]
811 assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, j.GetConclusion())
812 assert.Contains(t, j.GetError(), "environment name")
813 assert.Contains(t, j.GetError(), "whitespace")
814 }
815 envs, err := c.root.env.ListEnvironments(c.ctx, connect.NewRequest(&pb.ListEnvironmentsRequest{Repo: "proj"}))
816 require.NoError(t, err)
817 assert.Len(t, envs.Msg.GetEnvironments(), 1, "no trimmed copy was created")
818}
819
820// A settings failure that wraps ErrConflict (an environment creation that ran
821// out of retries) or ErrNotFound is not "nothing to repeat" and not "taken
822// meanwhile": the repeated poll fails, and no other job is assigned in its
823// place. Fetching without a poll id fails too and leaves every job ready.
824func TestCIRepeatedPollPropagatesSettingsConflicts(t *testing.T) {
825 for name, injected := range map[string]error{
826 "conflict": fmt.Errorf("create environment: %w", domain.ErrConflict),
827 "not found": fmt.Errorf("environment: %w", domain.ErrNotFound),
828 } {
829 t.Run(name, func(t *testing.T) {
830 c, flaky := newFlakyHarness(t)
831 runner := c.registerRunner("alpha", "ubuntu-latest")
832 c.setSecret(repoScope("proj"), "A", "value")
833 c.push("main", map[string]string{
834 ".github/workflows/one.yml": oneSecretJob,
835 ".github/workflows/two.yml": oneSecretJob,
836 }, "")
837 c.waitFor("two ready jobs", func() bool {
838 ready, err := c.forge.store.CI().ListReadyJobs(c.ctx, 0)
839 return err == nil && len(ready) == 2
840 })
841
842 first, err := runner.fetch("poll-1")
843 require.NoError(t, err)
844 require.NotNil(t, first)
845 ready, err := c.forge.store.CI().ListReadyJobs(c.ctx, 0)
846 require.NoError(t, err)
847 require.Len(t, ready, 1, "the other job waits")
848
849 flaky.err = injected
850 flaky.fail.Store(1)
851 res, err := runner.fetch("poll-1")
852 require.Error(t, err, "the repeat fails")
853 assert.Nil(t, res, "and gives no other job")
854 ready, err = c.forge.store.CI().ListReadyJobs(c.ctx, 0)
855 require.NoError(t, err)
856 assert.Len(t, ready, 1, "the second job was not assigned in its place")
857 assert.Equal(t, 1, c.repoTokens(), "no token was issued for another job")
858
859 // A poll without an id fails the same way and assigns nothing.
860 flaky.fail.Store(1)
861 _, err = runner.fetch("")
862 require.Error(t, err)
863 ready, err = c.forge.store.CI().ListReadyJobs(c.ctx, 0)
864 require.NoError(t, err)
865 assert.Len(t, ready, 1)
866
867 // Once it works again, the repeat is the first job, unchanged.
868 flaky.err = nil
869 again, err := runner.fetch("poll-1")
870 require.NoError(t, err)
871 require.NotNil(t, again)
872 assert.Equal(t, first.GetJobId(), again.GetJobId())
873 assert.Equal(t, first.GetJobToken(), again.GetJobToken())
874 })
875 }
876}