// SPDX-License-Identifier: AGPL-3.0-or-later
package app
import (
"strings"
"testing"
"time"
"connectrpc.com/connect"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/fieldmaskpb"
pb "bnema.dev/ermite-api/gen/ermite"
"bnema.dev/ermite/internal/domain"
)
// CI-2 results (phase E2): annotations and step summaries, from the runner's
// calls to the API, the pages and the CLI.
const resultsWorkflow = `name: ci
on:
push:
branches: [main]
jobs:
build:
runs-on: ubuntu-latest
steps:
- run: echo hello
lint:
runs-on: ubuntu-latest
steps:
- run: echo lint
`
func ann(level pb.AnnotationLevel, step int32, msg string) *pb.Annotation {
return &pb.Annotation{Level: level, Message: msg, StepNumber: step}
}
func (r *fakeRunner) update(j *pb.Job, steps []*pb.StepUpdate, anns ...*pb.Annotation) error {
_, err := r.jobAPI(j).runner.UpdateJob(r.c.ctx, connect.NewRequest(&pb.UpdateJobRequest{JobId: j.GetJobId(), Steps: steps, Annotations: anns}))
return err
}
func (r *fakeRunner) summary(j *pb.Job, step int32, md string) error {
_, err := r.jobAPI(j).runner.UploadSummary(r.c.ctx, connect.NewRequest(&pb.UploadSummaryRequest{JobId: j.GetJobId(), StepNumber: step, Markdown: md}))
return err
}
func (r *fakeRunner) heartbeat(j *pb.Job) {
r.c.t.Helper()
_, err := r.jobAPI(j).runner.Heartbeat(r.c.ctx, connect.NewRequest(&pb.HeartbeatRequest{JobId: j.GetJobId()}))
require.NoError(r.c.t, err)
}
func (c *ciHarness) annotations(runID int64, jobID string) *pb.ListAnnotationsResponse {
c.t.Helper()
res, err := c.root.run.ListAnnotations(c.ctx, connect.NewRequest(&pb.ListAnnotationsRequest{Repo: c.repo, RunId: runID, JobId: jobID}))
require.NoError(c.t, err)
return res.Msg
}
func (c *ciHarness) summaries(jobID string) *pb.GetStepSummariesResponse {
c.t.Helper()
res, err := c.root.run.GetStepSummaries(c.ctx, connect.NewRequest(&pb.GetStepSummariesRequest{Repo: c.repo, JobId: jobID}))
require.NoError(c.t, err)
return res.Msg
}
func code(err error) connect.Code { return connect.CodeOf(err) }
func TestCIAnnotationsAndSummariesOverTheAPI(t *testing.T) {
c := newCIHarness(t, nil)
runner := c.registerRunner("alpha", "ubuntu-latest")
c.push("main", map[string]string{".github/workflows/ci.yml": resultsWorkflow}, "")
c.waitFor("a run", func() bool { return len(c.runs()) == 1 })
runID := c.runs()[0].GetId()
ver, err := c.root.server.GetVersion(c.ctx, connect.NewRequest(&pb.GetVersionRequest{}))
require.NoError(t, err)
assert.Contains(t, ver.Msg.GetFeatures(), "ci.annotations")
assert.Contains(t, ver.Msg.GetFeatures(), "ci.summaries")
a, b := runner.await(), runner.await()
runner.heartbeat(a)
runner.heartbeat(b)
start := now()
running := []*pb.StepUpdate{{Number: 1, Name: "Set up job", State: pb.StepState_STEP_STATE_IN_PROGRESS, StartedAt: start}}
require.NoError(t, runner.update(a, running))
done := &pb.StepUpdate{Number: 1, Name: "Set up job", State: pb.StepState_STEP_STATE_COMPLETED, Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, StartedAt: start, CompletedAt: now()}
notice, warning, failure := pb.AnnotationLevel_ANNOTATION_LEVEL_NOTICE, pb.AnnotationLevel_ANNOTATION_LEVEL_WARNING, pb.AnnotationLevel_ANNOTATION_LEVEL_FAILURE
t.Run("an annotation of a step that UpdateJob never named is stored", func(t *testing.T) {
full := &pb.Annotation{Level: failure, Title: "Vet", Message: "unused variable", File: "main.go", StartLine: 3, EndLine: 4, StartColumn: 2, EndColumn: 8, StepNumber: 7}
require.NoError(t, runner.update(a, nil, full, ann(notice, 7, "second")))
got := c.annotations(runID, a.GetJobId())
require.Len(t, got.GetAnnotations(), 2)
assert.Equal(t, a.GetJobId(), got.GetAnnotations()[0].GetJobId())
assert.EqualValues(t, full.GetStartLine(), got.GetAnnotations()[0].GetAnnotation().GetStartLine())
assert.Equal(t, "Vet", got.GetAnnotations()[0].GetAnnotation().GetTitle())
assert.Equal(t, "second", got.GetAnnotations()[1].GetAnnotation().GetMessage(), "in the order the step produced them")
assert.False(t, got.GetTruncated())
})
t.Run("replacement per step, a retry changes nothing, a step absent is kept", func(t *testing.T) {
require.NoError(t, runner.update(a, nil, ann(warning, 3, "three")))
require.NoError(t, runner.update(a, nil, ann(warning, 3, "three")))
require.NoError(t, runner.update(a, nil, ann(notice, 7, "only one now")))
got := c.annotations(runID, a.GetJobId()).GetAnnotations()
require.Len(t, got, 2, "step 3 is kept, step 7 holds one annotation")
assert.EqualValues(t, 3, got[0].GetAnnotation().GetStepNumber(), "ordered by step")
assert.Equal(t, "only one now", got[1].GetAnnotation().GetMessage())
})
t.Run("a COMPLETED step can still have its annotations replaced", func(t *testing.T) {
require.NoError(t, runner.update(a, []*pb.StepUpdate{done}, ann(notice, 1, "before")))
require.NoError(t, runner.update(a, nil, ann(failure, 1, "after")))
require.NoError(t, runner.update(a, []*pb.StepUpdate{done}), "the same step again changes nothing")
got := c.annotations(runID, a.GetJobId()).GetAnnotations()
assert.Equal(t, "after", got[0].GetAnnotation().GetMessage())
assert.Equal(t, failure, got[0].GetAnnotation().GetLevel())
// The rule that a completed step cannot change still holds for the step.
changed := &pb.StepUpdate{Number: 1, Name: "Set up job", State: pb.StepState_STEP_STATE_COMPLETED, Conclusion: pb.CiConclusion_CI_CONCLUSION_FAILURE, StartedAt: start, CompletedAt: done.GetCompletedAt()}
assert.Equal(t, connect.CodeFailedPrecondition, code(runner.update(a, []*pb.StepUpdate{changed})))
})
t.Run("an empty list for a step is not a request: only a step present is replaced", func(t *testing.T) {
before := len(c.annotations(runID, a.GetJobId()).GetAnnotations())
require.NoError(t, runner.update(a, nil))
assert.Len(t, c.annotations(runID, a.GetJobId()).GetAnnotations(), before)
})
t.Run("the limits, and a call applied whole or not at all", func(t *testing.T) {
before := c.annotations(runID, a.GetJobId()).GetAnnotations()
many := func(level pb.AnnotationLevel, n int) []*pb.Annotation {
var out []*pb.Annotation
for range n {
out = append(out, ann(level, 9, "m"))
}
return out
}
require.NoError(t, runner.update(a, nil, many(warning, 10)...), "ten of a level is allowed")
require.NoError(t, runner.update(a, nil, append(many(warning, 10), many(failure, 10)...)...), "the limit is per level")
bad := map[string][]*pb.Annotation{
"eleven of a level": many(notice, 11),
"no level": {{Message: "m", StepNumber: 1}},
"no message": {ann(notice, 1, "")},
"message too long": {ann(notice, 1, strings.Repeat("é", 4097))},
"title too long": {{Level: notice, Message: "m", Title: strings.Repeat("t", 4097), StepNumber: 1}},
"file too long": {{Level: notice, Message: "m", File: strings.Repeat("f", 1025), StepNumber: 1}},
"step zero": {ann(notice, 0, "m")},
"step beyond 1000": {ann(notice, 1001, "m")},
"end before start": {{Level: notice, Message: "m", StartLine: 5, EndLine: 4, StepNumber: 1}},
"a good one and bad": {ann(notice, 1, "good"), ann(notice, 1, "")},
}
for name, anns := range bad {
assert.Equal(t, connect.CodeInvalidArgument, code(runner.update(a, nil, anns...)), name)
}
// Characters, not bytes: 4096 two-byte characters pass.
require.NoError(t, runner.update(a, nil, ann(notice, 11, strings.Repeat("é", 4096))))
require.NoError(t, runner.update(a, nil, &pb.Annotation{Level: notice, Message: "m", Title: strings.Repeat("t", 4096), File: strings.Repeat("f", 1024), StepNumber: 11}))
// An invalid call stored nothing: steps 1, 3 and 7 are as they were.
after := c.annotations(runID, a.GetJobId()).GetAnnotations()
require.GreaterOrEqual(t, len(after), len(before))
var step1 []string
for _, x := range after {
if x.GetAnnotation().GetStepNumber() == 1 {
step1 = append(step1, x.GetAnnotation().GetMessage())
}
}
assert.Equal(t, []string{"after"}, step1)
// A step of a bad call that is valid is not applied either.
assert.Equal(t, connect.CodeInvalidArgument, code(runner.update(a, []*pb.StepUpdate{{Number: 5, State: pb.StepState_STEP_STATE_IN_PROGRESS, StartedAt: now()}}, ann(notice, 1, ""))))
for _, s := range c.getRun(runID).GetJobs()[0].GetSteps() {
assert.NotEqual(t, int32(5), s.GetNumber())
}
})
t.Run("summaries: stored, replaced, removed, bounded", func(t *testing.T) {
require.NoError(t, runner.summary(a, 2, "# one"))
require.NoError(t, runner.summary(a, 2, "# one"))
require.NoError(t, runner.summary(a, 2, "# two"))
require.NoError(t, runner.summary(a, 12, "later"))
got := c.summaries(a.GetJobId())
require.Len(t, got.GetSummaries(), 2)
assert.EqualValues(t, 2, got.GetSummaries()[0].GetStepNumber())
assert.Equal(t, "# two", got.GetSummaries()[0].GetMarkdown())
assert.EqualValues(t, 12, got.GetSummaries()[1].GetStepNumber(), "numeric order")
require.NoError(t, runner.summary(a, 12, ""))
assert.Len(t, c.summaries(a.GetJobId()).GetSummaries(), 1, "an empty markdown removes it")
assert.Equal(t, connect.CodeInvalidArgument, code(runner.summary(a, 0, "x")))
assert.Equal(t, connect.CodeInvalidArgument, code(runner.summary(a, 1001, "x")))
assert.Equal(t, connect.CodeInvalidArgument, code(runner.summary(a, 2, strings.Repeat("x", 1<<20+1))))
require.NoError(t, runner.summary(a, 3, strings.Repeat("x", 1<<20)), "exactly 1 MiB is accepted")
assert.Len(t, c.summaries(a.GetJobId()).GetSummaries(), 2)
assert.Equal(t, "# two", c.summaries(a.GetJobId()).GetSummaries()[0].GetMarkdown(), "an invalid call changes nothing")
})
t.Run("at most 20 summaries per job, with truncated", func(t *testing.T) {
for i := int32(1); i <= 22; i++ {
require.NoError(t, runner.summary(b, i, "s"))
}
got := c.summaries(b.GetJobId())
assert.Len(t, got.GetSummaries(), 20)
assert.True(t, got.GetTruncated())
assert.EqualValues(t, 20, got.GetSummaries()[19].GetStepNumber())
})
t.Run("the token of another job is refused", func(t *testing.T) {
other := runner.jobAPI(b).runner
_, err := other.UpdateJob(c.ctx, connect.NewRequest(&pb.UpdateJobRequest{JobId: a.GetJobId(), Annotations: []*pb.Annotation{ann(notice, 1, "x")}}))
assert.Equal(t, connect.CodePermissionDenied, code(err))
_, err = other.UploadSummary(c.ctx, connect.NewRequest(&pb.UploadSummaryRequest{JobId: a.GetJobId(), StepNumber: 1, Markdown: "x"}))
assert.Equal(t, connect.CodePermissionDenied, code(err))
_, err = other.CompleteJob(c.ctx, connect.NewRequest(&pb.CompleteJobRequest{JobId: a.GetJobId(), Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, Annotations: []*pb.Annotation{ann(notice, 1, "x")}}))
assert.Equal(t, connect.CodePermissionDenied, code(err))
// No credential, a runner token and a stored token are not job tokens either.
for _, secret := range []string{"", runner.secret, adminToken} {
cl := c.api(secret).runner
_, err = cl.UploadSummary(c.ctx, connect.NewRequest(&pb.UploadSummaryRequest{JobId: a.GetJobId(), StepNumber: 1, Markdown: "x"}))
assert.Equal(t, connect.CodeUnauthenticated, code(err), secret)
}
assert.Equal(t, "# two", c.summaries(a.GetJobId()).GetSummaries()[0].GetMarkdown())
})
t.Run("CompleteJob applies annotations with the steps and the outcome", func(t *testing.T) {
steps := []*pb.StepUpdate{
done,
{
Number: 2, Name: "Run make", State: pb.StepState_STEP_STATE_COMPLETED, Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS,
Outcome: pb.CiConclusion_CI_CONCLUSION_FAILURE, StartedAt: start, CompletedAt: now(),
},
}
_, err := runner.jobAPI(a).runner.CompleteJob(c.ctx, connect.NewRequest(&pb.CompleteJobRequest{
JobId: a.GetJobId(), Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, Steps: steps,
Annotations: []*pb.Annotation{ann(warning, 2, "from complete"), ann(notice, 1, "step one, replaced")},
}))
require.NoError(t, err)
got := c.annotations(runID, a.GetJobId()).GetAnnotations()
var msgs []string
for _, x := range got {
if s := x.GetAnnotation().GetStepNumber(); s == 1 || s == 2 {
msgs = append(msgs, x.GetAnnotation().GetMessage())
}
}
assert.Equal(t, []string{"step one, replaced", "from complete"}, msgs)
job := c.getRun(runID).GetJobs()[0]
assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, job.GetSteps()[1].GetConclusion())
assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_FAILURE, job.GetSteps()[1].GetOutcome(), "continue-on-error: failed, then succeeded")
assert.Equal(t, pb.CiConclusion_CI_CONCLUSION_SUCCESS, job.GetSteps()[0].GetOutcome(), "a runner that sent none: read as the conclusion")
})
t.Run("a job that ended rejects annotations and summaries; a retry of CompleteJob changes nothing", func(t *testing.T) {
before := len(c.annotations(runID, a.GetJobId()).GetAnnotations())
err := runner.update(a, nil, ann(notice, 1, "too late"))
require.Equal(t, connect.CodeFailedPrecondition, code(err))
assert.NotEmpty(t, errDetails(err), "with a JobEnded detail")
err = runner.summary(a, 1, "too late")
assert.Equal(t, connect.CodeFailedPrecondition, code(err))
assert.NotEmpty(t, errDetails(err))
res, err := runner.jobAPI(a).runner.CompleteJob(c.ctx, connect.NewRequest(&pb.CompleteJobRequest{
JobId: a.GetJobId(), Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, Annotations: []*pb.Annotation{ann(notice, 1, "ignored")},
}))
require.NoError(t, err)
assert.False(t, res.Msg.GetApplied())
assert.Len(t, c.annotations(runID, a.GetJobId()).GetAnnotations(), before)
for _, x := range c.annotations(runID, a.GetJobId()).GetAnnotations() {
assert.NotEqual(t, "too late", x.GetAnnotation().GetMessage())
assert.NotEqual(t, "ignored", x.GetAnnotation().GetMessage())
}
assert.Equal(t, "# two", c.summaries(a.GetJobId()).GetSummaries()[0].GetMarkdown(), "still readable after the end")
})
t.Run("reading: the run, one job, a job that is not of the run, access", func(t *testing.T) {
all := c.annotations(runID, "")
jobs := map[string]int{}
for _, x := range all.GetAnnotations() {
jobs[x.GetJobId()]++
}
assert.Len(t, jobs, 1, "job b has none")
assert.Empty(t, c.annotations(runID, b.GetJobId()).GetAnnotations())
_, err := c.root.run.ListAnnotations(c.ctx, connect.NewRequest(&pb.ListAnnotationsRequest{Repo: c.repo, RunId: runID, JobId: "99-0-0"}))
assert.Equal(t, connect.CodeNotFound, code(err))
_, err = c.root.run.ListAnnotations(c.ctx, connect.NewRequest(&pb.ListAnnotationsRequest{Repo: c.repo, RunId: runID, Attempt: 2}))
assert.Equal(t, connect.CodeNotFound, code(err))
_, err = c.root.run.ListAnnotations(c.ctx, connect.NewRequest(&pb.ListAnnotationsRequest{Repo: c.repo, RunId: 99}))
assert.Equal(t, connect.CodeNotFound, code(err))
_, err = c.root.run.GetStepSummaries(c.ctx, connect.NewRequest(&pb.GetStepSummariesRequest{Repo: c.repo, JobId: "99-0-0"}))
assert.Equal(t, connect.CodeNotFound, code(err))
_, err = c.root.run.GetStepSummaries(c.ctx, connect.NewRequest(&pb.GetStepSummariesRequest{Repo: c.repo, JobId: a.GetJobId(), Attempt: 3}))
assert.Equal(t, connect.CodeNotFound, code(err))
// The repository is private: an anonymous caller sees nothing of it, a
// reader sees everything, as for GetRun.
anon := c.api("").run
_, err = anon.ListAnnotations(c.ctx, connect.NewRequest(&pb.ListAnnotationsRequest{Repo: c.repo, RunId: runID}))
assert.Equal(t, connect.CodeNotFound, code(err))
_, err = anon.GetStepSummaries(c.ctx, connect.NewRequest(&pb.GetStepSummariesRequest{Repo: c.repo, JobId: a.GetJobId()}))
assert.Equal(t, connect.CodeNotFound, code(err))
reader := c.api(c.token(scopes(domain.ScopeRepoRead))).run
_, err = reader.ListAnnotations(c.ctx, connect.NewRequest(&pb.ListAnnotationsRequest{Repo: c.repo, RunId: runID}))
assert.NoError(t, err)
_, err = reader.GetStepSummaries(c.ctx, connect.NewRequest(&pb.GetStepSummariesRequest{Repo: c.repo, JobId: a.GetJobId()}))
assert.NoError(t, err)
})
t.Run("the summaries of an ended job stay readable", func(t *testing.T) {
_, err := runner.jobAPI(b).runner.CompleteJob(c.ctx, connect.NewRequest(&pb.CompleteJobRequest{JobId: b.GetJobId(), Conclusion: pb.CiConclusion_CI_CONCLUSION_FAILURE}))
require.NoError(t, err)
assert.Len(t, c.summaries(b.GetJobId()).GetSummaries(), 20)
})
}
// errDetails returns the Connect error details of err.
func errDetails(err error) []*connect.ErrorDetail {
var ce *connect.Error
if connect.CodeOf(err) == connect.CodeUnknown {
return nil
}
if e, ok := err.(*connect.Error); ok {
ce = e
}
if ce == nil {
return nil
}
return ce.Details()
}
// The pages: the Annotations box of the run, the annotations at their step on
// the job page, the summaries rendered under the run, with nothing of the
// workflow's markup reaching the page as markup.
func TestWebAnnotationsAndSummaries(t *testing.T) {
ci := newCIHarness(t, nil)
b := browser{ci.harness}
runner := ci.registerRunner("alpha", "ubuntu-latest")
_, err := ci.root.repo.UpdateRepo(ci.ctx, connect.NewRequest(&pb.UpdateRepoRequest{
Repo: "proj", Visibility: pb.Visibility_VISIBILITY_PUBLIC, UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"visibility"}},
}))
require.NoError(t, err)
sha := ci.push("main", map[string]string{".github/workflows/ci.yml": strings.Split(resultsWorkflow, " lint:")[0], "src/main.go": "package main\n"}, "")
ci.waitFor("the run", func() bool { return len(ci.runs()) == 1 })
job := runner.await()
runner.heartbeat(job)
start := now()
steps := []*pb.StepUpdate{
{Number: 1, Name: "Set up job", State: pb.StepState_STEP_STATE_COMPLETED, Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, StartedAt: start, CompletedAt: now()},
{Number: 2, Name: "Run vet", State: pb.StepState_STEP_STATE_COMPLETED, Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, StartedAt: start, CompletedAt: now()},
}
require.NoError(t, runner.update(job, steps))
_, err = runner.jobAPI(job).runner.AppendLog(ci.ctx, connect.NewRequest(&pb.AppendLogRequest{JobId: job.GetJobId(), Step: 2, Lines: []*pb.LogLine{{Time: now(), Text: "vet"}}}))
require.NoError(t, err)
evil := ``
require.NoError(t, runner.update(job, nil,
&pb.Annotation{Level: pb.AnnotationLevel_ANNOTATION_LEVEL_FAILURE, Title: "Vet failed", Message: "unused " + evil, File: "src/main.go", StartLine: 1, EndLine: 3, StepNumber: 2},
&pb.Annotation{Level: pb.AnnotationLevel_ANNOTATION_LEVEL_WARNING, Message: "outside", File: "../../etc/passwd", StartLine: 1, StepNumber: 2},
&pb.Annotation{Level: pb.AnnotationLevel_ANNOTATION_LEVEL_NOTICE, Message: "plain notice", StepNumber: 1},
))
require.NoError(t, runner.summary(job, 2, strings.Join([]string{
"# Results",
"",
"",
"",
"[click](javascript:alert(1)) and [ok](https://example.org/x)",
"",
"",
"",
"
",
"",
"| a | b |", "|---|---|", "| 1 | 2 |",
}, "\n")))
_, err = runner.jobAPI(job).runner.CompleteJob(ci.ctx, connect.NewRequest(&pb.CompleteJobRequest{
JobId: job.GetJobId(), Conclusion: pb.CiConclusion_CI_CONCLUSION_SUCCESS, Steps: steps,
}))
require.NoError(t, err)
ci.waitFor("the run completes", func() bool { return ci.runs()[0].GetStatus() == pb.CiStatus_CI_STATUS_COMPLETED })
run := b.get("/proj/actions/runs/1", nil)
require.Equal(t, 200, run.status, run.body)
assert.Contains(t, run.body, "Annotations")
assert.Contains(t, run.body, `aria-label="error"`, "the level has a symbol and a word")
assert.Contains(t, run.body, `aria-label="warning"`)
assert.Contains(t, run.body, "Vet <i>failed</i>", "a title is text")
assert.Contains(t, run.body, "unused <img src=x onerror=alert(1)>")
assert.NotContains(t, run.body, evil)
assert.Contains(t, run.body, "Run <b>vet</b>", "the step name is text")
assert.Contains(t, run.body, `href="/proj/blob/`+sha+`/src/main.go#L1-L3"`, "file:line links to the file at the commit")
assert.Contains(t, run.body, "src/main.go:1-3")
assert.Contains(t, run.body, "../../etc/passwd:1")
assert.NotContains(t, run.body, `href="/proj/blob/`+sha+`/../`, "a path that leaves the repository is not a link")
assert.NotContains(t, run.body, `etc/passwd:1`)
assert.Contains(t, run.body, `href="/proj/actions/runs/1/job/`+job.GetJobId()+`"`, "the job of the annotation")
// The summary is rendered by the README renderer: markup dropped.
assert.Contains(t, run.body, "