abba0da feat(api): add CompleteJob, JobEnded, poll ids and versions to the CI protocol · ermite-api

M README.md +1 -1


@@ -22,7 +22,7 @@ Handlers are mounted under `/-/api/` by the server. Errors are Connect codes.
22	22		\| `MergeRequestService` \| `CreateMergeRequest`, `ListMergeRequests`, `GetMergeRequest` (commits, diff, comments), `AddComment`, `ResolveComment`, `CloseMergeRequest`, `MergeMergeRequest`, `MarkRead` \|
23	23		\| `SessionService` \| `CreateLoginLink`, `SignOutEverywhere` (owner only) \|
24	24		\| `EventService` \| `Watch` (streamed, resumable) \|
25		-	\| `RunnerService` \| `RegisterRunner`, `FetchJob` (long poll), `UpdateJob`, `AppendLog` (offset-idempotent), `Heartbeat`, `WatchJob` (cancel requests, streamed); for CI runners \|
	25	+	\| `RunnerService` \| `RegisterRunner`, `FetchJob` (long poll), `UpdateJob`, `AppendLog` (offset-idempotent), `Heartbeat`, `WatchJob` (cancel requests, streamed), `CompleteJob` (idempotent); for CI runners \|
26	26		\| `RunService` \| `ListRuns`, `GetRun`, `GetJob`, `GetJobLog` (streamed, follow), `CancelRun`, `RerunRun`, `ListChecks`, `ListRunners`, `RemoveRunner` \|
27	27		\| `ServerService` \| `GetVersion` (server version and feature list), `GetSigningKey` (commit signing key) \|
28	28


@@ -22,7 +22,7 @@ Handlers are mounted under `/-/api/` by the server. Errors are Connect codes.
22		\| `MergeRequestService` \| `CreateMergeRequest`, `ListMergeRequests`, `GetMergeRequest` (commits, diff, comments), `AddComment`, `ResolveComment`, `CloseMergeRequest`, `MergeMergeRequest`, `MarkRead` \|	22		\| `MergeRequestService` \| `CreateMergeRequest`, `ListMergeRequests`, `GetMergeRequest` (commits, diff, comments), `AddComment`, `ResolveComment`, `CloseMergeRequest`, `MergeMergeRequest`, `MarkRead` \|
23		\| `SessionService` \| `CreateLoginLink`, `SignOutEverywhere` (owner only) \|	23		\| `SessionService` \| `CreateLoginLink`, `SignOutEverywhere` (owner only) \|
24		\| `EventService` \| `Watch` (streamed, resumable) \|	24		\| `EventService` \| `Watch` (streamed, resumable) \|
25	-	\| `RunnerService` \| `RegisterRunner`, `FetchJob` (long poll), `UpdateJob`, `AppendLog` (offset-idempotent), `Heartbeat`, `WatchJob` (cancel requests, streamed); for CI runners \|	25	+	\| `RunnerService` \| `RegisterRunner`, `FetchJob` (long poll), `UpdateJob`, `AppendLog` (offset-idempotent), `Heartbeat`, `WatchJob` (cancel requests, streamed), `CompleteJob` (idempotent); for CI runners \|
26		\| `RunService` \| `ListRuns`, `GetRun`, `GetJob`, `GetJobLog` (streamed, follow), `CancelRun`, `RerunRun`, `ListChecks`, `ListRunners`, `RemoveRunner` \|	26		\| `RunService` \| `ListRuns`, `GetRun`, `GetJob`, `GetJobLog` (streamed, follow), `CancelRun`, `RerunRun`, `ListChecks`, `ListRunners`, `RemoveRunner` \|
27		\| `ServerService` \| `GetVersion` (server version and feature list), `GetSigningKey` (commit signing key) \|	27		\| `ServerService` \| `GetVersion` (server version and feature list), `GetSigningKey` (commit signing key) \|
28			28

M gen/ermite/ermiteconnect/event.connect.go +8 -4


@@ -56,8 +56,10 @@ type EventServiceClient interface {
56	56		// "home"), subject to the same visibility. They carry the state after the
57	57		// change and are published when a run or a job is created, changes status
58	58		// or conclusion, or when one of its steps starts or ends (never per log
59		-	// line: follow a log with RunService.GetJobLog). A job update is published
60		-	// before the update of its run. There is no event for a change of title or
	59	+	// line: follow a log with RunService.GetJobLog). The order of these events
	60	+	// is not guaranteed, between jobs and runs or between two updates of the same
	61	+	// one: a client keeps the copy with the higher Run.version or
	62	+	// RunJob.version and drops the other. There is no event for a change of title or
61	63		// description, nor for a resolved comment: after a gap, or to see such
62	64		// changes, clients read the merge request again. A merge publishes
63	65		// merge_request_merged, then pushed for the target branch.

@@ -126,8 +128,10 @@ type EventServiceHandler interface {
126	128		// "home"), subject to the same visibility. They carry the state after the
127	129		// change and are published when a run or a job is created, changes status
128	130		// or conclusion, or when one of its steps starts or ends (never per log
129		-	// line: follow a log with RunService.GetJobLog). A job update is published
130		-	// before the update of its run. There is no event for a change of title or
	131	+	// line: follow a log with RunService.GetJobLog). The order of these events
	132	+	// is not guaranteed, between jobs and runs or between two updates of the same
	133	+	// one: a client keeps the copy with the higher Run.version or
	134	+	// RunJob.version and drops the other. There is no event for a change of title or
131	135		// description, nor for a resolved comment: after a gap, or to see such
132	136		// changes, clients read the merge request again. A merge publishes
133	137		// merge_request_merged, then pushed for the target branch.

 	// "home"), subject to the same visibility. They carry the state after the
 	// change and are published when a run or a job is created, changes status
 	// or conclusion, or when one of its steps starts or ends (never per log
-	// line: follow a log with RunService.GetJobLog). A job update is published
+	// line: follow a log with RunService.GetJobLog). The order of these events
-	// before the update of its run. There is no event for a change of title or
+	// is not guaranteed, between jobs and runs or between two updates of the same
+	// one: a client keeps the copy with the higher Run.version or
+	// RunJob.version and drops the other. There is no event for a change of title or
 	// description, nor for a resolved comment: after a gap, or to see such
 	// changes, clients read the merge request again. A merge publishes
 	// merge_request_merged, then pushed for the target branch.
 	// "home"), subject to the same visibility. They carry the state after the
 	// change and are published when a run or a job is created, changes status
 	// or conclusion, or when one of its steps starts or ends (never per log
-	// line: follow a log with RunService.GetJobLog). A job update is published
+	// line: follow a log with RunService.GetJobLog). The order of these events
-	// before the update of its run. There is no event for a change of title or
+	// is not guaranteed, between jobs and runs or between two updates of the same
+	// one: a client keeps the copy with the higher Run.version or
+	// RunJob.version and drops the other. There is no event for a change of title or
 	// description, nor for a resolved comment: after a gap, or to see such
 	// changes, clients read the merge request again. A merge publishes
 	// merge_request_merged, then pushed for the target branch.

M gen/ermite/ermiteconnect/runner.connect.go +247 -74


@@ -48,6 +48,9 @@ const (
48	48		RunnerServiceHeartbeatProcedure = "/ermite.RunnerService/Heartbeat"
49	49		// RunnerServiceWatchJobProcedure is the fully-qualified name of the RunnerService's WatchJob RPC.
50	50		RunnerServiceWatchJobProcedure = "/ermite.RunnerService/WatchJob"
	51	+	// RunnerServiceCompleteJobProcedure is the fully-qualified name of the RunnerService's CompleteJob
	52	+	// RPC.
	53	+	RunnerServiceCompleteJobProcedure = "/ermite.RunnerService/CompleteJob"
51	54		)
52	55
53	56		// RunnerServiceClient is a client for the ermite.RunnerService service.

@@ -58,37 +61,54 @@ type RunnerServiceClient interface {
58	61		// (a runner serves every repository), or the root token. Fails with
59	62		// unauthenticated without a valid token, permission_denied without the
60	63		// scope or with a repository-restricted token, invalid_argument if a field
61		-	// breaks a rule, and already_exists if name is taken by another runner.
	64	+	// breaks a rule, and already_exists if name is taken by another runner and
	65	+	// replace is not set.
62	66		RegisterRunner(context.Context, connect.Request[ermite.RegisterRunnerRequest]) (connect.Response[ermite.RegisterRunnerResponse], error)
63	67		// FetchJob asks for a job to run (long poll). The server holds the call
64	68		// until a queued job matches the runner, or about 50 seconds (never more
65	69		// than 50) pass, then answers; without a job the response has no job and the
66		-	// runner calls again at once. Clients set a call timeout above 50 seconds
67		-	// (60 is suggested) and must not poll when they have no free capacity.
	70	+	// runner calls again after retry_after. Clients set a call timeout above 50
	71	+	// seconds (60 is suggested) and must not poll when they have no free
	72	+	// capacity.
68	73		//
69	74		// A job matches when every label of its runs-on is among the labels of the
70	75		// runner (case-insensitive). A job is given to exactly one runner, even when
71	76		// many runners and many polls compete; the assignment is recorded before
72		-	// the response is sent, and the job is then in progress. If the response is
73		-	// lost the job is not offered again: with no Heartbeat for 2 minutes the
74		-	// server fails it as a lost runner.
	77	+	// the response is sent. The job stays QUEUED for the owner until it is
	78	+	// acknowledged.
	79	+	//
	80	+	// Acknowledgement. The runner's first Heartbeat for the job is the
	81	+	// acknowledgement, and the runner sends it as soon as it has the job, before
	82	+	// anything else. If none arrives within 30 seconds of the assignment (the
	83	+	// response was lost, or the runner died), the server cancels the assignment:
	84	+	// the job token is revoked, the job goes back to QUEUED with a fresh token
	85	+	// and can be given to any runner, the same one included; nothing the old
	86	+	// assignment stored is kept. This is not a failure of the job. A job whose
	87	+	// assignment is cancelled 3 times in a row is failed with CI_CONCLUSION_FAILURE
	88	+	// and an error saying so.
	89	+	//
	90	+	// Retry after a lost response. poll_id lets a client retry a poll safely:
	91	+	// a call with the same poll_id as an earlier call of the same runner that
	92	+	// assigned a job returns that same assignment (same Job, same token) as long
	93	+	// as it has not been acknowledged or cancelled, instead of taking another.
	94	+	//
	95	+	// A runner that crashed or whose host restarted cannot resume a job: it does
	96	+	// not keep the job token, so the job fails with the lost-runner error after
	97	+	// the heartbeat timeout (see Heartbeat).
75	98		//
76	99		// Needs the runner token. Fails with unauthenticated when the runner was
77	100		// removed (also ending a call in progress) and invalid_argument if labels
78		-	// breaks a rule.
	101	+	// or poll_id break a rule.
79	102		FetchJob(context.Context, connect.Request[ermite.FetchJobRequest]) (connect.Response[ermite.FetchJobResponse], error)
80		-	// UpdateJob reports the state of the steps of a job and, once, how the job
81		-	// ended. Needs the job token. Each call is applied whole or not at all, and
82		-	// repeating an identical call changes nothing, so a runner may retry after a
83		-	// network error.
	103	+	// UpdateJob reports the state of the steps of a job while it runs. Needs the
	104	+	// job token. Each call is applied whole or not at all, and repeating an
	105	+	// identical call changes nothing, so a runner may retry after a network
	106	+	// error. To end the job use CompleteJob.
84	107		//
85		-	// Fails with invalid_argument if a rule is broken (including the output
86		-	// limits), and with failed_precondition if a step goes back (a COMPLETED
87		-	// step cannot change, a step cannot go from IN_PROGRESS to PENDING).
88		-	// The call that carries a conclusion ends the job: its job token stops
89		-	// working, the server completes the job and no further call is accepted,
90		-	// so a retry of that last call after a lost response gets unauthenticated,
91		-	// which means it was applied.
	108	+	// Fails with invalid_argument if a rule is broken, with failed_precondition
	109	+	// if a step goes back (a COMPLETED step cannot change, a step cannot go from
	110	+	// IN_PROGRESS to PENDING) and with failed_precondition and a JobEnded detail
	111	+	// if the job has ended.
92	112		UpdateJob(context.Context, connect.Request[ermite.UpdateJobRequest]) (connect.Response[ermite.UpdateJobResponse], error)
93	113		// AppendLog appends lines to the log of a job. Needs the job token.
94	114		//

@@ -100,46 +120,102 @@ type RunnerServiceClient interface {
100	120		// - offset equal to the server's line count: the lines are appended;
101	121		// - offset below it: the first (count - offset) lines were already stored
102	122		// (a retry after a lost response) and are skipped, and the rest is
103		-	// appended; a request entirely stored changes nothing;
	123	+	// appended; a request entirely stored changes nothing. The skipped lines
	124	+	// are not compared with the stored ones: a retry whose lines differ from
	125	+	// the first attempt is silently dropped, so a client must resend the
	126	+	// same lines at the same offset;
104	127		// - offset above it: out_of_range (a gap is a bug of the client; the
105	128		// message gives the line count the server holds).
106	129		//
107	130		// The response gives the line count after the call, so a client can check
108		-	// itself. A request has at most 1,000 lines whose text adds up to at most
109		-	// 64 KiB (65,536 bytes), all in the same step; more is invalid_argument. A
110		-	// line of more than 64 KiB is cut by the client into several lines.
	131	+	// itself. Offsets resume only after a network error within the same runner
	132	+	// process; they are not a way to resume a job after a crash.
	133	+	//
	134	+	// A request has at most 1,000 lines whose text adds up to at most 64 KiB
	135	+	// (65,536 bytes), all in the same step; more is invalid_argument. A line
	136	+	// longer than that is cut by the client into pieces, see LogLine.continued.
	137	+	//
	138	+	// Steps. The step of a request must be at least the step of the last line
	139	+	// the server holds, because the log is ordered by step (applied lines only:
	140	+	// lines skipped as already stored are not checked); a lower step is
	141	+	// failed_precondition. Lines for a step that UpdateJob has not announced
	142	+	// create it: it starts IN_PROGRESS at the time of its first line, with an
	143	+	// empty name until an UpdateJob gives one.
111	144		//
112	145		// The log of a job is limited to 512 MiB (536,870,912 bytes) as stored by
113	146		// the server (text, timestamps and separators). The call that would exceed
114	147		// it appends nothing; the server adds a last line saying that the limit was
115	148		// reached, ends the job with CI_CONCLUSION_FAILURE and RunJob.error set, and
116		-	// answers resource_exhausted. The runner then stops the job and sends
117		-	// nothing more: its calls get unauthenticated.
	149	+	// answers failed_precondition with a JobEnded detail like any call after the
	150	+	// end of a job. The runner then stops the job.
	151	+	//
	152	+	// Lines sent after the end of the job are rejected, so a runner flushes all
	153	+	// its AppendLog lines before it calls CompleteJob.
118	154		//
119	155		// Fails with invalid_argument if a rule is broken (step number, text with a
120		-	// line feed or invalid UTF-8, missing time).
	156	+	// line feed or invalid UTF-8, missing time, a continued first line).
121	157		AppendLog(context.Context, connect.Request[ermite.AppendLogRequest]) (connect.Response[ermite.AppendLogResponse], error)
122	158		// Heartbeat tells the server that the runner is alive and still working on
123		-	// the job. Needs the job token. The runner calls it every 30 seconds for as
124		-	// long as the job runs, from assignment to the end. Without a heartbeat for
125		-	// 2 minutes the server ends the job with CI_CONCLUSION_FAILURE and the message
126		-	// "The self-hosted runner: <name> lost communication with the server...".
127		-	// Only Heartbeat counts: other calls do not keep a job alive.
	159	+	// the job. Needs the job token. The first one acknowledges the assignment
	160	+	// (see FetchJob). The runner then calls it every 30 seconds for as long as
	161	+	// the job runs. The deadline is 2 minutes after the later of the last
	162	+	// heartbeat and the start of the server (so a server restart does not fail
	163	+	// the jobs of runners that could not reach it): past it the server ends the
	164	+	// job with CI_CONCLUSION_FAILURE and the message "The self-hosted runner:
	165	+	// <name> lost communication with the server...". Only Heartbeat counts:
	166	+	// other calls do not keep a job alive.
	167	+	//
	168	+	// A runner that gets unavailable (or any network error) retries Heartbeat
	169	+	// every 5 seconds until it succeeds, instead of waiting for its next 30
	170	+	// second tick, and keeps the job running meanwhile.
128	171		Heartbeat(context.Context, connect.Request[ermite.HeartbeatRequest]) (connect.Response[ermite.HeartbeatResponse], error)
129		-	// WatchJob streams the requests to stop a job. Needs the job token. The
	172	+	// WatchJob streams what the runner must react to. Needs the job token. The
130	173		// runner opens it when it starts the job and keeps it open; it reopens it
131		-	// after any error other than unauthenticated.
	174	+	// after any error other than unauthenticated and the failed_precondition
	175	+	// that says the job ended, and after a clean close of
	176	+	// the stream that did not end with a JobEnded message.
132	177		//
133	178		// When the stream opens and a cancellation has already been requested, the
134	179		// first message is that request, so a reconnect never misses one. Only the
135	180		// first reason is sent: a job already cancelled by the owner does not get a
136		-	// second message when its timeout expires. The server also sends a
137		-	// keep-alive message (reason CANCEL_REASON_UNSPECIFIED) at least every 30
138		-	// seconds, which the client ignores. The stream ends without error when
139		-	// the job ends. The runner stops the job as the official runner does
	181	+	// second message when its time budget expires. The server also sends a
	182	+	// Keepalive at least every 30 seconds, which the client ignores. When the
	183	+	// job ends the server sends JobEnded, then closes the stream without error.
	184	+	// On a job that has already ended the call fails like the other RPCs
	185	+	// (failed_precondition with a JobEnded detail).
	186	+	//
	187	+	// On a CancelRequest the runner stops the job as the official runner does
140	188		// (interrupt, then terminate, then kill) and reports the conclusion given
141		-	// for the reason with UpdateJob.
	189	+	// for the reason with CompleteJob.
	190	+	//
	191	+	// The server does not trust the runner to obey. If no conclusion arrives
	192	+	// within 60 seconds after a cancellation request, whatever the reason (user,
	193	+	// fail-fast, timeout), the server ends the job itself: CI_CONCLUSION_CANCELLED
	194	+	// for user and fail-fast, CI_CONCLUSION_TIMED_OUT for timeout, with an error
	195	+	// saying that the runner did not answer, and the steps not completed are
	196	+	// completed as cancelled. The runner's later CompleteJob returns that result.
142	197		WatchJob(context.Context, connect.Request[ermite.WatchJobRequest]) (connect.ServerStreamForClient[ermite.WatchJobResponse], error)
	198	+	// CompleteJob ends a job: it reports the final steps, the outputs and the
	199	+	// conclusion. Needs the job token, which stays valid for this RPC for 24
	200	+	// hours after the end of the job, however it ended. The call is applied whole
	201	+	// or not at all. Once the job has ended, the token works for no other RPC
	202	+	// (they answer failed_precondition with a JobEnded detail).
	203	+	//
	204	+	// It is idempotent. The first call that finds the job running ends it and
	205	+	// answers applied = true. Any later call, a retry after a lost response or a
	206	+	// call after the server ended the job itself, is not validated and changes
	207	+	// nothing: it answers OK with the stored conclusion and error and
	208	+	// applied = false (a runner whose conclusion differs from the stored one
	209	+	// simply logs it; the outputs it sent are dropped).
	210	+	//
	211	+	// Lines sent with AppendLog after this call are rejected: a runner flushes
	212	+	// all its log lines first. Steps that were not reported COMPLETED when the job
	213	+	// ends are completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when
	214	+	// they never started and the job did not end by cancellation).
	215	+	//
	216	+	// Fails with invalid_argument if a rule is broken (including the output
	217	+	// limits) and failed_precondition if a step goes back, as in UpdateJob.
	218	+	CompleteJob(context.Context, connect.Request[ermite.CompleteJobRequest]) (connect.Response[ermite.CompleteJobResponse], error)
143	219		}
144	220
145	221		// NewRunnerServiceClient constructs a client for the ermite.RunnerService service. By default, it

@@ -189,6 +265,12 @@ func NewRunnerServiceClient(httpClient connect.HTTPClient, baseURL string, opts
189	265		connect.WithSchema(runnerServiceMethods.ByName("WatchJob")),
190	266		connect.WithClientOptions(opts...),
191	267		),
	268	+	completeJob: connect.NewClient[ermite.CompleteJobRequest, ermite.CompleteJobResponse](
	269	+	httpClient,
	270	+	baseURL+RunnerServiceCompleteJobProcedure,
	271	+	connect.WithSchema(runnerServiceMethods.ByName("CompleteJob")),
	272	+	connect.WithClientOptions(opts...),
	273	+	),
192	274		}
193	275		}
194	276

@@ -200,6 +282,7 @@ type runnerServiceClient struct {
200	282		appendLog *connect.Client[ermite.AppendLogRequest, ermite.AppendLogResponse]
201	283		heartbeat *connect.Client[ermite.HeartbeatRequest, ermite.HeartbeatResponse]
202	284		watchJob *connect.Client[ermite.WatchJobRequest, ermite.WatchJobResponse]
	285	+	completeJob *connect.Client[ermite.CompleteJobRequest, ermite.CompleteJobResponse]
203	286		}
204	287
205	288		// RegisterRunner calls ermite.RunnerService.RegisterRunner.

@@ -232,6 +315,11 @@ func (c runnerServiceClient) WatchJob(ctx context.Context, req connect.Request
232	315		return c.watchJob.CallServerStream(ctx, req)
233	316		}
234	317
	318	+	// CompleteJob calls ermite.RunnerService.CompleteJob.
	319	+	func (c runnerServiceClient) CompleteJob(ctx context.Context, req connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error) {
	320	+	return c.completeJob.CallUnary(ctx, req)
	321	+	}
	322	+
235	323		// RunnerServiceHandler is an implementation of the ermite.RunnerService service.
236	324		type RunnerServiceHandler interface {
237	325		// RegisterRunner registers a runner and returns its runner token, which is

@@ -240,37 +328,54 @@ type RunnerServiceHandler interface {
240	328		// (a runner serves every repository), or the root token. Fails with
241	329		// unauthenticated without a valid token, permission_denied without the
242	330		// scope or with a repository-restricted token, invalid_argument if a field
243		-	// breaks a rule, and already_exists if name is taken by another runner.
	331	+	// breaks a rule, and already_exists if name is taken by another runner and
	332	+	// replace is not set.
244	333		RegisterRunner(context.Context, connect.Request[ermite.RegisterRunnerRequest]) (connect.Response[ermite.RegisterRunnerResponse], error)
245	334		// FetchJob asks for a job to run (long poll). The server holds the call
246	335		// until a queued job matches the runner, or about 50 seconds (never more
247	336		// than 50) pass, then answers; without a job the response has no job and the
248		-	// runner calls again at once. Clients set a call timeout above 50 seconds
249		-	// (60 is suggested) and must not poll when they have no free capacity.
	337	+	// runner calls again after retry_after. Clients set a call timeout above 50
	338	+	// seconds (60 is suggested) and must not poll when they have no free
	339	+	// capacity.
250	340		//
251	341		// A job matches when every label of its runs-on is among the labels of the
252	342		// runner (case-insensitive). A job is given to exactly one runner, even when
253	343		// many runners and many polls compete; the assignment is recorded before
254		-	// the response is sent, and the job is then in progress. If the response is
255		-	// lost the job is not offered again: with no Heartbeat for 2 minutes the
256		-	// server fails it as a lost runner.
	344	+	// the response is sent. The job stays QUEUED for the owner until it is
	345	+	// acknowledged.
	346	+	//
	347	+	// Acknowledgement. The runner's first Heartbeat for the job is the
	348	+	// acknowledgement, and the runner sends it as soon as it has the job, before
	349	+	// anything else. If none arrives within 30 seconds of the assignment (the
	350	+	// response was lost, or the runner died), the server cancels the assignment:
	351	+	// the job token is revoked, the job goes back to QUEUED with a fresh token
	352	+	// and can be given to any runner, the same one included; nothing the old
	353	+	// assignment stored is kept. This is not a failure of the job. A job whose
	354	+	// assignment is cancelled 3 times in a row is failed with CI_CONCLUSION_FAILURE
	355	+	// and an error saying so.
	356	+	//
	357	+	// Retry after a lost response. poll_id lets a client retry a poll safely:
	358	+	// a call with the same poll_id as an earlier call of the same runner that
	359	+	// assigned a job returns that same assignment (same Job, same token) as long
	360	+	// as it has not been acknowledged or cancelled, instead of taking another.
	361	+	//
	362	+	// A runner that crashed or whose host restarted cannot resume a job: it does
	363	+	// not keep the job token, so the job fails with the lost-runner error after
	364	+	// the heartbeat timeout (see Heartbeat).
257	365		//
258	366		// Needs the runner token. Fails with unauthenticated when the runner was
259	367		// removed (also ending a call in progress) and invalid_argument if labels
260		-	// breaks a rule.
	368	+	// or poll_id break a rule.
261	369		FetchJob(context.Context, connect.Request[ermite.FetchJobRequest]) (connect.Response[ermite.FetchJobResponse], error)
262		-	// UpdateJob reports the state of the steps of a job and, once, how the job
263		-	// ended. Needs the job token. Each call is applied whole or not at all, and
264		-	// repeating an identical call changes nothing, so a runner may retry after a
265		-	// network error.
	370	+	// UpdateJob reports the state of the steps of a job while it runs. Needs the
	371	+	// job token. Each call is applied whole or not at all, and repeating an
	372	+	// identical call changes nothing, so a runner may retry after a network
	373	+	// error. To end the job use CompleteJob.
266	374		//
267		-	// Fails with invalid_argument if a rule is broken (including the output
268		-	// limits), and with failed_precondition if a step goes back (a COMPLETED
269		-	// step cannot change, a step cannot go from IN_PROGRESS to PENDING).
270		-	// The call that carries a conclusion ends the job: its job token stops
271		-	// working, the server completes the job and no further call is accepted,
272		-	// so a retry of that last call after a lost response gets unauthenticated,
273		-	// which means it was applied.
	375	+	// Fails with invalid_argument if a rule is broken, with failed_precondition
	376	+	// if a step goes back (a COMPLETED step cannot change, a step cannot go from
	377	+	// IN_PROGRESS to PENDING) and with failed_precondition and a JobEnded detail
	378	+	// if the job has ended.
274	379		UpdateJob(context.Context, connect.Request[ermite.UpdateJobRequest]) (connect.Response[ermite.UpdateJobResponse], error)
275	380		// AppendLog appends lines to the log of a job. Needs the job token.
276	381		//

@@ -282,46 +387,102 @@ type RunnerServiceHandler interface {
282	387		// - offset equal to the server's line count: the lines are appended;
283	388		// - offset below it: the first (count - offset) lines were already stored
284	389		// (a retry after a lost response) and are skipped, and the rest is
285		-	// appended; a request entirely stored changes nothing;
	390	+	// appended; a request entirely stored changes nothing. The skipped lines
	391	+	// are not compared with the stored ones: a retry whose lines differ from
	392	+	// the first attempt is silently dropped, so a client must resend the
	393	+	// same lines at the same offset;
286	394		// - offset above it: out_of_range (a gap is a bug of the client; the
287	395		// message gives the line count the server holds).
288	396		//
289	397		// The response gives the line count after the call, so a client can check
290		-	// itself. A request has at most 1,000 lines whose text adds up to at most
291		-	// 64 KiB (65,536 bytes), all in the same step; more is invalid_argument. A
292		-	// line of more than 64 KiB is cut by the client into several lines.
	398	+	// itself. Offsets resume only after a network error within the same runner
	399	+	// process; they are not a way to resume a job after a crash.
	400	+	//
	401	+	// A request has at most 1,000 lines whose text adds up to at most 64 KiB
	402	+	// (65,536 bytes), all in the same step; more is invalid_argument. A line
	403	+	// longer than that is cut by the client into pieces, see LogLine.continued.
	404	+	//
	405	+	// Steps. The step of a request must be at least the step of the last line
	406	+	// the server holds, because the log is ordered by step (applied lines only:
	407	+	// lines skipped as already stored are not checked); a lower step is
	408	+	// failed_precondition. Lines for a step that UpdateJob has not announced
	409	+	// create it: it starts IN_PROGRESS at the time of its first line, with an
	410	+	// empty name until an UpdateJob gives one.
293	411		//
294	412		// The log of a job is limited to 512 MiB (536,870,912 bytes) as stored by
295	413		// the server (text, timestamps and separators). The call that would exceed
296	414		// it appends nothing; the server adds a last line saying that the limit was
297	415		// reached, ends the job with CI_CONCLUSION_FAILURE and RunJob.error set, and
298		-	// answers resource_exhausted. The runner then stops the job and sends
299		-	// nothing more: its calls get unauthenticated.
	416	+	// answers failed_precondition with a JobEnded detail like any call after the
	417	+	// end of a job. The runner then stops the job.
	418	+	//
	419	+	// Lines sent after the end of the job are rejected, so a runner flushes all
	420	+	// its AppendLog lines before it calls CompleteJob.
300	421		//
301	422		// Fails with invalid_argument if a rule is broken (step number, text with a
302		-	// line feed or invalid UTF-8, missing time).
	423	+	// line feed or invalid UTF-8, missing time, a continued first line).
303	424		AppendLog(context.Context, connect.Request[ermite.AppendLogRequest]) (connect.Response[ermite.AppendLogResponse], error)
304	425		// Heartbeat tells the server that the runner is alive and still working on
305		-	// the job. Needs the job token. The runner calls it every 30 seconds for as
306		-	// long as the job runs, from assignment to the end. Without a heartbeat for
307		-	// 2 minutes the server ends the job with CI_CONCLUSION_FAILURE and the message
308		-	// "The self-hosted runner: <name> lost communication with the server...".
309		-	// Only Heartbeat counts: other calls do not keep a job alive.
	426	+	// the job. Needs the job token. The first one acknowledges the assignment
	427	+	// (see FetchJob). The runner then calls it every 30 seconds for as long as
	428	+	// the job runs. The deadline is 2 minutes after the later of the last
	429	+	// heartbeat and the start of the server (so a server restart does not fail
	430	+	// the jobs of runners that could not reach it): past it the server ends the
	431	+	// job with CI_CONCLUSION_FAILURE and the message "The self-hosted runner:
	432	+	// <name> lost communication with the server...". Only Heartbeat counts:
	433	+	// other calls do not keep a job alive.
	434	+	//
	435	+	// A runner that gets unavailable (or any network error) retries Heartbeat
	436	+	// every 5 seconds until it succeeds, instead of waiting for its next 30
	437	+	// second tick, and keeps the job running meanwhile.
310	438		Heartbeat(context.Context, connect.Request[ermite.HeartbeatRequest]) (connect.Response[ermite.HeartbeatResponse], error)
311		-	// WatchJob streams the requests to stop a job. Needs the job token. The
	439	+	// WatchJob streams what the runner must react to. Needs the job token. The
312	440		// runner opens it when it starts the job and keeps it open; it reopens it
313		-	// after any error other than unauthenticated.
	441	+	// after any error other than unauthenticated and the failed_precondition
	442	+	// that says the job ended, and after a clean close of
	443	+	// the stream that did not end with a JobEnded message.
314	444		//
315	445		// When the stream opens and a cancellation has already been requested, the
316	446		// first message is that request, so a reconnect never misses one. Only the
317	447		// first reason is sent: a job already cancelled by the owner does not get a
318		-	// second message when its timeout expires. The server also sends a
319		-	// keep-alive message (reason CANCEL_REASON_UNSPECIFIED) at least every 30
320		-	// seconds, which the client ignores. The stream ends without error when
321		-	// the job ends. The runner stops the job as the official runner does
	448	+	// second message when its time budget expires. The server also sends a
	449	+	// Keepalive at least every 30 seconds, which the client ignores. When the
	450	+	// job ends the server sends JobEnded, then closes the stream without error.
	451	+	// On a job that has already ended the call fails like the other RPCs
	452	+	// (failed_precondition with a JobEnded detail).
	453	+	//
	454	+	// On a CancelRequest the runner stops the job as the official runner does
322	455		// (interrupt, then terminate, then kill) and reports the conclusion given
323		-	// for the reason with UpdateJob.
	456	+	// for the reason with CompleteJob.
	457	+	//
	458	+	// The server does not trust the runner to obey. If no conclusion arrives
	459	+	// within 60 seconds after a cancellation request, whatever the reason (user,
	460	+	// fail-fast, timeout), the server ends the job itself: CI_CONCLUSION_CANCELLED
	461	+	// for user and fail-fast, CI_CONCLUSION_TIMED_OUT for timeout, with an error
	462	+	// saying that the runner did not answer, and the steps not completed are
	463	+	// completed as cancelled. The runner's later CompleteJob returns that result.
324	464		WatchJob(context.Context, connect.Request[ermite.WatchJobRequest], connect.ServerStream[ermite.WatchJobResponse]) error
	465	+	// CompleteJob ends a job: it reports the final steps, the outputs and the
	466	+	// conclusion. Needs the job token, which stays valid for this RPC for 24
	467	+	// hours after the end of the job, however it ended. The call is applied whole
	468	+	// or not at all. Once the job has ended, the token works for no other RPC
	469	+	// (they answer failed_precondition with a JobEnded detail).
	470	+	//
	471	+	// It is idempotent. The first call that finds the job running ends it and
	472	+	// answers applied = true. Any later call, a retry after a lost response or a
	473	+	// call after the server ended the job itself, is not validated and changes
	474	+	// nothing: it answers OK with the stored conclusion and error and
	475	+	// applied = false (a runner whose conclusion differs from the stored one
	476	+	// simply logs it; the outputs it sent are dropped).
	477	+	//
	478	+	// Lines sent with AppendLog after this call are rejected: a runner flushes
	479	+	// all its log lines first. Steps that were not reported COMPLETED when the job
	480	+	// ends are completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when
	481	+	// they never started and the job did not end by cancellation).
	482	+	//
	483	+	// Fails with invalid_argument if a rule is broken (including the output
	484	+	// limits) and failed_precondition if a step goes back, as in UpdateJob.
	485	+	CompleteJob(context.Context, connect.Request[ermite.CompleteJobRequest]) (connect.Response[ermite.CompleteJobResponse], error)
325	486		}
326	487
327	488		// NewRunnerServiceHandler builds an HTTP handler from the service implementation. It returns the

@@ -367,6 +528,12 @@ func NewRunnerServiceHandler(svc RunnerServiceHandler, opts ...connect.HandlerOp
367	528		connect.WithSchema(runnerServiceMethods.ByName("WatchJob")),
368	529		connect.WithHandlerOptions(opts...),
369	530		)
	531	+	runnerServiceCompleteJobHandler := connect.NewUnaryHandler(
	532	+	RunnerServiceCompleteJobProcedure,
	533	+	svc.CompleteJob,
	534	+	connect.WithSchema(runnerServiceMethods.ByName("CompleteJob")),
	535	+	connect.WithHandlerOptions(opts...),
	536	+	)
370	537		return "/ermite.RunnerService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
371	538		switch r.URL.Path {
372	539		case RunnerServiceRegisterRunnerProcedure:

@@ -381,6 +548,8 @@ func NewRunnerServiceHandler(svc RunnerServiceHandler, opts ...connect.HandlerOp
381	548		runnerServiceHeartbeatHandler.ServeHTTP(w, r)
382	549		case RunnerServiceWatchJobProcedure:
383	550		runnerServiceWatchJobHandler.ServeHTTP(w, r)
	551	+	case RunnerServiceCompleteJobProcedure:
	552	+	runnerServiceCompleteJobHandler.ServeHTTP(w, r)
384	553		default:
385	554		http.NotFound(w, r)
386	555		}

@@ -413,3 +582,7 @@ func (UnimplementedRunnerServiceHandler) Heartbeat(context.Context, *connect.Req
413	582		func (UnimplementedRunnerServiceHandler) WatchJob(context.Context, connect.Request[ermite.WatchJobRequest], connect.ServerStream[ermite.WatchJobResponse]) error {
414	583		return connect.NewError(connect.CodeUnimplemented, errors.New("ermite.RunnerService.WatchJob is not implemented"))
415	584		}
	585	+
	586	+	func (UnimplementedRunnerServiceHandler) CompleteJob(context.Context, connect.Request[ermite.CompleteJobRequest]) (connect.Response[ermite.CompleteJobResponse], error) {
	587	+	return nil, connect.NewError(connect.CodeUnimplemented, errors.New("ermite.RunnerService.CompleteJob is not implemented"))
	588	+	}

 	RunnerServiceHeartbeatProcedure = "/ermite.RunnerService/Heartbeat"
 	// RunnerServiceWatchJobProcedure is the fully-qualified name of the RunnerService's WatchJob RPC.
 	RunnerServiceWatchJobProcedure = "/ermite.RunnerService/WatchJob"
+	// RunnerServiceCompleteJobProcedure is the fully-qualified name of the RunnerService's CompleteJob
+	// RPC.
+	RunnerServiceCompleteJobProcedure = "/ermite.RunnerService/CompleteJob"
 )
 // RunnerServiceClient is a client for the ermite.RunnerService service.
 	// (a runner serves every repository), or the root token. Fails with
 	// unauthenticated without a valid token, permission_denied without the
 	// scope or with a repository-restricted token, invalid_argument if a field
-	// breaks a rule, and already_exists if name is taken by another runner.
+	// breaks a rule, and already_exists if name is taken by another runner and
+	// replace is not set.
 	RegisterRunner(context.Context, *connect.Request[ermite.RegisterRunnerRequest]) (*connect.Response[ermite.RegisterRunnerResponse], error)
 	// FetchJob asks for a job to run (long poll). The server holds the call
 	// until a queued job matches the runner, or about 50 seconds (never more
 	// than 50) pass, then answers; without a job the response has no job and the
-	// runner calls again at once. Clients set a call timeout above 50 seconds
+	// runner calls again after retry_after. Clients set a call timeout above 50
-	// (60 is suggested) and must not poll when they have no free capacity.
+	// seconds (60 is suggested) and must not poll when they have no free
+	// capacity.
 	//
 	// A job matches when every label of its runs-on is among the labels of the
 	// runner (case-insensitive). A job is given to exactly one runner, even when
 	// many runners and many polls compete; the assignment is recorded before
-	// the response is sent, and the job is then in progress. If the response is
+	// the response is sent. The job stays QUEUED for the owner until it is
-	// lost the job is not offered again: with no Heartbeat for 2 minutes the
+	// acknowledged.
-	// server fails it as a lost runner.
+	//
+	// Acknowledgement. The runner's first Heartbeat for the job is the
+	// acknowledgement, and the runner sends it as soon as it has the job, before
+	// anything else. If none arrives within 30 seconds of the assignment (the
+	// response was lost, or the runner died), the server cancels the assignment:
+	// the job token is revoked, the job goes back to QUEUED with a fresh token
+	// and can be given to any runner, the same one included; nothing the old
+	// assignment stored is kept. This is not a failure of the job. A job whose
+	// assignment is cancelled 3 times in a row is failed with CI_CONCLUSION_FAILURE
+	// and an error saying so.
+	//
+	// Retry after a lost response. poll_id lets a client retry a poll safely:
+	// a call with the same poll_id as an earlier call of the same runner that
+	// assigned a job returns that same assignment (same Job, same token) as long
+	// as it has not been acknowledged or cancelled, instead of taking another.
+	//
+	// A runner that crashed or whose host restarted cannot resume a job: it does
+	// not keep the job token, so the job fails with the lost-runner error after
+	// the heartbeat timeout (see Heartbeat).
 	//
 	// Needs the runner token. Fails with unauthenticated when the runner was
 	// removed (also ending a call in progress) and invalid_argument if labels
-	// breaks a rule.
+	// or poll_id break a rule.
 	FetchJob(context.Context, *connect.Request[ermite.FetchJobRequest]) (*connect.Response[ermite.FetchJobResponse], error)
-	// UpdateJob reports the state of the steps of a job and, once, how the job
+	// UpdateJob reports the state of the steps of a job while it runs. Needs the
-	// ended. Needs the job token. Each call is applied whole or not at all, and
+	// job token. Each call is applied whole or not at all, and repeating an
-	// repeating an identical call changes nothing, so a runner may retry after a
+	// identical call changes nothing, so a runner may retry after a network
-	// network error.
+	// error. To end the job use CompleteJob.
 	//
-	// Fails with invalid_argument if a rule is broken (including the output
+	// Fails with invalid_argument if a rule is broken, with failed_precondition
-	// limits), and with failed_precondition if a step goes back (a COMPLETED
+	// if a step goes back (a COMPLETED step cannot change, a step cannot go from
-	// step cannot change, a step cannot go from IN_PROGRESS to PENDING).
+	// IN_PROGRESS to PENDING) and with failed_precondition and a JobEnded detail
-	// The call that carries a conclusion ends the job: its job token stops
+	// if the job has ended.
-	// working, the server completes the job and no further call is accepted,
-	// so a retry of that last call after a lost response gets unauthenticated,
-	// which means it was applied.
 	UpdateJob(context.Context, *connect.Request[ermite.UpdateJobRequest]) (*connect.Response[ermite.UpdateJobResponse], error)
 	// AppendLog appends lines to the log of a job. Needs the job token.
 	//
 	//   - offset equal to the server's line count: the lines are appended;
 	//   - offset below it: the first (count - offset) lines were already stored
 	//     (a retry after a lost response) and are skipped, and the rest is
-	//     appended; a request entirely stored changes nothing;
+	//     appended; a request entirely stored changes nothing. The skipped lines
+	//     are not compared with the stored ones: a retry whose lines differ from
+	//     the first attempt is silently dropped, so a client must resend the
+	//     same lines at the same offset;
 	//   - offset above it: out_of_range (a gap is a bug of the client; the
 	//     message gives the line count the server holds).
 	//
 	// The response gives the line count after the call, so a client can check
-	// itself. A request has at most 1,000 lines whose text adds up to at most
+	// itself. Offsets resume only after a network error within the same runner
-	// 64 KiB (65,536 bytes), all in the same step; more is invalid_argument. A
+	// process; they are not a way to resume a job after a crash.
-	// line of more than 64 KiB is cut by the client into several lines.
+	//
+	// A request has at most 1,000 lines whose text adds up to at most 64 KiB
+	// (65,536 bytes), all in the same step; more is invalid_argument. A line
+	// longer than that is cut by the client into pieces, see LogLine.continued.
+	//
+	// Steps. The step of a request must be at least the step of the last line
+	// the server holds, because the log is ordered by step (applied lines only:
+	// lines skipped as already stored are not checked); a lower step is
+	// failed_precondition. Lines for a step that UpdateJob has not announced
+	// create it: it starts IN_PROGRESS at the time of its first line, with an
+	// empty name until an UpdateJob gives one.
 	//
 	// The log of a job is limited to 512 MiB (536,870,912 bytes) as stored by
 	// the server (text, timestamps and separators). The call that would exceed
 	// it appends nothing; the server adds a last line saying that the limit was
 	// reached, ends the job with CI_CONCLUSION_FAILURE and RunJob.error set, and
-	// answers resource_exhausted. The runner then stops the job and sends
+	// answers failed_precondition with a JobEnded detail like any call after the
-	// nothing more: its calls get unauthenticated.
+	// end of a job. The runner then stops the job.
+	//
+	// Lines sent after the end of the job are rejected, so a runner flushes all
+	// its AppendLog lines before it calls CompleteJob.
 	//
 	// Fails with invalid_argument if a rule is broken (step number, text with a
-	// line feed or invalid UTF-8, missing time).
+	// line feed or invalid UTF-8, missing time, a continued first line).
 	AppendLog(context.Context, *connect.Request[ermite.AppendLogRequest]) (*connect.Response[ermite.AppendLogResponse], error)
 	// Heartbeat tells the server that the runner is alive and still working on
-	// the job. Needs the job token. The runner calls it every 30 seconds for as
+	// the job. Needs the job token. The first one acknowledges the assignment
-	// long as the job runs, from assignment to the end. Without a heartbeat for
+	// (see FetchJob). The runner then calls it every 30 seconds for as long as
-	// 2 minutes the server ends the job with CI_CONCLUSION_FAILURE and the message
+	// the job runs. The deadline is 2 minutes after the later of the last
-	// "The self-hosted runner: <name> lost communication with the server...".
+	// heartbeat and the start of the server (so a server restart does not fail
-	// Only Heartbeat counts: other calls do not keep a job alive.
+	// the jobs of runners that could not reach it): past it the server ends the
+	// job with CI_CONCLUSION_FAILURE and the message "The self-hosted runner:
+	// <name> lost communication with the server...". Only Heartbeat counts:
+	// other calls do not keep a job alive.
+	//
+	// A runner that gets unavailable (or any network error) retries Heartbeat
+	// every 5 seconds until it succeeds, instead of waiting for its next 30
+	// second tick, and keeps the job running meanwhile.
 	Heartbeat(context.Context, *connect.Request[ermite.HeartbeatRequest]) (*connect.Response[ermite.HeartbeatResponse], error)
-	// WatchJob streams the requests to stop a job. Needs the job token. The
+	// WatchJob streams what the runner must react to. Needs the job token. The
 	// runner opens it when it starts the job and keeps it open; it reopens it
-	// after any error other than unauthenticated.
+	// after any error other than unauthenticated and the failed_precondition
+	// that says the job ended, and after a clean close of
+	// the stream that did not end with a JobEnded message.
 	//
 	// When the stream opens and a cancellation has already been requested, the
 	// first message is that request, so a reconnect never misses one. Only the
 	// first reason is sent: a job already cancelled by the owner does not get a
-	// second message when its timeout expires. The server also sends a
+	// second message when its time budget expires. The server also sends a
-	// keep-alive message (reason CANCEL_REASON_UNSPECIFIED) at least every 30
+	// Keepalive at least every 30 seconds, which the client ignores. When the
-	// seconds, which the client ignores. The stream ends without error when
+	// job ends the server sends JobEnded, then closes the stream without error.
-	// the job ends. The runner stops the job as the official runner does
+	// On a job that has already ended the call fails like the other RPCs
+	// (failed_precondition with a JobEnded detail).
+	//
+	// On a CancelRequest the runner stops the job as the official runner does
 	// (interrupt, then terminate, then kill) and reports the conclusion given
-	// for the reason with UpdateJob.
+	// for the reason with CompleteJob.
+	//
+	// The server does not trust the runner to obey. If no conclusion arrives
+	// within 60 seconds after a cancellation request, whatever the reason (user,
+	// fail-fast, timeout), the server ends the job itself: CI_CONCLUSION_CANCELLED
+	// for user and fail-fast, CI_CONCLUSION_TIMED_OUT for timeout, with an error
+	// saying that the runner did not answer, and the steps not completed are
+	// completed as cancelled. The runner's later CompleteJob returns that result.
 	WatchJob(context.Context, *connect.Request[ermite.WatchJobRequest]) (*connect.ServerStreamForClient[ermite.WatchJobResponse], error)
+	// CompleteJob ends a job: it reports the final steps, the outputs and the
+	// conclusion. Needs the job token, which stays valid for this RPC for 24
+	// hours after the end of the job, however it ended. The call is applied whole
+	// or not at all. Once the job has ended, the token works for no other RPC
+	// (they answer failed_precondition with a JobEnded detail).
+	//
+	// It is idempotent. The first call that finds the job running ends it and
+	// answers applied = true. Any later call, a retry after a lost response or a
+	// call after the server ended the job itself, is not validated and changes
+	// nothing: it answers OK with the stored conclusion and error and
+	// applied = false (a runner whose conclusion differs from the stored one
+	// simply logs it; the outputs it sent are dropped).
+	//
+	// Lines sent with AppendLog after this call are rejected: a runner flushes
+	// all its log lines first. Steps that were not reported COMPLETED when the job
+	// ends are completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when
+	// they never started and the job did not end by cancellation).
+	//
+	// Fails with invalid_argument if a rule is broken (including the output
+	// limits) and failed_precondition if a step goes back, as in UpdateJob.
+	CompleteJob(context.Context, *connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error)
 }
 // NewRunnerServiceClient constructs a client for the ermite.RunnerService service. By default, it
 			connect.WithSchema(runnerServiceMethods.ByName("WatchJob")),
 			connect.WithClientOptions(opts...),
 		),
+		completeJob: connect.NewClient[ermite.CompleteJobRequest, ermite.CompleteJobResponse](
+			httpClient,
+			baseURL+RunnerServiceCompleteJobProcedure,
+			connect.WithSchema(runnerServiceMethods.ByName("CompleteJob")),
+			connect.WithClientOptions(opts...),
+		),
 	}
 }
 	appendLog      *connect.Client[ermite.AppendLogRequest, ermite.AppendLogResponse]
 	heartbeat      *connect.Client[ermite.HeartbeatRequest, ermite.HeartbeatResponse]
 	watchJob       *connect.Client[ermite.WatchJobRequest, ermite.WatchJobResponse]
+	completeJob    *connect.Client[ermite.CompleteJobRequest, ermite.CompleteJobResponse]
 }
 // RegisterRunner calls ermite.RunnerService.RegisterRunner.
 	return c.watchJob.CallServerStream(ctx, req)
 }
+// CompleteJob calls ermite.RunnerService.CompleteJob.
+func (c *runnerServiceClient) CompleteJob(ctx context.Context, req *connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error) {
+	return c.completeJob.CallUnary(ctx, req)
+}
 // RunnerServiceHandler is an implementation of the ermite.RunnerService service.
 type RunnerServiceHandler interface {
 	// RegisterRunner registers a runner and returns its runner token, which is
 	// (a runner serves every repository), or the root token. Fails with
 	// unauthenticated without a valid token, permission_denied without the
 	// scope or with a repository-restricted token, invalid_argument if a field
-	// breaks a rule, and already_exists if name is taken by another runner.
+	// breaks a rule, and already_exists if name is taken by another runner and
+	// replace is not set.
 	RegisterRunner(context.Context, *connect.Request[ermite.RegisterRunnerRequest]) (*connect.Response[ermite.RegisterRunnerResponse], error)
 	// FetchJob asks for a job to run (long poll). The server holds the call
 	// until a queued job matches the runner, or about 50 seconds (never more
 	// than 50) pass, then answers; without a job the response has no job and the
-	// runner calls again at once. Clients set a call timeout above 50 seconds
+	// runner calls again after retry_after. Clients set a call timeout above 50
-	// (60 is suggested) and must not poll when they have no free capacity.
+	// seconds (60 is suggested) and must not poll when they have no free
+	// capacity.
 	//
 	// A job matches when every label of its runs-on is among the labels of the
 	// runner (case-insensitive). A job is given to exactly one runner, even when
 	// many runners and many polls compete; the assignment is recorded before
-	// the response is sent, and the job is then in progress. If the response is
+	// the response is sent. The job stays QUEUED for the owner until it is
-	// lost the job is not offered again: with no Heartbeat for 2 minutes the
+	// acknowledged.
-	// server fails it as a lost runner.
+	//
+	// Acknowledgement. The runner's first Heartbeat for the job is the
+	// acknowledgement, and the runner sends it as soon as it has the job, before
+	// anything else. If none arrives within 30 seconds of the assignment (the
+	// response was lost, or the runner died), the server cancels the assignment:
+	// the job token is revoked, the job goes back to QUEUED with a fresh token
+	// and can be given to any runner, the same one included; nothing the old
+	// assignment stored is kept. This is not a failure of the job. A job whose
+	// assignment is cancelled 3 times in a row is failed with CI_CONCLUSION_FAILURE
+	// and an error saying so.
+	//
+	// Retry after a lost response. poll_id lets a client retry a poll safely:
+	// a call with the same poll_id as an earlier call of the same runner that
+	// assigned a job returns that same assignment (same Job, same token) as long
+	// as it has not been acknowledged or cancelled, instead of taking another.
+	//
+	// A runner that crashed or whose host restarted cannot resume a job: it does
+	// not keep the job token, so the job fails with the lost-runner error after
+	// the heartbeat timeout (see Heartbeat).
 	//
 	// Needs the runner token. Fails with unauthenticated when the runner was
 	// removed (also ending a call in progress) and invalid_argument if labels
-	// breaks a rule.
+	// or poll_id break a rule.
 	FetchJob(context.Context, *connect.Request[ermite.FetchJobRequest]) (*connect.Response[ermite.FetchJobResponse], error)
-	// UpdateJob reports the state of the steps of a job and, once, how the job
+	// UpdateJob reports the state of the steps of a job while it runs. Needs the
-	// ended. Needs the job token. Each call is applied whole or not at all, and
+	// job token. Each call is applied whole or not at all, and repeating an
-	// repeating an identical call changes nothing, so a runner may retry after a
+	// identical call changes nothing, so a runner may retry after a network
-	// network error.
+	// error. To end the job use CompleteJob.
 	//
-	// Fails with invalid_argument if a rule is broken (including the output
+	// Fails with invalid_argument if a rule is broken, with failed_precondition
-	// limits), and with failed_precondition if a step goes back (a COMPLETED
+	// if a step goes back (a COMPLETED step cannot change, a step cannot go from
-	// step cannot change, a step cannot go from IN_PROGRESS to PENDING).
+	// IN_PROGRESS to PENDING) and with failed_precondition and a JobEnded detail
-	// The call that carries a conclusion ends the job: its job token stops
+	// if the job has ended.
-	// working, the server completes the job and no further call is accepted,
-	// so a retry of that last call after a lost response gets unauthenticated,
-	// which means it was applied.
 	UpdateJob(context.Context, *connect.Request[ermite.UpdateJobRequest]) (*connect.Response[ermite.UpdateJobResponse], error)
 	// AppendLog appends lines to the log of a job. Needs the job token.
 	//
 	//   - offset equal to the server's line count: the lines are appended;
 	//   - offset below it: the first (count - offset) lines were already stored
 	//     (a retry after a lost response) and are skipped, and the rest is
-	//     appended; a request entirely stored changes nothing;
+	//     appended; a request entirely stored changes nothing. The skipped lines
+	//     are not compared with the stored ones: a retry whose lines differ from
+	//     the first attempt is silently dropped, so a client must resend the
+	//     same lines at the same offset;
 	//   - offset above it: out_of_range (a gap is a bug of the client; the
 	//     message gives the line count the server holds).
 	//
 	// The response gives the line count after the call, so a client can check
-	// itself. A request has at most 1,000 lines whose text adds up to at most
+	// itself. Offsets resume only after a network error within the same runner
-	// 64 KiB (65,536 bytes), all in the same step; more is invalid_argument. A
+	// process; they are not a way to resume a job after a crash.
-	// line of more than 64 KiB is cut by the client into several lines.
+	//
+	// A request has at most 1,000 lines whose text adds up to at most 64 KiB
+	// (65,536 bytes), all in the same step; more is invalid_argument. A line
+	// longer than that is cut by the client into pieces, see LogLine.continued.
+	//
+	// Steps. The step of a request must be at least the step of the last line
+	// the server holds, because the log is ordered by step (applied lines only:
+	// lines skipped as already stored are not checked); a lower step is
+	// failed_precondition. Lines for a step that UpdateJob has not announced
+	// create it: it starts IN_PROGRESS at the time of its first line, with an
+	// empty name until an UpdateJob gives one.
 	//
 	// The log of a job is limited to 512 MiB (536,870,912 bytes) as stored by
 	// the server (text, timestamps and separators). The call that would exceed
 	// it appends nothing; the server adds a last line saying that the limit was
 	// reached, ends the job with CI_CONCLUSION_FAILURE and RunJob.error set, and
-	// answers resource_exhausted. The runner then stops the job and sends
+	// answers failed_precondition with a JobEnded detail like any call after the
-	// nothing more: its calls get unauthenticated.
+	// end of a job. The runner then stops the job.
+	//
+	// Lines sent after the end of the job are rejected, so a runner flushes all
+	// its AppendLog lines before it calls CompleteJob.
 	//
 	// Fails with invalid_argument if a rule is broken (step number, text with a
-	// line feed or invalid UTF-8, missing time).
+	// line feed or invalid UTF-8, missing time, a continued first line).
 	AppendLog(context.Context, *connect.Request[ermite.AppendLogRequest]) (*connect.Response[ermite.AppendLogResponse], error)
 	// Heartbeat tells the server that the runner is alive and still working on
-	// the job. Needs the job token. The runner calls it every 30 seconds for as
+	// the job. Needs the job token. The first one acknowledges the assignment
-	// long as the job runs, from assignment to the end. Without a heartbeat for
+	// (see FetchJob). The runner then calls it every 30 seconds for as long as
-	// 2 minutes the server ends the job with CI_CONCLUSION_FAILURE and the message
+	// the job runs. The deadline is 2 minutes after the later of the last
-	// "The self-hosted runner: <name> lost communication with the server...".
+	// heartbeat and the start of the server (so a server restart does not fail
-	// Only Heartbeat counts: other calls do not keep a job alive.
+	// the jobs of runners that could not reach it): past it the server ends the
+	// job with CI_CONCLUSION_FAILURE and the message "The self-hosted runner:
+	// <name> lost communication with the server...". Only Heartbeat counts:
+	// other calls do not keep a job alive.
+	//
+	// A runner that gets unavailable (or any network error) retries Heartbeat
+	// every 5 seconds until it succeeds, instead of waiting for its next 30
+	// second tick, and keeps the job running meanwhile.
 	Heartbeat(context.Context, *connect.Request[ermite.HeartbeatRequest]) (*connect.Response[ermite.HeartbeatResponse], error)
-	// WatchJob streams the requests to stop a job. Needs the job token. The
+	// WatchJob streams what the runner must react to. Needs the job token. The
 	// runner opens it when it starts the job and keeps it open; it reopens it
-	// after any error other than unauthenticated.
+	// after any error other than unauthenticated and the failed_precondition
+	// that says the job ended, and after a clean close of
+	// the stream that did not end with a JobEnded message.
 	//
 	// When the stream opens and a cancellation has already been requested, the
 	// first message is that request, so a reconnect never misses one. Only the
 	// first reason is sent: a job already cancelled by the owner does not get a
-	// second message when its timeout expires. The server also sends a
+	// second message when its time budget expires. The server also sends a
-	// keep-alive message (reason CANCEL_REASON_UNSPECIFIED) at least every 30
+	// Keepalive at least every 30 seconds, which the client ignores. When the
-	// seconds, which the client ignores. The stream ends without error when
+	// job ends the server sends JobEnded, then closes the stream without error.
-	// the job ends. The runner stops the job as the official runner does
+	// On a job that has already ended the call fails like the other RPCs
+	// (failed_precondition with a JobEnded detail).
+	//
+	// On a CancelRequest the runner stops the job as the official runner does
 	// (interrupt, then terminate, then kill) and reports the conclusion given
-	// for the reason with UpdateJob.
+	// for the reason with CompleteJob.
+	//
+	// The server does not trust the runner to obey. If no conclusion arrives
+	// within 60 seconds after a cancellation request, whatever the reason (user,
+	// fail-fast, timeout), the server ends the job itself: CI_CONCLUSION_CANCELLED
+	// for user and fail-fast, CI_CONCLUSION_TIMED_OUT for timeout, with an error
+	// saying that the runner did not answer, and the steps not completed are
+	// completed as cancelled. The runner's later CompleteJob returns that result.
 	WatchJob(context.Context, *connect.Request[ermite.WatchJobRequest], *connect.ServerStream[ermite.WatchJobResponse]) error
+	// CompleteJob ends a job: it reports the final steps, the outputs and the
+	// conclusion. Needs the job token, which stays valid for this RPC for 24
+	// hours after the end of the job, however it ended. The call is applied whole
+	// or not at all. Once the job has ended, the token works for no other RPC
+	// (they answer failed_precondition with a JobEnded detail).
+	//
+	// It is idempotent. The first call that finds the job running ends it and
+	// answers applied = true. Any later call, a retry after a lost response or a
+	// call after the server ended the job itself, is not validated and changes
+	// nothing: it answers OK with the stored conclusion and error and
+	// applied = false (a runner whose conclusion differs from the stored one
+	// simply logs it; the outputs it sent are dropped).
+	//
+	// Lines sent with AppendLog after this call are rejected: a runner flushes
+	// all its log lines first. Steps that were not reported COMPLETED when the job
+	// ends are completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when
+	// they never started and the job did not end by cancellation).
+	//
+	// Fails with invalid_argument if a rule is broken (including the output
+	// limits) and failed_precondition if a step goes back, as in UpdateJob.
+	CompleteJob(context.Context, *connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error)
 }
 // NewRunnerServiceHandler builds an HTTP handler from the service implementation. It returns the
 		connect.WithSchema(runnerServiceMethods.ByName("WatchJob")),
 		connect.WithHandlerOptions(opts...),
 	)
+	runnerServiceCompleteJobHandler := connect.NewUnaryHandler(
+		RunnerServiceCompleteJobProcedure,
+		svc.CompleteJob,
+		connect.WithSchema(runnerServiceMethods.ByName("CompleteJob")),
+		connect.WithHandlerOptions(opts...),
+	)
 	return "/ermite.RunnerService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch r.URL.Path {
 		case RunnerServiceRegisterRunnerProcedure:
 			runnerServiceHeartbeatHandler.ServeHTTP(w, r)
 		case RunnerServiceWatchJobProcedure:
 			runnerServiceWatchJobHandler.ServeHTTP(w, r)
+		case RunnerServiceCompleteJobProcedure:
+			runnerServiceCompleteJobHandler.ServeHTTP(w, r)
 		default:
 			http.NotFound(w, r)
 		}
 func (UnimplementedRunnerServiceHandler) WatchJob(context.Context, *connect.Request[ermite.WatchJobRequest], *connect.ServerStream[ermite.WatchJobResponse]) error {
 	return connect.NewError(connect.CodeUnimplemented, errors.New("ermite.RunnerService.WatchJob is not implemented"))
 }
+func (UnimplementedRunnerServiceHandler) CompleteJob(context.Context, *connect.Request[ermite.CompleteJobRequest]) (*connect.Response[ermite.CompleteJobResponse], error) {
+	return nil, connect.NewError(connect.CodeUnimplemented, errors.New("ermite.RunnerService.CompleteJob is not implemented"))
+}

M gen/ermite/event.pb.go +1 -1 generated file: collapsed

load diff

M gen/ermite/run.pb.go +48 -8 generated file: collapsed

load diff

M gen/ermite/runner.pb.go +593 -164 generated file: collapsed

load diff

M gen_test.go +105 -14


@@ -3,8 +3,11 @@
3	3		package api_test
4	4
5	5		import (
	6	+	"errors"
6	7		"testing"
7	8
	9	+	"connectrpc.com/connect"
	10	+
8	11		"google.golang.org/protobuf/proto"
9	12		"google.golang.org/protobuf/reflect/protoreflect"
10	13		"google.golang.org/protobuf/types/known/fieldmaskpb"

@@ -50,6 +53,7 @@ func TestProcedures(t *testing.T) {
50	53		{ermiteconnect.RunnerServiceAppendLogProcedure, "/ermite.RunnerService/AppendLog"},
51	54		{ermiteconnect.RunnerServiceHeartbeatProcedure, "/ermite.RunnerService/Heartbeat"},
52	55		{ermiteconnect.RunnerServiceWatchJobProcedure, "/ermite.RunnerService/WatchJob"},
	56	+	{ermiteconnect.RunnerServiceCompleteJobProcedure, "/ermite.RunnerService/CompleteJob"},
53	57		{ermiteconnect.RunServiceListRunsProcedure, "/ermite.RunService/ListRuns"},
54	58		{ermiteconnect.RunServiceGetRunProcedure, "/ermite.RunService/GetRun"},
55	59		{ermiteconnect.RunServiceGetJobProcedure, "/ermite.RunService/GetJob"},

@@ -87,20 +91,40 @@ func TestEnumNumbers(t *testing.T) {
87	91		if got := ermite.DiffSide_DIFF_SIDE_NEW; got != 2 {
88	92		t.Errorf("DIFF_SIDE_NEW = %d, want 2", got)
89	93		}
90		-	if got := ermite.CiStatus_CI_STATUS_COMPLETED; got != 3 {
91		-	t.Errorf("RUN_STATUS_COMPLETED = %d, want 3", got)
92		-	}
93		-	if got := ermite.CiConclusion_CI_CONCLUSION_ACTION_REQUIRED; got != 7 {
94		-	t.Errorf("CONCLUSION_ACTION_REQUIRED = %d, want 7", got)
95		-	}
96		-	if got := ermite.StepState_STEP_STATE_COMPLETED; got != 3 {
97		-	t.Errorf("STEP_STATE_COMPLETED = %d, want 3", got)
98		-	}
99		-	if got := ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_PRIVILEGED; got != 3 {
100		-	t.Errorf("RUNNER_ISOLATION_CONTAINER_PRIVILEGED = %d, want 3", got)
101		-	}
102		-	if got := ermite.CancelReason_CANCEL_REASON_TIMEOUT; got != 3 {
103		-	t.Errorf("CANCEL_REASON_TIMEOUT = %d, want 3", got)
	94	+	enums := []struct {
	95	+	name string
	96	+	got int32
	97	+	want int32
	98	+	}{
	99	+	{"CI_STATUS_UNSPECIFIED", int32(ermite.CiStatus_CI_STATUS_UNSPECIFIED), 0},
	100	+	{"CI_STATUS_QUEUED", int32(ermite.CiStatus_CI_STATUS_QUEUED), 1},
	101	+	{"CI_STATUS_IN_PROGRESS", int32(ermite.CiStatus_CI_STATUS_IN_PROGRESS), 2},
	102	+	{"CI_STATUS_COMPLETED", int32(ermite.CiStatus_CI_STATUS_COMPLETED), 3},
	103	+	{"CI_CONCLUSION_UNSPECIFIED", int32(ermite.CiConclusion_CI_CONCLUSION_UNSPECIFIED), 0},
	104	+	{"CI_CONCLUSION_SUCCESS", int32(ermite.CiConclusion_CI_CONCLUSION_SUCCESS), 1},
	105	+	{"CI_CONCLUSION_FAILURE", int32(ermite.CiConclusion_CI_CONCLUSION_FAILURE), 2},
	106	+	{"CI_CONCLUSION_CANCELLED", int32(ermite.CiConclusion_CI_CONCLUSION_CANCELLED), 3},
	107	+	{"CI_CONCLUSION_SKIPPED", int32(ermite.CiConclusion_CI_CONCLUSION_SKIPPED), 4},
	108	+	{"CI_CONCLUSION_TIMED_OUT", int32(ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT), 5},
	109	+	{"CI_CONCLUSION_NEUTRAL", int32(ermite.CiConclusion_CI_CONCLUSION_NEUTRAL), 6},
	110	+	{"CI_CONCLUSION_ACTION_REQUIRED", int32(ermite.CiConclusion_CI_CONCLUSION_ACTION_REQUIRED), 7},
	111	+	{"STEP_STATE_UNSPECIFIED", int32(ermite.StepState_STEP_STATE_UNSPECIFIED), 0},
	112	+	{"STEP_STATE_PENDING", int32(ermite.StepState_STEP_STATE_PENDING), 1},
	113	+	{"STEP_STATE_IN_PROGRESS", int32(ermite.StepState_STEP_STATE_IN_PROGRESS), 2},
	114	+	{"STEP_STATE_COMPLETED", int32(ermite.StepState_STEP_STATE_COMPLETED), 3},
	115	+	{"RUNNER_ISOLATION_UNSPECIFIED", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_UNSPECIFIED), 0},
	116	+	{"RUNNER_ISOLATION_VM", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_VM), 1},
	117	+	{"RUNNER_ISOLATION_CONTAINER_SYSBOX", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_SYSBOX), 2},
	118	+	{"RUNNER_ISOLATION_CONTAINER_PRIVILEGED", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_PRIVILEGED), 3},
	119	+	{"CANCEL_REASON_UNSPECIFIED", int32(ermite.CancelReason_CANCEL_REASON_UNSPECIFIED), 0},
	120	+	{"CANCEL_REASON_USER", int32(ermite.CancelReason_CANCEL_REASON_USER), 1},
	121	+	{"CANCEL_REASON_FAIL_FAST", int32(ermite.CancelReason_CANCEL_REASON_FAIL_FAST), 2},
	122	+	{"CANCEL_REASON_TIMEOUT", int32(ermite.CancelReason_CANCEL_REASON_TIMEOUT), 3},
	123	+	}
	124	+	for _, e := range enums {
	125	+	if e.got != e.want {
	126	+	t.Errorf("%s = %d, want %d", e.name, e.got, e.want)
	127	+	}
104	128		}
105	129		}
106	130

@@ -238,3 +262,70 @@ func TestJobRoundTrip(t *testing.T) {
238	262		t.Errorf("round trip mismatch: %v != %v", &out, in)
239	263		}
240	264		}
	265	+
	266	+	// TestWatchJobRoundTrip checks the three kinds of WatchJob message and the
	267	+	// JobEnded detail.
	268	+	func TestWatchJobRoundTrip(t *testing.T) {
	269	+	msgs := []*ermite.WatchJobResponse{
	270	+	{Msg: &ermite.WatchJobResponse_Keepalive{Keepalive: &ermite.Keepalive{}}},
	271	+	{Msg: &ermite.WatchJobResponse_Cancel{Cancel: &ermite.CancelRequest{
	272	+	Reason: ermite.CancelReason_CANCEL_REASON_FAIL_FAST, Message: "m",
	273	+	}}},
	274	+	{Msg: &ermite.WatchJobResponse_Ended{Ended: &ermite.JobEnded{
	275	+	Conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED, Error: "e",
	276	+	}}},
	277	+	}
	278	+	for _, in := range msgs {
	279	+	b, err := proto.Marshal(in)
	280	+	if err != nil {
	281	+	t.Fatal(err)
	282	+	}
	283	+	var out ermite.WatchJobResponse
	284	+	if err := proto.Unmarshal(b, &out); err != nil {
	285	+	t.Fatal(err)
	286	+	}
	287	+	if !proto.Equal(in, &out) {
	288	+	t.Errorf("round trip mismatch: %v != %v", &out, in)
	289	+	}
	290	+	}
	291	+	// The detail travels in a Connect error.
	292	+	d, err := connect.NewErrorDetail(&ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT})
	293	+	if err != nil {
	294	+	t.Fatal(err)
	295	+	}
	296	+	cerr := connect.NewError(connect.CodeFailedPrecondition, errors.New("job ended"))
	297	+	cerr.AddDetail(d)
	298	+	v, err := cerr.Details()[0].Value()
	299	+	if err != nil {
	300	+	t.Fatal(err)
	301	+	}
	302	+	if got := v.(*ermite.JobEnded).GetConclusion(); got != ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
	303	+	t.Errorf("detail conclusion = %v", got)
	304	+	}
	305	+	}
	306	+
	307	+	// TestCompleteJobRoundTrip checks the final report of a job and its answer.
	308	+	func TestCompleteJobRoundTrip(t *testing.T) {
	309	+	in := &ermite.CompleteJobRequest{
	310	+	JobId: "j1",
	311	+	Conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED,
	312	+	Outputs: map[string]string{"a": "b"},
	313	+	RunnerShutdown: true,
	314	+	Steps: []*ermite.StepUpdate{{Number: 2, State: ermite.StepState_STEP_STATE_COMPLETED}},
	315	+	}
	316	+	b, err := proto.Marshal(in)
	317	+	if err != nil {
	318	+	t.Fatal(err)
	319	+	}
	320	+	var out ermite.CompleteJobRequest
	321	+	if err := proto.Unmarshal(b, &out); err != nil {
	322	+	t.Fatal(err)
	323	+	}
	324	+	if !proto.Equal(in, &out) {
	325	+	t.Errorf("round trip mismatch: %v != %v", &out, in)
	326	+	}
	327	+	r := &ermite.CompleteJobResponse{Applied: false, Conclusion: ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT}
	328	+	if r.GetApplied() \|\| r.GetConclusion() != ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
	329	+	t.Errorf("unexpected response %v", r)
	330	+	}
	331	+	}


@@ -3,8 +3,11 @@
3		package api_test	3		package api_test
4			4
5		import (	5		import (
			6	+	"errors"
6		"testing"	7		"testing"
7			8
			9	+	"connectrpc.com/connect"
			10	+
8		"google.golang.org/protobuf/proto"	11		"google.golang.org/protobuf/proto"
9		"google.golang.org/protobuf/reflect/protoreflect"	12		"google.golang.org/protobuf/reflect/protoreflect"
10		"google.golang.org/protobuf/types/known/fieldmaskpb"	13		"google.golang.org/protobuf/types/known/fieldmaskpb"

@@ -50,6 +53,7 @@ func TestProcedures(t *testing.T) {
50		{ermiteconnect.RunnerServiceAppendLogProcedure, "/ermite.RunnerService/AppendLog"},	53		{ermiteconnect.RunnerServiceAppendLogProcedure, "/ermite.RunnerService/AppendLog"},
51		{ermiteconnect.RunnerServiceHeartbeatProcedure, "/ermite.RunnerService/Heartbeat"},	54		{ermiteconnect.RunnerServiceHeartbeatProcedure, "/ermite.RunnerService/Heartbeat"},
52		{ermiteconnect.RunnerServiceWatchJobProcedure, "/ermite.RunnerService/WatchJob"},	55		{ermiteconnect.RunnerServiceWatchJobProcedure, "/ermite.RunnerService/WatchJob"},
			56	+	{ermiteconnect.RunnerServiceCompleteJobProcedure, "/ermite.RunnerService/CompleteJob"},
53		{ermiteconnect.RunServiceListRunsProcedure, "/ermite.RunService/ListRuns"},	57		{ermiteconnect.RunServiceListRunsProcedure, "/ermite.RunService/ListRuns"},
54		{ermiteconnect.RunServiceGetRunProcedure, "/ermite.RunService/GetRun"},	58		{ermiteconnect.RunServiceGetRunProcedure, "/ermite.RunService/GetRun"},
55		{ermiteconnect.RunServiceGetJobProcedure, "/ermite.RunService/GetJob"},	59		{ermiteconnect.RunServiceGetJobProcedure, "/ermite.RunService/GetJob"},

@@ -87,20 +91,40 @@ func TestEnumNumbers(t *testing.T) {
87		if got := ermite.DiffSide_DIFF_SIDE_NEW; got != 2 {	91		if got := ermite.DiffSide_DIFF_SIDE_NEW; got != 2 {
88		t.Errorf("DIFF_SIDE_NEW = %d, want 2", got)	92		t.Errorf("DIFF_SIDE_NEW = %d, want 2", got)
89		}	93		}
90	-	if got := ermite.CiStatus_CI_STATUS_COMPLETED; got != 3 {	94	+	enums := []struct {
91	-	t.Errorf("RUN_STATUS_COMPLETED = %d, want 3", got)	95	+	name string
92	-	}	96	+	got int32
93	-	if got := ermite.CiConclusion_CI_CONCLUSION_ACTION_REQUIRED; got != 7 {	97	+	want int32
94	-	t.Errorf("CONCLUSION_ACTION_REQUIRED = %d, want 7", got)	98	+	}{
95	-	}	99	+	{"CI_STATUS_UNSPECIFIED", int32(ermite.CiStatus_CI_STATUS_UNSPECIFIED), 0},
96	-	if got := ermite.StepState_STEP_STATE_COMPLETED; got != 3 {	100	+	{"CI_STATUS_QUEUED", int32(ermite.CiStatus_CI_STATUS_QUEUED), 1},
97	-	t.Errorf("STEP_STATE_COMPLETED = %d, want 3", got)	101	+	{"CI_STATUS_IN_PROGRESS", int32(ermite.CiStatus_CI_STATUS_IN_PROGRESS), 2},
98	-	}	102	+	{"CI_STATUS_COMPLETED", int32(ermite.CiStatus_CI_STATUS_COMPLETED), 3},
99	-	if got := ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_PRIVILEGED; got != 3 {	103	+	{"CI_CONCLUSION_UNSPECIFIED", int32(ermite.CiConclusion_CI_CONCLUSION_UNSPECIFIED), 0},
100	-	t.Errorf("RUNNER_ISOLATION_CONTAINER_PRIVILEGED = %d, want 3", got)	104	+	{"CI_CONCLUSION_SUCCESS", int32(ermite.CiConclusion_CI_CONCLUSION_SUCCESS), 1},
101	-	}	105	+	{"CI_CONCLUSION_FAILURE", int32(ermite.CiConclusion_CI_CONCLUSION_FAILURE), 2},
102	-	if got := ermite.CancelReason_CANCEL_REASON_TIMEOUT; got != 3 {	106	+	{"CI_CONCLUSION_CANCELLED", int32(ermite.CiConclusion_CI_CONCLUSION_CANCELLED), 3},
103	-	t.Errorf("CANCEL_REASON_TIMEOUT = %d, want 3", got)	107	+	{"CI_CONCLUSION_SKIPPED", int32(ermite.CiConclusion_CI_CONCLUSION_SKIPPED), 4},
			108	+	{"CI_CONCLUSION_TIMED_OUT", int32(ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT), 5},
			109	+	{"CI_CONCLUSION_NEUTRAL", int32(ermite.CiConclusion_CI_CONCLUSION_NEUTRAL), 6},
			110	+	{"CI_CONCLUSION_ACTION_REQUIRED", int32(ermite.CiConclusion_CI_CONCLUSION_ACTION_REQUIRED), 7},
			111	+	{"STEP_STATE_UNSPECIFIED", int32(ermite.StepState_STEP_STATE_UNSPECIFIED), 0},
			112	+	{"STEP_STATE_PENDING", int32(ermite.StepState_STEP_STATE_PENDING), 1},
			113	+	{"STEP_STATE_IN_PROGRESS", int32(ermite.StepState_STEP_STATE_IN_PROGRESS), 2},
			114	+	{"STEP_STATE_COMPLETED", int32(ermite.StepState_STEP_STATE_COMPLETED), 3},
			115	+	{"RUNNER_ISOLATION_UNSPECIFIED", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_UNSPECIFIED), 0},
			116	+	{"RUNNER_ISOLATION_VM", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_VM), 1},
			117	+	{"RUNNER_ISOLATION_CONTAINER_SYSBOX", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_SYSBOX), 2},
			118	+	{"RUNNER_ISOLATION_CONTAINER_PRIVILEGED", int32(ermite.RunnerIsolation_RUNNER_ISOLATION_CONTAINER_PRIVILEGED), 3},
			119	+	{"CANCEL_REASON_UNSPECIFIED", int32(ermite.CancelReason_CANCEL_REASON_UNSPECIFIED), 0},
			120	+	{"CANCEL_REASON_USER", int32(ermite.CancelReason_CANCEL_REASON_USER), 1},
			121	+	{"CANCEL_REASON_FAIL_FAST", int32(ermite.CancelReason_CANCEL_REASON_FAIL_FAST), 2},
			122	+	{"CANCEL_REASON_TIMEOUT", int32(ermite.CancelReason_CANCEL_REASON_TIMEOUT), 3},
			123	+	}
			124	+	for _, e := range enums {
			125	+	if e.got != e.want {
			126	+	t.Errorf("%s = %d, want %d", e.name, e.got, e.want)
			127	+	}
104		}	128		}
105		}	129		}
106			130

@@ -238,3 +262,70 @@ func TestJobRoundTrip(t *testing.T) {
238		t.Errorf("round trip mismatch: %v != %v", &out, in)	262		t.Errorf("round trip mismatch: %v != %v", &out, in)
239		}	263		}
240		}	264		}
			265	+
			266	+	// TestWatchJobRoundTrip checks the three kinds of WatchJob message and the
			267	+	// JobEnded detail.
			268	+	func TestWatchJobRoundTrip(t *testing.T) {
			269	+	msgs := []*ermite.WatchJobResponse{
			270	+	{Msg: &ermite.WatchJobResponse_Keepalive{Keepalive: &ermite.Keepalive{}}},
			271	+	{Msg: &ermite.WatchJobResponse_Cancel{Cancel: &ermite.CancelRequest{
			272	+	Reason: ermite.CancelReason_CANCEL_REASON_FAIL_FAST, Message: "m",
			273	+	}}},
			274	+	{Msg: &ermite.WatchJobResponse_Ended{Ended: &ermite.JobEnded{
			275	+	Conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED, Error: "e",
			276	+	}}},
			277	+	}
			278	+	for _, in := range msgs {
			279	+	b, err := proto.Marshal(in)
			280	+	if err != nil {
			281	+	t.Fatal(err)
			282	+	}
			283	+	var out ermite.WatchJobResponse
			284	+	if err := proto.Unmarshal(b, &out); err != nil {
			285	+	t.Fatal(err)
			286	+	}
			287	+	if !proto.Equal(in, &out) {
			288	+	t.Errorf("round trip mismatch: %v != %v", &out, in)
			289	+	}
			290	+	}
			291	+	// The detail travels in a Connect error.
			292	+	d, err := connect.NewErrorDetail(&ermite.JobEnded{Conclusion: ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT})
			293	+	if err != nil {
			294	+	t.Fatal(err)
			295	+	}
			296	+	cerr := connect.NewError(connect.CodeFailedPrecondition, errors.New("job ended"))
			297	+	cerr.AddDetail(d)
			298	+	v, err := cerr.Details()[0].Value()
			299	+	if err != nil {
			300	+	t.Fatal(err)
			301	+	}
			302	+	if got := v.(*ermite.JobEnded).GetConclusion(); got != ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
			303	+	t.Errorf("detail conclusion = %v", got)
			304	+	}
			305	+	}
			306	+
			307	+	// TestCompleteJobRoundTrip checks the final report of a job and its answer.
			308	+	func TestCompleteJobRoundTrip(t *testing.T) {
			309	+	in := &ermite.CompleteJobRequest{
			310	+	JobId: "j1",
			311	+	Conclusion: ermite.CiConclusion_CI_CONCLUSION_CANCELLED,
			312	+	Outputs: map[string]string{"a": "b"},
			313	+	RunnerShutdown: true,
			314	+	Steps: []*ermite.StepUpdate{{Number: 2, State: ermite.StepState_STEP_STATE_COMPLETED}},
			315	+	}
			316	+	b, err := proto.Marshal(in)
			317	+	if err != nil {
			318	+	t.Fatal(err)
			319	+	}
			320	+	var out ermite.CompleteJobRequest
			321	+	if err := proto.Unmarshal(b, &out); err != nil {
			322	+	t.Fatal(err)
			323	+	}
			324	+	if !proto.Equal(in, &out) {
			325	+	t.Errorf("round trip mismatch: %v != %v", &out, in)
			326	+	}
			327	+	r := &ermite.CompleteJobResponse{Applied: false, Conclusion: ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT}
			328	+	if r.GetApplied() \|\| r.GetConclusion() != ermite.CiConclusion_CI_CONCLUSION_TIMED_OUT {
			329	+	t.Errorf("unexpected response %v", r)
			330	+	}
			331	+	}

M proto/ermite/event.proto +5 -3


@@ -26,8 +26,10 @@ service EventService {
26	26		// "home"), subject to the same visibility. They carry the state after the
27	27		// change and are published when a run or a job is created, changes status
28	28		// or conclusion, or when one of its steps starts or ends (never per log
29		-	// line: follow a log with RunService.GetJobLog). A job update is published
30		-	// before the update of its run. There is no event for a change of title or
	29	+	// line: follow a log with RunService.GetJobLog). The order of these events
	30	+	// is not guaranteed, between jobs and runs or between two updates of the same
	31	+	// one: a client keeps the copy with the higher Run.version or
	32	+	// RunJob.version and drops the other. There is no event for a change of title or
31	33		// description, nor for a resolved comment: after a gap, or to see such
32	34		// changes, clients read the merge request again. A merge publishes
33	35		// merge_request_merged, then pushed for the target branch.

@@ -171,7 +173,7 @@ message CommentAdded {
171	173		// Gap tells the client that events were lost: the requested resume point is
172	174		// older than the server's buffer. It has no fields. The client should reload
173	175		// the state it displays (repositories, refs, merge requests and their
174		-	// comments) and carry on with the next events.
	176	+	// comments, CI runs and their jobs) and carry on with the next events.
175	177		message Gap {}
176	178
177	179		// RunUpdated reports a CI run that was created or whose status, conclusion or

   // "home"), subject to the same visibility. They carry the state after the
   // change and are published when a run or a job is created, changes status
   // or conclusion, or when one of its steps starts or ends (never per log
-  // line: follow a log with RunService.GetJobLog). A job update is published
+  // line: follow a log with RunService.GetJobLog). The order of these events
-  // before the update of its run. There is no event for a change of title or
+  // is not guaranteed, between jobs and runs or between two updates of the same
+  // one: a client keeps the copy with the higher Run.version or
+  // RunJob.version and drops the other. There is no event for a change of title or
   // description, nor for a resolved comment: after a gap, or to see such
   // changes, clients read the merge request again. A merge publishes
   // merge_request_merged, then pushed for the target branch.
 // Gap tells the client that events were lost: the requested resume point is
 // older than the server's buffer. It has no fields. The client should reload
 // the state it displays (repositories, refs, merge requests and their
-// comments) and carry on with the next events.
+// comments, CI runs and their jobs) and carry on with the next events.
 message Gap {}
 // RunUpdated reports a CI run that was created or whose status, conclusion or

M proto/ermite/run.proto +16 -0


@@ -68,6 +68,11 @@ message Run {
68	68		google.protobuf.Timestamp started_at = 16;
69	69		// CompletedAt is when this attempt completed. Unset until then.
70	70		google.protobuf.Timestamp completed_at = 17;
	71	+	// Version counts the changes of this run on the server: it grows by at least
	72	+	// one at every change and never goes back, also across attempts. Events are
	73	+	// not delivered in order, and a client that holds two copies of a run keeps
	74	+	// the one with the higher version and drops the other.
	75	+	int64 version = 18;
71	76		}
72	77
73	78		// RunJob is a job of a run as people see it: the state of one job instance

@@ -127,6 +132,11 @@ message RunJob {
127	132		google.protobuf.Timestamp started_at = 17;
128	133		// CompletedAt is when the job completed. Unset until then.
129	134		google.protobuf.Timestamp completed_at = 18;
	135	+	// Version counts the changes of this job on the server (status, conclusion,
	136	+	// runner, steps, with any attempt): it grows by at least one at every change
	137	+	// and never goes back. A client that holds two copies of a job keeps the
	138	+	// one with the higher version and drops the other, as for Run.version.
	139	+	int64 version = 19;
130	140		}
131	141
132	142		// Step is one step of a job, with where its lines are in the job log.

@@ -405,6 +415,12 @@ message JobLogLine {
405	415		// Text is the line without its terminator, as in LogLine.text, including the
406	416		// "##[group]" and "##[endgroup]" markers. Clients show it escaped.
407	417		string text = 3;
	418	+	// Continued is true when the line continues the line before it, which was
	419	+	// too long for the runner to send whole (see LogLine.continued). Pieces keep
	420	+	// their own line numbers, so offsets are the same as the runner's; a client
	421	+	// that shows lines joins a continued line to the previous one. The raw log
	422	+	// file already has them joined.
	423	+	bool continued = 4;
408	424		}
409	425
410	426		// CancelRunRequest names the run to cancel.


@@ -68,6 +68,11 @@ message Run {
68	google.protobuf.Timestamp started_at = 16;	68		google.protobuf.Timestamp started_at = 16;
69	// CompletedAt is when this attempt completed. Unset until then.	69		// CompletedAt is when this attempt completed. Unset until then.
70	google.protobuf.Timestamp completed_at = 17;	70		google.protobuf.Timestamp completed_at = 17;
		71	+	// Version counts the changes of this run on the server: it grows by at least
		72	+	// one at every change and never goes back, also across attempts. Events are
		73	+	// not delivered in order, and a client that holds two copies of a run keeps
		74	+	// the one with the higher version and drops the other.
		75	+	int64 version = 18;
71	}	76		}
72		77
73	// RunJob is a job of a run as people see it: the state of one job instance	78		// RunJob is a job of a run as people see it: the state of one job instance

@@ -127,6 +132,11 @@ message RunJob {
127	google.protobuf.Timestamp started_at = 17;	132		google.protobuf.Timestamp started_at = 17;
128	// CompletedAt is when the job completed. Unset until then.	133		// CompletedAt is when the job completed. Unset until then.
129	google.protobuf.Timestamp completed_at = 18;	134		google.protobuf.Timestamp completed_at = 18;
		135	+	// Version counts the changes of this job on the server (status, conclusion,
		136	+	// runner, steps, with any attempt): it grows by at least one at every change
		137	+	// and never goes back. A client that holds two copies of a job keeps the
		138	+	// one with the higher version and drops the other, as for Run.version.
		139	+	int64 version = 19;
130	}	140		}
131		141
132	// Step is one step of a job, with where its lines are in the job log.	142		// Step is one step of a job, with where its lines are in the job log.

@@ -405,6 +415,12 @@ message JobLogLine {
405	// Text is the line without its terminator, as in LogLine.text, including the	415		// Text is the line without its terminator, as in LogLine.text, including the
406	// "##[group]" and "##[endgroup]" markers. Clients show it escaped.	416		// "##[group]" and "##[endgroup]" markers. Clients show it escaped.
407	string text = 3;	417		string text = 3;
		418	+	// Continued is true when the line continues the line before it, which was
		419	+	// too long for the runner to send whole (see LogLine.continued). Pieces keep
		420	+	// their own line numbers, so offsets are the same as the runner's; a client
		421	+	// that shows lines joins a continued line to the previous one. The raw log
		422	+	// file already has them joined.
		423	+	bool continued = 4;
408	}	424		}
409		425
410	// CancelRunRequest names the run to cancel.	426		// CancelRunRequest names the run to cancel.

M proto/ermite/runner.proto +273 -101


@@ -4,6 +4,7 @@ syntax = "proto3";
4	4
5	5		package ermite;
6	6
	7	+	import "google/protobuf/duration.proto";
7	8		import "google/protobuf/timestamp.proto";
8	9
9	10		// CiStatus is where a run, a job or a check is in its life. It is the same

@@ -79,8 +80,7 @@ enum RunnerIsolation {
79	80
80	81		// CancelReason says why the server asks a runner to stop a job.
81	82		enum CancelReason {
82		-	// Not set. A WatchJobResponse with this reason is a keep-alive and carries
83		-	// no request.
	83	+	// Not set. Never sent in a CancelRequest.
84	84		CANCEL_REASON_UNSPECIFIED = 0;
85	85		// The owner cancelled the run (RunService.CancelRun, web or CLI). The job
86	86		// ends CI_CONCLUSION_CANCELLED.

@@ -88,38 +88,55 @@ enum CancelReason {
88	88		// Another job of the same matrix failed and the strategy has fail-fast. The
89	89		// job ends CI_CONCLUSION_CANCELLED.
90	90		CANCEL_REASON_FAIL_FAST = 2;
91		-	// The job exceeded its timeout_minutes without the runner ending it. The
92		-	// job ends CI_CONCLUSION_TIMED_OUT.
	91	+	// The job exceeded its time budget (see Job.timeout_minutes). The job ends
	92	+	// CI_CONCLUSION_TIMED_OUT.
93	93		CANCEL_REASON_TIMEOUT = 3;
94	94		}
95	95
96	96		// RunnerService is what a CI runner (ermite-runner) calls to get jobs and
97	97		// report on them. It is not for people: owners use RunService.
98	98		//
99		-	// Three credentials are used, each by different RPCs:
	99	+	// Three credentials are used, each by different RPCs. All are sent as
	100	+	// "Authorization: Bearer <secret>":
100	101		//
101	102		// - RegisterRunner: a stored token with the runner:register scope, or the
102		-	// root token, as "Authorization: Bearer <secret>".
103		-	// - FetchJob: the runner token returned by RegisterRunner, as
104		-	// "Authorization: Bearer <runner_token>". It identifies the runner. It
105		-	// only works for FetchJob.
106		-	// - UpdateJob, AppendLog, Heartbeat, WatchJob: the job token of the Job that
107		-	// FetchJob returned, as "Authorization: Bearer <job_token>". It is valid
108		-	// for that one job and attempt, from its assignment until it ends. The
109		-	// job_id of the request must be the one of the token.
	103	+	// root token.
	104	+	// - FetchJob: the runner token returned by RegisterRunner. It identifies the
	105	+	// runner and only works for FetchJob. It is "ermiterun_" followed by 43
	106	+	// base64url characters (32 random bytes, no padding).
	107	+	// - UpdateJob, AppendLog, Heartbeat, WatchJob, CompleteJob: the job token of
	108	+	// the Job that FetchJob returned. It is valid for that one job and that
	109	+	// one assignment. It is "ermitejob_" followed by 43 base64url characters
	110	+	// (32 random bytes, no padding). The job_id of the request must be the
	111	+	// one of the token.
110	112		//
111		-	// A request without a credential, with an unknown, expired or revoked one, or
112		-	// with a credential of the wrong kind for the RPC, gets unauthenticated. A
113		-	// job token that is valid but belongs to another job than the job_id of the
114		-	// request gets permission_denied. A job that has ended (concluded, cancelled,
115		-	// failed by the server, or whose runner was lost) no longer has a valid job
116		-	// token: every later call gets unauthenticated, which a runner treats as "the
117		-	// job is over" and stops sending anything for it. Repeated failed
118		-	// authentications are rate limited per client address.
	113	+	// Errors. unauthenticated means only that the credential is bad: missing,
	114	+	// unknown, expired, revoked, of the wrong kind for the RPC, or a job token
	115	+	// that was replaced (the job was given again, see FetchJob) or is older than
	116	+	// the 24 hours below. A runner that gets it for a job abandons the job at
	117	+	// once. A job token that is valid but belongs to another job than the job_id
	118	+	// of the request gets permission_denied. Repeated failed authentications are
	119	+	// rate limited per client address.
119	120		//
120		-	// Runner tokens and job tokens are secrets: the server never logs them and
121		-	// clients must not log them. The job token stays in the runner host process; it
122		-	// is never given to the sandbox that runs the job's code.
	121	+	// The end of a job. A job ends when its conclusion is set, by CompleteJob or
	122	+	// by the server (cancellation not honoured, lost runner, log limit, queue
	123	+	// timeout). From then on UpdateJob, AppendLog, Heartbeat and WatchJob answer
	124	+	// failed_precondition with a JobEnded error detail (a JobEnded message in the
	125	+	// Connect error details) that gives the conclusion; the runner stops the job
	126	+	// and sends nothing more. The job token is still recognised for 24 hours after
	127	+	// the end, so that these calls can say so and so that CompleteJob, the only
	128	+	// call that still does something, returns the stored result to a runner that
	129	+	// lost the answer of its last call or that was cut off by the server. After
	130	+	// the 24 hours every call gets unauthenticated.
	131	+	//
	132	+	// Secrets and trust. Runner tokens and job tokens are secrets: the server
	133	+	// never logs them and clients must not log them. The job token stays in the
	134	+	// runner host process; it is never given to the sandbox that runs the job's
	135	+	// code. A runner receives the jobs of every repository, with their workflow
	136	+	// files, contexts and (from CI-2) their code, and from CI-3 their secrets: the
	137	+	// runner:register scope therefore effectively gives read access to the code
	138	+	// and the secrets of every repository, private ones included. Give it only to
	139	+	// runners you operate.
123	140		//
124	141		// Servers that implement this service list "ci.runners" in
125	142		// GetVersionResponse.features.

@@ -130,37 +147,54 @@ service RunnerService {
130	147		// (a runner serves every repository), or the root token. Fails with
131	148		// unauthenticated without a valid token, permission_denied without the
132	149		// scope or with a repository-restricted token, invalid_argument if a field
133		-	// breaks a rule, and already_exists if name is taken by another runner.
	150	+	// breaks a rule, and already_exists if name is taken by another runner and
	151	+	// replace is not set.
134	152		rpc RegisterRunner(RegisterRunnerRequest) returns (RegisterRunnerResponse);
135	153		// FetchJob asks for a job to run (long poll). The server holds the call
136	154		// until a queued job matches the runner, or about 50 seconds (never more
137	155		// than 50) pass, then answers; without a job the response has no job and the
138		-	// runner calls again at once. Clients set a call timeout above 50 seconds
139		-	// (60 is suggested) and must not poll when they have no free capacity.
	156	+	// runner calls again after retry_after. Clients set a call timeout above 50
	157	+	// seconds (60 is suggested) and must not poll when they have no free
	158	+	// capacity.
140	159		//
141	160		// A job matches when every label of its runs-on is among the labels of the
142	161		// runner (case-insensitive). A job is given to exactly one runner, even when
143	162		// many runners and many polls compete; the assignment is recorded before
144		-	// the response is sent, and the job is then in progress. If the response is
145		-	// lost the job is not offered again: with no Heartbeat for 2 minutes the
146		-	// server fails it as a lost runner.
	163	+	// the response is sent. The job stays QUEUED for the owner until it is
	164	+	// acknowledged.
	165	+	//
	166	+	// Acknowledgement. The runner's first Heartbeat for the job is the
	167	+	// acknowledgement, and the runner sends it as soon as it has the job, before
	168	+	// anything else. If none arrives within 30 seconds of the assignment (the
	169	+	// response was lost, or the runner died), the server cancels the assignment:
	170	+	// the job token is revoked, the job goes back to QUEUED with a fresh token
	171	+	// and can be given to any runner, the same one included; nothing the old
	172	+	// assignment stored is kept. This is not a failure of the job. A job whose
	173	+	// assignment is cancelled 3 times in a row is failed with CI_CONCLUSION_FAILURE
	174	+	// and an error saying so.
	175	+	//
	176	+	// Retry after a lost response. poll_id lets a client retry a poll safely:
	177	+	// a call with the same poll_id as an earlier call of the same runner that
	178	+	// assigned a job returns that same assignment (same Job, same token) as long
	179	+	// as it has not been acknowledged or cancelled, instead of taking another.
	180	+	//
	181	+	// A runner that crashed or whose host restarted cannot resume a job: it does
	182	+	// not keep the job token, so the job fails with the lost-runner error after
	183	+	// the heartbeat timeout (see Heartbeat).
147	184		//
148	185		// Needs the runner token. Fails with unauthenticated when the runner was
149	186		// removed (also ending a call in progress) and invalid_argument if labels
150		-	// breaks a rule.
	187	+	// or poll_id break a rule.
151	188		rpc FetchJob(FetchJobRequest) returns (FetchJobResponse);
152		-	// UpdateJob reports the state of the steps of a job and, once, how the job
153		-	// ended. Needs the job token. Each call is applied whole or not at all, and
154		-	// repeating an identical call changes nothing, so a runner may retry after a
155		-	// network error.
	189	+	// UpdateJob reports the state of the steps of a job while it runs. Needs the
	190	+	// job token. Each call is applied whole or not at all, and repeating an
	191	+	// identical call changes nothing, so a runner may retry after a network
	192	+	// error. To end the job use CompleteJob.
156	193		//
157		-	// Fails with invalid_argument if a rule is broken (including the output
158		-	// limits), and with failed_precondition if a step goes back (a COMPLETED
159		-	// step cannot change, a step cannot go from IN_PROGRESS to PENDING).
160		-	// The call that carries a conclusion ends the job: its job token stops
161		-	// working, the server completes the job and no further call is accepted,
162		-	// so a retry of that last call after a lost response gets unauthenticated,
163		-	// which means it was applied.
	194	+	// Fails with invalid_argument if a rule is broken, with failed_precondition
	195	+	// if a step goes back (a COMPLETED step cannot change, a step cannot go from
	196	+	// IN_PROGRESS to PENDING) and with failed_precondition and a JobEnded detail
	197	+	// if the job has ended.
164	198		rpc UpdateJob(UpdateJobRequest) returns (UpdateJobResponse);
165	199		// AppendLog appends lines to the log of a job. Needs the job token.
166	200		//

@@ -172,46 +206,102 @@ service RunnerService {
172	206		// - offset equal to the server's line count: the lines are appended;
173	207		// - offset below it: the first (count - offset) lines were already stored
174	208		// (a retry after a lost response) and are skipped, and the rest is
175		-	// appended; a request entirely stored changes nothing;
	209	+	// appended; a request entirely stored changes nothing. The skipped lines
	210	+	// are not compared with the stored ones: a retry whose lines differ from
	211	+	// the first attempt is silently dropped, so a client must resend the
	212	+	// same lines at the same offset;
176	213		// - offset above it: out_of_range (a gap is a bug of the client; the
177	214		// message gives the line count the server holds).
178	215		//
179	216		// The response gives the line count after the call, so a client can check
180		-	// itself. A request has at most 1,000 lines whose text adds up to at most
181		-	// 64 KiB (65,536 bytes), all in the same step; more is invalid_argument. A
182		-	// line of more than 64 KiB is cut by the client into several lines.
	217	+	// itself. Offsets resume only after a network error within the same runner
	218	+	// process; they are not a way to resume a job after a crash.
	219	+	//
	220	+	// A request has at most 1,000 lines whose text adds up to at most 64 KiB
	221	+	// (65,536 bytes), all in the same step; more is invalid_argument. A line
	222	+	// longer than that is cut by the client into pieces, see LogLine.continued.
	223	+	//
	224	+	// Steps. The step of a request must be at least the step of the last line
	225	+	// the server holds, because the log is ordered by step (applied lines only:
	226	+	// lines skipped as already stored are not checked); a lower step is
	227	+	// failed_precondition. Lines for a step that UpdateJob has not announced
	228	+	// create it: it starts IN_PROGRESS at the time of its first line, with an
	229	+	// empty name until an UpdateJob gives one.
183	230		//
184	231		// The log of a job is limited to 512 MiB (536,870,912 bytes) as stored by
185	232		// the server (text, timestamps and separators). The call that would exceed
186	233		// it appends nothing; the server adds a last line saying that the limit was
187	234		// reached, ends the job with CI_CONCLUSION_FAILURE and RunJob.error set, and
188		-	// answers resource_exhausted. The runner then stops the job and sends
189		-	// nothing more: its calls get unauthenticated.
	235	+	// answers failed_precondition with a JobEnded detail like any call after the
	236	+	// end of a job. The runner then stops the job.
	237	+	//
	238	+	// Lines sent after the end of the job are rejected, so a runner flushes all
	239	+	// its AppendLog lines before it calls CompleteJob.
190	240		//
191	241		// Fails with invalid_argument if a rule is broken (step number, text with a
192		-	// line feed or invalid UTF-8, missing time).
	242	+	// line feed or invalid UTF-8, missing time, a continued first line).
193	243		rpc AppendLog(AppendLogRequest) returns (AppendLogResponse);
194	244		// Heartbeat tells the server that the runner is alive and still working on
195		-	// the job. Needs the job token. The runner calls it every 30 seconds for as
196		-	// long as the job runs, from assignment to the end. Without a heartbeat for
197		-	// 2 minutes the server ends the job with CI_CONCLUSION_FAILURE and the message
198		-	// "The self-hosted runner: <name> lost communication with the server...".
199		-	// Only Heartbeat counts: other calls do not keep a job alive.
	245	+	// the job. Needs the job token. The first one acknowledges the assignment
	246	+	// (see FetchJob). The runner then calls it every 30 seconds for as long as
	247	+	// the job runs. The deadline is 2 minutes after the later of the last
	248	+	// heartbeat and the start of the server (so a server restart does not fail
	249	+	// the jobs of runners that could not reach it): past it the server ends the
	250	+	// job with CI_CONCLUSION_FAILURE and the message "The self-hosted runner:
	251	+	// <name> lost communication with the server...". Only Heartbeat counts:
	252	+	// other calls do not keep a job alive.
	253	+	//
	254	+	// A runner that gets unavailable (or any network error) retries Heartbeat
	255	+	// every 5 seconds until it succeeds, instead of waiting for its next 30
	256	+	// second tick, and keeps the job running meanwhile.
200	257		rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse);
201		-	// WatchJob streams the requests to stop a job. Needs the job token. The
	258	+	// WatchJob streams what the runner must react to. Needs the job token. The
202	259		// runner opens it when it starts the job and keeps it open; it reopens it
203		-	// after any error other than unauthenticated.
	260	+	// after any error other than unauthenticated and the failed_precondition
	261	+	// that says the job ended, and after a clean close of
	262	+	// the stream that did not end with a JobEnded message.
204	263		//
205	264		// When the stream opens and a cancellation has already been requested, the
206	265		// first message is that request, so a reconnect never misses one. Only the
207	266		// first reason is sent: a job already cancelled by the owner does not get a
208		-	// second message when its timeout expires. The server also sends a
209		-	// keep-alive message (reason CANCEL_REASON_UNSPECIFIED) at least every 30
210		-	// seconds, which the client ignores. The stream ends without error when
211		-	// the job ends. The runner stops the job as the official runner does
	267	+	// second message when its time budget expires. The server also sends a
	268	+	// Keepalive at least every 30 seconds, which the client ignores. When the
	269	+	// job ends the server sends JobEnded, then closes the stream without error.
	270	+	// On a job that has already ended the call fails like the other RPCs
	271	+	// (failed_precondition with a JobEnded detail).
	272	+	//
	273	+	// On a CancelRequest the runner stops the job as the official runner does
212	274		// (interrupt, then terminate, then kill) and reports the conclusion given
213		-	// for the reason with UpdateJob.
	275	+	// for the reason with CompleteJob.
	276	+	//
	277	+	// The server does not trust the runner to obey. If no conclusion arrives
	278	+	// within 60 seconds after a cancellation request, whatever the reason (user,
	279	+	// fail-fast, timeout), the server ends the job itself: CI_CONCLUSION_CANCELLED
	280	+	// for user and fail-fast, CI_CONCLUSION_TIMED_OUT for timeout, with an error
	281	+	// saying that the runner did not answer, and the steps not completed are
	282	+	// completed as cancelled. The runner's later CompleteJob returns that result.
214	283		rpc WatchJob(WatchJobRequest) returns (stream WatchJobResponse);
	284	+	// CompleteJob ends a job: it reports the final steps, the outputs and the
	285	+	// conclusion. Needs the job token, which stays valid for this RPC for 24
	286	+	// hours after the end of the job, however it ended. The call is applied whole
	287	+	// or not at all. Once the job has ended, the token works for no other RPC
	288	+	// (they answer failed_precondition with a JobEnded detail).
	289	+	//
	290	+	// It is idempotent. The first call that finds the job running ends it and
	291	+	// answers applied = true. Any later call, a retry after a lost response or a
	292	+	// call after the server ended the job itself, is not validated and changes
	293	+	// nothing: it answers OK with the stored conclusion and error and
	294	+	// applied = false (a runner whose conclusion differs from the stored one
	295	+	// simply logs it; the outputs it sent are dropped).
	296	+	//
	297	+	// Lines sent with AppendLog after this call are rejected: a runner flushes
	298	+	// all its log lines first. Steps that were not reported COMPLETED when the job
	299	+	// ends are completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when
	300	+	// they never started and the job did not end by cancellation).
	301	+	//
	302	+	// Fails with invalid_argument if a rule is broken (including the output
	303	+	// limits) and failed_precondition if a step goes back, as in UpdateJob.
	304	+	rpc CompleteJob(CompleteJobRequest) returns (CompleteJobResponse);
215	305		}
216	306
217	307		// RegisterRunnerRequest describes the runner to register.

@@ -237,6 +327,13 @@ message RegisterRunnerRequest {
237	327		// Os is the operating system: "linux". Other values are invalid_argument
238	328		// (the set may grow with compatible additions).
239	329		string os = 6;
	330	+	// Replace removes the runner that already has this name, if any, before
	331	+	// registering this one: its runner token is revoked at once, as in
	332	+	// RunService.RemoveRunner, and a job it is running is not affected. The new
	333	+	// runner gets a new runner_id. Needs the same scope as registering. Without
	334	+	// replace, a name in use is already_exists. It is how a runner re-registers
	335	+	// after losing its state.
	336	+	bool replace = 7;
240	337		}
241	338
242	339		// RegisterRunnerResponse holds the new runner and its secret.

@@ -245,9 +342,10 @@ message RegisterRunnerResponse {
245	342		// a secret.
246	343		string runner_id = 1;
247	344		// RunnerToken is the secret of the runner, to present as "Authorization:
248		-	// Bearer <runner_token>" to FetchJob. It starts with "ermiterun_" and is
249		-	// returned once: the server keeps only its SHA-256, the runner must store
250		-	// it (mode 0600). It stops working when the runner is removed.
	345	+	// Bearer <runner_token>" to FetchJob: "ermiterun_" and 43 base64url
	346	+	// characters (32 random bytes, no padding). It is returned once: the server
	347	+	// keeps only its SHA-256, the runner must store it (mode 0600). It stops
	348	+	// working when the runner is removed or replaced.
251	349		string runner_token = 2;
252	350		}
253	351

@@ -258,6 +356,13 @@ message FetchJobRequest {
258	356		// a label of the runner (invalid_argument if not) and only jobs whose runs-on
259	357		// labels are all among them are offered.
260	358		repeated string labels = 1;
	359	+	// PollId identifies this poll, chosen by the client (for example 16 random
	360	+	// bytes in base64url), 1 to 64 characters among letters, digits, "_" and
	361	+	// "-". A client that retries a call whose response it did not get sends the
	362	+	// same poll_id, and gets the assignment the first call made, if it made one
	363	+	// (see FetchJob). A new poll uses a new id. Empty means no retry protection:
	364	+	// a lost assignment is then recovered by the acknowledgement timeout alone.
	365	+	string poll_id = 2;
261	366		}
262	367
263	368		// FetchJobResponse holds the job given to the runner, if any.

@@ -265,6 +370,11 @@ message FetchJobResponse {
265	370		// Job is the job to run. Unset when no job was available before the end of
266	371		// the poll.
267	372		Job job = 1;
	373	+	// RetryAfter is how long the client waits before its next FetchJob. Unset or
	374	+	// zero means call again at once. The server sets it when it has no job for
	375	+	// the runner and wants to slow polling down (load, shutdown in progress);
	376	+	// clients obey it and may wait longer, never less. Unset when job is set.
	377	+	google.protobuf.Duration retry_after = 2;
268	378		}
269	379
270	380		// Job is a job given to a runner: everything needed to run it, nothing more.

@@ -282,10 +392,13 @@ message Job {
282	392		// one more after each RunService.RerunRun. It is the "github.run_attempt".
283	393		int32 attempt = 3;
284	394		// JobToken is the secret that authenticates this job's UpdateJob,
285		-	// AppendLog, Heartbeat and WatchJob calls ("Authorization: Bearer
286		-	// <job_token>"). It is random, valid until the job ends, survives a server
287		-	// restart and is shown only in this response. It must stay in the runner
288		-	// host process and never reach the job's code.
	395	+	// AppendLog, Heartbeat, WatchJob and CompleteJob calls ("Authorization:
	396	+	// Bearer <job_token>"): "ermitejob_" and 43 base64url characters (32 random
	397	+	// bytes, no padding). It is random, valid from the assignment until the job
	398	+	// ends (and for CompleteJob 24 hours more), survives a server restart and
	399	+	// is shown only in this response, or in a repeat of the same poll_id. It
	400	+	// must stay in the runner host process and never reach the job's code. A new
	401	+	// assignment of the same job has a new token.
289	402		string job_token = 4;
290	403		// WorkflowPath is the path of the workflow file in the repository, for
291	404		// example ".github/workflows/ci.yml".

@@ -310,8 +423,11 @@ message Job {
310	423		repeated string labels = 10;
311	424		// TimeoutMinutes is the timeout of the job in minutes: the value of the
312	425		// workflow's timeout-minutes, or 360 when it has none. At least 1. The
313		-	// runner enforces it from the start of the job; the server ends a job that
314		-	// exceeds it by announcing CANCEL_REASON_TIMEOUT.
	426	+	// runner enforces it from the moment it receives the job. The server counts
	427	+	// from the moment of the assignment and adds 60 seconds of slack, so the
	428	+	// runner's own timeout always comes first: when the server's deadline
	429	+	// passes it announces CANCEL_REASON_TIMEOUT, and ends the job itself after
	430	+	// 60 more seconds without a conclusion (see WatchJob).
315	431		int32 timeout_minutes = 11;
316	432		// Repo is the name of the repository.
317	433		string repo = 12;

@@ -323,7 +439,7 @@ message Job {
323	439		string ref = 14;
324	440		}
325	441
326		-	// UpdateJobRequest reports the steps and, in the last call, the end of a job.
	442	+	// UpdateJobRequest reports the steps of a running job.
327	443		message UpdateJobRequest {
328	444		// JobId is the Job.job_id. It must be the job of the token.
329	445		string job_id = 1;

@@ -333,28 +449,6 @@ message UpdateJobRequest {
333	449		// implicit "Set up job" is 1, the steps of the workflow follow in order, and
334	450		// the implicit "Complete job" is last.
335	451		repeated StepUpdate steps = 2;
336		-	// Outputs are the outputs of the job, evaluated by the runner. Only allowed
337		-	// in the call that carries the conclusion (invalid_argument otherwise).
338		-	// Names are 1 to 255 characters among letters, digits, "_" and "-" and
339		-	// begin with a letter or "_" and are case-insensitive: two names that
340		-	// differ only by case are invalid_argument. Each value is at most 1 MiB, and
341		-	// all names and values together at most 1 MiB (1,048,576 bytes); beyond that
342		-	// the whole call is rejected with invalid_argument and the runner reports
343		-	// the failure of the job without outputs, with a line in the log.
344		-	map<string, string> outputs = 3;
345		-	// Conclusion ends the job when set. A runner reports CI_CONCLUSION_SUCCESS,
346		-	// CI_CONCLUSION_FAILURE, CI_CONCLUSION_CANCELLED or CI_CONCLUSION_TIMED_OUT; any
347		-	// other value is invalid_argument. CI_CONCLUSION_UNSPECIFIED means the job goes
348		-	// on. Steps that were not reported COMPLETED when the job ends are
349		-	// completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when they
350		-	// never started and the job did not fail by cancellation).
351		-	CiConclusion conclusion = 4;
352		-	// RunnerShutdown is true when the runner ends the job because it was told to
353		-	// stop (service stopped, signal). Only allowed with CI_CONCLUSION_CANCELLED.
354		-	// The server then annotates the job with GitHub's message "The runner has
355		-	// received a shutdown signal. This can happen when the runner service is
356		-	// stopped, or a manually started runner is canceled."
357		-	bool runner_shutdown = 5;
358	452		}
359	453
360	454		// StepUpdate is the state of one step as the runner knows it.

@@ -383,13 +477,67 @@ message StepUpdate {
383	477		// UpdateJobResponse is empty.
384	478		message UpdateJobResponse {}
385	479
	480	+	// CompleteJobRequest ends a job.
	481	+	message CompleteJobRequest {
	482	+	// JobId is the Job.job_id. It must be the job of the token.
	483	+	string job_id = 1;
	484	+	// Conclusion is how the job ended. Required: a runner reports
	485	+	// CI_CONCLUSION_SUCCESS, CI_CONCLUSION_FAILURE, CI_CONCLUSION_CANCELLED or
	486	+	// CI_CONCLUSION_TIMED_OUT; any other value is invalid_argument. Skipped is
	487	+	// decided by the server and never reported by a runner.
	488	+	CiConclusion conclusion = 2;
	489	+	// Outputs are the outputs of the job, evaluated by the runner. Names are 1
	490	+	// to 255 characters among letters, digits, "_" and "-" and begin with a
	491	+	// letter or "_" and are case-insensitive: two names that differ only by case
	492	+	// are invalid_argument. Each value is at most 1 MiB, and all names and
	493	+	// values together at most 1 MiB (1,048,576 bytes); beyond that the whole call
	494	+	// is rejected with invalid_argument and the runner reports the failure of
	495	+	// the job without outputs, with a line in the log.
	496	+	map<string, string> outputs = 3;
	497	+	// RunnerShutdown is true when the runner ends the job because it was told to
	498	+	// stop (service stopped, signal). Only allowed with CI_CONCLUSION_CANCELLED.
	499	+	// The server then annotates the job with GitHub's message "The runner has
	500	+	// received a shutdown signal. This can happen when the runner service is
	501	+	// stopped, or a manually started runner is canceled." The conclusion
	502	+	// CANCELLED for this case is to be verified against a recording of GitHub.
	503	+	bool runner_shutdown = 4;
	504	+	// Steps are the final steps, with the same meaning as UpdateJobRequest.steps,
	505	+	// applied in the same atomic step as the conclusion.
	506	+	repeated StepUpdate steps = 5;
	507	+	}
	508	+
	509	+	// CompleteJobResponse is the stored result of a job.
	510	+	message CompleteJobResponse {
	511	+	// Conclusion is the conclusion of the job as the server stored it. It is the
	512	+	// one of the request when applied is true.
	513	+	CiConclusion conclusion = 1;
	514	+	// Error is the error text stored with the job, as RunJob.error. Empty when
	515	+	// there is none.
	516	+	string error = 2;
	517	+	// Applied is true when this call ended the job, false when the job had
	518	+	// already ended (a retry, or the server ended it first) and nothing changed.
	519	+	bool applied = 3;
	520	+	}
	521	+
	522	+	// JobEnded says that a job has ended. It is sent in WatchJob and attached, as
	523	+	// an error detail, to the failed_precondition errors that RunnerService returns
	524	+	// for a job that has ended.
	525	+	message JobEnded {
	526	+	// Conclusion is how the job ended.
	527	+	CiConclusion conclusion = 1;
	528	+	// Error is the error text stored with the job, as RunJob.error. Empty when
	529	+	// there is none.
	530	+	string error = 2;
	531	+	}
	532	+
386	533		// AppendLogRequest carries log lines of one step.
387	534		message AppendLogRequest {
388	535		// JobId is the Job.job_id. It must be the job of the token.
389	536		string job_id = 1;
390	537		// Step is the number of the step the lines belong to (see
391		-	// UpdateJobRequest.steps), at least 1. A step that UpdateJob has not
392		-	// announced yet is created by the first lines.
	538	+	// UpdateJobRequest.steps), at least 1, and at least the step of the last
	539	+	// stored line. A step that UpdateJob has not announced yet is created by the
	540	+	// first lines. See the rules on AppendLog.
393	541		int32 step = 2;
394	542		// Offset is the number of the first line of the request in the log of the
395	543		// job, counted from 0 across all steps. See the rules on AppendLog.

@@ -409,6 +557,15 @@ message LogLine {
409	557		// carriage return is kept. It is the line as GitHub's raw log has it,
410	558		// including "##[group]" and "##[endgroup]" markers and "##[error]" lines.
411	559		string text = 2;
	560	+	// Continued is true when this line is the continuation of the line before
	561	+	// it, which was too long to send whole. A runner cuts a line of more than
	562	+	// 64 KiB into pieces at UTF-8 character boundaries, each piece of at most
	563	+	// 64 KiB, and sets continued on every piece but the first. Each piece is one
	564	+	// line for numbering and offsets; the server and the raw log join the
	565	+	// pieces back, so the original line is rebuilt exactly. The first line of a
	566	+	// job cannot be continued (invalid_argument) and a continued line is in
	567	+	// the same step as the line before it (failed_precondition otherwise).
	568	+	bool continued = 3;
412	569		}
413	570
414	571		// AppendLogResponse tells how much of the log the server holds.

@@ -433,13 +590,28 @@ message WatchJobRequest {
433	590		string job_id = 1;
434	591		}
435	592
436		-	// WatchJobResponse is one message of the stream: a request to cancel the job,
437		-	// or a keep-alive.
	593	+	// WatchJobResponse is one message of the stream.
438	594		message WatchJobResponse {
439		-	// Reason is why the job must stop. CANCEL_REASON_UNSPECIFIED marks a
440		-	// keep-alive: the other fields are empty and the client ignores it. New
441		-	// reasons may be added; a runner that does not know one cancels the job and
442		-	// reports CI_CONCLUSION_CANCELLED.
	595	+	// Msg is what the message carries. New kinds may be added: a client ignores
	596	+	// the ones it does not know.
	597	+	oneof msg {
	598	+	// Keepalive: nothing to do, the stream is alive.
	599	+	Keepalive keepalive = 1;
	600	+	// Cancel: the job must stop.
	601	+	CancelRequest cancel = 2;
	602	+	// Ended: the job has ended. The last message of a stream: a close without
	603	+	// it means the client reconnects.
	604	+	JobEnded ended = 3;
	605	+	}
	606	+	}
	607	+
	608	+	// Keepalive has no fields.
	609	+	message Keepalive {}
	610	+
	611	+	// CancelRequest asks the runner to stop a job.
	612	+	message CancelRequest {
	613	+	// Reason is why the job must stop. New reasons may be added; a runner that
	614	+	// does not know one cancels the job and reports CI_CONCLUSION_CANCELLED.
443	615		CancelReason reason = 1;
444	616		// Message is a short human-readable explanation, for the job log, for
445	617		// example "Canceled by the owner". At most 1,024 characters.

 package ermite;
+import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 // CiStatus is where a run, a job or a check is in its life. It is the same
 // CancelReason says why the server asks a runner to stop a job.
 enum CancelReason {
-  // Not set. A WatchJobResponse with this reason is a keep-alive and carries
+  // Not set. Never sent in a CancelRequest.
-  // no request.
   CANCEL_REASON_UNSPECIFIED = 0;
   // The owner cancelled the run (RunService.CancelRun, web or CLI). The job
   // ends CI_CONCLUSION_CANCELLED.
   // Another job of the same matrix failed and the strategy has fail-fast. The
   // job ends CI_CONCLUSION_CANCELLED.
   CANCEL_REASON_FAIL_FAST = 2;
-  // The job exceeded its timeout_minutes without the runner ending it. The
+  // The job exceeded its time budget (see Job.timeout_minutes). The job ends
-  // job ends CI_CONCLUSION_TIMED_OUT.
+  // CI_CONCLUSION_TIMED_OUT.
   CANCEL_REASON_TIMEOUT = 3;
 }
 // RunnerService is what a CI runner (ermite-runner) calls to get jobs and
 // report on them. It is not for people: owners use RunService.
 //
-// Three credentials are used, each by different RPCs:
+// Three credentials are used, each by different RPCs. All are sent as
+// "Authorization: Bearer <secret>":
 //
 //   - RegisterRunner: a stored token with the runner:register scope, or the
-//     root token, as "Authorization: Bearer <secret>".
+//     root token.
-//   - FetchJob: the runner token returned by RegisterRunner, as
+//   - FetchJob: the runner token returned by RegisterRunner. It identifies the
-//     "Authorization: Bearer <runner_token>". It identifies the runner. It
+//     runner and only works for FetchJob. It is "ermiterun_" followed by 43
-//     only works for FetchJob.
+//     base64url characters (32 random bytes, no padding).
-//   - UpdateJob, AppendLog, Heartbeat, WatchJob: the job token of the Job that
+//   - UpdateJob, AppendLog, Heartbeat, WatchJob, CompleteJob: the job token of
-//     FetchJob returned, as "Authorization: Bearer <job_token>". It is valid
+//     the Job that FetchJob returned. It is valid for that one job and that
-//     for that one job and attempt, from its assignment until it ends. The
+//     one assignment. It is "ermitejob_" followed by 43 base64url characters
-//     job_id of the request must be the one of the token.
+//     (32 random bytes, no padding). The job_id of the request must be the
+//     one of the token.
 //
-// A request without a credential, with an unknown, expired or revoked one, or
+// Errors. unauthenticated means only that the credential is bad: missing,
-// with a credential of the wrong kind for the RPC, gets unauthenticated. A
+// unknown, expired, revoked, of the wrong kind for the RPC, or a job token
-// job token that is valid but belongs to another job than the job_id of the
+// that was replaced (the job was given again, see FetchJob) or is older than
-// request gets permission_denied. A job that has ended (concluded, cancelled,
+// the 24 hours below. A runner that gets it for a job abandons the job at
-// failed by the server, or whose runner was lost) no longer has a valid job
+// once. A job token that is valid but belongs to another job than the job_id
-// token: every later call gets unauthenticated, which a runner treats as "the
+// of the request gets permission_denied. Repeated failed authentications are
-// job is over" and stops sending anything for it. Repeated failed
+// rate limited per client address.
-// authentications are rate limited per client address.
 //
-// Runner tokens and job tokens are secrets: the server never logs them and
+// The end of a job. A job ends when its conclusion is set, by CompleteJob or
-// clients must not log them. The job token stays in the runner host process; it
+// by the server (cancellation not honoured, lost runner, log limit, queue
-// is never given to the sandbox that runs the job's code.
+// timeout). From then on UpdateJob, AppendLog, Heartbeat and WatchJob answer
+// failed_precondition with a JobEnded error detail (a JobEnded message in the
+// Connect error details) that gives the conclusion; the runner stops the job
+// and sends nothing more. The job token is still recognised for 24 hours after
+// the end, so that these calls can say so and so that CompleteJob, the only
+// call that still does something, returns the stored result to a runner that
+// lost the answer of its last call or that was cut off by the server. After
+// the 24 hours every call gets unauthenticated.
+//
+// Secrets and trust. Runner tokens and job tokens are secrets: the server
+// never logs them and clients must not log them. The job token stays in the
+// runner host process; it is never given to the sandbox that runs the job's
+// code. A runner receives the jobs of every repository, with their workflow
+// files, contexts and (from CI-2) their code, and from CI-3 their secrets: the
+// runner:register scope therefore effectively gives read access to the code
+// and the secrets of every repository, private ones included. Give it only to
+// runners you operate.
 //
 // Servers that implement this service list "ci.runners" in
 // GetVersionResponse.features.
   // (a runner serves every repository), or the root token. Fails with
   // unauthenticated without a valid token, permission_denied without the
   // scope or with a repository-restricted token, invalid_argument if a field
-  // breaks a rule, and already_exists if name is taken by another runner.
+  // breaks a rule, and already_exists if name is taken by another runner and
+  // replace is not set.
   rpc RegisterRunner(RegisterRunnerRequest) returns (RegisterRunnerResponse);
   // FetchJob asks for a job to run (long poll). The server holds the call
   // until a queued job matches the runner, or about 50 seconds (never more
   // than 50) pass, then answers; without a job the response has no job and the
-  // runner calls again at once. Clients set a call timeout above 50 seconds
+  // runner calls again after retry_after. Clients set a call timeout above 50
-  // (60 is suggested) and must not poll when they have no free capacity.
+  // seconds (60 is suggested) and must not poll when they have no free
+  // capacity.
   //
   // A job matches when every label of its runs-on is among the labels of the
   // runner (case-insensitive). A job is given to exactly one runner, even when
   // many runners and many polls compete; the assignment is recorded before
-  // the response is sent, and the job is then in progress. If the response is
+  // the response is sent. The job stays QUEUED for the owner until it is
-  // lost the job is not offered again: with no Heartbeat for 2 minutes the
+  // acknowledged.
-  // server fails it as a lost runner.
+  //
+  // Acknowledgement. The runner's first Heartbeat for the job is the
+  // acknowledgement, and the runner sends it as soon as it has the job, before
+  // anything else. If none arrives within 30 seconds of the assignment (the
+  // response was lost, or the runner died), the server cancels the assignment:
+  // the job token is revoked, the job goes back to QUEUED with a fresh token
+  // and can be given to any runner, the same one included; nothing the old
+  // assignment stored is kept. This is not a failure of the job. A job whose
+  // assignment is cancelled 3 times in a row is failed with CI_CONCLUSION_FAILURE
+  // and an error saying so.
+  //
+  // Retry after a lost response. poll_id lets a client retry a poll safely:
+  // a call with the same poll_id as an earlier call of the same runner that
+  // assigned a job returns that same assignment (same Job, same token) as long
+  // as it has not been acknowledged or cancelled, instead of taking another.
+  //
+  // A runner that crashed or whose host restarted cannot resume a job: it does
+  // not keep the job token, so the job fails with the lost-runner error after
+  // the heartbeat timeout (see Heartbeat).
   //
   // Needs the runner token. Fails with unauthenticated when the runner was
   // removed (also ending a call in progress) and invalid_argument if labels
-  // breaks a rule.
+  // or poll_id break a rule.
   rpc FetchJob(FetchJobRequest) returns (FetchJobResponse);
-  // UpdateJob reports the state of the steps of a job and, once, how the job
+  // UpdateJob reports the state of the steps of a job while it runs. Needs the
-  // ended. Needs the job token. Each call is applied whole or not at all, and
+  // job token. Each call is applied whole or not at all, and repeating an
-  // repeating an identical call changes nothing, so a runner may retry after a
+  // identical call changes nothing, so a runner may retry after a network
-  // network error.
+  // error. To end the job use CompleteJob.
   //
-  // Fails with invalid_argument if a rule is broken (including the output
+  // Fails with invalid_argument if a rule is broken, with failed_precondition
-  // limits), and with failed_precondition if a step goes back (a COMPLETED
+  // if a step goes back (a COMPLETED step cannot change, a step cannot go from
-  // step cannot change, a step cannot go from IN_PROGRESS to PENDING).
+  // IN_PROGRESS to PENDING) and with failed_precondition and a JobEnded detail
-  // The call that carries a conclusion ends the job: its job token stops
+  // if the job has ended.
-  // working, the server completes the job and no further call is accepted,
-  // so a retry of that last call after a lost response gets unauthenticated,
-  // which means it was applied.
   rpc UpdateJob(UpdateJobRequest) returns (UpdateJobResponse);
   // AppendLog appends lines to the log of a job. Needs the job token.
   //
   //   - offset equal to the server's line count: the lines are appended;
   //   - offset below it: the first (count - offset) lines were already stored
   //     (a retry after a lost response) and are skipped, and the rest is
-  //     appended; a request entirely stored changes nothing;
+  //     appended; a request entirely stored changes nothing. The skipped lines
+  //     are not compared with the stored ones: a retry whose lines differ from
+  //     the first attempt is silently dropped, so a client must resend the
+  //     same lines at the same offset;
   //   - offset above it: out_of_range (a gap is a bug of the client; the
   //     message gives the line count the server holds).
   //
   // The response gives the line count after the call, so a client can check
-  // itself. A request has at most 1,000 lines whose text adds up to at most
+  // itself. Offsets resume only after a network error within the same runner
-  // 64 KiB (65,536 bytes), all in the same step; more is invalid_argument. A
+  // process; they are not a way to resume a job after a crash.
-  // line of more than 64 KiB is cut by the client into several lines.
+  //
+  // A request has at most 1,000 lines whose text adds up to at most 64 KiB
+  // (65,536 bytes), all in the same step; more is invalid_argument. A line
+  // longer than that is cut by the client into pieces, see LogLine.continued.
+  //
+  // Steps. The step of a request must be at least the step of the last line
+  // the server holds, because the log is ordered by step (applied lines only:
+  // lines skipped as already stored are not checked); a lower step is
+  // failed_precondition. Lines for a step that UpdateJob has not announced
+  // create it: it starts IN_PROGRESS at the time of its first line, with an
+  // empty name until an UpdateJob gives one.
   //
   // The log of a job is limited to 512 MiB (536,870,912 bytes) as stored by
   // the server (text, timestamps and separators). The call that would exceed
   // it appends nothing; the server adds a last line saying that the limit was
   // reached, ends the job with CI_CONCLUSION_FAILURE and RunJob.error set, and
-  // answers resource_exhausted. The runner then stops the job and sends
+  // answers failed_precondition with a JobEnded detail like any call after the
-  // nothing more: its calls get unauthenticated.
+  // end of a job. The runner then stops the job.
+  //
+  // Lines sent after the end of the job are rejected, so a runner flushes all
+  // its AppendLog lines before it calls CompleteJob.
   //
   // Fails with invalid_argument if a rule is broken (step number, text with a
-  // line feed or invalid UTF-8, missing time).
+  // line feed or invalid UTF-8, missing time, a continued first line).
   rpc AppendLog(AppendLogRequest) returns (AppendLogResponse);
   // Heartbeat tells the server that the runner is alive and still working on
-  // the job. Needs the job token. The runner calls it every 30 seconds for as
+  // the job. Needs the job token. The first one acknowledges the assignment
-  // long as the job runs, from assignment to the end. Without a heartbeat for
+  // (see FetchJob). The runner then calls it every 30 seconds for as long as
-  // 2 minutes the server ends the job with CI_CONCLUSION_FAILURE and the message
+  // the job runs. The deadline is 2 minutes after the later of the last
-  // "The self-hosted runner: <name> lost communication with the server...".
+  // heartbeat and the start of the server (so a server restart does not fail
-  // Only Heartbeat counts: other calls do not keep a job alive.
+  // the jobs of runners that could not reach it): past it the server ends the
+  // job with CI_CONCLUSION_FAILURE and the message "The self-hosted runner:
+  // <name> lost communication with the server...". Only Heartbeat counts:
+  // other calls do not keep a job alive.
+  //
+  // A runner that gets unavailable (or any network error) retries Heartbeat
+  // every 5 seconds until it succeeds, instead of waiting for its next 30
+  // second tick, and keeps the job running meanwhile.
   rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse);
-  // WatchJob streams the requests to stop a job. Needs the job token. The
+  // WatchJob streams what the runner must react to. Needs the job token. The
   // runner opens it when it starts the job and keeps it open; it reopens it
-  // after any error other than unauthenticated.
+  // after any error other than unauthenticated and the failed_precondition
+  // that says the job ended, and after a clean close of
+  // the stream that did not end with a JobEnded message.
   //
   // When the stream opens and a cancellation has already been requested, the
   // first message is that request, so a reconnect never misses one. Only the
   // first reason is sent: a job already cancelled by the owner does not get a
-  // second message when its timeout expires. The server also sends a
+  // second message when its time budget expires. The server also sends a
-  // keep-alive message (reason CANCEL_REASON_UNSPECIFIED) at least every 30
+  // Keepalive at least every 30 seconds, which the client ignores. When the
-  // seconds, which the client ignores. The stream ends without error when
+  // job ends the server sends JobEnded, then closes the stream without error.
-  // the job ends. The runner stops the job as the official runner does
+  // On a job that has already ended the call fails like the other RPCs
+  // (failed_precondition with a JobEnded detail).
+  //
+  // On a CancelRequest the runner stops the job as the official runner does
   // (interrupt, then terminate, then kill) and reports the conclusion given
-  // for the reason with UpdateJob.
+  // for the reason with CompleteJob.
+  //
+  // The server does not trust the runner to obey. If no conclusion arrives
+  // within 60 seconds after a cancellation request, whatever the reason (user,
+  // fail-fast, timeout), the server ends the job itself: CI_CONCLUSION_CANCELLED
+  // for user and fail-fast, CI_CONCLUSION_TIMED_OUT for timeout, with an error
+  // saying that the runner did not answer, and the steps not completed are
+  // completed as cancelled. The runner's later CompleteJob returns that result.
   rpc WatchJob(WatchJobRequest) returns (stream WatchJobResponse);
+  // CompleteJob ends a job: it reports the final steps, the outputs and the
+  // conclusion. Needs the job token, which stays valid for this RPC for 24
+  // hours after the end of the job, however it ended. The call is applied whole
+  // or not at all. Once the job has ended, the token works for no other RPC
+  // (they answer failed_precondition with a JobEnded detail).
+  //
+  // It is idempotent. The first call that finds the job running ends it and
+  // answers applied = true. Any later call, a retry after a lost response or a
+  // call after the server ended the job itself, is not validated and changes
+  // nothing: it answers OK with the stored conclusion and error and
+  // applied = false (a runner whose conclusion differs from the stored one
+  // simply logs it; the outputs it sent are dropped).
+  //
+  // Lines sent with AppendLog after this call are rejected: a runner flushes
+  // all its log lines first. Steps that were not reported COMPLETED when the job
+  // ends are completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when
+  // they never started and the job did not end by cancellation).
+  //
+  // Fails with invalid_argument if a rule is broken (including the output
+  // limits) and failed_precondition if a step goes back, as in UpdateJob.
+  rpc CompleteJob(CompleteJobRequest) returns (CompleteJobResponse);
 }
 // RegisterRunnerRequest describes the runner to register.
   // Os is the operating system: "linux". Other values are invalid_argument
   // (the set may grow with compatible additions).
   string os = 6;
+  // Replace removes the runner that already has this name, if any, before
+  // registering this one: its runner token is revoked at once, as in
+  // RunService.RemoveRunner, and a job it is running is not affected. The new
+  // runner gets a new runner_id. Needs the same scope as registering. Without
+  // replace, a name in use is already_exists. It is how a runner re-registers
+  // after losing its state.
+  bool replace = 7;
 }
 // RegisterRunnerResponse holds the new runner and its secret.
   // a secret.
   string runner_id = 1;
   // RunnerToken is the secret of the runner, to present as "Authorization:
-  // Bearer <runner_token>" to FetchJob. It starts with "ermiterun_" and is
+  // Bearer <runner_token>" to FetchJob: "ermiterun_" and 43 base64url
-  // returned once: the server keeps only its SHA-256, the runner must store
+  // characters (32 random bytes, no padding). It is returned once: the server
-  // it (mode 0600). It stops working when the runner is removed.
+  // keeps only its SHA-256, the runner must store it (mode 0600). It stops
+  // working when the runner is removed or replaced.
   string runner_token = 2;
 }
   // a label of the runner (invalid_argument if not) and only jobs whose runs-on
   // labels are all among them are offered.
   repeated string labels = 1;
+  // PollId identifies this poll, chosen by the client (for example 16 random
+  // bytes in base64url), 1 to 64 characters among letters, digits, "_" and
+  // "-". A client that retries a call whose response it did not get sends the
+  // same poll_id, and gets the assignment the first call made, if it made one
+  // (see FetchJob). A new poll uses a new id. Empty means no retry protection:
+  // a lost assignment is then recovered by the acknowledgement timeout alone.
+  string poll_id = 2;
 }
 // FetchJobResponse holds the job given to the runner, if any.
   // Job is the job to run. Unset when no job was available before the end of
   // the poll.
   Job job = 1;
+  // RetryAfter is how long the client waits before its next FetchJob. Unset or
+  // zero means call again at once. The server sets it when it has no job for
+  // the runner and wants to slow polling down (load, shutdown in progress);
+  // clients obey it and may wait longer, never less. Unset when job is set.
+  google.protobuf.Duration retry_after = 2;
 }
 // Job is a job given to a runner: everything needed to run it, nothing more.
   // one more after each RunService.RerunRun. It is the "github.run_attempt".
   int32 attempt = 3;
   // JobToken is the secret that authenticates this job's UpdateJob,
-  // AppendLog, Heartbeat and WatchJob calls ("Authorization: Bearer
+  // AppendLog, Heartbeat, WatchJob and CompleteJob calls ("Authorization:
-  // <job_token>"). It is random, valid until the job ends, survives a server
+  // Bearer <job_token>"): "ermitejob_" and 43 base64url characters (32 random
-  // restart and is shown only in this response. It must stay in the runner
+  // bytes, no padding). It is random, valid from the assignment until the job
-  // host process and never reach the job's code.
+  // ends (and for CompleteJob 24 hours more), survives a server restart and
+  // is shown only in this response, or in a repeat of the same poll_id. It
+  // must stay in the runner host process and never reach the job's code. A new
+  // assignment of the same job has a new token.
   string job_token = 4;
   // WorkflowPath is the path of the workflow file in the repository, for
   // example ".github/workflows/ci.yml".
   repeated string labels = 10;
   // TimeoutMinutes is the timeout of the job in minutes: the value of the
   // workflow's timeout-minutes, or 360 when it has none. At least 1. The
-  // runner enforces it from the start of the job; the server ends a job that
+  // runner enforces it from the moment it receives the job. The server counts
-  // exceeds it by announcing CANCEL_REASON_TIMEOUT.
+  // from the moment of the assignment and adds 60 seconds of slack, so the
+  // runner's own timeout always comes first: when the server's deadline
+  // passes it announces CANCEL_REASON_TIMEOUT, and ends the job itself after
+  // 60 more seconds without a conclusion (see WatchJob).
   int32 timeout_minutes = 11;
   // Repo is the name of the repository.
   string repo = 12;
   string ref = 14;
 }
-// UpdateJobRequest reports the steps and, in the last call, the end of a job.
+// UpdateJobRequest reports the steps of a running job.
 message UpdateJobRequest {
   // JobId is the Job.job_id. It must be the job of the token.
   string job_id = 1;
   // implicit "Set up job" is 1, the steps of the workflow follow in order, and
   // the implicit "Complete job" is last.
   repeated StepUpdate steps = 2;
-  // Outputs are the outputs of the job, evaluated by the runner. Only allowed
-  // in the call that carries the conclusion (invalid_argument otherwise).
-  // Names are 1 to 255 characters among letters, digits, "_" and "-" and
-  // begin with a letter or "_" and are case-insensitive: two names that
-  // differ only by case are invalid_argument. Each value is at most 1 MiB, and
-  // all names and values together at most 1 MiB (1,048,576 bytes); beyond that
-  // the whole call is rejected with invalid_argument and the runner reports
-  // the failure of the job without outputs, with a line in the log.
-  map<string, string> outputs = 3;
-  // Conclusion ends the job when set. A runner reports CI_CONCLUSION_SUCCESS,
-  // CI_CONCLUSION_FAILURE, CI_CONCLUSION_CANCELLED or CI_CONCLUSION_TIMED_OUT; any
-  // other value is invalid_argument. CI_CONCLUSION_UNSPECIFIED means the job goes
-  // on. Steps that were not reported COMPLETED when the job ends are
-  // completed by the server as CI_CONCLUSION_CANCELLED (or SKIPPED when they
-  // never started and the job did not fail by cancellation).
-  CiConclusion conclusion = 4;
-  // RunnerShutdown is true when the runner ends the job because it was told to
-  // stop (service stopped, signal). Only allowed with CI_CONCLUSION_CANCELLED.
-  // The server then annotates the job with GitHub's message "The runner has
-  // received a shutdown signal. This can happen when the runner service is
-  // stopped, or a manually started runner is canceled."
-  bool runner_shutdown = 5;
 }
 // StepUpdate is the state of one step as the runner knows it.
 // UpdateJobResponse is empty.
 message UpdateJobResponse {}
+// CompleteJobRequest ends a job.
+message CompleteJobRequest {
+  // JobId is the Job.job_id. It must be the job of the token.
+  string job_id = 1;
+  // Conclusion is how the job ended. Required: a runner reports
+  // CI_CONCLUSION_SUCCESS, CI_CONCLUSION_FAILURE, CI_CONCLUSION_CANCELLED or
+  // CI_CONCLUSION_TIMED_OUT; any other value is invalid_argument. Skipped is
+  // decided by the server and never reported by a runner.
+  CiConclusion conclusion = 2;
+  // Outputs are the outputs of the job, evaluated by the runner. Names are 1
+  // to 255 characters among letters, digits, "_" and "-" and begin with a
+  // letter or "_" and are case-insensitive: two names that differ only by case
+  // are invalid_argument. Each value is at most 1 MiB, and all names and
+  // values together at most 1 MiB (1,048,576 bytes); beyond that the whole call
+  // is rejected with invalid_argument and the runner reports the failure of
+  // the job without outputs, with a line in the log.
+  map<string, string> outputs = 3;
+  // RunnerShutdown is true when the runner ends the job because it was told to
+  // stop (service stopped, signal). Only allowed with CI_CONCLUSION_CANCELLED.
+  // The server then annotates the job with GitHub's message "The runner has
+  // received a shutdown signal. This can happen when the runner service is
+  // stopped, or a manually started runner is canceled." The conclusion
+  // CANCELLED for this case is to be verified against a recording of GitHub.
+  bool runner_shutdown = 4;
+  // Steps are the final steps, with the same meaning as UpdateJobRequest.steps,
+  // applied in the same atomic step as the conclusion.
+  repeated StepUpdate steps = 5;
+}
+// CompleteJobResponse is the stored result of a job.
+message CompleteJobResponse {
+  // Conclusion is the conclusion of the job as the server stored it. It is the
+  // one of the request when applied is true.
+  CiConclusion conclusion = 1;
+  // Error is the error text stored with the job, as RunJob.error. Empty when
+  // there is none.
+  string error = 2;
+  // Applied is true when this call ended the job, false when the job had
+  // already ended (a retry, or the server ended it first) and nothing changed.
+  bool applied = 3;
+}
+// JobEnded says that a job has ended. It is sent in WatchJob and attached, as
+// an error detail, to the failed_precondition errors that RunnerService returns
+// for a job that has ended.
+message JobEnded {
+  // Conclusion is how the job ended.
+  CiConclusion conclusion = 1;
+  // Error is the error text stored with the job, as RunJob.error. Empty when
+  // there is none.
+  string error = 2;
+}
 // AppendLogRequest carries log lines of one step.
 message AppendLogRequest {
   // JobId is the Job.job_id. It must be the job of the token.
   string job_id = 1;
   // Step is the number of the step the lines belong to (see
-  // UpdateJobRequest.steps), at least 1. A step that UpdateJob has not
+  // UpdateJobRequest.steps), at least 1, and at least the step of the last
-  // announced yet is created by the first lines.
+  // stored line. A step that UpdateJob has not announced yet is created by the
+  // first lines. See the rules on AppendLog.
   int32 step = 2;
   // Offset is the number of the first line of the request in the log of the
   // job, counted from 0 across all steps. See the rules on AppendLog.
   // carriage return is kept. It is the line as GitHub's raw log has it,
   // including "##[group]" and "##[endgroup]" markers and "##[error]" lines.
   string text = 2;
+  // Continued is true when this line is the continuation of the line before
+  // it, which was too long to send whole. A runner cuts a line of more than
+  // 64 KiB into pieces at UTF-8 character boundaries, each piece of at most
+  // 64 KiB, and sets continued on every piece but the first. Each piece is one
+  // line for numbering and offsets; the server and the raw log join the
+  // pieces back, so the original line is rebuilt exactly. The first line of a
+  // job cannot be continued (invalid_argument) and a continued line is in
+  // the same step as the line before it (failed_precondition otherwise).
+  bool continued = 3;
 }
 // AppendLogResponse tells how much of the log the server holds.
   string job_id = 1;
 }
-// WatchJobResponse is one message of the stream: a request to cancel the job,
+// WatchJobResponse is one message of the stream.
-// or a keep-alive.
 message WatchJobResponse {
-  // Reason is why the job must stop. CANCEL_REASON_UNSPECIFIED marks a
+  // Msg is what the message carries. New kinds may be added: a client ignores
-  // keep-alive: the other fields are empty and the client ignores it. New
+  // the ones it does not know.
-  // reasons may be added; a runner that does not know one cancels the job and
+  oneof msg {
-  // reports CI_CONCLUSION_CANCELLED.
+    // Keepalive: nothing to do, the stream is alive.
+    Keepalive keepalive = 1;
+    // Cancel: the job must stop.
+    CancelRequest cancel = 2;
+    // Ended: the job has ended. The last message of a stream: a close without
+    // it means the client reconnects.
+    JobEnded ended = 3;
+  }
+}
+// Keepalive has no fields.
+message Keepalive {}
+// CancelRequest asks the runner to stop a job.
+message CancelRequest {
+  // Reason is why the job must stop. New reasons may be added; a runner that
+  // does not know one cancels the job and reports CI_CONCLUSION_CANCELLED.
   CancelReason reason = 1;
   // Message is a short human-readable explanation, for the job log, for
   // example "Canceled by the owner". At most 1,024 characters.