1package daemon
2
3import (
4 "context"
5 "errors"
6 "sort"
7
8 "github.com/bnema/vev/internal/domain"
9)
10
11func (d *Daemon) reportSnapshotFailure(capture *snapshotCapture, phase string, cause error) {
12 if d == nil || capture == nil || capture.session == nil || cause == nil {
13 return
14 }
15 signature := snapshotFailureSignature(phase, cause)
16 d.snapshotNoticeMu.Lock()
17 changed := d.snapshotActiveFailureSignature != signature
18 d.snapshotActiveFailureSignature = signature
19 d.snapshotNoticeMu.Unlock()
20
21 n := domain.Notification{
22 Code: domain.NoticeSnapshotWrite,
23 Severity: domain.NoticeError,
24 Message: "couldn't save session state; recent state may be lost on restart",
25 // The history is intentionally diagnostic only at the error-class level.
26 // Raw causes can include user paths and terminal content.
27 Details: signature,
28 Time: d.clock.Now(),
29 }
30 n, _ = d.notices.recordSnapshotFailure(n)
31 d.log.Warn("writing session snapshot failed", "phase", phase, "class", signature, "session", capture.name)
32 if changed {
33 d.deliverGlobal(n)
34 }
35}
36
37func (d *Daemon) clearSnapshotFailure() {
38 if d == nil {
39 return
40 }
41 d.snapshotNoticeMu.Lock()
42 d.snapshotActiveFailureSignature = ""
43 d.snapshotNoticeMu.Unlock()
44}
45
46func (d *Daemon) startSnapshotEncodeWorker() {
47 if d == nil {
48 return
49 }
50 d.snapshotWorkerMu.Lock()
51 if d.snapshotWorkerCancel != nil {
52 d.snapshotWorkerMu.Unlock()
53 return
54 }
55 // Shutdown owns the worker lifetime so it can flush captures after Serve
56 // cancels its parent context. The worker is always stopped explicitly.
57 workerCtx, cancel := context.WithCancel(context.Background())
58 d.snapshotWorkerID++
59 workerID := d.snapshotWorkerID
60 d.snapshotWorkerCtx = workerCtx
61 d.snapshotWorkerCancel = cancel
62 d.snapshotWorkerDone = make(chan struct{})
63 d.snapshotWorkerFlush = make(chan struct{})
64 d.snapshotWorkerFinalWake = make(chan struct{}, 1)
65 d.snapshotFinalJobs = make(map[*session]*snapshotCapture)
66 d.snapshotFinalOrder = nil
67 d.snapshotWorkerClosing = false
68 d.snapshotWorkerInFlight = nil
69 done := d.snapshotWorkerDone
70 flush := d.snapshotWorkerFlush
71 finalWake := d.snapshotWorkerFinalWake
72 d.snapshotWorkerMu.Unlock()
73 go d.runSnapshotEncodeWorker(workerCtx, workerID, done, flush, finalWake)
74}
75
76// runSnapshotEncodeWorker owns the worker state machine. Each event either
77// continues processing, drains the terminal queues for shutdown, or stops;
78// publication details are kept out of the select loop.
79func (d *Daemon) runSnapshotEncodeWorker(workerCtx context.Context, workerID uint64, done chan<- struct{}, flush, finalWake <-chan struct{}) {
80 defer close(done)
81 for {
82 select {
83 case <-workerCtx.Done():
84 return
85 case <-flush:
86 d.flushSnapshotCaptures(workerCtx, workerID)
87 return
88 case <-finalWake:
89 if !d.drainFinalSnapshotCaptures(workerCtx, workerID) {
90 return
91 }
92 case capture := <-d.snapshotJobs:
93 if !d.publishSnapshotCapture(workerCtx, workerID, capture) {
94 return
95 }
96 }
97 }
98}
99
100func (d *Daemon) flushSnapshotCaptures(workerCtx context.Context, workerID uint64) {
101 for {
102 select {
103 case capture := <-d.snapshotJobs:
104 if !d.publishSnapshotCapture(workerCtx, workerID, capture) {
105 return
106 }
107 default:
108 _ = d.drainFinalSnapshotCaptures(workerCtx, workerID)
109 return
110 }
111 }
112}
113
114func (d *Daemon) drainFinalSnapshotCaptures(workerCtx context.Context, workerID uint64) bool {
115 for capture := d.takeFinalSnapshotCapture(); capture != nil; capture = d.takeFinalSnapshotCapture() {
116 if !d.publishSnapshotCapture(workerCtx, workerID, capture) {
117 return false
118 }
119 }
120 return true
121}
122
123func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint64, capture *snapshotCapture) bool {
124 if workerCtx.Err() != nil || !startSnapshotPublication(capture) || !d.setSnapshotWorkerInFlight(workerID, capture) {
125 d.finishSnapshotCapture(capture, false)
126 return workerCtx.Err() == nil
127 }
128 publication, err := d.incrementalPublication(capture)
129 publicationContext := capture.publicationContext
130 if publicationContext == nil {
131 publicationContext = workerCtx
132 }
133 if err == nil && workerCtx.Err() == nil {
134 if err = publicationContext.Err(); err == nil {
135 if d.recovery != nil {
136 var record domain.CatalogueRecord
137 record, err = d.recovery.PublishCheckpoint(publicationContext, capture.name, publication)
138 if err == nil {
139 if record.Committed == nil {
140 err = errors.New("snapshot: checkpoint publication returned no committed checkpoint")
141 } else {
142 capture.checkpoint = *record.Committed
143 }
144 }
145 } else {
146 err = d.snapshotRepository.Publish(publicationContext, publication)
147 }
148 }
149 }
150 d.clearSnapshotWorkerInFlight(workerID, capture)
151 if err == nil && workerCtx.Err() == nil {
152 markSnapshotCaptureObjectsPublished(capture)
153 }
154 if err != nil && workerCtx.Err() == nil && publicationContext.Err() == nil {
155 // Global, not session-scoped: by now the session may already be torn
156 // down, so a session notice would be dead-on-arrival. No lock is held
157 // here (clearSnapshotWorkerInFlight released its own).
158 d.reportSnapshotFailure(capture, "publish", err)
159 }
160 d.finishSnapshotCapture(capture, err == nil && workerCtx.Err() == nil)
161 return workerCtx.Err() == nil
162}
163
164func (d *Daemon) takeFinalSnapshotCapture() *snapshotCapture {
165 d.snapshotWorkerMu.Lock()
166 defer d.snapshotWorkerMu.Unlock()
167 for len(d.snapshotFinalOrder) > 0 {
168 sess := d.snapshotFinalOrder[0]
169 d.snapshotFinalOrder[0] = nil
170 d.snapshotFinalOrder = d.snapshotFinalOrder[1:]
171 capture := d.snapshotFinalJobs[sess]
172 delete(d.snapshotFinalJobs, sess)
173 if capture != nil {
174 return capture
175 }
176 }
177 return nil
178}
179
180func (d *Daemon) setSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) bool {
181 d.snapshotWorkerMu.Lock()
182 defer d.snapshotWorkerMu.Unlock()
183 if d.snapshotWorkerID != workerID || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil {
184 return false
185 }
186 d.snapshotWorkerInFlight = capture
187 return true
188}
189
190func (d *Daemon) clearSnapshotWorkerInFlight(workerID uint64, capture *snapshotCapture) {
191 d.snapshotWorkerMu.Lock()
192 defer d.snapshotWorkerMu.Unlock()
193 if d.snapshotWorkerID == workerID && d.snapshotWorkerInFlight == capture {
194 d.snapshotWorkerInFlight = nil
195 }
196}
197
198func (d *Daemon) stopSnapshotEncodeWorker() {
199 if d == nil {
200 return
201 }
202 _ = d.StopDurableWriters(context.Background())
203 d.WaitDurableWriters()
204}
205
206// StopDurableWriters stops maintenance scheduling, cancels cooperative
207// maintenance calls, stops snapshot admission, and requests the snapshot
208// worker's final drain. The context bounds checkpoint success only:
209// cancellation never detaches a writer from its owner.
210func (d *Daemon) StopDurableWriters(ctx context.Context) []string {
211 if d == nil {
212 return nil
213 }
214 cancel, done := d.requestDurableWriterStop()
215 if done == nil {
216 return nil
217 }
218 if ctx == nil {
219 ctx = context.Background()
220 }
221 select {
222 case <-done:
223 return nil
224 case <-ctx.Done():
225 // Snapshot identity before cancellation can let a cooperative worker clear
226 // its in-flight capture. The caller persists timeout notices before the
227 // unconditional ownership join.
228 names := d.durableWriterFailureNames()
229 cancel()
230 return names
231 }
232}
233
234// WaitDurableWriters is the unconditional ownership barrier. It has no timeout
235// or status branch: every writer goroutine must exit before Serve may return.
236// Restoration is joined with the others because it mutates durable state too;
237// it observes serveCtx cancellation, so waiting for it cannot outlive an
238// uncooperative repository call any longer than the snapshot worker does.
239func (d *Daemon) WaitDurableWriters() {
240 if d == nil {
241 return
242 }
243 d.snapshotWorkerMu.Lock()
244 snapshotDone := d.snapshotWorkerDone
245 maintenanceDone := d.maintenanceWorkerDone
246 restoreDone := d.restoreWorkerDone
247 d.snapshotWorkerMu.Unlock()
248 if snapshotDone != nil {
249 <-snapshotDone
250 d.finishStoppedSnapshotWorker(false)
251 }
252 if maintenanceDone != nil {
253 <-maintenanceDone
254 }
255 // Restoration never blocks on the workers above (snapshot admission is
256 // non-blocking), so joining it last cannot deadlock.
257 if restoreDone != nil {
258 <-restoreDone
259 }
260}
261
262func (d *Daemon) requestDurableWriterStop() (context.CancelFunc, <-chan struct{}) {
263 d.snapshotWorkerMu.Lock()
264 defer d.snapshotWorkerMu.Unlock()
265 d.cancelDurableMaintenanceLocked()
266 cancel := d.snapshotWorkerCancel
267 if cancel == nil {
268 return nil, nil
269 }
270 if !d.snapshotWorkerClosing {
271 d.snapshotWorkerClosing = true
272 close(d.snapshotWorkerFlush)
273 }
274 return cancel, d.snapshotWorkerDone
275}
276
277// durableWriterFailureNames includes admitted buffered captures as well as the
278// active and final queues. snapshotAdmitted tracks normal captures from queue
279// admission through completion, so worker dequeue cannot make one disappear.
280func (d *Daemon) durableWriterFailureNames() []string {
281 d.snapshotWorkerMu.Lock()
282 defer d.snapshotWorkerMu.Unlock()
283 seen := make(map[string]struct{})
284 add := func(capture *snapshotCapture) {
285 if capture != nil && capture.name != "" {
286 seen[capture.name] = struct{}{}
287 }
288 }
289 add(d.snapshotWorkerInFlight)
290 for _, capture := range d.snapshotFinalJobs {
291 add(capture)
292 }
293 for capture := range d.snapshotAdmitted {
294 add(capture)
295 }
296 names := make([]string, 0, len(seen))
297 for name := range seen {
298 names = append(names, name)
299 }
300 sort.Strings(names)
301 return names
302}
303
304func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) {
305 d.snapshotWorkerMu.Lock()
306 inFlight := d.snapshotWorkerInFlight
307 d.snapshotWorkerCtx = nil
308 d.snapshotWorkerCancel = nil
309 d.snapshotWorkerDone = nil
310 d.snapshotWorkerFlush = nil
311 d.snapshotWorkerFinalWake = nil
312 d.snapshotWorkerClosing = false
313 d.snapshotWorkerInFlight = nil
314 queued := make([]*snapshotCapture, 0, len(d.snapshotJobs)+len(d.snapshotFinalJobs))
315 for _, capture := range d.snapshotFinalJobs {
316 queued = append(queued, capture)
317 }
318 d.snapshotFinalJobs = nil
319 d.snapshotFinalOrder = nil
320 for {
321 select {
322 case capture := <-d.snapshotJobs:
323 queued = append(queued, capture)
324 default:
325 d.snapshotWorkerMu.Unlock()
326 if abandoned {
327 d.finishSnapshotCapture(inFlight, false)
328 }
329 for _, capture := range queued {
330 d.finishSnapshotCapture(capture, false)
331 }
332 return
333 }
334 }
335}