1package cef
2
3import (
4 "time"
5
6 purecef "github.com/bnema/purego-cef/cef"
7 "github.com/rs/zerolog"
8
9 "github.com/bnema/dumber/internal/logging"
10)
11
12// GPU process crash recovery.
13//
14// When Chromium's GPU process dies and is relaunched, existing OSR browsers
15// stop receiving OnAcceleratedPaint even though the CEF UI/IO threads and GTK
16// stay alive. Recovery is two-staged per live view:
17//
18// 1. a render pulse (the same one used by the stall watchdog) on the CEF UI
19// thread, which is cheap and often enough to re-establish the compositor
20// output;
21// 2. if no accelerated paint arrived within gpuRecoveryVerifyDelay, a page
22// reload, which recreates the renderer's compositor frame sink.
23//
24// Unlike the stall watchdog, this path is always on and not env-gated.
25//
26// Concurrency model:
27// - Every recovery attempt takes a per-view generation (gpuRecoveryGen). The
28// pulse task and verify callback of an older attempt bail once a newer one
29// started (a new crash, or a deferred view being shown), so a stale verify
30// never reloads early and attempts never stack up.
31// - Visibility (effectiveVisible), the deferral flag (gpuRecoveryPending) and
32// the generation bump are all read/written under wv.mu. "Is the view hidden?
33// then mark it pending" and "view became visible: consume pending" are
34// therefore atomic with respect to each other: a deferral cannot be lost to
35// a concurrent show.
36// - The view bridge is cleared under wv.mu by teardown; recovery only reads
37// it through renderPulseTarget/liveBridge snapshots taken under wv.mu.
38// - The pulse runs on the CEF UI thread, the same thread that applies
39// visibility changes, so the hidden check in the pulse task is also
40// exact with respect to its WasHidden(0).
41
42const (
43 // gpuRecoveryDelay lets the relaunched GPU process initialize (GL/Vulkan
44 // context, GPU channel hand-off to the browser and renderers) before the
45 // first pulse. OnBeforeChildProcessLaunch fires when the process is spawned,
46 // not when it is ready; initialization takes a few hundred ms, so 1s leaves
47 // margin without a visibly long freeze. It also acts as the debounce window:
48 // every relaunch notification arriving while a pass is pending is absorbed.
49 gpuRecoveryDelay = 1 * time.Second
50 // gpuRecoveryVerifyDelay is how long a pulse gets to yield a new paint.
51 gpuRecoveryVerifyDelay = renderRecoveryVerifyDelay
52)
53
54// gpuRecoveryPaintCount reads the accelerated paint counter of a bridge. It is
55// a variable so tests can simulate paints without a live GTK/GL bridge.
56var gpuRecoveryPaintCount = func(bridge *Cef2gtkAdapter) int {
57 return bridge.Diagnostics().AcceleratedPaints
58}
59
60// noteGPUProcessLaunch is called for every GPU process launch with the
61// running launch count. The first launch is normal startup; later ones mean a
62// crash and relaunch. It only schedules work, so it is safe on the CEF
63// browser-process launcher thread.
64func (e *Engine) noteGPUProcessLaunch(launchCount uint64) {
65 if e == nil || launchCount <= 1 {
66 return
67 }
68 if !e.gpuRecoveryScheduled.CompareAndSwap(false, true) {
69 return
70 }
71 logging.FromContext(e.currentContext()).Warn().
72 Uint64("gpu_launches", launchCount).
73 Dur("delay", gpuRecoveryDelay).
74 Msg("cef: gpu process relaunched, scheduling render recovery")
75 cefScheduleAfter(gpuRecoveryDelay, e.runGPURecoveryPass)
76}
77
78// runGPURecoveryPass starts recovery for every live view.
79func (e *Engine) runGPURecoveryPass() {
80 e.gpuRecoveryScheduled.Store(false)
81 e.activeWebViews.Range(func(_, value any) bool {
82 if wv, ok := value.(*WebView); ok {
83 wv.startGPURecovery()
84 }
85 return true
86 })
87}
88
89// startGPURecovery begins a new recovery attempt for the view (superseding any
90// in-flight one): stage 1 posts the pulse, then stage 2 is scheduled.
91//
92// Hidden views legitimately produce no paints, so they can be neither pulsed
93// (the pulse would leave CEF believing the view is visible) nor judged. They
94// are marked pending instead and recovered when they become visible again.
95func (wv *WebView) startGPURecovery() {
96 target, ok := wv.renderPulseTarget()
97 if !ok || !target.hasBrowser {
98 return
99 }
100 gen, deferred := wv.claimGPURecovery()
101 log := logging.FromContext(wv.ctx)
102 if deferred {
103 log.Warn().
104 Uint64("webview_id", uint64(wv.id)).
105 Int32("browser_id", target.browserID).
106 Str("action", "defer").
107 Msg("cef: gpu recovery deferred until view is visible")
108 return
109 }
110 browserID := target.browserID
111 const msg = "cef: gpu recovery pulse"
112 wv.postRenderPulseTask(msg, func(host purecef.BrowserHost, bridge *Cef2gtkAdapter) {
113 if wv.gpuRecoveryGen.Load() != gen {
114 return // superseded by a newer attempt
115 }
116 // Runs on the CEF UI thread, where visibility changes are applied, so
117 // the view cannot be shown or hidden between this check and the pulse.
118 if wv.deferGPURecoveryIfHidden() {
119 logging.FromContext(wv.ctx).Warn().
120 Uint64("webview_id", uint64(wv.id)).
121 Int32("browser_id", browserID).
122 Str("action", "defer").
123 Msg(msg + " skipped: view hidden, deferred until visible")
124 return
125 }
126 // The baseline is taken here, right before the pulse, so paints that
127 // landed while the task was queued do not count as pulse recovery.
128 before := gpuRecoveryPaintCount(bridge)
129 pulseRender(host)
130 logging.FromContext(wv.ctx).Warn().
131 Uint64("webview_id", uint64(wv.id)).
132 Int32("browser_id", browserID).
133 Int("accelerated_paints_before", before).
134 Msg(msg + " executed")
135 // The verify window starts after the pulse ran, and only for a pulse
136 // that actually ran (not for a skipped or stale one).
137 cefScheduleAfter(gpuRecoveryVerifyDelay, func() { wv.verifyGPURecovery(gen, browserID, before) })
138 })
139}
140
141// verifyGPURecovery is stage 2: reload the view if the pulse produced no paint.
142func (wv *WebView) verifyGPURecovery(gen uint64, browserID int32, before int) {
143 if wv.gpuRecoveryGen.Load() != gen {
144 return // superseded by a newer attempt
145 }
146 bridge := wv.liveBridge()
147 if bridge == nil || wv.ctx == nil {
148 return
149 }
150 after := gpuRecoveryPaintCount(bridge)
151 event := func() *zerolog.Event {
152 return logging.FromContext(wv.ctx).Warn().
153 Uint64("webview_id", uint64(wv.id)).
154 Int32("browser_id", browserID).
155 Int("accelerated_paints_before", before).
156 Int("accelerated_paints_after", after)
157 }
158 if after > before {
159 event().Str("action", "none").Msg("cef: gpu recovery pulse recovered view")
160 return
161 }
162 // Reload waits on a synchronous GTK dispatch, during which the view can be
163 // hidden, destroyed or superseded. The guard revalidates all of that on the
164 // GTK thread right before navigating; no lock is held across the dispatch.
165 reloaded := false
166 deferred := false
167 err := wv.reloadGuarded(func() bool {
168 if wv.destroyed.Load() || wv.gpuRecoveryGen.Load() != gen {
169 return false
170 }
171 if wv.deferGPURecoveryIfHidden() {
172 deferred = true
173 return false
174 }
175 reloaded = true
176 return true
177 })
178 switch {
179 case err != nil:
180 logging.FromContext(wv.ctx).Warn().
181 Err(err).
182 Uint64("webview_id", uint64(wv.id)).
183 Int32("browser_id", browserID).
184 Msg("cef: gpu recovery reload failed")
185 case reloaded:
186 event().Str("action", "reload").Msg("cef: gpu recovery reload")
187 case deferred:
188 event().Str("action", "defer").Msg("cef: gpu recovery deferred until view is visible")
189 }
190}
191
192// claimGPURecovery starts a new attempt generation. If the view is hidden it
193// marks recovery pending instead; otherwise it clears any stale pending flag.
194// Everything happens under wv.mu so it is atomic with visibility changes.
195func (wv *WebView) claimGPURecovery() (gen uint64, deferred bool) {
196 wv.mu.Lock()
197 defer wv.mu.Unlock()
198 gen = wv.gpuRecoveryGen.Add(1)
199 if wv.hiddenLocked() {
200 wv.gpuRecoveryPending = true
201 return gen, true
202 }
203 wv.gpuRecoveryPending = false
204 return gen, false
205}
206
207// deferGPURecoveryIfHidden atomically checks visibility and, when hidden,
208// marks recovery pending so the next show resumes it.
209func (wv *WebView) deferGPURecoveryIfHidden() bool {
210 wv.mu.Lock()
211 defer wv.mu.Unlock()
212 if !wv.hiddenLocked() {
213 return false
214 }
215 wv.gpuRecoveryPending = true
216 return true
217}
218
219// effectivelyHidden reports whether CEF was last told the view is hidden. A
220// view whose visibility was never reported is treated as visible.
221func (wv *WebView) effectivelyHidden() bool {
222 wv.mu.RLock()
223 defer wv.mu.RUnlock()
224 return wv.hiddenLocked()
225}
226
227// hiddenLocked is effectivelyHidden for callers already holding wv.mu.
228func (wv *WebView) hiddenLocked() bool {
229 return wv.effectiveVisibilityKnown && !wv.effectiveVisible
230}