1package cef
2
3import (
4 "bytes"
5 "context"
6 "fmt"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "sort"
11 "strconv"
12 "strings"
13 "sync/atomic"
14 "time"
15
16 "github.com/bnema/dumber/internal/logging"
17)
18
19const (
20 procRoot = "/proc"
21 maxLoggedThreadsPerProcess = 8
22 maxDetailedThreadsPerProcess = 48
23 threadStatUTimeIndex = 11
24 threadStatSTimeIndex = 12
25 maxKernelStackLines = 6
26 renderStallProcessDiagnosticsCooldown = 30 * time.Second
27 renderStallBacktraceCooldown = 30 * time.Second
28 renderStallBacktraceCommandLimit = 64 * 1024
29)
30
31var (
32 renderStallProcessDiagnosticsLastUnixNS atomic.Int64
33 renderStallBacktraceLastUnixNS atomic.Int64
34)
35
36func (wv *WebView) logCEFProcessDiagnostics(reason string, classification renderStallClassification) {
37 if wv == nil || wv.ctx == nil {
38 return
39 }
40 if !reserveRenderStallProcessDiagnostics(time.Now()) {
41 return
42 }
43 pids := append([]int{os.Getpid()}, childPIDs(os.Getpid())...)
44 for _, pid := range pids {
45 cmdline := procCmdline(pid)
46 processType := chromiumProcessType(cmdline)
47 threads := procThreadSummaries(pid, maxLoggedThreadsPerProcess)
48 detailedThreads := []string(nil)
49 if classification.CEFUIBlocked && processType == "browser" {
50 detailedThreads = procDetailedThreadSummaries(pid, maxDetailedThreadsPerProcess)
51 }
52 logging.FromContext(wv.ctx).Warn().
53 Str("reason", reason).
54 Str("classification", classification.Category).
55 Bool("cef_ui_blocked", classification.CEFUIBlocked).
56 Bool("cef_io_alive", classification.CEFIOAlive).
57 Int("pid", pid).
58 Str("process_type", processType).
59 Strs("cmdline_flags", safeChromiumCmdlineFlags(cmdline)).
60 Int("thread_count", procThreadCount(pid)).
61 Strs("hot_threads", threads).
62 Strs("detailed_threads", detailedThreads).
63 Strs("gpu_fdinfo", gpuFDInfoSummaries(pid)).
64 Msg("cef: process diagnostic snapshot")
65 }
66 if classification.CEFUIBlocked {
67 wv.maybeCaptureRenderStallBacktrace(reason, classification)
68 }
69}
70
71func reserveRenderStallProcessDiagnostics(now time.Time) bool {
72 nowNS := now.UnixNano()
73 for {
74 lastNS := renderStallProcessDiagnosticsLastUnixNS.Load()
75 last := unixNSTime(lastNS)
76 if !last.IsZero() && now.Sub(last) < renderStallProcessDiagnosticsCooldown {
77 return false
78 }
79 if renderStallProcessDiagnosticsLastUnixNS.CompareAndSwap(lastNS, nowNS) {
80 return true
81 }
82 }
83}
84
85func safeChromiumCmdlineFlags(cmdline string) []string {
86 fields := strings.Fields(cmdline)
87 flags := make([]string, 0, len(fields))
88 seen := make(map[string]struct{})
89 for _, field := range fields {
90 if !strings.HasPrefix(field, "--") {
91 continue
92 }
93 name, value, hasValue := strings.Cut(field, "=")
94 var safe string
95 switch name {
96 case "--type", "--lang", "--enable-features", "--disable-features", "--ozone-platform",
97 "--use-gl", "--use-angle", "--enable-logging", "--log-severity",
98 "--force-device-scale-factor", "--high-dpi-support", "--enable-use-zoom-for-dsf":
99 if hasValue && value != "" {
100 safe = name + "=" + value
101 } else {
102 safe = name
103 }
104 case "--enable-gpu-rasterization", "--disable-gpu", "--disable-software-rasterizer", "--in-process-gpu", "--no-sandbox", "--no-zygote":
105 safe = name
106 default:
107 continue
108 }
109 if _, ok := seen[safe]; ok {
110 continue
111 }
112 seen[safe] = struct{}{}
113 flags = append(flags, safe)
114 }
115 return flags
116}
117
118func procPath(parts ...string) string {
119 return filepath.Join(append([]string{procRoot}, parts...)...)
120}
121
122func childPIDs(parent int) []int {
123 if out := childPIDsFromProcChildren(parent); len(out) > 0 {
124 return out
125 }
126 entries, err := os.ReadDir(procRoot)
127 if err != nil {
128 return nil
129 }
130 var out []int
131 for _, entry := range entries {
132 if !entry.IsDir() {
133 continue
134 }
135 pid, err := strconv.Atoi(entry.Name())
136 if err != nil {
137 continue
138 }
139 if procPPID(pid) == parent {
140 out = append(out, pid)
141 }
142 }
143 sort.Ints(out)
144 return out
145}
146
147func childPIDsFromProcChildren(parent int) []int {
148 path := procPath(strconv.Itoa(parent), "task", strconv.Itoa(parent), "children")
149 b, err := os.ReadFile(path)
150 if err != nil {
151 return nil
152 }
153 fields := strings.Fields(string(b))
154 out := make([]int, 0, len(fields))
155 for _, field := range fields {
156 pid, err := strconv.Atoi(field)
157 if err != nil {
158 continue
159 }
160 out = append(out, pid)
161 }
162 sort.Ints(out)
163 return out
164}
165
166func procPPID(pid int) int {
167 b, err := os.ReadFile(procPath(strconv.Itoa(pid), "status"))
168 if err != nil {
169 return 0
170 }
171 for line := range strings.SplitSeq(string(b), "\n") {
172 if !strings.HasPrefix(line, "PPid:") {
173 continue
174 }
175 fields := strings.Fields(line)
176 if len(fields) >= 2 {
177 ppid, _ := strconv.Atoi(fields[1])
178 return ppid
179 }
180 }
181 return 0
182}
183
184func procCmdline(pid int) string {
185 b, err := os.ReadFile(procPath(strconv.Itoa(pid), "cmdline"))
186 if err != nil {
187 return ""
188 }
189 return strings.TrimSpace(string(bytes.ReplaceAll(b, []byte{0}, []byte{' '})))
190}
191
192func chromiumProcessType(cmdline string) string {
193 for field := range strings.FieldsSeq(cmdline) {
194 if after, ok := strings.CutPrefix(field, "--type="); ok {
195 return after
196 }
197 }
198 return "browser"
199}
200
201func procThreadCount(pid int) int {
202 entries, err := os.ReadDir(procPath(strconv.Itoa(pid), "task"))
203 if err != nil {
204 return 0
205 }
206 return len(entries)
207}
208
209type procThreadSummary struct {
210 tid int
211 text string
212 ticks uint64
213}
214
215func procThreadSummaries(pid, limit int) []string {
216 return procThreadSummariesWithDetails(pid, limit, false)
217}
218
219func procDetailedThreadSummaries(pid, limit int) []string {
220 return procThreadSummariesWithDetails(pid, limit, true)
221}
222
223func procThreadSummariesWithDetails(pid, limit int, detailed bool) []string {
224 entries, err := os.ReadDir(procPath(strconv.Itoa(pid), "task"))
225 if err != nil {
226 return nil
227 }
228 items := make([]procThreadSummary, 0, len(entries))
229 for _, entry := range entries {
230 tid, err := strconv.Atoi(entry.Name())
231 if err != nil {
232 continue
233 }
234 b, err := os.ReadFile(procPath(strconv.Itoa(pid), "task", strconv.Itoa(tid), "stat"))
235 if err != nil {
236 continue
237 }
238 text, ticks := parseThreadStat(tid, string(b))
239 items = append(items, procThreadSummary{tid: tid, text: text, ticks: ticks})
240 }
241 sort.Slice(items, func(i, j int) bool { return items[i].ticks > items[j].ticks })
242 if limit > 0 && len(items) > limit {
243 items = items[:limit]
244 }
245 if detailed {
246 for i := range items {
247 text := items[i].text + " wchan=" + procThreadWchan(pid, items[i].tid)
248 if stack := procThreadKernelStack(pid, items[i].tid); stack != "" {
249 text = text + " kernel_stack=" + stack
250 }
251 items[i].text = text
252 }
253 }
254 out := make([]string, 0, len(items))
255 for _, item := range items {
256 out = append(out, item.text)
257 }
258 return out
259}
260
261func parseThreadStat(tid int, stat string) (string, uint64) {
262 open := strings.IndexByte(stat, '(')
263 closeIdx := strings.LastIndexByte(stat, ')')
264 if open < 0 || closeIdx <= open || closeIdx+2 >= len(stat) {
265 return strconv.Itoa(tid), 0
266 }
267 comm := stat[open+1 : closeIdx]
268 rest := strings.Fields(stat[closeIdx+2:])
269 state := "?"
270 if len(rest) > 0 {
271 state = rest[0]
272 }
273 var ticks uint64
274 if len(rest) > threadStatSTimeIndex {
275 utime, _ := strconv.ParseUint(rest[threadStatUTimeIndex], 10, 64)
276 stime, _ := strconv.ParseUint(rest[threadStatSTimeIndex], 10, 64)
277 ticks = utime + stime
278 }
279 return "tid=" + strconv.Itoa(tid) + " state=" + state + " ticks=" + strconv.FormatUint(ticks, 10) + " comm=" + comm, ticks
280}
281
282func procThreadWchan(pid, tid int) string {
283 b, err := os.ReadFile(procPath(strconv.Itoa(pid), "task", strconv.Itoa(tid), "wchan"))
284 if err != nil {
285 return ""
286 }
287 return strings.TrimSpace(string(b))
288}
289
290func procThreadKernelStack(pid, tid int) string {
291 b, err := os.ReadFile(procPath(strconv.Itoa(pid), "task", strconv.Itoa(tid), "stack"))
292 if err != nil {
293 return ""
294 }
295 lines := strings.Split(strings.TrimSpace(string(b)), "\n")
296 if len(lines) > maxKernelStackLines {
297 lines = lines[:maxKernelStackLines]
298 }
299 return strings.Join(lines, " | ")
300}
301
302func (wv *WebView) maybeCaptureRenderStallBacktrace(reason string, classification renderStallClassification) {
303 if wv == nil || wv.ctx == nil || wv.destroyed.Load() || !renderStallBacktraceEnabled() {
304 return
305 }
306 now := time.Now()
307 nowNS := now.UnixNano()
308 for {
309 lastNS := renderStallBacktraceLastUnixNS.Load()
310 last := unixNSTime(lastNS)
311 if !last.IsZero() && now.Sub(last) < renderStallBacktraceCooldown {
312 return
313 }
314 if renderStallBacktraceLastUnixNS.CompareAndSwap(lastNS, nowNS) {
315 break
316 }
317 }
318 pid := os.Getpid()
319 if _, err := exec.LookPath("gdb"); err != nil {
320 logging.FromContext(wv.ctx).Warn().
321 Err(err).
322 Str("reason", reason).
323 Str("classification", classification.Category).
324 Int("pid", pid).
325 Str("external_command", fmt.Sprintf("gdb -p %d -batch -ex 'thread apply all bt'", pid)).
326 Msg("cef: render stall native backtrace skipped; gdb not available")
327 return
328 }
329 if wv.ctx == nil || wv.destroyed.Load() || !renderStallBacktraceEnabled() {
330 return
331 }
332 logCtx := wv.ctx
333 go func() {
334 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
335 defer cancel()
336 cmd := exec.CommandContext(ctx, "gdb", "-p", strconv.Itoa(pid), "-batch", "-ex", "thread apply all bt")
337 out, err := cmd.CombinedOutput()
338 truncated := false
339 if len(out) > renderStallBacktraceCommandLimit {
340 out = out[:renderStallBacktraceCommandLimit]
341 truncated = true
342 }
343 log := logging.FromContext(logCtx).Warn().
344 Str("reason", reason).
345 Str("classification", classification.Category).
346 Int("pid", pid).
347 Bool("truncated", truncated).
348 Str("backtrace", string(out))
349 if ctx.Err() != nil {
350 log = log.Err(ctx.Err())
351 } else if err != nil {
352 log = log.Err(err)
353 }
354 log.Msg("cef: render stall native backtrace")
355 }()
356}
357
358func gpuFDInfoSummaries(pid int) []string {
359 entries, err := os.ReadDir(procPath(strconv.Itoa(pid), "fdinfo"))
360 if err != nil {
361 return nil
362 }
363 var out []string
364 for _, entry := range entries {
365 b, err := os.ReadFile(procPath(strconv.Itoa(pid), "fdinfo", entry.Name()))
366 if err != nil {
367 continue
368 }
369 for line := range strings.SplitSeq(string(b), "\n") {
370 if strings.Contains(line, "drm-engine-") || strings.HasPrefix(line, "drm-client-id") || strings.HasPrefix(line, "drm-pdev") {
371 out = append(out, "fd="+entry.Name()+" "+strings.TrimSpace(line))
372 }
373 }
374 }
375 if len(out) > 32 {
376 out = out[:32]
377 }
378 return out
379}