0f9d086 feat(app): dump every goroutine stack when an owner loop stalls · neferwl

M internal/adapters/drm/output.go +21 -1


@@ -78,6 +78,8 @@ type Output struct {
78	78		// seatDisable carries a request to leave the planes on Bypass before
79	79		// the seat is disabled (PrepareSeatDisable); Run answers it.
80	80		seatDisable chan chan struct{}
	81	+	// ping carries watchdog probes (Ping); Run answers each one.
	82	+	ping chan chan struct{}
81	83		// fbs are the two renderer images frames alternate between, back the
82	84		// one the next frame draws into (ADR 014: zero copy).
83	85		fbs [2]uint32

@@ -206,7 +208,7 @@ type CursorLoader func(c ports.CursorChange, scale float64, t ports.BufferTransf
206	208		func newOutput(card Card, c connector, mode modeInfo, crtc uint32) (Output, error) {
207	209		log := card.log
208	210		pipe := slices.Index(card.crtcs, crtc)
209		-	o := &Output{k: card.k, flipped: card.flips[crtc], frame: frameLifecycle{serials: &card.serials}, formats: card.formats, sampled: card.want.Sampled, device: card.want.Device, crtc: crtc, conn: c, mode: mode, log: log, renderLog: card.renderLog, monitor: readMonitor(card.path, c.name), scanout: !card.want.NoScanout, clientFBs: map[uint64]*clientFB{}, reason: "start", ready: make(chan error, 1), seatDisable: make(chan chan struct{}), traceFlips: card.want.TraceFlips, vrrFlipGap: card.want.VRRFlipGap}
	211	+	o := &Output{k: card.k, flipped: card.flips[crtc], frame: frameLifecycle{serials: &card.serials}, formats: card.formats, sampled: card.want.Sampled, device: card.want.Device, crtc: crtc, conn: c, mode: mode, log: log, renderLog: card.renderLog, monitor: readMonitor(card.path, c.name), scanout: !card.want.NoScanout, clientFBs: map[uint64]*clientFB{}, reason: "start", ready: make(chan error, 1), seatDisable: make(chan chan struct{}), ping: make(chan chan struct{}), traceFlips: card.want.TraceFlips, vrrFlipGap: card.want.VRRFlipGap}
210	212		var err error
211	213		if o.saved, err = getCrtc(card.fd, crtc); err != nil {
212	214		log.Warn().Err(err).Uint32("crtc", crtc).Msg("save crtc; it will not be restored on exit")

@@ -319,6 +321,21 @@ func cursorSize(fd int) int {
319	321		// Cursor is the hardware cursor, or nil when the CRTC has no cursor plane.
320	322		func (o Output) Cursor() Cursor { return o.cursor }
321	323
	324	+	// Ping returns once Run handled a probe, or once ctx ends: the watchdog
	325	+	// uses it to detect a stuck output loop.
	326	+	func (o *Output) Ping(ctx context.Context) {
	327	+	ack := make(chan struct{})
	328	+	select {
	329	+	case o.ping <- ack:
	330	+	case <-ctx.Done():
	331	+	return
	332	+	}
	333	+	select {
	334	+	case <-ack:
	335	+	case <-ctx.Done():
	336	+	}
	337	+	}
	338	+
322	339		// Info describes the output for core and clients. Unknown EDID fields are empty.
323	340		func (o *Output) Info() ports.OutputInfo {
324	341		known := func(v string) string {

@@ -1319,6 +1336,9 @@ func (o *Output) Run(ctx context.Context, newRenderer func(w, h int) (ports.Rend
1319	1336		refuse = capture.ErrOutputOff
1320	1337		}
1321	1338		dirty = requests.Admit(q, refuse, scene, haveScene) \|\| dirty
	1339	+	case ack := <-o.ping:
	1340	+	close(ack)
	1341	+	continue
1322	1342		case ack := <-o.seatDisable:
1323	1343		o.bypassForSeatDisable()
1324	1344		// The seat is about to be disabled: no frame may select the

 	// seatDisable carries a request to leave the planes on Bypass before
 	// the seat is disabled (PrepareSeatDisable); Run answers it.
 	seatDisable chan chan struct{}
+	// ping carries watchdog probes (Ping); Run answers each one.
+	ping chan chan struct{}
 	// fbs are the two renderer images frames alternate between, back the
 	// one the next frame draws into (ADR 014: zero copy).
 	fbs   [2]uint32
 func newOutput(card *Card, c connector, mode modeInfo, crtc uint32) (*Output, error) {
 	log := card.log
 	pipe := slices.Index(card.crtcs, crtc)
-	o := &Output{k: card.k, flipped: card.flips[crtc], frame: frameLifecycle{serials: &card.serials}, formats: card.formats, sampled: card.want.Sampled, device: card.want.Device, crtc: crtc, conn: c, mode: mode, log: log, renderLog: card.renderLog, monitor: readMonitor(card.path, c.name), scanout: !card.want.NoScanout, clientFBs: map[uint64]*clientFB{}, reason: "start", ready: make(chan error, 1), seatDisable: make(chan chan struct{}), traceFlips: card.want.TraceFlips, vrrFlipGap: card.want.VRRFlipGap}
+	o := &Output{k: card.k, flipped: card.flips[crtc], frame: frameLifecycle{serials: &card.serials}, formats: card.formats, sampled: card.want.Sampled, device: card.want.Device, crtc: crtc, conn: c, mode: mode, log: log, renderLog: card.renderLog, monitor: readMonitor(card.path, c.name), scanout: !card.want.NoScanout, clientFBs: map[uint64]*clientFB{}, reason: "start", ready: make(chan error, 1), seatDisable: make(chan chan struct{}), ping: make(chan chan struct{}), traceFlips: card.want.TraceFlips, vrrFlipGap: card.want.VRRFlipGap}
 	var err error
 	if o.saved, err = getCrtc(card.fd, crtc); err != nil {
 		log.Warn().Err(err).Uint32("crtc", crtc).Msg("save crtc; it will not be restored on exit")
 // Cursor is the hardware cursor, or nil when the CRTC has no cursor plane.
 func (o *Output) Cursor() *Cursor { return o.cursor }
+// Ping returns once Run handled a probe, or once ctx ends: the watchdog
+// uses it to detect a stuck output loop.
+func (o *Output) Ping(ctx context.Context) {
+	ack := make(chan struct{})
+	select {
+	case o.ping <- ack:
+	case <-ctx.Done():
+		return
+	}
+	select {
+	case <-ack:
+	case <-ctx.Done():
+	}
+}
 // Info describes the output for core and clients. Unknown EDID fields are empty.
 func (o *Output) Info() ports.OutputInfo {
 	known := func(v string) string {
 				refuse = capture.ErrOutputOff
 			}
 			dirty = requests.Admit(q, refuse, scene, haveScene) || dirty
+		case ack := <-o.ping:
+			close(ack)
+			continue
 		case ack := <-o.seatDisable:
 			o.bypassForSeatDisable()
 			// The seat is about to be disabled: no frame may select the

M internal/adapters/logging/logging.go +8 -0


@@ -31,6 +31,7 @@ var categories = map[string]string{"input-motion": "input", "input-keys": "input
31	31
32	32		type debugKey struct{}
33	33		type levelKey struct{}
	34	+	type dirKey struct{}
34	35		type debugSet map[string]bool
35	36
36	37		func ParseDebug(value string) (map[string]bool, error) {

@@ -97,6 +98,7 @@ func Open(ctx context.Context, backend, level, debug string) (context.Context, f
97	98		ctx = zerowrap.WithCtx(ctx, zerowrap.New(zerowrap.Config{Level: level, Format: "json", Output: w}))
98	99		ctx = context.WithValue(ctx, debugKey{}, debugSet(selected))
99	100		ctx = context.WithValue(ctx, levelKey{}, level)
	101	+	ctx = context.WithValue(ctx, dirKey{}, dir)
100	102		return ctx, w.Close, nil
101	103		}
102	104

@@ -141,6 +143,12 @@ func (w *consoleAndFile) Write(p []byte) (int, error) {
141	143		return w.file.Write(p)
142	144		}
143	145
	146	+	// Dir is the directory of the run log Open created, or "" without one.
	147	+	func Dir(ctx context.Context) string {
	148	+	dir, _ := ctx.Value(dirKey{}).(string)
	149	+	return dir
	150	+	}
	151	+
144	152		// Enabled reports whether a debug category was named in --debug.
145	153		func Enabled(ctx context.Context, category string) bool {
146	154		selected, _ := ctx.Value(debugKey{}).(debugSet)


@@ -31,6 +31,7 @@ var categories = map[string]string{"input-motion": "input", "input-keys": "input
31		31
32	type debugKey struct{}	32		type debugKey struct{}
33	type levelKey struct{}	33		type levelKey struct{}
		34	+	type dirKey struct{}
34	type debugSet map[string]bool	35		type debugSet map[string]bool
35		36
36	func ParseDebug(value string) (map[string]bool, error) {	37		func ParseDebug(value string) (map[string]bool, error) {

@@ -97,6 +98,7 @@ func Open(ctx context.Context, backend, level, debug string) (context.Context, f
97	ctx = zerowrap.WithCtx(ctx, zerowrap.New(zerowrap.Config{Level: level, Format: "json", Output: w}))	98		ctx = zerowrap.WithCtx(ctx, zerowrap.New(zerowrap.Config{Level: level, Format: "json", Output: w}))
98	ctx = context.WithValue(ctx, debugKey{}, debugSet(selected))	99		ctx = context.WithValue(ctx, debugKey{}, debugSet(selected))
99	ctx = context.WithValue(ctx, levelKey{}, level)	100		ctx = context.WithValue(ctx, levelKey{}, level)
		101	+	ctx = context.WithValue(ctx, dirKey{}, dir)
100	return ctx, w.Close, nil	102		return ctx, w.Close, nil
101	}	103		}
102		104

@@ -141,6 +143,12 @@ func (w *consoleAndFile) Write(p []byte) (int, error) {
141	return w.file.Write(p)	143		return w.file.Write(p)
142	}	144		}
143		145
		146	+	// Dir is the directory of the run log Open created, or "" without one.
		147	+	func Dir(ctx context.Context) string {
		148	+	dir, _ := ctx.Value(dirKey{}).(string)
		149	+	return dir
		150	+	}
		151	+
144	// Enabled reports whether a debug category was named in --debug.	152		// Enabled reports whether a debug category was named in --debug.
145	func Enabled(ctx context.Context, category string) bool {	153		func Enabled(ctx context.Context, category string) bool {
146	selected, _ := ctx.Value(debugKey{}).(debugSet)	154		selected, _ := ctx.Value(debugKey{}).(debugSet)

A internal/adapters/watchdog/watchdog.go +113 -0

@@ -0,0 +1,113 @@
	1	+	// Package watchdog writes every goroutine stack to a file when a probed
	2	+	// owner loop stops answering, so a frozen session leaves the exact place
	3	+	// it is stuck on disk.
	4	+	package watchdog
	5	+
	6	+	import (
	7	+	"context"
	8	+	"fmt"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"runtime/pprof"
	12	+	"time"
	13	+
	14	+	"github.com/bnema/zerowrap"
	15	+	)
	16	+
	17	+	// Probe asks an owner loop to run an empty task: Ping returns once the
	18	+	// loop ran it. A loop that is stuck never returns.
	19	+	type Probe struct {
	20	+	Name string
	21	+	Ping func()
	22	+	}
	23	+
	24	+	// Options set how often the probes run and how long a loop may take.
	25	+	type Options struct {
	26	+	Dir string // where stall dumps go (the run log directory)
	27	+	Every time.Duration // between two probes of a loop
	28	+	Timeout time.Duration // a probe slower than this is a stall
	29	+	Log zerowrap.Logger
	30	+	}
	31	+
	32	+	// Run probes each loop probes returns until ctx ends; the set may change
	33	+	// between ticks (outputs come and go). On a stall it writes one dump per
	34	+	// stall: a loop must answer again before it is dumped again.
	35	+	func Run(ctx context.Context, probes func() []Probe, o Options) {
	36	+	type state struct {
	37	+	answer chan struct{} // the probe in flight, nil when none
	38	+	since time.Time
	39	+	stalled bool
	40	+	seen bool
	41	+	}
	42	+	states := map[string]*state{}
	43	+	tick := time.NewTicker(o.Every)
	44	+	defer tick.Stop()
	45	+	for {
	46	+	select {
	47	+	case <-ctx.Done():
	48	+	return
	49	+	case now := <-tick.C:
	50	+	for _, s := range states {
	51	+	s.seen = false
	52	+	}
	53	+	for _, p := range probes() {
	54	+	s := states[p.Name]
	55	+	if s == nil {
	56	+	s = &state{}
	57	+	states[p.Name] = s
	58	+	}
	59	+	s.seen = true
	60	+	if s.answer != nil {
	61	+	select {
	62	+	case <-s.answer:
	63	+	if s.stalled {
	64	+	o.Log.Warn().Str("loop", p.Name).Dur("after", now.Sub(s.since)).Msg("loop answers again")
	65	+	}
	66	+	s.answer, s.stalled = nil, false
	67	+	default:
	68	+	if !s.stalled && now.Sub(s.since) >= o.Timeout {
	69	+	s.stalled = true
	70	+	path, err := dump(o.Dir, p.Name, now)
	71	+	o.Log.Error().Err(err).Str("loop", p.Name).Dur("waited", now.Sub(s.since)).Str("dump", path).Msg("loop stalled")
	72	+	}
	73	+	continue // one probe in flight per loop
	74	+	}
	75	+	}
	76	+	answer := make(chan struct{})
	77	+	s.answer, s.since = answer, now
	78	+	go func() { p.Ping(); close(answer) }()
	79	+	}
	80	+	// A loop that left (an unplugged output) is forgotten; its
	81	+	// probe goroutine ends with the loop's context.
	82	+	for name, s := range states {
	83	+	if !s.seen {
	84	+	delete(states, name)
	85	+	}
	86	+	}
	87	+	}
	88	+	}
	89	+	}
	90	+
	91	+	// dump writes every goroutine stack to dir and syncs it: a frozen machine
	92	+	// is often force rebooted, which loses what the page cache still holds.
	93	+	func dump(dir, name string, now time.Time) (string, error) {
	94	+	path := filepath.Join(dir, fmt.Sprintf("stall-%s-%s.txt", now.Format("20060102-150405.000"), name))
	95	+	f, err := os.OpenFile(path, os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	96	+	if err != nil {
	97	+	return path, err
	98	+	}
	99	+	fmt.Fprintf(f, "loop %q stopped answering at %s\n\n", name, now.Format(time.RFC3339Nano))
	100	+	werr := pprof.Lookup("goroutine").WriteTo(f, 2)
	101	+	serr := f.Sync()
	102	+	cerr := f.Close()
	103	+	if d, err := os.Open(dir); err == nil {
	104	+	_ = d.Sync()
	105	+	_ = d.Close()
	106	+	}
	107	+	for _, err := range []error{werr, serr, cerr} {
	108	+	if err != nil {
	109	+	return path, err
	110	+	}
	111	+	}
	112	+	return path, nil
	113	+	}

@@ -0,0 +1,113 @@
	1	+	// Package watchdog writes every goroutine stack to a file when a probed
	2	+	// owner loop stops answering, so a frozen session leaves the exact place
	3	+	// it is stuck on disk.
	4	+	package watchdog
	5	+
	6	+	import (
	7	+	"context"
	8	+	"fmt"
	9	+	"os"
	10	+	"path/filepath"
	11	+	"runtime/pprof"
	12	+	"time"
	13	+
	14	+	"github.com/bnema/zerowrap"
	15	+	)
	16	+
	17	+	// Probe asks an owner loop to run an empty task: Ping returns once the
	18	+	// loop ran it. A loop that is stuck never returns.
	19	+	type Probe struct {
	20	+	Name string
	21	+	Ping func()
	22	+	}
	23	+
	24	+	// Options set how often the probes run and how long a loop may take.
	25	+	type Options struct {
	26	+	Dir string // where stall dumps go (the run log directory)
	27	+	Every time.Duration // between two probes of a loop
	28	+	Timeout time.Duration // a probe slower than this is a stall
	29	+	Log zerowrap.Logger
	30	+	}
	31	+
	32	+	// Run probes each loop probes returns until ctx ends; the set may change
	33	+	// between ticks (outputs come and go). On a stall it writes one dump per
	34	+	// stall: a loop must answer again before it is dumped again.
	35	+	func Run(ctx context.Context, probes func() []Probe, o Options) {
	36	+	type state struct {
	37	+	answer chan struct{} // the probe in flight, nil when none
	38	+	since time.Time
	39	+	stalled bool
	40	+	seen bool
	41	+	}
	42	+	states := map[string]*state{}
	43	+	tick := time.NewTicker(o.Every)
	44	+	defer tick.Stop()
	45	+	for {
	46	+	select {
	47	+	case <-ctx.Done():
	48	+	return
	49	+	case now := <-tick.C:
	50	+	for _, s := range states {
	51	+	s.seen = false
	52	+	}
	53	+	for _, p := range probes() {
	54	+	s := states[p.Name]
	55	+	if s == nil {
	56	+	s = &state{}
	57	+	states[p.Name] = s
	58	+	}
	59	+	s.seen = true
	60	+	if s.answer != nil {
	61	+	select {
	62	+	case <-s.answer:
	63	+	if s.stalled {
	64	+	o.Log.Warn().Str("loop", p.Name).Dur("after", now.Sub(s.since)).Msg("loop answers again")
	65	+	}
	66	+	s.answer, s.stalled = nil, false
	67	+	default:
	68	+	if !s.stalled && now.Sub(s.since) >= o.Timeout {
	69	+	s.stalled = true
	70	+	path, err := dump(o.Dir, p.Name, now)
	71	+	o.Log.Error().Err(err).Str("loop", p.Name).Dur("waited", now.Sub(s.since)).Str("dump", path).Msg("loop stalled")
	72	+	}
	73	+	continue // one probe in flight per loop
	74	+	}
	75	+	}
	76	+	answer := make(chan struct{})
	77	+	s.answer, s.since = answer, now
	78	+	go func() { p.Ping(); close(answer) }()
	79	+	}
	80	+	// A loop that left (an unplugged output) is forgotten; its
	81	+	// probe goroutine ends with the loop's context.
	82	+	for name, s := range states {
	83	+	if !s.seen {
	84	+	delete(states, name)
	85	+	}
	86	+	}
	87	+	}
	88	+	}
	89	+	}
	90	+
	91	+	// dump writes every goroutine stack to dir and syncs it: a frozen machine
	92	+	// is often force rebooted, which loses what the page cache still holds.
	93	+	func dump(dir, name string, now time.Time) (string, error) {
	94	+	path := filepath.Join(dir, fmt.Sprintf("stall-%s-%s.txt", now.Format("20060102-150405.000"), name))
	95	+	f, err := os.OpenFile(path, os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, 0o600)
	96	+	if err != nil {
	97	+	return path, err
	98	+	}
	99	+	fmt.Fprintf(f, "loop %q stopped answering at %s\n\n", name, now.Format(time.RFC3339Nano))
	100	+	werr := pprof.Lookup("goroutine").WriteTo(f, 2)
	101	+	serr := f.Sync()
	102	+	cerr := f.Close()
	103	+	if d, err := os.Open(dir); err == nil {
	104	+	_ = d.Sync()
	105	+	_ = d.Close()
	106	+	}
	107	+	for _, err := range []error{werr, serr, cerr} {
	108	+	if err != nil {
	109	+	return path, err
	110	+	}
	111	+	}
	112	+	return path, nil
	113	+	}

A internal/adapters/watchdog/watchdog_test.go +66 -0

@@ -0,0 +1,66 @@
	1	+	package watchdog
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"path/filepath"
	7	+	"strings"
	8	+	"sync"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+	)
	14	+
	15	+	// stuckOwnerLoop blocks like an owner goroutine that never reaches its
	16	+	// select again; the dump must name it.
	17	+	func stuckOwnerLoop(release <-chan struct{}) { <-release }
	18	+
	19	+	func TestStalledLoopIsDumpedOnce(t *testing.T) {
	20	+	dir := t.TempDir()
	21	+	release := make(chan struct{})
	22	+	var mu sync.Mutex
	23	+	answered := 0
	24	+	probes := func() []Probe {
	25	+	return []Probe{
	26	+	{Name: "stuck", Ping: func() { stuckOwnerLoop(release) }},
	27	+	{Name: "fine", Ping: func() { mu.Lock(); answered++; mu.Unlock() }},
	28	+	}
	29	+	}
	30	+	ctx, cancel := context.WithCancel(context.Background())
	31	+	done := make(chan struct{})
	32	+	go func() {
	33	+	defer close(done)
	34	+	Run(ctx, probes, Options{Dir: dir, Every: 5 * time.Millisecond, Timeout: 30 * time.Millisecond, Log: zerowrap.Default()})
	35	+	}()
	36	+	deadline := time.Now().Add(2 * time.Second)
	37	+	var dumps []string
	38	+	for len(dumps) == 0 && time.Now().Before(deadline) {
	39	+	time.Sleep(5 * time.Millisecond)
	40	+	dumps, _ = filepath.Glob(filepath.Join(dir, "stall-*-stuck.txt"))
	41	+	}
	42	+	if len(dumps) != 1 {
	43	+	t.Fatalf("dumps %v", dumps)
	44	+	}
	45	+	// Still stuck: no second dump.
	46	+	time.Sleep(100 * time.Millisecond)
	47	+	if again, _ := filepath.Glob(filepath.Join(dir, "stall-*.txt")); len(again) != 1 {
	48	+	t.Fatalf("dumps after the first %v", again)
	49	+	}
	50	+	data, err := os.ReadFile(dumps[0])
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	if !strings.Contains(string(data), "stuckOwnerLoop") {
	55	+	t.Fatalf("dump does not name the stuck function:\n%s", data)
	56	+	}
	57	+	mu.Lock()
	58	+	n := answered
	59	+	mu.Unlock()
	60	+	if n < 2 {
	61	+	t.Fatalf("a healthy loop was probed %d times", n)
	62	+	}
	63	+	close(release)
	64	+	cancel()
	65	+	<-done
	66	+	}

@@ -0,0 +1,66 @@
	1	+	package watchdog
	2	+
	3	+	import (
	4	+	"context"
	5	+	"os"
	6	+	"path/filepath"
	7	+	"strings"
	8	+	"sync"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/bnema/zerowrap"
	13	+	)
	14	+
	15	+	// stuckOwnerLoop blocks like an owner goroutine that never reaches its
	16	+	// select again; the dump must name it.
	17	+	func stuckOwnerLoop(release <-chan struct{}) { <-release }
	18	+
	19	+	func TestStalledLoopIsDumpedOnce(t *testing.T) {
	20	+	dir := t.TempDir()
	21	+	release := make(chan struct{})
	22	+	var mu sync.Mutex
	23	+	answered := 0
	24	+	probes := func() []Probe {
	25	+	return []Probe{
	26	+	{Name: "stuck", Ping: func() { stuckOwnerLoop(release) }},
	27	+	{Name: "fine", Ping: func() { mu.Lock(); answered++; mu.Unlock() }},
	28	+	}
	29	+	}
	30	+	ctx, cancel := context.WithCancel(context.Background())
	31	+	done := make(chan struct{})
	32	+	go func() {
	33	+	defer close(done)
	34	+	Run(ctx, probes, Options{Dir: dir, Every: 5 * time.Millisecond, Timeout: 30 * time.Millisecond, Log: zerowrap.Default()})
	35	+	}()
	36	+	deadline := time.Now().Add(2 * time.Second)
	37	+	var dumps []string
	38	+	for len(dumps) == 0 && time.Now().Before(deadline) {
	39	+	time.Sleep(5 * time.Millisecond)
	40	+	dumps, _ = filepath.Glob(filepath.Join(dir, "stall-*-stuck.txt"))
	41	+	}
	42	+	if len(dumps) != 1 {
	43	+	t.Fatalf("dumps %v", dumps)
	44	+	}
	45	+	// Still stuck: no second dump.
	46	+	time.Sleep(100 * time.Millisecond)
	47	+	if again, _ := filepath.Glob(filepath.Join(dir, "stall-*.txt")); len(again) != 1 {
	48	+	t.Fatalf("dumps after the first %v", again)
	49	+	}
	50	+	data, err := os.ReadFile(dumps[0])
	51	+	if err != nil {
	52	+	t.Fatal(err)
	53	+	}
	54	+	if !strings.Contains(string(data), "stuckOwnerLoop") {
	55	+	t.Fatalf("dump does not name the stuck function:\n%s", data)
	56	+	}
	57	+	mu.Lock()
	58	+	n := answered
	59	+	mu.Unlock()
	60	+	if n < 2 {
	61	+	t.Fatalf("a healthy loop was probed %d times", n)
	62	+	}
	63	+	close(release)
	64	+	cancel()
	65	+	<-done
	66	+	}

A internal/adapters/wayland/probe.go +5 -0

@@ -0,0 +1,5 @@
	1	+	package wayland
	2	+
	3	+	// Ping returns once the display loop ran an empty task, or once the display
	4	+	// stopped. The watchdog uses it to detect a stuck display loop.
	5	+	func (s *Server) Ping() { s.display.Do(func() {}) }

@@ -0,0 +1,5 @@
	1	+	package wayland
	2	+
	3	+	// Ping returns once the display loop ran an empty task, or once the display
	4	+	// stopped. The watchdog uses it to detect a stuck display loop.
	5	+	func (s *Server) Ping() { s.display.Do(func() {}) }

M internal/app/drm.go +37 -0


@@ -19,6 +19,7 @@ import (
19	19		"github.com/bnema/neferwl/internal/adapters/logging"
20	20		"github.com/bnema/neferwl/internal/adapters/sched"
21	21		"github.com/bnema/neferwl/internal/adapters/seat"
	22	+	"github.com/bnema/neferwl/internal/adapters/watchdog"
22	23		"github.com/bnema/neferwl/internal/ports"
23	24		"github.com/bnema/zerowrap"
24	25		)

@@ -119,6 +120,35 @@ type drmBackend struct {
119	120		// reads them while the output loop adds and removes them.
120	121		mu sync.Mutex
121	122		preparers map[seatDisablePreparer]struct{}
	123	+	// probes are the running output loops the watchdog pings, by
	124	+	// connector, guarded by mu.
	125	+	probes map[string]func(context.Context)
	126	+	}
	127	+
	128	+	func (b *drmBackend) addProbe(name string, ping func(context.Context)) {
	129	+	b.mu.Lock()
	130	+	defer b.mu.Unlock()
	131	+	if b.probes == nil {
	132	+	b.probes = map[string]func(context.Context){}
	133	+	}
	134	+	b.probes[name] = ping
	135	+	}
	136	+
	137	+	func (b *drmBackend) removeProbe(name string) {
	138	+	b.mu.Lock()
	139	+	defer b.mu.Unlock()
	140	+	delete(b.probes, name)
	141	+	}
	142	+
	143	+	// watchdogProbes lists the running output loops for the watchdog.
	144	+	func (b *drmBackend) watchdogProbes(ctx context.Context) []watchdog.Probe {
	145	+	b.mu.Lock()
	146	+	defer b.mu.Unlock()
	147	+	out := make([]watchdog.Probe, 0, len(b.probes))
	148	+	for name, ping := range b.probes {
	149	+	out = append(out, watchdog.Probe{Name: "output-" + name, Ping: func() { ping(ctx) }})
	150	+	}
	151	+	return out
122	152		}
123	153
124	154		// seatDisablePreparer is an output that can reset its planes before the

@@ -490,6 +520,13 @@ func (b *drmBackend) runOutputs(ctx context.Context, want func(ports.Config) drm
490	520		defer o.Close()
491	521		b.addPreparer(o)
492	522		defer b.removePreparer(o) // before Close: it handles no request any more
	523	+	b.addProbe(name, func(pctx context.Context) {
	524	+	pctx, stop := context.WithCancel(pctx)
	525	+	defer stop()
	526	+	context.AfterFunc(octx, stop) // the probe ends with this output
	527	+	o.Ping(pctx)
	528	+	})
	529	+	defer b.removeProbe(name)
493	530		if realtime {
494	531		runtime.LockOSThread()
495	532		// Keep the thread locked until this goroutine exits: an RT


@@ -19,6 +19,7 @@ import (
19	"github.com/bnema/neferwl/internal/adapters/logging"	19		"github.com/bnema/neferwl/internal/adapters/logging"
20	"github.com/bnema/neferwl/internal/adapters/sched"	20		"github.com/bnema/neferwl/internal/adapters/sched"
21	"github.com/bnema/neferwl/internal/adapters/seat"	21		"github.com/bnema/neferwl/internal/adapters/seat"
		22	+	"github.com/bnema/neferwl/internal/adapters/watchdog"
22	"github.com/bnema/neferwl/internal/ports"	23		"github.com/bnema/neferwl/internal/ports"
23	"github.com/bnema/zerowrap"	24		"github.com/bnema/zerowrap"
24	)	25		)

@@ -119,6 +120,35 @@ type drmBackend struct {
119	// reads them while the output loop adds and removes them.	120		// reads them while the output loop adds and removes them.
120	mu sync.Mutex	121		mu sync.Mutex
121	preparers map[seatDisablePreparer]struct{}	122		preparers map[seatDisablePreparer]struct{}
		123	+	// probes are the running output loops the watchdog pings, by
		124	+	// connector, guarded by mu.
		125	+	probes map[string]func(context.Context)
		126	+	}
		127	+
		128	+	func (b *drmBackend) addProbe(name string, ping func(context.Context)) {
		129	+	b.mu.Lock()
		130	+	defer b.mu.Unlock()
		131	+	if b.probes == nil {
		132	+	b.probes = map[string]func(context.Context){}
		133	+	}
		134	+	b.probes[name] = ping
		135	+	}
		136	+
		137	+	func (b *drmBackend) removeProbe(name string) {
		138	+	b.mu.Lock()
		139	+	defer b.mu.Unlock()
		140	+	delete(b.probes, name)
		141	+	}
		142	+
		143	+	// watchdogProbes lists the running output loops for the watchdog.
		144	+	func (b *drmBackend) watchdogProbes(ctx context.Context) []watchdog.Probe {
		145	+	b.mu.Lock()
		146	+	defer b.mu.Unlock()
		147	+	out := make([]watchdog.Probe, 0, len(b.probes))
		148	+	for name, ping := range b.probes {
		149	+	out = append(out, watchdog.Probe{Name: "output-" + name, Ping: func() { ping(ctx) }})
		150	+	}
		151	+	return out
122	}	152		}
123		153
124	// seatDisablePreparer is an output that can reset its planes before the	154		// seatDisablePreparer is an output that can reset its planes before the

@@ -490,6 +520,13 @@ func (b *drmBackend) runOutputs(ctx context.Context, want func(ports.Config) drm
490	defer o.Close()	520		defer o.Close()
491	b.addPreparer(o)	521		b.addPreparer(o)
492	defer b.removePreparer(o) // before Close: it handles no request any more	522		defer b.removePreparer(o) // before Close: it handles no request any more
		523	+	b.addProbe(name, func(pctx context.Context) {
		524	+	pctx, stop := context.WithCancel(pctx)
		525	+	defer stop()
		526	+	context.AfterFunc(octx, stop) // the probe ends with this output
		527	+	o.Ping(pctx)
		528	+	})
		529	+	defer b.removeProbe(name)
493	if realtime {	530		if realtime {
494	runtime.LockOSThread()	531		runtime.LockOSThread()
495	// Keep the thread locked until this goroutine exits: an RT	532		// Keep the thread locked until this goroutine exits: an RT

M internal/app/session.go +17 -0


@@ -23,6 +23,7 @@ import (
23	23		"github.com/bnema/neferwl/internal/adapters/sessionsecurity"
24	24		"github.com/bnema/neferwl/internal/adapters/statefile"
25	25		"github.com/bnema/neferwl/internal/adapters/vulkan"
	26	+	"github.com/bnema/neferwl/internal/adapters/watchdog"
26	27		"github.com/bnema/neferwl/internal/adapters/wayland"
27	28		"github.com/bnema/neferwl/internal/adapters/workspaceid"
28	29		"github.com/bnema/neferwl/internal/adapters/xkb"

@@ -302,6 +303,22 @@ func (s *session) run(ctx context.Context, inject func(chan<- ports.InputEvent))
302	303		close(script)
303	304		}
304	305		go func() { defer workers.Done(); done <- server.Run(ctx) }()
	306	+	if dir := logging.Dir(ctx); dir != "" {
	307	+	workers.Add(1)
	308	+	go func() {
	309	+	defer workers.Done()
	310	+	watchdog.Run(ctx, func() []watchdog.Probe {
	311	+	probes := []watchdog.Probe{
	312	+	{Name: "wayland", Ping: server.Ping},
	313	+	{Name: "core", Ping: func() { c.Ping(ctx) }},
	314	+	}
	315	+	if hw != nil {
	316	+	probes = append(probes, hw.watchdogProbes(ctx)...)
	317	+	}
	318	+	return probes
	319	+	}, watchdog.Options{Dir: dir, Every: 500 * time.Millisecond, Timeout: 2 * time.Second, Log: logging.For(ctx, "app")})
	320	+	}()
	321	+	}
305	322		go func() {
306	323		defer workers.Done()
307	324		// Without a watch the allowlist stays as loaded: restart to apply.


@@ -23,6 +23,7 @@ import (
23	"github.com/bnema/neferwl/internal/adapters/sessionsecurity"	23		"github.com/bnema/neferwl/internal/adapters/sessionsecurity"
24	"github.com/bnema/neferwl/internal/adapters/statefile"	24		"github.com/bnema/neferwl/internal/adapters/statefile"
25	"github.com/bnema/neferwl/internal/adapters/vulkan"	25		"github.com/bnema/neferwl/internal/adapters/vulkan"
		26	+	"github.com/bnema/neferwl/internal/adapters/watchdog"
26	"github.com/bnema/neferwl/internal/adapters/wayland"	27		"github.com/bnema/neferwl/internal/adapters/wayland"
27	"github.com/bnema/neferwl/internal/adapters/workspaceid"	28		"github.com/bnema/neferwl/internal/adapters/workspaceid"
28	"github.com/bnema/neferwl/internal/adapters/xkb"	29		"github.com/bnema/neferwl/internal/adapters/xkb"

@@ -302,6 +303,22 @@ func (s *session) run(ctx context.Context, inject func(chan<- ports.InputEvent))
302	close(script)	303		close(script)
303	}	304		}
304	go func() { defer workers.Done(); done <- server.Run(ctx) }()	305		go func() { defer workers.Done(); done <- server.Run(ctx) }()
		306	+	if dir := logging.Dir(ctx); dir != "" {
		307	+	workers.Add(1)
		308	+	go func() {
		309	+	defer workers.Done()
		310	+	watchdog.Run(ctx, func() []watchdog.Probe {
		311	+	probes := []watchdog.Probe{
		312	+	{Name: "wayland", Ping: server.Ping},
		313	+	{Name: "core", Ping: func() { c.Ping(ctx) }},
		314	+	}
		315	+	if hw != nil {
		316	+	probes = append(probes, hw.watchdogProbes(ctx)...)
		317	+	}
		318	+	return probes
		319	+	}, watchdog.Options{Dir: dir, Every: 500 * time.Millisecond, Timeout: 2 * time.Second, Log: logging.For(ctx, "app")})
		320	+	}()
		321	+	}
305	go func() {	322		go func() {
306	defer workers.Done()	323		defer workers.Done()
307	// Without a watch the allowlist stays as loaded: restart to apply.	324		// Without a watch the allowlist stays as loaded: restart to apply.

M internal/core/core.go +22 -2


@@ -138,7 +138,9 @@ type Core struct {
138	138		swipe *swipeGesture
139	139		// gesture is the touchpad gesture forwarded to a client
140	140		// (clientgesture.go).
141		-	gesture clientGesture
	141	+	gesture clientGesture
	142	+	// ping carries watchdog probes (Ping); Run answers each one.
	143	+	ping chan chan struct{}
142	144		frameC <-chan time.Time
143	145		frameTimer ports.Timer
144	146		// shots is the snapshot of what the screens show before an action

@@ -401,7 +403,7 @@ func New(cfg ports.Config, ch Channels, opts Options) (*Core, error) {
401	403		return nil, fmt.Errorf("scenes, layouts, constraints, state and workspaces must have capacity 1")
402	404		}
403	405		// A placeholder screen holds windows until the first output arrives.
404		-	c := &Core{inputKeys: map[string]bool{}, slots: map[slotKey]slotState{}, placement: newSpawnPlacement(), windows: newWindowRegistry(), popups: map[WindowID]popupState{}, ch: ch, opts: opts, pressed: map[string]bool{}, buttons: map[uint32]bool{}, swallow: map[uint32]bool{}, configures: newConfigures()}
	406	+	c := &Core{inputKeys: map[string]bool{}, slots: map[slotKey]slotState{}, placement: newSpawnPlacement(), windows: newWindowRegistry(), popups: map[WindowID]popupState{}, ch: ch, opts: opts, pressed: map[string]bool{}, buttons: map[uint32]bool{}, swallow: map[uint32]bool{}, configures: newConfigures(), ping: make(chan chan struct{})}
405	407		c.screens = []*screen{{mon: newMonitorWithIDs("", "", &c.nextWorkspaceID), scale: 1, cfgScale: 1}}
406	408		if err := c.apply(cfg); err != nil {
407	409		return nil, err

@@ -1023,6 +1025,21 @@ func (c *Core) hit(x, y float64) (WindowID, float64, float64) {
1023	1025		return id, sx, sy
1024	1026		}
1025	1027
	1028	+	// Ping returns once Run handled a probe, or once ctx ends: the watchdog
	1029	+	// uses it to detect a stuck owner loop.
	1030	+	func (c *Core) Ping(ctx context.Context) {
	1031	+	ack := make(chan struct{})
	1032	+	select {
	1033	+	case c.ping <- ack:
	1034	+	case <-ctx.Done():
	1035	+	return
	1036	+	}
	1037	+	select {
	1038	+	case <-ack:
	1039	+	case <-ctx.Done():
	1040	+	}
	1041	+	}
	1042	+
1026	1043		func (c *Core) Run(ctx context.Context) error {
1027	1044		c.syncSecurity()
1028	1045		defer c.stopFrame()

@@ -1084,6 +1101,9 @@ func (c *Core) Run(ctx context.Context) error {
1084	1101		return nil
1085	1102		}
1086	1103		continue
	1104	+	case ack := <-c.ping:
	1105	+	close(ack)
	1106	+	continue
1087	1107		case <-c.frameC:
1088	1108		if c.securityCheckpoint(ctx) != nil {
1089	1109		return nil

 	swipe *swipeGesture
 	// gesture is the touchpad gesture forwarded to a client
 	// (clientgesture.go).
-	gesture    clientGesture
+	gesture clientGesture
+	// ping carries watchdog probes (Ping); Run answers each one.
+	ping       chan chan struct{}
 	frameC     <-chan time.Time
 	frameTimer ports.Timer
 	// shots is the snapshot of what the screens show before an action
 		return nil, fmt.Errorf("scenes, layouts, constraints, state and workspaces must have capacity 1")
 	}
 	// A placeholder screen holds windows until the first output arrives.
-	c := &Core{inputKeys: map[string]bool{}, slots: map[slotKey]*slotState{}, placement: newSpawnPlacement(), windows: newWindowRegistry(), popups: map[WindowID]*popupState{}, ch: ch, opts: opts, pressed: map[string]bool{}, buttons: map[uint32]bool{}, swallow: map[uint32]bool{}, configures: newConfigures()}
+	c := &Core{inputKeys: map[string]bool{}, slots: map[slotKey]*slotState{}, placement: newSpawnPlacement(), windows: newWindowRegistry(), popups: map[WindowID]*popupState{}, ch: ch, opts: opts, pressed: map[string]bool{}, buttons: map[uint32]bool{}, swallow: map[uint32]bool{}, configures: newConfigures(), ping: make(chan chan struct{})}
 	c.screens = []*screen{{mon: newMonitorWithIDs("", "", &c.nextWorkspaceID), scale: 1, cfgScale: 1}}
 	if err := c.apply(cfg); err != nil {
 		return nil, err
 	return id, sx, sy
 }
+// Ping returns once Run handled a probe, or once ctx ends: the watchdog
+// uses it to detect a stuck owner loop.
+func (c *Core) Ping(ctx context.Context) {
+	ack := make(chan struct{})
+	select {
+	case c.ping <- ack:
+	case <-ctx.Done():
+		return
+	}
+	select {
+	case <-ack:
+	case <-ctx.Done():
+	}
+}
 func (c *Core) Run(ctx context.Context) error {
 	c.syncSecurity()
 	defer c.stopFrame()
 				return nil
 			}
 			continue
+		case ack := <-c.ping:
+			close(ack)
+			continue
 		case <-c.frameC:
 			if c.securityCheckpoint(ctx) != nil {
 				return nil