fix(spawn): name a daemon that holds the lifecycle lock without answering

commit: d6feeac4e601e606a29e654378cebb2f42a32e3d
author: brice · 29d ago · 2026-09-23 19:53 UTC
parent: f44b572
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +60 -0

M internal/app/spawn.go +13 -0


@@ -25,6 +25,12 @@ import (
25	25		// dialable within the retry budget, despite a spawn attempt.
26	26		var ErrDaemonUnreachable = errors.New("vev: daemon did not become reachable")
27	27
	28	+	// ErrLifecycleHeldByOther reports that another process kept the daemon
	29	+	// lifecycle lock for the whole start budget without publishing the carriage:
	30	+	// typically an older vev daemon that predates the broker. Spawning cannot
	31	+	// recover this; the user has to stop that process.
	32	+	var ErrLifecycleHeldByOther = errors.New("vev: another vev daemon holds the lifecycle lock but does not answer; stop the old daemon (for example `pkill -f -- '--daemon'`) and retry")
	33	+
28	34		// spawnLockName is the mkdir-based lock directory guarding daemon spawn, so
29	35		// concurrent first-clients elect a single spawner instead of racing to
30	36		// re-exec multiple daemons.

@@ -116,6 +122,7 @@ func waitForTargetOrLifecycle[T any](ctx context.Context, lockDir string, dial f
116	122		target T
117	123		owner lifecycleOwnership
118	124		}
	125	+	busy := false
119	126		result, err := retryAttempts(ctx, cfg, func() (targetOrLifecycle, bool, error) {
120	127		if target, err := dial(ctx); err == nil {
121	128		return targetOrLifecycle{target: target}, true, nil

@@ -127,8 +134,14 @@ func waitForTargetOrLifecycle[T any](ctx context.Context, lockDir string, dial f
127	134		if !errors.Is(err, lifecycle.ErrBusy) {
128	135		return targetOrLifecycle{}, false, err
129	136		}
	137	+	busy = true
130	138		return targetOrLifecycle{}, false, nil
131	139		})
	140	+	if busy && errors.Is(err, ErrDaemonUnreachable) {
	141	+	// The lock stayed held and the target never answered for the whole
	142	+	// budget: a live owner that is not the daemon this binary expects.
	143	+	err = fmt.Errorf("%w (%w)", ErrLifecycleHeldByOther, err)
	144	+	}
132	145		return result.target, result.owner, err
133	146		}
134	147


@@ -25,6 +25,12 @@ import (
25	// dialable within the retry budget, despite a spawn attempt.	25		// dialable within the retry budget, despite a spawn attempt.
26	var ErrDaemonUnreachable = errors.New("vev: daemon did not become reachable")	26		var ErrDaemonUnreachable = errors.New("vev: daemon did not become reachable")
27		27
		28	+	// ErrLifecycleHeldByOther reports that another process kept the daemon
		29	+	// lifecycle lock for the whole start budget without publishing the carriage:
		30	+	// typically an older vev daemon that predates the broker. Spawning cannot
		31	+	// recover this; the user has to stop that process.
		32	+	var ErrLifecycleHeldByOther = errors.New("vev: another vev daemon holds the lifecycle lock but does not answer; stop the old daemon (for example `pkill -f -- '--daemon'`) and retry")
		33	+
28	// spawnLockName is the mkdir-based lock directory guarding daemon spawn, so	34		// spawnLockName is the mkdir-based lock directory guarding daemon spawn, so
29	// concurrent first-clients elect a single spawner instead of racing to	35		// concurrent first-clients elect a single spawner instead of racing to
30	// re-exec multiple daemons.	36		// re-exec multiple daemons.

@@ -116,6 +122,7 @@ func waitForTargetOrLifecycle[T any](ctx context.Context, lockDir string, dial f
116	target T	122		target T
117	owner lifecycleOwnership	123		owner lifecycleOwnership
118	}	124		}
		125	+	busy := false
119	result, err := retryAttempts(ctx, cfg, func() (targetOrLifecycle, bool, error) {	126		result, err := retryAttempts(ctx, cfg, func() (targetOrLifecycle, bool, error) {
120	if target, err := dial(ctx); err == nil {	127		if target, err := dial(ctx); err == nil {
121	return targetOrLifecycle{target: target}, true, nil	128		return targetOrLifecycle{target: target}, true, nil

@@ -127,8 +134,14 @@ func waitForTargetOrLifecycle[T any](ctx context.Context, lockDir string, dial f
127	if !errors.Is(err, lifecycle.ErrBusy) {	134		if !errors.Is(err, lifecycle.ErrBusy) {
128	return targetOrLifecycle{}, false, err	135		return targetOrLifecycle{}, false, err
129	}	136		}
		137	+	busy = true
130	return targetOrLifecycle{}, false, nil	138		return targetOrLifecycle{}, false, nil
131	})	139		})
		140	+	if busy && errors.Is(err, ErrDaemonUnreachable) {
		141	+	// The lock stayed held and the target never answered for the whole
		142	+	// budget: a live owner that is not the daemon this binary expects.
		143	+	err = fmt.Errorf("%w (%w)", ErrLifecycleHeldByOther, err)
		144	+	}
132	return result.target, result.owner, err	145		return result.target, result.owner, err
133	}	146		}
134		147

A internal/app/spawn_lifecycle_held_test.go +47 -0

@@ -0,0 +1,47 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/require"
	10	+
	11	+	"github.com/bnema/vev/internal/adapters/lifecycle"
	12	+	"github.com/bnema/vev/pkg/safedir"
	13	+	)
	14	+
	15	+	func TestWaitForTargetOrLifecycleNamesAHeldLock(t *testing.T) {
	16	+	cfg := backoffConfig{initial: time.Millisecond, max: 2 * time.Millisecond, total: 20 * time.Millisecond}
	17	+	refuse := func(context.Context) (struct{}, error) { return struct{}{}, errors.New("no carriage") }
	18	+	tests := []struct {
	19	+	name string
	20	+	held bool
	21	+	wantHeld bool
	22	+	}{
	23	+	{name: "lock held by a silent owner", held: true, wantHeld: true},
	24	+	{name: "free lock is acquired", held: false, wantHeld: false},
	25	+	}
	26	+	for _, tt := range tests {
	27	+	t.Run(tt.name, func(t *testing.T) {
	28	+	dir := t.TempDir() + "/runtime"
	29	+	require.NoError(t, safedir.EnsurePrivate(dir))
	30	+	if tt.held {
	31	+	owner, err := lifecycle.TryAcquire(dir)
	32	+	require.NoError(t, err)
	33	+	t.Cleanup(func() { _ = owner.Release() })
	34	+	}
	35	+
	36	+	_, owner, err := waitForTargetOrLifecycle(context.Background(), dir, refuse, cfg)
	37	+
	38	+	if tt.wantHeld {
	39	+	require.ErrorIs(t, err, ErrLifecycleHeldByOther)
	40	+	require.ErrorIs(t, err, ErrDaemonUnreachable)
	41	+	return
	42	+	}
	43	+	require.NoError(t, err)
	44	+	require.NoError(t, owner.Release())
	45	+	})
	46	+	}
	47	+	}

@@ -0,0 +1,47 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"testing"
	7	+	"time"
	8	+
	9	+	"github.com/stretchr/testify/require"
	10	+
	11	+	"github.com/bnema/vev/internal/adapters/lifecycle"
	12	+	"github.com/bnema/vev/pkg/safedir"
	13	+	)
	14	+
	15	+	func TestWaitForTargetOrLifecycleNamesAHeldLock(t *testing.T) {
	16	+	cfg := backoffConfig{initial: time.Millisecond, max: 2 * time.Millisecond, total: 20 * time.Millisecond}
	17	+	refuse := func(context.Context) (struct{}, error) { return struct{}{}, errors.New("no carriage") }
	18	+	tests := []struct {
	19	+	name string
	20	+	held bool
	21	+	wantHeld bool
	22	+	}{
	23	+	{name: "lock held by a silent owner", held: true, wantHeld: true},
	24	+	{name: "free lock is acquired", held: false, wantHeld: false},
	25	+	}
	26	+	for _, tt := range tests {
	27	+	t.Run(tt.name, func(t *testing.T) {
	28	+	dir := t.TempDir() + "/runtime"
	29	+	require.NoError(t, safedir.EnsurePrivate(dir))
	30	+	if tt.held {
	31	+	owner, err := lifecycle.TryAcquire(dir)
	32	+	require.NoError(t, err)
	33	+	t.Cleanup(func() { _ = owner.Release() })
	34	+	}
	35	+
	36	+	_, owner, err := waitForTargetOrLifecycle(context.Background(), dir, refuse, cfg)
	37	+
	38	+	if tt.wantHeld {
	39	+	require.ErrorIs(t, err, ErrLifecycleHeldByOther)
	40	+	require.ErrorIs(t, err, ErrDaemonUnreachable)
	41	+	return
	42	+	}
	43	+	require.NoError(t, err)
	44	+	require.NoError(t, owner.Release())
	45	+	})
	46	+	}
	47	+	}