resident_failure_recovery_test.go RawHistoryFolder
1package subagents
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "testing"
12 "time"
13
14 "github.com/bnema/zut/packages/core"
15 "github.com/bnema/zut/packages/provider"
16)
17
18func TestResidentFailureRecoveryKeepsRequiredUnmetUntilResume(t *testing.T) {
19 for _, mode := range []string{"read-only", "shared", "worktree"} {
20 for _, restart := range []bool{false, true} {
21 t.Run(fmt.Sprintf("%s/restart=%t", mode, restart), func(t *testing.T) {
22 root, workspace := t.TempDir(), t.TempDir()
23 if mode == "worktree" {
24 workspace = initTestRepo(t)
25 }
26 spec := ResidentChildSpec{ID: "failure-recovery", SessionID: "session", InitialTurnID: "initial", Provider: "openai", Model: "test", Required: true, RepositoryRoot: workspace, WorkspaceMode: WorkspaceShared}
27 if mode == "worktree" {
28 spec.WorkspaceMode, spec.WorkspaceCapture = WorkspaceWorktree, CapturePatch
29 }
30 turnErr := errors.New("turn failed")
31 factory := func(spec ResidentChildSpec, journal *ResidentJournal) (ResidentRuntime, error) {
32 return ResidentTurnRunner(func(_ context.Context, prompt string) error {
33 if prompt == "continue remaining work" {
34 messages, err := ReadResidentTranscriptMessages(journal.Dir())
35 if err != nil || len(messages) < 3 {
36 return fmt.Errorf("missing restored progress: %d messages, %v", len(messages), err)
37 }
38 if mode != "read-only" {
39 data, err := os.ReadFile(filepath.Join(spec.Workspace, "README.md"))
40 if err != nil || string(data) != "partial artifact\n" {
41 return fmt.Errorf("lost artifact: %q, %v", data, err)
42 }
43 }
44 if err := journal.RecordAgentEvent(core.EvUsage{Cumulative: provider.Usage{InputTokens: 105}}); err != nil {
45 return err
46 }
47 return journal.RecordAgentEvent(core.EvAssistantMessage{Message: provider.Message{Role: provider.RoleAssistant, Content: []provider.Content{provider.TextBlock{Text: "verified remaining work"}}}})
48 }
49 tool, output := "read", "inspected README; verification unfinished"
50 if mode != "read-only" {
51 tool, output = "write", "wrote README.md; verification unfinished"
52 if err := os.WriteFile(filepath.Join(spec.Workspace, "README.md"), []byte("partial artifact\n"), 0o600); err != nil {
53 return err
54 }
55 }
56 args := json.RawMessage(`{"path":"README.md"}`)
57 for _, event := range []core.AgentEvent{
58 core.EvUserMessage{Message: provider.Message{Role: provider.RoleUser, Content: []provider.Content{provider.TextBlock{Text: prompt}}}},
59 core.EvAssistantMessage{Message: provider.Message{Role: provider.RoleAssistant, Content: []provider.Content{provider.TextBlock{Text: "working on README.md; verification unfinished"}, provider.ToolCallBlock{ID: "call", Name: tool, Arguments: args}}}},
60 core.EvToolCall{ID: "call", Name: tool, Args: args},
61 core.EvToolResult{ID: "call", Result: core.ToolResult{Content: []provider.Content{provider.TextBlock{Text: output}}}},
62 core.EvUsage{Cumulative: provider.Usage{InputTokens: 100}},
63 } {
64 if err := journal.RecordAgentEvent(event); err != nil {
65 return err
66 }
67 }
68 return turnErr
69 }), nil
70 }
71 manager := NewResidentManager(root, factory)
72 t.Cleanup(func() { _ = manager.Close(context.Background()) })
73 completions := make(chan ResidentCompletion, 2)
74 manager.SetCompletionObserver(func(c ResidentCompletion) bool { completions <- c; return true })
75 if _, err := manager.Spawn(t.Context(), spec, "inspect and verify README"); err != nil {
76 t.Fatal(err)
77 }
78 completion := awaitResidentCompletion(t, completions)
79 if !errors.Is(completion.Err, turnErr) {
80 t.Fatalf("terminal notification = %#v", completion)
81 }
82 assertFailedPartial := func() ResidentResult {
83 t.Helper()
84 snapshot, ok := manager.SnapshotFor(spec.ID)
85 if !ok || snapshot.State != ResidentFailed || len(manager.UnmetRequired()) != 1 {
86 t.Fatalf("partial status = %#v; unmet = %#v", snapshot, manager.UnmetRequired())
87 }
88 result, err := manager.Result(spec.ID)
89 if err != nil || result.ErrorCode != "turn_failed" || result.Handoff != "" {
90 t.Fatalf("saved partial result = %#v, %v", result, err)
91 }
92 return result
93 }
94 result := assertFailedPartial()
95 if mode == "worktree" {
96 if result.PatchRef != PatchRef(spec.ID) || len(result.ChangedFiles) != 1 {
97 t.Fatalf("missing partial patch: %#v", result)
98 }
99 patch, err := os.ReadFile(filepath.Join(root, spec.ID, residentPatchName))
100 if err != nil || !strings.Contains(string(patch), "partial artifact") {
101 t.Fatalf("partial patch = %q, %v", patch, err)
102 }
103 }
104 if restart {
105 if err := manager.Close(t.Context()); err != nil {
106 t.Fatal(err)
107 }
108 manager = NewResidentManager(root, factory)
109 manager.SetCompletionObserver(func(c ResidentCompletion) bool { completions <- c; return true })
110 if errs := manager.Reconcile(); len(errs) != 0 {
111 t.Fatal(errs)
112 }
113 assertFailedPartial()
114 }
115 if err := manager.Resume(t.Context(), spec.ID, "continue remaining work"); err != nil {
116 t.Fatal(err)
117 }
118 if completion := awaitResidentCompletion(t, completions); completion.Err != nil {
119 t.Fatal(completion.Err)
120 }
121 if unmet := manager.UnmetRequired(); len(unmet) != 0 {
122 t.Fatalf("successful recovery remains unmet: %#v", unmet)
123 }
124 snapshot, _ := manager.SnapshotFor(spec.ID)
125 if snapshot.Usage.InputTokens != 105 {
126 t.Fatalf("recovery lost cumulative accounting: %#v", snapshot)
127 }
128 if err := manager.Close(t.Context()); err != nil {
129 t.Fatal(err)
130 }
131 metadata, err := ReconcileResidentJournal(filepath.Join(root, spec.ID))
132 if err != nil || metadata.Usage.InputTokens != 105 {
133 t.Fatalf("recovery usage did not survive reconciliation: %#v, %v", metadata, err)
134 }
135 })
136 }
137 }
138}
139
140func awaitResidentCompletion(t *testing.T, completions <-chan ResidentCompletion) ResidentCompletion {
141 t.Helper()
142 select {
143 case completion := <-completions:
144 return completion
145 case <-time.After(5 * time.Second):
146 t.Fatal("no terminal notification")
147 return ResidentCompletion{}
148 }
149}