1package core
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "io"
8 "reflect"
9 "strings"
10 "sync/atomic"
11 "testing"
12 "time"
13 "unicode/utf8"
14
15 "github.com/bnema/zut/packages/provider"
16)
17
18type retryFakeClient struct {
19 calls int32
20 firstErr error
21}
22
23func (c *retryFakeClient) Name() string { return "retry-fake" }
24
25func (c *retryFakeClient) Stream(ctx context.Context, req provider.Request) (<-chan provider.Event, error) {
26 if req.Lifecycle != nil {
27 req.Lifecycle.RequestAttempt(1, 1)
28 }
29 call := atomic.AddInt32(&c.calls, 1)
30 out := make(chan provider.Event, 4)
31 go func() {
32 defer close(out)
33 out <- provider.EventStart{Provider: "retry-fake", Model: req.Model}
34 if call == 1 {
35 err := c.firstErr
36 if err == nil {
37 err = fmt.Errorf("anthropic overloaded_error: Overloaded")
38 }
39 out <- provider.EventDone{Stop: provider.StopError, Err: err}
40 return
41 }
42 out <- provider.EventTextDelta{Delta: "ok"}
43 out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{
44 Role: provider.RoleAssistant,
45 Content: []provider.Content{provider.TextBlock{Text: "ok"}},
46 }}
47 }()
48 return out, nil
49}
50
51type silentStreamClient struct{}
52
53func (silentStreamClient) Name() string { return "silent-stream" }
54
55func (silentStreamClient) Stream(ctx context.Context, _ provider.Request) (<-chan provider.Event, error) {
56 out := make(chan provider.Event)
57 go func() {
58 defer close(out)
59 <-ctx.Done()
60 }()
61 return out, nil
62}
63
64type activeStreamClient struct{}
65
66func (activeStreamClient) Name() string { return "active-stream" }
67
68func (activeStreamClient) Stream(context.Context, provider.Request) (<-chan provider.Event, error) {
69 out := make(chan provider.Event, 2)
70 out <- provider.EventStart{}
71 out <- provider.EventDone{Stop: provider.StopEnd}
72 close(out)
73 return out, nil
74}
75
76type requestOpenFailureClient struct{ calls int32 }
77
78func (c *requestOpenFailureClient) Name() string { return "request-open-failure" }
79
80func (c *requestOpenFailureClient) Stream(context.Context, provider.Request) (<-chan provider.Event, error) {
81 atomic.AddInt32(&c.calls, 1)
82 return nil, errors.New("provider returned error: 503 service unavailable")
83}
84
85type readableAfterCancellationClient struct {
86 cancel context.CancelFunc
87 drained chan struct{}
88}
89
90func (c readableAfterCancellationClient) Name() string { return "readable-after-cancellation" }
91
92func (c readableAfterCancellationClient) Stream(_ context.Context, _ provider.Request) (<-chan provider.Event, error) {
93 out := make(chan provider.Event, 1)
94 out <- provider.EventStart{}
95 c.cancel()
96 go func() {
97 out <- provider.EventTextDelta{Delta: "discard after cancellation"}
98 close(out)
99 close(c.drained)
100 }()
101 return out, nil
102}
103
104func TestNewAgentUsesCodexStreamGuardDefaults(t *testing.T) {
105 a := NewAgent(&retryFakeClient{}, "fake-model", "system", Registry{})
106 if a.MaxSteps != 0 {
107 t.Fatalf("max steps = %d, want unlimited", a.MaxSteps)
108 }
109 if a.MaxRetries != 5 {
110 t.Fatalf("stream retries = %d, want 5", a.MaxRetries)
111 }
112 if a.StreamIdleTimeout != 5*time.Minute {
113 t.Fatalf("stream idle timeout = %s, want 5m", a.StreamIdleTimeout)
114 }
115}
116
117func TestAgentStopsReadingStreamAfterCancellation(t *testing.T) {
118 ctx, cancel := context.WithCancel(context.Background())
119 drained := make(chan struct{})
120 a := NewAgent(readableAfterCancellationClient{cancel: cancel, drained: drained}, "fake-model", "system", Registry{})
121 a.MaxRetries = 0
122 if err := a.Prompt(ctx, "hello", nil, nil); !errors.Is(err, context.Canceled) {
123 t.Fatalf("Prompt error = %v, want context.Canceled", err)
124 }
125 select {
126 case <-drained:
127 case <-time.After(time.Second):
128 t.Fatal("provider producer remained blocked after turn cancellation")
129 }
130}
131
132func TestAgentStopsSilentStreamAtIdleDeadline(t *testing.T) {
133 a := NewAgent(silentStreamClient{}, "fake-model", "system", Registry{})
134 a.MaxRetries = 0
135 a.StreamIdleTimeout = time.Minute
136 idle := make(chan time.Time, 1)
137 a.streamIdleTimer = func(time.Duration) (<-chan time.Time, func(), func()) {
138 idle <- time.Now()
139 return idle, func() {}, func() {}
140 }
141 if err := a.Prompt(context.Background(), "hello", nil, nil); !errors.Is(err, ErrStreamIdleTimeout) {
142 t.Fatalf("Prompt error = %v, want ErrStreamIdleTimeout", err)
143 }
144}
145
146func TestAgentResetsIdleDeadlineForEveryStreamEvent(t *testing.T) {
147 a := NewAgent(activeStreamClient{}, "fake-model", "system", Registry{})
148 a.MaxRetries = 0
149 a.StreamIdleTimeout = time.Minute
150 var resets atomic.Int32
151 a.streamIdleTimer = func(time.Duration) (<-chan time.Time, func(), func()) {
152 return make(chan time.Time), func() {}, func() { resets.Add(1) }
153 }
154 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
155 t.Fatal(err)
156 }
157 if got := resets.Load(); got != 2 {
158 t.Fatalf("idle timer resets = %d, want one for each stream event", got)
159 }
160}
161
162func TestAgentRetriesSilentStreamWithoutEndingTurn(t *testing.T) {
163 client := &retryFakeClient{firstErr: ErrStreamIdleTimeout}
164 a := NewAgent(client, "fake-model", "system", Registry{})
165 a.MaxRetries = 1
166 a.RetryBaseDelay = time.Millisecond
167 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
168 t.Fatalf("Prompt returned %v", err)
169 }
170 if got := atomic.LoadInt32(&client.calls); got != 2 {
171 t.Fatalf("Stream calls = %d; want reconnect after idle timeout", got)
172 }
173 if got := extractText(a.Messages()[1]); got != "ok" {
174 t.Fatalf("final assistant text = %q; want ok", got)
175 }
176}
177
178func TestAgentRetriesOverloadedStreamError(t *testing.T) {
179 client := &retryFakeClient{}
180 a := NewAgent(client, "fake-model", "system", Registry{})
181 a.RetryBaseDelay = time.Millisecond
182
183 var turnErrs []string
184 err := a.Prompt(context.Background(), "hello", nil, func(ev AgentEvent) {
185 if e, ok := ev.(EvTurnEnd); ok && e.Err != nil {
186 turnErrs = append(turnErrs, e.Err.Error())
187 }
188 })
189 if err != nil {
190 t.Fatalf("Prompt returned %v", err)
191 }
192 if got := atomic.LoadInt32(&client.calls); got != 2 {
193 t.Fatalf("Stream calls = %d; want 2", got)
194 }
195 if len(turnErrs) != 1 || !strings.Contains(turnErrs[0], "overloaded_error") {
196 t.Fatalf("turn errors = %v; want one overloaded error before retry", turnErrs)
197 }
198 msgs := a.Messages()
199 if len(msgs) != 2 {
200 t.Fatalf("message count = %d; want user + final assistant", len(msgs))
201 }
202 if got := extractText(msgs[1]); got != "ok" {
203 t.Fatalf("final assistant text = %q; want ok", got)
204 }
205}
206
207func TestAgentDoesNotReplayProviderRequestRetriesAtStreamLayer(t *testing.T) {
208 client := &requestOpenFailureClient{}
209 a := NewAgent(client, "fake-model", "system", Registry{})
210 a.RetryBaseDelay = time.Millisecond
211
212 err := a.Prompt(context.Background(), "hello", nil, nil)
213 if err == nil || !strings.Contains(err.Error(), "503") {
214 t.Fatalf("Prompt error = %v, want provider failure", err)
215 }
216 if got := atomic.LoadInt32(&client.calls); got != 1 {
217 t.Fatalf("Stream calls = %d, want one provider-owned request sequence", got)
218 }
219}
220
221func TestAgentRetriesUnexpectedEOFStreamError(t *testing.T) {
222 client := &retryFakeClient{firstErr: fmt.Errorf("read SSE: %w", io.ErrUnexpectedEOF)}
223 a := NewAgent(client, "fake-model", "system", Registry{})
224 a.RetryBaseDelay = time.Millisecond
225
226 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
227 t.Fatalf("Prompt returned %v", err)
228 }
229 if got := atomic.LoadInt32(&client.calls); got != 2 {
230 t.Fatalf("Stream calls = %d; want 2", got)
231 }
232}
233
234func TestAgentEmitsRetryLifecycleEvents(t *testing.T) {
235 client := &retryFakeClient{}
236 a := NewAgent(client, "fake-model", "system", Registry{})
237 a.RetryBaseDelay = time.Millisecond
238
239 var lifecycle []string
240 if err := a.Prompt(context.Background(), "hello", nil, func(ev AgentEvent) {
241 switch e := ev.(type) {
242 case EvRequestStarted:
243 lifecycle = append(lifecycle, fmt.Sprintf("request:%s:%d/%d", e.Scope, e.Attempt, e.MaxAttempts))
244 case EvRetryScheduled:
245 lifecycle = append(lifecycle, fmt.Sprintf("retry:%s:%d/%d", e.Scope, e.Attempt, e.MaxAttempts))
246 }
247 }); err != nil {
248 t.Fatalf("Prompt returned %v", err)
249 }
250
251 want := []string{
252 "request:agent:1/6",
253 "request:provider:1/1",
254 "retry:agent:2/6",
255 "request:agent:2/6",
256 "request:provider:1/1",
257 }
258 if strings.Join(lifecycle, ",") != strings.Join(want, ",") {
259 t.Fatalf("lifecycle = %v, want %v", lifecycle, want)
260 }
261}
262
263func TestAgentReportsSanitizedRetryLifecycle(t *testing.T) {
264 client := &retryFakeClient{}
265 a := NewAgent(client, "fake-model", "system", Registry{})
266 a.RetryBaseDelay = time.Millisecond
267
268 var records []RetryLifecycleRecord
269 a.OnRetryLifecycle = func(record RetryLifecycleRecord) {
270 records = append(records, record)
271 }
272 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
273 t.Fatalf("Prompt returned %v", err)
274 }
275
276 want := []RetryLifecycleRecord{
277 {
278 Event: RetryLifecycleRequestFailed, Scope: RetryScopeAgent,
279 Attempt: 1, MaxAttempts: 6, Reason: RetryReasonOverload,
280 },
281 {
282 Event: RetryLifecycleRetryScheduled, Scope: RetryScopeAgent,
283 Attempt: 2, MaxAttempts: 6, Reason: RetryReasonOverload, DelayMS: 1,
284 },
285 }
286 if !reflect.DeepEqual(records, want) {
287 t.Fatalf("retry lifecycle = %#v, want %#v", records, want)
288 }
289}
290
291func TestRequestLifecycleSinkForwardsProviderFailureReason(t *testing.T) {
292 var records []RetryLifecycleRecord
293 sink := &requestLifecycleSink{
294 sink: func(AgentEvent) {},
295 observe: func(record RetryLifecycleRecord) { records = append(records, record) },
296 }
297 sink.RequestFailed(1, 3, provider.RequestFailureOverload, false)
298 sink.RetryScheduled(2, 3, 250*time.Millisecond)
299
300 want := []RetryLifecycleRecord{
301 {
302 Event: RetryLifecycleRequestFailed, Scope: RetryScopeProvider,
303 Attempt: 1, MaxAttempts: 3, Reason: RetryReasonOverload,
304 },
305 {
306 Event: RetryLifecycleRetryScheduled, Scope: RetryScopeProvider,
307 Attempt: 2, MaxAttempts: 3, Reason: RetryReasonOverload, DelayMS: 250,
308 },
309 }
310 if !reflect.DeepEqual(records, want) {
311 t.Fatalf("provider retry lifecycle = %#v, want %#v", records, want)
312 }
313}
314
315func TestAgentReportsTerminalRetryFailure(t *testing.T) {
316 client := &retryFakeClient{}
317 a := NewAgent(client, "fake-model", "system", Registry{})
318 a.MaxRetries = 0
319
320 var records []RetryLifecycleRecord
321 a.OnRetryLifecycle = func(record RetryLifecycleRecord) {
322 records = append(records, record)
323 }
324 if err := a.Prompt(context.Background(), "hello", nil, nil); err == nil {
325 t.Fatal("Prompt succeeded; want terminal provider error")
326 }
327 if len(records) != 1 {
328 t.Fatalf("retry lifecycle = %#v, want one terminal record", records)
329 }
330 record := records[0]
331 if record.Event != RetryLifecycleRequestFailed || record.Scope != RetryScopeAgent ||
332 record.Attempt != 1 || record.MaxAttempts != 1 || record.Reason != RetryReasonOverload || !record.Terminal {
333 t.Fatalf("terminal retry record = %#v", record)
334 }
335}
336
337// codexRetryFakeClient reproduces a transient OpenAI Codex backend
338// failure on the first call and succeeds afterwards.
339type codexRetryFakeClient struct {
340 calls int32
341 firstErr string
342}
343
344func (c *codexRetryFakeClient) Name() string { return "openai-codex" }
345
346func (c *codexRetryFakeClient) Stream(ctx context.Context, req provider.Request) (<-chan provider.Event, error) {
347 call := atomic.AddInt32(&c.calls, 1)
348 out := make(chan provider.Event, 4)
349 go func() {
350 defer close(out)
351 out <- provider.EventStart{Provider: "openai-codex", Model: req.Model}
352 if call == 1 {
353 out <- provider.EventDone{Stop: provider.StopError, Err: fmt.Errorf("%s", c.firstErr)}
354 return
355 }
356 out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{
357 Role: provider.RoleAssistant,
358 Content: []provider.Content{provider.TextBlock{Text: "ok"}},
359 }}
360 }()
361 return out, nil
362}
363
364func TestAgentRetriesCodexProcessingError(t *testing.T) {
365 cases := []struct {
366 name string
367 err string
368 }{
369 {
370 name: "processing error",
371 err: "codex error: An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID 60c8ebbd-20bd-42e4-b756-6e844041cfc0 in your message.",
372 },
373 {
374 name: "servers overloaded",
375 err: "codex error: Our servers are currently overloaded. Please try again later.",
376 },
377 {
378 name: "try again later only",
379 err: "codex error: Something went wrong. Please try again later.",
380 },
381 }
382 for _, tc := range cases {
383 t.Run(tc.name, func(t *testing.T) {
384 client := &codexRetryFakeClient{firstErr: tc.err}
385 a := NewAgent(client, "gpt-5.6-sol", "system", Registry{})
386 a.RetryBaseDelay = time.Millisecond
387
388 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
389 t.Fatalf("Prompt returned %v; want retry to succeed", err)
390 }
391 if got := atomic.LoadInt32(&client.calls); got != 2 {
392 t.Fatalf("Stream calls = %d; want 2", got)
393 }
394 msgs := a.Messages()
395 if len(msgs) != 2 || extractText(msgs[1]) != "ok" {
396 t.Fatalf("messages = %v; want user + ok assistant", msgs)
397 }
398 })
399 }
400}
401
402// TestCanRetryErrorCapacityMessages pins the classification of Codex
403// capacity wording and makes sure quota/usage limits stay terminal even
404// when they carry "try again later" style advice.
405func TestCanRetryErrorCapacityMessages(t *testing.T) {
406 a := NewAgent(nil, "gpt-5.6-sol", "system", Registry{})
407 cases := []struct {
408 msg string
409 want bool
410 }{
411 {"codex error: Our servers are currently overloaded. Please try again later.", true},
412 {"codex error: Our servers are busy right now.", true},
413 {"codex error: Please try again later.", true},
414 {"codex error: You have hit your monthly usage limit. Try again later.", false},
415 {"codex error: quota exceeded, try again later", false},
416 {"codex error: unsupported parameter: reasoning", false},
417 }
418 for _, tc := range cases {
419 if got := a.canRetryError(errors.New(tc.msg), 0); got != tc.want {
420 t.Errorf("canRetryError(%q) = %v; want %v", tc.msg, got, tc.want)
421 }
422 }
423}
424
425type partialRetryFakeClient struct {
426 calls int32
427}
428
429func (c *partialRetryFakeClient) Name() string { return "partial-retry-fake" }
430
431func (c *partialRetryFakeClient) Stream(ctx context.Context, req provider.Request) (<-chan provider.Event, error) {
432 call := atomic.AddInt32(&c.calls, 1)
433 out := make(chan provider.Event, 4)
434 go func() {
435 defer close(out)
436 out <- provider.EventStart{Provider: "partial-retry-fake", Model: req.Model}
437 if call == 1 {
438 out <- provider.EventTextDelta{Delta: "partial"}
439 out <- provider.EventDone{Stop: provider.StopError, Err: fmt.Errorf("provider returned error: 503"), Message: provider.Message{
440 Role: provider.RoleAssistant,
441 Content: []provider.Content{provider.TextBlock{Text: "partial"}},
442 }}
443 return
444 }
445 out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{
446 Role: provider.RoleAssistant,
447 Content: []provider.Content{provider.TextBlock{Text: "recovered"}},
448 }}
449 }()
450 return out, nil
451}
452
453func TestAgentDropsPartialAssistantBeforeRetry(t *testing.T) {
454 client := &partialRetryFakeClient{}
455 a := NewAgent(client, "fake-model", "system", Registry{})
456 a.RetryBaseDelay = time.Millisecond
457
458 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
459 t.Fatalf("Prompt returned %v", err)
460 }
461 msgs := a.Messages()
462 if len(msgs) != 2 {
463 t.Fatalf("message count = %d; want user + recovered assistant", len(msgs))
464 }
465 if got := extractText(msgs[1]); got != "recovered" {
466 t.Fatalf("final assistant text = %q; want recovered", got)
467 }
468}
469
470// captureClient records the last Request it received so tests can
471// assert what the agent put on the wire.
472type captureClient struct {
473 lastReq provider.Request
474}
475
476func (c *captureClient) Name() string { return "capture" }
477
478func (c *captureClient) Stream(ctx context.Context, req provider.Request) (<-chan provider.Event, error) {
479 c.lastReq = req
480 out := make(chan provider.Event, 3)
481 go func() {
482 defer close(out)
483 out <- provider.EventStart{Provider: "capture", Model: req.Model}
484 out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{
485 Role: provider.RoleAssistant,
486 Content: []provider.Content{provider.TextBlock{Text: "ok"}},
487 }}
488 }()
489 return out, nil
490}
491
492func TestAgentPersistsTurnContextAsDeveloperHistory(t *testing.T) {
493 client := &captureClient{}
494 a := NewAgent(client, "fake-model", "system", Registry{})
495 a.BeforeTurnContext = func(context.Context, int) (bool, string, string) {
496 return true, "", "current phase: parse files"
497 }
498 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
499 t.Fatalf("Prompt returned %v", err)
500 }
501 if strings.Contains(client.lastReq.System, "current phase: parse files") {
502 t.Fatalf("stable request system prompt = %q; dynamic context must not rewrite it", client.lastReq.System)
503 }
504 if len(client.lastReq.Messages) != 2 || client.lastReq.Messages[0].Role != provider.RoleDeveloper || !strings.Contains(extractText(client.lastReq.Messages[0]), "current phase: parse files") {
505 t.Fatalf("request context history = %#v, want developer context before user task", client.lastReq.Messages)
506 }
507 if len(a.Messages()) != 3 || a.Messages()[0].Role != provider.RoleDeveloper {
508 t.Fatalf("persisted transcript = %#v, want developer context", a.Messages())
509 }
510}
511
512func TestBoundedTurnContextPreservesUTF8AndLimit(t *testing.T) {
513 contextText := strings.Repeat("界", maxTurnContextBytes)
514 got := boundedTurnContext(contextText)
515 if len(got) > maxTurnContextBytes {
516 t.Fatalf("bounded context bytes = %d, want <= %d", len(got), maxTurnContextBytes)
517 }
518 if !utf8.ValidString(got) {
519 t.Fatal("bounded context is not valid UTF-8")
520 }
521 if !strings.HasSuffix(got, turnContextTruncatedMarker) {
522 tail := got
523 if len(tail) > 64 {
524 tail = tail[len(tail)-64:]
525 }
526 t.Fatalf("bounded context missing truncation marker: %q", tail)
527 }
528}
529
530func TestAgentPropagatesMaxTokens(t *testing.T) {
531 client := &captureClient{}
532 a := NewAgent(client, "fake-model", "system", Registry{})
533 a.MaxTokens = 64000
534
535 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
536 t.Fatalf("Prompt returned %v", err)
537 }
538 if client.lastReq.MaxTokens != 64000 {
539 t.Fatalf("request MaxTokens = %d; want 64000 (Agent.MaxTokens not propagated)", client.lastReq.MaxTokens)
540 }
541}
542
543func TestAgentPropagatesTemperature(t *testing.T) {
544 client := &captureClient{}
545 a := NewAgent(client, "fake-model", "system", Registry{})
546 temp := float32(0)
547 a.Temperature = &temp
548
549 if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil {
550 t.Fatalf("Prompt returned %v", err)
551 }
552 if client.lastReq.Temperature == nil || *client.lastReq.Temperature != temp {
553 t.Fatalf("request Temperature = %v; want %v", client.lastReq.Temperature, temp)
554 }
555}
556
557type cancelledRetryClient struct {
558 cancel context.CancelFunc
559}
560
561func (c *cancelledRetryClient) Name() string { return "cancelled-retry" }
562
563func (c *cancelledRetryClient) Stream(_ context.Context, _ provider.Request) (<-chan provider.Event, error) {
564 out := make(chan provider.Event, 1)
565 c.cancel()
566 out <- provider.EventDone{Stop: provider.StopError, Err: errors.New("provider overloaded")}
567 close(out)
568 return out, nil
569}
570
571func TestAgentDoesNotReportRetryAfterCancellation(t *testing.T) {
572 ctx, cancel := context.WithCancel(context.Background())
573 client := &cancelledRetryClient{cancel: cancel}
574 agent := NewAgent(client, "fake-model", "system", Registry{})
575 agent.MaxRetries = 1
576
577 var retries []EvRetryScheduled
578 var lifecycle []RetryLifecycleRecord
579 agent.OnRetryLifecycle = func(record RetryLifecycleRecord) {
580 lifecycle = append(lifecycle, record)
581 }
582 err := agent.Prompt(ctx, "hello", nil, func(event AgentEvent) {
583 if retry, ok := event.(EvRetryScheduled); ok {
584 retries = append(retries, retry)
585 }
586 })
587 if !errors.Is(err, context.Canceled) {
588 t.Fatalf("Prompt error = %v, want context.Canceled", err)
589 }
590 if len(retries) != 0 {
591 t.Fatalf("retry events = %#v, want none after cancellation", retries)
592 }
593 if len(lifecycle) != 1 || lifecycle[0].Event != RetryLifecycleRequestFailed || !lifecycle[0].Terminal {
594 t.Fatalf("retry lifecycle = %#v, want one terminal failure after cancellation", lifecycle)
595 }
596}