add model rescue picker and silent transient retries

commit: 0a85be3c335a18ceece037fc743c44164dd1f109
author: patriceckhart · 5mo ago · 2026-05-06 18:40 UTC
parent: 0f332fa
signature: unsigned
refs: tag: v0.1.61

9 files · +853 -53

M README.md +12 -0


@@ -262,6 +262,18 @@ Every interactive or print/json run (unless `--no-session`) writes a JSONL trans
262	262
263	263		The context meter in the status line uses the model's advertised context window to show how much of it your last turn consumed.
264	264
	265	+	### Model fallback (rescue)
	266	+
	267	+	When a turn fails because of a recoverable provider error — expired token (`401`), permission denied (`403`), rate limit (`429`), provider outage (`502`/`503`/`504`), or a transient network failure — zot opens a rescue picker over the chat instead of just painting a red banner.
	268	+
	269	+	The picker is the same vertical list / fuzzy filter UI as `/model`, but it only shows models from providers you're currently logged in to (env vars, `auth.json`, Kimi CLI fallback, ollama). The failed model is excluded. Press `↑`/`↓` to choose, `enter` to retry the same prompt on the new model, `esc` to dismiss.
	270	+
	271	+	Before the actual provider request fires, the OpenAI / Anthropic / Kimi / OpenAI-Codex clients also do up to two silent retries with short backoff (250ms, 750ms) on `502`/`503`/`504` and connection-reset / EOF-before-headers errors. Most edge-proxy blips disappear without you ever seeing the rescue picker.
	272	+
	273	+	A rescue retry always drops launch-time `--api-key` and `--base-url` before rebuilding the agent. Those overrides are usually the reason the rescue triggered (bad key, typo'd base URL, corporate gateway only valid for the originally-picked provider), so the retry re-resolves credentials from env vars / `auth.json` / provider defaults instead. The status bar tags the swap as `rescue retry: switched to <provider> / <model> (ignored --api-key / --base-url)` so the behavior is visible. Use `/model` if you want overrides to stick.
	274	+
	275	+	No configuration is required — the candidate list is built dynamically from your active credentials. Bad-request / context-length / serialization errors are NOT routed to the rescue picker, because switching models won't fix them; those still surface as a normal error.
	276	+
265	277		### Custom models
266	278
267	279		Place a `models.json` in `$ZOT_HOME` (macOS: `~/Library/Application Support/zot/`, Linux: `~/.local/state/zot/`) to add models that aren't in the baked-in catalog or to override existing entries:


@@ -262,6 +262,18 @@ Every interactive or print/json run (unless `--no-session`) writes a JSONL trans
262		262
263	The context meter in the status line uses the model's advertised context window to show how much of it your last turn consumed.	263		The context meter in the status line uses the model's advertised context window to show how much of it your last turn consumed.
264		264
		265	+	### Model fallback (rescue)
		266	+
		267	+	When a turn fails because of a recoverable provider error — expired token (`401`), permission denied (`403`), rate limit (`429`), provider outage (`502`/`503`/`504`), or a transient network failure — zot opens a rescue picker over the chat instead of just painting a red banner.
		268	+
		269	+	The picker is the same vertical list / fuzzy filter UI as `/model`, but it only shows models from providers you're currently logged in to (env vars, `auth.json`, Kimi CLI fallback, ollama). The failed model is excluded. Press `↑`/`↓` to choose, `enter` to retry the same prompt on the new model, `esc` to dismiss.
		270	+
		271	+	Before the actual provider request fires, the OpenAI / Anthropic / Kimi / OpenAI-Codex clients also do up to two silent retries with short backoff (250ms, 750ms) on `502`/`503`/`504` and connection-reset / EOF-before-headers errors. Most edge-proxy blips disappear without you ever seeing the rescue picker.
		272	+
		273	+	A rescue retry always drops launch-time `--api-key` and `--base-url` before rebuilding the agent. Those overrides are usually the reason the rescue triggered (bad key, typo'd base URL, corporate gateway only valid for the originally-picked provider), so the retry re-resolves credentials from env vars / `auth.json` / provider defaults instead. The status bar tags the swap as `rescue retry: switched to <provider> / <model> (ignored --api-key / --base-url)` so the behavior is visible. Use `/model` if you want overrides to stick.
		274	+
		275	+	No configuration is required — the candidate list is built dynamically from your active credentials. Bad-request / context-length / serialization errors are NOT routed to the rescue picker, because switching models won't fix them; those still surface as a normal error.
		276	+
265	### Custom models	277		### Custom models
266		278
267	Place a `models.json` in `$ZOT_HOME` (macOS: `~/Library/Application Support/zot/`, Linux: `~/.local/state/zot/`) to add models that aren't in the baked-in catalog or to override existing entries:	279		Place a `models.json` in `$ZOT_HOME` (macOS: `~/Library/Application Support/zot/`, Linux: `~/.local/state/zot/`) to add models that aren't in the baked-in catalog or to override existing entries:

M internal/agent/cli.go +32 -0


@@ -475,6 +475,32 @@ func runInteractive(ctx context.Context, args Args, version string) error {
475	475		return wireAgentExt(resolved.NewAgent()), resolved.Provider, resolved.Model, nil
476	476		}
477	477
	478	+	// Rebuild agent for the rescue picker after a recoverable failure.
	479	+	// Unlike buildAgentFor, this drops launch-time --api-key and
	480	+	// --base-url overrides because those are typically the cause of the
	481	+	// rescue (a bad key, a typo'd base URL, or a corporate gateway that
	482	+	// only the originally-picked provider needed). Re-resolving without
	483	+	// them lets the rescue retry use env vars / auth.json / provider
	484	+	// defaults the way zot would have without the overrides.
	485	+	buildAgentForRescue := func(providerOverride, modelOverride string) (*core.Agent, string, string, error) {
	486	+	next := args
	487	+	next.APIKey = ""
	488	+	next.BaseURL = ""
	489	+	if providerOverride != "" {
	490	+	next.Provider = providerOverride
	491	+	}
	492	+	if modelOverride != "" {
	493	+	next.Model = modelOverride
	494	+	}
	495	+	resolved, err := Resolve(next, true)
	496	+	if err != nil {
	497	+	return nil, "", "", err
	498	+	}
	499	+	resolved.UseSandbox(sharedSandbox)
	500	+	resolved.MergeExtensionTools(extToolAdapter)
	501	+	return wireAgentExt(resolved.NewAgent()), resolved.Provider, resolved.Model, nil
	502	+	}
	503	+
478	504		var ag *core.Agent
479	505		if r.HasCredential() {
480	506		ag = wireAgentExt(r.NewAgent())

@@ -573,6 +599,11 @@ func runInteractive(ctx context.Context, args Args, version string) error {
573	599		a, p, m, err := baseBuildAgentFor(providerOverride, modelOverride)
574	600		return wireAgentPersist(a), p, m, err
575	601		}
	602	+	baseBuildAgentForRescue := buildAgentForRescue
	603	+	buildAgentForRescue = func(providerOverride, modelOverride string) (*core.Agent, string, string, error) {
	604	+	a, p, m, err := baseBuildAgentForRescue(providerOverride, modelOverride)
	605	+	return wireAgentPersist(a), p, m, err
	606	+	}
576	607
577	608		// loadSession replaces the current session with the one at path and
578	609		// hands its messages to the agent. Used by the /sessions picker.

@@ -687,6 +718,7 @@ func runInteractive(ctx context.Context, args Args, version string) error {
687	718		BuildAgent: buildAgent,
688	719		SetKimiCLIFallbackDisabled: SetKimiCLIFallbackDisabled,
689	720		BuildAgentFor: buildAgentFor,
	721	+	BuildAgentForRescue: buildAgentForRescue,
690	722		LoggedInProviders: func() []string {
691	723		var out []string
692	724		for _, p := range []string{"anthropic", "openai", "kimi"} {


@@ -475,6 +475,32 @@ func runInteractive(ctx context.Context, args Args, version string) error {
475	return wireAgentExt(resolved.NewAgent()), resolved.Provider, resolved.Model, nil	475		return wireAgentExt(resolved.NewAgent()), resolved.Provider, resolved.Model, nil
476	}	476		}
477		477
		478	+	// Rebuild agent for the rescue picker after a recoverable failure.
		479	+	// Unlike buildAgentFor, this drops launch-time --api-key and
		480	+	// --base-url overrides because those are typically the cause of the
		481	+	// rescue (a bad key, a typo'd base URL, or a corporate gateway that
		482	+	// only the originally-picked provider needed). Re-resolving without
		483	+	// them lets the rescue retry use env vars / auth.json / provider
		484	+	// defaults the way zot would have without the overrides.
		485	+	buildAgentForRescue := func(providerOverride, modelOverride string) (*core.Agent, string, string, error) {
		486	+	next := args
		487	+	next.APIKey = ""
		488	+	next.BaseURL = ""
		489	+	if providerOverride != "" {
		490	+	next.Provider = providerOverride
		491	+	}
		492	+	if modelOverride != "" {
		493	+	next.Model = modelOverride
		494	+	}
		495	+	resolved, err := Resolve(next, true)
		496	+	if err != nil {
		497	+	return nil, "", "", err
		498	+	}
		499	+	resolved.UseSandbox(sharedSandbox)
		500	+	resolved.MergeExtensionTools(extToolAdapter)
		501	+	return wireAgentExt(resolved.NewAgent()), resolved.Provider, resolved.Model, nil
		502	+	}
		503	+
478	var ag *core.Agent	504		var ag *core.Agent
479	if r.HasCredential() {	505		if r.HasCredential() {
480	ag = wireAgentExt(r.NewAgent())	506		ag = wireAgentExt(r.NewAgent())

@@ -573,6 +599,11 @@ func runInteractive(ctx context.Context, args Args, version string) error {
573	a, p, m, err := baseBuildAgentFor(providerOverride, modelOverride)	599		a, p, m, err := baseBuildAgentFor(providerOverride, modelOverride)
574	return wireAgentPersist(a), p, m, err	600		return wireAgentPersist(a), p, m, err
575	}	601		}
		602	+	baseBuildAgentForRescue := buildAgentForRescue
		603	+	buildAgentForRescue = func(providerOverride, modelOverride string) (*core.Agent, string, string, error) {
		604	+	a, p, m, err := baseBuildAgentForRescue(providerOverride, modelOverride)
		605	+	return wireAgentPersist(a), p, m, err
		606	+	}
576		607
577	// loadSession replaces the current session with the one at path and	608		// loadSession replaces the current session with the one at path and
578	// hands its messages to the agent. Used by the /sessions picker.	609		// hands its messages to the agent. Used by the /sessions picker.

@@ -687,6 +718,7 @@ func runInteractive(ctx context.Context, args Args, version string) error {
687	BuildAgent: buildAgent,	718		BuildAgent: buildAgent,
688	SetKimiCLIFallbackDisabled: SetKimiCLIFallbackDisabled,	719		SetKimiCLIFallbackDisabled: SetKimiCLIFallbackDisabled,
689	BuildAgentFor: buildAgentFor,	720		BuildAgentFor: buildAgentFor,
		721	+	BuildAgentForRescue: buildAgentForRescue,
690	LoggedInProviders: func() []string {	722		LoggedInProviders: func() []string {
691	var out []string	723		var out []string
692	for _, p := range []string{"anthropic", "openai", "kimi"} {	724		for _, p := range []string{"anthropic", "openai", "kimi"} {

M internal/agent/modes/interactive.go +152 -9


@@ -63,6 +63,15 @@ type InteractiveConfig struct {
63	63		// If providerOverride is empty, the current provider is kept.
64	64		BuildAgentFor func(providerOverride, modelOverride string) (*core.Agent, string, string, error)
65	65
	66	+	// BuildAgentForRescue rebuilds the agent for the rescue picker that
	67	+	// opens after a recoverable provider failure. Unlike BuildAgentFor,
	68	+	// this builder drops launch-time --api-key and --base-url overrides
	69	+	// because those are usually the reason rescue triggered. Re-resolves
	70	+	// credentials from env vars / auth.json / provider defaults so the
	71	+	// retry has a real chance of succeeding. Falls back to BuildAgentFor
	72	+	// when nil so embedders that don't wire it keep working.
	73	+	BuildAgentForRescue func(providerOverride, modelOverride string) (*core.Agent, string, string, error)
	74	+
66	75		// LoggedInProviders returns the list of provider names that
67	76		// currently have credentials. Used by /model to filter the
68	77		// picker to only show reachable models.

@@ -268,6 +277,7 @@ type Interactive struct {
268	277
269	278		dialog *loginDialog
270	279		modelDialog *modelDialog
	280	+	rescueDialog *rescueDialog
271	281		sessionDialog *sessionDialog
272	282		jumpDialog *jumpDialog
273	283		btwDialog *btwDialog

@@ -323,6 +333,13 @@ type Interactive struct {
323	333		// on a background goroutine. Keeping this off the input goroutine
324	334		// lets ctrl+c/exit remain responsive for very large JSONL sessions.
325	335		sessionLoading bool
	336	+
	337	+	// pendingRescuePrompt / pendingRescueImages stash the prompt and
	338	+	// images that should be re-run after the user picks a model in
	339	+	// the rescue dialog. Cleared once applyRescueSelection consumes
	340	+	// them (or when the dialog is dismissed via esc).
	341	+	pendingRescuePrompt string
	342	+	pendingRescueImages []provider.ImageBlock
326	343		}
327	344
328	345		// welcomeVersionDuration is how long the welcome banner shows the

@@ -347,6 +364,7 @@ func NewInteractive(cfg InteractiveConfig) *Interactive {
347	364		dirty: make(chan struct{}, 8),
348	365		dialog: newLoginDialog(),
349	366		modelDialog: newModelDialog(),
	367	+	rescueDialog: newRescueDialog(),
350	368		sessionDialog: newSessionDialog(),
351	369		jumpDialog: newJumpDialog(),
352	370		btwDialog: newBtwDialog(),

@@ -852,6 +870,8 @@ func (i *Interactive) redraw() {
852	870		dialog = i.dialog.Render(i.cfg.Theme, cols)
853	871		case i.modelDialog.Active():
854	872		dialog = i.modelDialog.Render(i.cfg.Theme, cols)
	873	+	case i.rescueDialog.Active():
	874	+	dialog = i.rescueDialog.Render(i.cfg.Theme, cols)
855	875		case i.sessionDialog.Active():
856	876		// Reserve rows for the editor (~3), status line (1-2),
857	877		// dialog chrome (header + hint + rule + indicators, ~5),

@@ -1436,6 +1456,19 @@ func (i *Interactive) handleKey(ctx context.Context, k tui.Key) (done bool) {
1436	1456		}
1437	1457		return false
1438	1458		}
	1459	+	if i.rescueDialog.Active() {
	1460	+	if k.Kind == tui.KeyCtrlC {
	1461	+	i.rescueDialog.Close()
	1462	+	i.invalidate()
	1463	+	return false
	1464	+	}
	1465	+	act := i.rescueDialog.HandleKey(k)
	1466	+	if act.Select {
	1467	+	i.applyRescueSelection(act.Provider, act.Model, act.Prompt)
	1468	+	}
	1469	+	i.invalidate()
	1470	+	return false
	1471	+	}
1439	1472		if i.sessionDialog.Active() {
1440	1473		if k.Kind == tui.KeyCtrlC {
1441	1474		i.sessionDialog.Close()

@@ -2835,6 +2868,25 @@ func (i *Interactive) scrollToLastTurn(msgs []provider.Message) {
2835	2868		}
2836	2869
2837	2870		func (i *Interactive) applyModelSelection(prov, model string) {
	2871	+	i.swapModel(prov, model, i.cfg.BuildAgentFor, false)
	2872	+	}
	2873	+
	2874	+	// applyRescueModelSelection is like applyModelSelection but routes
	2875	+	// through BuildAgentForRescue so launch-time --api-key / --base-url
	2876	+	// overrides are dropped before the new agent is built. Falls back to
	2877	+	// the regular builder when the host doesn't wire a rescue builder.
	2878	+	func (i *Interactive) applyRescueModelSelection(prov, model string) {
	2879	+	builder := i.cfg.BuildAgentForRescue
	2880	+	if builder == nil {
	2881	+	builder = i.cfg.BuildAgentFor
	2882	+	}
	2883	+	i.swapModel(prov, model, builder, true)
	2884	+	}
	2885	+
	2886	+	// swapModel applies a /model selection (or a rescue selection) using
	2887	+	// the supplied builder. rescue=true tags the success message so the
	2888	+	// user can see that launch-time overrides were ignored.
	2889	+	func (i Interactive) swapModel(prov, model string, builder func(string, string) (core.Agent, string, string, error), rescue bool) {
2838	2890		if model == "" {
2839	2891		return
2840	2892		}

@@ -2845,8 +2897,11 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2845	2897		i.mu.Unlock()
2846	2898		return
2847	2899		}
2848		-	// Same provider? Just swap the model on the existing agent.
2849		-	if i.agent != nil && m.Provider == i.cfg.Provider {
	2900	+	// Same provider AND not a rescue retry: just swap the model on
	2901	+	// the existing agent — no rebuild needed because the underlying
	2902	+	// client is reusable. Rescue retries always rebuild so a stale
	2903	+	// auth header / base URL can't carry over.
	2904	+	if !rescue && i.agent != nil && m.Provider == i.cfg.Provider {
2850	2905		i.mu.Lock()
2851	2906		i.cfg.Model = m.ID
2852	2907		i.agent.Model = m.ID

@@ -2858,8 +2913,7 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2858	2913		}
2859	2914		return
2860	2915		}
2861		-	// Different provider: rebuild agent (needs credentials for target provider).
2862		-	if i.cfg.BuildAgentFor == nil {
	2916	+	if builder == nil {
2863	2917		i.mu.Lock()
2864	2918		i.statusErr = "cannot switch provider: no builder configured"
2865	2919		i.mu.Unlock()

@@ -2876,7 +2930,7 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2876	2930		carryCost = i.agent.Cost()
2877	2931		}
2878	2932
2879		-	ag, p, md, err := i.cfg.BuildAgentFor(m.Provider, m.ID)
	2933	+	ag, p, md, err := builder(m.Provider, m.ID)
2880	2934		if err != nil {
2881	2935		i.mu.Lock()
2882	2936		i.statusErr = err.Error()

@@ -2899,7 +2953,11 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2899	2953		i.agent = ag
2900	2954		i.cfg.Provider = p
2901	2955		i.cfg.Model = md
2902		-	i.statusOK = "switched to " + p + " / " + md
	2956	+	if rescue {
	2957	+	i.statusOK = "rescue retry: switched to " + p + " / " + md + " (ignored --api-key / --base-url)"
	2958	+	} else {
	2959	+	i.statusOK = "switched to " + p + " / " + md
	2960	+	}
2903	2961		i.statusErr = ""
2904	2962		// Render cache keys are width+content based, so the new agent's
2905	2963		// identical messages will reuse the existing entries. Nothing

@@ -3113,6 +3171,33 @@ func (i *Interactive) startTurnWithImages(parent context.Context, prompt string,
3113	3171		if err != nil && ctx.Err() == nil {
3114	3172		i.statusErr = err.Error()
3115	3173		}
	3174	+	// Decide whether to offer a model rescue picker for recoverable
	3175	+	// provider failures (auth/rate/temporary). The picker opens after
	3176	+	// the mutex is released so it can take its own locks freely.
	3177	+	var (
	3178	+	offer bool
	3179	+	rescueWhy string
	3180	+	rescueImgs []provider.ImageBlock
	3181	+	rescueModel string
	3182	+	rescueProv string
	3183	+	rescueFprov string
	3184	+	)
	3185	+	if err != nil && ctx.Err() == nil {
	3186	+	if ok, reason := classifyRescueError(err); ok {
	3187	+	offer = true
	3188	+	rescueWhy = reason
	3189	+	rescueImgs = images
	3190	+	rescueModel = i.cfg.Model
	3191	+	rescueProv = i.cfg.Provider
	3192	+	rescueFprov = extractFailedProvider(err)
	3193	+	if rescueFprov == "" {
	3194	+	rescueFprov = i.cfg.Provider
	3195	+	}
	3196	+	// Suppress the red banner — the rescue dialog already
	3197	+	// surfaces the failure.
	3198	+	i.statusErr = ""
	3199	+	}
	3200	+	}
3116	3201		// Detect HTTP 413 "payload too large" responses. The provider
3117	3202		// rejected the request because the request body exceeded its
3118	3203		// per-request limit. Token-based auto-compact can miss this

@@ -3162,6 +3247,8 @@ func (i *Interactive) startTurnWithImages(parent context.Context, prompt string,
3162	3247		switch {
3163	3248		case hasNext:
3164	3249		i.startTurn(parent, next)
	3250	+	case offer:
	3251	+	i.openRescueDialog(rescueProv, rescueFprov, rescueModel, rescueWhy, prompt, rescueImgs)
3165	3252		case payloadTooLarge:
3166	3253		i.runCompact(parent, true)
3167	3254		case shouldAutoCompact:

@@ -3170,6 +3257,56 @@ func (i *Interactive) startTurnWithImages(parent context.Context, prompt string,
3170	3257		}()
3171	3258		}
3172	3259
	3260	+	// openRescueDialog surfaces the rescue model picker after a recoverable
	3261	+	// provider failure. The pending prompt + images are stashed on the
	3262	+	// Interactive so a later applyRescueSelection can re-run the same turn
	3263	+	// against the freshly-picked model. activeProvider/failedProvider are
	3264	+	// usually the same, but some clients embed different prefixes in their
	3265	+	// errors than the configured provider id, so we accept both.
	3266	+	func (i *Interactive) openRescueDialog(activeProvider, failedProvider, failedModel, reason, prompt string, images []provider.ImageBlock) {
	3267	+	if i.rescueDialog == nil {
	3268	+	return
	3269	+	}
	3270	+	loggedIn := []string{}
	3271	+	if i.cfg.LoggedInProviders != nil {
	3272	+	loggedIn = i.cfg.LoggedInProviders()
	3273	+	}
	3274	+	fprov := failedProvider
	3275	+	if fprov == "" {
	3276	+	fprov = activeProvider
	3277	+	}
	3278	+	i.mu.Lock()
	3279	+	i.pendingRescuePrompt = prompt
	3280	+	i.pendingRescueImages = images
	3281	+	i.mu.Unlock()
	3282	+	i.rescueDialog.Open(failedModel, loggedIn, fprov, failedModel, reason, prompt)
	3283	+	i.invalidate()
	3284	+	}
	3285	+
	3286	+	// applyRescueSelection switches model (cross-provider if needed) and
	3287	+	// re-runs the same prompt+images that just failed. Mirrors
	3288	+	// applyModelSelection's transcript-carry logic so the user keeps full
	3289	+	// session continuity across the swap.
	3290	+	func (i *Interactive) applyRescueSelection(prov, model, prompt string) {
	3291	+	if model == "" {
	3292	+	return
	3293	+	}
	3294	+	i.applyRescueModelSelection(prov, model)
	3295	+	i.mu.Lock()
	3296	+	images := i.pendingRescueImages
	3297	+	if prompt == "" {
	3298	+	prompt = i.pendingRescuePrompt
	3299	+	}
	3300	+	i.pendingRescuePrompt = ""
	3301	+	i.pendingRescueImages = nil
	3302	+	i.mu.Unlock()
	3303	+	parent := i.runCtx
	3304	+	if parent == nil {
	3305	+	parent = context.Background()
	3306	+	}
	3307	+	i.startTurnWithImages(parent, prompt, images)
	3308	+	}
	3309	+
3173	3310		func stripAutoCompactNotes(notes []string) []string {
3174	3311		if len(notes) == 0 {
3175	3312		return notes

@@ -3365,9 +3502,15 @@ func (i *Interactive) handleEvent(ev core.AgentEvent) {
3365	3502		i.statusOK = "cancelled"
3366	3503		return
3367	3504		}
3368		-	if e.Err != nil && !strings.Contains(e.Err.Error(), "context canceled") {
3369		-	i.statusErr = e.Err.Error()
3370		-	}
	3505	+	// Don't surface mid-loop stream errors as a red banner here.
	3506	+	// EvTurnEnd fires after every step in a multi-step tool loop,
	3507	+	// so a transient 503 / network blip would briefly paint a red
	3508	+	// banner over the still-streaming chat before the agent loop
	3509	+	// either retries or exits. The final error (if any) is set by
	3510	+	// startTurnWithImages once Prompt() returns, and recoverable
	3511	+	// failures are routed to the rescue picker instead — which
	3512	+	// keeps the chat clean while the agent is working.
	3513	+	_ = e.Err
3371	3514		}
3372	3515		}
3373	3516


@@ -63,6 +63,15 @@ type InteractiveConfig struct {
63		// If providerOverride is empty, the current provider is kept.	63		// If providerOverride is empty, the current provider is kept.
64		BuildAgentFor func(providerOverride, modelOverride string) (*core.Agent, string, string, error)	64		BuildAgentFor func(providerOverride, modelOverride string) (*core.Agent, string, string, error)
65			65
			66	+	// BuildAgentForRescue rebuilds the agent for the rescue picker that
			67	+	// opens after a recoverable provider failure. Unlike BuildAgentFor,
			68	+	// this builder drops launch-time --api-key and --base-url overrides
			69	+	// because those are usually the reason rescue triggered. Re-resolves
			70	+	// credentials from env vars / auth.json / provider defaults so the
			71	+	// retry has a real chance of succeeding. Falls back to BuildAgentFor
			72	+	// when nil so embedders that don't wire it keep working.
			73	+	BuildAgentForRescue func(providerOverride, modelOverride string) (*core.Agent, string, string, error)
			74	+
66		// LoggedInProviders returns the list of provider names that	75		// LoggedInProviders returns the list of provider names that
67		// currently have credentials. Used by /model to filter the	76		// currently have credentials. Used by /model to filter the
68		// picker to only show reachable models.	77		// picker to only show reachable models.

@@ -268,6 +277,7 @@ type Interactive struct {
268			277
269		dialog *loginDialog	278		dialog *loginDialog
270		modelDialog *modelDialog	279		modelDialog *modelDialog
			280	+	rescueDialog *rescueDialog
271		sessionDialog *sessionDialog	281		sessionDialog *sessionDialog
272		jumpDialog *jumpDialog	282		jumpDialog *jumpDialog
273		btwDialog *btwDialog	283		btwDialog *btwDialog

@@ -323,6 +333,13 @@ type Interactive struct {
323		// on a background goroutine. Keeping this off the input goroutine	333		// on a background goroutine. Keeping this off the input goroutine
324		// lets ctrl+c/exit remain responsive for very large JSONL sessions.	334		// lets ctrl+c/exit remain responsive for very large JSONL sessions.
325		sessionLoading bool	335		sessionLoading bool
			336	+
			337	+	// pendingRescuePrompt / pendingRescueImages stash the prompt and
			338	+	// images that should be re-run after the user picks a model in
			339	+	// the rescue dialog. Cleared once applyRescueSelection consumes
			340	+	// them (or when the dialog is dismissed via esc).
			341	+	pendingRescuePrompt string
			342	+	pendingRescueImages []provider.ImageBlock
326		}	343		}
327			344
328		// welcomeVersionDuration is how long the welcome banner shows the	345		// welcomeVersionDuration is how long the welcome banner shows the

@@ -347,6 +364,7 @@ func NewInteractive(cfg InteractiveConfig) *Interactive {
347		dirty: make(chan struct{}, 8),	364		dirty: make(chan struct{}, 8),
348		dialog: newLoginDialog(),	365		dialog: newLoginDialog(),
349		modelDialog: newModelDialog(),	366		modelDialog: newModelDialog(),
			367	+	rescueDialog: newRescueDialog(),
350		sessionDialog: newSessionDialog(),	368		sessionDialog: newSessionDialog(),
351		jumpDialog: newJumpDialog(),	369		jumpDialog: newJumpDialog(),
352		btwDialog: newBtwDialog(),	370		btwDialog: newBtwDialog(),

@@ -852,6 +870,8 @@ func (i *Interactive) redraw() {
852		dialog = i.dialog.Render(i.cfg.Theme, cols)	870		dialog = i.dialog.Render(i.cfg.Theme, cols)
853		case i.modelDialog.Active():	871		case i.modelDialog.Active():
854		dialog = i.modelDialog.Render(i.cfg.Theme, cols)	872		dialog = i.modelDialog.Render(i.cfg.Theme, cols)
			873	+	case i.rescueDialog.Active():
			874	+	dialog = i.rescueDialog.Render(i.cfg.Theme, cols)
855		case i.sessionDialog.Active():	875		case i.sessionDialog.Active():
856		// Reserve rows for the editor (~3), status line (1-2),	876		// Reserve rows for the editor (~3), status line (1-2),
857		// dialog chrome (header + hint + rule + indicators, ~5),	877		// dialog chrome (header + hint + rule + indicators, ~5),

@@ -1436,6 +1456,19 @@ func (i *Interactive) handleKey(ctx context.Context, k tui.Key) (done bool) {
1436		}	1456		}
1437		return false	1457		return false
1438		}	1458		}
			1459	+	if i.rescueDialog.Active() {
			1460	+	if k.Kind == tui.KeyCtrlC {
			1461	+	i.rescueDialog.Close()
			1462	+	i.invalidate()
			1463	+	return false
			1464	+	}
			1465	+	act := i.rescueDialog.HandleKey(k)
			1466	+	if act.Select {
			1467	+	i.applyRescueSelection(act.Provider, act.Model, act.Prompt)
			1468	+	}
			1469	+	i.invalidate()
			1470	+	return false
			1471	+	}
1439		if i.sessionDialog.Active() {	1472		if i.sessionDialog.Active() {
1440		if k.Kind == tui.KeyCtrlC {	1473		if k.Kind == tui.KeyCtrlC {
1441		i.sessionDialog.Close()	1474		i.sessionDialog.Close()

@@ -2835,6 +2868,25 @@ func (i *Interactive) scrollToLastTurn(msgs []provider.Message) {
2835		}	2868		}
2836			2869
2837		func (i *Interactive) applyModelSelection(prov, model string) {	2870		func (i *Interactive) applyModelSelection(prov, model string) {
			2871	+	i.swapModel(prov, model, i.cfg.BuildAgentFor, false)
			2872	+	}
			2873	+
			2874	+	// applyRescueModelSelection is like applyModelSelection but routes
			2875	+	// through BuildAgentForRescue so launch-time --api-key / --base-url
			2876	+	// overrides are dropped before the new agent is built. Falls back to
			2877	+	// the regular builder when the host doesn't wire a rescue builder.
			2878	+	func (i *Interactive) applyRescueModelSelection(prov, model string) {
			2879	+	builder := i.cfg.BuildAgentForRescue
			2880	+	if builder == nil {
			2881	+	builder = i.cfg.BuildAgentFor
			2882	+	}
			2883	+	i.swapModel(prov, model, builder, true)
			2884	+	}
			2885	+
			2886	+	// swapModel applies a /model selection (or a rescue selection) using
			2887	+	// the supplied builder. rescue=true tags the success message so the
			2888	+	// user can see that launch-time overrides were ignored.
			2889	+	func (i Interactive) swapModel(prov, model string, builder func(string, string) (core.Agent, string, string, error), rescue bool) {
2838		if model == "" {	2890		if model == "" {
2839		return	2891		return
2840		}	2892		}

@@ -2845,8 +2897,11 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2845		i.mu.Unlock()	2897		i.mu.Unlock()
2846		return	2898		return
2847		}	2899		}
2848	-	// Same provider? Just swap the model on the existing agent.	2900	+	// Same provider AND not a rescue retry: just swap the model on
2849	-	if i.agent != nil && m.Provider == i.cfg.Provider {	2901	+	// the existing agent — no rebuild needed because the underlying
			2902	+	// client is reusable. Rescue retries always rebuild so a stale
			2903	+	// auth header / base URL can't carry over.
			2904	+	if !rescue && i.agent != nil && m.Provider == i.cfg.Provider {
2850		i.mu.Lock()	2905		i.mu.Lock()
2851		i.cfg.Model = m.ID	2906		i.cfg.Model = m.ID
2852		i.agent.Model = m.ID	2907		i.agent.Model = m.ID

@@ -2858,8 +2913,7 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2858		}	2913		}
2859		return	2914		return
2860		}	2915		}
2861	-	// Different provider: rebuild agent (needs credentials for target provider).	2916	+	if builder == nil {
2862	-	if i.cfg.BuildAgentFor == nil {
2863		i.mu.Lock()	2917		i.mu.Lock()
2864		i.statusErr = "cannot switch provider: no builder configured"	2918		i.statusErr = "cannot switch provider: no builder configured"
2865		i.mu.Unlock()	2919		i.mu.Unlock()

@@ -2876,7 +2930,7 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2876		carryCost = i.agent.Cost()	2930		carryCost = i.agent.Cost()
2877		}	2931		}
2878			2932
2879	-	ag, p, md, err := i.cfg.BuildAgentFor(m.Provider, m.ID)	2933	+	ag, p, md, err := builder(m.Provider, m.ID)
2880		if err != nil {	2934		if err != nil {
2881		i.mu.Lock()	2935		i.mu.Lock()
2882		i.statusErr = err.Error()	2936		i.statusErr = err.Error()

@@ -2899,7 +2953,11 @@ func (i *Interactive) applyModelSelection(prov, model string) {
2899		i.agent = ag	2953		i.agent = ag
2900		i.cfg.Provider = p	2954		i.cfg.Provider = p
2901		i.cfg.Model = md	2955		i.cfg.Model = md
2902	-	i.statusOK = "switched to " + p + " / " + md	2956	+	if rescue {
			2957	+	i.statusOK = "rescue retry: switched to " + p + " / " + md + " (ignored --api-key / --base-url)"
			2958	+	} else {
			2959	+	i.statusOK = "switched to " + p + " / " + md
			2960	+	}
2903		i.statusErr = ""	2961		i.statusErr = ""
2904		// Render cache keys are width+content based, so the new agent's	2962		// Render cache keys are width+content based, so the new agent's
2905		// identical messages will reuse the existing entries. Nothing	2963		// identical messages will reuse the existing entries. Nothing

@@ -3113,6 +3171,33 @@ func (i *Interactive) startTurnWithImages(parent context.Context, prompt string,
3113		if err != nil && ctx.Err() == nil {	3171		if err != nil && ctx.Err() == nil {
3114		i.statusErr = err.Error()	3172		i.statusErr = err.Error()
3115		}	3173		}
			3174	+	// Decide whether to offer a model rescue picker for recoverable
			3175	+	// provider failures (auth/rate/temporary). The picker opens after
			3176	+	// the mutex is released so it can take its own locks freely.
			3177	+	var (
			3178	+	offer bool
			3179	+	rescueWhy string
			3180	+	rescueImgs []provider.ImageBlock
			3181	+	rescueModel string
			3182	+	rescueProv string
			3183	+	rescueFprov string
			3184	+	)
			3185	+	if err != nil && ctx.Err() == nil {
			3186	+	if ok, reason := classifyRescueError(err); ok {
			3187	+	offer = true
			3188	+	rescueWhy = reason
			3189	+	rescueImgs = images
			3190	+	rescueModel = i.cfg.Model
			3191	+	rescueProv = i.cfg.Provider
			3192	+	rescueFprov = extractFailedProvider(err)
			3193	+	if rescueFprov == "" {
			3194	+	rescueFprov = i.cfg.Provider
			3195	+	}
			3196	+	// Suppress the red banner — the rescue dialog already
			3197	+	// surfaces the failure.
			3198	+	i.statusErr = ""
			3199	+	}
			3200	+	}
3116		// Detect HTTP 413 "payload too large" responses. The provider	3201		// Detect HTTP 413 "payload too large" responses. The provider
3117		// rejected the request because the request body exceeded its	3202		// rejected the request because the request body exceeded its
3118		// per-request limit. Token-based auto-compact can miss this	3203		// per-request limit. Token-based auto-compact can miss this

@@ -3162,6 +3247,8 @@ func (i *Interactive) startTurnWithImages(parent context.Context, prompt string,
3162		switch {	3247		switch {
3163		case hasNext:	3248		case hasNext:
3164		i.startTurn(parent, next)	3249		i.startTurn(parent, next)
			3250	+	case offer:
			3251	+	i.openRescueDialog(rescueProv, rescueFprov, rescueModel, rescueWhy, prompt, rescueImgs)
3165		case payloadTooLarge:	3252		case payloadTooLarge:
3166		i.runCompact(parent, true)	3253		i.runCompact(parent, true)
3167		case shouldAutoCompact:	3254		case shouldAutoCompact:

@@ -3170,6 +3257,56 @@ func (i *Interactive) startTurnWithImages(parent context.Context, prompt string,
3170		}()	3257		}()
3171		}	3258		}
3172			3259
			3260	+	// openRescueDialog surfaces the rescue model picker after a recoverable
			3261	+	// provider failure. The pending prompt + images are stashed on the
			3262	+	// Interactive so a later applyRescueSelection can re-run the same turn
			3263	+	// against the freshly-picked model. activeProvider/failedProvider are
			3264	+	// usually the same, but some clients embed different prefixes in their
			3265	+	// errors than the configured provider id, so we accept both.
			3266	+	func (i *Interactive) openRescueDialog(activeProvider, failedProvider, failedModel, reason, prompt string, images []provider.ImageBlock) {
			3267	+	if i.rescueDialog == nil {
			3268	+	return
			3269	+	}
			3270	+	loggedIn := []string{}
			3271	+	if i.cfg.LoggedInProviders != nil {
			3272	+	loggedIn = i.cfg.LoggedInProviders()
			3273	+	}
			3274	+	fprov := failedProvider
			3275	+	if fprov == "" {
			3276	+	fprov = activeProvider
			3277	+	}
			3278	+	i.mu.Lock()
			3279	+	i.pendingRescuePrompt = prompt
			3280	+	i.pendingRescueImages = images
			3281	+	i.mu.Unlock()
			3282	+	i.rescueDialog.Open(failedModel, loggedIn, fprov, failedModel, reason, prompt)
			3283	+	i.invalidate()
			3284	+	}
			3285	+
			3286	+	// applyRescueSelection switches model (cross-provider if needed) and
			3287	+	// re-runs the same prompt+images that just failed. Mirrors
			3288	+	// applyModelSelection's transcript-carry logic so the user keeps full
			3289	+	// session continuity across the swap.
			3290	+	func (i *Interactive) applyRescueSelection(prov, model, prompt string) {
			3291	+	if model == "" {
			3292	+	return
			3293	+	}
			3294	+	i.applyRescueModelSelection(prov, model)
			3295	+	i.mu.Lock()
			3296	+	images := i.pendingRescueImages
			3297	+	if prompt == "" {
			3298	+	prompt = i.pendingRescuePrompt
			3299	+	}
			3300	+	i.pendingRescuePrompt = ""
			3301	+	i.pendingRescueImages = nil
			3302	+	i.mu.Unlock()
			3303	+	parent := i.runCtx
			3304	+	if parent == nil {
			3305	+	parent = context.Background()
			3306	+	}
			3307	+	i.startTurnWithImages(parent, prompt, images)
			3308	+	}
			3309	+
3173		func stripAutoCompactNotes(notes []string) []string {	3310		func stripAutoCompactNotes(notes []string) []string {
3174		if len(notes) == 0 {	3311		if len(notes) == 0 {
3175		return notes	3312		return notes

@@ -3365,9 +3502,15 @@ func (i *Interactive) handleEvent(ev core.AgentEvent) {
3365		i.statusOK = "cancelled"	3502		i.statusOK = "cancelled"
3366		return	3503		return
3367		}	3504		}
3368	-	if e.Err != nil && !strings.Contains(e.Err.Error(), "context canceled") {	3505	+	// Don't surface mid-loop stream errors as a red banner here.
3369	-	i.statusErr = e.Err.Error()	3506	+	// EvTurnEnd fires after every step in a multi-step tool loop,
3370	-	}	3507	+	// so a transient 503 / network blip would briefly paint a red
			3508	+	// banner over the still-streaming chat before the agent loop
			3509	+	// either retries or exits. The final error (if any) is set by
			3510	+	// startTurnWithImages once Prompt() returns, and recoverable
			3511	+	// failures are routed to the rescue picker instead — which
			3512	+	// keeps the chat clean while the agent is working.
			3513	+	_ = e.Err
3371		}	3514		}
3372		}	3515		}
3373			3516

A internal/agent/modes/rescue_dialog.go +300 -0

@@ -0,0 +1,300 @@
	1	+	package modes
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"strings"
	6	+
	7	+	"github.com/patriceckhart/zot/internal/provider"
	8	+	"github.com/patriceckhart/zot/internal/tui"
	9	+	)
	10	+
	11	+	// rescueDialog offers a quick model swap when the active turn fails
	12	+	// with a recoverable provider error (auth/rate/temporary). It looks
	13	+	// and behaves like modelDialog: vertical list of available models,
	14	+	// up/down to select, enter to retry, esc to cancel. The candidate
	15	+	// list is built dynamically from the providers the user is currently
	16	+	// logged in to — no static fallback config.
	17	+	type rescueDialog struct {
	18	+	active bool
	19	+	all []provider.Model // candidates, sorted
	20	+	view []provider.Model // filtered view (typed query)
	21	+	cursor int
	22	+	current string // currently active model id (excluded from view)
	23	+	query string
	24	+	failedAt string // failed provider/model pair, e.g. "kimi/kimi-for-coding"
	25	+	reason string // short human-readable reason ("token expired", "rate limited", ...)
	26	+	prompt string // the user prompt that should be retried on Select
	27	+	}
	28	+
	29	+	type rescueDialogAction struct {
	30	+	Select bool
	31	+	Provider string
	32	+	Model string
	33	+	Prompt string
	34	+	Close bool
	35	+	}
	36	+
	37	+	func newRescueDialog() *rescueDialog { return &rescueDialog{} }
	38	+
	39	+	// Open shows the dialog. current is the currently active model id so
	40	+	// it can be excluded from the candidate list. loggedInProviders is
	41	+	// the set of provider names with usable credentials right now.
	42	+	func (d *rescueDialog) Open(current string, loggedInProviders []string, failedProvider, failedModel, reason, prompt string) {
	43	+	d.active = true
	44	+	d.current = current
	45	+	d.query = ""
	46	+	d.reason = reason
	47	+	d.prompt = prompt
	48	+	d.failedAt = strings.TrimSpace(failedProvider + "/" + failedModel)
	49	+
	50	+	all := provider.Active()
	51	+	provSet := map[string]bool{}
	52	+	for _, p := range loggedInProviders {
	53	+	provSet[p] = true
	54	+	}
	55	+	var filtered []provider.Model
	56	+	for _, m := range all {
	57	+	if !provSet[m.Provider] {
	58	+	continue
	59	+	}
	60	+	// Drop the exact failed pair so users can't retry on the
	61	+	// model that just failed.
	62	+	if m.Provider == failedProvider && m.ID == failedModel {
	63	+	continue
	64	+	}
	65	+	// Drop the currently-active model id (if it differs from
	66	+	// the failed one for some reason). The picker is meant to
	67	+	// help the user move off the broken pair.
	68	+	if m.ID == current {
	69	+	continue
	70	+	}
	71	+	filtered = append(filtered, m)
	72	+	}
	73	+	d.all = sortedModels(filtered)
	74	+	d.refilter()
	75	+	}
	76	+
	77	+	func (d *rescueDialog) Close() { d.active = false }
	78	+	func (d *rescueDialog) Active() bool { return d != nil && d.active }
	79	+
	80	+	func (d *rescueDialog) refilter() {
	81	+	needle := normalizeModelQuery(d.query)
	82	+	if needle == "" {
	83	+	d.view = append([]provider.Model(nil), d.all...)
	84	+	} else {
	85	+	out := make([]provider.Model, 0, len(d.all))
	86	+	for _, m := range d.all {
	87	+	if strings.Contains(normalizeModelQuery(m.Provider+" "+m.ID+" "+m.DisplayName), needle) {
	88	+	out = append(out, m)
	89	+	}
	90	+	}
	91	+	d.view = out
	92	+	}
	93	+	d.cursor = 0
	94	+	}
	95	+
	96	+	// Render mirrors modelDialog so a rescue prompt feels identical to
	97	+	// every other picker in the TUI.
	98	+	func (d *rescueDialog) Render(th tui.Theme, width int) []string {
	99	+	if !d.Active() {
	100	+	return nil
	101	+	}
	102	+	var lines []string
	103	+	header := "rescue turn"
	104	+	if d.failedAt != "" && d.failedAt != "/" {
	105	+	header = "rescue turn — " + d.failedAt + " failed"
	106	+	}
	107	+	lines = append(lines, frameHeader(th, header, width))
	108	+
	109	+	if d.reason != "" {
	110	+	lines = append(lines, th.FG256(th.Warning, " "+d.reason))
	111	+	}
	112	+
	113	+	hint := "retry this turn with another model (↑/↓, enter, esc to cancel)"
	114	+	if d.query != "" {
	115	+	hint = fmt.Sprintf("filter: %s (%d match)", d.query, len(d.view))
	116	+	if len(d.view) != 1 {
	117	+	hint = fmt.Sprintf("filter: %s (%d matches)", d.query, len(d.view))
	118	+	}
	119	+	} else {
	120	+	hint += " - type to filter"
	121	+	}
	122	+	lines = append(lines, th.FG256(th.Muted, hint))
	123	+
	124	+	if len(d.view) == 0 {
	125	+	if len(d.all) == 0 {
	126	+	lines = append(lines, th.FG256(th.Muted, " no other models available — log in to another provider with /login"))
	127	+	} else {
	128	+	lines = append(lines, th.FG256(th.Muted, " no models match "+fmt.Sprintf("%q", d.query)))
	129	+	}
	130	+	lines = append(lines, frameRule(th, width))
	131	+	return lines
	132	+	}
	133	+
	134	+	const visible = 12
	135	+	start := 0
	136	+	end := len(d.view)
	137	+	if end > visible {
	138	+	start = d.cursor - visible/2
	139	+	if start < 0 {
	140	+	start = 0
	141	+	}
	142	+	if start+visible > end {
	143	+	start = end - visible
	144	+	}
	145	+	end = start + visible
	146	+	}
	147	+	for i := start; i < end; i++ {
	148	+	m := d.view[i]
	149	+	reason := " "
	150	+	if m.Reasoning {
	151	+	reason = "✦"
	152	+	}
	153	+	tag := ""
	154	+	switch {
	155	+	case m.Speculative:
	156	+	tag = "[speculative] "
	157	+	case m.Source == "live":
	158	+	tag = "[live] "
	159	+	}
	160	+	plain := fmt.Sprintf(" %-10s %-28s %s %s%s", m.Provider, m.ID, reason, tag, m.DisplayName)
	161	+	if i == d.cursor {
	162	+	lines = append(lines, th.PadHighlight(plain, width))
	163	+	} else {
	164	+	lines = append(lines, th.FG256(th.Muted, plain))
	165	+	}
	166	+	}
	167	+	if start > 0 {
	168	+	lines = append(lines, th.FG256(th.Muted, fmt.Sprintf(" … %d more above", start)))
	169	+	}
	170	+	if end < len(d.view) {
	171	+	lines = append(lines, th.FG256(th.Muted, fmt.Sprintf(" … %d more below", len(d.view)-end)))
	172	+	}
	173	+
	174	+	lines = append(lines, frameRule(th, width))
	175	+	return lines
	176	+	}
	177	+
	178	+	func (d *rescueDialog) HandleKey(k tui.Key) rescueDialogAction {
	179	+	switch k.Kind {
	180	+	case tui.KeyUp:
	181	+	if d.cursor > 0 {
	182	+	d.cursor--
	183	+	}
	184	+	case tui.KeyDown:
	185	+	if d.cursor < len(d.view)-1 {
	186	+	d.cursor++
	187	+	}
	188	+	case tui.KeyBackspace:
	189	+	if len(d.query) > 0 {
	190	+	r := []rune(d.query)
	191	+	d.query = string(r[:len(r)-1])
	192	+	d.refilter()
	193	+	}
	194	+	case tui.KeyRune:
	195	+	if k.Alt \|\| k.Ctrl {
	196	+	break
	197	+	}
	198	+	if k.Rune >= 0x20 && k.Rune < 0x7f {
	199	+	d.query += string(k.Rune)
	200	+	d.refilter()
	201	+	}
	202	+	case tui.KeyEsc:
	203	+	d.Close()
	204	+	return rescueDialogAction{Close: true}
	205	+	case tui.KeyEnter:
	206	+	if len(d.view) == 0 {
	207	+	d.Close()
	208	+	return rescueDialogAction{Close: true}
	209	+	}
	210	+	m := d.view[d.cursor]
	211	+	prompt := d.prompt
	212	+	d.Close()
	213	+	return rescueDialogAction{Select: true, Provider: m.Provider, Model: m.ID, Prompt: prompt}
	214	+	}
	215	+	return rescueDialogAction{}
	216	+	}
	217	+
	218	+	// classifyRescueError inspects an agent error and decides whether
	219	+	// it's a recoverable provider failure that the user should be offered
	220	+	// a rescue picker for, plus a short human-readable reason. Returns
	221	+	// (false, "") for errors we should NOT auto-rescue (bad request,
	222	+	// context length, transcript serialization issues, etc.) so the
	223	+	// regular red status banner still surfaces them.
	224	+	func classifyRescueError(err error) (bool, string) {
	225	+	if err == nil {
	226	+	return false, ""
	227	+	}
	228	+	msg := err.Error()
	229	+	low := strings.ToLower(msg)
	230	+
	231	+	// Don't trigger on payload-too-large; that path already has its
	232	+	// own auto-compact handling.
	233	+	if isPayloadTooLargeError(err) {
	234	+	return false, ""
	235	+	}
	236	+
	237	+	// Network failures.
	238	+	if strings.Contains(low, "timeout") \|\|
	239	+	strings.Contains(low, "deadline exceeded") \|\|
	240	+	strings.Contains(low, "connection refused") \|\|
	241	+	strings.Contains(low, "connection reset") \|\|
	242	+	strings.Contains(low, "no such host") \|\|
	243	+	strings.Contains(low, "tls handshake") \|\|
	244	+	strings.Contains(low, "eof") {
	245	+	return true, "network failure: " + shortError(msg)
	246	+	}
	247	+
	248	+	switch {
	249	+	case containsAny(low, "http 401", " 401:", "invalid_authentication", "token expired", "api key appears to be invalid"):
	250	+	return true, "authentication failed: " + shortError(msg)
	251	+	case containsAny(low, "http 403", " 403:", "permission denied", "forbidden"):
	252	+	return true, "permission denied: " + shortError(msg)
	253	+	case containsAny(low, "http 429", " 429:", "rate limit", "rate_limit", "too many requests", "quota"):
	254	+	return true, "rate limited: " + shortError(msg)
	255	+	case containsAny(low, "http 500", "http 502", "http 503", "http 504", " 500:", " 502:", " 503:", " 504:", "upstream connect error", "service unavailable", "internal server error", "bad gateway", "gateway timeout"):
	256	+	return true, "provider unavailable: " + shortError(msg)
	257	+	}
	258	+
	259	+	// Anything else (400 bad request, validation errors, etc.) is
	260	+	// usually not fixed by switching models; let it surface as-is.
	261	+	return false, ""
	262	+	}
	263	+
	264	+	func containsAny(haystack string, needles ...string) bool {
	265	+	for _, n := range needles {
	266	+	if strings.Contains(haystack, n) {
	267	+	return true
	268	+	}
	269	+	}
	270	+	return false
	271	+	}
	272	+
	273	+	// shortError trims a long http-payload error to something readable in
	274	+	// the rescue dialog header without dropping the most useful prefix.
	275	+	func shortError(msg string) string {
	276	+	msg = strings.TrimSpace(msg)
	277	+	const max = 140
	278	+	if len(msg) <= max {
	279	+	return msg
	280	+	}
	281	+	return msg[:max] + "…"
	282	+	}
	283	+
	284	+	// extractFailedProvider tries to pull the failing provider name out
	285	+	// of an error string ("kimi: http 401: ...", "openai: http 503: ...").
	286	+	// Returns "" when nothing recognisable is found.
	287	+	func extractFailedProvider(err error) string {
	288	+	if err == nil {
	289	+	return ""
	290	+	}
	291	+	msg := err.Error()
	292	+	if i := strings.Index(msg, ":"); i > 0 {
	293	+	head := strings.TrimSpace(msg[:i])
	294	+	switch head {
	295	+	case "anthropic", "openai", "openai-codex", "kimi", "ollama":
	296	+	return head
	297	+	}
	298	+	}
	299	+	return ""
	300	+	}

@@ -0,0 +1,300 @@
	1	+	package modes
	2	+
	3	+	import (
	4	+	"fmt"
	5	+	"strings"
	6	+
	7	+	"github.com/patriceckhart/zot/internal/provider"
	8	+	"github.com/patriceckhart/zot/internal/tui"
	9	+	)
	10	+
	11	+	// rescueDialog offers a quick model swap when the active turn fails
	12	+	// with a recoverable provider error (auth/rate/temporary). It looks
	13	+	// and behaves like modelDialog: vertical list of available models,
	14	+	// up/down to select, enter to retry, esc to cancel. The candidate
	15	+	// list is built dynamically from the providers the user is currently
	16	+	// logged in to — no static fallback config.
	17	+	type rescueDialog struct {
	18	+	active bool
	19	+	all []provider.Model // candidates, sorted
	20	+	view []provider.Model // filtered view (typed query)
	21	+	cursor int
	22	+	current string // currently active model id (excluded from view)
	23	+	query string
	24	+	failedAt string // failed provider/model pair, e.g. "kimi/kimi-for-coding"
	25	+	reason string // short human-readable reason ("token expired", "rate limited", ...)
	26	+	prompt string // the user prompt that should be retried on Select
	27	+	}
	28	+
	29	+	type rescueDialogAction struct {
	30	+	Select bool
	31	+	Provider string
	32	+	Model string
	33	+	Prompt string
	34	+	Close bool
	35	+	}
	36	+
	37	+	func newRescueDialog() *rescueDialog { return &rescueDialog{} }
	38	+
	39	+	// Open shows the dialog. current is the currently active model id so
	40	+	// it can be excluded from the candidate list. loggedInProviders is
	41	+	// the set of provider names with usable credentials right now.
	42	+	func (d *rescueDialog) Open(current string, loggedInProviders []string, failedProvider, failedModel, reason, prompt string) {
	43	+	d.active = true
	44	+	d.current = current
	45	+	d.query = ""
	46	+	d.reason = reason
	47	+	d.prompt = prompt
	48	+	d.failedAt = strings.TrimSpace(failedProvider + "/" + failedModel)
	49	+
	50	+	all := provider.Active()
	51	+	provSet := map[string]bool{}
	52	+	for _, p := range loggedInProviders {
	53	+	provSet[p] = true
	54	+	}
	55	+	var filtered []provider.Model
	56	+	for _, m := range all {
	57	+	if !provSet[m.Provider] {
	58	+	continue
	59	+	}
	60	+	// Drop the exact failed pair so users can't retry on the
	61	+	// model that just failed.
	62	+	if m.Provider == failedProvider && m.ID == failedModel {
	63	+	continue
	64	+	}
	65	+	// Drop the currently-active model id (if it differs from
	66	+	// the failed one for some reason). The picker is meant to
	67	+	// help the user move off the broken pair.
	68	+	if m.ID == current {
	69	+	continue
	70	+	}
	71	+	filtered = append(filtered, m)
	72	+	}
	73	+	d.all = sortedModels(filtered)
	74	+	d.refilter()
	75	+	}
	76	+
	77	+	func (d *rescueDialog) Close() { d.active = false }
	78	+	func (d *rescueDialog) Active() bool { return d != nil && d.active }
	79	+
	80	+	func (d *rescueDialog) refilter() {
	81	+	needle := normalizeModelQuery(d.query)
	82	+	if needle == "" {
	83	+	d.view = append([]provider.Model(nil), d.all...)
	84	+	} else {
	85	+	out := make([]provider.Model, 0, len(d.all))
	86	+	for _, m := range d.all {
	87	+	if strings.Contains(normalizeModelQuery(m.Provider+" "+m.ID+" "+m.DisplayName), needle) {
	88	+	out = append(out, m)
	89	+	}
	90	+	}
	91	+	d.view = out
	92	+	}
	93	+	d.cursor = 0
	94	+	}
	95	+
	96	+	// Render mirrors modelDialog so a rescue prompt feels identical to
	97	+	// every other picker in the TUI.
	98	+	func (d *rescueDialog) Render(th tui.Theme, width int) []string {
	99	+	if !d.Active() {
	100	+	return nil
	101	+	}
	102	+	var lines []string
	103	+	header := "rescue turn"
	104	+	if d.failedAt != "" && d.failedAt != "/" {
	105	+	header = "rescue turn — " + d.failedAt + " failed"
	106	+	}
	107	+	lines = append(lines, frameHeader(th, header, width))
	108	+
	109	+	if d.reason != "" {
	110	+	lines = append(lines, th.FG256(th.Warning, " "+d.reason))
	111	+	}
	112	+
	113	+	hint := "retry this turn with another model (↑/↓, enter, esc to cancel)"
	114	+	if d.query != "" {
	115	+	hint = fmt.Sprintf("filter: %s (%d match)", d.query, len(d.view))
	116	+	if len(d.view) != 1 {
	117	+	hint = fmt.Sprintf("filter: %s (%d matches)", d.query, len(d.view))
	118	+	}
	119	+	} else {
	120	+	hint += " - type to filter"
	121	+	}
	122	+	lines = append(lines, th.FG256(th.Muted, hint))
	123	+
	124	+	if len(d.view) == 0 {
	125	+	if len(d.all) == 0 {
	126	+	lines = append(lines, th.FG256(th.Muted, " no other models available — log in to another provider with /login"))
	127	+	} else {
	128	+	lines = append(lines, th.FG256(th.Muted, " no models match "+fmt.Sprintf("%q", d.query)))
	129	+	}
	130	+	lines = append(lines, frameRule(th, width))
	131	+	return lines
	132	+	}
	133	+
	134	+	const visible = 12
	135	+	start := 0
	136	+	end := len(d.view)
	137	+	if end > visible {
	138	+	start = d.cursor - visible/2
	139	+	if start < 0 {
	140	+	start = 0
	141	+	}
	142	+	if start+visible > end {
	143	+	start = end - visible
	144	+	}
	145	+	end = start + visible
	146	+	}
	147	+	for i := start; i < end; i++ {
	148	+	m := d.view[i]
	149	+	reason := " "
	150	+	if m.Reasoning {
	151	+	reason = "✦"
	152	+	}
	153	+	tag := ""
	154	+	switch {
	155	+	case m.Speculative:
	156	+	tag = "[speculative] "
	157	+	case m.Source == "live":
	158	+	tag = "[live] "
	159	+	}
	160	+	plain := fmt.Sprintf(" %-10s %-28s %s %s%s", m.Provider, m.ID, reason, tag, m.DisplayName)
	161	+	if i == d.cursor {
	162	+	lines = append(lines, th.PadHighlight(plain, width))
	163	+	} else {
	164	+	lines = append(lines, th.FG256(th.Muted, plain))
	165	+	}
	166	+	}
	167	+	if start > 0 {
	168	+	lines = append(lines, th.FG256(th.Muted, fmt.Sprintf(" … %d more above", start)))
	169	+	}
	170	+	if end < len(d.view) {
	171	+	lines = append(lines, th.FG256(th.Muted, fmt.Sprintf(" … %d more below", len(d.view)-end)))
	172	+	}
	173	+
	174	+	lines = append(lines, frameRule(th, width))
	175	+	return lines
	176	+	}
	177	+
	178	+	func (d *rescueDialog) HandleKey(k tui.Key) rescueDialogAction {
	179	+	switch k.Kind {
	180	+	case tui.KeyUp:
	181	+	if d.cursor > 0 {
	182	+	d.cursor--
	183	+	}
	184	+	case tui.KeyDown:
	185	+	if d.cursor < len(d.view)-1 {
	186	+	d.cursor++
	187	+	}
	188	+	case tui.KeyBackspace:
	189	+	if len(d.query) > 0 {
	190	+	r := []rune(d.query)
	191	+	d.query = string(r[:len(r)-1])
	192	+	d.refilter()
	193	+	}
	194	+	case tui.KeyRune:
	195	+	if k.Alt \|\| k.Ctrl {
	196	+	break
	197	+	}
	198	+	if k.Rune >= 0x20 && k.Rune < 0x7f {
	199	+	d.query += string(k.Rune)
	200	+	d.refilter()
	201	+	}
	202	+	case tui.KeyEsc:
	203	+	d.Close()
	204	+	return rescueDialogAction{Close: true}
	205	+	case tui.KeyEnter:
	206	+	if len(d.view) == 0 {
	207	+	d.Close()
	208	+	return rescueDialogAction{Close: true}
	209	+	}
	210	+	m := d.view[d.cursor]
	211	+	prompt := d.prompt
	212	+	d.Close()
	213	+	return rescueDialogAction{Select: true, Provider: m.Provider, Model: m.ID, Prompt: prompt}
	214	+	}
	215	+	return rescueDialogAction{}
	216	+	}
	217	+
	218	+	// classifyRescueError inspects an agent error and decides whether
	219	+	// it's a recoverable provider failure that the user should be offered
	220	+	// a rescue picker for, plus a short human-readable reason. Returns
	221	+	// (false, "") for errors we should NOT auto-rescue (bad request,
	222	+	// context length, transcript serialization issues, etc.) so the
	223	+	// regular red status banner still surfaces them.
	224	+	func classifyRescueError(err error) (bool, string) {
	225	+	if err == nil {
	226	+	return false, ""
	227	+	}
	228	+	msg := err.Error()
	229	+	low := strings.ToLower(msg)
	230	+
	231	+	// Don't trigger on payload-too-large; that path already has its
	232	+	// own auto-compact handling.
	233	+	if isPayloadTooLargeError(err) {
	234	+	return false, ""
	235	+	}
	236	+
	237	+	// Network failures.
	238	+	if strings.Contains(low, "timeout") \|\|
	239	+	strings.Contains(low, "deadline exceeded") \|\|
	240	+	strings.Contains(low, "connection refused") \|\|
	241	+	strings.Contains(low, "connection reset") \|\|
	242	+	strings.Contains(low, "no such host") \|\|
	243	+	strings.Contains(low, "tls handshake") \|\|
	244	+	strings.Contains(low, "eof") {
	245	+	return true, "network failure: " + shortError(msg)
	246	+	}
	247	+
	248	+	switch {
	249	+	case containsAny(low, "http 401", " 401:", "invalid_authentication", "token expired", "api key appears to be invalid"):
	250	+	return true, "authentication failed: " + shortError(msg)
	251	+	case containsAny(low, "http 403", " 403:", "permission denied", "forbidden"):
	252	+	return true, "permission denied: " + shortError(msg)
	253	+	case containsAny(low, "http 429", " 429:", "rate limit", "rate_limit", "too many requests", "quota"):
	254	+	return true, "rate limited: " + shortError(msg)
	255	+	case containsAny(low, "http 500", "http 502", "http 503", "http 504", " 500:", " 502:", " 503:", " 504:", "upstream connect error", "service unavailable", "internal server error", "bad gateway", "gateway timeout"):
	256	+	return true, "provider unavailable: " + shortError(msg)
	257	+	}
	258	+
	259	+	// Anything else (400 bad request, validation errors, etc.) is
	260	+	// usually not fixed by switching models; let it surface as-is.
	261	+	return false, ""
	262	+	}
	263	+
	264	+	func containsAny(haystack string, needles ...string) bool {
	265	+	for _, n := range needles {
	266	+	if strings.Contains(haystack, n) {
	267	+	return true
	268	+	}
	269	+	}
	270	+	return false
	271	+	}
	272	+
	273	+	// shortError trims a long http-payload error to something readable in
	274	+	// the rescue dialog header without dropping the most useful prefix.
	275	+	func shortError(msg string) string {
	276	+	msg = strings.TrimSpace(msg)
	277	+	const max = 140
	278	+	if len(msg) <= max {
	279	+	return msg
	280	+	}
	281	+	return msg[:max] + "…"
	282	+	}
	283	+
	284	+	// extractFailedProvider tries to pull the failing provider name out
	285	+	// of an error string ("kimi: http 401: ...", "openai: http 503: ...").
	286	+	// Returns "" when nothing recognisable is found.
	287	+	func extractFailedProvider(err error) string {
	288	+	if err == nil {
	289	+	return ""
	290	+	}
	291	+	msg := err.Error()
	292	+	if i := strings.Index(msg, ":"); i > 0 {
	293	+	head := strings.TrimSpace(msg[:i])
	294	+	switch head {
	295	+	case "anthropic", "openai", "openai-codex", "kimi", "ollama":
	296	+	return head
	297	+	}
	298	+	}
	299	+	return ""
	300	+	}

M internal/provider/anthropic.go +25 -22


@@ -453,29 +453,32 @@ func (c *anthropicClient) Stream(ctx context.Context, req Request) (<-chan Event
453	453		}
454	454		}
455	455
456		-	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/v1/messages", bytes.NewReader(body))
457		-	if err != nil {
458		-	return nil, err
	456	+	newReq := func() (*http.Request, error) {
	457	+	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/v1/messages", bytes.NewReader(body))
	458	+	if err != nil {
	459	+	return nil, err
	460	+	}
	461	+	httpReq.Header.Set("content-type", "application/json")
	462	+	httpReq.Header.Set("anthropic-version", anthropicAPIVersion)
	463	+	if c.oauthTok != "" {
	464	+	// Claude-Code-shaped request: identical headers and values as the
	465	+	// official CLI. Any drift triggers Anthropic's anti-abuse check and
	466	+	// rate-limits (or outright blocks) the request.
	467	+	httpReq.Header.Set("accept", "application/json")
	468	+	httpReq.Header.Set("authorization", "Bearer "+c.oauthTok)
	469	+	httpReq.Header.Set("anthropic-beta", "claude-code-20250219,oauth-2025-04-20,fine-grained-tool-streaming-2025-05-14")
	470	+	httpReq.Header.Set("anthropic-dangerous-direct-browser-access", "true")
	471	+	httpReq.Header.Set("user-agent", "claude-cli/"+claudeCodeVersion)
	472	+	httpReq.Header.Set("x-app", "cli")
	473	+	// Remove x-api-key entirely by NOT setting it.
	474	+	} else {
	475	+	httpReq.Header.Set("accept", "text/event-stream")
	476	+	httpReq.Header.Set("x-api-key", c.apiKey)
	477	+	}
	478	+	return httpReq, nil
459	479		}
460		-	httpReq.Header.Set("content-type", "application/json")
461		-	httpReq.Header.Set("anthropic-version", anthropicAPIVersion)
462		-	if c.oauthTok != "" {
463		-	// Claude-Code-shaped request: identical headers and values as the
464		-	// official CLI. Any drift triggers Anthropic's anti-abuse check and
465		-	// rate-limits (or outright blocks) the request.
466		-	httpReq.Header.Set("accept", "application/json")
467		-	httpReq.Header.Set("authorization", "Bearer "+c.oauthTok)
468		-	httpReq.Header.Set("anthropic-beta", "claude-code-20250219,oauth-2025-04-20,fine-grained-tool-streaming-2025-05-14")
469		-	httpReq.Header.Set("anthropic-dangerous-direct-browser-access", "true")
470		-	httpReq.Header.Set("user-agent", "claude-cli/"+claudeCodeVersion)
471		-	httpReq.Header.Set("x-app", "cli")
472		-	// Remove x-api-key entirely by NOT setting it.
473		-	} else {
474		-	httpReq.Header.Set("accept", "text/event-stream")
475		-	httpReq.Header.Set("x-api-key", c.apiKey)
476		-	}
477		-
478		-	resp, err := c.http.Do(httpReq)
	480	+
	481	+	resp, err := doStreamWithRetry(ctx, c.http, newReq)
479	482		if err != nil {
480	483		return nil, fmt.Errorf("anthropic: %w", err)
481	484		}


@@ -453,29 +453,32 @@ func (c *anthropicClient) Stream(ctx context.Context, req Request) (<-chan Event
453		}	453		}
454		}	454		}
455			455
456	-	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/v1/messages", bytes.NewReader(body))	456	+	newReq := func() (*http.Request, error) {
457	-	if err != nil {	457	+	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/v1/messages", bytes.NewReader(body))
458	-	return nil, err	458	+	if err != nil {
			459	+	return nil, err
			460	+	}
			461	+	httpReq.Header.Set("content-type", "application/json")
			462	+	httpReq.Header.Set("anthropic-version", anthropicAPIVersion)
			463	+	if c.oauthTok != "" {
			464	+	// Claude-Code-shaped request: identical headers and values as the
			465	+	// official CLI. Any drift triggers Anthropic's anti-abuse check and
			466	+	// rate-limits (or outright blocks) the request.
			467	+	httpReq.Header.Set("accept", "application/json")
			468	+	httpReq.Header.Set("authorization", "Bearer "+c.oauthTok)
			469	+	httpReq.Header.Set("anthropic-beta", "claude-code-20250219,oauth-2025-04-20,fine-grained-tool-streaming-2025-05-14")
			470	+	httpReq.Header.Set("anthropic-dangerous-direct-browser-access", "true")
			471	+	httpReq.Header.Set("user-agent", "claude-cli/"+claudeCodeVersion)
			472	+	httpReq.Header.Set("x-app", "cli")
			473	+	// Remove x-api-key entirely by NOT setting it.
			474	+	} else {
			475	+	httpReq.Header.Set("accept", "text/event-stream")
			476	+	httpReq.Header.Set("x-api-key", c.apiKey)
			477	+	}
			478	+	return httpReq, nil
459		}	479		}
460	-	httpReq.Header.Set("content-type", "application/json")	480	+
461	-	httpReq.Header.Set("anthropic-version", anthropicAPIVersion)	481	+	resp, err := doStreamWithRetry(ctx, c.http, newReq)
462	-	if c.oauthTok != "" {
463	-	// Claude-Code-shaped request: identical headers and values as the
464	-	// official CLI. Any drift triggers Anthropic's anti-abuse check and
465	-	// rate-limits (or outright blocks) the request.
466	-	httpReq.Header.Set("accept", "application/json")
467	-	httpReq.Header.Set("authorization", "Bearer "+c.oauthTok)
468	-	httpReq.Header.Set("anthropic-beta", "claude-code-20250219,oauth-2025-04-20,fine-grained-tool-streaming-2025-05-14")
469	-	httpReq.Header.Set("anthropic-dangerous-direct-browser-access", "true")
470	-	httpReq.Header.Set("user-agent", "claude-cli/"+claudeCodeVersion)
471	-	httpReq.Header.Set("x-app", "cli")
472	-	// Remove x-api-key entirely by NOT setting it.
473	-	} else {
474	-	httpReq.Header.Set("accept", "text/event-stream")
475	-	httpReq.Header.Set("x-api-key", c.apiKey)
476	-	}
477	-
478	-	resp, err := c.http.Do(httpReq)
479		if err != nil {	482		if err != nil {
480		return nil, fmt.Errorf("anthropic: %w", err)	483		return nil, fmt.Errorf("anthropic: %w", err)
481		}	484		}

M internal/provider/openai.go +13 -10


@@ -354,18 +354,21 @@ func (c *openaiClient) Stream(ctx context.Context, req Request) (<-chan Event, e
354	354		if err != nil {
355	355		return nil, err
356	356		}
357		-	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+apiPath, bytes.NewReader(body))
358		-	if err != nil {
359		-	return nil, err
360		-	}
361		-	httpReq.Header.Set("content-type", "application/json")
362		-	httpReq.Header.Set("accept", "text/event-stream")
363		-	httpReq.Header.Set("authorization", "Bearer "+c.apiKey)
364		-	for k, v := range c.headers {
365		-	httpReq.Header.Set(k, v)
	357	+	newReq := func() (*http.Request, error) {
	358	+	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+apiPath, bytes.NewReader(body))
	359	+	if err != nil {
	360	+	return nil, err
	361	+	}
	362	+	httpReq.Header.Set("content-type", "application/json")
	363	+	httpReq.Header.Set("accept", "text/event-stream")
	364	+	httpReq.Header.Set("authorization", "Bearer "+c.apiKey)
	365	+	for k, v := range c.headers {
	366	+	httpReq.Header.Set(k, v)
	367	+	}
	368	+	return httpReq, nil
366	369		}
367	370
368		-	resp, err := c.http.Do(httpReq)
	371	+	resp, err := doStreamWithRetry(ctx, c.http, newReq)
369	372		if err != nil {
370	373		return nil, fmt.Errorf("%s: %w", c.Name(), err)
371	374		}


@@ -354,18 +354,21 @@ func (c *openaiClient) Stream(ctx context.Context, req Request) (<-chan Event, e
354		if err != nil {	354		if err != nil {
355		return nil, err	355		return nil, err
356		}	356		}
357	-	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+apiPath, bytes.NewReader(body))	357	+	newReq := func() (*http.Request, error) {
358	-	if err != nil {	358	+	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+apiPath, bytes.NewReader(body))
359	-	return nil, err	359	+	if err != nil {
360	-	}	360	+	return nil, err
361	-	httpReq.Header.Set("content-type", "application/json")	361	+	}
362	-	httpReq.Header.Set("accept", "text/event-stream")	362	+	httpReq.Header.Set("content-type", "application/json")
363	-	httpReq.Header.Set("authorization", "Bearer "+c.apiKey)	363	+	httpReq.Header.Set("accept", "text/event-stream")
364	-	for k, v := range c.headers {	364	+	httpReq.Header.Set("authorization", "Bearer "+c.apiKey)
365	-	httpReq.Header.Set(k, v)	365	+	for k, v := range c.headers {
			366	+	httpReq.Header.Set(k, v)
			367	+	}
			368	+	return httpReq, nil
366		}	369		}
367			370
368	-	resp, err := c.http.Do(httpReq)	371	+	resp, err := doStreamWithRetry(ctx, c.http, newReq)
369		if err != nil {	372		if err != nil {
370		return nil, fmt.Errorf("%s: %w", c.Name(), err)	373		return nil, fmt.Errorf("%s: %w", c.Name(), err)
371		}	374		}

M internal/provider/openai_codex.go +15 -12


@@ -290,19 +290,22 @@ func (c *codexClient) Stream(ctx context.Context, req Request) (<-chan Event, er
290	290		return nil, err
291	291		}
292	292
293		-	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL, bytes.NewReader(body))
294		-	if err != nil {
295		-	return nil, err
	293	+	newReq := func() (*http.Request, error) {
	294	+	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL, bytes.NewReader(body))
	295	+	if err != nil {
	296	+	return nil, err
	297	+	}
	298	+	httpReq.Header.Set("content-type", "application/json")
	299	+	httpReq.Header.Set("accept", "text/event-stream")
	300	+	httpReq.Header.Set("authorization", "Bearer "+c.token)
	301	+	httpReq.Header.Set("chatgpt-account-id", c.accountID)
	302	+	httpReq.Header.Set("openai-beta", "responses=experimental")
	303	+	httpReq.Header.Set("originator", "zot")
	304	+	httpReq.Header.Set("user-agent", fmt.Sprintf("zot (%s %s)", runtime.GOOS, runtime.GOARCH))
	305	+	return httpReq, nil
296	306		}
297		-	httpReq.Header.Set("content-type", "application/json")
298		-	httpReq.Header.Set("accept", "text/event-stream")
299		-	httpReq.Header.Set("authorization", "Bearer "+c.token)
300		-	httpReq.Header.Set("chatgpt-account-id", c.accountID)
301		-	httpReq.Header.Set("openai-beta", "responses=experimental")
302		-	httpReq.Header.Set("originator", "zot")
303		-	httpReq.Header.Set("user-agent", fmt.Sprintf("zot (%s %s)", runtime.GOOS, runtime.GOARCH))
304		-
305		-	resp, err := c.http.Do(httpReq)
	307	+
	308	+	resp, err := doStreamWithRetry(ctx, c.http, newReq)
306	309		if err != nil {
307	310		return nil, fmt.Errorf("openai-codex: %w", err)
308	311		}


@@ -290,19 +290,22 @@ func (c *codexClient) Stream(ctx context.Context, req Request) (<-chan Event, er
290		return nil, err	290		return nil, err
291		}	291		}
292			292
293	-	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL, bytes.NewReader(body))	293	+	newReq := func() (*http.Request, error) {
294	-	if err != nil {	294	+	httpReq, err := http.NewRequestWithContext(ctx, "POST", c.baseURL, bytes.NewReader(body))
295	-	return nil, err	295	+	if err != nil {
			296	+	return nil, err
			297	+	}
			298	+	httpReq.Header.Set("content-type", "application/json")
			299	+	httpReq.Header.Set("accept", "text/event-stream")
			300	+	httpReq.Header.Set("authorization", "Bearer "+c.token)
			301	+	httpReq.Header.Set("chatgpt-account-id", c.accountID)
			302	+	httpReq.Header.Set("openai-beta", "responses=experimental")
			303	+	httpReq.Header.Set("originator", "zot")
			304	+	httpReq.Header.Set("user-agent", fmt.Sprintf("zot (%s %s)", runtime.GOOS, runtime.GOARCH))
			305	+	return httpReq, nil
296		}	306		}
297	-	httpReq.Header.Set("content-type", "application/json")	307	+
298	-	httpReq.Header.Set("accept", "text/event-stream")	308	+	resp, err := doStreamWithRetry(ctx, c.http, newReq)
299	-	httpReq.Header.Set("authorization", "Bearer "+c.token)
300	-	httpReq.Header.Set("chatgpt-account-id", c.accountID)
301	-	httpReq.Header.Set("openai-beta", "responses=experimental")
302	-	httpReq.Header.Set("originator", "zot")
303	-	httpReq.Header.Set("user-agent", fmt.Sprintf("zot (%s %s)", runtime.GOOS, runtime.GOARCH))
304	-
305	-	resp, err := c.http.Do(httpReq)
306		if err != nil {	309		if err != nil {
307		return nil, fmt.Errorf("openai-codex: %w", err)	310		return nil, fmt.Errorf("openai-codex: %w", err)
308		}	311		}

A internal/provider/retry.go +158 -0

@@ -0,0 +1,158 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"errors"
	7	+	"io"
	8	+	"net/http"
	9	+	"strings"
	10	+	"time"
	11	+	)
	12	+
	13	+	// streamRetryAttempts is the maximum number of silent retries for
	14	+	// transient connect failures before the streaming request is opened.
	15	+	// 2 retries (so up to 3 total attempts) covers the common case where
	16	+	// the upstream provider's edge proxy briefly returns 502/503/504 or
	17	+	// resets the connection before headers, without making the user wait
	18	+	// noticeably long if the outage is real.
	19	+	const streamRetryAttempts = 2
	20	+
	21	+	// streamRetryBackoff returns the wait duration before retry attempt n
	22	+	// (1-based). Short, fixed backoff: 250ms, then 750ms. Anything longer
	23	+	// would feel like the agent froze; anything shorter starts hammering
	24	+	// the provider when it's actually struggling.
	25	+	func streamRetryBackoff(attempt int) time.Duration {
	26	+	switch attempt {
	27	+	case 1:
	28	+	return 250 * time.Millisecond
	29	+	default:
	30	+	return 750 * time.Millisecond
	31	+	}
	32	+	}
	33	+
	34	+	// doStreamWithRetry performs an HTTP request that begins a streaming
	35	+	// response, with up to streamRetryAttempts silent retries for transient
	36	+	// connect failures. Successful responses (status 200) and non-transient
	37	+	// failures (4xx, malformed bodies, etc.) are returned immediately.
	38	+	//
	39	+	// Retries fire on:
	40	+	// - net.Conn dial errors (connection refused, no such host, EOF
	41	+	// before headers, "upstream connect error", "connection reset")
	42	+	// - HTTP 502/503/504
	43	+	//
	44	+	// The handler is responsible for re-reading the request body each
	45	+	// attempt; we re-create the request via newReq() because http.Request
	46	+	// bodies are single-use. Callers in this package always Marshal a JSON
	47	+	// body up-front, so newReq just rebuilds with bytes.NewReader on the
	48	+	// captured payload.
	49	+	//
	50	+	// If every attempt fails, the last error/response is returned so the
	51	+	// caller can format a normal error message. On context cancellation
	52	+	// the loop bails out immediately with ctx.Err().
	53	+	func doStreamWithRetry(ctx context.Context, client http.Client, newReq func() (http.Request, error)) (*http.Response, error) {
	54	+	var lastErr error
	55	+	for attempt := 0; attempt <= streamRetryAttempts; attempt++ {
	56	+	if attempt > 0 {
	57	+	select {
	58	+	case <-ctx.Done():
	59	+	return nil, ctx.Err()
	60	+	case <-time.After(streamRetryBackoff(attempt)):
	61	+	}
	62	+	}
	63	+	req, err := newReq()
	64	+	if err != nil {
	65	+	return nil, err
	66	+	}
	67	+	resp, err := client.Do(req)
	68	+	if err != nil {
	69	+	lastErr = err
	70	+	if !isTransientConnectError(err) \|\| ctx.Err() != nil {
	71	+	return nil, err
	72	+	}
	73	+	continue
	74	+	}
	75	+	if isTransientHTTPStatus(resp.StatusCode) {
	76	+	// Drain a bit of the body so we can include it in the
	77	+	// final error if every retry exhausts. We cap the read
	78	+	// at 4 KiB because edge proxies sometimes send pages.
	79	+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
	80	+	resp.Body.Close()
	81	+	lastErr = &transientHTTPError{Status: resp.StatusCode, Body: strings.TrimSpace(string(body))}
	82	+	if attempt == streamRetryAttempts {
	83	+	// Wrap as a real *http.Response shape the caller
	84	+	// expects so it formats the error identically to a
	85	+	// non-retried failure.
	86	+	return synthesizeResponse(resp.StatusCode, body), nil
	87	+	}
	88	+	continue
	89	+	}
	90	+	return resp, nil
	91	+	}
	92	+	if lastErr == nil {
	93	+	lastErr = errors.New("retry loop exhausted")
	94	+	}
	95	+	return nil, lastErr
	96	+	}
	97	+
	98	+	// transientHTTPError is the placeholder error returned while we're
	99	+	// still inside the retry loop. Callers never observe it directly;
	100	+	// the loop converts the final exhausted attempt into a synthesized
	101	+	// http.Response so the caller's existing "http NNN: body" formatting
	102	+	// keeps working unchanged.
	103	+	type transientHTTPError struct {
	104	+	Status int
	105	+	Body string
	106	+	}
	107	+
	108	+	func (e *transientHTTPError) Error() string { return "transient http error" }
	109	+
	110	+	// synthesizeResponse rebuilds a closable response wrapping the captured
	111	+	// body bytes so the caller's existing non-200 handling path keeps
	112	+	// working without needing to know retries happened.
	113	+	func synthesizeResponse(status int, body []byte) *http.Response {
	114	+	return &http.Response{
	115	+	StatusCode: status,
	116	+	Body: io.NopCloser(bytes.NewReader(body)),
	117	+	}
	118	+	}
	119	+
	120	+	// isTransientConnectError reports whether err looks like a transient
	121	+	// network failure that's worth retrying. The classic "upstream connect
	122	+	// error or disconnect/reset before headers" from edge proxies (Envoy,
	123	+	// Cloudflare, GFE) shows up as several different concrete error types
	124	+	// across HTTP/1, HTTP/2, and TLS handshakes — easier to match by
	125	+	// substring on the rendered message than to enumerate every type.
	126	+	func isTransientConnectError(err error) bool {
	127	+	if err == nil {
	128	+	return false
	129	+	}
	130	+	if errors.Is(err, context.DeadlineExceeded) \|\| errors.Is(err, context.Canceled) {
	131	+	return false
	132	+	}
	133	+	msg := strings.ToLower(err.Error())
	134	+	return strings.Contains(msg, "connection reset") \|\|
	135	+	strings.Contains(msg, "connection refused") \|\|
	136	+	strings.Contains(msg, "broken pipe") \|\|
	137	+	strings.Contains(msg, "unexpected eof") \|\|
	138	+	strings.Contains(msg, "eof") \|\|
	139	+	strings.Contains(msg, "no such host") \|\|
	140	+	strings.Contains(msg, "tls handshake") \|\|
	141	+	strings.Contains(msg, "i/o timeout") \|\|
	142	+	strings.Contains(msg, "transport failure") \|\|
	143	+	strings.Contains(msg, "upstream connect error") \|\|
	144	+	strings.Contains(msg, "disconnect/reset before headers")
	145	+	}
	146	+
	147	+	// isTransientHTTPStatus reports whether a non-200 status code is
	148	+	// likely transient (502/503/504). 5xx outside this range is surfaced
	149	+	// immediately because it usually indicates a real backend bug.
	150	+	func isTransientHTTPStatus(code int) bool {
	151	+	switch code {
	152	+	case http.StatusBadGateway,
	153	+	http.StatusServiceUnavailable,
	154	+	http.StatusGatewayTimeout:
	155	+	return true
	156	+	}
	157	+	return false
	158	+	}

@@ -0,0 +1,158 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"bytes"
	5	+	"context"
	6	+	"errors"
	7	+	"io"
	8	+	"net/http"
	9	+	"strings"
	10	+	"time"
	11	+	)
	12	+
	13	+	// streamRetryAttempts is the maximum number of silent retries for
	14	+	// transient connect failures before the streaming request is opened.
	15	+	// 2 retries (so up to 3 total attempts) covers the common case where
	16	+	// the upstream provider's edge proxy briefly returns 502/503/504 or
	17	+	// resets the connection before headers, without making the user wait
	18	+	// noticeably long if the outage is real.
	19	+	const streamRetryAttempts = 2
	20	+
	21	+	// streamRetryBackoff returns the wait duration before retry attempt n
	22	+	// (1-based). Short, fixed backoff: 250ms, then 750ms. Anything longer
	23	+	// would feel like the agent froze; anything shorter starts hammering
	24	+	// the provider when it's actually struggling.
	25	+	func streamRetryBackoff(attempt int) time.Duration {
	26	+	switch attempt {
	27	+	case 1:
	28	+	return 250 * time.Millisecond
	29	+	default:
	30	+	return 750 * time.Millisecond
	31	+	}
	32	+	}
	33	+
	34	+	// doStreamWithRetry performs an HTTP request that begins a streaming
	35	+	// response, with up to streamRetryAttempts silent retries for transient
	36	+	// connect failures. Successful responses (status 200) and non-transient
	37	+	// failures (4xx, malformed bodies, etc.) are returned immediately.
	38	+	//
	39	+	// Retries fire on:
	40	+	// - net.Conn dial errors (connection refused, no such host, EOF
	41	+	// before headers, "upstream connect error", "connection reset")
	42	+	// - HTTP 502/503/504
	43	+	//
	44	+	// The handler is responsible for re-reading the request body each
	45	+	// attempt; we re-create the request via newReq() because http.Request
	46	+	// bodies are single-use. Callers in this package always Marshal a JSON
	47	+	// body up-front, so newReq just rebuilds with bytes.NewReader on the
	48	+	// captured payload.
	49	+	//
	50	+	// If every attempt fails, the last error/response is returned so the
	51	+	// caller can format a normal error message. On context cancellation
	52	+	// the loop bails out immediately with ctx.Err().
	53	+	func doStreamWithRetry(ctx context.Context, client http.Client, newReq func() (http.Request, error)) (*http.Response, error) {
	54	+	var lastErr error
	55	+	for attempt := 0; attempt <= streamRetryAttempts; attempt++ {
	56	+	if attempt > 0 {
	57	+	select {
	58	+	case <-ctx.Done():
	59	+	return nil, ctx.Err()
	60	+	case <-time.After(streamRetryBackoff(attempt)):
	61	+	}
	62	+	}
	63	+	req, err := newReq()
	64	+	if err != nil {
	65	+	return nil, err
	66	+	}
	67	+	resp, err := client.Do(req)
	68	+	if err != nil {
	69	+	lastErr = err
	70	+	if !isTransientConnectError(err) \|\| ctx.Err() != nil {
	71	+	return nil, err
	72	+	}
	73	+	continue
	74	+	}
	75	+	if isTransientHTTPStatus(resp.StatusCode) {
	76	+	// Drain a bit of the body so we can include it in the
	77	+	// final error if every retry exhausts. We cap the read
	78	+	// at 4 KiB because edge proxies sometimes send pages.
	79	+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
	80	+	resp.Body.Close()
	81	+	lastErr = &transientHTTPError{Status: resp.StatusCode, Body: strings.TrimSpace(string(body))}
	82	+	if attempt == streamRetryAttempts {
	83	+	// Wrap as a real *http.Response shape the caller
	84	+	// expects so it formats the error identically to a
	85	+	// non-retried failure.
	86	+	return synthesizeResponse(resp.StatusCode, body), nil
	87	+	}
	88	+	continue
	89	+	}
	90	+	return resp, nil
	91	+	}
	92	+	if lastErr == nil {
	93	+	lastErr = errors.New("retry loop exhausted")
	94	+	}
	95	+	return nil, lastErr
	96	+	}
	97	+
	98	+	// transientHTTPError is the placeholder error returned while we're
	99	+	// still inside the retry loop. Callers never observe it directly;
	100	+	// the loop converts the final exhausted attempt into a synthesized
	101	+	// http.Response so the caller's existing "http NNN: body" formatting
	102	+	// keeps working unchanged.
	103	+	type transientHTTPError struct {
	104	+	Status int
	105	+	Body string
	106	+	}
	107	+
	108	+	func (e *transientHTTPError) Error() string { return "transient http error" }
	109	+
	110	+	// synthesizeResponse rebuilds a closable response wrapping the captured
	111	+	// body bytes so the caller's existing non-200 handling path keeps
	112	+	// working without needing to know retries happened.
	113	+	func synthesizeResponse(status int, body []byte) *http.Response {
	114	+	return &http.Response{
	115	+	StatusCode: status,
	116	+	Body: io.NopCloser(bytes.NewReader(body)),
	117	+	}
	118	+	}
	119	+
	120	+	// isTransientConnectError reports whether err looks like a transient
	121	+	// network failure that's worth retrying. The classic "upstream connect
	122	+	// error or disconnect/reset before headers" from edge proxies (Envoy,
	123	+	// Cloudflare, GFE) shows up as several different concrete error types
	124	+	// across HTTP/1, HTTP/2, and TLS handshakes — easier to match by
	125	+	// substring on the rendered message than to enumerate every type.
	126	+	func isTransientConnectError(err error) bool {
	127	+	if err == nil {
	128	+	return false
	129	+	}
	130	+	if errors.Is(err, context.DeadlineExceeded) \|\| errors.Is(err, context.Canceled) {
	131	+	return false
	132	+	}
	133	+	msg := strings.ToLower(err.Error())
	134	+	return strings.Contains(msg, "connection reset") \|\|
	135	+	strings.Contains(msg, "connection refused") \|\|
	136	+	strings.Contains(msg, "broken pipe") \|\|
	137	+	strings.Contains(msg, "unexpected eof") \|\|
	138	+	strings.Contains(msg, "eof") \|\|
	139	+	strings.Contains(msg, "no such host") \|\|
	140	+	strings.Contains(msg, "tls handshake") \|\|
	141	+	strings.Contains(msg, "i/o timeout") \|\|
	142	+	strings.Contains(msg, "transport failure") \|\|
	143	+	strings.Contains(msg, "upstream connect error") \|\|
	144	+	strings.Contains(msg, "disconnect/reset before headers")
	145	+	}
	146	+
	147	+	// isTransientHTTPStatus reports whether a non-200 status code is
	148	+	// likely transient (502/503/504). 5xx outside this range is surfaced
	149	+	// immediately because it usually indicates a real backend bug.
	150	+	func isTransientHTTPStatus(code int) bool {
	151	+	switch code {
	152	+	case http.StatusBadGateway,
	153	+	http.StatusServiceUnavailable,
	154	+	http.StatusGatewayTimeout:
	155	+	return true
	156	+	}
	157	+	return false
	158	+	}

A internal/provider/retry_test.go +146 -0

@@ -0,0 +1,146 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"context"
	5	+	"net/http"
	6	+	"net/http/httptest"
	7	+	"strings"
	8	+	"sync/atomic"
	9	+	"testing"
	10	+	)
	11	+
	12	+	// TestOpenAIStreamRetriesOn503 verifies the OpenAI/Kimi stream client
	13	+	// silently retries when the upstream returns 503 a couple of times,
	14	+	// then succeeds. The user-visible error path should NOT fire because
	15	+	// a later attempt landed on a 200.
	16	+	func TestOpenAIStreamRetriesOn503(t *testing.T) {
	17	+	var attempts atomic.Int32
	18	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	19	+	n := attempts.Add(1)
	20	+	if n < 3 {
	21	+	w.WriteHeader(http.StatusServiceUnavailable)
	22	+	_, _ = w.Write([]byte("upstream connect error"))
	23	+	return
	24	+	}
	25	+	w.Header().Set("content-type", "text/event-stream")
	26	+	w.WriteHeader(http.StatusOK)
	27	+	fl, _ := w.(http.Flusher)
	28	+	write := func(s string) {
	29	+	_, _ = w.Write([]byte(s))
	30	+	if fl != nil {
	31	+	fl.Flush()
	32	+	}
	33	+	}
	34	+	write("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\n")
	35	+	write("data: [DONE]\n\n")
	36	+	}))
	37	+	defer srv.Close()
	38	+
	39	+	c := NewOpenAI("x", srv.URL)
	40	+	evs, err := c.Stream(context.Background(), Request{Model: "gpt-5"})
	41	+	if err != nil {
	42	+	t.Fatalf("stream returned error: %v", err)
	43	+	}
	44	+	gotText := ""
	45	+	for ev := range evs {
	46	+	if td, ok := ev.(EventTextDelta); ok {
	47	+	gotText += td.Delta
	48	+	}
	49	+	}
	50	+	if gotText != "ok" {
	51	+	t.Fatalf("text=%q", gotText)
	52	+	}
	53	+	if got := attempts.Load(); got != 3 {
	54	+	t.Fatalf("attempts=%d want 3", got)
	55	+	}
	56	+	}
	57	+
	58	+	// TestOpenAIStreamSurfaces503AfterRetriesExhausted verifies that when
	59	+	// every retry attempt returns 503 the caller sees a normal "http 503"
	60	+	// error, not a retry-internal placeholder. The body bytes must be
	61	+	// preserved so the rescue picker / red banner can show them.
	62	+	func TestOpenAIStreamSurfaces503AfterRetriesExhausted(t *testing.T) {
	63	+	var attempts atomic.Int32
	64	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	65	+	attempts.Add(1)
	66	+	w.WriteHeader(http.StatusServiceUnavailable)
	67	+	_, _ = w.Write([]byte("upstream connect error"))
	68	+	}))
	69	+	defer srv.Close()
	70	+
	71	+	c := NewOpenAI("x", srv.URL)
	72	+	_, err := c.Stream(context.Background(), Request{Model: "gpt-5"})
	73	+	if err == nil {
	74	+	t.Fatalf("expected error after exhausted retries")
	75	+	}
	76	+	if !strings.Contains(err.Error(), "503") {
	77	+	t.Fatalf("error %q should mention 503", err)
	78	+	}
	79	+	if got := attempts.Load(); got != streamRetryAttempts+1 {
	80	+	t.Fatalf("attempts=%d want %d", got, streamRetryAttempts+1)
	81	+	}
	82	+	}
	83	+
	84	+	// TestOpenAIStreamDoesNotRetryOn400 ensures non-transient errors are
	85	+	// returned immediately without retrying. We don't want to amplify
	86	+	// load on a provider that's actively rejecting our request shape.
	87	+	func TestOpenAIStreamDoesNotRetryOn400(t *testing.T) {
	88	+	var attempts atomic.Int32
	89	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	90	+	attempts.Add(1)
	91	+	w.WriteHeader(http.StatusBadRequest)
	92	+	_, _ = w.Write([]byte(`{"error":{"message":"nope"}}`))
	93	+	}))
	94	+	defer srv.Close()
	95	+
	96	+	c := NewOpenAI("x", srv.URL)
	97	+	_, err := c.Stream(context.Background(), Request{Model: "gpt-5"})
	98	+	if err == nil \|\| !strings.Contains(err.Error(), "400") {
	99	+	t.Fatalf("want 400 err got %v", err)
	100	+	}
	101	+	if got := attempts.Load(); got != 1 {
	102	+	t.Fatalf("attempts=%d want 1", got)
	103	+	}
	104	+	}
	105	+
	106	+	func TestIsTransientHTTPStatus(t *testing.T) {
	107	+	for _, code := range []int{502, 503, 504} {
	108	+	if !isTransientHTTPStatus(code) {
	109	+	t.Errorf("%d should be transient", code)
	110	+	}
	111	+	}
	112	+	for _, code := range []int{200, 400, 401, 403, 404, 429, 500, 501} {
	113	+	if isTransientHTTPStatus(code) {
	114	+	t.Errorf("%d should NOT be transient", code)
	115	+	}
	116	+	}
	117	+	}
	118	+
	119	+	func TestIsTransientConnectError(t *testing.T) {
	120	+	good := []string{
	121	+	"read tcp: connection reset by peer",
	122	+	"dial tcp: connection refused",
	123	+	"unexpected EOF",
	124	+	"upstream connect error or disconnect/reset before headers",
	125	+	"tls handshake error: bad cert",
	126	+	"i/o timeout",
	127	+	}
	128	+	for _, m := range good {
	129	+	if !isTransientConnectError(stringErr(m)) {
	130	+	t.Errorf("%q should be transient", m)
	131	+	}
	132	+	}
	133	+	bad := []string{
	134	+	"context canceled",
	135	+	"json: cannot unmarshal",
	136	+	}
	137	+	for _, m := range bad {
	138	+	if isTransientConnectError(stringErr(m)) {
	139	+	t.Errorf("%q should NOT be transient", m)
	140	+	}
	141	+	}
	142	+	}
	143	+
	144	+	type stringErr string
	145	+
	146	+	func (s stringErr) Error() string { return string(s) }

@@ -0,0 +1,146 @@
	1	+	package provider
	2	+
	3	+	import (
	4	+	"context"
	5	+	"net/http"
	6	+	"net/http/httptest"
	7	+	"strings"
	8	+	"sync/atomic"
	9	+	"testing"
	10	+	)
	11	+
	12	+	// TestOpenAIStreamRetriesOn503 verifies the OpenAI/Kimi stream client
	13	+	// silently retries when the upstream returns 503 a couple of times,
	14	+	// then succeeds. The user-visible error path should NOT fire because
	15	+	// a later attempt landed on a 200.
	16	+	func TestOpenAIStreamRetriesOn503(t *testing.T) {
	17	+	var attempts atomic.Int32
	18	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	19	+	n := attempts.Add(1)
	20	+	if n < 3 {
	21	+	w.WriteHeader(http.StatusServiceUnavailable)
	22	+	_, _ = w.Write([]byte("upstream connect error"))
	23	+	return
	24	+	}
	25	+	w.Header().Set("content-type", "text/event-stream")
	26	+	w.WriteHeader(http.StatusOK)
	27	+	fl, _ := w.(http.Flusher)
	28	+	write := func(s string) {
	29	+	_, _ = w.Write([]byte(s))
	30	+	if fl != nil {
	31	+	fl.Flush()
	32	+	}
	33	+	}
	34	+	write("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\n")
	35	+	write("data: [DONE]\n\n")
	36	+	}))
	37	+	defer srv.Close()
	38	+
	39	+	c := NewOpenAI("x", srv.URL)
	40	+	evs, err := c.Stream(context.Background(), Request{Model: "gpt-5"})
	41	+	if err != nil {
	42	+	t.Fatalf("stream returned error: %v", err)
	43	+	}
	44	+	gotText := ""
	45	+	for ev := range evs {
	46	+	if td, ok := ev.(EventTextDelta); ok {
	47	+	gotText += td.Delta
	48	+	}
	49	+	}
	50	+	if gotText != "ok" {
	51	+	t.Fatalf("text=%q", gotText)
	52	+	}
	53	+	if got := attempts.Load(); got != 3 {
	54	+	t.Fatalf("attempts=%d want 3", got)
	55	+	}
	56	+	}
	57	+
	58	+	// TestOpenAIStreamSurfaces503AfterRetriesExhausted verifies that when
	59	+	// every retry attempt returns 503 the caller sees a normal "http 503"
	60	+	// error, not a retry-internal placeholder. The body bytes must be
	61	+	// preserved so the rescue picker / red banner can show them.
	62	+	func TestOpenAIStreamSurfaces503AfterRetriesExhausted(t *testing.T) {
	63	+	var attempts atomic.Int32
	64	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	65	+	attempts.Add(1)
	66	+	w.WriteHeader(http.StatusServiceUnavailable)
	67	+	_, _ = w.Write([]byte("upstream connect error"))
	68	+	}))
	69	+	defer srv.Close()
	70	+
	71	+	c := NewOpenAI("x", srv.URL)
	72	+	_, err := c.Stream(context.Background(), Request{Model: "gpt-5"})
	73	+	if err == nil {
	74	+	t.Fatalf("expected error after exhausted retries")
	75	+	}
	76	+	if !strings.Contains(err.Error(), "503") {
	77	+	t.Fatalf("error %q should mention 503", err)
	78	+	}
	79	+	if got := attempts.Load(); got != streamRetryAttempts+1 {
	80	+	t.Fatalf("attempts=%d want %d", got, streamRetryAttempts+1)
	81	+	}
	82	+	}
	83	+
	84	+	// TestOpenAIStreamDoesNotRetryOn400 ensures non-transient errors are
	85	+	// returned immediately without retrying. We don't want to amplify
	86	+	// load on a provider that's actively rejecting our request shape.
	87	+	func TestOpenAIStreamDoesNotRetryOn400(t *testing.T) {
	88	+	var attempts atomic.Int32
	89	+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	90	+	attempts.Add(1)
	91	+	w.WriteHeader(http.StatusBadRequest)
	92	+	_, _ = w.Write([]byte(`{"error":{"message":"nope"}}`))
	93	+	}))
	94	+	defer srv.Close()
	95	+
	96	+	c := NewOpenAI("x", srv.URL)
	97	+	_, err := c.Stream(context.Background(), Request{Model: "gpt-5"})
	98	+	if err == nil \|\| !strings.Contains(err.Error(), "400") {
	99	+	t.Fatalf("want 400 err got %v", err)
	100	+	}
	101	+	if got := attempts.Load(); got != 1 {
	102	+	t.Fatalf("attempts=%d want 1", got)
	103	+	}
	104	+	}
	105	+
	106	+	func TestIsTransientHTTPStatus(t *testing.T) {
	107	+	for _, code := range []int{502, 503, 504} {
	108	+	if !isTransientHTTPStatus(code) {
	109	+	t.Errorf("%d should be transient", code)
	110	+	}
	111	+	}
	112	+	for _, code := range []int{200, 400, 401, 403, 404, 429, 500, 501} {
	113	+	if isTransientHTTPStatus(code) {
	114	+	t.Errorf("%d should NOT be transient", code)
	115	+	}
	116	+	}
	117	+	}
	118	+
	119	+	func TestIsTransientConnectError(t *testing.T) {
	120	+	good := []string{
	121	+	"read tcp: connection reset by peer",
	122	+	"dial tcp: connection refused",
	123	+	"unexpected EOF",
	124	+	"upstream connect error or disconnect/reset before headers",
	125	+	"tls handshake error: bad cert",
	126	+	"i/o timeout",
	127	+	}
	128	+	for _, m := range good {
	129	+	if !isTransientConnectError(stringErr(m)) {
	130	+	t.Errorf("%q should be transient", m)
	131	+	}
	132	+	}
	133	+	bad := []string{
	134	+	"context canceled",
	135	+	"json: cannot unmarshal",
	136	+	}
	137	+	for _, m := range bad {
	138	+	if isTransientConnectError(stringErr(m)) {
	139	+	t.Errorf("%q should NOT be transient", m)
	140	+	}
	141	+	}
	142	+	}
	143	+
	144	+	type stringErr string
	145	+
	146	+	func (s stringErr) Error() string { return string(s) }