Retry Codex server capacity errors
- commit
- a2cd138cb3441062216614e8ead001bc29404d7a
- author
- patriceckhart · · 2026-07-30 09:22 UTC
- parent
- b289bf6
- signature
- unsigned
- refs
- tag: v0.3.27
2 files · +67 -16
M packages/core/agent.go +6 -0
| @@ -478,6 +478,12 @@ func (a *Agent) canRetryError(err error, attempt int) bool { | |||
| 478 | 478 | // Explicit retry guidance emitted by provider backends (OpenAI | |
| 479 | 479 | // Responses, AWS Bedrock stream exceptions) with varying prefixes. | |
| 480 | 480 | "you can retry your request", "try your request again", "please retry your request", | |
| 481 | + | // Capacity messages from the ChatGPT/Codex backend, e.g. | |
| 482 | + | // "Our servers are currently overloaded. Please try again later." | |
| 483 | + | // The trailing advice also shows up on its own for transient | |
| 484 | + | // capacity failures; usage/quota limits are filtered out above by | |
| 485 | + | // isNonRetryableProviderLimit before this list is consulted. | |
| 486 | + | "servers are currently overloaded", "servers are busy", "try again later", | |
| 481 | 487 | } | |
| 482 | 488 | for _, needle := range needles { | |
| 483 | 489 | if strings.Contains(msg, needle) { | |
| @@ -478,6 +478,12 @@ func (a *Agent) canRetryError(err error, attempt int) bool { | |||||
| 478 | // Explicit retry guidance emitted by provider backends (OpenAI | 478 | // Explicit retry guidance emitted by provider backends (OpenAI | ||
| 479 | // Responses, AWS Bedrock stream exceptions) with varying prefixes. | 479 | // Responses, AWS Bedrock stream exceptions) with varying prefixes. | ||
| 480 | "you can retry your request", "try your request again", "please retry your request", | 480 | "you can retry your request", "try your request again", "please retry your request", | ||
| 481 | + | // Capacity messages from the ChatGPT/Codex backend, e.g. | |||
| 482 | + | // "Our servers are currently overloaded. Please try again later." | |||
| 483 | + | // The trailing advice also shows up on its own for transient | |||
| 484 | + | // capacity failures; usage/quota limits are filtered out above by | |||
| 485 | + | // isNonRetryableProviderLimit before this list is consulted. | |||
| 486 | + | "servers are currently overloaded", "servers are busy", "try again later", | |||
| 481 | } | 487 | } | ||
| 482 | for _, needle := range needles { | 488 | for _, needle := range needles { | ||
| 483 | if strings.Contains(msg, needle) { | 489 | if strings.Contains(msg, needle) { | ||
M packages/core/agent_retry_test.go +61 -16
| @@ -2,6 +2,7 @@ package core | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "errors" | |
| 5 | 6 | "fmt" | |
| 6 | 7 | "strings" | |
| 7 | 8 | "sync/atomic" | |
| @@ -65,11 +66,11 @@ func TestAgentRetriesOverloadedStreamError(t *testing.T) { | |||
| 65 | 66 | } | |
| 66 | 67 | } | |
| 67 | 68 | ||
| 68 | - | // codexRetryFakeClient reproduces the generic OpenAI Codex backend | |
| 69 | - | // failure ("An error occurred while processing your request. You can | |
| 70 | - | // retry your request, ...") on the first call and succeeds afterwards. | |
| 69 | + | // codexRetryFakeClient reproduces a transient OpenAI Codex backend | |
| 70 | + | // failure on the first call and succeeds afterwards. | |
| 71 | 71 | type codexRetryFakeClient struct { | |
| 72 | - | calls int32 | |
| 72 | + | calls int32 | |
| 73 | + | firstErr string | |
| 73 | 74 | } | |
| 74 | 75 | ||
| 75 | 76 | func (c *codexRetryFakeClient) Name() string { return "openai-codex" } | |
| @@ -81,7 +82,7 @@ func (c *codexRetryFakeClient) Stream(ctx context.Context, req provider.Request) | |||
| 81 | 82 | defer close(out) | |
| 82 | 83 | out <- provider.EventStart{Provider: "openai-codex", Model: req.Model} | |
| 83 | 84 | if call == 1 { | |
| 84 | - | out <- provider.EventDone{Stop: provider.StopError, Err: fmt.Errorf("codex error: An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID 60c8ebbd-20bd-42e4-b756-6e844041cfc0 in your message.")} | |
| 85 | + | out <- provider.EventDone{Stop: provider.StopError, Err: fmt.Errorf("%s", c.firstErr)} | |
| 85 | 86 | return | |
| 86 | 87 | } | |
| 87 | 88 | out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{ | |
| @@ -93,19 +94,63 @@ func (c *codexRetryFakeClient) Stream(ctx context.Context, req provider.Request) | |||
| 93 | 94 | } | |
| 94 | 95 | ||
| 95 | 96 | func TestAgentRetriesCodexProcessingError(t *testing.T) { | |
| 96 | - | client := &codexRetryFakeClient{} | |
| 97 | - | a := NewAgent(client, "gpt-5.6-sol", "system", Registry{}) | |
| 98 | - | a.RetryBaseDelay = time.Millisecond | |
| 99 | - | ||
| 100 | - | if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil { | |
| 101 | - | t.Fatalf("Prompt returned %v; want retry to succeed", err) | |
| 97 | + | cases := []struct { | |
| 98 | + | name string | |
| 99 | + | err string | |
| 100 | + | }{ | |
| 101 | + | { | |
| 102 | + | name: "processing error", | |
| 103 | + | err: "codex error: An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID 60c8ebbd-20bd-42e4-b756-6e844041cfc0 in your message.", | |
| 104 | + | }, | |
| 105 | + | { | |
| 106 | + | name: "servers overloaded", | |
| 107 | + | err: "codex error: Our servers are currently overloaded. Please try again later.", | |
| 108 | + | }, | |
| 109 | + | { | |
| 110 | + | name: "try again later only", | |
| 111 | + | err: "codex error: Something went wrong. Please try again later.", | |
| 112 | + | }, | |
| 102 | 113 | } | |
| 103 | - | if got := atomic.LoadInt32(&client.calls); got != 2 { | |
| 104 | - | t.Fatalf("Stream calls = %d; want 2", got) | |
| 114 | + | for _, tc := range cases { | |
| 115 | + | t.Run(tc.name, func(t *testing.T) { | |
| 116 | + | client := &codexRetryFakeClient{firstErr: tc.err} | |
| 117 | + | a := NewAgent(client, "gpt-5.6-sol", "system", Registry{}) | |
| 118 | + | a.RetryBaseDelay = time.Millisecond | |
| 119 | + | ||
| 120 | + | if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil { | |
| 121 | + | t.Fatalf("Prompt returned %v; want retry to succeed", err) | |
| 122 | + | } | |
| 123 | + | if got := atomic.LoadInt32(&client.calls); got != 2 { | |
| 124 | + | t.Fatalf("Stream calls = %d; want 2", got) | |
| 125 | + | } | |
| 126 | + | msgs := a.Messages() | |
| 127 | + | if len(msgs) != 2 || extractText(msgs[1]) != "ok" { | |
| 128 | + | t.Fatalf("messages = %v; want user + ok assistant", msgs) | |
| 129 | + | } | |
| 130 | + | }) | |
| 105 | 131 | } | |
| 106 | - | msgs := a.Messages() | |
| 107 | - | if len(msgs) != 2 || extractText(msgs[1]) != "ok" { | |
| 108 | - | t.Fatalf("messages = %v; want user + ok assistant", msgs) | |
| 132 | + | } | |
| 133 | + | ||
| 134 | + | // TestCanRetryErrorCapacityMessages pins the classification of Codex | |
| 135 | + | // capacity wording and makes sure quota/usage limits stay terminal even | |
| 136 | + | // when they carry "try again later" style advice. | |
| 137 | + | func TestCanRetryErrorCapacityMessages(t *testing.T) { | |
| 138 | + | a := NewAgent(nil, "gpt-5.6-sol", "system", Registry{}) | |
| 139 | + | cases := []struct { | |
| 140 | + | msg string | |
| 141 | + | want bool | |
| 142 | + | }{ | |
| 143 | + | {"codex error: Our servers are currently overloaded. Please try again later.", true}, | |
| 144 | + | {"codex error: Our servers are busy right now.", true}, | |
| 145 | + | {"codex error: Please try again later.", true}, | |
| 146 | + | {"codex error: You have hit your monthly usage limit. Try again later.", false}, | |
| 147 | + | {"codex error: quota exceeded, try again later", false}, | |
| 148 | + | {"codex error: unsupported parameter: reasoning", false}, | |
| 149 | + | } | |
| 150 | + | for _, tc := range cases { | |
| 151 | + | if got := a.canRetryError(errors.New(tc.msg), 0); got != tc.want { | |
| 152 | + | t.Errorf("canRetryError(%q) = %v; want %v", tc.msg, got, tc.want) | |
| 153 | + | } | |
| 109 | 154 | } | |
| 110 | 155 | } | |
| 111 | 156 | ||
| @@ -2,6 +2,7 @@ package core | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "errors" | |||
| 5 | "fmt" | 6 | "fmt" | ||
| 6 | "strings" | 7 | "strings" | ||
| 7 | "sync/atomic" | 8 | "sync/atomic" | ||
| @@ -65,11 +66,11 @@ func TestAgentRetriesOverloadedStreamError(t *testing.T) { | |||||
| 65 | } | 66 | } | ||
| 66 | } | 67 | } | ||
| 67 | 68 | ||||
| 68 | - | // codexRetryFakeClient reproduces the generic OpenAI Codex backend | 69 | + | // codexRetryFakeClient reproduces a transient OpenAI Codex backend |
| 69 | - | // failure ("An error occurred while processing your request. You can | 70 | + | // failure on the first call and succeeds afterwards. |
| 70 | - | // retry your request, ...") on the first call and succeeds afterwards. | |||
| 71 | type codexRetryFakeClient struct { | 71 | type codexRetryFakeClient struct { | ||
| 72 | - | calls int32 | 72 | + | calls int32 |
| 73 | + | firstErr string | |||
| 73 | } | 74 | } | ||
| 74 | 75 | ||||
| 75 | func (c *codexRetryFakeClient) Name() string { return "openai-codex" } | 76 | func (c *codexRetryFakeClient) Name() string { return "openai-codex" } | ||
| @@ -81,7 +82,7 @@ func (c *codexRetryFakeClient) Stream(ctx context.Context, req provider.Request) | |||||
| 81 | defer close(out) | 82 | defer close(out) | ||
| 82 | out <- provider.EventStart{Provider: "openai-codex", Model: req.Model} | 83 | out <- provider.EventStart{Provider: "openai-codex", Model: req.Model} | ||
| 83 | if call == 1 { | 84 | if call == 1 { | ||
| 84 | - | out <- provider.EventDone{Stop: provider.StopError, Err: fmt.Errorf("codex error: An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID 60c8ebbd-20bd-42e4-b756-6e844041cfc0 in your message.")} | 85 | + | out <- provider.EventDone{Stop: provider.StopError, Err: fmt.Errorf("%s", c.firstErr)} |
| 85 | return | 86 | return | ||
| 86 | } | 87 | } | ||
| 87 | out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{ | 88 | out <- provider.EventDone{Stop: provider.StopEnd, Message: provider.Message{ | ||
| @@ -93,19 +94,63 @@ func (c *codexRetryFakeClient) Stream(ctx context.Context, req provider.Request) | |||||
| 93 | } | 94 | } | ||
| 94 | 95 | ||||
| 95 | func TestAgentRetriesCodexProcessingError(t *testing.T) { | 96 | func TestAgentRetriesCodexProcessingError(t *testing.T) { | ||
| 96 | - | client := &codexRetryFakeClient{} | 97 | + | cases := []struct { |
| 97 | - | a := NewAgent(client, "gpt-5.6-sol", "system", Registry{}) | 98 | + | name string |
| 98 | - | a.RetryBaseDelay = time.Millisecond | 99 | + | err string |
| 99 | - | 100 | + | }{ | |
| 100 | - | if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil { | 101 | + | { |
| 101 | - | t.Fatalf("Prompt returned %v; want retry to succeed", err) | 102 | + | name: "processing error", |
| 103 | + | err: "codex error: An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID 60c8ebbd-20bd-42e4-b756-6e844041cfc0 in your message.", | |||
| 104 | + | }, | |||
| 105 | + | { | |||
| 106 | + | name: "servers overloaded", | |||
| 107 | + | err: "codex error: Our servers are currently overloaded. Please try again later.", | |||
| 108 | + | }, | |||
| 109 | + | { | |||
| 110 | + | name: "try again later only", | |||
| 111 | + | err: "codex error: Something went wrong. Please try again later.", | |||
| 112 | + | }, | |||
| 102 | } | 113 | } | ||
| 103 | - | if got := atomic.LoadInt32(&client.calls); got != 2 { | 114 | + | for _, tc := range cases { |
| 104 | - | t.Fatalf("Stream calls = %d; want 2", got) | 115 | + | t.Run(tc.name, func(t *testing.T) { |
| 116 | + | client := &codexRetryFakeClient{firstErr: tc.err} | |||
| 117 | + | a := NewAgent(client, "gpt-5.6-sol", "system", Registry{}) | |||
| 118 | + | a.RetryBaseDelay = time.Millisecond | |||
| 119 | + | ||||
| 120 | + | if err := a.Prompt(context.Background(), "hello", nil, nil); err != nil { | |||
| 121 | + | t.Fatalf("Prompt returned %v; want retry to succeed", err) | |||
| 122 | + | } | |||
| 123 | + | if got := atomic.LoadInt32(&client.calls); got != 2 { | |||
| 124 | + | t.Fatalf("Stream calls = %d; want 2", got) | |||
| 125 | + | } | |||
| 126 | + | msgs := a.Messages() | |||
| 127 | + | if len(msgs) != 2 || extractText(msgs[1]) != "ok" { | |||
| 128 | + | t.Fatalf("messages = %v; want user + ok assistant", msgs) | |||
| 129 | + | } | |||
| 130 | + | }) | |||
| 105 | } | 131 | } | ||
| 106 | - | msgs := a.Messages() | 132 | + | } |
| 107 | - | if len(msgs) != 2 || extractText(msgs[1]) != "ok" { | 133 | + | |
| 108 | - | t.Fatalf("messages = %v; want user + ok assistant", msgs) | 134 | + | // TestCanRetryErrorCapacityMessages pins the classification of Codex |
| 135 | + | // capacity wording and makes sure quota/usage limits stay terminal even | |||
| 136 | + | // when they carry "try again later" style advice. | |||
| 137 | + | func TestCanRetryErrorCapacityMessages(t *testing.T) { | |||
| 138 | + | a := NewAgent(nil, "gpt-5.6-sol", "system", Registry{}) | |||
| 139 | + | cases := []struct { | |||
| 140 | + | msg string | |||
| 141 | + | want bool | |||
| 142 | + | }{ | |||
| 143 | + | {"codex error: Our servers are currently overloaded. Please try again later.", true}, | |||
| 144 | + | {"codex error: Our servers are busy right now.", true}, | |||
| 145 | + | {"codex error: Please try again later.", true}, | |||
| 146 | + | {"codex error: You have hit your monthly usage limit. Try again later.", false}, | |||
| 147 | + | {"codex error: quota exceeded, try again later", false}, | |||
| 148 | + | {"codex error: unsupported parameter: reasoning", false}, | |||
| 149 | + | } | |||
| 150 | + | for _, tc := range cases { | |||
| 151 | + | if got := a.canRetryError(errors.New(tc.msg), 0); got != tc.want { | |||
| 152 | + | t.Errorf("canRetryError(%q) = %v; want %v", tc.msg, got, tc.want) | |||
| 153 | + | } | |||
| 109 | } | 154 | } | ||
| 110 | } | 155 | } | ||
| 111 | 156 | ||||