From 43330a142c12fb600da0f5f2816e557e443d65c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 17 Nov 2026 11:14:08 +0100 Subject: [PATCH 1/5] feat: configurable retries; an empty answer is retried instead of ending the turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One RetryPolicy (llm::retry, retry.json, Settings → General) bounds the retries of a failed LLM request — at most N or until it succeeds — and the wait between them. The clients use it for rate limits and transient errors, the agent loop for broken or stalled streams. An answer without content ended the turn as if the model had answered: nothing was recorded, the history still ended in the user's message, and the session showed Resume without an error. The agent loop now treats it as a failed request (EmptyResponse) and retries it under the same policy. --- crates/agent_core/src/hooks.rs | 2 +- crates/agent_core/src/runtime.rs | 51 +++-- crates/agent_core/src/runtime/tests.rs | 38 ++++ crates/agent_core/src/types.rs | 7 + crates/agent_core/src/ui.rs | 2 +- crates/code_assistant_core/src/agent/tests.rs | 10 +- crates/code_assistant_core/src/plugins/mod.rs | 2 +- .../src/plugins/recovery.rs | 89 +++++++-- .../src/session/instance.rs | 12 +- crates/code_assistant_core/src/ui/mod.rs | 4 +- crates/llm/src/aicore/anthropic.rs | 4 + crates/llm/src/aicore/openai.rs | 4 + crates/llm/src/aicore/openai_responses.rs | 4 + crates/llm/src/aicore/vertex.rs | 4 + crates/llm/src/anthropic.rs | 15 +- crates/llm/src/cerebras.rs | 4 + crates/llm/src/factory.rs | 7 +- crates/llm/src/groq.rs | 4 + crates/llm/src/lib.rs | 10 +- crates/llm/src/minimax.rs | 4 + crates/llm/src/mistralai.rs | 4 + crates/llm/src/moonshot.rs | 4 + crates/llm/src/openai.rs | 14 +- crates/llm/src/openai_responses.rs | 14 +- crates/llm/src/openrouter.rs | 4 + crates/llm/src/retry.rs | 176 ++++++++++++++++++ crates/llm/src/utils.rs | 163 ++++++++-------- crates/llm/src/vertex.rs | 15 +- crates/llm/src/zai.rs | 4 + .../src/messages/activity_indicator.rs | 5 +- .../src/settings_screen/general_section.rs | 162 ++++++++++++++++ crates/ui_terminal/src/renderer.rs | 9 +- crates/ui_terminal/src/ui.rs | 7 +- 33 files changed, 693 insertions(+), 165 deletions(-) create mode 100644 crates/llm/src/retry.rs diff --git a/crates/agent_core/src/hooks.rs b/crates/agent_core/src/hooks.rs index 889d91f23..d6d688457 100644 --- a/crates/agent_core/src/hooks.rs +++ b/crates/agent_core/src/hooks.rs @@ -151,7 +151,7 @@ pub enum RecoveryAction { RetryStream { delay: Duration, attempt: u32, - max_attempts: u32, + max_attempts: Option, }, /// Not recoverable; propagate the error. Fail, diff --git a/crates/agent_core/src/runtime.rs b/crates/agent_core/src/runtime.rs index b45d5957e..15c4aa672 100644 --- a/crates/agent_core/src/runtime.rs +++ b/crates/agent_core/src/runtime.rs @@ -428,10 +428,16 @@ impl AgentRuntime { let reserved_node_id = self.reserve_node_id(); // 1. Get LLM response (without adding to history yet) - let (llm_response, request_id) = match self + let response = self .get_next_assistant_message(messages, reserved_node_id) .await - { + .and_then(|(response, request_id)| { + if response.content.is_empty() { + return Err(crate::types::EmptyResponse.into()); + } + Ok((response, request_id)) + }); + let (llm_response, request_id) = match response { Ok(result) => { // Successful response — reset the retry counters streaming_retry_count = 0; @@ -476,14 +482,12 @@ impl AgentRuntime { self.cancellation.check()?; // 2. Add original LLM response to message history using the pre-allocated node_id - if !llm_response.content.is_empty() { - self.append_message_with_node_id( - Message::new_assistant_content(llm_response.content.clone()) - .with_request_id(request_id) - .with_usage(llm_response.usage.clone()), - reserved_node_id, - )?; - } + self.append_message_with_node_id( + Message::new_assistant_content(llm_response.content.clone()) + .with_request_id(request_id) + .with_usage(llm_response.usage.clone()), + reserved_node_id, + )?; // 3. Extract tool requests from LLM response and get truncated response let (tool_requests, flow, truncated_response) = self @@ -1155,37 +1159,32 @@ impl AgentRuntime { Ok(()) } - /// Informs the user that a transient streaming failure is being retried and - /// tells the UI to discard all partial content from the failed request. + /// Tells the user that a failed request is sent again, and the UI to + /// discard all partial content from it. async fn prepare_streaming_retry( &self, error: &anyhow::Error, attempt: u32, - max_attempts: u32, + max_attempts: Option, delay: std::time::Duration, ) { warn!( - "Transient streaming error (attempt {}/{}), retrying in {:?}: {}", - attempt, max_attempts, delay, error + "LLM request failed (retry {}), retrying in {:?}: {:#}", + llm::retry::attempt_label(attempt, max_attempts), + delay, + error ); - // get_next_assistant_message already sent StreamingStopped{error: ...} for - // the failed request, so the UI knows streaming ended. Now we tell it to + // get_next_assistant_message already sent StreamingStopped for the + // failed request, so the UI knows streaming ended. Now we tell it to // also remove whatever was already rendered. let _ = self .send_ui(AgentUiEvent::RollbackStreaming { request_id: self.next_request_id - 1, }) .await; - - let _ = self - .send_ui(AgentUiEvent::ShowTransientStatus { - message: format!( - "Stream interrupted — retrying ({}/{})\u{2026}", - attempt, max_attempts - ), - }) - .await; + self.ui + .notify_retrying(&error.to_string(), attempt, max_attempts); } /// Project the largest results from the most recent turn as small error diff --git a/crates/agent_core/src/runtime/tests.rs b/crates/agent_core/src/runtime/tests.rs index 1442f1def..c1778ea57 100644 --- a/crates/agent_core/src/runtime/tests.rs +++ b/crates/agent_core/src/runtime/tests.rs @@ -766,3 +766,41 @@ fn a_message_queued_after_compaction_joins_the_handoff() { assert!(texts[0].starts_with(""), "{texts:?}"); assert_eq!(texts[1], "Also this"); } + +/// Retries every failed request at once, as often as asked. +struct RetryAlways; + +impl RecoveryPolicy for RetryAlways { + fn classify(&self, _: &anyhow::Error, completed_retries: u32) -> RecoveryAction { + RecoveryAction::RetryStream { + delay: std::time::Duration::ZERO, + attempt: completed_retries + 1, + max_attempts: None, + } + } +} + +#[tokio::test] +async fn an_empty_answer_is_a_failure_not_the_end_of_the_turn() { + let (mut runtime, llm) = runtime_answering(vec![vec![]]); + runtime.append_message(Message::new_user("Hi")).unwrap(); + + let error = runtime.run_single_iteration().await.unwrap_err(); + + assert!(error.is::(), "{error:#}"); + assert_eq!(llm.requests().len(), 1); +} + +#[tokio::test] +async fn an_empty_answer_is_retried_under_the_recovery_policy() { + let (mut runtime, llm) = runtime_answering(vec![vec![], vec![ContentBlock::new_text("Hello")]]); + runtime.hooks.recovery = Box::new(RetryAlways); + runtime.append_message(Message::new_user("Hi")).unwrap(); + + runtime.run_single_iteration().await.unwrap(); + + assert_eq!(llm.requests().len(), 2); + let messages: Vec<_> = runtime.conversation.active_messages().collect(); + assert_eq!(messages.len(), 2, "the empty answer is not recorded"); + assert_eq!(messages[1].role, MessageRole::Assistant); +} diff --git a/crates/agent_core/src/types.rs b/crates/agent_core/src/types.rs index 2f45fe531..1819d39a4 100644 --- a/crates/agent_core/src/types.rs +++ b/crates/agent_core/src/types.rs @@ -86,6 +86,13 @@ impl ToolResult for ParseError { } } +/// The model answered with nothing: no text, no reasoning, no tool call. +/// Ending the turn there would leave the request unanswered and the session +/// waiting for a resume, so the agent loop treats it as a failed request. +#[derive(Debug, thiserror::Error)] +#[error("The model returned an empty response")] +pub struct EmptyResponse; + /// Placeholder result used when a tool's output was too large and caused a /// "prompt too long" error from the LLM provider. The original tool execution /// is replaced with this so the LLM gets actionable feedback on the next turn. diff --git a/crates/agent_core/src/ui.rs b/crates/agent_core/src/ui.rs index 242bf9317..af6483a87 100644 --- a/crates/agent_core/src/ui.rs +++ b/crates/agent_core/src/ui.rs @@ -227,7 +227,7 @@ pub trait AgentUi: Send + Sync { /// The LLM request failed before anything was streamed and is sent /// again (`attempt` of `max_attempts`). Cleared by the next response. - fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: u32) {} + fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: Option) {} } /// Predicate deciding whether a tool's invocation is hidden from the UI. diff --git a/crates/code_assistant_core/src/agent/tests.rs b/crates/code_assistant_core/src/agent/tests.rs index d8311fd27..884b0b4bf 100644 --- a/crates/code_assistant_core/src/agent/tests.rs +++ b/crates/code_assistant_core/src/agent/tests.rs @@ -524,7 +524,7 @@ async fn test_context_compaction_inserts_summary() -> Result<()> { }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -667,7 +667,7 @@ async fn test_compaction_reminds_about_active_skills() -> Result<()> { rate_limit_info: None, }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -764,7 +764,7 @@ async fn test_compaction_prompt_not_persisted_in_history() -> Result<()> { }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -889,7 +889,7 @@ async fn test_context_compaction_uses_only_messages_after_previous_summary() -> }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -2575,7 +2575,7 @@ fn text_response(text: &str) -> LLMResponse { fn idle_response() -> LLMResponse { LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, } diff --git a/crates/code_assistant_core/src/plugins/mod.rs b/crates/code_assistant_core/src/plugins/mod.rs index 233a378d4..a1ebf7ce3 100644 --- a/crates/code_assistant_core/src/plugins/mod.rs +++ b/crates/code_assistant_core/src/plugins/mod.rs @@ -36,7 +36,7 @@ pub fn default_hooks() -> HookRegistry { observers: vec![], dispatch: Box::new(SpawnAgentParallelPolicy), compaction: Box::new(TokenRatioCompaction::new(0.8)), - recovery: Box::new(DefaultRecovery), + recovery: Box::new(DefaultRecovery::new(llm::RetryPolicy::load())), system_prompt: Box::new(CodeAssistantSystemPrompt::new()), } } diff --git a/crates/code_assistant_core/src/plugins/recovery.rs b/crates/code_assistant_core/src/plugins/recovery.rs index fa8791421..116aeb0fe 100644 --- a/crates/code_assistant_core/src/plugins/recovery.rs +++ b/crates/code_assistant_core/src/plugins/recovery.rs @@ -1,16 +1,25 @@ //! Recovery policy: classifies failed LLM requests by provider error text. use agent_core::hooks::{RecoveryAction, RecoveryPolicy}; +use agent_core::types::EmptyResponse; +use llm::RetryPolicy; use std::time::Duration; -/// Maximum number of retries for transient streaming errors (e.g. HTTP chunk -/// errors, connection resets). -const MAX_STREAMING_RETRIES: u32 = 2; +/// Wait before the first retry of a broken stream or an empty answer +/// (doubles on each further one, up to the policy's maximum). +const RETRY_BASE_DELAY: Duration = Duration::from_secs(2); -/// Base delay between streaming retries (doubles on each attempt). -const STREAMING_RETRY_BASE_DELAY: Duration = Duration::from_secs(2); +/// Recovers from oversized prompts by compacting, and sends a request again +/// after a broken stream or an empty answer, as often as `policy` allows. +pub struct DefaultRecovery { + policy: RetryPolicy, +} -pub struct DefaultRecovery; +impl DefaultRecovery { + pub fn new(policy: RetryPolicy) -> Self { + Self { policy } + } +} impl RecoveryPolicy for DefaultRecovery { fn classify(&self, error: &anyhow::Error, completed_retries: u32) -> RecoveryAction { @@ -20,11 +29,12 @@ impl RecoveryPolicy for DefaultRecovery { if is_prompt_too_long_error(error) { return RecoveryAction::ReduceContext; } - if is_retryable_streaming_error(error) && completed_retries < MAX_STREAMING_RETRIES { + let retryable = error.is::() || is_retryable_streaming_error(error); + if retryable && self.policy.allows_retry(completed_retries) { return RecoveryAction::RetryStream { - delay: STREAMING_RETRY_BASE_DELAY * 2u32.saturating_pow(completed_retries), + delay: self.policy.backoff(RETRY_BASE_DELAY, completed_retries), attempt: completed_retries + 1, - max_attempts: MAX_STREAMING_RETRIES, + max_attempts: self.policy.max_attempts(), }; } RecoveryAction::Fail @@ -87,6 +97,8 @@ fn is_retryable_streaming_error(error: &anyhow::Error) -> bool { // Timeout errors (read timeouts, not connect timeouts) || msg.contains("operation timed out") || msg.contains("timed out reading") + // No data for a while (`llm::streaming`) + || msg.contains("stream stalled") // Server errors that are transient || msg.contains("502 bad gateway") || msg.contains("503 service") @@ -98,11 +110,19 @@ fn is_retryable_streaming_error(error: &anyhow::Error) -> bool { mod tests { use super::*; + fn two_retries() -> DefaultRecovery { + DefaultRecovery::new(RetryPolicy { + retry_forever: false, + max_retries: 2, + max_delay_secs: 300, + }) + } + #[test] fn oversized_prompt_reduces_context() { let err = anyhow::anyhow!("Invalid request: prompt is too long: 500000 tokens"); assert!(matches!( - DefaultRecovery.classify(&err, 0), + two_retries().classify(&err, 0), RecoveryAction::ReduceContext )); } @@ -110,7 +130,7 @@ mod tests { #[test] fn transient_stream_error_retries_with_backoff() { let err = anyhow::anyhow!("error reading a body from connection: connection reset"); - match DefaultRecovery.classify(&err, 0) { + match two_retries().classify(&err, 0) { RecoveryAction::RetryStream { delay, attempt, @@ -118,11 +138,11 @@ mod tests { } => { assert_eq!(delay, Duration::from_secs(2)); assert_eq!(attempt, 1); - assert_eq!(max_attempts, MAX_STREAMING_RETRIES); + assert_eq!(max_attempts, Some(2)); } _ => panic!("expected RetryStream"), } - match DefaultRecovery.classify(&err, 1) { + match two_retries().classify(&err, 1) { RecoveryAction::RetryStream { delay, attempt, .. } => { assert_eq!(delay, Duration::from_secs(4)); assert_eq!(attempt, 2); @@ -135,14 +155,14 @@ mod tests { fn retries_are_bounded() { let err = anyhow::anyhow!("connection reset"); assert!(matches!( - DefaultRecovery.classify(&err, MAX_STREAMING_RETRIES), + two_retries().classify(&err, 2), RecoveryAction::Fail )); } #[test] fn image_limit_errors_compact_with_the_named_limit() { - let classify = |message: &str| DefaultRecovery.classify(&anyhow::anyhow!("{message}"), 0); + let classify = |message: &str| two_retries().classify(&anyhow::anyhow!("{message}"), 0); let openai = r#"Invalid request: {"error":{"message":"Exceeded maximum number of images (50) allowed in the request.","param":"input","type":"invalid_request_error"}}"#; assert!(matches!( classify(openai), @@ -158,11 +178,48 @@ mod tests { )); } + #[test] + fn empty_answers_and_stalled_streams_are_retried() { + for err in [ + anyhow::Error::from(EmptyResponse), + anyhow::anyhow!("LLM stream stalled: no data received for 300s"), + ] { + assert!( + matches!( + two_retries().classify(&err, 0), + RecoveryAction::RetryStream { attempt: 1, .. } + ), + "{err}" + ); + } + } + + #[test] + fn retrying_forever_keeps_retrying_with_capped_delay() { + let recovery = DefaultRecovery::new(RetryPolicy { + retry_forever: true, + max_retries: 0, + max_delay_secs: 30, + }); + match recovery.classify(&EmptyResponse.into(), 100) { + RecoveryAction::RetryStream { + delay, + attempt, + max_attempts, + } => { + assert_eq!(delay, Duration::from_secs(30)); + assert_eq!(attempt, 101); + assert_eq!(max_attempts, None); + } + _ => panic!("expected RetryStream"), + } + } + #[test] fn unknown_errors_fail() { let err = anyhow::anyhow!("invalid api key"); assert!(matches!( - DefaultRecovery.classify(&err, 0), + two_retries().classify(&err, 0), RecoveryAction::Fail )); } diff --git a/crates/code_assistant_core/src/session/instance.rs b/crates/code_assistant_core/src/session/instance.rs index 18f512fc2..169d21d5f 100644 --- a/crates/code_assistant_core/src/session/instance.rs +++ b/crates/code_assistant_core/src/session/instance.rs @@ -36,7 +36,7 @@ pub enum SessionActivityState { Retrying { reason: String, attempt: u32, - max_attempts: u32, + max_attempts: Option, }, /// Agent terminated with an error Errored { message: String }, @@ -162,7 +162,7 @@ impl SessionActivity { &self, reason: &str, attempt: u32, - max_attempts: u32, + max_attempts: Option, ) -> Option { self.try_transition(SessionActivityState::Retrying { reason: reason.to_string(), @@ -1394,7 +1394,7 @@ impl UserInterface for SessionEventPublisher { self.publish_activity_change(self.activity.on_rate_limit_cleared()); } - fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: u32) { + fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: Option) { self.publish_activity_change(self.activity.on_retrying(reason, attempt, max_attempts)); } @@ -1588,12 +1588,12 @@ mod tests { let retrying = SessionActivityState::Retrying { reason: "no response".into(), attempt: 1, - max_attempts: 3, + max_attempts: Some(3), }; let activity = activity_with(SessionActivityState::WaitingForResponse); assert_eq!( - activity.on_retrying("no response", 1, 3), + activity.on_retrying("no response", 1, Some(3)), Some(retrying.clone()) ); assert_eq!( @@ -1609,7 +1609,7 @@ mod tests { // A retry notice after completion doesn't revive the session. let activity = activity_with(SessionActivityState::Idle); - assert_eq!(activity.on_retrying("no response", 1, 3), None); + assert_eq!(activity.on_retrying("no response", 1, Some(3)), None); } fn test_publisher(activity: SessionActivity, session_id: &str) -> SessionEventPublisher { diff --git a/crates/code_assistant_core/src/ui/mod.rs b/crates/code_assistant_core/src/ui/mod.rs index 384616149..8526e5730 100644 --- a/crates/code_assistant_core/src/ui/mod.rs +++ b/crates/code_assistant_core/src/ui/mod.rs @@ -43,7 +43,7 @@ pub trait UserInterface: Send + Sync { /// The LLM request failed before anything was streamed and is sent /// again (`attempt` of `max_attempts`). Cleared by the next response. - fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: u32) {} + fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: Option) {} /// Ask the user multiple-choice questions and wait for the answers /// (`ask_question` tool). Default: unsupported — only frontends with a @@ -106,7 +106,7 @@ impl agent_core::AgentUi for AgentUiAdapter { self.inner.clear_rate_limit(); } - fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: u32) { + fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: Option) { self.inner.notify_retrying(reason, attempt, max_attempts); } } diff --git a/crates/llm/src/aicore/anthropic.rs b/crates/llm/src/aicore/anthropic.rs index 1a2930613..bfbc467b1 100644 --- a/crates/llm/src/aicore/anthropic.rs +++ b/crates/llm/src/aicore/anthropic.rs @@ -142,4 +142,8 @@ impl LLMProvider for AiCoreAnthropicClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.anthropic_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/aicore/openai.rs b/crates/llm/src/aicore/openai.rs index 4c2a7ce32..b6cb62b47 100644 --- a/crates/llm/src/aicore/openai.rs +++ b/crates/llm/src/aicore/openai.rs @@ -116,4 +116,8 @@ impl LLMProvider for AiCoreOpenAIClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.openai_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/aicore/openai_responses.rs b/crates/llm/src/aicore/openai_responses.rs index 255b43ff1..6f948bd3c 100644 --- a/crates/llm/src/aicore/openai_responses.rs +++ b/crates/llm/src/aicore/openai_responses.rs @@ -144,4 +144,8 @@ impl LLMProvider for AiCoreOpenAIResponsesClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.responses_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/aicore/vertex.rs b/crates/llm/src/aicore/vertex.rs index 23b782941..b68fbce5a 100644 --- a/crates/llm/src/aicore/vertex.rs +++ b/crates/llm/src/aicore/vertex.rs @@ -132,4 +132,8 @@ impl LLMProvider for AiCoreVertexClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.vertex_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/anthropic.rs b/crates/llm/src/anthropic.rs index d600b3cd8..2ac0b28b5 100644 --- a/crates/llm/src/anthropic.rs +++ b/crates/llm/src/anthropic.rs @@ -537,6 +537,8 @@ pub struct AnthropicClient { client: Client, base_url: String, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, recorder: Option, playback: Option, @@ -559,6 +561,7 @@ impl AnthropicClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), recorder: None, playback: None, auth_provider: Box::new(ApiKeyAuth::new(api_key)), @@ -579,6 +582,7 @@ impl AnthropicClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), recorder: Some(APIRecorder::new(recording_path)), playback: None, auth_provider: Box::new(ApiKeyAuth::new(api_key)), @@ -600,6 +604,7 @@ impl AnthropicClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), recorder: None, playback: None, auth_provider, @@ -636,13 +641,13 @@ impl AnthropicClient { &mut self, request: &serde_json::Value, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { // If playback is enabled, skip HTTP and use recorded data if self.playback.is_some() { return self.playback_request(request, streaming_callback).await; } + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -656,7 +661,7 @@ impl AnthropicClient { if utils::handle_retryable_error::( &e, attempts, - max_retries, + &policy, streaming_callback, ) .await @@ -1231,6 +1236,10 @@ impl AnthropicClient { #[async_trait] impl LLMProvider for AnthropicClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1314,7 +1323,7 @@ impl LLMProvider for AnthropicClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&anthropic_request, streaming_callback, 3) + .send_with_retry(&anthropic_request, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/cerebras.rs b/crates/llm/src/cerebras.rs index a818b9691..63c84bccc 100644 --- a/crates/llm/src/cerebras.rs +++ b/crates/llm/src/cerebras.rs @@ -61,6 +61,10 @@ impl LLMProvider for CerebrasClient { // Delegate to inner OpenAI client with custom request processing self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/factory.rs b/crates/llm/src/factory.rs index b92a375d1..7a27be82a 100644 --- a/crates/llm/src/factory.rs +++ b/crates/llm/src/factory.rs @@ -6,6 +6,7 @@ use crate::auth::TokenManager; use crate::capabilities::{Effort, ModelCapabilities}; use crate::config_merge::merge_patch; use crate::provider_config::{ConfigurationSystem, LlmApiClientKind, ModelConfig, ProviderConfig}; +use crate::retry::RetryPolicy; use crate::{ AnthropicClient, CerebrasClient, GroqClient, LLMProvider, MinimaxClient, MistralAiClient, MoonshotClient, OllamaClient, OpenAIClient, OpenAIResponsesClient, OpenAIResponsesWsClient, @@ -315,7 +316,7 @@ pub async fn create_llm_client_from_configs( ..model_config.clone() }; - match provider_type { + let mut client = match provider_type { LLMProviderType::AiCore => { create_ai_core_client(model_config, provider_config, record_path.clone()).await } @@ -350,7 +351,9 @@ pub async fn create_llm_client_from_configs( LLMProviderType::OpenRouter => { create_openrouter_client(model_config, provider_config).await } - } + }?; + client.set_retry_policy(RetryPolicy::load()); + Ok(client) } /// The patch merged into every request for this model: the fields its diff --git a/crates/llm/src/groq.rs b/crates/llm/src/groq.rs index 816da3f38..572b5de87 100644 --- a/crates/llm/src/groq.rs +++ b/crates/llm/src/groq.rs @@ -61,4 +61,8 @@ impl LLMProvider for GroqClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index ff8b4011b..b8cc3ffa6 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -37,6 +37,7 @@ pub mod prompt_caching; pub mod provider_config; pub mod realtime; pub mod recording; +pub mod retry; pub mod streaming; pub mod types; pub mod vertex; @@ -57,6 +58,7 @@ pub use openai::OpenAIClient; pub use openai_responses::OpenAIResponsesClient; pub use openai_responses_ws::OpenAIResponsesWsClient; pub use openrouter::OpenRouterClient; +pub use retry::RetryPolicy; pub use types::*; pub use vertex::{VertexClient, sanitize_google_schema}; pub use zai::ZaiClient; @@ -88,7 +90,8 @@ pub enum StreamingChunk { reason: String, /// The retry about to happen, starting at 1 attempt: u32, - max_attempts: u32, + /// `None` when retries go on until the request succeeds + max_attempts: Option, }, /// Indicates that streaming from the LLM has completed StreamingComplete, @@ -114,4 +117,9 @@ pub trait LLMProvider: Send + Sync { request: LLMRequest, streaming_callback: Option<&StreamingCallback>, ) -> Result; + + /// How often and how patiently failed requests are sent again. Clients + /// that never retry (Ollama runs locally; the WebSocket client only + /// reconnects) ignore it, the agent loop still retries under it. + fn set_retry_policy(&mut self, _policy: RetryPolicy) {} } diff --git a/crates/llm/src/minimax.rs b/crates/llm/src/minimax.rs index 838dda889..b57603bc7 100644 --- a/crates/llm/src/minimax.rs +++ b/crates/llm/src/minimax.rs @@ -49,6 +49,10 @@ impl LLMProvider for MinimaxClient { // Delegate to inner Anthropic client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/mistralai.rs b/crates/llm/src/mistralai.rs index e191b404e..382cc0437 100644 --- a/crates/llm/src/mistralai.rs +++ b/crates/llm/src/mistralai.rs @@ -65,6 +65,10 @@ impl LLMProvider for MistralAiClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/moonshot.rs b/crates/llm/src/moonshot.rs index 353f55c41..f1c1238bd 100644 --- a/crates/llm/src/moonshot.rs +++ b/crates/llm/src/moonshot.rs @@ -68,6 +68,10 @@ impl LLMProvider for MoonshotClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/openai.rs b/crates/llm/src/openai.rs index aed5b7ec6..b8b922bff 100644 --- a/crates/llm/src/openai.rs +++ b/crates/llm/src/openai.rs @@ -305,6 +305,8 @@ pub struct OpenAIClient { client: Client, base_url: String, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, model_temperatures: HashMap, model_top_ps: HashMap, // Customization points @@ -326,6 +328,7 @@ impl OpenAIClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), model_temperatures, model_top_ps, auth_provider: Box::new(ApiKeyAuth::new(api_key)), @@ -347,6 +350,7 @@ impl OpenAIClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), model_temperatures, model_top_ps, auth_provider, @@ -627,8 +631,8 @@ impl OpenAIClient { &self, request: &OpenAIRequest, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -647,7 +651,7 @@ impl OpenAIClient { if utils::handle_retryable_error::( &e, attempts, - max_retries, + &policy, streaming_callback, ) .await @@ -1128,6 +1132,10 @@ impl OpenAIClient { #[async_trait] impl LLMProvider for OpenAIClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1182,7 +1190,7 @@ impl LLMProvider for OpenAIClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&openai_request, streaming_callback, 3) + .send_with_retry(&openai_request, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/openai_responses.rs b/crates/llm/src/openai_responses.rs index b46ef37a7..fb00817cf 100644 --- a/crates/llm/src/openai_responses.rs +++ b/crates/llm/src/openai_responses.rs @@ -593,6 +593,8 @@ pub struct OpenAIResponsesClient { client: Client, base_url: String, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, auth_provider: Box, request_customizer: Box, recorder: Option, @@ -614,6 +616,7 @@ impl OpenAIResponsesClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), auth_provider: Box::new(ApiKeyAuth::new(api_key)), request_customizer: Box::new(DefaultRequestCustomizer), recorder: None, @@ -633,6 +636,7 @@ impl OpenAIResponsesClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), auth_provider, request_customizer, recorder: None, @@ -819,13 +823,13 @@ impl OpenAIResponsesClient { &mut self, request: &ResponsesRequest, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { // If playback is enabled, skip HTTP and use recorded data if self.playback.is_some() { return self.playback_request(request, streaming_callback).await; } + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -838,7 +842,7 @@ impl OpenAIResponsesClient { if utils::handle_retryable_error::( &e, attempts, - max_retries, + &policy, streaming_callback, ) .await @@ -1422,6 +1426,10 @@ impl<'a> StreamProcessor<'a> { #[async_trait] impl LLMProvider for OpenAIResponsesClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1476,7 +1484,7 @@ impl LLMProvider for OpenAIResponsesClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&responses_request, streaming_callback, 3) + .send_with_retry(&responses_request, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/openrouter.rs b/crates/llm/src/openrouter.rs index 68cdb8dfb..e0aac8a0c 100644 --- a/crates/llm/src/openrouter.rs +++ b/crates/llm/src/openrouter.rs @@ -35,4 +35,8 @@ impl LLMProvider for OpenRouterClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } diff --git a/crates/llm/src/retry.rs b/crates/llm/src/retry.rs new file mode 100644 index 000000000..e094725f7 --- /dev/null +++ b/crates/llm/src/retry.rs @@ -0,0 +1,176 @@ +//! How often and how patiently a failed LLM request is sent again +//! (`retry.json`): rate limits, overloaded or unreachable servers, broken +//! streams and empty answers all follow this one policy. + +use crate::provider_config::ConfigurationSystem; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tracing::warn; + +const FILE_NAME: &str = "retry.json"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct RetryPolicy { + /// Send a failed request again until it succeeds or the user stops the + /// run; `max_retries` is ignored then. + pub retry_forever: bool, + /// Retries of one request before the run fails. + pub max_retries: u32, + /// Upper bound for the wait before a retry, also when the provider asks + /// for a longer one. + pub max_delay_secs: u64, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + retry_forever: false, + max_retries: 5, + max_delay_secs: 300, + } + } +} + +impl RetryPolicy { + pub fn path() -> PathBuf { + ConfigurationSystem::config_file_path(FILE_NAME) + } + + /// The policy from the config directory; a missing or malformed file + /// yields the defaults. + pub fn load() -> Self { + Self::load_from(&Self::path()) + } + + pub fn load_from(path: &Path) -> Self { + let Ok(content) = std::fs::read_to_string(path) else { + return Self::default(); + }; + serde_json::from_str(&content).unwrap_or_else(|e| { + warn!("Failed to parse {}: {e}; using defaults", path.display()); + Self::default() + }) + } + + pub fn save(&self) -> Result<()> { + self.save_to(&Self::path()) + } + + pub fn save_to(&self, path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, serde_json::to_string_pretty(self)?) + .with_context(|| format!("Failed to write {}", path.display())) + } + + /// Whether another retry may follow `completed` ones. + pub fn allows_retry(&self, completed: u32) -> bool { + self.retry_forever || completed < self.max_retries + } + + /// The number of retries, `None` when there is no limit. + pub fn max_attempts(&self) -> Option { + (!self.retry_forever).then_some(self.max_retries) + } + + /// `delay` cut to the configured maximum. + pub fn cap(&self, delay: Duration) -> Duration { + delay.min(Duration::from_secs(self.max_delay_secs)) + } + + /// Exponential backoff for the retry after `completed` ones: `base`, + /// then doubled for each further retry, at most the configured maximum. + pub fn backoff(&self, base: Duration, completed: u32) -> Duration { + let factor = 2u32.saturating_pow(completed.min(31)); + self.cap(base.saturating_mul(factor)) + } +} + +/// "3/5" for the third of five retries, "3" without a limit. +pub fn attempt_label(attempt: u32, max_attempts: Option) -> String { + match max_attempts { + Some(max) => format!("{attempt}/{max}"), + None => attempt.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_limited_policy_stops_after_max_retries() { + let policy = RetryPolicy { + retry_forever: false, + max_retries: 2, + max_delay_secs: 60, + }; + assert!(policy.allows_retry(0)); + assert!(policy.allows_retry(1)); + assert!(!policy.allows_retry(2)); + assert_eq!(policy.max_attempts(), Some(2)); + } + + #[test] + fn retrying_forever_never_stops() { + let policy = RetryPolicy { + retry_forever: true, + max_retries: 0, + max_delay_secs: 60, + }; + assert!(policy.allows_retry(0)); + assert!(policy.allows_retry(u32::MAX)); + assert_eq!(policy.max_attempts(), None); + } + + #[test] + fn backoff_doubles_up_to_the_maximum_delay() { + let policy = RetryPolicy { + max_delay_secs: 10, + ..RetryPolicy::default() + }; + let base = Duration::from_secs(2); + assert_eq!(policy.backoff(base, 0), Duration::from_secs(2)); + assert_eq!(policy.backoff(base, 1), Duration::from_secs(4)); + assert_eq!(policy.backoff(base, 2), Duration::from_secs(8)); + assert_eq!(policy.backoff(base, 3), Duration::from_secs(10)); + assert_eq!(policy.backoff(base, 1000), Duration::from_secs(10)); + } + + #[test] + fn a_requested_wait_is_capped() { + let policy = RetryPolicy { + max_delay_secs: 30, + ..RetryPolicy::default() + }; + assert_eq!( + policy.cap(Duration::from_secs(3600)), + Duration::from_secs(30) + ); + assert_eq!(policy.cap(Duration::from_secs(5)), Duration::from_secs(5)); + } + + #[test] + fn missing_fields_take_the_defaults_and_the_file_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(FILE_NAME); + assert_eq!(RetryPolicy::load_from(&path), RetryPolicy::default()); + + std::fs::write(&path, r#"{"retry_forever": true}"#).unwrap(); + let loaded = RetryPolicy::load_from(&path); + assert!(loaded.retry_forever); + assert_eq!(loaded.max_delay_secs, RetryPolicy::default().max_delay_secs); + + let policy = RetryPolicy { + retry_forever: false, + max_retries: 7, + max_delay_secs: 42, + }; + policy.save_to(&path).unwrap(); + assert_eq!(RetryPolicy::load_from(&path), policy); + } +} diff --git a/crates/llm/src/utils.rs b/crates/llm/src/utils.rs index b51f43064..1fb93fed5 100644 --- a/crates/llm/src/utils.rs +++ b/crates/llm/src/utils.rs @@ -1,4 +1,6 @@ -use crate::{ApiError, ApiErrorContext, RateLimitHandler, StreamingCallback, StreamingChunk}; +use crate::{ + ApiError, ApiErrorContext, RateLimitHandler, RetryPolicy, StreamingCallback, StreamingChunk, +}; use anyhow::Result; use reqwest::{Client, RequestBuilder, Response, StatusCode}; use std::time::Duration; @@ -130,13 +132,15 @@ pub async fn check_response_error( error: &anyhow::Error, attempts: u32, - max_retries: u32, + policy: &RetryPolicy, streaming_callback: Option<&StreamingCallback>, ) -> bool { let (api_error, rate_limits) = if let Some(ctx) = error.downcast_ref::>() { @@ -148,80 +152,42 @@ pub async fn handle_retryable_error< } else { return false; }; - match api_error { - ApiError::RateLimit(_) => { - if let Some(rate_limits) = rate_limits { - if attempts < max_retries { - // Get base delay from rate limit headers - let mut delay = rate_limits.get_retry_delay(); - - // Apply exponential backoff based on retry attempts - // Double the delay for each retry attempt after the first - if attempts > 0 { - let multiplier = 2u32.pow(attempts); - delay *= multiplier; - } - let delay_secs = delay.as_secs(); - warn!( - "Rate limit hit (attempt {}/{}), waiting {} seconds before retry", - attempts, max_retries, delay_secs - ); - - return wait_rate_limit(delay, streaming_callback).await; - } - } else { - // Fallback if no rate limit info available - if attempts < max_retries { - // Implement exponential backoff with a more appropriate base delay - // Start with 60 seconds for the first retry, then double for each subsequent attempt - let base_delay_secs = 60u64; - let delay = Duration::from_secs(base_delay_secs * 2u64.pow(attempts)); - warn!( - "Rate limit hit but no timing info available (attempt {}/{}), using exponential backoff: {} seconds", - attempts, - max_retries, - delay.as_secs() - ); - return wait_rate_limit(delay, streaming_callback).await; - } - } - } - ApiError::Authentication(msg) if is_transient_auth_error(msg) => { - if attempts < max_retries { - let delay = Duration::from_secs(2u64.pow(attempts.saturating_sub(1))); - warn!( - "Transient auth error: {} (attempt {}/{}), retrying in {} seconds", - error, - attempts, - max_retries, - delay.as_secs() - ); - return announce_retry(api_error, attempts, max_retries, delay, streaming_callback) - .await; - } - } - ApiError::ServiceError(_) | ApiError::NetworkError(_) | ApiError::Overloaded(_) => { - if attempts < max_retries { - let delay = Duration::from_secs(2u64.pow(attempts.saturating_sub(1))); - warn!( - "Error: {} (attempt {}/{}), retrying in {} seconds", - error, - attempts, - max_retries, - delay.as_secs() - ); - return announce_retry(api_error, attempts, max_retries, delay, streaming_callback) - .await; - } - } - _ => { - warn!( - "Unhandled error (attempt {}/{}): {:?}", - attempts, max_retries, error - ); - } + let retryable = match api_error { + ApiError::RateLimit(_) => true, + ApiError::Authentication(msg) => is_transient_auth_error(msg), + ApiError::ServiceError(_) | ApiError::NetworkError(_) | ApiError::Overloaded(_) => true, + _ => false, + }; + if !retryable { + warn!("Unhandled error (attempt {attempts}): {error:?}"); + return false; } - false + if !policy.allows_retry(attempts) { + warn!("Giving up after {attempts} retries: {error}"); + return false; + } + if let ApiError::RateLimit(_) = api_error { + // The provider's own wait (headers), else a minute; doubled for + // each further retry. + let base = rate_limits + .map(|rate_limits| rate_limits.get_retry_delay()) + .unwrap_or(Duration::from_secs(60)); + let delay = policy.backoff(base, attempts); + warn!( + "Rate limit hit (retry {}), waiting {} seconds", + attempts + 1, + delay.as_secs() + ); + return wait_rate_limit(delay, streaming_callback).await; + } + let delay = policy.backoff(Duration::from_secs(1), attempts.saturating_sub(1)); + warn!( + "Error: {} (retry {}), retrying in {} seconds", + error, + attempts + 1, + delay.as_secs() + ); + announce_retry(api_error, attempts, policy, delay, streaming_callback).await } /// Wait out a rate limit, showing a countdown when there is a callback. @@ -268,7 +234,7 @@ async fn wait_rate_limit(delay: Duration, streaming_callback: Option<&StreamingC async fn announce_retry( error: &ApiError, attempts: u32, - max_retries: u32, + policy: &RetryPolicy, delay: Duration, streaming_callback: Option<&StreamingCallback>, ) -> bool { @@ -276,7 +242,7 @@ async fn announce_retry( let chunk = StreamingChunk::Retrying { reason: error.to_string(), attempt: attempts + 1, - max_attempts: max_retries, + max_attempts: policy.max_attempts(), }; if callback(&chunk).is_err() { return false; @@ -303,15 +269,36 @@ mod tests { fn log_status(&self) {} } + /// Three retries without waiting. + fn three_retries() -> RetryPolicy { + RetryPolicy { + retry_forever: false, + max_retries: 3, + max_delay_secs: 0, + } + } + #[tokio::test] async fn bare_network_errors_are_retried() { let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); - assert!(handle_retryable_error::(&error, 0, 3, None).await); - assert!(!handle_retryable_error::(&error, 3, 3, None).await); + let policy = three_retries(); + assert!(handle_retryable_error::(&error, 0, &policy, None).await); + assert!(!handle_retryable_error::(&error, 3, &policy, None).await); // Context added on the way up must not hide the error. let error = error.context("LLM request failed"); - assert!(handle_retryable_error::(&error, 1, 3, None).await); + assert!(handle_retryable_error::(&error, 1, &policy, None).await); + } + + #[tokio::test] + async fn retrying_forever_has_no_limit() { + let error: anyhow::Error = ApiError::RateLimit("slow down".into()).into(); + let policy = RetryPolicy { + retry_forever: true, + max_retries: 0, + max_delay_secs: 0, + }; + assert!(handle_retryable_error::(&error, 1000, &policy, None).await); } #[tokio::test] @@ -336,25 +323,29 @@ mod tests { } }); let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); - assert!(handle_retryable_error::(&error, 0, 3, Some(&callback)).await); + let policy = three_retries(); + assert!(handle_retryable_error::(&error, 0, &policy, Some(&callback)).await); assert_eq!( *seen.lock().unwrap(), - vec![("Network error: connection reset".to_string(), 1, 3)] + vec![("Network error: connection reset".to_string(), 1, Some(3))] ); // A stop request (callback error) ends the retry loop. let cancelled: StreamingCallback = Box::new(|_: &StreamingChunk| Err(anyhow::anyhow!("stopped"))); - assert!(!handle_retryable_error::(&error, 0, 3, Some(&cancelled)).await); + assert!( + !handle_retryable_error::(&error, 0, &policy, Some(&cancelled)).await + ); } #[tokio::test] async fn other_errors_are_not_retried() { let error = anyhow::anyhow!("LLM stream stalled: no data received for 300s"); - assert!(!handle_retryable_error::(&error, 0, 3, None).await); + let policy = three_retries(); + assert!(!handle_retryable_error::(&error, 0, &policy, None).await); let error: anyhow::Error = ApiError::InvalidRequest("bad".into()).into(); - assert!(!handle_retryable_error::(&error, 0, 3, None).await); + assert!(!handle_retryable_error::(&error, 0, &policy, None).await); } #[test] diff --git a/crates/llm/src/vertex.rs b/crates/llm/src/vertex.rs index d5a0abe33..c1e8c4826 100644 --- a/crates/llm/src/vertex.rs +++ b/crates/llm/src/vertex.rs @@ -801,6 +801,8 @@ fn merge_vertex_partial_value(target: &mut serde_json::Value, value: serde_json: pub struct VertexClient { client: Client, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, base_url: String, recorder: Option, custom_config: Option, @@ -818,6 +820,7 @@ impl VertexClient { Self { client: utils::http_client(), model, + retry_policy: crate::RetryPolicy::default(), base_url, recorder: None, custom_config: None, @@ -836,6 +839,7 @@ impl VertexClient { Self { client: utils::http_client(), model, + retry_policy: crate::RetryPolicy::default(), base_url, recorder: Some(APIRecorder::new(recording_path)), custom_config: None, @@ -854,6 +858,7 @@ impl VertexClient { Self { client: utils::http_client(), model, + retry_policy: crate::RetryPolicy::default(), base_url, recorder: None, custom_config: None, @@ -1011,8 +1016,8 @@ impl VertexClient { request: &VertexRequest, request_id: u64, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -1030,7 +1035,7 @@ impl VertexClient { if utils::handle_retryable_error::( &e, attempts, - max_retries, + &policy, streaming_callback, ) .await @@ -1435,6 +1440,10 @@ impl VertexClient { #[async_trait] impl LLMProvider for VertexClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1508,7 +1517,7 @@ impl LLMProvider for VertexClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&vertex_request, request_id, streaming_callback, 3) + .send_with_retry(&vertex_request, request_id, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/zai.rs b/crates/llm/src/zai.rs index 7832b4072..ad31ea4c9 100644 --- a/crates/llm/src/zai.rs +++ b/crates/llm/src/zai.rs @@ -68,6 +68,10 @@ impl LLMProvider for ZaiClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/ui_gpui/src/messages/activity_indicator.rs b/crates/ui_gpui/src/messages/activity_indicator.rs index 408522f9c..af30095bc 100644 --- a/crates/ui_gpui/src/messages/activity_indicator.rs +++ b/crates/ui_gpui/src/messages/activity_indicator.rs @@ -104,7 +104,10 @@ pub fn render_activity_indicator( max_attempts, } => ( cx.theme().warning, - Some(format!("{reason} — retrying ({attempt}/{max_attempts})…")), + Some(format!( + "{reason} — retrying ({})…", + llm::retry::attempt_label(attempt, max_attempts) + )), ), SessionActivityState::Thinking => (cx.theme().primary, Some("Thinking…".to_string())), SessionActivityState::WaitingForResponse => (cx.theme().primary, None), diff --git a/crates/ui_gpui/src/settings_screen/general_section.rs b/crates/ui_gpui/src/settings_screen/general_section.rs index 04638d605..ce12a3924 100644 --- a/crates/ui_gpui/src/settings_screen/general_section.rs +++ b/crates/ui_gpui/src/settings_screen/general_section.rs @@ -12,6 +12,7 @@ use gpui_kit::component::{ActiveTheme, Icon, Sizable, Size}; use gpui_kit::{ App, Context, Entity, FocusHandle, Focusable, SharedString, Subscription, div, prelude::*, px, }; +use llm::RetryPolicy; use tracing::warn; pub struct GeneralSection { @@ -38,6 +39,12 @@ pub struct GeneralSection { _auto_mode_model_subscription: Subscription, auto_mode_rules_input: Entity, _auto_mode_rules_subscription: Subscription, + /// How failed LLM requests are sent again; stored in `retry.json`. + retry_forever: bool, + max_retries_input: Entity, + _max_retries_subscription: Subscription, + max_retry_delay_input: Entity, + _max_retry_delay_subscription: Subscription, } impl GeneralSection { @@ -94,6 +101,24 @@ impl GeneralSection { window, Self::on_auto_mode_rules_input, ); + let retry = RetryPolicy::load(); + let max_retries_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(RetryPolicy::default().max_retries.to_string()) + .default_value(retry.max_retries.to_string()) + }); + let max_retries_subscription = + cx.subscribe_in(&max_retries_input, window, Self::on_max_retries_input); + let max_retry_delay_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(RetryPolicy::default().max_delay_secs.to_string()) + .default_value(retry.max_delay_secs.to_string()) + }); + let max_retry_delay_subscription = cx.subscribe_in( + &max_retry_delay_input, + window, + Self::on_max_retry_delay_input, + ); let mut this = Self { focus_handle: cx.focus_handle(), default_model_select, @@ -109,6 +134,11 @@ impl GeneralSection { _auto_mode_model_subscription: auto_mode_model_subscription, auto_mode_rules_input, _auto_mode_rules_subscription: auto_mode_rules_subscription, + retry_forever: retry.retry_forever, + max_retries_input, + _max_retries_subscription: max_retries_subscription, + max_retry_delay_input, + _max_retry_delay_subscription: max_retry_delay_subscription, }; this.reload(window, cx); this @@ -273,6 +303,54 @@ impl GeneralSection { Self::save_handoff_config(|config| config.max_images_per_request = max_images_per_request); } + fn set_retry_forever(&mut self, enabled: bool, cx: &mut Context) { + self.retry_forever = enabled; + Self::save_retry_policy(|policy| policy.retry_forever = enabled); + cx.notify(); + } + + /// Save the number of retries whenever the input holds a valid number. + fn on_max_retries_input( + &mut self, + input: &Entity, + event: &InputEvent, + _window: &mut gpui_kit::Window, + cx: &mut Context, + ) { + if !matches!(event, InputEvent::Change) { + return; + } + let Ok(max_retries) = input.read(cx).value().trim().parse::() else { + return; + }; + Self::save_retry_policy(|policy| policy.max_retries = max_retries); + } + + /// Save the longest wait whenever the input holds a valid number. + fn on_max_retry_delay_input( + &mut self, + input: &Entity, + event: &InputEvent, + _window: &mut gpui_kit::Window, + cx: &mut Context, + ) { + if !matches!(event, InputEvent::Change) { + return; + } + let Ok(max_delay_secs) = input.read(cx).value().trim().parse::() else { + return; + }; + Self::save_retry_policy(|policy| policy.max_delay_secs = max_delay_secs); + } + + fn save_retry_policy(change: impl FnOnce(&mut RetryPolicy)) { + let mut policy = RetryPolicy::load(); + change(&mut policy); + if let Err(e) = policy.save() { + warn!("Failed to save the retry settings: {e:#}"); + } + } + fn save_handoff_config(change: impl FnOnce(&mut HandoffConfig)) { let mut config = HandoffConfig::load(); change(&mut config); @@ -482,6 +560,90 @@ impl Render for GeneralSection { ), ), ) + // Failed requests + .child( + div() + .flex() + .flex_col() + .gap_2() + .child( + div() + .text_xs() + .font_weight(gpui_kit::FontWeight::MEDIUM) + .text_color(cx.theme().muted_foreground) + .child("Failed requests"), + ) + .child( + div() + .flex() + .items_center() + .gap_2() + .child( + Switch::new("retry-forever") + .checked(self.retry_forever) + .with_size(Size::Small) + .on_click({ + let view = cx.entity(); + move |enabled, _window, app| { + let enabled = *enabled; + view.update(app, |this, cx| { + this.set_retry_forever(enabled, cx); + }); + } + }), + ) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("Retry until the request succeeds"), + ), + ) + .when(!self.retry_forever, |el| { + el.child( + div() + .flex() + .items_center() + .gap_2() + .child(div().w(px(80.)).child(Input::new(&self.max_retries_input))) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("retries at most"), + ), + ) + }) + .child( + div() + .flex() + .items_center() + .gap_2() + .child( + div() + .w(px(80.)) + .child(Input::new(&self.max_retry_delay_input)), + ) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("seconds between retries at most"), + ), + ) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child( + "Applies to rate limits, overloaded or unreachable servers, \ + broken streams and empty answers. The wait doubles with each \ + retry up to this maximum, also when the provider asks for a \ + longer one. Stopping the session ends the waiting. Takes \ + effect with the next request a session starts.", + ), + ), + ) // Image limit .child( div() diff --git a/crates/ui_terminal/src/renderer.rs b/crates/ui_terminal/src/renderer.rs index 0eda53764..aadaa9c05 100644 --- a/crates/ui_terminal/src/renderer.rs +++ b/crates/ui_terminal/src/renderer.rs @@ -31,7 +31,7 @@ pub enum SpinnerState { Retrying { start_time: Instant, attempt: u32, - max_attempts: u32, + max_attempts: Option, }, } @@ -66,7 +66,10 @@ impl SpinnerState { attempt, max_attempts, .. - } => Some(format!("Retrying ({attempt}/{max_attempts})")), + } => Some(format!( + "Retrying ({})", + llm::retry::attempt_label(*attempt, *max_attempts) + )), } } } @@ -605,7 +608,7 @@ impl TerminalRenderer { } /// Show that the LLM request is sent again - pub fn show_retry_spinner(&mut self, attempt: u32, max_attempts: u32) { + pub fn show_retry_spinner(&mut self, attempt: u32, max_attempts: Option) { self.spinner_state = SpinnerState::Retrying { start_time: Instant::now(), attempt, diff --git a/crates/ui_terminal/src/ui.rs b/crates/ui_terminal/src/ui.rs index 9f871b073..f7911e087 100644 --- a/crates/ui_terminal/src/ui.rs +++ b/crates/ui_terminal/src/ui.rs @@ -837,8 +837,11 @@ impl UserInterface for TerminalUI { }); } - fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: u32) { - debug!("Retrying LLM request ({attempt}/{max_attempts}): {reason}"); + fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: Option) { + debug!( + "Retrying LLM request ({}): {reason}", + llm::retry::attempt_label(attempt, max_attempts) + ); let rt = tokio::runtime::Handle::current(); let renderer = self.renderer.clone(); From 278e4f231988bf8386149edcdb668de7fd401c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 17 Nov 2026 11:14:08 +0100 Subject: [PATCH 2/5] docs: retrying failed requests --- AGENTS.md | 9 +++++++++ docs/retry-policy.md | 48 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 docs/retry-policy.md diff --git a/AGENTS.md b/AGENTS.md index e63ed4d31..fcb24f6e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,15 @@ Layer 5 (binary): code_assistant — CLI, config, feature-gated frontend `SessionService::restart_mcp_server`) without restarting the app - Details: `docs/mcp-client-mode.md`, `docs/project-scoped-mcp-servers.md` +### Retrying Failed Requests +- One policy (`llm::RetryPolicy`, `/retry.json`, *Settings → + General*): retry forever or at most N times, and the longest wait between + retries +- LLM clients retry rate limits and transient server/network errors; the + agent loop (`plugins::DefaultRecovery`) retries broken streams and empty + answers (`agent_core::types::EmptyResponse`), which used to end a turn + silently with *Resume*; see `docs/retry-policy.md` + ### Built-in Tool Toggles - Groups of built-in tools (browser, web, terminal, sub-agents, wakeups, session history) switch on/off like MCP servers: globally in `tools.json` diff --git a/docs/retry-policy.md b/docs/retry-policy.md new file mode 100644 index 000000000..0d4436ae8 --- /dev/null +++ b/docs/retry-policy.md @@ -0,0 +1,48 @@ +# Retrying failed requests + +A failed LLM request is sent again under one policy, `llm::RetryPolicy`, +stored in `/retry.json` and edited under *Settings → General → +Failed requests*: + +```json +{ + "retry_forever": false, + "max_retries": 5, + "max_delay_secs": 300 +} +``` + +- `retry_forever` sends a request again until it succeeds or the user stops + the session; `max_retries` is ignored then. +- `max_retries` bounds the retries of one request; after the last one the + session fails and shows the error. +- `max_delay_secs` bounds the wait before a retry, also when the provider asks + for a longer one (`retry-after`, rate limit reset headers). + +Missing fields take the defaults shown above. A changed policy applies to the +next request a session starts: clients and the agent's hooks are created per +run. + +## Two places retry + +- **The LLM clients** (`llm::utils::handle_retryable_error`) retry what fails + before an answer streams: rate limits (wait taken from the provider's + headers, else a minute, doubled per retry), transient authentication + errors, overloaded, unreachable or failing servers (one second, doubled per + retry). The factory hands each client the policy + (`LLMProvider::set_retry_policy`); Ollama and the WebSocket client keep + their own behaviour. +- **The agent loop** (`code_assistant_core::plugins::DefaultRecovery`) retries + what the clients do not: a broken or stalled stream and an empty answer + (`agent_core::types::EmptyResponse`). It starts at two seconds, doubling. + +While waiting, the session shows `RateLimited` (countdown) or `Retrying` +(reason and `attempt/max`, only `attempt` without a limit). + +## Empty answers + +An answer without any content used to end the turn as if the model had +answered: nothing was recorded, so the history still ended in the user's +message or the tool results, and the session showed *Resume* without an +error. Now the agent loop treats it as a failed request, retries it under the +policy, and fails visibly when the retries run out. From ecaed26e09271e73bc196e5e27b804bac82f0810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 17 Nov 2026 11:27:33 +0100 Subject: [PATCH 3/5] fix: retry an error in one place only; wait at least a second MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent loop recognised errors by their text and retried overloaded and network errors the client had already given up on, so N retries became (N+1)² − 1 requests. A client now marks such an error RetriesExhausted and the agent loop leaves it alone. A maximum wait of 0 counts as one second, so retrying forever cannot send requests back to back. --- .../src/plugins/recovery.rs | 15 ++ crates/llm/src/anthropic.rs | 12 +- crates/llm/src/openai.rs | 12 +- crates/llm/src/openai_responses.rs | 12 +- crates/llm/src/retry.rs | 25 ++- crates/llm/src/utils.rs | 144 +++++++++++++++--- crates/llm/src/vertex.rs | 12 +- docs/retry-policy.md | 8 +- 8 files changed, 178 insertions(+), 62 deletions(-) diff --git a/crates/code_assistant_core/src/plugins/recovery.rs b/crates/code_assistant_core/src/plugins/recovery.rs index 116aeb0fe..61f423341 100644 --- a/crates/code_assistant_core/src/plugins/recovery.rs +++ b/crates/code_assistant_core/src/plugins/recovery.rs @@ -3,6 +3,7 @@ use agent_core::hooks::{RecoveryAction, RecoveryPolicy}; use agent_core::types::EmptyResponse; use llm::RetryPolicy; +use llm::retry::RetriesExhausted; use std::time::Duration; /// Wait before the first retry of a broken stream or an empty answer @@ -29,6 +30,10 @@ impl RecoveryPolicy for DefaultRecovery { if is_prompt_too_long_error(error) { return RecoveryAction::ReduceContext; } + // The client already sent it again as often as the policy allows. + if error.is::() { + return RecoveryAction::Fail; + } let retryable = error.is::() || is_retryable_streaming_error(error); if retryable && self.policy.allows_retry(completed_retries) { return RecoveryAction::RetryStream { @@ -215,6 +220,16 @@ mod tests { } } + #[test] + fn errors_the_client_gave_up_on_are_not_retried_again() { + let err = anyhow::anyhow!("Service overloaded: overloaded_error") + .context(RetriesExhausted { retries: 2 }); + assert!(matches!( + two_retries().classify(&err, 0), + RecoveryAction::Fail + )); + } + #[test] fn unknown_errors_fail() { let err = anyhow::anyhow!("invalid api key"); diff --git a/crates/llm/src/anthropic.rs b/crates/llm/src/anthropic.rs index 2ac0b28b5..3252b39ed 100644 --- a/crates/llm/src/anthropic.rs +++ b/crates/llm/src/anthropic.rs @@ -658,18 +658,14 @@ impl AnthropicClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } diff --git a/crates/llm/src/openai.rs b/crates/llm/src/openai.rs index b8b922bff..d08988734 100644 --- a/crates/llm/src/openai.rs +++ b/crates/llm/src/openai.rs @@ -648,18 +648,14 @@ impl OpenAIClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } diff --git a/crates/llm/src/openai_responses.rs b/crates/llm/src/openai_responses.rs index fb00817cf..11ce8a867 100644 --- a/crates/llm/src/openai_responses.rs +++ b/crates/llm/src/openai_responses.rs @@ -839,18 +839,14 @@ impl OpenAIResponsesClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } diff --git a/crates/llm/src/retry.rs b/crates/llm/src/retry.rs index e094725f7..fc764ee53 100644 --- a/crates/llm/src/retry.rs +++ b/crates/llm/src/retry.rs @@ -20,7 +20,7 @@ pub struct RetryPolicy { /// Retries of one request before the run fails. pub max_retries: u32, /// Upper bound for the wait before a retry, also when the provider asks - /// for a longer one. + /// for a longer one; at least one second. pub max_delay_secs: u64, } @@ -77,9 +77,10 @@ impl RetryPolicy { (!self.retry_forever).then_some(self.max_retries) } - /// `delay` cut to the configured maximum. + /// `delay` cut to the configured maximum. A maximum below one second + /// counts as one, so retrying forever never turns into a busy loop. pub fn cap(&self, delay: Duration) -> Duration { - delay.min(Duration::from_secs(self.max_delay_secs)) + delay.min(Duration::from_secs(self.max_delay_secs.max(1))) } /// Exponential backoff for the retry after `completed` ones: `base`, @@ -90,6 +91,14 @@ impl RetryPolicy { } } +/// Context on an error the client sent again as often as the policy +/// allows; whoever receives it should not retry it once more. +#[derive(Debug, thiserror::Error)] +#[error("gave up after {retries} retries")] +pub struct RetriesExhausted { + pub retries: u32, +} + /// "3/5" for the third of five retries, "3" without a limit. pub fn attempt_label(attempt: u32, max_attempts: Option) -> String { match max_attempts { @@ -154,6 +163,16 @@ mod tests { assert_eq!(policy.cap(Duration::from_secs(5)), Duration::from_secs(5)); } + #[test] + fn a_maximum_of_zero_still_waits_a_second() { + let policy = RetryPolicy { + max_delay_secs: 0, + ..RetryPolicy::default() + }; + assert_eq!(policy.cap(Duration::from_secs(60)), Duration::from_secs(1)); + assert_eq!(policy.cap(Duration::ZERO), Duration::ZERO); + } + #[test] fn missing_fields_take_the_defaults_and_the_file_round_trips() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/llm/src/utils.rs b/crates/llm/src/utils.rs index 1fb93fed5..1515bdfbb 100644 --- a/crates/llm/src/utils.rs +++ b/crates/llm/src/utils.rs @@ -1,5 +1,6 @@ use crate::{ ApiError, ApiErrorContext, RateLimitHandler, RetryPolicy, StreamingCallback, StreamingChunk, + retry::RetriesExhausted, }; use anyhow::Result; use reqwest::{Client, RequestBuilder, Response, StatusCode}; @@ -128,9 +129,11 @@ pub async fn check_response_error( - error: &anyhow::Error, + error: anyhow::Error, attempts: u32, policy: &RetryPolicy, streaming_callback: Option<&StreamingCallback>, -) -> bool { - let (api_error, rate_limits) = if let Some(ctx) = error.downcast_ref::>() { - (&ctx.error, ctx.rate_limits.as_ref()) - } else if let Some(api_error) = error.downcast_ref::() { +) -> anyhow::Result<()> { + if retry_after_wait::(&error, attempts, policy, streaming_callback).await { + return Ok(()); + } + if is_retryable::(&error) { + return Err(error.context(RetriesExhausted { retries: attempts })); + } + Err(error) +} + +/// The provider error behind `error`, with its rate limit context. +fn api_error( + error: &anyhow::Error, +) -> Option<(&ApiError, Option<&T>)> { + if let Some(ctx) = error.downcast_ref::>() { + Some((&ctx.error, ctx.rate_limits.as_ref())) + } else { // Raised before a response arrived (connect failure, timeout while // waiting for headers), so there is no rate limit context. - (api_error, None) - } else { - return false; - }; - let retryable = match api_error { - ApiError::RateLimit(_) => true, - ApiError::Authentication(msg) => is_transient_auth_error(msg), - ApiError::ServiceError(_) | ApiError::NetworkError(_) | ApiError::Overloaded(_) => true, + error.downcast_ref::().map(|e| (e, None)) + } +} + +fn is_retryable( + error: &anyhow::Error, +) -> bool { + match api_error::(error) { + Some((ApiError::Authentication(msg), _)) => is_transient_auth_error(msg), + Some(( + ApiError::RateLimit(_) + | ApiError::ServiceError(_) + | ApiError::NetworkError(_) + | ApiError::Overloaded(_), + _, + )) => true, _ => false, + } +} + +/// Wait before the next retry when `error` is retryable and the policy +/// allows one; false when not, or when the user cancelled while waiting. +async fn retry_after_wait( + error: &anyhow::Error, + attempts: u32, + policy: &RetryPolicy, + streaming_callback: Option<&StreamingCallback>, +) -> bool { + let Some((api_error, rate_limits)) = api_error::(error) else { + return false; }; - if !retryable { + if !is_retryable::(error) { warn!("Unhandled error (attempt {attempts}): {error:?}"); return false; } @@ -269,6 +306,33 @@ mod tests { fn log_status(&self) {} } + /// A copy of a test error, which carries one `ApiError` or plain text. + fn clone(error: &anyhow::Error) -> anyhow::Error { + match error.downcast_ref::() { + Some(ApiError::NetworkError(m)) => ApiError::NetworkError(m.clone()).into(), + Some(ApiError::RateLimit(m)) => ApiError::RateLimit(m.clone()).into(), + Some(ApiError::InvalidRequest(m)) => ApiError::InvalidRequest(m.clone()).into(), + _ => anyhow::anyhow!("{error:#}"), + } + } + + #[tokio::test] + async fn giving_up_on_a_retryable_error_marks_it_exhausted() { + let policy = three_retries(); + let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); + let given_up = handle_retryable_error::(error, 3, &policy, None) + .await + .unwrap_err(); + assert!(given_up.is::(), "{given_up:#}"); + assert!(given_up.is::(), "the cause stays reachable"); + + let error: anyhow::Error = ApiError::InvalidRequest("bad".into()).into(); + let given_up = handle_retryable_error::(error, 0, &policy, None) + .await + .unwrap_err(); + assert!(!given_up.is::(), "never retryable"); + } + /// Three retries without waiting. fn three_retries() -> RetryPolicy { RetryPolicy { @@ -282,12 +346,24 @@ mod tests { async fn bare_network_errors_are_retried() { let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); let policy = three_retries(); - assert!(handle_retryable_error::(&error, 0, &policy, None).await); - assert!(!handle_retryable_error::(&error, 3, &policy, None).await); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, None) + .await + .is_ok() + ); + assert!( + handle_retryable_error::(clone(&error), 3, &policy, None) + .await + .is_err() + ); // Context added on the way up must not hide the error. let error = error.context("LLM request failed"); - assert!(handle_retryable_error::(&error, 1, &policy, None).await); + assert!( + handle_retryable_error::(clone(&error), 1, &policy, None) + .await + .is_ok() + ); } #[tokio::test] @@ -298,7 +374,11 @@ mod tests { max_retries: 0, max_delay_secs: 0, }; - assert!(handle_retryable_error::(&error, 1000, &policy, None).await); + assert!( + handle_retryable_error::(clone(&error), 1000, &policy, None) + .await + .is_ok() + ); } #[tokio::test] @@ -324,7 +404,11 @@ mod tests { }); let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); let policy = three_retries(); - assert!(handle_retryable_error::(&error, 0, &policy, Some(&callback)).await); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, Some(&callback)) + .await + .is_ok() + ); assert_eq!( *seen.lock().unwrap(), vec![("Network error: connection reset".to_string(), 1, Some(3))] @@ -334,7 +418,9 @@ mod tests { let cancelled: StreamingCallback = Box::new(|_: &StreamingChunk| Err(anyhow::anyhow!("stopped"))); assert!( - !handle_retryable_error::(&error, 0, &policy, Some(&cancelled)).await + handle_retryable_error::(clone(&error), 0, &policy, Some(&cancelled)) + .await + .is_err() ); } @@ -342,10 +428,18 @@ mod tests { async fn other_errors_are_not_retried() { let error = anyhow::anyhow!("LLM stream stalled: no data received for 300s"); let policy = three_retries(); - assert!(!handle_retryable_error::(&error, 0, &policy, None).await); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, None) + .await + .is_err() + ); let error: anyhow::Error = ApiError::InvalidRequest("bad".into()).into(); - assert!(!handle_retryable_error::(&error, 0, &policy, None).await); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, None) + .await + .is_err() + ); } #[test] diff --git a/crates/llm/src/vertex.rs b/crates/llm/src/vertex.rs index c1e8c4826..5755522be 100644 --- a/crates/llm/src/vertex.rs +++ b/crates/llm/src/vertex.rs @@ -1032,18 +1032,14 @@ impl VertexClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } diff --git a/docs/retry-policy.md b/docs/retry-policy.md index 0d4436ae8..e6aee5b02 100644 --- a/docs/retry-policy.md +++ b/docs/retry-policy.md @@ -17,7 +17,8 @@ Failed requests*: - `max_retries` bounds the retries of one request; after the last one the session fails and shows the error. - `max_delay_secs` bounds the wait before a retry, also when the provider asks - for a longer one (`retry-after`, rate limit reset headers). + for a longer one (`retry-after`, rate limit reset headers). Below one second + it counts as one, so retrying forever never sends requests back to back. Missing fields take the defaults shown above. A changed policy applies to the next request a session starts: clients and the agent's hooks are created per @@ -31,10 +32,13 @@ run. errors, overloaded, unreachable or failing servers (one second, doubled per retry). The factory hands each client the policy (`LLMProvider::set_retry_policy`); Ollama and the WebSocket client keep - their own behaviour. + their own behaviour. When a client gives up on an error it could have + retried, it marks it `llm::retry::RetriesExhausted`. - **The agent loop** (`code_assistant_core::plugins::DefaultRecovery`) retries what the clients do not: a broken or stalled stream and an empty answer (`agent_core::types::EmptyResponse`). It starts at two seconds, doubling. + An error marked `RetriesExhausted` is not retried again here, so the retries + of the two places never multiply. While waiting, the session shows `RateLimited` (countdown) or `Retrying` (reason and `attempt/max`, only `attempt` without a limit). From 95c1631a88e82c9fb9e3e028aad309c6823bb613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 17 Nov 2026 13:04:26 +0100 Subject: [PATCH 4/5] refactor: drop retrying forever; retries are always limited A model that answers a prompt empty may do so every time, so unlimited retries could send paid requests until someone stops the session. The policy is now just the number of retries and the longest wait. --- AGENTS.md | 3 +- crates/agent_core/src/hooks.rs | 2 +- crates/agent_core/src/runtime.rs | 7 +--- crates/agent_core/src/runtime/tests.rs | 4 +- crates/agent_core/src/ui.rs | 2 +- .../src/plugins/recovery.rs | 26 ++++-------- .../src/session/instance.rs | 12 +++--- crates/code_assistant_core/src/ui/mod.rs | 4 +- crates/llm/src/lib.rs | 3 +- crates/llm/src/retry.rs | 40 ++----------------- crates/llm/src/utils.rs | 20 +--------- .../src/messages/activity_indicator.rs | 5 +-- .../src/settings_screen/general_section.rs | 40 +------------------ crates/ui_terminal/src/renderer.rs | 9 ++--- crates/ui_terminal/src/ui.rs | 7 +--- docs/retry-policy.md | 7 +--- 16 files changed, 40 insertions(+), 151 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fcb24f6e6..1dc8ee528 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,8 +95,7 @@ Layer 5 (binary): code_assistant — CLI, config, feature-gated frontend ### Retrying Failed Requests - One policy (`llm::RetryPolicy`, `/retry.json`, *Settings → - General*): retry forever or at most N times, and the longest wait between - retries + General*): at most N retries, and the longest wait between retries - LLM clients retry rate limits and transient server/network errors; the agent loop (`plugins::DefaultRecovery`) retries broken streams and empty answers (`agent_core::types::EmptyResponse`), which used to end a turn diff --git a/crates/agent_core/src/hooks.rs b/crates/agent_core/src/hooks.rs index d6d688457..889d91f23 100644 --- a/crates/agent_core/src/hooks.rs +++ b/crates/agent_core/src/hooks.rs @@ -151,7 +151,7 @@ pub enum RecoveryAction { RetryStream { delay: Duration, attempt: u32, - max_attempts: Option, + max_attempts: u32, }, /// Not recoverable; propagate the error. Fail, diff --git a/crates/agent_core/src/runtime.rs b/crates/agent_core/src/runtime.rs index 15c4aa672..eca2ef372 100644 --- a/crates/agent_core/src/runtime.rs +++ b/crates/agent_core/src/runtime.rs @@ -1165,14 +1165,11 @@ impl AgentRuntime { &self, error: &anyhow::Error, attempt: u32, - max_attempts: Option, + max_attempts: u32, delay: std::time::Duration, ) { warn!( - "LLM request failed (retry {}), retrying in {:?}: {:#}", - llm::retry::attempt_label(attempt, max_attempts), - delay, - error + "LLM request failed (retry {attempt}/{max_attempts}), retrying in {delay:?}: {error:#}" ); // get_next_assistant_message already sent StreamingStopped for the diff --git a/crates/agent_core/src/runtime/tests.rs b/crates/agent_core/src/runtime/tests.rs index c1778ea57..538225260 100644 --- a/crates/agent_core/src/runtime/tests.rs +++ b/crates/agent_core/src/runtime/tests.rs @@ -767,7 +767,7 @@ fn a_message_queued_after_compaction_joins_the_handoff() { assert_eq!(texts[1], "Also this"); } -/// Retries every failed request at once, as often as asked. +/// Retries every failed request at once. struct RetryAlways; impl RecoveryPolicy for RetryAlways { @@ -775,7 +775,7 @@ impl RecoveryPolicy for RetryAlways { RecoveryAction::RetryStream { delay: std::time::Duration::ZERO, attempt: completed_retries + 1, - max_attempts: None, + max_attempts: u32::MAX, } } } diff --git a/crates/agent_core/src/ui.rs b/crates/agent_core/src/ui.rs index af6483a87..242bf9317 100644 --- a/crates/agent_core/src/ui.rs +++ b/crates/agent_core/src/ui.rs @@ -227,7 +227,7 @@ pub trait AgentUi: Send + Sync { /// The LLM request failed before anything was streamed and is sent /// again (`attempt` of `max_attempts`). Cleared by the next response. - fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: Option) {} + fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: u32) {} } /// Predicate deciding whether a tool's invocation is hidden from the UI. diff --git a/crates/code_assistant_core/src/plugins/recovery.rs b/crates/code_assistant_core/src/plugins/recovery.rs index 61f423341..a9719ff54 100644 --- a/crates/code_assistant_core/src/plugins/recovery.rs +++ b/crates/code_assistant_core/src/plugins/recovery.rs @@ -39,7 +39,7 @@ impl RecoveryPolicy for DefaultRecovery { return RecoveryAction::RetryStream { delay: self.policy.backoff(RETRY_BASE_DELAY, completed_retries), attempt: completed_retries + 1, - max_attempts: self.policy.max_attempts(), + max_attempts: self.policy.max_retries, }; } RecoveryAction::Fail @@ -117,7 +117,6 @@ mod tests { fn two_retries() -> DefaultRecovery { DefaultRecovery::new(RetryPolicy { - retry_forever: false, max_retries: 2, max_delay_secs: 300, }) @@ -143,7 +142,7 @@ mod tests { } => { assert_eq!(delay, Duration::from_secs(2)); assert_eq!(attempt, 1); - assert_eq!(max_attempts, Some(2)); + assert_eq!(max_attempts, 2); } _ => panic!("expected RetryStream"), } @@ -200,24 +199,15 @@ mod tests { } #[test] - fn retrying_forever_keeps_retrying_with_capped_delay() { + fn the_retry_delay_is_capped() { let recovery = DefaultRecovery::new(RetryPolicy { - retry_forever: true, - max_retries: 0, + max_retries: 200, max_delay_secs: 30, }); - match recovery.classify(&EmptyResponse.into(), 100) { - RecoveryAction::RetryStream { - delay, - attempt, - max_attempts, - } => { - assert_eq!(delay, Duration::from_secs(30)); - assert_eq!(attempt, 101); - assert_eq!(max_attempts, None); - } - _ => panic!("expected RetryStream"), - } + assert!(matches!( + recovery.classify(&EmptyResponse.into(), 100), + RecoveryAction::RetryStream { delay, attempt: 101, .. } if delay == Duration::from_secs(30) + )); } #[test] diff --git a/crates/code_assistant_core/src/session/instance.rs b/crates/code_assistant_core/src/session/instance.rs index 169d21d5f..18f512fc2 100644 --- a/crates/code_assistant_core/src/session/instance.rs +++ b/crates/code_assistant_core/src/session/instance.rs @@ -36,7 +36,7 @@ pub enum SessionActivityState { Retrying { reason: String, attempt: u32, - max_attempts: Option, + max_attempts: u32, }, /// Agent terminated with an error Errored { message: String }, @@ -162,7 +162,7 @@ impl SessionActivity { &self, reason: &str, attempt: u32, - max_attempts: Option, + max_attempts: u32, ) -> Option { self.try_transition(SessionActivityState::Retrying { reason: reason.to_string(), @@ -1394,7 +1394,7 @@ impl UserInterface for SessionEventPublisher { self.publish_activity_change(self.activity.on_rate_limit_cleared()); } - fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: Option) { + fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: u32) { self.publish_activity_change(self.activity.on_retrying(reason, attempt, max_attempts)); } @@ -1588,12 +1588,12 @@ mod tests { let retrying = SessionActivityState::Retrying { reason: "no response".into(), attempt: 1, - max_attempts: Some(3), + max_attempts: 3, }; let activity = activity_with(SessionActivityState::WaitingForResponse); assert_eq!( - activity.on_retrying("no response", 1, Some(3)), + activity.on_retrying("no response", 1, 3), Some(retrying.clone()) ); assert_eq!( @@ -1609,7 +1609,7 @@ mod tests { // A retry notice after completion doesn't revive the session. let activity = activity_with(SessionActivityState::Idle); - assert_eq!(activity.on_retrying("no response", 1, Some(3)), None); + assert_eq!(activity.on_retrying("no response", 1, 3), None); } fn test_publisher(activity: SessionActivity, session_id: &str) -> SessionEventPublisher { diff --git a/crates/code_assistant_core/src/ui/mod.rs b/crates/code_assistant_core/src/ui/mod.rs index 8526e5730..384616149 100644 --- a/crates/code_assistant_core/src/ui/mod.rs +++ b/crates/code_assistant_core/src/ui/mod.rs @@ -43,7 +43,7 @@ pub trait UserInterface: Send + Sync { /// The LLM request failed before anything was streamed and is sent /// again (`attempt` of `max_attempts`). Cleared by the next response. - fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: Option) {} + fn notify_retrying(&self, _reason: &str, _attempt: u32, _max_attempts: u32) {} /// Ask the user multiple-choice questions and wait for the answers /// (`ask_question` tool). Default: unsupported — only frontends with a @@ -106,7 +106,7 @@ impl agent_core::AgentUi for AgentUiAdapter { self.inner.clear_rate_limit(); } - fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: Option) { + fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: u32) { self.inner.notify_retrying(reason, attempt, max_attempts); } } diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index b8cc3ffa6..aa5ce763d 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -90,8 +90,7 @@ pub enum StreamingChunk { reason: String, /// The retry about to happen, starting at 1 attempt: u32, - /// `None` when retries go on until the request succeeds - max_attempts: Option, + max_attempts: u32, }, /// Indicates that streaming from the LLM has completed StreamingComplete, diff --git a/crates/llm/src/retry.rs b/crates/llm/src/retry.rs index fc764ee53..4f8c79e54 100644 --- a/crates/llm/src/retry.rs +++ b/crates/llm/src/retry.rs @@ -14,9 +14,6 @@ const FILE_NAME: &str = "retry.json"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct RetryPolicy { - /// Send a failed request again until it succeeds or the user stops the - /// run; `max_retries` is ignored then. - pub retry_forever: bool, /// Retries of one request before the run fails. pub max_retries: u32, /// Upper bound for the wait before a retry, also when the provider asks @@ -27,7 +24,6 @@ pub struct RetryPolicy { impl Default for RetryPolicy { fn default() -> Self { Self { - retry_forever: false, max_retries: 5, max_delay_secs: 300, } @@ -69,16 +65,11 @@ impl RetryPolicy { /// Whether another retry may follow `completed` ones. pub fn allows_retry(&self, completed: u32) -> bool { - self.retry_forever || completed < self.max_retries - } - - /// The number of retries, `None` when there is no limit. - pub fn max_attempts(&self) -> Option { - (!self.retry_forever).then_some(self.max_retries) + completed < self.max_retries } /// `delay` cut to the configured maximum. A maximum below one second - /// counts as one, so retrying forever never turns into a busy loop. + /// counts as one, so retries never follow each other back to back. pub fn cap(&self, delay: Duration) -> Duration { delay.min(Duration::from_secs(self.max_delay_secs.max(1))) } @@ -99,14 +90,6 @@ pub struct RetriesExhausted { pub retries: u32, } -/// "3/5" for the third of five retries, "3" without a limit. -pub fn attempt_label(attempt: u32, max_attempts: Option) -> String { - match max_attempts { - Some(max) => format!("{attempt}/{max}"), - None => attempt.to_string(), - } -} - #[cfg(test)] mod tests { use super::*; @@ -114,26 +97,12 @@ mod tests { #[test] fn a_limited_policy_stops_after_max_retries() { let policy = RetryPolicy { - retry_forever: false, max_retries: 2, max_delay_secs: 60, }; assert!(policy.allows_retry(0)); assert!(policy.allows_retry(1)); assert!(!policy.allows_retry(2)); - assert_eq!(policy.max_attempts(), Some(2)); - } - - #[test] - fn retrying_forever_never_stops() { - let policy = RetryPolicy { - retry_forever: true, - max_retries: 0, - max_delay_secs: 60, - }; - assert!(policy.allows_retry(0)); - assert!(policy.allows_retry(u32::MAX)); - assert_eq!(policy.max_attempts(), None); } #[test] @@ -179,13 +148,12 @@ mod tests { let path = dir.path().join(FILE_NAME); assert_eq!(RetryPolicy::load_from(&path), RetryPolicy::default()); - std::fs::write(&path, r#"{"retry_forever": true}"#).unwrap(); + std::fs::write(&path, r#"{"max_retries": 9}"#).unwrap(); let loaded = RetryPolicy::load_from(&path); - assert!(loaded.retry_forever); + assert_eq!(loaded.max_retries, 9); assert_eq!(loaded.max_delay_secs, RetryPolicy::default().max_delay_secs); let policy = RetryPolicy { - retry_forever: false, max_retries: 7, max_delay_secs: 42, }; diff --git a/crates/llm/src/utils.rs b/crates/llm/src/utils.rs index 1515bdfbb..91b220811 100644 --- a/crates/llm/src/utils.rs +++ b/crates/llm/src/utils.rs @@ -279,7 +279,7 @@ async fn announce_retry( let chunk = StreamingChunk::Retrying { reason: error.to_string(), attempt: attempts + 1, - max_attempts: policy.max_attempts(), + max_attempts: policy.max_retries, }; if callback(&chunk).is_err() { return false; @@ -336,7 +336,6 @@ mod tests { /// Three retries without waiting. fn three_retries() -> RetryPolicy { RetryPolicy { - retry_forever: false, max_retries: 3, max_delay_secs: 0, } @@ -366,21 +365,6 @@ mod tests { ); } - #[tokio::test] - async fn retrying_forever_has_no_limit() { - let error: anyhow::Error = ApiError::RateLimit("slow down".into()).into(); - let policy = RetryPolicy { - retry_forever: true, - max_retries: 0, - max_delay_secs: 0, - }; - assert!( - handle_retryable_error::(clone(&error), 1000, &policy, None) - .await - .is_ok() - ); - } - #[tokio::test] async fn retries_are_announced_and_can_be_cancelled() { use std::sync::{Arc, Mutex}; @@ -411,7 +395,7 @@ mod tests { ); assert_eq!( *seen.lock().unwrap(), - vec![("Network error: connection reset".to_string(), 1, Some(3))] + vec![("Network error: connection reset".to_string(), 1, 3)] ); // A stop request (callback error) ends the retry loop. diff --git a/crates/ui_gpui/src/messages/activity_indicator.rs b/crates/ui_gpui/src/messages/activity_indicator.rs index af30095bc..408522f9c 100644 --- a/crates/ui_gpui/src/messages/activity_indicator.rs +++ b/crates/ui_gpui/src/messages/activity_indicator.rs @@ -104,10 +104,7 @@ pub fn render_activity_indicator( max_attempts, } => ( cx.theme().warning, - Some(format!( - "{reason} — retrying ({})…", - llm::retry::attempt_label(attempt, max_attempts) - )), + Some(format!("{reason} — retrying ({attempt}/{max_attempts})…")), ), SessionActivityState::Thinking => (cx.theme().primary, Some("Thinking…".to_string())), SessionActivityState::WaitingForResponse => (cx.theme().primary, None), diff --git a/crates/ui_gpui/src/settings_screen/general_section.rs b/crates/ui_gpui/src/settings_screen/general_section.rs index ce12a3924..a786c0528 100644 --- a/crates/ui_gpui/src/settings_screen/general_section.rs +++ b/crates/ui_gpui/src/settings_screen/general_section.rs @@ -40,7 +40,6 @@ pub struct GeneralSection { auto_mode_rules_input: Entity, _auto_mode_rules_subscription: Subscription, /// How failed LLM requests are sent again; stored in `retry.json`. - retry_forever: bool, max_retries_input: Entity, _max_retries_subscription: Subscription, max_retry_delay_input: Entity, @@ -134,7 +133,6 @@ impl GeneralSection { _auto_mode_model_subscription: auto_mode_model_subscription, auto_mode_rules_input, _auto_mode_rules_subscription: auto_mode_rules_subscription, - retry_forever: retry.retry_forever, max_retries_input, _max_retries_subscription: max_retries_subscription, max_retry_delay_input, @@ -303,12 +301,6 @@ impl GeneralSection { Self::save_handoff_config(|config| config.max_images_per_request = max_images_per_request); } - fn set_retry_forever(&mut self, enabled: bool, cx: &mut Context) { - self.retry_forever = enabled; - Self::save_retry_policy(|policy| policy.retry_forever = enabled); - cx.notify(); - } - /// Save the number of retries whenever the input holds a valid number. fn on_max_retries_input( &mut self, @@ -578,42 +570,14 @@ impl Render for GeneralSection { .flex() .items_center() .gap_2() - .child( - Switch::new("retry-forever") - .checked(self.retry_forever) - .with_size(Size::Small) - .on_click({ - let view = cx.entity(); - move |enabled, _window, app| { - let enabled = *enabled; - view.update(app, |this, cx| { - this.set_retry_forever(enabled, cx); - }); - } - }), - ) + .child(div().w(px(80.)).child(Input::new(&self.max_retries_input))) .child( div() .text_sm() .text_color(cx.theme().foreground) - .child("Retry until the request succeeds"), + .child("retries at most"), ), ) - .when(!self.retry_forever, |el| { - el.child( - div() - .flex() - .items_center() - .gap_2() - .child(div().w(px(80.)).child(Input::new(&self.max_retries_input))) - .child( - div() - .text_sm() - .text_color(cx.theme().foreground) - .child("retries at most"), - ), - ) - }) .child( div() .flex() diff --git a/crates/ui_terminal/src/renderer.rs b/crates/ui_terminal/src/renderer.rs index aadaa9c05..0eda53764 100644 --- a/crates/ui_terminal/src/renderer.rs +++ b/crates/ui_terminal/src/renderer.rs @@ -31,7 +31,7 @@ pub enum SpinnerState { Retrying { start_time: Instant, attempt: u32, - max_attempts: Option, + max_attempts: u32, }, } @@ -66,10 +66,7 @@ impl SpinnerState { attempt, max_attempts, .. - } => Some(format!( - "Retrying ({})", - llm::retry::attempt_label(*attempt, *max_attempts) - )), + } => Some(format!("Retrying ({attempt}/{max_attempts})")), } } } @@ -608,7 +605,7 @@ impl TerminalRenderer { } /// Show that the LLM request is sent again - pub fn show_retry_spinner(&mut self, attempt: u32, max_attempts: Option) { + pub fn show_retry_spinner(&mut self, attempt: u32, max_attempts: u32) { self.spinner_state = SpinnerState::Retrying { start_time: Instant::now(), attempt, diff --git a/crates/ui_terminal/src/ui.rs b/crates/ui_terminal/src/ui.rs index f7911e087..9f871b073 100644 --- a/crates/ui_terminal/src/ui.rs +++ b/crates/ui_terminal/src/ui.rs @@ -837,11 +837,8 @@ impl UserInterface for TerminalUI { }); } - fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: Option) { - debug!( - "Retrying LLM request ({}): {reason}", - llm::retry::attempt_label(attempt, max_attempts) - ); + fn notify_retrying(&self, reason: &str, attempt: u32, max_attempts: u32) { + debug!("Retrying LLM request ({attempt}/{max_attempts}): {reason}"); let rt = tokio::runtime::Handle::current(); let renderer = self.renderer.clone(); diff --git a/docs/retry-policy.md b/docs/retry-policy.md index e6aee5b02..df7f91015 100644 --- a/docs/retry-policy.md +++ b/docs/retry-policy.md @@ -6,19 +6,16 @@ Failed requests*: ```json { - "retry_forever": false, "max_retries": 5, "max_delay_secs": 300 } ``` -- `retry_forever` sends a request again until it succeeds or the user stops - the session; `max_retries` is ignored then. - `max_retries` bounds the retries of one request; after the last one the session fails and shows the error. - `max_delay_secs` bounds the wait before a retry, also when the provider asks for a longer one (`retry-after`, rate limit reset headers). Below one second - it counts as one, so retrying forever never sends requests back to back. + it counts as one, so retries never follow each other back to back. Missing fields take the defaults shown above. A changed policy applies to the next request a session starts: clients and the agent's hooks are created per @@ -41,7 +38,7 @@ run. of the two places never multiply. While waiting, the session shows `RateLimited` (countdown) or `Retrying` -(reason and `attempt/max`, only `attempt` without a limit). +(reason and `attempt/max`). ## Empty answers From 78837743ef442c8e770c4a2c39e8ea379d9097eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20A=C3=9Fmus?= Date: Tue, 17 Nov 2026 17:18:25 +0100 Subject: [PATCH 5/5] chore(deny): note the new zip advisories and chrono being unmaintained None of the zip code paths is reached: nothing extracts to the file system, reads archives as a stream, decrypts entries or builds legacy-zip. chrono has no maintained release to move to. Also update the yanked async-executor. --- Cargo.lock | 4 ++-- deny.toml | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbb2651f6..46de65ca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -571,9 +571,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37" dependencies = [ "async-task", "concurrent-queue", diff --git a/deny.toml b/deny.toml index fc05670fc..2c2301ee9 100644 --- a/deny.toml +++ b/deny.toml @@ -35,6 +35,15 @@ ignore = [ { id = "RUSTSEC-2026-0368", reason = "zip extract() follows symlinks out of the target dir; not reachable, see RUSTSEC-2026-0367" }, { id = "RUSTSEC-2026-0369", reason = "zip extract() applies permissions through symlinks; not reachable, see RUSTSEC-2026-0367" }, { id = "RUSTSEC-2026-0429", reason = "zip extract() aborts on a symlink entry with a large declared size; not reachable, see RUSTSEC-2026-0367" }, + { id = "RUSTSEC-2026-0546", reason = "zip extract() writes through chained symlinks; not reachable, see RUSTSEC-2026-0367" }, + { id = "RUSTSEC-2026-0548", reason = "ZipStreamReader trusts local-header names; nothing reads archives as a stream" }, + { id = "RUSTSEC-2026-0545", reason = "zip AES entries skip the HMAC on short data; nothing decrypts entries (app_update reads without a password)" }, + { id = "RUSTSEC-2026-0547", reason = "zip legacy Shrink/Reduce/Implode decoders over-allocate; the legacy-zip feature is not built" }, + # chrono has no maintainer any more and no release to move to. Our crates + # use it for timestamps; gpui, rmcp, oauth2, schemars and the office + # converters pull it in as well, so leaving it means more than our own + # code. Drop this once the workspace has moved to jiff (or time). + { id = "RUSTSEC-2026-0543", reason = "chrono is unmaintained; no replacement release, used across the dependency graph" }, # zip 8.x also pins zstd 0.13; the fixes are in zstd 0.14 / zstd-safe 8.2. # zip builds zstd without default features (no `legacy`, no `seekable`) # and makes a fresh context per stream. Drop with the zip entries above.