diff --git a/AGENTS.md b/AGENTS.md index e63ed4d31..1dc8ee528 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,14 @@ Layer 5 (binary): code_assistant — CLI, config, feature-gated frontend `SessionService::restart_mcp_server`) without restarting the app - Details: `docs/mcp-client-mode.md`, `docs/project-scoped-mcp-servers.md` +### Retrying Failed Requests +- One policy (`llm::RetryPolicy`, `/retry.json`, *Settings → + General*): at most N retries, and the longest wait between retries +- LLM clients retry rate limits and transient server/network errors; the + agent loop (`plugins::DefaultRecovery`) retries broken streams and empty + answers (`agent_core::types::EmptyResponse`), which used to end a turn + silently with *Resume*; see `docs/retry-policy.md` + ### Built-in Tool Toggles - Groups of built-in tools (browser, web, terminal, sub-agents, wakeups, session history) switch on/off like MCP servers: globally in `tools.json` diff --git a/Cargo.lock b/Cargo.lock index bbb2651f6..46de65ca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -571,9 +571,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37" dependencies = [ "async-task", "concurrent-queue", diff --git a/crates/agent_core/src/runtime.rs b/crates/agent_core/src/runtime.rs index b45d5957e..eca2ef372 100644 --- a/crates/agent_core/src/runtime.rs +++ b/crates/agent_core/src/runtime.rs @@ -428,10 +428,16 @@ impl AgentRuntime { let reserved_node_id = self.reserve_node_id(); // 1. Get LLM response (without adding to history yet) - let (llm_response, request_id) = match self + let response = self .get_next_assistant_message(messages, reserved_node_id) .await - { + .and_then(|(response, request_id)| { + if response.content.is_empty() { + return Err(crate::types::EmptyResponse.into()); + } + Ok((response, request_id)) + }); + let (llm_response, request_id) = match response { Ok(result) => { // Successful response — reset the retry counters streaming_retry_count = 0; @@ -476,14 +482,12 @@ impl AgentRuntime { self.cancellation.check()?; // 2. Add original LLM response to message history using the pre-allocated node_id - if !llm_response.content.is_empty() { - self.append_message_with_node_id( - Message::new_assistant_content(llm_response.content.clone()) - .with_request_id(request_id) - .with_usage(llm_response.usage.clone()), - reserved_node_id, - )?; - } + self.append_message_with_node_id( + Message::new_assistant_content(llm_response.content.clone()) + .with_request_id(request_id) + .with_usage(llm_response.usage.clone()), + reserved_node_id, + )?; // 3. Extract tool requests from LLM response and get truncated response let (tool_requests, flow, truncated_response) = self @@ -1155,8 +1159,8 @@ impl AgentRuntime { Ok(()) } - /// Informs the user that a transient streaming failure is being retried and - /// tells the UI to discard all partial content from the failed request. + /// Tells the user that a failed request is sent again, and the UI to + /// discard all partial content from it. async fn prepare_streaming_retry( &self, error: &anyhow::Error, @@ -1165,27 +1169,19 @@ impl AgentRuntime { delay: std::time::Duration, ) { warn!( - "Transient streaming error (attempt {}/{}), retrying in {:?}: {}", - attempt, max_attempts, delay, error + "LLM request failed (retry {attempt}/{max_attempts}), retrying in {delay:?}: {error:#}" ); - // get_next_assistant_message already sent StreamingStopped{error: ...} for - // the failed request, so the UI knows streaming ended. Now we tell it to + // get_next_assistant_message already sent StreamingStopped for the + // failed request, so the UI knows streaming ended. Now we tell it to // also remove whatever was already rendered. let _ = self .send_ui(AgentUiEvent::RollbackStreaming { request_id: self.next_request_id - 1, }) .await; - - let _ = self - .send_ui(AgentUiEvent::ShowTransientStatus { - message: format!( - "Stream interrupted — retrying ({}/{})\u{2026}", - attempt, max_attempts - ), - }) - .await; + self.ui + .notify_retrying(&error.to_string(), attempt, max_attempts); } /// Project the largest results from the most recent turn as small error diff --git a/crates/agent_core/src/runtime/tests.rs b/crates/agent_core/src/runtime/tests.rs index 1442f1def..538225260 100644 --- a/crates/agent_core/src/runtime/tests.rs +++ b/crates/agent_core/src/runtime/tests.rs @@ -766,3 +766,41 @@ fn a_message_queued_after_compaction_joins_the_handoff() { assert!(texts[0].starts_with(""), "{texts:?}"); assert_eq!(texts[1], "Also this"); } + +/// Retries every failed request at once. +struct RetryAlways; + +impl RecoveryPolicy for RetryAlways { + fn classify(&self, _: &anyhow::Error, completed_retries: u32) -> RecoveryAction { + RecoveryAction::RetryStream { + delay: std::time::Duration::ZERO, + attempt: completed_retries + 1, + max_attempts: u32::MAX, + } + } +} + +#[tokio::test] +async fn an_empty_answer_is_a_failure_not_the_end_of_the_turn() { + let (mut runtime, llm) = runtime_answering(vec![vec![]]); + runtime.append_message(Message::new_user("Hi")).unwrap(); + + let error = runtime.run_single_iteration().await.unwrap_err(); + + assert!(error.is::(), "{error:#}"); + assert_eq!(llm.requests().len(), 1); +} + +#[tokio::test] +async fn an_empty_answer_is_retried_under_the_recovery_policy() { + let (mut runtime, llm) = runtime_answering(vec![vec![], vec![ContentBlock::new_text("Hello")]]); + runtime.hooks.recovery = Box::new(RetryAlways); + runtime.append_message(Message::new_user("Hi")).unwrap(); + + runtime.run_single_iteration().await.unwrap(); + + assert_eq!(llm.requests().len(), 2); + let messages: Vec<_> = runtime.conversation.active_messages().collect(); + assert_eq!(messages.len(), 2, "the empty answer is not recorded"); + assert_eq!(messages[1].role, MessageRole::Assistant); +} diff --git a/crates/agent_core/src/types.rs b/crates/agent_core/src/types.rs index 2f45fe531..1819d39a4 100644 --- a/crates/agent_core/src/types.rs +++ b/crates/agent_core/src/types.rs @@ -86,6 +86,13 @@ impl ToolResult for ParseError { } } +/// The model answered with nothing: no text, no reasoning, no tool call. +/// Ending the turn there would leave the request unanswered and the session +/// waiting for a resume, so the agent loop treats it as a failed request. +#[derive(Debug, thiserror::Error)] +#[error("The model returned an empty response")] +pub struct EmptyResponse; + /// Placeholder result used when a tool's output was too large and caused a /// "prompt too long" error from the LLM provider. The original tool execution /// is replaced with this so the LLM gets actionable feedback on the next turn. diff --git a/crates/code_assistant_core/src/agent/tests.rs b/crates/code_assistant_core/src/agent/tests.rs index d8311fd27..884b0b4bf 100644 --- a/crates/code_assistant_core/src/agent/tests.rs +++ b/crates/code_assistant_core/src/agent/tests.rs @@ -524,7 +524,7 @@ async fn test_context_compaction_inserts_summary() -> Result<()> { }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -667,7 +667,7 @@ async fn test_compaction_reminds_about_active_skills() -> Result<()> { rate_limit_info: None, }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -764,7 +764,7 @@ async fn test_compaction_prompt_not_persisted_in_history() -> Result<()> { }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -889,7 +889,7 @@ async fn test_context_compaction_uses_only_messages_after_previous_summary() -> }; let idle_response = LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, }; @@ -2575,7 +2575,7 @@ fn text_response(text: &str) -> LLMResponse { fn idle_response() -> LLMResponse { LLMResponse { - content: Vec::new(), + content: vec![ContentBlock::new_text("Done")], usage: Usage::zero(), rate_limit_info: None, } diff --git a/crates/code_assistant_core/src/plugins/mod.rs b/crates/code_assistant_core/src/plugins/mod.rs index 233a378d4..a1ebf7ce3 100644 --- a/crates/code_assistant_core/src/plugins/mod.rs +++ b/crates/code_assistant_core/src/plugins/mod.rs @@ -36,7 +36,7 @@ pub fn default_hooks() -> HookRegistry { observers: vec![], dispatch: Box::new(SpawnAgentParallelPolicy), compaction: Box::new(TokenRatioCompaction::new(0.8)), - recovery: Box::new(DefaultRecovery), + recovery: Box::new(DefaultRecovery::new(llm::RetryPolicy::load())), system_prompt: Box::new(CodeAssistantSystemPrompt::new()), } } diff --git a/crates/code_assistant_core/src/plugins/recovery.rs b/crates/code_assistant_core/src/plugins/recovery.rs index fa8791421..a9719ff54 100644 --- a/crates/code_assistant_core/src/plugins/recovery.rs +++ b/crates/code_assistant_core/src/plugins/recovery.rs @@ -1,16 +1,26 @@ //! Recovery policy: classifies failed LLM requests by provider error text. use agent_core::hooks::{RecoveryAction, RecoveryPolicy}; +use agent_core::types::EmptyResponse; +use llm::RetryPolicy; +use llm::retry::RetriesExhausted; use std::time::Duration; -/// Maximum number of retries for transient streaming errors (e.g. HTTP chunk -/// errors, connection resets). -const MAX_STREAMING_RETRIES: u32 = 2; +/// Wait before the first retry of a broken stream or an empty answer +/// (doubles on each further one, up to the policy's maximum). +const RETRY_BASE_DELAY: Duration = Duration::from_secs(2); -/// Base delay between streaming retries (doubles on each attempt). -const STREAMING_RETRY_BASE_DELAY: Duration = Duration::from_secs(2); +/// Recovers from oversized prompts by compacting, and sends a request again +/// after a broken stream or an empty answer, as often as `policy` allows. +pub struct DefaultRecovery { + policy: RetryPolicy, +} -pub struct DefaultRecovery; +impl DefaultRecovery { + pub fn new(policy: RetryPolicy) -> Self { + Self { policy } + } +} impl RecoveryPolicy for DefaultRecovery { fn classify(&self, error: &anyhow::Error, completed_retries: u32) -> RecoveryAction { @@ -20,11 +30,16 @@ impl RecoveryPolicy for DefaultRecovery { if is_prompt_too_long_error(error) { return RecoveryAction::ReduceContext; } - if is_retryable_streaming_error(error) && completed_retries < MAX_STREAMING_RETRIES { + // The client already sent it again as often as the policy allows. + if error.is::() { + return RecoveryAction::Fail; + } + let retryable = error.is::() || is_retryable_streaming_error(error); + if retryable && self.policy.allows_retry(completed_retries) { return RecoveryAction::RetryStream { - delay: STREAMING_RETRY_BASE_DELAY * 2u32.saturating_pow(completed_retries), + delay: self.policy.backoff(RETRY_BASE_DELAY, completed_retries), attempt: completed_retries + 1, - max_attempts: MAX_STREAMING_RETRIES, + max_attempts: self.policy.max_retries, }; } RecoveryAction::Fail @@ -87,6 +102,8 @@ fn is_retryable_streaming_error(error: &anyhow::Error) -> bool { // Timeout errors (read timeouts, not connect timeouts) || msg.contains("operation timed out") || msg.contains("timed out reading") + // No data for a while (`llm::streaming`) + || msg.contains("stream stalled") // Server errors that are transient || msg.contains("502 bad gateway") || msg.contains("503 service") @@ -98,11 +115,18 @@ fn is_retryable_streaming_error(error: &anyhow::Error) -> bool { mod tests { use super::*; + fn two_retries() -> DefaultRecovery { + DefaultRecovery::new(RetryPolicy { + max_retries: 2, + max_delay_secs: 300, + }) + } + #[test] fn oversized_prompt_reduces_context() { let err = anyhow::anyhow!("Invalid request: prompt is too long: 500000 tokens"); assert!(matches!( - DefaultRecovery.classify(&err, 0), + two_retries().classify(&err, 0), RecoveryAction::ReduceContext )); } @@ -110,7 +134,7 @@ mod tests { #[test] fn transient_stream_error_retries_with_backoff() { let err = anyhow::anyhow!("error reading a body from connection: connection reset"); - match DefaultRecovery.classify(&err, 0) { + match two_retries().classify(&err, 0) { RecoveryAction::RetryStream { delay, attempt, @@ -118,11 +142,11 @@ mod tests { } => { assert_eq!(delay, Duration::from_secs(2)); assert_eq!(attempt, 1); - assert_eq!(max_attempts, MAX_STREAMING_RETRIES); + assert_eq!(max_attempts, 2); } _ => panic!("expected RetryStream"), } - match DefaultRecovery.classify(&err, 1) { + match two_retries().classify(&err, 1) { RecoveryAction::RetryStream { delay, attempt, .. } => { assert_eq!(delay, Duration::from_secs(4)); assert_eq!(attempt, 2); @@ -135,14 +159,14 @@ mod tests { fn retries_are_bounded() { let err = anyhow::anyhow!("connection reset"); assert!(matches!( - DefaultRecovery.classify(&err, MAX_STREAMING_RETRIES), + two_retries().classify(&err, 2), RecoveryAction::Fail )); } #[test] fn image_limit_errors_compact_with_the_named_limit() { - let classify = |message: &str| DefaultRecovery.classify(&anyhow::anyhow!("{message}"), 0); + let classify = |message: &str| two_retries().classify(&anyhow::anyhow!("{message}"), 0); let openai = r#"Invalid request: {"error":{"message":"Exceeded maximum number of images (50) allowed in the request.","param":"input","type":"invalid_request_error"}}"#; assert!(matches!( classify(openai), @@ -158,11 +182,49 @@ mod tests { )); } + #[test] + fn empty_answers_and_stalled_streams_are_retried() { + for err in [ + anyhow::Error::from(EmptyResponse), + anyhow::anyhow!("LLM stream stalled: no data received for 300s"), + ] { + assert!( + matches!( + two_retries().classify(&err, 0), + RecoveryAction::RetryStream { attempt: 1, .. } + ), + "{err}" + ); + } + } + + #[test] + fn the_retry_delay_is_capped() { + let recovery = DefaultRecovery::new(RetryPolicy { + max_retries: 200, + max_delay_secs: 30, + }); + assert!(matches!( + recovery.classify(&EmptyResponse.into(), 100), + RecoveryAction::RetryStream { delay, attempt: 101, .. } if delay == Duration::from_secs(30) + )); + } + + #[test] + fn errors_the_client_gave_up_on_are_not_retried_again() { + let err = anyhow::anyhow!("Service overloaded: overloaded_error") + .context(RetriesExhausted { retries: 2 }); + assert!(matches!( + two_retries().classify(&err, 0), + RecoveryAction::Fail + )); + } + #[test] fn unknown_errors_fail() { let err = anyhow::anyhow!("invalid api key"); assert!(matches!( - DefaultRecovery.classify(&err, 0), + two_retries().classify(&err, 0), RecoveryAction::Fail )); } diff --git a/crates/llm/src/aicore/anthropic.rs b/crates/llm/src/aicore/anthropic.rs index 1a2930613..bfbc467b1 100644 --- a/crates/llm/src/aicore/anthropic.rs +++ b/crates/llm/src/aicore/anthropic.rs @@ -142,4 +142,8 @@ impl LLMProvider for AiCoreAnthropicClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.anthropic_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/aicore/openai.rs b/crates/llm/src/aicore/openai.rs index 4c2a7ce32..b6cb62b47 100644 --- a/crates/llm/src/aicore/openai.rs +++ b/crates/llm/src/aicore/openai.rs @@ -116,4 +116,8 @@ impl LLMProvider for AiCoreOpenAIClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.openai_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/aicore/openai_responses.rs b/crates/llm/src/aicore/openai_responses.rs index 255b43ff1..6f948bd3c 100644 --- a/crates/llm/src/aicore/openai_responses.rs +++ b/crates/llm/src/aicore/openai_responses.rs @@ -144,4 +144,8 @@ impl LLMProvider for AiCoreOpenAIResponsesClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.responses_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/aicore/vertex.rs b/crates/llm/src/aicore/vertex.rs index 23b782941..b68fbce5a 100644 --- a/crates/llm/src/aicore/vertex.rs +++ b/crates/llm/src/aicore/vertex.rs @@ -132,4 +132,8 @@ impl LLMProvider for AiCoreVertexClient { .send_message(request, streaming_callback) .await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.vertex_client.set_retry_policy(policy); + } } diff --git a/crates/llm/src/anthropic.rs b/crates/llm/src/anthropic.rs index d600b3cd8..3252b39ed 100644 --- a/crates/llm/src/anthropic.rs +++ b/crates/llm/src/anthropic.rs @@ -537,6 +537,8 @@ pub struct AnthropicClient { client: Client, base_url: String, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, recorder: Option, playback: Option, @@ -559,6 +561,7 @@ impl AnthropicClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), recorder: None, playback: None, auth_provider: Box::new(ApiKeyAuth::new(api_key)), @@ -579,6 +582,7 @@ impl AnthropicClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), recorder: Some(APIRecorder::new(recording_path)), playback: None, auth_provider: Box::new(ApiKeyAuth::new(api_key)), @@ -600,6 +604,7 @@ impl AnthropicClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), recorder: None, playback: None, auth_provider, @@ -636,13 +641,13 @@ impl AnthropicClient { &mut self, request: &serde_json::Value, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { // If playback is enabled, skip HTTP and use recorded data if self.playback.is_some() { return self.playback_request(request, streaming_callback).await; } + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -653,18 +658,14 @@ impl AnthropicClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, - max_retries, + &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } @@ -1231,6 +1232,10 @@ impl AnthropicClient { #[async_trait] impl LLMProvider for AnthropicClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1314,7 +1319,7 @@ impl LLMProvider for AnthropicClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&anthropic_request, streaming_callback, 3) + .send_with_retry(&anthropic_request, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/cerebras.rs b/crates/llm/src/cerebras.rs index a818b9691..63c84bccc 100644 --- a/crates/llm/src/cerebras.rs +++ b/crates/llm/src/cerebras.rs @@ -61,6 +61,10 @@ impl LLMProvider for CerebrasClient { // Delegate to inner OpenAI client with custom request processing self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/factory.rs b/crates/llm/src/factory.rs index b92a375d1..7a27be82a 100644 --- a/crates/llm/src/factory.rs +++ b/crates/llm/src/factory.rs @@ -6,6 +6,7 @@ use crate::auth::TokenManager; use crate::capabilities::{Effort, ModelCapabilities}; use crate::config_merge::merge_patch; use crate::provider_config::{ConfigurationSystem, LlmApiClientKind, ModelConfig, ProviderConfig}; +use crate::retry::RetryPolicy; use crate::{ AnthropicClient, CerebrasClient, GroqClient, LLMProvider, MinimaxClient, MistralAiClient, MoonshotClient, OllamaClient, OpenAIClient, OpenAIResponsesClient, OpenAIResponsesWsClient, @@ -315,7 +316,7 @@ pub async fn create_llm_client_from_configs( ..model_config.clone() }; - match provider_type { + let mut client = match provider_type { LLMProviderType::AiCore => { create_ai_core_client(model_config, provider_config, record_path.clone()).await } @@ -350,7 +351,9 @@ pub async fn create_llm_client_from_configs( LLMProviderType::OpenRouter => { create_openrouter_client(model_config, provider_config).await } - } + }?; + client.set_retry_policy(RetryPolicy::load()); + Ok(client) } /// The patch merged into every request for this model: the fields its diff --git a/crates/llm/src/groq.rs b/crates/llm/src/groq.rs index 816da3f38..572b5de87 100644 --- a/crates/llm/src/groq.rs +++ b/crates/llm/src/groq.rs @@ -61,4 +61,8 @@ impl LLMProvider for GroqClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index ff8b4011b..aa5ce763d 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -37,6 +37,7 @@ pub mod prompt_caching; pub mod provider_config; pub mod realtime; pub mod recording; +pub mod retry; pub mod streaming; pub mod types; pub mod vertex; @@ -57,6 +58,7 @@ pub use openai::OpenAIClient; pub use openai_responses::OpenAIResponsesClient; pub use openai_responses_ws::OpenAIResponsesWsClient; pub use openrouter::OpenRouterClient; +pub use retry::RetryPolicy; pub use types::*; pub use vertex::{VertexClient, sanitize_google_schema}; pub use zai::ZaiClient; @@ -114,4 +116,9 @@ pub trait LLMProvider: Send + Sync { request: LLMRequest, streaming_callback: Option<&StreamingCallback>, ) -> Result; + + /// How often and how patiently failed requests are sent again. Clients + /// that never retry (Ollama runs locally; the WebSocket client only + /// reconnects) ignore it, the agent loop still retries under it. + fn set_retry_policy(&mut self, _policy: RetryPolicy) {} } diff --git a/crates/llm/src/minimax.rs b/crates/llm/src/minimax.rs index 838dda889..b57603bc7 100644 --- a/crates/llm/src/minimax.rs +++ b/crates/llm/src/minimax.rs @@ -49,6 +49,10 @@ impl LLMProvider for MinimaxClient { // Delegate to inner Anthropic client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/mistralai.rs b/crates/llm/src/mistralai.rs index e191b404e..382cc0437 100644 --- a/crates/llm/src/mistralai.rs +++ b/crates/llm/src/mistralai.rs @@ -65,6 +65,10 @@ impl LLMProvider for MistralAiClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/moonshot.rs b/crates/llm/src/moonshot.rs index 353f55c41..f1c1238bd 100644 --- a/crates/llm/src/moonshot.rs +++ b/crates/llm/src/moonshot.rs @@ -68,6 +68,10 @@ impl LLMProvider for MoonshotClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/llm/src/openai.rs b/crates/llm/src/openai.rs index aed5b7ec6..d08988734 100644 --- a/crates/llm/src/openai.rs +++ b/crates/llm/src/openai.rs @@ -305,6 +305,8 @@ pub struct OpenAIClient { client: Client, base_url: String, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, model_temperatures: HashMap, model_top_ps: HashMap, // Customization points @@ -326,6 +328,7 @@ impl OpenAIClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), model_temperatures, model_top_ps, auth_provider: Box::new(ApiKeyAuth::new(api_key)), @@ -347,6 +350,7 @@ impl OpenAIClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), model_temperatures, model_top_ps, auth_provider, @@ -627,8 +631,8 @@ impl OpenAIClient { &self, request: &OpenAIRequest, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -644,18 +648,14 @@ impl OpenAIClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, - max_retries, + &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } @@ -1128,6 +1128,10 @@ impl OpenAIClient { #[async_trait] impl LLMProvider for OpenAIClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1182,7 +1186,7 @@ impl LLMProvider for OpenAIClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&openai_request, streaming_callback, 3) + .send_with_retry(&openai_request, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/openai_responses.rs b/crates/llm/src/openai_responses.rs index b46ef37a7..11ce8a867 100644 --- a/crates/llm/src/openai_responses.rs +++ b/crates/llm/src/openai_responses.rs @@ -593,6 +593,8 @@ pub struct OpenAIResponsesClient { client: Client, base_url: String, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, auth_provider: Box, request_customizer: Box, recorder: Option, @@ -614,6 +616,7 @@ impl OpenAIResponsesClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), auth_provider: Box::new(ApiKeyAuth::new(api_key)), request_customizer: Box::new(DefaultRequestCustomizer), recorder: None, @@ -633,6 +636,7 @@ impl OpenAIResponsesClient { client: utils::http_client(), base_url, model, + retry_policy: crate::RetryPolicy::default(), auth_provider, request_customizer, recorder: None, @@ -819,13 +823,13 @@ impl OpenAIResponsesClient { &mut self, request: &ResponsesRequest, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { // If playback is enabled, skip HTTP and use recorded data if self.playback.is_some() { return self.playback_request(request, streaming_callback).await; } + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -835,18 +839,14 @@ impl OpenAIResponsesClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, - max_retries, + &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } @@ -1422,6 +1422,10 @@ impl<'a> StreamProcessor<'a> { #[async_trait] impl LLMProvider for OpenAIResponsesClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1476,7 +1480,7 @@ impl LLMProvider for OpenAIResponsesClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&responses_request, streaming_callback, 3) + .send_with_retry(&responses_request, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/openrouter.rs b/crates/llm/src/openrouter.rs index 68cdb8dfb..e0aac8a0c 100644 --- a/crates/llm/src/openrouter.rs +++ b/crates/llm/src/openrouter.rs @@ -35,4 +35,8 @@ impl LLMProvider for OpenRouterClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } diff --git a/crates/llm/src/retry.rs b/crates/llm/src/retry.rs new file mode 100644 index 000000000..4f8c79e54 --- /dev/null +++ b/crates/llm/src/retry.rs @@ -0,0 +1,163 @@ +//! How often and how patiently a failed LLM request is sent again +//! (`retry.json`): rate limits, overloaded or unreachable servers, broken +//! streams and empty answers all follow this one policy. + +use crate::provider_config::ConfigurationSystem; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tracing::warn; + +const FILE_NAME: &str = "retry.json"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct RetryPolicy { + /// Retries of one request before the run fails. + pub max_retries: u32, + /// Upper bound for the wait before a retry, also when the provider asks + /// for a longer one; at least one second. + pub max_delay_secs: u64, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_retries: 5, + max_delay_secs: 300, + } + } +} + +impl RetryPolicy { + pub fn path() -> PathBuf { + ConfigurationSystem::config_file_path(FILE_NAME) + } + + /// The policy from the config directory; a missing or malformed file + /// yields the defaults. + pub fn load() -> Self { + Self::load_from(&Self::path()) + } + + pub fn load_from(path: &Path) -> Self { + let Ok(content) = std::fs::read_to_string(path) else { + return Self::default(); + }; + serde_json::from_str(&content).unwrap_or_else(|e| { + warn!("Failed to parse {}: {e}; using defaults", path.display()); + Self::default() + }) + } + + pub fn save(&self) -> Result<()> { + self.save_to(&Self::path()) + } + + pub fn save_to(&self, path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, serde_json::to_string_pretty(self)?) + .with_context(|| format!("Failed to write {}", path.display())) + } + + /// Whether another retry may follow `completed` ones. + pub fn allows_retry(&self, completed: u32) -> bool { + completed < self.max_retries + } + + /// `delay` cut to the configured maximum. A maximum below one second + /// counts as one, so retries never follow each other back to back. + pub fn cap(&self, delay: Duration) -> Duration { + delay.min(Duration::from_secs(self.max_delay_secs.max(1))) + } + + /// Exponential backoff for the retry after `completed` ones: `base`, + /// then doubled for each further retry, at most the configured maximum. + pub fn backoff(&self, base: Duration, completed: u32) -> Duration { + let factor = 2u32.saturating_pow(completed.min(31)); + self.cap(base.saturating_mul(factor)) + } +} + +/// Context on an error the client sent again as often as the policy +/// allows; whoever receives it should not retry it once more. +#[derive(Debug, thiserror::Error)] +#[error("gave up after {retries} retries")] +pub struct RetriesExhausted { + pub retries: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_limited_policy_stops_after_max_retries() { + let policy = RetryPolicy { + max_retries: 2, + max_delay_secs: 60, + }; + assert!(policy.allows_retry(0)); + assert!(policy.allows_retry(1)); + assert!(!policy.allows_retry(2)); + } + + #[test] + fn backoff_doubles_up_to_the_maximum_delay() { + let policy = RetryPolicy { + max_delay_secs: 10, + ..RetryPolicy::default() + }; + let base = Duration::from_secs(2); + assert_eq!(policy.backoff(base, 0), Duration::from_secs(2)); + assert_eq!(policy.backoff(base, 1), Duration::from_secs(4)); + assert_eq!(policy.backoff(base, 2), Duration::from_secs(8)); + assert_eq!(policy.backoff(base, 3), Duration::from_secs(10)); + assert_eq!(policy.backoff(base, 1000), Duration::from_secs(10)); + } + + #[test] + fn a_requested_wait_is_capped() { + let policy = RetryPolicy { + max_delay_secs: 30, + ..RetryPolicy::default() + }; + assert_eq!( + policy.cap(Duration::from_secs(3600)), + Duration::from_secs(30) + ); + assert_eq!(policy.cap(Duration::from_secs(5)), Duration::from_secs(5)); + } + + #[test] + fn a_maximum_of_zero_still_waits_a_second() { + let policy = RetryPolicy { + max_delay_secs: 0, + ..RetryPolicy::default() + }; + assert_eq!(policy.cap(Duration::from_secs(60)), Duration::from_secs(1)); + assert_eq!(policy.cap(Duration::ZERO), Duration::ZERO); + } + + #[test] + fn missing_fields_take_the_defaults_and_the_file_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(FILE_NAME); + assert_eq!(RetryPolicy::load_from(&path), RetryPolicy::default()); + + std::fs::write(&path, r#"{"max_retries": 9}"#).unwrap(); + let loaded = RetryPolicy::load_from(&path); + assert_eq!(loaded.max_retries, 9); + assert_eq!(loaded.max_delay_secs, RetryPolicy::default().max_delay_secs); + + let policy = RetryPolicy { + max_retries: 7, + max_delay_secs: 42, + }; + policy.save_to(&path).unwrap(); + assert_eq!(RetryPolicy::load_from(&path), policy); + } +} diff --git a/crates/llm/src/utils.rs b/crates/llm/src/utils.rs index b51f43064..91b220811 100644 --- a/crates/llm/src/utils.rs +++ b/crates/llm/src/utils.rs @@ -1,4 +1,7 @@ -use crate::{ApiError, ApiErrorContext, RateLimitHandler, StreamingCallback, StreamingChunk}; +use crate::{ + ApiError, ApiErrorContext, RateLimitHandler, RetryPolicy, StreamingCallback, StreamingChunk, + retry::RetriesExhausted, +}; use anyhow::Result; use reqwest::{Client, RequestBuilder, Response, StatusCode}; use std::time::Duration; @@ -126,102 +129,102 @@ pub async fn check_response_error( - error: &anyhow::Error, + error: anyhow::Error, attempts: u32, - max_retries: u32, + policy: &RetryPolicy, streaming_callback: Option<&StreamingCallback>, -) -> bool { - let (api_error, rate_limits) = if let Some(ctx) = error.downcast_ref::>() { - (&ctx.error, ctx.rate_limits.as_ref()) - } else if let Some(api_error) = error.downcast_ref::() { +) -> anyhow::Result<()> { + if retry_after_wait::(&error, attempts, policy, streaming_callback).await { + return Ok(()); + } + if is_retryable::(&error) { + return Err(error.context(RetriesExhausted { retries: attempts })); + } + Err(error) +} + +/// The provider error behind `error`, with its rate limit context. +fn api_error( + error: &anyhow::Error, +) -> Option<(&ApiError, Option<&T>)> { + if let Some(ctx) = error.downcast_ref::>() { + Some((&ctx.error, ctx.rate_limits.as_ref())) + } else { // Raised before a response arrived (connect failure, timeout while // waiting for headers), so there is no rate limit context. - (api_error, None) - } else { + error.downcast_ref::().map(|e| (e, None)) + } +} + +fn is_retryable( + error: &anyhow::Error, +) -> bool { + match api_error::(error) { + Some((ApiError::Authentication(msg), _)) => is_transient_auth_error(msg), + Some(( + ApiError::RateLimit(_) + | ApiError::ServiceError(_) + | ApiError::NetworkError(_) + | ApiError::Overloaded(_), + _, + )) => true, + _ => false, + } +} + +/// Wait before the next retry when `error` is retryable and the policy +/// allows one; false when not, or when the user cancelled while waiting. +async fn retry_after_wait( + error: &anyhow::Error, + attempts: u32, + policy: &RetryPolicy, + streaming_callback: Option<&StreamingCallback>, +) -> bool { + let Some((api_error, rate_limits)) = api_error::(error) else { return false; }; - match api_error { - ApiError::RateLimit(_) => { - if let Some(rate_limits) = rate_limits { - if attempts < max_retries { - // Get base delay from rate limit headers - let mut delay = rate_limits.get_retry_delay(); - - // Apply exponential backoff based on retry attempts - // Double the delay for each retry attempt after the first - if attempts > 0 { - let multiplier = 2u32.pow(attempts); - delay *= multiplier; - } - let delay_secs = delay.as_secs(); - warn!( - "Rate limit hit (attempt {}/{}), waiting {} seconds before retry", - attempts, max_retries, delay_secs - ); - - return wait_rate_limit(delay, streaming_callback).await; - } - } else { - // Fallback if no rate limit info available - if attempts < max_retries { - // Implement exponential backoff with a more appropriate base delay - // Start with 60 seconds for the first retry, then double for each subsequent attempt - let base_delay_secs = 60u64; - let delay = Duration::from_secs(base_delay_secs * 2u64.pow(attempts)); - warn!( - "Rate limit hit but no timing info available (attempt {}/{}), using exponential backoff: {} seconds", - attempts, - max_retries, - delay.as_secs() - ); - return wait_rate_limit(delay, streaming_callback).await; - } - } - } - ApiError::Authentication(msg) if is_transient_auth_error(msg) => { - if attempts < max_retries { - let delay = Duration::from_secs(2u64.pow(attempts.saturating_sub(1))); - warn!( - "Transient auth error: {} (attempt {}/{}), retrying in {} seconds", - error, - attempts, - max_retries, - delay.as_secs() - ); - return announce_retry(api_error, attempts, max_retries, delay, streaming_callback) - .await; - } - } - ApiError::ServiceError(_) | ApiError::NetworkError(_) | ApiError::Overloaded(_) => { - if attempts < max_retries { - let delay = Duration::from_secs(2u64.pow(attempts.saturating_sub(1))); - warn!( - "Error: {} (attempt {}/{}), retrying in {} seconds", - error, - attempts, - max_retries, - delay.as_secs() - ); - return announce_retry(api_error, attempts, max_retries, delay, streaming_callback) - .await; - } - } - _ => { - warn!( - "Unhandled error (attempt {}/{}): {:?}", - attempts, max_retries, error - ); - } + if !is_retryable::(error) { + warn!("Unhandled error (attempt {attempts}): {error:?}"); + return false; + } + if !policy.allows_retry(attempts) { + warn!("Giving up after {attempts} retries: {error}"); + return false; } - false + if let ApiError::RateLimit(_) = api_error { + // The provider's own wait (headers), else a minute; doubled for + // each further retry. + let base = rate_limits + .map(|rate_limits| rate_limits.get_retry_delay()) + .unwrap_or(Duration::from_secs(60)); + let delay = policy.backoff(base, attempts); + warn!( + "Rate limit hit (retry {}), waiting {} seconds", + attempts + 1, + delay.as_secs() + ); + return wait_rate_limit(delay, streaming_callback).await; + } + let delay = policy.backoff(Duration::from_secs(1), attempts.saturating_sub(1)); + warn!( + "Error: {} (retry {}), retrying in {} seconds", + error, + attempts + 1, + delay.as_secs() + ); + announce_retry(api_error, attempts, policy, delay, streaming_callback).await } /// Wait out a rate limit, showing a countdown when there is a callback. @@ -268,7 +271,7 @@ async fn wait_rate_limit(delay: Duration, streaming_callback: Option<&StreamingC async fn announce_retry( error: &ApiError, attempts: u32, - max_retries: u32, + policy: &RetryPolicy, delay: Duration, streaming_callback: Option<&StreamingCallback>, ) -> bool { @@ -276,7 +279,7 @@ async fn announce_retry( let chunk = StreamingChunk::Retrying { reason: error.to_string(), attempt: attempts + 1, - max_attempts: max_retries, + max_attempts: policy.max_retries, }; if callback(&chunk).is_err() { return false; @@ -303,15 +306,63 @@ mod tests { fn log_status(&self) {} } + /// A copy of a test error, which carries one `ApiError` or plain text. + fn clone(error: &anyhow::Error) -> anyhow::Error { + match error.downcast_ref::() { + Some(ApiError::NetworkError(m)) => ApiError::NetworkError(m.clone()).into(), + Some(ApiError::RateLimit(m)) => ApiError::RateLimit(m.clone()).into(), + Some(ApiError::InvalidRequest(m)) => ApiError::InvalidRequest(m.clone()).into(), + _ => anyhow::anyhow!("{error:#}"), + } + } + + #[tokio::test] + async fn giving_up_on_a_retryable_error_marks_it_exhausted() { + let policy = three_retries(); + let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); + let given_up = handle_retryable_error::(error, 3, &policy, None) + .await + .unwrap_err(); + assert!(given_up.is::(), "{given_up:#}"); + assert!(given_up.is::(), "the cause stays reachable"); + + let error: anyhow::Error = ApiError::InvalidRequest("bad".into()).into(); + let given_up = handle_retryable_error::(error, 0, &policy, None) + .await + .unwrap_err(); + assert!(!given_up.is::(), "never retryable"); + } + + /// Three retries without waiting. + fn three_retries() -> RetryPolicy { + RetryPolicy { + max_retries: 3, + max_delay_secs: 0, + } + } + #[tokio::test] async fn bare_network_errors_are_retried() { let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); - assert!(handle_retryable_error::(&error, 0, 3, None).await); - assert!(!handle_retryable_error::(&error, 3, 3, None).await); + let policy = three_retries(); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, None) + .await + .is_ok() + ); + assert!( + handle_retryable_error::(clone(&error), 3, &policy, None) + .await + .is_err() + ); // Context added on the way up must not hide the error. let error = error.context("LLM request failed"); - assert!(handle_retryable_error::(&error, 1, 3, None).await); + assert!( + handle_retryable_error::(clone(&error), 1, &policy, None) + .await + .is_ok() + ); } #[tokio::test] @@ -336,7 +387,12 @@ mod tests { } }); let error: anyhow::Error = ApiError::NetworkError("connection reset".into()).into(); - assert!(handle_retryable_error::(&error, 0, 3, Some(&callback)).await); + let policy = three_retries(); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, Some(&callback)) + .await + .is_ok() + ); assert_eq!( *seen.lock().unwrap(), vec![("Network error: connection reset".to_string(), 1, 3)] @@ -345,16 +401,29 @@ mod tests { // A stop request (callback error) ends the retry loop. let cancelled: StreamingCallback = Box::new(|_: &StreamingChunk| Err(anyhow::anyhow!("stopped"))); - assert!(!handle_retryable_error::(&error, 0, 3, Some(&cancelled)).await); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, Some(&cancelled)) + .await + .is_err() + ); } #[tokio::test] async fn other_errors_are_not_retried() { let error = anyhow::anyhow!("LLM stream stalled: no data received for 300s"); - assert!(!handle_retryable_error::(&error, 0, 3, None).await); + let policy = three_retries(); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, None) + .await + .is_err() + ); let error: anyhow::Error = ApiError::InvalidRequest("bad".into()).into(); - assert!(!handle_retryable_error::(&error, 0, 3, None).await); + assert!( + handle_retryable_error::(clone(&error), 0, &policy, None) + .await + .is_err() + ); } #[test] diff --git a/crates/llm/src/vertex.rs b/crates/llm/src/vertex.rs index d5a0abe33..5755522be 100644 --- a/crates/llm/src/vertex.rs +++ b/crates/llm/src/vertex.rs @@ -801,6 +801,8 @@ fn merge_vertex_partial_value(target: &mut serde_json::Value, value: serde_json: pub struct VertexClient { client: Client, model: String, + /// How often and how patiently failed requests are sent again. + retry_policy: crate::RetryPolicy, base_url: String, recorder: Option, custom_config: Option, @@ -818,6 +820,7 @@ impl VertexClient { Self { client: utils::http_client(), model, + retry_policy: crate::RetryPolicy::default(), base_url, recorder: None, custom_config: None, @@ -836,6 +839,7 @@ impl VertexClient { Self { client: utils::http_client(), model, + retry_policy: crate::RetryPolicy::default(), base_url, recorder: Some(APIRecorder::new(recording_path)), custom_config: None, @@ -854,6 +858,7 @@ impl VertexClient { Self { client: utils::http_client(), model, + retry_policy: crate::RetryPolicy::default(), base_url, recorder: None, custom_config: None, @@ -1011,8 +1016,8 @@ impl VertexClient { request: &VertexRequest, request_id: u64, streaming_callback: Option<&StreamingCallback>, - max_retries: u32, ) -> Result { + let policy = self.retry_policy; let mut attempts = 0; loop { @@ -1027,18 +1032,14 @@ impl VertexClient { return Ok(response); } Err(e) => { - if utils::handle_retryable_error::( - &e, + utils::handle_retryable_error::( + e, attempts, - max_retries, + &policy, streaming_callback, ) - .await - { - attempts += 1; - continue; - } - return Err(e); + .await?; + attempts += 1; } } } @@ -1435,6 +1436,10 @@ impl VertexClient { #[async_trait] impl LLMProvider for VertexClient { + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.retry_policy = policy; + } + async fn send_message( &mut self, request: LLMRequest, @@ -1508,7 +1513,7 @@ impl LLMProvider for VertexClient { let request_start = std::time::SystemTime::now(); let mut response = self - .send_with_retry(&vertex_request, request_id, streaming_callback, 3) + .send_with_retry(&vertex_request, request_id, streaming_callback) .await?; let response_end = std::time::SystemTime::now(); diff --git a/crates/llm/src/zai.rs b/crates/llm/src/zai.rs index 7832b4072..ad31ea4c9 100644 --- a/crates/llm/src/zai.rs +++ b/crates/llm/src/zai.rs @@ -68,6 +68,10 @@ impl LLMProvider for ZaiClient { // Delegate to inner OpenAI client since the APIs are compatible self.inner.send_message(request, streaming_callback).await } + + fn set_retry_policy(&mut self, policy: crate::RetryPolicy) { + self.inner.set_retry_policy(policy); + } } #[cfg(test)] diff --git a/crates/ui_gpui/src/settings_screen/general_section.rs b/crates/ui_gpui/src/settings_screen/general_section.rs index 04638d605..a786c0528 100644 --- a/crates/ui_gpui/src/settings_screen/general_section.rs +++ b/crates/ui_gpui/src/settings_screen/general_section.rs @@ -12,6 +12,7 @@ use gpui_kit::component::{ActiveTheme, Icon, Sizable, Size}; use gpui_kit::{ App, Context, Entity, FocusHandle, Focusable, SharedString, Subscription, div, prelude::*, px, }; +use llm::RetryPolicy; use tracing::warn; pub struct GeneralSection { @@ -38,6 +39,11 @@ pub struct GeneralSection { _auto_mode_model_subscription: Subscription, auto_mode_rules_input: Entity, _auto_mode_rules_subscription: Subscription, + /// How failed LLM requests are sent again; stored in `retry.json`. + max_retries_input: Entity, + _max_retries_subscription: Subscription, + max_retry_delay_input: Entity, + _max_retry_delay_subscription: Subscription, } impl GeneralSection { @@ -94,6 +100,24 @@ impl GeneralSection { window, Self::on_auto_mode_rules_input, ); + let retry = RetryPolicy::load(); + let max_retries_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(RetryPolicy::default().max_retries.to_string()) + .default_value(retry.max_retries.to_string()) + }); + let max_retries_subscription = + cx.subscribe_in(&max_retries_input, window, Self::on_max_retries_input); + let max_retry_delay_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(RetryPolicy::default().max_delay_secs.to_string()) + .default_value(retry.max_delay_secs.to_string()) + }); + let max_retry_delay_subscription = cx.subscribe_in( + &max_retry_delay_input, + window, + Self::on_max_retry_delay_input, + ); let mut this = Self { focus_handle: cx.focus_handle(), default_model_select, @@ -109,6 +133,10 @@ impl GeneralSection { _auto_mode_model_subscription: auto_mode_model_subscription, auto_mode_rules_input, _auto_mode_rules_subscription: auto_mode_rules_subscription, + max_retries_input, + _max_retries_subscription: max_retries_subscription, + max_retry_delay_input, + _max_retry_delay_subscription: max_retry_delay_subscription, }; this.reload(window, cx); this @@ -273,6 +301,48 @@ impl GeneralSection { Self::save_handoff_config(|config| config.max_images_per_request = max_images_per_request); } + /// Save the number of retries whenever the input holds a valid number. + fn on_max_retries_input( + &mut self, + input: &Entity, + event: &InputEvent, + _window: &mut gpui_kit::Window, + cx: &mut Context, + ) { + if !matches!(event, InputEvent::Change) { + return; + } + let Ok(max_retries) = input.read(cx).value().trim().parse::() else { + return; + }; + Self::save_retry_policy(|policy| policy.max_retries = max_retries); + } + + /// Save the longest wait whenever the input holds a valid number. + fn on_max_retry_delay_input( + &mut self, + input: &Entity, + event: &InputEvent, + _window: &mut gpui_kit::Window, + cx: &mut Context, + ) { + if !matches!(event, InputEvent::Change) { + return; + } + let Ok(max_delay_secs) = input.read(cx).value().trim().parse::() else { + return; + }; + Self::save_retry_policy(|policy| policy.max_delay_secs = max_delay_secs); + } + + fn save_retry_policy(change: impl FnOnce(&mut RetryPolicy)) { + let mut policy = RetryPolicy::load(); + change(&mut policy); + if let Err(e) = policy.save() { + warn!("Failed to save the retry settings: {e:#}"); + } + } + fn save_handoff_config(change: impl FnOnce(&mut HandoffConfig)) { let mut config = HandoffConfig::load(); change(&mut config); @@ -482,6 +552,62 @@ impl Render for GeneralSection { ), ), ) + // Failed requests + .child( + div() + .flex() + .flex_col() + .gap_2() + .child( + div() + .text_xs() + .font_weight(gpui_kit::FontWeight::MEDIUM) + .text_color(cx.theme().muted_foreground) + .child("Failed requests"), + ) + .child( + div() + .flex() + .items_center() + .gap_2() + .child(div().w(px(80.)).child(Input::new(&self.max_retries_input))) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("retries at most"), + ), + ) + .child( + div() + .flex() + .items_center() + .gap_2() + .child( + div() + .w(px(80.)) + .child(Input::new(&self.max_retry_delay_input)), + ) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child("seconds between retries at most"), + ), + ) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child( + "Applies to rate limits, overloaded or unreachable servers, \ + broken streams and empty answers. The wait doubles with each \ + retry up to this maximum, also when the provider asks for a \ + longer one. Stopping the session ends the waiting. Takes \ + effect with the next request a session starts.", + ), + ), + ) // Image limit .child( div() diff --git a/deny.toml b/deny.toml index fc05670fc..2c2301ee9 100644 --- a/deny.toml +++ b/deny.toml @@ -35,6 +35,15 @@ ignore = [ { id = "RUSTSEC-2026-0368", reason = "zip extract() follows symlinks out of the target dir; not reachable, see RUSTSEC-2026-0367" }, { id = "RUSTSEC-2026-0369", reason = "zip extract() applies permissions through symlinks; not reachable, see RUSTSEC-2026-0367" }, { id = "RUSTSEC-2026-0429", reason = "zip extract() aborts on a symlink entry with a large declared size; not reachable, see RUSTSEC-2026-0367" }, + { id = "RUSTSEC-2026-0546", reason = "zip extract() writes through chained symlinks; not reachable, see RUSTSEC-2026-0367" }, + { id = "RUSTSEC-2026-0548", reason = "ZipStreamReader trusts local-header names; nothing reads archives as a stream" }, + { id = "RUSTSEC-2026-0545", reason = "zip AES entries skip the HMAC on short data; nothing decrypts entries (app_update reads without a password)" }, + { id = "RUSTSEC-2026-0547", reason = "zip legacy Shrink/Reduce/Implode decoders over-allocate; the legacy-zip feature is not built" }, + # chrono has no maintainer any more and no release to move to. Our crates + # use it for timestamps; gpui, rmcp, oauth2, schemars and the office + # converters pull it in as well, so leaving it means more than our own + # code. Drop this once the workspace has moved to jiff (or time). + { id = "RUSTSEC-2026-0543", reason = "chrono is unmaintained; no replacement release, used across the dependency graph" }, # zip 8.x also pins zstd 0.13; the fixes are in zstd 0.14 / zstd-safe 8.2. # zip builds zstd without default features (no `legacy`, no `seekable`) # and makes a fresh context per stream. Drop with the zip entries above. diff --git a/docs/retry-policy.md b/docs/retry-policy.md new file mode 100644 index 000000000..df7f91015 --- /dev/null +++ b/docs/retry-policy.md @@ -0,0 +1,49 @@ +# Retrying failed requests + +A failed LLM request is sent again under one policy, `llm::RetryPolicy`, +stored in `/retry.json` and edited under *Settings → General → +Failed requests*: + +```json +{ + "max_retries": 5, + "max_delay_secs": 300 +} +``` + +- `max_retries` bounds the retries of one request; after the last one the + session fails and shows the error. +- `max_delay_secs` bounds the wait before a retry, also when the provider asks + for a longer one (`retry-after`, rate limit reset headers). Below one second + it counts as one, so retries never follow each other back to back. + +Missing fields take the defaults shown above. A changed policy applies to the +next request a session starts: clients and the agent's hooks are created per +run. + +## Two places retry + +- **The LLM clients** (`llm::utils::handle_retryable_error`) retry what fails + before an answer streams: rate limits (wait taken from the provider's + headers, else a minute, doubled per retry), transient authentication + errors, overloaded, unreachable or failing servers (one second, doubled per + retry). The factory hands each client the policy + (`LLMProvider::set_retry_policy`); Ollama and the WebSocket client keep + their own behaviour. When a client gives up on an error it could have + retried, it marks it `llm::retry::RetriesExhausted`. +- **The agent loop** (`code_assistant_core::plugins::DefaultRecovery`) retries + what the clients do not: a broken or stalled stream and an empty answer + (`agent_core::types::EmptyResponse`). It starts at two seconds, doubling. + An error marked `RetriesExhausted` is not retried again here, so the retries + of the two places never multiply. + +While waiting, the session shows `RateLimited` (countdown) or `Retrying` +(reason and `attempt/max`). + +## Empty answers + +An answer without any content used to end the turn as if the model had +answered: nothing was recorded, so the history still ended in the user's +message or the tool results, and the session showed *Resume* without an +error. Now the agent loop treats it as a failed request, retries it under the +policy, and fails visibly when the retries run out.