Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,14 @@ Layer 5 (binary): code_assistant — CLI, config, feature-gated frontend
`SessionService::restart_mcp_server`) without restarting the app
- Details: `docs/mcp-client-mode.md`, `docs/project-scoped-mcp-servers.md`

### Retrying Failed Requests
- One policy (`llm::RetryPolicy`, `<config_dir>/retry.json`, *Settings →
General*): at most N retries, and the longest wait between retries
- LLM clients retry rate limits and transient server/network errors; the
agent loop (`plugins::DefaultRecovery`) retries broken streams and empty
answers (`agent_core::types::EmptyResponse`), which used to end a turn
silently with *Resume*; see `docs/retry-policy.md`

### Built-in Tool Toggles
- Groups of built-in tools (browser, web, terminal, sub-agents, wakeups,
session history) switch on/off like MCP servers: globally in `tools.json`
Expand Down
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

46 changes: 21 additions & 25 deletions crates/agent_core/src/runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -428,10 +428,16 @@ impl AgentRuntime {
let reserved_node_id = self.reserve_node_id();

// 1. Get LLM response (without adding to history yet)
let (llm_response, request_id) = match self
let response = self
.get_next_assistant_message(messages, reserved_node_id)
.await
{
.and_then(|(response, request_id)| {
if response.content.is_empty() {
return Err(crate::types::EmptyResponse.into());
}
Ok((response, request_id))
});
let (llm_response, request_id) = match response {
Ok(result) => {
// Successful response — reset the retry counters
streaming_retry_count = 0;
Expand Down Expand Up @@ -476,14 +482,12 @@ impl AgentRuntime {

self.cancellation.check()?;
// 2. Add original LLM response to message history using the pre-allocated node_id
if !llm_response.content.is_empty() {
self.append_message_with_node_id(
Message::new_assistant_content(llm_response.content.clone())
.with_request_id(request_id)
.with_usage(llm_response.usage.clone()),
reserved_node_id,
)?;
}
self.append_message_with_node_id(
Message::new_assistant_content(llm_response.content.clone())
.with_request_id(request_id)
.with_usage(llm_response.usage.clone()),
reserved_node_id,
)?;

// 3. Extract tool requests from LLM response and get truncated response
let (tool_requests, flow, truncated_response) = self
Expand Down Expand Up @@ -1155,8 +1159,8 @@ impl AgentRuntime {
Ok(())
}

/// Informs the user that a transient streaming failure is being retried and
/// tells the UI to discard all partial content from the failed request.
/// Tells the user that a failed request is sent again, and the UI to
/// discard all partial content from it.
async fn prepare_streaming_retry(
&self,
error: &anyhow::Error,
Expand All @@ -1165,27 +1169,19 @@ impl AgentRuntime {
delay: std::time::Duration,
) {
warn!(
"Transient streaming error (attempt {}/{}), retrying in {:?}: {}",
attempt, max_attempts, delay, error
"LLM request failed (retry {attempt}/{max_attempts}), retrying in {delay:?}: {error:#}"
);

// get_next_assistant_message already sent StreamingStopped{error: ...} for
// the failed request, so the UI knows streaming ended. Now we tell it to
// get_next_assistant_message already sent StreamingStopped for the
// failed request, so the UI knows streaming ended. Now we tell it to
// also remove whatever was already rendered.
let _ = self
.send_ui(AgentUiEvent::RollbackStreaming {
request_id: self.next_request_id - 1,
})
.await;

let _ = self
.send_ui(AgentUiEvent::ShowTransientStatus {
message: format!(
"Stream interrupted — retrying ({}/{})\u{2026}",
attempt, max_attempts
),
})
.await;
self.ui
.notify_retrying(&error.to_string(), attempt, max_attempts);
}

/// Project the largest results from the most recent turn as small error
Expand Down
38 changes: 38 additions & 0 deletions crates/agent_core/src/runtime/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -766,3 +766,41 @@ fn a_message_queued_after_compaction_joins_the_handoff() {
assert!(texts[0].starts_with("<handoff>"), "{texts:?}");
assert_eq!(texts[1], "Also this");
}

/// Retries every failed request at once.
struct RetryAlways;

impl RecoveryPolicy for RetryAlways {
fn classify(&self, _: &anyhow::Error, completed_retries: u32) -> RecoveryAction {
RecoveryAction::RetryStream {
delay: std::time::Duration::ZERO,
attempt: completed_retries + 1,
max_attempts: u32::MAX,
}
}
}

#[tokio::test]
async fn an_empty_answer_is_a_failure_not_the_end_of_the_turn() {
let (mut runtime, llm) = runtime_answering(vec![vec![]]);
runtime.append_message(Message::new_user("Hi")).unwrap();

let error = runtime.run_single_iteration().await.unwrap_err();

assert!(error.is::<crate::types::EmptyResponse>(), "{error:#}");
assert_eq!(llm.requests().len(), 1);
}

#[tokio::test]
async fn an_empty_answer_is_retried_under_the_recovery_policy() {
let (mut runtime, llm) = runtime_answering(vec![vec![], vec![ContentBlock::new_text("Hello")]]);
runtime.hooks.recovery = Box::new(RetryAlways);
runtime.append_message(Message::new_user("Hi")).unwrap();

runtime.run_single_iteration().await.unwrap();

assert_eq!(llm.requests().len(), 2);
let messages: Vec<_> = runtime.conversation.active_messages().collect();
assert_eq!(messages.len(), 2, "the empty answer is not recorded");
assert_eq!(messages[1].role, MessageRole::Assistant);
}
7 changes: 7 additions & 0 deletions crates/agent_core/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,13 @@ impl ToolResult for ParseError {
}
}

/// The model answered with nothing: no text, no reasoning, no tool call.
/// Ending the turn there would leave the request unanswered and the session
/// waiting for a resume, so the agent loop treats it as a failed request.
#[derive(Debug, thiserror::Error)]
#[error("The model returned an empty response")]
pub struct EmptyResponse;

/// Placeholder result used when a tool's output was too large and caused a
/// "prompt too long" error from the LLM provider. The original tool execution
/// is replaced with this so the LLM gets actionable feedback on the next turn.
Expand Down
10 changes: 5 additions & 5 deletions crates/code_assistant_core/src/agent/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,7 @@ async fn test_context_compaction_inserts_summary() -> Result<()> {
};

let idle_response = LLMResponse {
content: Vec::new(),
content: vec![ContentBlock::new_text("Done")],
usage: Usage::zero(),
rate_limit_info: None,
};
Expand Down Expand Up @@ -667,7 +667,7 @@ async fn test_compaction_reminds_about_active_skills() -> Result<()> {
rate_limit_info: None,
};
let idle_response = LLMResponse {
content: Vec::new(),
content: vec![ContentBlock::new_text("Done")],
usage: Usage::zero(),
rate_limit_info: None,
};
Expand Down Expand Up @@ -764,7 +764,7 @@ async fn test_compaction_prompt_not_persisted_in_history() -> Result<()> {
};

let idle_response = LLMResponse {
content: Vec::new(),
content: vec![ContentBlock::new_text("Done")],
usage: Usage::zero(),
rate_limit_info: None,
};
Expand Down Expand Up @@ -889,7 +889,7 @@ async fn test_context_compaction_uses_only_messages_after_previous_summary() ->
};

let idle_response = LLMResponse {
content: Vec::new(),
content: vec![ContentBlock::new_text("Done")],
usage: Usage::zero(),
rate_limit_info: None,
};
Expand Down Expand Up @@ -2575,7 +2575,7 @@ fn text_response(text: &str) -> LLMResponse {

fn idle_response() -> LLMResponse {
LLMResponse {
content: Vec::new(),
content: vec![ContentBlock::new_text("Done")],
usage: Usage::zero(),
rate_limit_info: None,
}
Expand Down
2 changes: 1 addition & 1 deletion crates/code_assistant_core/src/plugins/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub fn default_hooks() -> HookRegistry {
observers: vec![],
dispatch: Box::new(SpawnAgentParallelPolicy),
compaction: Box::new(TokenRatioCompaction::new(0.8)),
recovery: Box::new(DefaultRecovery),
recovery: Box::new(DefaultRecovery::new(llm::RetryPolicy::load())),
system_prompt: Box::new(CodeAssistantSystemPrompt::new()),
}
}
94 changes: 78 additions & 16 deletions crates/code_assistant_core/src/plugins/recovery.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,26 @@
//! Recovery policy: classifies failed LLM requests by provider error text.

use agent_core::hooks::{RecoveryAction, RecoveryPolicy};
use agent_core::types::EmptyResponse;
use llm::RetryPolicy;
use llm::retry::RetriesExhausted;
use std::time::Duration;

/// Maximum number of retries for transient streaming errors (e.g. HTTP chunk
/// errors, connection resets).
const MAX_STREAMING_RETRIES: u32 = 2;
/// Wait before the first retry of a broken stream or an empty answer
/// (doubles on each further one, up to the policy's maximum).
const RETRY_BASE_DELAY: Duration = Duration::from_secs(2);

/// Base delay between streaming retries (doubles on each attempt).
const STREAMING_RETRY_BASE_DELAY: Duration = Duration::from_secs(2);
/// Recovers from oversized prompts by compacting, and sends a request again
/// after a broken stream or an empty answer, as often as `policy` allows.
pub struct DefaultRecovery {
policy: RetryPolicy,
}

pub struct DefaultRecovery;
impl DefaultRecovery {
pub fn new(policy: RetryPolicy) -> Self {
Self { policy }
}
}

impl RecoveryPolicy for DefaultRecovery {
fn classify(&self, error: &anyhow::Error, completed_retries: u32) -> RecoveryAction {
Expand All @@ -20,11 +30,16 @@ impl RecoveryPolicy for DefaultRecovery {
if is_prompt_too_long_error(error) {
return RecoveryAction::ReduceContext;
}
if is_retryable_streaming_error(error) && completed_retries < MAX_STREAMING_RETRIES {
// The client already sent it again as often as the policy allows.
if error.is::<RetriesExhausted>() {
return RecoveryAction::Fail;
}
let retryable = error.is::<EmptyResponse>() || is_retryable_streaming_error(error);
if retryable && self.policy.allows_retry(completed_retries) {
return RecoveryAction::RetryStream {
delay: STREAMING_RETRY_BASE_DELAY * 2u32.saturating_pow(completed_retries),
delay: self.policy.backoff(RETRY_BASE_DELAY, completed_retries),
attempt: completed_retries + 1,
max_attempts: MAX_STREAMING_RETRIES,
max_attempts: self.policy.max_retries,
};
}
RecoveryAction::Fail
Expand Down Expand Up @@ -87,6 +102,8 @@ fn is_retryable_streaming_error(error: &anyhow::Error) -> bool {
// Timeout errors (read timeouts, not connect timeouts)
|| msg.contains("operation timed out")
|| msg.contains("timed out reading")
// No data for a while (`llm::streaming`)
|| msg.contains("stream stalled")
// Server errors that are transient
|| msg.contains("502 bad gateway")
|| msg.contains("503 service")
Expand All @@ -98,31 +115,38 @@ fn is_retryable_streaming_error(error: &anyhow::Error) -> bool {
mod tests {
use super::*;

fn two_retries() -> DefaultRecovery {
DefaultRecovery::new(RetryPolicy {
max_retries: 2,
max_delay_secs: 300,
})
}

#[test]
fn oversized_prompt_reduces_context() {
let err = anyhow::anyhow!("Invalid request: prompt is too long: 500000 tokens");
assert!(matches!(
DefaultRecovery.classify(&err, 0),
two_retries().classify(&err, 0),
RecoveryAction::ReduceContext
));
}

#[test]
fn transient_stream_error_retries_with_backoff() {
let err = anyhow::anyhow!("error reading a body from connection: connection reset");
match DefaultRecovery.classify(&err, 0) {
match two_retries().classify(&err, 0) {
RecoveryAction::RetryStream {
delay,
attempt,
max_attempts,
} => {
assert_eq!(delay, Duration::from_secs(2));
assert_eq!(attempt, 1);
assert_eq!(max_attempts, MAX_STREAMING_RETRIES);
assert_eq!(max_attempts, 2);
}
_ => panic!("expected RetryStream"),
}
match DefaultRecovery.classify(&err, 1) {
match two_retries().classify(&err, 1) {
RecoveryAction::RetryStream { delay, attempt, .. } => {
assert_eq!(delay, Duration::from_secs(4));
assert_eq!(attempt, 2);
Expand All @@ -135,14 +159,14 @@ mod tests {
fn retries_are_bounded() {
let err = anyhow::anyhow!("connection reset");
assert!(matches!(
DefaultRecovery.classify(&err, MAX_STREAMING_RETRIES),
two_retries().classify(&err, 2),
RecoveryAction::Fail
));
}

#[test]
fn image_limit_errors_compact_with_the_named_limit() {
let classify = |message: &str| DefaultRecovery.classify(&anyhow::anyhow!("{message}"), 0);
let classify = |message: &str| two_retries().classify(&anyhow::anyhow!("{message}"), 0);
let openai = r#"Invalid request: {"error":{"message":"Exceeded maximum number of images (50) allowed in the request.","param":"input","type":"invalid_request_error"}}"#;
assert!(matches!(
classify(openai),
Expand All @@ -158,11 +182,49 @@ mod tests {
));
}

#[test]
fn empty_answers_and_stalled_streams_are_retried() {
for err in [
anyhow::Error::from(EmptyResponse),
anyhow::anyhow!("LLM stream stalled: no data received for 300s"),
] {
assert!(
matches!(
two_retries().classify(&err, 0),
RecoveryAction::RetryStream { attempt: 1, .. }
),
"{err}"
);
}
}

#[test]
fn the_retry_delay_is_capped() {
let recovery = DefaultRecovery::new(RetryPolicy {
max_retries: 200,
max_delay_secs: 30,
});
assert!(matches!(
recovery.classify(&EmptyResponse.into(), 100),
RecoveryAction::RetryStream { delay, attempt: 101, .. } if delay == Duration::from_secs(30)
));
}

#[test]
fn errors_the_client_gave_up_on_are_not_retried_again() {
let err = anyhow::anyhow!("Service overloaded: overloaded_error")
.context(RetriesExhausted { retries: 2 });
assert!(matches!(
two_retries().classify(&err, 0),
RecoveryAction::Fail
));
}

#[test]
fn unknown_errors_fail() {
let err = anyhow::anyhow!("invalid api key");
assert!(matches!(
DefaultRecovery.classify(&err, 0),
two_retries().classify(&err, 0),
RecoveryAction::Fail
));
}
Expand Down
Loading
Loading