diff --git a/codex-rs/.config/nextest.toml b/codex-rs/.config/nextest.toml index 6d873605633f..17decdc0b855 100644 --- a/codex-rs/.config/nextest.toml +++ b/codex-rs/.config/nextest.toml @@ -43,6 +43,9 @@ max-threads = 1 [test-groups.windows_sandbox_legacy_sessions] max-threads = 1 +[test-groups.windows_sandbox_accounts] +max-threads = 1 + [test-groups.windows_process_heavy] max-threads = 2 @@ -82,6 +85,23 @@ test-group = 'app_server_integration_local' filter = 'package(codex-core) & kind(test) & test(apply_patch_cli)' test-group = 'core_apply_patch_cli_integration' +[[profile.default.overrides]] +# These cases share the runner's elevated sandbox accounts. Queue them before +# starting their timeout; their file lock still coordinates other test processes. +platform = 'cfg(windows)' +filter = ''' + (package(codex-core) & ( + test(windows_sandbox_cli_preserves_managed_deny_reads_across_launches) | + test(windows_elevated_does_not_create_missing_workspace_metadata) | + test(windows_elevated_enforces_deny_read_and_protects_setup_marker) | + test(windows_elevated_unified_exec_enforces_large_recursive_deny_reads) | + test(windows_elevated_approved_git_pull_preserves_deny_read) + )) | + (package(codex-exec-server) & test(file_system_elevated_relative_read_denial_uses_policy_cwd)) | + (package(codex-windows-sandbox) & test(elevated_non_tty_cmd_forwards_env_output_and_exit)) +''' +test-group = 'windows_sandbox_accounts' + [[profile.default.overrides]] # These tests create restricted-token Windows child processes and private desktops. # Serialize them to avoid exhausting Windows session/global desktop resources in CI. diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index e8b1096596fb..a26f2fd61e6f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2021,8 +2021,10 @@ name = "codex-ansi-escape" version = "0.0.0" dependencies = [ "ansi-to-tui", + "pretty_assertions", "ratatui", "tracing", + "tracing-subscriber", ] [[package]] @@ -2976,6 +2978,7 @@ dependencies = [ "codex-utils-string", "codex-web-search-extension", "codex-windows-sandbox", + "codex-windows-sandbox-test-support", "core_test_support", "ctor 0.6.3", "dirs", @@ -3213,6 +3216,7 @@ dependencies = [ "codex-utils-rustls-provider", "codex-websocket-auth", "codex-websocket-client", + "codex-windows-sandbox-test-support", "ctor 0.6.3", "dirs", "futures", @@ -3250,6 +3254,7 @@ version = "0.0.0" dependencies = [ "base64 0.22.1", "codex-file-system", + "codex-install-context", "codex-mxc-sandbox", "codex-network-proxy", "codex-protocol", @@ -3813,6 +3818,7 @@ dependencies = [ "jsonwebtoken", "keyring", "once_cell", + "opentelemetry_sdk", "os_info", "pretty_assertions", "rand 0.9.3", @@ -4264,6 +4270,7 @@ dependencies = [ "codex-utils-path-uri", "codex-utils-redacted-string", "codex-utils-string", + "derive_more 2.1.1", "encoding_rs", "gix-url", "globset", @@ -4385,6 +4392,7 @@ dependencies = [ "libc", "memchr", "oauth2", + "opentelemetry_sdk", "pretty_assertions", "rmcp", "serde", @@ -5280,6 +5288,7 @@ dependencies = [ "codex-utils-process", "codex-utils-pty", "codex-utils-string", + "codex-windows-sandbox-test-support", "dirs-next", "dunce", "glob", @@ -5311,6 +5320,13 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "codex-windows-sandbox-test-support" +version = "0.0.0" +dependencies = [ + "dirs", +] + [[package]] name = "codex-workload-identity" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 4a063dc3a85f..0e01c4534be3 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -112,6 +112,7 @@ members = [ "v8-poc", "websocket-auth", "websocket-client", + "windows-sandbox-rs/tests/support", "windows-sandbox-service", "worktree", "workload-identity", @@ -316,6 +317,7 @@ codex-user-verification = { path = "user-verification" } codex-v8-poc = { path = "v8-poc" } codex-workload-identity = { path = "workload-identity" } codex-windows-sandbox = { path = "windows-sandbox-rs" } +codex-windows-sandbox-test-support = { path = "windows-sandbox-rs/tests/support" } core_test_support = { path = "core/tests/common" } # External diff --git a/codex-rs/ansi-escape/Cargo.toml b/codex-rs/ansi-escape/Cargo.toml index edb5b6e514cc..60b1aedf29c7 100644 --- a/codex-rs/ansi-escape/Cargo.toml +++ b/codex-rs/ansi-escape/Cargo.toml @@ -7,7 +7,6 @@ license.workspace = true [lib] name = "codex_ansi_escape" path = "src/lib.rs" -test = false doctest = false [lints] @@ -17,3 +16,7 @@ workspace = true ansi-to-tui = { workspace = true } ratatui = { workspace = true } tracing = { workspace = true, features = ["log"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/codex-rs/ansi-escape/src/ansi_escape_tests.rs b/codex-rs/ansi-escape/src/ansi_escape_tests.rs new file mode 100644 index 000000000000..d51deb73aa66 --- /dev/null +++ b/codex-rs/ansi-escape/src/ansi_escape_tests.rs @@ -0,0 +1,76 @@ +//! Check that multiline warnings omit payloads and preserve first-line rendering. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::sync::Mutex; + +use pretty_assertions::assert_eq; +use ratatui::style::Stylize; +use ratatui::text::Line; +use tracing::Event; +use tracing::Level; +use tracing::Subscriber; +use tracing::field::Field; +use tracing::field::Visit; +use tracing_subscriber::Layer; +use tracing_subscriber::layer::Context; +use tracing_subscriber::layer::SubscriberExt; + +use super::ansi_escape_line; + +#[derive(Debug, PartialEq, Eq)] +struct CapturedEvent { + level: Level, + fields: BTreeMap, +} + +impl Visit for CapturedEvent { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.fields + .insert(field.name().to_string(), format!("{value:?}")); + } +} + +struct CaptureLayer(Arc>>); + +impl Layer for CaptureLayer { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let mut captured = CapturedEvent { + level: *event.metadata().level(), + fields: BTreeMap::new(), + }; + event.record(&mut captured); + self.0.lock().expect("capture events").push(captured); + } +} + +#[test] +fn multiline_warning_contains_counts_and_preserves_the_styled_first_line() { + let events = Arc::new(Mutex::new(Vec::new())); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(Arc::clone(&events))); + let mut expected_events = Vec::new(); + + tracing::subscriber::with_default(subscriber, || { + for tail_bytes in [8, 128 * 1024] { + let tail = "x".repeat(tail_bytes); + let input = format!("\x1b[31mfirst\x1b[0m\n{tail}\nlast"); + assert_eq!(ansi_escape_line(&input), Line::from("first".red())); + expected_events.push(CapturedEvent { + level: Level::WARN, + fields: BTreeMap::from([ + ("input_bytes".to_string(), input.len().to_string()), + ("line_count".to_string(), "3".to_string()), + ( + "message".to_string(), + "ansi_escape_line: expected a single line".to_string(), + ), + ]), + }); + } + }); + + assert_eq!( + *events.lock().expect("read captured events"), + expected_events + ); +} diff --git a/codex-rs/ansi-escape/src/lib.rs b/codex-rs/ansi-escape/src/lib.rs index b47cf14f8ea6..caaabf2a7aef 100644 --- a/codex-rs/ansi-escape/src/lib.rs +++ b/codex-rs/ansi-escape/src/lib.rs @@ -31,7 +31,11 @@ pub fn ansi_escape_line(s: &str) -> Line<'static> { [] => "".into(), [only] => only.clone(), [first, rest @ ..] => { - tracing::warn!("ansi_escape_line: expected a single line, got {first:?} and {rest:?}"); + tracing::warn!( + input_bytes = s.len(), + line_count = rest.len() + 1, + "ansi_escape_line: expected a single line" + ); first.clone() } } @@ -56,3 +60,7 @@ pub fn ansi_escape(s: &str) -> Text<'static> { }, } } + +#[cfg(test)] +#[path = "ansi_escape_tests.rs"] +mod tests; diff --git a/codex-rs/app-server-daemon/src/backend/pid.rs b/codex-rs/app-server-daemon/src/backend/pid.rs index 22e34d9095d2..0e69a3f0e90c 100644 --- a/codex-rs/app-server-daemon/src/backend/pid.rs +++ b/codex-rs/app-server-daemon/src/backend/pid.rs @@ -394,6 +394,8 @@ impl PidBackend { } }; if matches!(self.command_kind, PidCommandKind::AppServer { .. }) { + // Match first-party clients' default while preserving explicit analytics opt-outs. + args.push("--analytics-default-enabled".into()); for (name, enabled) in &self.feature_overrides { args.extend(["-c".into(), format!("features.{name}={enabled}").into()]); } diff --git a/codex-rs/app-server-daemon/src/backend/pid_tests.rs b/codex-rs/app-server-daemon/src/backend/pid_tests.rs index 19d262c09706..dbd6a4075ac6 100644 --- a/codex-rs/app-server-daemon/src/backend/pid_tests.rs +++ b/codex-rs/app-server-daemon/src/backend/pid_tests.rs @@ -602,7 +602,13 @@ fn app_server_remote_control_uses_runtime_flag() { assert_eq!( backend.command_args(), - vec!["app-server", "--remote-control", "--listen", "unix://"] + vec![ + "app-server", + "--remote-control", + "--listen", + "unix://", + "--analytics-default-enabled" + ] ); } @@ -616,7 +622,12 @@ fn app_server_disabled_remote_control_uses_compatible_args_and_runtime_env() { assert_eq!( backend.command_args(), - vec!["app-server", "--listen", "unix://"] + vec![ + "app-server", + "--listen", + "unix://", + "--analytics-default-enabled" + ] ); assert_eq!( backend.command_env(), diff --git a/codex-rs/app-server-daemon/src/update_loop_tests.rs b/codex-rs/app-server-daemon/src/update_loop_tests.rs index d932742371bd..0847655a30cf 100644 --- a/codex-rs/app-server-daemon/src/update_loop_tests.rs +++ b/codex-rs/app-server-daemon/src/update_loop_tests.rs @@ -584,9 +584,9 @@ async fn daemon_start_and_restart_preserve_launch_features() { features ); let expected = if features.is_empty() { - "app-server\n--listen\nunix://\n--managed-daemon\n" + "app-server\n--listen\nunix://\n--analytics-default-enabled\n--managed-daemon\n" } else { - "app-server\n--listen\nunix://\n-c\nfeatures.api_key_model_discovery=true\n-c\nfeatures.code_mode_host=false\n--managed-daemon\n" + "app-server\n--listen\nunix://\n--analytics-default-enabled\n-c\nfeatures.api_key_model_discovery=true\n-c\nfeatures.code_mode_host=false\n--managed-daemon\n" }; assert_eq!(std::fs::read_to_string(&args_path).unwrap(), expected); let reused = daemon diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index dbac6c0f66ba..458e63efca2f 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -762,7 +762,7 @@ "type": "string" }, "CyberAccessProgram": { - "description": "Requested cyber treatment for a ChatGPT-authenticated Codex turn. Authorization and model-tier restrictions remain server-owned.", + "description": "Requested cyber treatment for an OpenAI model turn. Authorization and model-tier restrictions remain server-owned.", "enum": [ "standard", "daybreakBlue", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index a4dc6548afbd..fb7684cabf94 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10712,7 +10712,7 @@ "type": "object" }, "CyberAccessProgram": { - "description": "Requested cyber treatment for a ChatGPT-authenticated Codex turn. Authorization and model-tier restrictions remain server-owned.", + "description": "Requested cyber treatment for an OpenAI model turn. Authorization and model-tier restrictions remain server-owned.", "enum": [ "standard", "daybreakBlue", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 75ecacf9c26f..acdc575506b7 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6568,7 +6568,7 @@ "type": "object" }, "CyberAccessProgram": { - "description": "Requested cyber treatment for a ChatGPT-authenticated Codex turn. Authorization and model-tier restrictions remain server-owned.", + "description": "Requested cyber treatment for an OpenAI model turn. Authorization and model-tier restrictions remain server-owned.", "enum": [ "standard", "daybreakBlue", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json index 8169ec1e8d3f..ee3ca9ae1449 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "CyberAccessProgram": { - "description": "Requested cyber treatment for a ChatGPT-authenticated Codex turn. Authorization and model-tier restrictions remain server-owned.", + "description": "Requested cyber treatment for an OpenAI model turn. Authorization and model-tier restrictions remain server-owned.", "enum": [ "standard", "daybreakBlue", diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index c0cb6dfd4a0a..751bb8bc4e2b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -121,7 +121,7 @@ "type": "object" }, "CyberAccessProgram": { - "description": "Requested cyber treatment for a ChatGPT-authenticated Codex turn. Authorization and model-tier restrictions remain server-owned.", + "description": "Requested cyber treatment for an OpenAI model turn. Authorization and model-tier restrictions remain server-owned.", "enum": [ "standard", "daybreakBlue", diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index 5838b537c5ad..6aa5e5bfed28 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index e8bfb39944d8..72fe29bf62f8 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/CyberAccessProgram.ts b/codex-rs/app-server-protocol/schema/typescript/v2/CyberAccessProgram.ts index ccd4118a8320..0b87ec9cc6f9 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/CyberAccessProgram.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/CyberAccessProgram.ts @@ -3,7 +3,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. /** - * Requested cyber treatment for a ChatGPT-authenticated Codex turn. + * Requested cyber treatment for an OpenAI model turn. * Authorization and model-tier restrictions remain server-owned. */ export type CyberAccessProgram = "standard" | "daybreakBlue" | "daybreakRed"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs index f938d179e697..f7a7ec09eab8 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs @@ -118,7 +118,7 @@ pub struct AdditionalContextEntry { pub kind: AdditionalContextKind, } -/// Requested cyber treatment for a ChatGPT-authenticated Codex turn. +/// Requested cyber treatment for an OpenAI model turn. /// Authorization and model-tier restrictions remain server-owned. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] @@ -271,7 +271,7 @@ pub struct TurnStartParams { #[ts(optional = nullable)] pub multi_agent_mode: Option, - /// EXPERIMENTAL - Request a workspace-authorized cyber program for this + /// EXPERIMENTAL - Request an authorized cyber program for this /// turn. Omission preserves automatic behavior. This does not grant access. #[experimental("turn/start.cyberAccessProgram")] #[ts(optional = nullable)] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 268eee9022a5..64aef0e2deed 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -111,6 +111,15 @@ after a client tries to archive or delete it. After the owner releases the worker, its saved conversation can be archived or deleted normally. Ordinary client-controlled threads keep their existing behavior. +## Environment information (experimental) + +`environment/info` connects to a configured environment by `environmentId` and +returns its detected `shell` plus its default `cwd` as a canonical +environment-native `file:` URI. Connection failures are returned as request +errors. After connecting, the live metadata request has a 30-second timeout. A +timeout closes the probed connection and starts normal session recovery without +retrying the failed request. + ## User verification (experimental) Codex app-server advertises `openai/elicitation.userVerification` to the diff --git a/codex-rs/app-server/src/in_process_bootstrap.rs b/codex-rs/app-server/src/in_process_bootstrap.rs index b39ca0605dd1..a21c3b0b96d0 100644 --- a/codex-rs/app-server/src/in_process_bootstrap.rs +++ b/codex-rs/app-server/src/in_process_bootstrap.rs @@ -92,7 +92,8 @@ impl EmbeddedNetworkPolicy { .restrict_to_endpoints(endpoint.parse().into_iter().collect()), ), ) - .with_local_bootstrap_factory(factory); + .with_local_bootstrap_factory(factory) + .with_application_network_policy(self.effective.policy()); auth } } diff --git a/codex-rs/app-server/src/in_process_bootstrap_tests.rs b/codex-rs/app-server/src/in_process_bootstrap_tests.rs index bbc034797119..c6aa41d29709 100644 --- a/codex-rs/app-server/src/in_process_bootstrap_tests.rs +++ b/codex-rs/app-server/src/in_process_bootstrap_tests.rs @@ -1,7 +1,10 @@ -//! Embedded startup must keep the project selected by the caller. +//! Embedded startup preserves caller configuration and bootstrap access across authentication. +use super::EmbeddedNetworkPolicy; use super::configure; use crate::config_manager::ConfigManager; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; use codex_arg0::Arg0DispatchPaths; use codex_config::CloudConfigBundleLoader; use codex_config::LoaderOverrides; @@ -9,10 +12,137 @@ use codex_config::NoopThreadConfigLoader; use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; use codex_core::config::set_project_trust_level; +use codex_http_client::DestinationPolicy; +use codex_http_client::HttpClientFactory; +use codex_http_client::NetworkPermit; +use codex_http_client::NetworkPolicyDenied; +use codex_login::AuthCredentialsStoreMode; +use codex_login::AuthKeyringBackendKind; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::ExternalAuth; +use codex_login::ExternalAuthFuture; +use codex_login::ExternalAuthRefreshContext; use codex_protocol::config_types::TrustLevel; use pretty_assertions::assert_eq; use std::sync::Arc; +struct BootstrapExternalAuth(CodexAuth); + +impl ExternalAuth for BootstrapExternalAuth { + fn resolve(&self) -> ExternalAuthFuture<'_, CodexAuth> { + Box::pin(async { Ok(self.0.clone()) }) + } + + fn refresh(&self, _context: ExternalAuthRefreshContext) -> ExternalAuthFuture<'_, CodexAuth> { + self.resolve() + } +} + +#[tokio::test] +async fn bootstrap_discovery_survives_identity_installation_but_account_access_is_revoked() +-> anyhow::Result<()> { + let home = tempfile::tempdir()?; + let overrides = LoaderOverrides::without_managed_config_for_tests(); + let config = ConfigBuilder::default() + .codex_home(home.path().to_path_buf()) + .loader_overrides(overrides.clone()) + .build() + .await?; + let policy = EmbeddedNetworkPolicy::load(&overrides).await; + let auth_config = policy.bind_bootstrap_auth(config.auth_config()); + let bootstrap = auth_config.auth_route_config.http_client_factory().clone(); + let bundle_url = + codex_backend_client::Client::new(config.chatgpt_base_url.clone(), bootstrap.clone()) + .config_bundle_url() + .parse()?; + let content_url = "https://chatgpt.com/backend-api/codex/responses".parse()?; + let manager = AuthManager::new( + home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::Ephemeral, + /*forced_chatgpt_workspace_id*/ None, + Some(config.chatgpt_base_url.clone()), + AuthKeyringBackendKind::default(), + auth_config.auth_route_config, + ) + .await; + assert!(manager.auth_cached().is_none()); + + bootstrap.network_policy().acquire(&bundle_url)?; + let mut previous_access: Option<(HttpClientFactory, NetworkPermit)> = None; + for identity in ["bootstrap-account-one", "bootstrap-account-two"] { + let token = format!( + "e30.{}.signature", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&serde_json::json!({ + "sub": "bootstrap-user", + "https://api.openai.com/auth": { + "chatgpt_user_id": "bootstrap-user", + "chatgpt_account_id": identity, + }, + }))?), + ); + let auth = CodexAuth::from_external_chatgpt_tokens(&token, identity, Some("enterprise"))?; + manager + .set_external_auth(Arc::new(BootstrapExternalAuth(auth))) + .await?; + + if let Some((client, permit)) = previous_access.take() { + assert_eq!(permit.check(), Err(NetworkPolicyDenied::Revoked)); + assert_eq!( + client.network_policy().acquire(&content_url).err(), + Some(NetworkPolicyDenied::Revoked) + ); + } + + // Discovery must remain possible after both initial login and a workspace change. + bootstrap.network_policy().acquire(&bundle_url)?; + assert_eq!( + bootstrap.network_policy().acquire(&content_url).err(), + Some(NetworkPolicyDenied::Destination) + ); + assert_eq!( + manager + .application_network_policy() + .acquire(&content_url) + .err(), + Some(NetworkPolicyDenied::Unavailable) + ); + + assert!(policy.effective.publish( + policy.effective.policy().revision(), + DestinationPolicy::Unrestricted, + )); + let client = manager.http_client_factory(); + let permit = client.network_policy().acquire(&content_url)?; + previous_access = Some((client, permit)); + } + + let (client, permit) = previous_access.expect("the last identity has active account access"); + manager.clear_external_auth(); + assert_eq!(permit.check(), Err(NetworkPolicyDenied::Revoked)); + assert_eq!( + client.network_policy().acquire(&content_url).err(), + Some(NetworkPolicyDenied::Revoked) + ); + assert_eq!( + manager + .application_network_policy() + .acquire(&content_url) + .err(), + Some(NetworkPolicyDenied::Unavailable) + ); + bootstrap.network_policy().acquire(&bundle_url)?; + + // Losing local requirements must still close the bootstrap route. + policy.local.unavailable(policy.local.policy().revision()); + assert_eq!( + bootstrap.network_policy().acquire(&bundle_url).err(), + Some(NetworkPolicyDenied::Unavailable) + ); + Ok(()) +} + #[tokio::test] async fn startup_reloads_the_callers_selected_project() -> anyhow::Result<()> { let temp = tempfile::tempdir()?; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 13a28a10535f..f5a9a65425ae 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -83,6 +83,7 @@ use codex_feedback::CodexFeedback; use codex_goal_extension::GoalService; use codex_home::CodexHomeUserInstructionsProvider; use codex_login::AuthManager; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::ThreadId; use codex_protocol::mcp::ClientMcpExtensions; use codex_protocol::protocol::SessionSource; @@ -364,10 +365,11 @@ impl MessageProcessor { outgoing.clone(), thread_state_manager.clone(), )), - Some(app_server_time_provider( - outgoing.clone(), - thread_state_manager.clone(), - )), + Some({ + let time_provider = + app_server_time_provider(outgoing.clone(), thread_state_manager.clone()); + time_provider + }), ); match code_mode_session_provider { Some(provider) => manager.with_code_mode_session_provider(provider), @@ -1031,15 +1033,21 @@ impl MessageProcessor { return; } let processor_for_request = Arc::clone(&processor); + let originator = AuthStorageOriginator::from_client_name( + session.app_server_client_name().unwrap_or("none"), + ); // Keep queued requests small to avoid large stack temporaries during construction. - let result = Box::pin(processor_for_request.handle_initialized_client_request( - connection_request_id, - codex_request, - request_context, - session, - event_stream_ready, - )) - .await; + let result = originator + .scope(Box::pin( + processor_for_request.handle_initialized_client_request( + connection_request_id, + codex_request, + request_context, + session, + event_stream_ready, + ), + )) + .await; if let Err(error) = result { processor.outgoing.send_error(error_request_id, error).await; } diff --git a/codex-rs/app-server/src/request_processors/account_processor.rs b/codex-rs/app-server/src/request_processors/account_processor.rs index e1a0cd33d2fb..401b4cc58470 100644 --- a/codex-rs/app-server/src/request_processors/account_processor.rs +++ b/codex-rs/app-server/src/request_processors/account_processor.rs @@ -13,6 +13,7 @@ use codex_login::LoginOnboardingEntrypoint; use codex_login::login_with_bedrock_access_keys; use codex_mcp::ema_auth_scope; use codex_model_provider::is_supported_amazon_bedrock_region; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_rmcp_client::EnterpriseOAuthCredentialGuard; mod bedrock_setup; @@ -678,7 +679,7 @@ impl AccountRequestProcessor { let processor = self.clone(); let active_login = self.active_login.clone(); let auth_url = server.auth_url.clone(); - tokio::spawn(async move { + let task = async move { let (success, error_msg, onboarding_entrypoint) = match tokio::time::timeout( LOGIN_CHATGPT_TIMEOUT, server.block_until_done_with_callback_result(), @@ -715,7 +716,8 @@ impl AccountRequestProcessor { if guard.as_ref().map(ActiveLogin::login_id) == Some(login_id) { *guard = None; } - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); Ok(LoginAccountResponse::Chatgpt { login_id: login_id.to_string(), @@ -759,7 +761,7 @@ impl AccountRequestProcessor { let processor = self.clone(); let active_login = self.active_login.clone(); - tokio::spawn(async move { + let task = async move { let (success, error_msg) = tokio::select! { _ = cancel.cancelled() => { (false, Some("Login was not completed".to_string())) @@ -785,7 +787,8 @@ impl AccountRequestProcessor { if guard.as_ref().map(ActiveLogin::login_id) == Some(login_id) { *guard = None; } - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); Ok(LoginAccountResponse::ChatgptDeviceCode { login_id: login_id.to_string(), diff --git a/codex-rs/app-server/src/request_processors/config_processor.rs b/codex-rs/app-server/src/request_processors/config_processor.rs index 56a8b4b316ee..72ddca52f6e3 100644 --- a/codex-rs/app-server/src/request_processors/config_processor.rs +++ b/codex-rs/app-server/src/request_processors/config_processor.rs @@ -64,6 +64,7 @@ const BACKGROUND_PAGINATED_ROLLOUT_MIGRATION_FEATURE: &str = "background_paginated_rollout_migration"; const SUPPORTED_EXPERIMENTAL_FEATURE_ENABLEMENT: &[&str] = &[ + "api_key_cyber_access_programs", "api_key_model_discovery", "auth_elicitation", BACKGROUND_PAGINATED_ROLLOUT_MIGRATION_FEATURE, diff --git a/codex-rs/app-server/src/request_processors/mcp_processor.rs b/codex-rs/app-server/src/request_processors/mcp_processor.rs index ac2a55ba798d..16adf8eb2133 100644 --- a/codex-rs/app-server/src/request_processors/mcp_processor.rs +++ b/codex-rs/app-server/src/request_processors/mcp_processor.rs @@ -7,6 +7,7 @@ use codex_mcp::McpServerSource; use codex_mcp::ReadResourceRequestParams; use codex_mcp::ema_auth_scope; use codex_mcp::resolve_oauth_callback; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_rmcp_client::EnterpriseOAuthLoginRequest; use std::future::Future; @@ -379,12 +380,13 @@ impl McpRequestProcessor { None => None, }; let processor = self.clone(); - tokio::spawn(async move { + let task = async move { let result = processor .list_mcp_server_status_response(params, thread) .await; outgoing.send_result(request, result).await; - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); Ok(()) } @@ -598,7 +600,7 @@ impl McpRequestProcessor { let (_, thread) = self.load_thread(&thread_id).await?; let request_id = request_id.clone(); - tokio::spawn(async move { + let task = async move { let origin_call_id = origin_call_id.filter(|_| server == codex_mcp::CODEX_APPS_MCP_SERVER_NAME); let result = match origin_call_id.as_deref() { @@ -611,7 +613,8 @@ impl McpRequestProcessor { }; Self::send_mcp_resource_read_response(outgoing, request_id, result, origin_call_id) .await; - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); return Ok(()); } @@ -633,7 +636,7 @@ impl McpRequestProcessor { McpRuntimeContext::new(Arc::clone(&environment_manager), config.cwd.to_path_buf()); let request_id = request_id.clone(); - tokio::spawn(async move { + let task = async move { let result = read_mcp_resource_without_thread( &mcp_config, auth.as_ref(), @@ -649,7 +652,8 @@ impl McpRequestProcessor { outgoing, request_id, result, /*origin_call_id*/ None, ) .await; - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); Ok(()) } @@ -687,14 +691,15 @@ impl McpRequestProcessor { let meta = with_mcp_tool_call_thread_id_meta(params.meta, &thread_id); let request_id = request_id.clone(); - tokio::spawn(async move { + let task = async move { let result = thread .call_mcp_tool(¶ms.server, ¶ms.tool, params.arguments, meta) .await .map(McpServerToolCallResponse::from) .map_err(mcp_operation_error); outgoing.send_result(request_id, result).await; - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); Ok(()) } } diff --git a/codex-rs/app-server/tests/suite/v2/account.rs b/codex-rs/app-server/tests/suite/v2/account.rs index 239ed7097fc8..e2a1c1a1b22b 100644 --- a/codex-rs/app-server/tests/suite/v2/account.rs +++ b/codex-rs/app-server/tests/suite/v2/account.rs @@ -2166,6 +2166,7 @@ async fn login_account_chatgpt_device_code_succeeds_and_notifies() -> Result<()> }, )?; write_models_cache(codex_home.path()).await?; + let collector = super::auth_storage_originator::configure_collector(codex_home.path()).await?; mock_device_code_usercode(&mock_server, /*interval_seconds*/ 0).await; mock_device_code_token_success(&mock_server).await; @@ -2184,10 +2185,17 @@ async fn login_account_chatgpt_device_code_succeeds_and_notifies() -> Result<()> .without_auto_env() .with_env_overrides(&[ ("OPENAI_API_KEY", None), + ("OTEL_METRIC_EXPORT_INTERVAL", Some("200")), (LOGIN_ISSUER_ENV_VAR, Some(issuer.as_str())), ]) - .build_initialized_with_timeout(DEFAULT_READ_TIMEOUT) + .build() .await?; + mcp.initialize_with_client_info(ClientInfo { + name: "codex_vscode".into(), + title: None, + version: "1".into(), + }) + .await?; let request_id = mcp.send_login_account_chatgpt_device_code_request().await?; let login: LoginAccountResponse = @@ -2231,6 +2239,7 @@ async fn login_account_chatgpt_device_code_succeeds_and_notifies() -> Result<()> codex_home.path().join("auth.json").exists(), "auth.json should be created when device code login succeeds" ); + super::auth_storage_originator::assert_saved_originators(&collector, &["codex_vscode"]).await?; Ok(()) } diff --git a/codex-rs/app-server/tests/suite/v2/auth_storage_originator_tests.rs b/codex-rs/app-server/tests/suite/v2/auth_storage_originator_tests.rs new file mode 100644 index 000000000000..86817d5fe57d --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/auth_storage_originator_tests.rs @@ -0,0 +1,252 @@ +//! Checks client attribution on exported storage metrics through the public RPC API. + +use super::connection_handling_websocket::connect_websocket; +use super::connection_handling_websocket::read_response_for_id; +use super::connection_handling_websocket::send_initialize_request; +use super::connection_handling_websocket::send_request; +use super::connection_handling_websocket::spawn_websocket_server; +use anyhow::Context; +use anyhow::Result; +use app_test_support::ChatGptAuthFixture; +use app_test_support::MockResponsesConfig; +use app_test_support::TestAppServer; +use app_test_support::write_chatgpt_auth; +use app_test_support::write_models_cache; +use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::TurnStatus; +use codex_app_server_protocol::UserInput; +use codex_login::AuthCredentialsStoreMode; +use codex_login::REFRESH_TOKEN_URL_OVERRIDE_ENV_VAR; +use core_test_support::responses; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use std::collections::BTreeMap; +use std::io::Write; +use std::path::Path; +use std::time::Duration; +use tempfile::TempDir; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +pub(super) async fn configure_collector(codex_home: &Path) -> Result { + let collector = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/metrics")) + .respond_with(ResponseTemplate::new(/*s*/ 200)) + .mount(&collector) + .await; + let endpoint = format!("{}/v1/metrics", collector.uri()); + let mut config = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(codex_home.join("config.toml"))?; + writeln!( + config, + "\n[analytics]\nenabled = true\n[otel.metrics_exporter.otlp-http]\nendpoint = {endpoint:?}\nprotocol = \"json\"" + )?; + Ok(collector) +} + +pub(super) async fn assert_saved_originators( + collector: &MockServer, + expected: &[&str], +) -> Result<()> { + let expected: BTreeMap<_, _> = expected + .iter() + .map(|value| (value.to_string(), 1_u64)) + .collect(); + let actual = tokio::time::timeout(Duration::from_secs(/*secs*/ 60), async { + loop { + let mut actual = BTreeMap::::new(); + for request in collector.received_requests().await.unwrap_or_default() { + let body: Value = serde_json::from_slice(&request.body)?; + for resource in body["resourceMetrics"].as_array().into_iter().flatten() { + for scope in resource["scopeMetrics"].as_array().into_iter().flatten() { + for metric in scope["metrics"].as_array().into_iter().flatten() { + if metric["name"] != "codex.auth_storage.operation" { + continue; + } + for point in + metric["sum"]["dataPoints"].as_array().into_iter().flatten() + { + let tags: BTreeMap<_, _> = point["attributes"] + .as_array() + .into_iter() + .flatten() + .filter_map(|tag| { + Some(( + tag["key"].as_str()?, + tag["value"]["stringValue"].as_str()?, + )) + }) + .collect(); + if tags.get("credential_kind") == Some(&"codex") + && tags.get("operation") == Some(&"save") + { + let count = point["asInt"] + .as_u64() + .or_else(|| point["asInt"].as_str()?.parse().ok()) + .context("storage count must be an integer")?; + *actual + .entry( + tags.get("originator") + .unwrap_or(&"missing") + .to_string(), + ) + .or_default() += count; + } + } + } + } + } + } + if actual.values().sum::() >= expected.len() as u64 { + break Ok::<_, anyhow::Error>(actual); + } + tokio::time::sleep(Duration::from_millis(/*millis*/ 25)).await; + } + }) + .await??; + assert_eq!(actual, expected); + Ok(()) +} + +#[tokio::test] +async fn concurrent_connections_export_their_own_storage_originator() -> Result<()> { + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("config.toml"), + "cli_auth_credentials_store = \"file\"\n", + )?; + let collector = configure_collector(codex_home.path()).await?; + let (_process, address) = spawn_websocket_server(codex_home.path()).await?; + let mut clients = Vec::new(); + for name in ["Codex Desktop", "codex_vscode", "private-client-name"] { + let mut client = connect_websocket(address).await?; + send_initialize_request(&mut client, /*id*/ 1, name).await?; + read_response_for_id(&mut client, /*id*/ 1).await?; + clients.push(client); + } + // All clients initialize before any storage request; a process-global identity fails this test. + for client in &mut clients { + send_request( + client, + "account/login/start", + /*id*/ 2, + Some(json!({"type": "apiKey", "apiKey": "sk-test-key"})), + ) + .await?; + } + for client in &mut clients { + let response = read_response_for_id(client, /*id*/ 2).await?; + assert_eq!(response.result, json!({"type": "apiKey"})); + } + assert_saved_originators(&collector, &["codex_desktop", "codex_vscode", "other"]).await +} + +#[tokio::test] +async fn turn_token_refresh_exports_thread_storage_originator() -> Result<()> { + let codex_home = TempDir::new()?; + let backend = MockServer::start().await; + MockResponsesConfig::new(&backend.uri()) + .with_root_config("cli_auth_credentials_store = \"file\"") + .with_provider_config("requires_openai_auth = true") + .write(codex_home.path())?; + write_models_cache(codex_home.path()).await?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("initial-access-token") + .account_id("account-123") + .chatgpt_user_id("user-123"), + AuthCredentialsStoreMode::File, + )?; + let collector = configure_collector(codex_home.path()).await?; + + Mock::given(method("POST")) + .and(path("/oauth/token")) + .respond_with(ResponseTemplate::new(/*s*/ 200).set_body_json(json!({ + "access_token": "refreshed-access-token", + "refresh_token": "refreshed-refresh-token", + }))) + .expect(/*r*/ 1) + .mount(&backend) + .await; + // Fresh credentials prevent startup refresh. The turn first reloads storage on 401, + // then refreshes and persists the tokens when the retry is also unauthorized. + let responses_mock = responses::mount_response_sequence( + &backend, + vec![ + ResponseTemplate::new(/*s*/ 401), + ResponseTemplate::new(/*s*/ 401), + responses::sse_response(responses::sse(vec![ + responses::ev_response_created("resp-turn"), + responses::ev_completed("resp-turn"), + ])), + ], + ) + .await; + let refresh_url = format!("{}/oauth/token", backend.uri()); + let mut server = TestAppServer::builder() + .with_codex_home(codex_home.path()) + .with_env_overrides(&[ + ("OPENAI_API_KEY", None), + ( + REFRESH_TOKEN_URL_OVERRIDE_ENV_VAR, + Some(refresh_url.as_str()), + ), + ]) + .build() + .await?; + server + .initialize_with_client_info(ClientInfo { + name: "codex_vscode".to_string(), + title: None, + version: "0.1.0".to_string(), + }) + .await?; + let request = server + .send_thread_start_request_with_auto_env(ThreadStartParams::default()) + .await?; + let thread: ThreadStartResponse = server.read_response(request).await?; + let request = server + .send_turn_start_request(TurnStartParams { + thread_id: thread.thread.id, + input: vec![UserInput::Text { + text: "Hello".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let _: TurnStartResponse = server.read_response(request).await?; + let notification = server + .read_stream_until_notification_message("turn/completed") + .await?; + let completed: TurnCompletedNotification = + serde_json::from_value(notification.params.context("turn/completed params")?)?; + assert_eq!(completed.turn.status, TurnStatus::Completed); + assert_eq!( + responses_mock + .requests() + .iter() + .map(|request| request.header("authorization")) + .collect::>(), + vec![ + Some("Bearer initial-access-token".to_string()), + Some("Bearer initial-access-token".to_string()), + Some("Bearer refreshed-access-token".to_string()), + ] + ); + assert_saved_originators(&collector, &["codex_vscode"]).await?; + backend.verify().await; + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs index 235d669d6288..e083aaa7655b 100644 --- a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs +++ b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs @@ -507,7 +507,8 @@ pub(super) async fn spawn_websocket_server_with_args( .stdout(Stdio::null()) .stderr(Stdio::piped()) .env("CODEX_HOME", codex_home) - .env("RUST_LOG", "warn"); + .env("RUST_LOG", "warn") + .env("OTEL_METRIC_EXPORT_INTERVAL", "200"); let mut process = cmd .kill_on_drop(true) diff --git a/codex-rs/app-server/tests/suite/v2/cyber_access_program.rs b/codex-rs/app-server/tests/suite/v2/cyber_access_program.rs index fea72f3f56ac..6cfedf7706e1 100644 --- a/codex-rs/app-server/tests/suite/v2/cyber_access_program.rs +++ b/codex-rs/app-server/tests/suite/v2/cyber_access_program.rs @@ -3,7 +3,10 @@ use app_test_support::ChatGptAuthFixture; use app_test_support::TestAppServer; use app_test_support::write_chatgpt_auth; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::CyberAccessProgram; +use codex_app_server_protocol::ExperimentalFeatureEnablementSetParams; +use codex_app_server_protocol::ExperimentalFeatureEnablementSetResponse; use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadForkParams; @@ -16,13 +19,18 @@ use codex_app_server_protocol::ThreadResumeParams; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartedNotification; +use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; use codex_login::AuthCredentialsStoreMode; +use codex_login::AuthKeyringBackendKind; +use codex_login::login_with_api_key; use core_test_support::responses; use pretty_assertions::assert_eq; use serde_json::json; +use std::collections::BTreeMap; +use std::collections::HashMap; use std::path::Path; use tempfile::TempDir; use wiremock::Mock; @@ -192,6 +200,167 @@ async fn turn_start_forwards_explicit_cyber_access_program() -> Result<()> { Ok(()) } +#[tokio::test] +async fn api_key_cyber_access_program_requires_both_features() -> Result<()> { + core_test_support::skip_if_no_network!(Ok(())); + let server = responses::start_mock_server().await; + let expected_programs = [ + None, + None, + None, + Some(json!({"cyber": "daybreak_blue"})), + Some(json!({"cyber": "daybreak_red"})), + Some(json!({"cyber": "standard"})), + ]; + let requests = responses::mount_sse_sequence( + &server, + expected_programs + .iter() + .enumerate() + .map(|(index, _)| responses::sse_completed(&format!("resp-{index}"))) + .collect(), + ) + .await; + let home = TempDir::new()?; + let mut app = start_api_key_app(home.path(), &server).await?; + set_api_key_cyber_access_programs(&mut app, /*enabled*/ true).await?; + for disabled_feature in [ + Some("api_key_cyber_access_programs"), + Some("api_key_model_discovery"), + None, + ] { + let thread = app + .start_thread(ThreadStartParams { + config: disabled_feature + .map(|feature| HashMap::from([(format!("features.{feature}"), json!(false))])), + ..Default::default() + }) + .await? + .thread; + for program in [ + None, + Some(CyberAccessProgram::DaybreakBlue), + Some(CyberAccessProgram::DaybreakRed), + Some(CyberAccessProgram::Standard), + ] { + let completed = app + .start_turn_and_wait_for_completion(TurnStartParams { + thread_id: thread.id.clone(), + cyber_access_program: program, + input: vec![UserInput::Text { + text: "hello".to_owned(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + if program.is_some() && disabled_feature.is_some() { + assert_eq!(completed.turn.status, TurnStatus::Failed); + assert_eq!( + completed.turn.error, + Some(TurnError { + message: "Cyber access programs are disabled for this API-key session." + .to_owned(), + codex_error_info: Some(CodexErrorInfo::Other), + additional_details: None, + misalignment: None, + }) + ); + } else { + assert_eq!(completed.turn.status, TurnStatus::Completed); + } + } + } + assert_eq!( + requests + .requests() + .iter() + .map(|request| ( + request.header("authorization"), + request.body_json()["model"].clone(), + request.body_json().get("access_programs").cloned(), + )) + .collect::>(), + expected_programs + .into_iter() + .map(|program| ( + Some("Bearer test-key".to_owned()), + json!("gpt-6-sol"), + program + )) + .collect::>() + ); + app.shutdown_gracefully().await?; + Ok(()) +} + +#[tokio::test] +async fn api_key_cyber_access_program_server_rejection_does_not_fall_back() -> Result<()> { + core_test_support::skip_if_no_network!(Ok(())); + let server = responses::start_mock_server().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(/*s*/ 403).set_body_json(json!({ + "error": { + "type": "invalid_request_error", + "code": "access_program_not_enabled", + "message": "Daybreak Blue is not enabled for this project." + } + }))) + .expect(/*r*/ 1..) + .mount(&server) + .await; + let home = TempDir::new()?; + let mut app = start_api_key_app(home.path(), &server).await?; + set_api_key_cyber_access_programs(&mut app, /*enabled*/ true).await?; + let thread = app.start_thread(ThreadStartParams::default()).await?.thread; + let completed = app + .start_turn_and_wait_for_completion(TurnStartParams { + thread_id: thread.id, + cyber_access_program: Some(CyberAccessProgram::DaybreakBlue), + input: vec![UserInput::Text { + text: "hello".to_owned(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + assert_eq!(completed.turn.status, TurnStatus::Failed); + let error = completed.turn.error.expect("server rejection"); + assert_eq!( + error.codex_error_info, + Some(CodexErrorInfo::HttpConnectionFailed { + http_status_code: Some(403), + }) + ); + assert!( + error + .message + .contains("Daybreak Blue is not enabled for this project.") + ); + let requests = responses::received_responses_requests(&server).await; + assert!(!requests.is_empty()); + // Existing transport retries may repeat the request; none may drop the program. + assert_eq!( + requests + .iter() + .map(|request| ( + request.header("authorization"), + request.body_json()["access_programs"].clone() + )) + .collect::>(), + vec![ + ( + Some("Bearer test-key".to_owned()), + json!({"cyber": "daybreak_blue"}) + ); + requests.len() + ] + ); + app.shutdown_gracefully().await?; + Ok(()) +} + #[tokio::test] async fn daybreak_thread_metadata_persists_independently_across_restart_and_fork() -> Result<()> { core_test_support::skip_if_no_network!(Ok(())); @@ -461,3 +630,55 @@ async fn start_chatgpt_app(home: &Path, server: &MockServer) -> Result Result { + Mock::given(method("GET")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(/*s*/ 426)) + .mount(server) + .await; + // The existing inference override disables remote catalog discovery. Forwarding + // must therefore work without granting programs through a test catalog. + std::fs::write( + home.join("config.toml"), + format!( + "model = \"gpt-6-sol\"\nopenai_base_url = \"{}/v1\"\n[features]\napi_key_model_discovery = true\n", + server.uri() + ), + )?; + login_with_api_key( + home, + "test-key", + AuthCredentialsStoreMode::File, + AuthKeyringBackendKind::default(), + )?; + TestAppServer::builder() + .with_codex_home(home) + .without_managed_config() + .with_env_overrides(&[ + ("OPENAI_API_KEY", None), + ("CODEX_API_KEY", None), + ("OPENAI_BASE_URL", None), + ]) + .build_initialized() + .await +} + +async fn set_api_key_cyber_access_programs(app: &mut TestAppServer, enabled: bool) -> Result<()> { + let enablement = BTreeMap::from([("api_key_cyber_access_programs".to_owned(), enabled)]); + let response: ExperimentalFeatureEnablementSetResponse = app + .request( + |request_id| ClientRequest::ExperimentalFeatureEnablementSet { + request_id, + params: ExperimentalFeatureEnablementSetParams { + enablement: enablement.clone(), + }, + }, + ) + .await?; + assert_eq!( + response, + ExperimentalFeatureEnablementSetResponse { enablement } + ); + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index d24896903618..2d82aa822106 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -9,6 +9,8 @@ mod app_list; mod app_read; mod application_network; mod attestation; +#[path = "auth_storage_originator_tests.rs"] +mod auth_storage_originator; mod auto_env; mod bedrock_setup; mod client_metadata; diff --git a/codex-rs/cli/src/doctor/output/detail.rs b/codex-rs/cli/src/doctor/output/detail.rs index bca2baeaa49f..d7e730c3c5df 100644 --- a/codex-rs/cli/src/doctor/output/detail.rs +++ b/codex-rs/cli/src/doctor/output/detail.rs @@ -750,6 +750,7 @@ fn value<'a>(parsed: &'a [ParsedDetail], label: &str) -> Option<&'a str> { fn display_label(label: &str) -> String { match label { + "auth file" => "auth.json path (file backend/fallback)", "codex-linux-sandbox helper" => "linux helper", "optional reachability failed" => "optional reachability", "check for update on startup" => "startup update check", diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index 8b9f081dada9..1d6421b060ba 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -46,6 +46,8 @@ const API_KEY_LOGIN_DISABLED_MESSAGE: &str = const ACCESS_TOKEN_LOGIN_DISABLED_MESSAGE: &str = "Access token login is disabled. Use API key login instead."; const LOGIN_SUCCESS_MESSAGE: &str = "Successfully logged in"; +const DEFAULT_LOGIN_LOG_FILTER: &str = + "codex_cli=info,codex_core=info,codex_login=info,codex_otel::auth_storage=warn"; /// Installs a small file-backed tracing layer for direct `codex login` flows. /// @@ -94,7 +96,7 @@ fn init_login_file_logging(config: &Config) -> Option { let (non_blocking, guard) = non_blocking(log_file); let env_filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("codex_cli=info,codex_core=info,codex_login=info")); + .unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOGIN_LOG_FILTER)); let file_layer = tracing_subscriber::fmt::layer() .with_writer(non_blocking) .with_target(true) @@ -619,6 +621,10 @@ fn safe_format_key(key: &str) -> String { format!("{prefix}***{suffix}") } +#[cfg(test)] +#[path = "login_storage_tests.rs"] +mod storage_tests; + #[cfg(test)] mod tests { use codex_config::types::AuthCredentialsStoreMode; diff --git a/codex-rs/cli/src/login_storage_tests.rs b/codex-rs/cli/src/login_storage_tests.rs new file mode 100644 index 000000000000..ded91dcfa1d7 --- /dev/null +++ b/codex-rs/cli/src/login_storage_tests.rs @@ -0,0 +1,87 @@ +//! Verifies storage failures remain visible in direct login diagnostics. + +use super::DEFAULT_LOGIN_LOG_FILTER; +use codex_config::types::AuthCredentialsStoreMode; +use codex_login::AuthKeyringBackendKind; +use codex_login::login_with_api_key; +use pretty_assertions::assert_eq; +use tempfile::tempdir; +use tracing_subscriber::EnvFilter; + +#[test] +fn login_default_filter_records_sanitized_storage_fallback() -> anyhow::Result<()> { + let home = tempdir()?; + // Fail before accessing the real keyring, then allow the plaintext fallback to succeed. + std::fs::write(home.path().join("secrets"), "not a directory")?; + let log_path = home.path().join("login.log"); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_env_filter(EnvFilter::new(DEFAULT_LOGIN_LOG_FILTER)) + .with_writer(std::fs::File::create(&log_path)?) + .finish(); + tracing::subscriber::with_default(subscriber, || { + login_with_api_key( + home.path(), + "private-api-key", + AuthCredentialsStoreMode::Auto, + AuthKeyringBackendKind::Secrets, + ) + })?; + + let logs = std::fs::read_to_string(log_path)?; + assert_eq!(logs.lines().count(), 2); + let failure = logs.lines().next().unwrap(); + assert!( + failure.contains("failed to save auth to keyring, falling back to file storage"), + "{logs}" + ); + assert!(failure.contains("failed to create secrets dir"), "{logs}"); + assert!( + logs.contains("credential storage file fallback completed"), + "{logs}" + ); + assert!(logs.contains("fallback_reason=\"secure_error\""), "{logs}"); + assert!(logs.contains("outcome=\"success\""), "{logs}"); + assert!(!logs.contains("private-api-key")); + let completion = logs.lines().last().unwrap(); + assert!(!completion.contains(home.path().to_str().unwrap())); + let cause = std::fs::create_dir_all(home.path().join("secrets")) + .unwrap_err() + .to_string(); + insta::assert_snapshot!( + logs.replace( + home.path().join("secrets").to_str().unwrap(), + "[SECRETS_PATH]" + ) + .replace(&cause, "[OS_ERROR]") + ); + Ok(()) +} + +#[test] +fn login_storage_error_displays_context_and_cause() -> anyhow::Result<()> { + let home = tempdir()?; + let secrets_path = home.path().join("secrets"); + std::fs::write(&secrets_path, "not a directory")?; + let cause = std::fs::create_dir_all(&secrets_path) + .unwrap_err() + .to_string(); + let error = login_with_api_key( + home.path(), + "private-api-key", + AuthCredentialsStoreMode::Keyring, + AuthKeyringBackendKind::Secrets, + ) + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains(&cause), "{message}"); + assert!(!message.contains("private-api-key")); + // The temporary path and native filesystem error differ across platforms. + insta::assert_snapshot!( + message + .replace(secrets_path.to_str().unwrap(), "[SECRETS_PATH]") + .replace(&cause, "[OS_ERROR]") + ); + Ok(()) +} diff --git a/codex-rs/cli/src/snapshots/codex_cli__login__storage_tests__login_default_filter_records_sanitized_storage_fallback.snap b/codex-rs/cli/src/snapshots/codex_cli__login__storage_tests__login_default_filter_records_sanitized_storage_fallback.snap new file mode 100644 index 000000000000..8536486dcc8a --- /dev/null +++ b/codex-rs/cli/src/snapshots/codex_cli__login__storage_tests__login_default_filter_records_sanitized_storage_fallback.snap @@ -0,0 +1,7 @@ +--- +source: cli/src/login_storage_tests.rs +assertion_line: 47 +expression: "logs.replace(home.path().join(\"secrets\").to_str().unwrap(),\n\"[SECRETS_PATH]\").replace(&cause, \"[OS_ERROR]\")" +--- + WARN codex_login::auth::storage: failed to save auth to keyring, falling back to file storage: failed to write OAuth tokens to encrypted auth storage: failed to create secrets dir [SECRETS_PATH]: [OS_ERROR] + WARN codex_otel::auth_storage: credential storage file fallback completed credential_kind="codex" store_mode="auto" selected_store="secrets" actual_store="file" operation="save" secure_outcome="error" outcome="success" fallback_reason="secure_error" secure_error="other" storage_phase="policy" originator="none" diff --git a/codex-rs/cli/src/snapshots/codex_cli__login__storage_tests__login_storage_error_displays_context_and_cause.snap b/codex-rs/cli/src/snapshots/codex_cli__login__storage_tests__login_storage_error_displays_context_and_cause.snap new file mode 100644 index 000000000000..f4f7250e386c --- /dev/null +++ b/codex-rs/cli/src/snapshots/codex_cli__login__storage_tests__login_storage_error_displays_context_and_cause.snap @@ -0,0 +1,6 @@ +--- +source: cli/src/login_storage_tests.rs +assertion_line: 58 +expression: "message.replace(secrets_path.to_str().unwrap(),\n\"[SECRETS_PATH]\").replace(&cause, \"[OS_ERROR]\")" +--- +failed to write OAuth tokens to encrypted auth storage: failed to create secrets dir [SECRETS_PATH]: [OS_ERROR] diff --git a/codex-rs/cli/tests/daemon_telemetry.rs b/codex-rs/cli/tests/daemon_telemetry.rs index 48ea0b207eb3..5e265992f3e4 100644 --- a/codex-rs/cli/tests/daemon_telemetry.rs +++ b/codex-rs/cli/tests/daemon_telemetry.rs @@ -53,8 +53,14 @@ async fn foreground_update_respects_consent_and_reports_unconfirmed() -> anyhow: continue; } let body: Value = serde_json::from_slice(&requests[0].body)?; - let metric = &body["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]; - assert_eq!(metric["name"], "codex.daemon.update"); + let metric = body["resourceMetrics"] + .as_array() + .into_iter() + .flatten() + .flat_map(|resource| resource["scopeMetrics"].as_array().into_iter().flatten()) + .flat_map(|scope| scope["metrics"].as_array().into_iter().flatten()) + .find(|metric| metric["name"] == "codex.daemon.update") + .context("daemon update metric")?; assert!( metric["sum"]["dataPoints"][0]["attributes"] .as_array() diff --git a/codex-rs/cli/tests/exec_server.rs b/codex-rs/cli/tests/exec_server.rs index 75bb8344cfb5..001a69e61988 100644 --- a/codex-rs/cli/tests/exec_server.rs +++ b/codex-rs/cli/tests/exec_server.rs @@ -187,6 +187,8 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{collector_url}/v1/metrics", pr std::fs::create_dir(&bin_dir)?; let executable = bin_dir.join(format!("codex{}", std::env::consts::EXE_SUFFIX)); copy_executable(&codex_utils_cargo_bin::cargo_bin("codex")?, &executable)?; + let codex_path_dir = package.path().join("codex-path"); + std::fs::create_dir(&codex_path_dir)?; let manifest = package.path().join("codex-package.json"); std::fs::write(&manifest, r#"{"version":"1.2.3-alpha.4"}"#)?; @@ -250,10 +252,19 @@ metrics_exporter = {{ otlp-http = {{ endpoint = "{collector_url}/v1/metrics", pr .context("remote harness did not connect")???; let environment_info = client.environment_info().await?; + assert_eq!( + environment_info + .prepend_path_dirs + .iter() + .map(|path| std::fs::canonicalize(path.inferred_native_path_string())) + .collect::>>()?, + vec![codex_path_dir.canonicalize()?] + ); let expected_info = EnvironmentInfo { executor_version: "1.2.3-alpha.4".to_string(), // The build identity belongs to the spawned CLI, not this test process. provider_id: environment_info.provider_id.clone(), + prepend_path_dirs: environment_info.prepend_path_dirs.clone(), ..EnvironmentInfo::local() }; assert_eq!(environment_info, expected_info); diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index ad4fbf482881..65f7c5b4457e 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -65,6 +65,7 @@ use codex_config::McpServerTransportConfig; use codex_config::McpStartupReadiness; use codex_diagnostics::Gauge; use codex_diagnostics::GaugeGuard; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::mcp::CallToolResult; use codex_protocol::mcp::McpServerInfo; use codex_protocol::protocol::Event; @@ -772,6 +773,7 @@ impl McpConnectionSet { (server_name, outcome) }; + let startup = AuthStorageOriginator::current().scope(startup); if defer_startup { // Dormant servers must not hold the initial startup summary open. tokio::spawn(startup); @@ -893,15 +895,18 @@ impl McpConnectionSet { return Vec::new(); } + let originator = AuthStorageOriginator::current(); match tokio::task::spawn_blocking(move || { - candidates - .into_iter() - .filter_map(|(server_name, identity, config)| { - identity - .oauth_credentials_changed(&server_name, &config) - .then_some(server_name) - }) - .collect() + originator.sync_scope(|| { + candidates + .into_iter() + .filter_map(|(server_name, identity, config)| { + identity + .oauth_credentials_changed(&server_name, &config) + .then_some(server_name) + }) + .collect() + }) }) .await { diff --git a/codex-rs/codex-mcp/src/rmcp_client.rs b/codex-rs/codex-mcp/src/rmcp_client.rs index f40ca172240e..6d1d90b74ca8 100644 --- a/codex-rs/codex-mcp/src/rmcp_client.rs +++ b/codex-rs/codex-mcp/src/rmcp_client.rs @@ -57,6 +57,7 @@ use codex_connectors::ConnectorRuntimeContext; use codex_connectors::ConnectorRuntimeFetchSource; use codex_exec_server::Environment; use codex_login::AuthChangeState; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::mcp::ClientMcpExtensions; use codex_protocol::mcp::McpServerInfo; use codex_protocol::protocol::Event; @@ -286,6 +287,7 @@ fn codex_apps_reconnect_backoff(consecutive_failures: u32) -> Duration { #[derive(Clone)] struct ManagedClientStartup { + originator: AuthStorageOriginator, server_name: String, server: EffectiveMcpServer, store_mode: OAuthCredentialsStoreMode, @@ -316,6 +318,7 @@ impl ManagedClientStartup { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = None; let Self { + originator, server_name, server, store_mode, @@ -343,7 +346,7 @@ impl ManagedClientStartup { .startup_timeout_sec .unwrap_or(DEFAULT_STARTUP_TIMEOUT); let cancel_token_for_fut = cancel_token; - async move { + let startup = async move { let tool_catalog_fetch_ticket = tool_catalog_cache_context .as_ref() .map(McpToolCatalogCacheContext::begin_fetch); @@ -420,10 +423,8 @@ impl ManagedClientStartup { startup_complete.store(true, Ordering::Release); outcome - } - .in_current_span() - .boxed() - .shared() + }; + originator.scope(startup).in_current_span().boxed().shared() } } @@ -480,6 +481,7 @@ impl AsyncManagedClient { let startup_complete = Arc::new(AtomicBool::new(false)); let server_capabilities = Arc::new(StdMutex::new(None)); let startup = Arc::new(ManagedClientStartup { + originator: AuthStorageOriginator::current(), server_name, server, store_mode, diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 53c64fff363e..661bfdd43a99 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -151,6 +151,7 @@ codex-test-binary-support = { workspace = true } codex-utils-cargo-bin = { workspace = true } codex-utils-redacted-string = { workspace = true } codex-web-search-extension = { workspace = true } +codex-windows-sandbox-test-support = { workspace = true } core_test_support = { workspace = true } ctor = { workspace = true } insta = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index a3e5627aaccc..6aaa39c3f540 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -713,6 +713,9 @@ "analytics_plan_history": { "type": "boolean" }, + "api_key_cyber_access_programs": { + "type": "boolean" + }, "api_key_model_discovery": { "type": "boolean" }, @@ -954,6 +957,9 @@ "local_thread_store_shared_compression": { "type": "boolean" }, + "login_shell_package_path": { + "type": "boolean" + }, "mcp_2026_07_28": { "type": "boolean" }, @@ -6737,6 +6743,9 @@ "analytics_plan_history": { "type": "boolean" }, + "api_key_cyber_access_programs": { + "type": "boolean" + }, "api_key_model_discovery": { "type": "boolean" }, @@ -6978,6 +6987,9 @@ "local_thread_store_shared_compression": { "type": "boolean" }, + "login_shell_package_path": { + "type": "boolean" + }, "mcp_2026_07_28": { "type": "boolean" }, diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index f5ac495337fd..e852b632d8fe 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -272,6 +272,8 @@ pub struct ModelClient { restored_history: bool, request_contributors: Vec>, executed_tool_calls: Option, + // Resolved once when the session is created, like other session feature flags. + api_key_cyber_access_programs: cyber_access_program::ApiKeyCyberAccessPrograms, } /// A turn-scoped streaming session created from a [`ModelClient`]. @@ -541,6 +543,8 @@ impl ModelClient { restored_history: false, request_contributors, executed_tool_calls: None, + api_key_cyber_access_programs: + cyber_access_program::ApiKeyCyberAccessPrograms::UnsupportedProvider, } } @@ -565,10 +569,12 @@ impl ModelClient { prompt_cache_key_override: Option, event_sender: Sender, codex_responses_headers: Option>, + api_key_cyber_access_programs: cyber_access_program::ApiKeyCyberAccessPrograms, ) -> Self { self.prompt_cache_key_override = prompt_cache_key_override; self.event_sender = Some(event_sender); self.codex_responses_headers = codex_responses_headers; + self.api_key_cyber_access_programs = api_key_cyber_access_programs; self } @@ -1730,7 +1736,8 @@ impl ModelClientSession { request.access_programs = cyber_access_program::for_auth( client_setup.auth.as_ref(), prompt.cyber_access_program, - ); + self.client.api_key_cyber_access_programs, + )?; self.client .prepare_response_items_for_request(&mut request.input); if crate::guardian::is_basic_session_source(&self.client.state.session_source) { @@ -1889,7 +1896,8 @@ impl ModelClientSession { request.access_programs = cyber_access_program::for_auth( client_setup.auth.as_ref(), prompt.cyber_access_program, - ); + self.client.api_key_cyber_access_programs, + )?; let mut websocket_metadata = responses_metadata.clone(); websocket_metadata.routing_hint = self.client.build_routing_hint_header( client_setup.auth.as_ref(), diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index 07d57f5536d6..1562fb19588e 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -523,10 +523,14 @@ async fn responses_request_includes_internal_metadata_for_provider_grant_or_firs recorder.attach_direct_call_to_output(&mut item, Some((recorded, permit))); outputs.push(item); } - let original_outputs = outputs.clone(); + let persisted_outputs = outputs.clone(); recorder.attach_to_prompt(&mut outputs, &mut Default::default()); - assert_eq!(outputs, original_outputs); - let recorded = serde_json::to_value(&outputs)?; + let restored = serde_json::to_value(&outputs)?; + assert_eq!( + restored[1]["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], + resource_metadata, + ); + let recorded = serde_json::to_value(&persisted_outputs)?; assert_eq!( recorded[0]["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], resource_metadata, @@ -537,11 +541,12 @@ async fn responses_request_includes_internal_metadata_for_provider_grant_or_firs .as_str() .is_some_and(|value| value.starts_with("omitted_due_to_size_limit (overage_bytes=")) ); - let omitted_output = outputs.pop().expect("second direct output"); + let omitted_output = persisted_outputs[1].clone(); let mut without_omitted_metadata = ResponseItem::from(ResponseInputItem::FunctionCallOutput { call_id: "second".to_string(), output: FunctionCallOutputPayload::from_text("result for second".to_string()), }); + without_omitted_metadata.set_id(omitted_output.id().cloned()); without_omitted_metadata.append_executed_tool_calls(vec![ExecutedToolCall::new( "mcp__apps__read".to_string(), json!({"query": "second"}), @@ -801,7 +806,6 @@ fn responses_request_preserves_result_metadata_above_previous_aggregate_budget() let metadata_bytes = serde_json::to_vec(&body)?.len() - serde_json::to_vec(&without_metadata)?.len(); assert!(metadata_bytes > 128 * 1024); - assert!(metadata_bytes <= 2 * 1024 * 1024); assert_eq!(serde_json::to_value(&history)?, original_history); Ok(()) } diff --git a/codex-rs/core/src/cyber_access_program.rs b/codex-rs/core/src/cyber_access_program.rs index 4a508715fd47..7f75798562df 100644 --- a/codex-rs/core/src/cyber_access_program.rs +++ b/codex-rs/core/src/cyber_access_program.rs @@ -1,12 +1,55 @@ +//! Forward explicit programs while leaving entitlement and model policy to the server. + use codex_api::AccessPrograms; +use codex_features::Feature; use codex_login::CodexAuth; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result; use codex_protocol::turn_input::CyberAccessProgram; +#[derive(Clone, Copy, Debug)] +pub(crate) enum ApiKeyCyberAccessPrograms { + UnsupportedProvider, + Disabled, + Enabled, +} + +impl ApiKeyCyberAccessPrograms { + pub(crate) fn from_config(config: &crate::config::Config) -> Self { + if config.model_provider_id != codex_model_provider_info::OPENAI_PROVIDER_ID { + Self::UnsupportedProvider + } else if config.features.enabled(Feature::ApiKeyCyberAccessPrograms) + && config.features.enabled(Feature::ApiKeyModelDiscovery) + { + Self::Enabled + } else { + Self::Disabled + } + } +} + pub(crate) fn for_auth( auth: Option<&CodexAuth>, program: Option, -) -> Option { - program - .filter(|_| auth.is_some_and(CodexAuth::is_chatgpt_auth)) - .map(AccessPrograms::from) + policy: ApiKeyCyberAccessPrograms, +) -> Result> { + let Some(program) = program else { + return Ok(None); + }; + let Some(auth) = auth else { + return Ok(None); + }; + if auth.is_chatgpt_auth() { + return Ok(Some(program.into())); + } + if !auth.is_api_key_auth() { + return Ok(None); + } + match policy { + ApiKeyCyberAccessPrograms::UnsupportedProvider => Ok(None), + ApiKeyCyberAccessPrograms::Disabled => Err(CodexErr::InvalidRequest( + "Cyber access programs are disabled for this API-key session.".to_owned(), + )), + ApiKeyCyberAccessPrograms::Enabled => Ok(Some(program.into())), + } } diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index 889ae89a2e3a..e7c95576db7a 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -11,6 +11,7 @@ use codex_mcp::ElicitationReviewRequest; use codex_mcp::ElicitationReviewer; use codex_mcp::ElicitationReviewerHandle; use codex_mcp::MCP_TOOL_CODEX_APPS_META_KEY; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_prompts::ResolvedModelMessages; use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::capabilities::SelectedCapabilityRoot; @@ -343,7 +344,9 @@ impl Session { input.mcp_servers.contains_key(CODEX_APPS_MCP_SERVER_NAME), "unknown MCP server '{CODEX_APPS_MCP_SERVER_NAME}'" ); - let refreshed = self.services.mcp_runtime.replace_fresh(input).await; + let refreshed = AuthStorageOriginator::from_client_name(&desired.originator) + .scope(self.services.mcp_runtime.replace_fresh(input)) + .await; self.services.thread_extension_data.insert(selected_plugins); refreshed } @@ -375,11 +378,14 @@ impl Session { self.mark_mcp_runtime_dirty(); } - let recovered_oauth_servers = self - .services - .mcp_runtime - .updated_oauth_credentials_after_auth_failure() - .await; + let recovered_oauth_servers = + AuthStorageOriginator::from_client_name(&turn_context.originator) + .scope( + self.services + .mcp_runtime + .updated_oauth_credentials_after_auth_failure(), + ) + .await; if !recovered_oauth_servers.is_empty() && let Ok(_refresh) = self.mcp_refresh.acquire().await && self diff --git a/codex-rs/core/src/session/mcp_runtime.rs b/codex-rs/core/src/session/mcp_runtime.rs index 800527900f2f..375f88c45fdc 100644 --- a/codex-rs/core/src/session/mcp_runtime.rs +++ b/codex-rs/core/src/session/mcp_runtime.rs @@ -16,6 +16,7 @@ use codex_mcp::McpServerSource; use codex_mcp::McpStartupPolicy; use codex_mcp::PreparedMcpCall; use codex_mcp::ToolInfo; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::capabilities::SelectedCapabilityRoot; use std::collections::HashSet; @@ -300,7 +301,9 @@ impl Session { ready_selected_capability_roots, elicitation_reviewer, ); - self.services.mcp_runtime.replace(input).await; + AuthStorageOriginator::from_client_name(&desired.originator) + .scope(self.services.mcp_runtime.replace(input)) + .await; self.services.thread_extension_data.insert(selected_plugins); } diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 99066080af9a..b87697684690 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -91,6 +91,7 @@ use codex_models_manager::manager::SharedModelsManager; use codex_network_proxy::NetworkProxy; use codex_network_proxy::NetworkProxyAuditMetadata; use codex_network_proxy::normalize_host; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_otel::current_span_trace_id; use codex_otel::current_span_w3c_trace_context; use codex_otel::set_parent_from_w3c_trace_context; @@ -822,6 +823,7 @@ impl Session { ); let service_tier = get_service_tier(config.service_tier.clone(), fast_mode_enabled, &model_info); + let storage_originator = AuthStorageOriginator::from_client_name(&originator); let session_configuration = SessionConfiguration { provider: create_model_provider( config.model_provider.clone(), @@ -930,11 +932,11 @@ impl Session { // This task will run until Op::Shutdown is received. let session_for_loop = Arc::clone(&session); - let session_loop_handle = tokio::spawn(async move { + let session_loop_handle = tokio::spawn(storage_originator.scope(async move { submission_loop(session_for_loop, configured_config, rx_sub) .instrument(info_span!("session_loop", thread_id = %thread_id)) .await; - }); + })); let io = SessionIo { tx_sub, rx_event, @@ -3391,7 +3393,7 @@ impl Session { items } - fn assign_missing_response_item_id(item: &mut ResponseItem) { + pub(crate) fn assign_missing_response_item_id(item: &mut ResponseItem) { if item.id().is_some_and(|id| !id.is_empty()) { return; } diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 824cf589cd4e..e92aaeef8ae5 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1763,6 +1763,7 @@ impl Session { .or(fork_cache_key), tx_event.clone(), codex_responses_headers, + crate::cyber_access_program::ApiKeyCyberAccessPrograms::from_config(&config), ), executed_tool_calls: executed_tool_calls.clone(), code_mode_service: crate::tools::code_mode::CodeModeService::new( diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 37a35bfe3f44..c253383610fa 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -76,6 +76,23 @@ impl Shell { } } +/// The shell and startup mode Codex chose for an exec command. +#[derive(Debug, Clone)] +pub(crate) struct ShellInvocation { + pub(crate) shell: Shell, + pub(crate) use_login_shell: bool, +} + +impl ShellInvocation { + pub(crate) fn is_posix_login(&self) -> bool { + self.use_login_shell + && matches!( + self.shell.shell_type, + ShellType::Bash | ShellType::Zsh | ShellType::Sh + ) + } +} + #[cfg(all(test, unix))] fn ultimate_fallback_shell() -> Shell { codex_shell_command::shell_detect::ultimate_fallback_shell().into() diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 28a3b54b1c6f..4666142887a7 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -46,6 +46,7 @@ use codex_otel::TURN_MEMORY_METRIC; use codex_otel::TURN_NETWORK_PROXY_METRIC; use codex_otel::TURN_TOOL_CALL_METRIC; use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::EventMsg; @@ -361,46 +362,46 @@ impl Session { codex.turn.token_usage.reasoning_output_tokens = field::Empty, codex.turn.token_usage.total_tokens = field::Empty, ); - let handle = tokio::spawn( - async move { - let ctx_for_finish = Arc::clone(&ctx); - let task_result = task_for_run - .run( - Arc::clone(&session), - ctx, - task_input, - task_cancellation_token.child_token(), - ) - .instrument(trace_span!("session_task.run")) - .await; - let sess = Arc::clone(&session); - // Private reviewers save their transcript together with the terminal event. - // Errors and cancellation retain their existing save path. - if (!sess.is_private_guardian_reviewer().await - || task_cancellation_token.is_cancelled() - || task_result.is_err()) - && let Err(err) = sess.flush_rollout().await - { - warn!("failed to flush rollout before completing turn: {err}"); - sess.send_event( - ctx_for_finish.as_ref(), - EventMsg::Warning(WarningEvent { - message: format!( - "Failed to save the conversation transcript; Codex will continue retrying. Error: {err}" - ), - }), - ) + let storage_originator = AuthStorageOriginator::from_client_name(&turn_context.originator); + let task_future = async move { + let ctx_for_finish = Arc::clone(&ctx); + let task_result = task_for_run + .run( + Arc::clone(&session), + ctx, + task_input, + task_cancellation_token.child_token(), + ) + .instrument(trace_span!("session_task.run")) + .await; + let sess = Arc::clone(&session); + // Private reviewers save their transcript together with the terminal event. + // Errors and cancellation retain their existing save path. + if (!sess.is_private_guardian_reviewer().await + || task_cancellation_token.is_cancelled() + || task_result.is_err()) + && let Err(err) = sess.flush_rollout().await + { + warn!("failed to flush rollout before completing turn: {err}"); + sess.send_event( + ctx_for_finish.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "Failed to save the conversation transcript; Codex will continue retrying. Error: {err}" + ), + }), + ) + .await; + } + if !task_cancellation_token.is_cancelled() { + // Finish uniformly from the spawn site so all tasks share the same lifecycle. + sess.on_task_finished(Arc::clone(&ctx_for_finish), task_result) .await; - } - if !task_cancellation_token.is_cancelled() { - // Finish uniformly from the spawn site so all tasks share the same lifecycle. - sess.on_task_finished(Arc::clone(&ctx_for_finish), task_result) - .await; - } - done_clone.notify_waiters(); } - .instrument(task_span), - ); + done_clone.notify_waiters(); + } + .instrument(task_span); + let handle = tokio::spawn(storage_originator.scope(task_future)); let timer = turn_context .session_telemetry .start_timer(TURN_E2E_DURATION_METRIC, &[]) diff --git a/codex-rs/core/src/tools/executed_tool_calls.rs b/codex-rs/core/src/tools/executed_tool_calls.rs index ee05b483bd41..636d3cbd92ae 100644 --- a/codex-rs/core/src/tools/executed_tool_calls.rs +++ b/codex-rs/core/src/tools/executed_tool_calls.rs @@ -20,19 +20,20 @@ use codex_code_mode::CellId; use codex_features::Feature; use codex_features::Features; use codex_history::InitialHistory; +use codex_protocol::ResponseItemId; use codex_protocol::mcp::McpAttribution; use codex_protocol::mcp::McpAttributionSource; use codex_protocol::models::ExecutedToolCall; use codex_protocol::models::ExecutedToolCallArguments; use codex_protocol::models::ResponseItem; use codex_protocol::models::ToolResultMetadata; -use codex_protocol::models::bound_executed_tool_calls_for_prompt; -use codex_protocol::models::bound_executed_tool_calls_for_prompt_prioritizing_recent; use codex_protocol::models::executed_tool_call_metadata_bytes; +use codex_protocol::models::normalize_executed_tool_call_arguments; use codex_protocol::openai_models::ToolMode; use indexmap::IndexMap; use serde_json::Value as JsonValue; +use crate::session::session::Session; use crate::session::step_context::StepContext; use crate::tools::context::ToolCallSource; use crate::tools::context::ToolOutput; @@ -97,6 +98,8 @@ impl Drop for DirectCallPermit { struct ExecutedToolCallRecorderState { // Disabling drops this lifetime, permanently invalidating prepared Direct records. recording: Arc<()>, + // Full live Direct observations; persisted outputs keep their bounded projection. + direct_calls: HashMap, cells: HashMap, output_cells: HashMap, retained_calls: HashMap<(std::mem::Discriminant, String), RetainedToolCalls>, @@ -399,13 +402,19 @@ impl ExecutedToolCalls { let Some((call, permit)) = prepared else { return; }; - let state = self.lock_state(); - let Some(state) = state.as_ref() else { + let mut state = self.lock_state(); + let Some(state) = state.as_mut() else { return; }; if !permit.recording.ptr_eq(&Arc::downgrade(&state.recording)) { return; } + // Assign the output's ordinary history ID here so reused model call IDs + // cannot associate this observation with a different output. + Session::assign_missing_response_item_id(item); + if let Some(id) = item.id() { + state.direct_calls.insert(id.clone(), call.clone()); + } let complete = matches!(call.arguments(), ExecutedToolCallArguments::Raw(_)); item.append_executed_tool_calls(vec![call]); if complete { diff --git a/codex-rs/core/src/tools/executed_tool_calls/request_metadata.rs b/codex-rs/core/src/tools/executed_tool_calls/request_metadata.rs index 762eccee8923..1ae9eefb166e 100644 --- a/codex-rs/core/src/tools/executed_tool_calls/request_metadata.rs +++ b/codex-rs/core/src/tools/executed_tool_calls/request_metadata.rs @@ -1,59 +1,74 @@ -//! Attaches host observations to request metadata and applies its byte budget. -//! Final request budgeting runs after the recorder lock is released; it never changes tool execution. +//! Attaches host observations to request metadata without changing tool execution. use super::*; #[derive(Clone, Copy, PartialEq, Eq)] -enum MetadataBudgetScope { - AllCalls, - CodeModeOnly, +enum RequestKind { + Sampling, + Compaction, } impl ExecutedToolCalls { - /// Attaches Code Mode observations while preserving Direct records captured in history. pub(crate) fn attach_to_compaction_prompt(&self, items: &mut [ResponseItem]) { - let mut state = self.lock_state(); - let Some(state) = state.as_mut() else { - clear_direct_call_metadata(items); - return; - }; - Self::attach_pending_to_prompt_with_state( - state, - items, - &mut HashMap::new(), - MetadataBudgetScope::CodeModeOnly, - ); + self.attach_to_request(items, &mut HashMap::new(), RequestKind::Compaction); } - /// Attaches trusted completeness; request budgeting only revokes damaged inventories. pub(crate) fn attach_to_prompt( &self, items: &mut [ResponseItem], retry_cache: &mut ExecutedToolCallCache, ) { - let attached = { + self.attach_to_request(items, retry_cache, RequestKind::Sampling); + } + + fn attach_to_request( + &self, + items: &mut [ResponseItem], + retry_cache: &mut ExecutedToolCallCache, + request_kind: RequestKind, + ) { + let recording = { let mut state = self.lock_state(); - state.as_mut().map(|state| { - Self::attach_pending_to_prompt_with_state( - state, - items, - retry_cache, - MetadataBudgetScope::AllCalls, - ) - }) - }; - let Some(attached) = attached else { - // Direct records now live in history; disabling capture also stops replaying them. - clear_direct_call_metadata(items); - return; + let Some(state) = state.as_mut() else { + // Disabling capture also stops replaying Direct records from history. + clear_direct_call_metadata(items); + return; + }; + for item in items.iter_mut() { + if let Some(call) = item.id().and_then(|id| state.direct_calls.get(id)) { + item.clear_executed_tool_calls(); + item.append_executed_tool_calls(vec![call.clone()]); + if matches!(call.arguments(), ExecutedToolCallArguments::Raw(_)) { + item.mark_tool_calls_complete(); + } + } + } + // A shortened compaction attempt is not an installed history window. + // Only ordinary sampling can discard observations absent from the window. + if request_kind == RequestKind::Sampling && !state.direct_calls.is_empty() { + let output_ids = items + .iter() + .filter_map(ResponseItem::id) + .collect::>(); + state.direct_calls.retain(|id, _| output_ids.contains(id)); + } + Self::attach_pending_to_prompt_with_state(state, items, retry_cache, request_kind); + Arc::downgrade(&state.recording) }; - if attached || items.iter().any(has_direct_call_metadata) { - metadata_metrics::bound_prompt_metadata( - items, - bound_executed_tool_calls_for_prompt, - "request", - codex_otel::global().as_ref(), - ); + + // Existing history can contain arguments recorded by an older client. Keep the + // per-call limit, and prevent newly truncated cells from claiming completeness + // on a later wait. Do the serialization outside the recorder lock. + let truncated_origins = normalize_executed_tool_call_arguments(items); + if !truncated_origins.is_empty() { + let mut state = self.lock_state(); + if let Some(state) = state.as_mut() + && recording.ptr_eq(&Arc::downgrade(&state.recording)) + { + for origin in truncated_origins { + state.invalidate_origin(&origin); + } + } } } @@ -61,7 +76,7 @@ impl ExecutedToolCalls { state: &mut ExecutedToolCallRecorderState, items: &mut [ResponseItem], retry_cache: &mut ExecutedToolCallCache, - budget_scope: MetadataBudgetScope, + request_kind: RequestKind, ) -> bool { // Failed wrappers have no callback; only their bounded bitmap observation survives. state.pending_wrapper_origins.clear(); @@ -317,7 +332,7 @@ impl ExecutedToolCalls { } // Compaction may retry with a shortened history without installing that history. // Keep absent observations until a normal sampling request confirms the live window. - if budget_scope == MetadataBudgetScope::AllCalls && !pending_outputs.is_empty() { + if request_kind == RequestKind::Sampling && !pending_outputs.is_empty() { state .retained_calls .retain(|key, _| !pending_outputs.contains(key)); @@ -378,6 +393,7 @@ impl ExecutedToolCalls { .is_some_and(|cell_id| invalid_cells.contains(cell_id)) { retained.complete = false; + retained.call_index_by_id.clear(); retained.clear_late_truncated_metadata(); } } @@ -400,122 +416,6 @@ impl ExecutedToolCalls { } } - let metadata_bytes = items - .iter() - .filter(|item| { - budget_scope == MetadataBudgetScope::AllCalls || !has_direct_call_metadata(item) - }) - .fold(0_usize, |bytes, item| { - bytes.saturating_add(executed_tool_call_metadata_bytes(item)) - }); - if metadata_bytes > MAX_EXECUTED_TOOL_CALL_FULL_ARGUMENT_BYTES_PER_OUTPUT { - match budget_scope { - MetadataBudgetScope::AllCalls => { - metadata_metrics::bound_prompt_metadata( - items, - bound_executed_tool_calls_for_prompt_prioritizing_recent, - "retained", - codex_otel::global().as_ref(), - ); - } - MetadataBudgetScope::CodeModeOnly => { - // Validate bindings against the complete history above, but do not make - // captured Direct records compete with Code Mode's bounded recorder. - let (indices, mut bounded): (Vec<_>, Vec<_>) = items - .iter() - .enumerate() - .filter(|(_, item)| { - !has_direct_call_metadata(item) - && item.executed_tool_call_metadata().is_some() - }) - .map(|(index, item)| (index, item.clone())) - .unzip(); - metadata_metrics::bound_prompt_metadata( - &mut bounded, - bound_executed_tool_calls_for_prompt_prioritizing_recent, - "compaction", - codex_otel::global().as_ref(), - ); - for (index, item) in indices.into_iter().zip(bounded) { - items[index] = item; - } - } - } - let retained_before_bounding = std::mem::take(&mut state.retained_calls); - let mut bounded_outputs = HashSet::new(); - for item in items.iter_mut() { - if has_direct_call_metadata(item) { - continue; - } - let Some(call_id) = output_call_id(item) else { - continue; - }; - let key = (std::mem::discriminant(&*item), call_id.to_string()); - // History already carries unmanaged records; retaining them would append them again. - let Some(previous) = retained_before_bounding.get(&key) else { - continue; - }; - let metadata = item.executed_tool_call_metadata(); - let unique_output = bounded_outputs.insert(key.clone()); - if !unique_output && let Some(retained) = state.retained_calls.get_mut(&key) { - retained.call_index_by_id.clear(); - retained.clear_late_truncated_metadata(); - } - if let Some(runtime_cell_id) = &previous.runtime_cell_id - && metadata - .is_none_or(|metadata| !metadata.has_same_tool_calls(&previous.calls)) - && let Some(cell) = state.cells.get_mut(runtime_cell_id) - { - cell.completion = CellCompletion::Incomplete; - } - if let Some(metadata) = metadata - && (metadata - .executed_tool_calls - .as_ref() - .is_some_and(|calls| !calls.is_empty()) - || metadata.tool_calls_complete.is_some()) - { - // Metadata-only shedding preserves slots; changed calls or duplicate outputs do not. - let same_calls = unique_output && metadata.has_same_tool_calls(&previous.calls); - let call_index_by_id = if same_calls { - previous.call_index_by_id.clone() - } else { - HashMap::new() - }; - let retained = state.retained_calls.entry(key).or_default(); - retained.runtime_cell_id = previous.runtime_cell_id.clone(); - retained.calls = metadata.executed_tool_calls.clone().unwrap_or_default(); - if let Some(cell_id) = metadata.cell_id.as_ref() { - retained.cell_id = Some(cell_id.clone()); - } - retained.complete |= metadata.tool_calls_complete == Some(true); - retained.call_index_by_id = call_index_by_id; - // Budgeting preserves call order; changed inventories lose their indices. - if same_calls { - retained.truncated_call_index_by_id = - previous.truncated_call_index_by_id.clone(); - retained.late_truncated_indices = previous.late_truncated_indices.clone(); - } else { - // The budgeter never reorders calls. Remove only the new late - // evidence before losing the indices that identify it. - for index in &previous.late_truncated_indices { - if let Some(call) = retained.calls.get_mut(*index) { - call.set_tool_result_metadata(ToolResultMetadata::default()); - } - } - } - retained.result_metadata_updated = previous.result_metadata_updated; - } - } - if budget_scope == MetadataBudgetScope::CodeModeOnly { - state.retained_calls.extend( - retained_before_bounding - .into_iter() - .filter(|(key, _)| pending_outputs.contains(key)), - ); - } - } - attached } } diff --git a/codex-rs/core/src/tools/executed_tool_calls/request_metadata_tests.rs b/codex-rs/core/src/tools/executed_tool_calls/request_metadata_tests.rs index c92841f41313..b4ffbc8276af 100644 --- a/codex-rs/core/src/tools/executed_tool_calls/request_metadata_tests.rs +++ b/codex-rs/core/src/tools/executed_tool_calls/request_metadata_tests.rs @@ -20,12 +20,7 @@ impl ExecutedToolCalls { let Some(state) = state.as_mut() else { return false; }; - Self::attach_pending_to_prompt_with_state( - state, - items, - retry_cache, - MetadataBudgetScope::AllCalls, - ) + Self::attach_pending_to_prompt_with_state(state, items, retry_cache, RequestKind::Sampling) } } @@ -145,147 +140,133 @@ fn compaction_attaches_pending_and_retained_code_mode_metadata() { } #[test] -fn failed_compaction_preserves_observations_absent_from_trimmed_retries() { - let call_count = 100; - for rebudget in [false, true] { - let recorder = new_recorder(InitialHistory::New); - let mut history = Vec::new(); - for origin in ["old", "recent"] { - let cell = CellId::new(origin.to_string()); - recorder.start_cell(&cell, origin); - for index in 0..call_count { - record_nested_call(&recorder, &cell, &format!("nested-{index}")); - } - recorder.finish_cell_recording(&cell); - history.extend([exec_input(origin), exec_output(origin)]); - } - let mut full_attempt = history.clone(); - recorder.attach_to_compaction_prompt(&mut full_attempt); - assert_eq!(tool_calls_complete(&full_attempt[1]), Some(true)); - - if rebudget { - // Late results make the retained retry item exceed the budget independently of - // the older output that the compaction retry has temporarily removed. - let source = ToolCallSource::CodeMode { - cell_id: "recent".to_string(), - runtime_tool_call_id: "runtime".to_string(), - }; - for index in 0..call_count { - assert!(recorder.record_tool_result_metadata( - &source, - &format!("nested-{index}"), - &json!({"provider": "x".repeat(24 * 1024)}), - )); - } - } - let mut trimmed_attempt = history[2..].to_vec(); - recorder.attach_to_compaction_prompt(&mut trimmed_attempt); - assert_eq!(tool_calls_complete(&trimmed_attempt[1]), Some(true)); - if rebudget { - let encoded = serde_json::to_value(&trimmed_attempt[1]).unwrap(); - assert!( - encoded["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"] - .as_str() - .is_some_and(|marker| marker.starts_with("omitted_due_to_size_limit (overage_bytes=")), - ); - } - - // Exhausted retries leave the session's original history intact. - let mut expected = full_attempt; - expected[2..].clone_from_slice(&trimmed_attempt); - // The restored inventory must fit without causing another round of raw-result shedding. - assert!( - expected - .iter() - .map(executed_tool_call_metadata_bytes) - .sum::() - <= 2 * 1024 * 1024 - ); - let mut resumed = history.clone(); - recorder.attach_to_prompt(&mut resumed, &mut HashMap::new()); - assert_eq!(resumed, expected); - // Once ordinary sampling advances to a smaller window, the old record is pruned. - recorder.attach_to_prompt(&mut history[2..], &mut HashMap::new()); - let mut old = vec![exec_input("old"), exec_output("old")]; - recorder.attach_to_prompt(&mut old, &mut HashMap::new()); - assert_eq!(old, vec![exec_input("old"), exec_output("old")]); - } -} - -#[test] -fn compaction_bounds_code_mode_metadata_without_rebudgeting_direct_history() { +fn large_direct_and_code_mode_metadata_survive_sampling_and_compaction() { let recorder = new_recorder(InitialHistory::New); - let mut history = Vec::new(); - let arguments = json!({"value": "x".repeat(7 * 1024)}); - let argument_bytes = serialized_json_bytes(&arguments).unwrap(); - for index in 0..96 { - let origin = format!("exec-{index}"); - let cell = CellId::new(format!("cell-{index}")); - recorder.start_cell(&cell, &origin); - let mut call = ExecutedToolCall::new("nested_tool".to_string(), arguments.clone()); - call.set_tool_result_metadata(ToolResultMetadata::new(&json!({ - "provider": "x".repeat(24 * 1024), - }))); - recorder.record_nested_tool_call( - cell.clone(), - format!("nested-{index}"), - call, - argument_bytes, - ); - recorder.finish_cell_recording(&cell); - history.extend([exec_input(&origin), exec_output(&origin)]); - } - let direct_start = history.len(); - for index in 0..24 { - let mut item = output(&format!("direct-{index}")); - let mut call = ExecutedToolCall::new("direct_tool".to_string(), arguments.clone()); - call.set_tool_result_metadata(ToolResultMetadata::new(&json!({"id": index}))); - recorder.attach_direct_call_to_output( - &mut item, - Some((call, recorder.reserve_direct_call().unwrap())), - ); - history.push(item); - } - assert!( - history[direct_start..] - .iter() - .map(executed_tool_call_metadata_bytes) - .sum::() - > 128 * 1024 + recorder + .retained_direct_metadata_bytes + .store(MAX_RETAINED_DIRECT_METADATA_BYTES, Ordering::Relaxed); + + let direct_metadata = json!({"openai/resource_access": {"data": "d".repeat(900 * 1024)}}); + let mut direct_call = ExecutedToolCall::new("direct_tool".to_string(), json!({})); + direct_call.set_tool_result_metadata(ToolResultMetadata::new(&direct_metadata)); + let ordinary_direct = output("direct"); + let mut direct = ordinary_direct.clone(); + recorder.attach_direct_call_to_output( + &mut direct, + Some((direct_call, recorder.reserve_direct_call().unwrap())), ); + assert!(direct.executed_tool_call_metadata().is_none()); + + let cell = CellId::new("code-cell".to_string()); + recorder.start_cell(&cell, "exec"); + record_nested_call(&recorder, &cell, "nested"); + let code_metadata = json!({"openai/resource_access": {"data": "c".repeat(1400 * 1024)}}); + let source = ToolCallSource::CodeMode { + cell_id: cell.as_str().to_string(), + runtime_tool_call_id: "nested".to_string(), + }; + assert!(recorder.record_tool_result_metadata(&source, "nested", &code_metadata)); + recorder.finish_cell_recording(&cell); + let ordinary_code = ResponseItem::from(ResponseInputItem::CustomToolCallOutput { + call_id: "exec".to_string(), + name: None, + output: FunctionCallOutputPayload::from_text("ordinary result".to_string()), + }); + let history = vec![direct, exec_input("exec"), ordinary_code.clone()]; - let mut code_mode_only = history[..direct_start].to_vec(); - recorder.attach_to_compaction_prompt(&mut code_mode_only); let mut compact = history.clone(); recorder.attach_to_compaction_prompt(&mut compact); - assert_eq!(&compact[direct_start..], &history[direct_start..]); - assert_eq!(&compact[..direct_start], &code_mode_only); assert!( - compact[..direct_start] + compact .iter() .map(executed_tool_call_metadata_bytes) .sum::() - <= 2 * 1024 * 1024 + > 2 * 1024 * 1024 ); - let mut newest = exec_output("exec-95"); - let mut newest_call = ExecutedToolCall::new("nested_tool".to_string(), arguments); - newest_call.set_tool_result_metadata(ToolResultMetadata::new(&json!({ - "provider": "x".repeat(24 * 1024), - }))); - newest.append_executed_tool_calls(vec![newest_call]); - newest.set_tool_call_cell_id("exec-95"); - newest.mark_tool_calls_complete(); - assert_eq!(compact[direct_start - 1], newest); - let oldest = serde_json::to_value(&compact[1]).unwrap(); - assert!( - oldest["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"] - .as_str() - .is_some_and(|marker| marker.starts_with("omitted_due_to_size_limit (overage_bytes=")), + let direct_wire = serde_json::to_value(&compact[0]).unwrap(); + let code_wire = serde_json::to_value(&compact[2]).unwrap(); + assert_eq!( + direct_wire["output"], + serde_json::to_value(&ordinary_direct).unwrap()["output"] + ); + assert_eq!( + code_wire["output"], + serde_json::to_value(&ordinary_code).unwrap()["output"] + ); + assert_eq!( + direct_wire["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], + direct_metadata + ); + assert_eq!( + code_wire["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], + code_metadata + ); + assert_eq!(tool_calls_complete(&compact[2]), Some(true)); + + let mut sampled = history.clone(); + recorder.attach_to_prompt(&mut sampled, &mut HashMap::new()); + assert_eq!(sampled, compact); + + // A smaller window still has the full Code Mode result and its completion marker. + let mut next = history[1..].to_vec(); + recorder.attach_to_prompt(&mut next, &mut HashMap::new()); + assert_eq!(next[1], compact[2]); +} + +#[test] +fn failed_compaction_preserves_observations_absent_from_trimmed_retries() { + let recorder = new_recorder(InitialHistory::New); + recorder + .retained_direct_metadata_bytes + .store(MAX_RETAINED_DIRECT_METADATA_BYTES, Ordering::Relaxed); + let mut direct = output("direct"); + let mut direct_call = ExecutedToolCall::new("direct_tool".to_string(), json!({})); + direct_call.set_tool_result_metadata(ToolResultMetadata::new(&json!({"resource": "old"}))); + recorder.attach_direct_call_to_output( + &mut direct, + Some((direct_call, recorder.reserve_direct_call().unwrap())), ); - assert_eq!(tool_calls_complete(&compact[1]), Some(true)); - let mut retry = history; - recorder.attach_to_compaction_prompt(&mut retry); - assert_eq!(retry, compact); + assert!(direct.executed_tool_call_metadata().is_none()); + let mut history = vec![direct.clone()]; + for origin in ["old", "recent"] { + let cell = CellId::new(origin.to_string()); + recorder.start_cell(&cell, origin); + record_nested_call(&recorder, &cell, "nested"); + recorder.finish_cell_recording(&cell); + history.extend([exec_input(origin), exec_output(origin)]); + } + let mut full_attempt = history.clone(); + recorder.attach_to_compaction_prompt(&mut full_attempt); + assert_eq!(tool_calls_complete(&full_attempt[0]), Some(true)); + assert_eq!(tool_calls_complete(&full_attempt[2]), Some(true)); + + let source = ToolCallSource::CodeMode { + cell_id: "recent".to_string(), + runtime_tool_call_id: "runtime".to_string(), + }; + let large_metadata = json!({"provider": "x".repeat(2100 * 1024)}); + assert!(recorder.record_tool_result_metadata(&source, "nested", &large_metadata)); + let mut trimmed_attempt = history[3..].to_vec(); + recorder.attach_to_compaction_prompt(&mut trimmed_attempt); + assert_eq!(tool_calls_complete(&trimmed_attempt[1]), Some(true)); + let encoded = serde_json::to_value(&trimmed_attempt[1]).unwrap(); + assert_eq!( + encoded["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], + large_metadata + ); + + // Failed retries do not install the trimmed history or lose older observations. + let mut expected = full_attempt; + expected[3..].clone_from_slice(&trimmed_attempt); + let mut resumed = history.clone(); + recorder.attach_to_prompt(&mut resumed, &mut HashMap::new()); + assert_eq!(resumed, expected); + + // Normal sampling in the smaller window can prune the old observation. + recorder.attach_to_prompt(&mut history[3..], &mut HashMap::new()); + let mut old = vec![direct.clone(), exec_input("old"), exec_output("old")]; + recorder.attach_to_prompt(&mut old, &mut HashMap::new()); + assert_eq!(old, vec![direct, exec_input("old"), exec_output("old")]); } #[test] @@ -323,6 +304,7 @@ fn direct_retained_metadata_budget_sheds_results_then_calls_across_refresh() { Some((call.clone(), recorder.reserve_direct_call().unwrap())), ); let mut expected_first = output("first"); + expected_first.set_id(first.id().cloned()); expected_first.append_executed_tool_calls(vec![call.clone()]); expected_first.mark_tool_calls_complete(); assert_eq!(first, expected_first); @@ -356,6 +338,7 @@ fn direct_retained_metadata_budget_sheds_results_then_calls_across_refresh() { &mut resource_only_output, Some((call.clone(), recorder.reserve_direct_call().unwrap())), ); + expected_resource_only.set_id(resource_only_output.id().cloned()); assert_eq!(resource_only_output, expected_resource_only); assert_eq!( recorder @@ -380,6 +363,7 @@ fn direct_retained_metadata_budget_sheds_results_then_calls_across_refresh() { Some((call.clone(), recorder.reserve_direct_call().unwrap())), ); let mut expected_second = output("second"); + expected_second.set_id(second.id().cloned()); expected_second.append_executed_tool_calls(vec![ExecutedToolCall::new( "test_tool".to_string(), json!({"argument": "kept"}), @@ -550,8 +534,13 @@ async fn recorder_refreshes_without_changing_execution_features_or_claiming_miss crate::ConfigRefreshOutcome::Published ); let mut replay = output(&call.call_id); + // Replay the same history item across the configuration refresh. + replay.set_id(direct_output.id().cloned()); calls.attach_direct_call_to_output(&mut replay, prepared); - assert_eq!(replay, direct_output); + assert_eq!( + replay.executed_tool_call_metadata(), + direct_output.executed_tool_call_metadata() + ); } let cell = CellId::new(format!("cell-{index}")); @@ -704,79 +693,6 @@ fn executed_tool_call_recorder_bounds_pending_calls_and_preserves_overflow() { assert!(!state.cells.contains_key(&cell_id)); } -#[test] -fn executed_tool_call_recorder_bounds_retained_history_and_keeps_latest_calls() { - let recorder = new_recorder(InitialHistory::Forked(Vec::new())); - let mut history = Vec::new(); - let arguments = json!({ "payload": "x".repeat(7 * 1024) }); - let argument_bytes = serialized_json_bytes(&arguments).expect("tool arguments must serialize"); - let cell_count = 100; - let calls_per_cell = 4; - assert!(argument_bytes < MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES); - assert!( - argument_bytes * calls_per_cell < MAX_EXECUTED_TOOL_CALL_FULL_ARGUMENT_BYTES_PER_OUTPUT - ); - assert!(argument_bytes * calls_per_cell * cell_count > 2 * 1024 * 1024); - let mut prompt = Vec::new(); - - for index in 0..cell_count { - let call_id = format!("retained-{index}"); - let cell_id = CellId::new(format!("retained-cell-{index}")); - recorder.start_cell(&cell_id, &call_id); - let expected_calls = (0..calls_per_cell) - .map(|nested_index| { - ExecutedToolCall::new( - format!("retained_tool_{index}_{nested_index}"), - arguments.clone(), - ) - }) - .collect::>(); - for (nested_index, call) in expected_calls.iter().enumerate() { - recorder.record_nested_tool_call( - cell_id.clone(), - format!("nested-{index}-{nested_index}"), - call.clone(), - argument_bytes, - ); - } - recorder.finish_cell_recording(&cell_id); - history.extend([exec_input(&call_id), exec_output(&call_id)]); - prompt = history.clone(); - assert!(recorder.attach_pending_to_prompt(&mut prompt, &mut HashMap::new())); - codex_protocol::models::bound_executed_tool_calls_for_prompt(&mut prompt); - let latest_calls = prompt - .last() - .and_then(ResponseItem::executed_tool_call_metadata) - .and_then(|metadata| metadata.executed_tool_calls.as_ref()) - .expect("latest tool calls must remain in retained metadata"); - assert_eq!(latest_calls, &expected_calls); - } - - let state = recorder.lock_state(); - let state = state.as_ref().unwrap(); - let retained_bytes = state - .retained_calls - .values() - .map(|retained| serialized_json_bytes(&retained.calls)) - .sum::>() - .expect("retained calls must serialize"); - assert!(retained_bytes <= 2 * 1024 * 1024); - - let metadata = prompt - .iter() - .filter_map(ResponseItem::executed_tool_call_metadata) - .filter_map(|metadata| metadata.executed_tool_calls.as_ref()) - .flatten() - .map(|call| serde_json::to_value(call).expect("retained call must serialize")) - .collect::>(); - assert!(metadata.len() < cell_count * calls_per_cell); - // Dropped outputs do not add their omission counts to unrelated surviving calls. - assert!(metadata.iter().all(|call| { - let omitted = &call["arguments"]["_codex_executed_tool_call_truncated"]["omitted_calls"]; - omitted.is_null() || *omitted == json!(calls_per_cell - 1) - })); -} - #[test] fn tool_call_completeness_requires_finished_lossless_recording() { for scenario in ["unfinished", "unobserved", "late_call"] { @@ -1047,64 +963,6 @@ fn tool_call_completeness_survives_waits_without_changing_deltas() { } } -#[test] -fn unrelated_raw_overflow_retains_a_complete_exec_within_the_request_budget() { - let request_budget = 2 * 1024 * 1024; - let recorder = new_recorder(InitialHistory::New); - let cell = CellId::new("clean-overflow-cell".to_string()); - recorder.start_cell(&cell, "exec-clean"); - let clean_call = record_nested_call(&recorder, &cell, "nested-clean"); - recorder.finish_cell_recording(&cell); - - let mut unrelated = output("direct-oversized"); - let arguments = json!("x".repeat(7 * 1024)); - assert!(serialized_json_bytes(&arguments).unwrap() <= MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES); - unrelated.append_executed_tool_calls( - (0..request_budget / (7 * 1024) + 1) - .map(|index| ExecutedToolCall::new(format!("direct-{index}"), arguments.clone())) - .collect(), - ); - let original = [ - unrelated, - exec_input("exec-clean"), - exec_output("exec-clean"), - ]; - let mut prompt = original.clone(); - assert!( - prompt - .iter() - .map(executed_tool_call_metadata_bytes) - .sum::() - > request_budget - ); - recorder.attach_to_prompt(&mut prompt, &mut HashMap::new()); - let metadata = prompt[2] - .executed_tool_call_metadata() - .expect("the later clean output must retain its calls"); - assert_eq!(metadata.executed_tool_calls, Some(vec![clean_call])); - assert_eq!(metadata.tool_calls_complete, Some(true)); - assert!( - prompt[0] - .executed_tool_call_metadata() - .and_then(|metadata| metadata.executed_tool_calls.as_ref()) - .is_some_and(|calls| calls.iter().any(|call| matches!( - call.arguments(), - ExecutedToolCallArguments::Truncated { .. } - ))) - ); - assert!( - prompt - .iter() - .map(executed_tool_call_metadata_bytes) - .sum::() - <= request_budget - ); - - let mut retry = original; - recorder.attach_to_prompt(&mut retry, &mut HashMap::new()); - assert_eq!(retry, prompt); -} - #[test] fn paginated_history_allows_fresh_origins_but_rejects_known_ids() { for origin in ["fresh-exec", "old-exec"] { @@ -1298,72 +1156,46 @@ fn completeness_rejects_direct_id_collision_and_late_calls() { } #[test] -fn request_truncation_prevents_completion_after_compaction() { +fn newly_normalized_arguments_keep_later_wait_incomplete() { let recorder = new_recorder(InitialHistory::New); - let cell_id = CellId::new("compacted-cell".to_string()); - recorder.start_cell(&cell_id, "exec"); - + let cell = CellId::new("active-cell".to_string()); + recorder.start_cell(&cell, "exec"); let arguments = json!({ - "payload": "x".repeat(MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES - r#"{"payload":""}"#.len()), + "_codex_executed_tool_call_truncated": true, + "padding": "x".repeat(8_120), + }) + .to_string(); + let (call, original_bytes) = recorded_call(&ToolCall { + tool_name: codex_tools::ToolName::plain("nested_tool"), + call_id: "nested".to_string(), + payload: ToolPayload::Function { arguments }, + encrypted_function_args: None, }); - for index in 0..4 { - recorder.record_nested_tool_call( - cell_id.clone(), - format!("nested-{index}"), - ExecutedToolCall::new("nested_tool".to_string(), arguments.clone()), - MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES, - ); - } - - let mut initial = vec![exec_input("exec"), exec_output("exec")]; - for index in 0..63 { - let origin = format!("other-{index}"); - let other_cell = CellId::new(origin.clone()); - recorder.start_cell(&other_cell, &origin); - for nested in 0..4 { - recorder.record_nested_tool_call( - other_cell.clone(), - format!("other-{index}-{nested}"), - ExecutedToolCall::new("nested_tool".to_string(), arguments.clone()), - MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES, - ); - } - recorder.finish_cell_recording(&other_cell); - initial.extend([exec_input(&origin), exec_output(&origin)]); - } - assert!(recorder.attach_pending_to_prompt(&mut initial, &mut HashMap::new())); + assert!(original_bytes <= MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES); + assert!( + serialized_json_bytes(call.arguments()).unwrap() > MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES + ); + recorder.record_nested_tool_call(cell.clone(), "nested".to_string(), call, original_bytes); + let mut initial = [exec_input("exec"), exec_output("exec")]; + recorder.attach_to_prompt(&mut initial, &mut HashMap::new()); assert!( initial[1] .executed_tool_call_metadata() - .and_then(|metadata| metadata.executed_tool_calls.as_ref()) - .is_some_and(|calls| calls.iter().any(|call| matches!( + .unwrap() + .executed_tool_calls + .as_ref() + .unwrap() + .iter() + .any(|call| matches!( call.arguments(), ExecutedToolCallArguments::Truncated { .. } - ))) - ); - let source = ToolCallSource::CodeMode { - cell_id: cell_id.as_str().to_string(), - runtime_tool_call_id: "runtime-call".to_string(), - }; - for index in 0..4 { - assert!(!recorder.record_tool_result_metadata( - &source, - &format!("nested-{index}"), - &json!({}), - )); - } - - recorder.attach_pending_to_prompt(&mut [], &mut HashMap::new()); - recorder.register_cell(&cell_id, "wait"); - recorder.finish_cell_recording(&cell_id); - let mut final_output = [wait_input("wait", &cell_id), output("wait")]; - recorder.attach_pending_to_prompt(&mut final_output, &mut HashMap::new()); - assert_eq!( - final_output[1] - .executed_tool_call_metadata() - .and_then(|metadata| metadata.tool_calls_complete), - None, + )) ); + recorder.register_cell(&cell, "wait"); + recorder.finish_cell_recording(&cell); + let mut wait = [wait_input("wait", &cell), output("wait")]; + recorder.attach_to_prompt(&mut wait, &mut HashMap::new()); + assert_eq!(tool_calls_complete(&wait[1]), None); } #[test] @@ -1720,7 +1552,31 @@ fn late_truncated_metadata_is_not_attached_to_ambiguous_outputs() { } #[test] -fn late_truncated_metadata_obeys_the_existing_request_budget() { +fn late_raw_metadata_is_not_attached_after_duplicate_outputs() { + let recorder = new_recorder(InitialHistory::New); + let cell = CellId::new("ambiguous-cell".to_string()); + recorder.start_cell(&cell, "exec"); + record_nested_call(&recorder, &cell, "nested"); + let mut duplicate = [exec_input("exec"), exec_output("exec"), exec_output("exec")]; + recorder.attach_to_prompt(&mut duplicate, &mut HashMap::new()); + + let source = ToolCallSource::CodeMode { + cell_id: cell.as_str().to_string(), + runtime_tool_call_id: "nested".to_string(), + }; + assert!(!recorder.record_tool_result_metadata( + &source, + "nested", + &json!({"openai/resource_access": {"resources": ["result"]}}), + )); + let mut retry = [exec_input("exec"), exec_output("exec")]; + recorder.attach_to_prompt(&mut retry, &mut HashMap::new()); + assert!(!retry[1].has_tool_result_metadata()); + assert_eq!(tool_calls_complete(&retry[1]), None); +} + +#[test] +fn late_truncated_metadata_preserves_large_result_on_retry() { let recorder = new_recorder(InitialHistory::New); let cell = CellId::new("late-truncated-cell".to_string()); recorder.start_cell(&cell, "exec"); @@ -1744,7 +1600,7 @@ fn late_truncated_metadata_obeys_the_existing_request_budget() { let actual = serde_json::to_value(&retry[1]).unwrap(); assert_eq!( actual["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], - json!({"openai/resource_access": {"resources": ["result"]}}), + metadata, ); } } @@ -1804,142 +1660,6 @@ fn late_truncated_metadata_is_not_reused_after_budget_changes_calls() { ); } -#[test] -fn untracked_history_budget_loss_keeps_later_wait_incomplete() { - let recorder = new_recorder(InitialHistory::New); - let cell = CellId::new("running-cell".to_string()); - recorder.start_cell(&cell, "exec"); - recorder.record_nested_tool_call( - cell.clone(), - "nested".to_string(), - ExecutedToolCall::new("nested_tool".to_string(), json!("x".repeat(8190))), - /*original_bytes*/ 8192, - ); - let mut prompt = vec![exec_input("exec"), exec_output("exec")]; - let untracked_count = 256; - let per_output_budget = 2 * 1024 * 1024 / untracked_count; - let mut empty_output = output("untracked"); - empty_output.append_executed_tool_calls(vec![ExecutedToolCall::new( - "other_tool".to_string(), - json!(""), - )]); - // Fill the budget with intact unmanaged outputs, leaving no room for the tracked record. - let payload_bytes = per_output_budget - executed_tool_call_metadata_bytes(&empty_output); - for index in 0..untracked_count { - let mut item = output(&format!("untracked-{index}")); - item.append_executed_tool_calls(vec![ExecutedToolCall::new( - "other_tool".to_string(), - json!("x".repeat(payload_bytes)), - )]); - assert_eq!(executed_tool_call_metadata_bytes(&item), per_output_budget); - prompt.push(item); - } - recorder.attach_to_prompt(&mut prompt, &mut HashMap::new()); - assert!(prompt[1].executed_tool_call_metadata().is_none()); - - recorder.attach_to_prompt(&mut [], &mut HashMap::new()); - recorder.register_cell(&cell, "wait"); - recorder.finish_cell_recording(&cell); - let mut wait = [wait_input("wait", &cell), output("wait")]; - recorder.attach_to_prompt(&mut wait, &mut HashMap::new()); - assert_eq!(tool_calls_complete(&wait[1]), None); -} - -#[test] -fn result_metadata_shedding_preserves_completion_after_compaction() { - let recorder = new_recorder(InitialHistory::New); - let cell_id = CellId::new("compacted-cell".to_string()); - recorder.start_cell(&cell_id, "exec"); - - let call_count = 79; - let argument_bytes = MAX_EXECUTED_TOOL_CALL_FULL_ARGUMENT_BYTES_PER_OUTPUT / call_count - 64; - let arguments = json!({ - "payload": "x".repeat(argument_bytes - r#"{"payload":""}"#.len()), - }); - for index in 0..call_count { - recorder.record_nested_tool_call( - cell_id.clone(), - format!("nested-{index}"), - ExecutedToolCall::new("nested_tool".to_string(), arguments.clone()), - argument_bytes, - ); - } - let source = ToolCallSource::CodeMode { - cell_id: cell_id.as_str().to_string(), - runtime_tool_call_id: "runtime-call".to_string(), - }; - let mut retry_cache = HashMap::new(); - recorder.attach_pending_to_prompt( - &mut [exec_input("exec"), exec_output("exec")], - &mut retry_cache, - ); - // Each snapshot fits alone; together they push the calls over the aggregate budget. - let large_metadata = json!({ "provider_data": "x".repeat(26 * 1024) }); - for index in (0..call_count).filter(|index| *index != 1) { - assert!(recorder.record_tool_result_metadata( - &source, - &format!("nested-{index}"), - &large_metadata, - )); - } - - assert!(recorder.record_tool_result_metadata( - &source, - "nested-1", - &json!({ "status": "unavailable" }) - )); - let mut initial = [exec_output("exec")]; - assert!(recorder.attach_pending_to_prompt(&mut initial, &mut retry_cache)); - let calls = initial[0] - .executed_tool_call_metadata() - .and_then(|metadata| metadata.executed_tool_calls.as_ref()) - .expect("metadata shedding must preserve recorded calls"); - let mut expected_calls = - vec![ExecutedToolCall::new("nested_tool".to_string(), arguments); call_count]; - for call in &mut expected_calls { - call.set_tool_result_metadata(ToolResultMetadata::new(&large_metadata)); - } - expected_calls[1] - .set_tool_result_metadata(ToolResultMetadata::new(&json!({ "status": "unavailable" }))); - let mut unbounded = exec_output("exec"); - unbounded.append_executed_tool_calls(expected_calls.clone()); - unbounded.set_tool_call_cell_id("exec"); - let overage = executed_tool_call_metadata_bytes(&unbounded) - 2 * 1024 * 1024; - expected_calls[0].set_tool_result_metadata(ToolResultMetadata::new(&json!(format!( - "omitted_due_to_size_limit (overage_bytes={overage})" - )))); - assert_eq!(calls, &expected_calls); - let mut retry = [exec_output("exec")]; - assert!(recorder.attach_pending_to_prompt(&mut retry, &mut retry_cache)); - assert_eq!(retry, initial); - - // A different call can still attach a small result after the oversized metadata is shed. - let metadata = json!({ "id": "R2" }); - assert!(recorder.record_tool_result_metadata(&source, "nested-2", &metadata)); - let mut expected_calls = calls.clone(); - expected_calls[2].set_tool_result_metadata(ToolResultMetadata::new(&metadata)); - let mut expected = exec_output("exec"); - expected.append_executed_tool_calls(expected_calls); - expected.set_tool_call_cell_id("exec"); - let mut retry = [exec_output("exec")]; - assert!(recorder.attach_pending_to_prompt(&mut retry, &mut retry_cache)); - assert_eq!(retry, [expected]); - - recorder.attach_pending_to_prompt(&mut [], &mut retry_cache); - assert!(retry_cache.is_empty()); - assert!(!recorder.record_tool_result_metadata(&source, "nested-2", &metadata)); - recorder.register_cell(&cell_id, "wait"); - recorder.finish_cell_recording(&cell_id); - let mut final_output = [wait_input("wait", &cell_id), output("wait")]; - recorder.attach_pending_to_prompt(&mut final_output, &mut HashMap::new()); - assert_eq!( - final_output[1] - .executed_tool_call_metadata() - .and_then(|metadata| metadata.tool_calls_complete), - Some(true), - ); -} - #[test] fn mapping_pressure_preserves_existing_cells_and_late_partial_records() { let recorder = new_recorder(InitialHistory::New); diff --git a/codex-rs/core/src/tools/executed_tool_calls_direct_tests.rs b/codex-rs/core/src/tools/executed_tool_calls_direct_tests.rs index 4d0f1ea7f193..23f2033d252d 100644 --- a/codex-rs/core/src/tools/executed_tool_calls_direct_tests.rs +++ b/codex-rs/core/src/tools/executed_tool_calls_direct_tests.rs @@ -1,6 +1,7 @@ //! Tests Direct metadata admission and permit lifetimes. //! Metadata limits must leave ordinary tool outputs unchanged. +use codex_protocol::ResponseItemId; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; use pretty_assertions::assert_eq; @@ -15,6 +16,118 @@ fn output(call_id: &str) -> ResponseItem { }) } +#[test] +fn direct_request_uses_live_window_after_persisted_budget_is_exhausted() { + let mut features = Features::default(); + features.enable(Feature::ExecutedToolCallMetadata); + let recorder = ExecutedToolCalls::new(&features, &InitialHistory::New); + recorder + .retained_direct_metadata_bytes + .store(MAX_RETAINED_DIRECT_METADATA_BYTES, Ordering::Relaxed); + let metadata = json!({"openai/resource_access": {"connector": "example"}}); + let make_output = |id: &str| { + let mut call = ExecutedToolCall::new("connector_tool".to_string(), json!({})); + call.set_tool_result_metadata(ToolResultMetadata::new(&metadata)); + let mut item = output(id); + recorder.attach_direct_call_to_output( + &mut item, + Some((call, recorder.reserve_direct_call().unwrap())), + ); + item + }; + let old = make_output("old-window"); + let mut first = vec![old.clone()]; + recorder.attach_to_prompt(&mut first, &mut HashMap::new()); + let current = make_output("new-window"); + assert!(current.executed_tool_call_metadata().is_none()); + let serialized = serde_json::to_string(&codex_history::RolloutItem::ResponseItem( + current.clone().into(), + )) + .unwrap(); + assert!(!serialized.contains("example")); + + // The input to the next window no longer contains the old output. + let mut next = vec![current.clone()]; + recorder.attach_to_prompt(&mut next, &mut HashMap::new()); + for item in [&first[0], &next[0]] { + let encoded = serde_json::to_value(item).unwrap(); + assert_eq!(encoded["output"], "tool result"); + assert_eq!( + encoded["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0]["tool_result_metadata"], + metadata, + ); + } + let mut pruned = vec![old]; + recorder.attach_to_compaction_prompt(&mut pruned); + assert!(pruned[0].executed_tool_call_metadata().is_none()); + + // Capture toggles must not revive a previous recorder's live-only metadata. + recorder.refresh(&Features::default()); + recorder.refresh(&features); + let mut after_refresh = vec![current]; + recorder.attach_to_prompt(&mut after_refresh, &mut HashMap::new()); + assert!(after_refresh[0].executed_tool_call_metadata().is_none()); +} + +#[test] +fn direct_metadata_follows_output_ids_with_reused_call_ids_and_reverse_completion() { + for supplied_ids in [false, true] { + let mut features = Features::default(); + features.enable(Feature::ExecutedToolCallMetadata); + let recorder = ExecutedToolCalls::new(&features, &InitialHistory::New); + recorder + .retained_direct_metadata_bytes + .store(MAX_RETAINED_DIRECT_METADATA_BYTES, Ordering::Relaxed); + let metadata = [json!({"result": "first"}), json!({"result": "second"})]; + let mut prepared = metadata + .iter() + .map(|metadata| { + let mut call = ExecutedToolCall::new("connector_tool".to_string(), json!({})); + call.set_tool_result_metadata(ToolResultMetadata::new(metadata)); + Some((call, recorder.reserve_direct_call().unwrap())) + }) + .collect::>(); + let mut outputs = [output("reused"), output("reused")]; + let expected_ids = [ + ResponseItemId::with_suffix("fco", "first"), + ResponseItemId::with_suffix("fco", "second"), + ]; + if supplied_ids { + for (item, id) in outputs.iter_mut().zip(&expected_ids) { + item.set_id(Some(id.clone())); + } + } + // Completion order and prompt order must not choose which observation belongs here. + for index in [1, 0] { + recorder.attach_direct_call_to_output(&mut outputs[index], prepared[index].take()); + assert!(outputs[index].executed_tool_call_metadata().is_none()); + if supplied_ids { + assert_eq!(outputs[index].id(), Some(&expected_ids[index])); + } + } + let ids = outputs.clone().map(|item| item.id().unwrap().clone()); + assert_ne!(ids[0], ids[1]); + let mut request = outputs; + for reversed in [false, true] { + if reversed { + request.reverse(); + } + recorder.attach_to_prompt(&mut request, &mut HashMap::new()); + for item in &request { + let index = usize::from(item.id() == Some(&ids[1])); + let encoded = serde_json::to_value(item).unwrap(); + assert_eq!(encoded["call_id"], "reused"); + assert_eq!(encoded["output"], "tool result"); + assert_eq!( + encoded["internal_chat_message_metadata_passthrough"]["executed_tool_calls"][0] + ["tool_result_metadata"], + metadata[index], + ); + } + } + } +} + #[test] fn direct_budget_keeps_an_omission_marker_when_it_fits() { for metadata in [ @@ -50,6 +163,7 @@ fn direct_budget_keeps_an_omission_marker_when_it_fits() { Some((call, recorder.reserve_direct_call().unwrap())), ); + expected.set_id(item.id().cloned()); assert_eq!(item, expected); assert_eq!( recorder diff --git a/codex-rs/core/src/tools/handlers/unified_exec.rs b/codex-rs/core/src/tools/handlers/unified_exec.rs index 9f9f8a8cf1ea..763fa155766d 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec.rs @@ -1,5 +1,6 @@ use crate::sandboxing::SandboxPermissions; use crate::shell::Shell; +use crate::shell::ShellInvocation; use crate::shell::ShellType; use crate::shell::get_shell_by_model_provided_path; use crate::tools::context::ToolInvocation; @@ -74,7 +75,7 @@ fn default_tty() -> bool { #[derive(Debug)] pub(crate) struct ResolvedCommand { pub(crate) command: Vec, - pub(crate) shell_type: ShellType, + pub(crate) shell: ShellInvocation, } fn post_unified_exec_tool_use_payload( @@ -112,35 +113,31 @@ pub(crate) fn get_command( None => allow_login_shell, }; - match shell_mode { - UnifiedExecShellMode::Direct => { - let model_shell = args - .shell - .as_ref() - .map(|shell_str| get_shell_by_model_provided_path(&PathBuf::from(shell_str))); - let shell = model_shell.as_ref().unwrap_or(session_shell.as_ref()); - Ok(ResolvedCommand { - command: shell.derive_exec_args(&args.cmd, use_login_shell), - shell_type: shell.shell_type, - }) - } + let shell = match shell_mode { + UnifiedExecShellMode::Direct => args + .shell + .as_ref() + .map(|shell_str| get_shell_by_model_provided_path(&PathBuf::from(shell_str))) + .unwrap_or_else(|| session_shell.as_ref().clone()), UnifiedExecShellMode::ZshFork(zsh_fork_config) => { if args.shell.is_some() { return Err( "`shell` is not supported for local zsh-fork exec; omit `shell` to use zsh-fork, or target a remote environment where `shell` is supported.".to_string(), ); } - - Ok(ResolvedCommand { - command: vec![ - zsh_fork_config.shell_zsh_path.to_string_lossy().to_string(), - if use_login_shell { "-lc" } else { "-c" }.to_string(), - args.cmd.clone(), - ], + Shell { shell_type: ShellType::Zsh, - }) + shell_path: zsh_fork_config.shell_zsh_path.as_path().to_path_buf(), + } } - } + }; + Ok(ResolvedCommand { + command: shell.derive_exec_args(&args.cmd, use_login_shell), + shell: ShellInvocation { + shell, + use_login_shell, + }, + }) } pub(crate) fn shell_mode_for_environment( diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs index bdf6877389ff..ba5dd79efba7 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs @@ -297,7 +297,6 @@ impl ExecCommandHandler { ) .map_err(FunctionCallError::RespondToModel)?; let command = resolved_command.command; - let shell_type = resolved_command.shell_type; let ExecCommandArgs { mut tty, yield_time_ms, @@ -418,7 +417,7 @@ impl ExecCommandHandler { let process_id = manager.allocate_process_id().await; let request = ExecCommandRequest { command, - shell_type, + shell: resolved_command.shell, hook_command: hook_command.clone(), process_id, yield_time_ms, diff --git a/codex-rs/core/src/tools/handlers/unified_exec_tests.rs b/codex-rs/core/src/tools/handlers/unified_exec_tests.rs index 46fa63dbbb5a..4657b64fbcb1 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec_tests.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec_tests.rs @@ -130,7 +130,7 @@ fn test_get_command_resolves_powershell_by_type() -> anyhow::Result<()> { resolved.command, expected_shell.derive_exec_args("echo hello", /*use_login_shell*/ true) ); - assert_eq!(resolved.shell_type, expected_shell.shell_type); + assert_eq!(resolved.shell.shell, expected_shell); Ok(()) } diff --git a/codex-rs/core/src/tools/metadata_metrics.rs b/codex-rs/core/src/tools/metadata_metrics.rs index f4fee32b1fca..caae89dacfc9 100644 --- a/codex-rs/core/src/tools/metadata_metrics.rs +++ b/codex-rs/core/src/tools/metadata_metrics.rs @@ -11,20 +11,6 @@ const BYTE_BUCKETS: &[f64] = &[ 4194304.0, 8388608.0, 16777216.0, ]; -pub(super) fn bound_prompt_metadata( - items: &mut [ResponseItem], - bound: fn(&mut [ResponseItem]), - stage: &'static str, - metrics: Option<&MetricsClient>, -) { - let before = metrics.map(|_| metadata_bytes(items)); - bound(items); - if let Some(before) = before { - // Measure only the budget operation: attaching observations can itself add bytes. - record_shedding(stage, before, metadata_bytes(items), metrics); - } -} - pub(crate) fn metadata_bytes(items: &[ResponseItem]) -> usize { items.iter().fold(0_usize, |bytes, item| { bytes.saturating_add(executed_tool_call_metadata_bytes(item)) diff --git a/codex-rs/core/src/tools/metadata_metrics_tests.rs b/codex-rs/core/src/tools/metadata_metrics_tests.rs index 253f2d16fb9c..5d9a1b6cc968 100644 --- a/codex-rs/core/src/tools/metadata_metrics_tests.rs +++ b/codex-rs/core/src/tools/metadata_metrics_tests.rs @@ -1,17 +1,10 @@ use super::*; use codex_otel::MetricsConfig; -use codex_protocol::models::ExecutedToolCall; -use codex_protocol::models::FunctionCallOutputPayload; -use codex_protocol::models::ResponseInputItem; -use codex_protocol::models::ToolResultMetadata; -use codex_protocol::models::bound_executed_tool_calls_for_prompt; -use codex_protocol::models::bound_executed_tool_calls_for_prompt_prioritizing_recent; use opentelemetry_sdk::metrics::InMemoryMetricExporter; use opentelemetry_sdk::metrics::data::AggregatedMetrics; use opentelemetry_sdk::metrics::data::MetricData; use opentelemetry_sdk::metrics::data::ScopeMetrics; use pretty_assertions::assert_eq; -use serde_json::json; use std::collections::BTreeMap; fn metrics() -> MetricsClient { @@ -48,57 +41,6 @@ fn samples(metrics: &MetricsClient) -> BTreeMap { samples } -#[test] -fn prompt_reports_each_budget_application_without_changing_output() { - let metrics = metrics(); - let original = (0..96) - .map(|index| { - let mut item = ResponseItem::from(ResponseInputItem::FunctionCallOutput { - call_id: format!("call_{index}"), - output: FunctionCallOutputPayload::from_text("unchanged output".to_string()), - }); - let mut call = ExecutedToolCall::new("mcp__apps__read".to_string(), json!({})); - call.set_tool_result_metadata(ToolResultMetadata::new(&json!({ - "openai/resource_access": {"resources": ["x".repeat(24_000)]}, - }))); - item.append_executed_tool_calls(vec![call]); - item.mark_tool_calls_complete(); - item - }) - .collect::>(); - let mut expected_samples = BTreeMap::new(); - for (stage, bound) in [ - ( - "request", - bound_executed_tool_calls_for_prompt as fn(&mut [ResponseItem]), - ), - ( - "retained", - bound_executed_tool_calls_for_prompt_prioritizing_recent, - ), - ( - "compaction", - bound_executed_tool_calls_for_prompt_prioritizing_recent, - ), - ] { - let mut expected = original.clone(); - bound(&mut expected); - let shed = metadata_bytes(&original) - metadata_bytes(&expected); - assert!(shed > 0); - let mut actual = original.clone(); - bound_prompt_metadata(&mut actual, bound, stage, Some(&metrics)); - assert_eq!(actual, expected); - // Reapplying an already-satisfied budget must not add a zero-byte sample. - bound_prompt_metadata(&mut actual, bound, stage, Some(&metrics)); - assert_eq!(actual, expected); - let mut without_metrics = original.clone(); - bound_prompt_metadata(&mut without_metrics, bound, stage, /*metrics*/ None); - assert_eq!(without_metrics, expected); - expected_samples.insert(stage.to_string(), (1, shed as f64)); - } - assert_eq!(samples(&metrics), expected_samples); -} - #[test] fn direct_retention_reports_only_positive_known_shedding() { let metrics = metrics(); diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 3473cd50d6a7..2d94f27d8662 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -29,6 +29,9 @@ use codex_otel::ToolDecisionSource; use codex_protocol::error::CodexErrorDetails; use codex_protocol::error::SandboxErr; use codex_protocol::exec_output::ExecToolCallOutput; +use codex_protocol::models::PermissionProfile; +use codex_protocol::permissions::PROTECTED_METADATA_PATH_NAMES; +use codex_protocol::permissions::file_system_root; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::ReviewDecision; use codex_sandboxing::SandboxManager; @@ -222,15 +225,71 @@ impl ToolOrchestrator { // 2) First attempt under the selected sandbox. let unsandboxed_allowed = unsandboxed_execution_allowed(&file_system_sandbox_policy); - let sandbox_override = if unsandboxed_allowed { - sandbox_override_for_first_attempt( - tool.sandbox_permissions(req), - &requirement, - &file_system_sandbox_policy, - ) + let sandbox_override = sandbox_override_for_first_attempt( + tool.sandbox_permissions(req), + &requirement, + &file_system_sandbox_policy, + already_approved, + ); + let (escalated_profile, writable_root_metadata) = if sandbox_override + == SandboxOverride::EscalatedSandboxWithRestrictions + { + let context = environment.sandbox_context(/*additional_permissions*/ None); + let policy_context = context.policy_context(); + let baseline = permission_profile.file_system_sandbox_policy(); + let file_system = baseline.for_approved_command(&policy_context); + let writable_root_metadata = file_system_root(&policy_context).is_some_and(|root| { + file_system.can_write_path(&root, &policy_context) + && PROTECTED_METADATA_PATH_NAMES.iter().any(|name| { + root.join_descendant(name) + .is_ok_and(|path| file_system.can_write_path(&path, &policy_context)) + }) + }); + let escalated_profile = (file_system != baseline + && file_system + != baseline.materialize_project_roots_with_path_uris(workspace_roots)) + .then(|| { + PermissionProfile::from_runtime_permissions_with_enforcement( + permission_profile.enforcement(), + &file_system, + permission_profile.network_sandbox_policy(), + ) + }); + (escalated_profile, writable_root_metadata) } else { - SandboxOverride::NoOverride + (None, false) }; + if (escalated_profile.is_some() || writable_root_metadata) + && environment.environment.is_remote() + { + let info = environment.environment.info().await.map_err(|err| { + ToolError::Rejected(format!( + "could not verify exec-server support for escalation with denied reads: {err}" + )) + })?; + if !matches!(info.platform_os.as_deref(), Some("windows" | "macos")) + && !info + .capabilities + .linux_approved_root_write_preserves_restrictions + { + return Err(ToolError::Rejected( + "this exec-server cannot safely escalate with denied reads; upgrade the exec-server and retry" + .to_string(), + )); + } + } + let escalated_local_profile = escalated_profile + .as_ref() + .filter(|_| !executor_managed_process_sandbox) + .map(|profile| { + profile + .clone() + .materialize_project_roots_with_path_uris(workspace_roots) + }); + let initial_permissions = escalated_local_profile + .as_ref() + .or(escalated_profile.as_ref()) + .unwrap_or(&permissions); let network_approval_spec = tool.network_approval_spec(req, tool_ctx); // An explicit owner proxy requirement permits filtered egress, like an enabled // controller proxy. Traffic-only owner policies stay offline unless command @@ -265,11 +324,13 @@ impl ToolOrchestrator { let sandbox_preference = tool.sandbox_preference(); let sandbox_requested = match sandbox_override { SandboxOverride::BypassSandboxFirstAttempt => false, - SandboxOverride::NoOverride => sandbox_manager.should_sandbox( - &permissions, - sandbox_preference, - managed_network_active, - ), + SandboxOverride::NoOverride | SandboxOverride::EscalatedSandboxWithRestrictions => { + sandbox_manager.should_sandbox( + initial_permissions, + sandbox_preference, + managed_network_active, + ) + } }; let windows_sandbox_type = codex_protocol::sandbox::effective_windows_sandbox_type( sandbox_config.windows_sandbox_type, @@ -277,7 +338,7 @@ impl ToolOrchestrator { ); let initial_sandbox = if sandbox_requested && !executor_managed_process_sandbox { sandbox_manager.select_initial( - &permissions, + initial_permissions, sandbox_preference, windows_sandbox_type, managed_network_active, @@ -285,6 +346,9 @@ impl ToolOrchestrator { } else { SandboxType::None }; + if !executor_managed_process_sandbox { + sandbox_override.ensure_native_sandbox(initial_sandbox)?; + } let sandbox_policy_cwd = tool .sandbox_cwd(req) @@ -298,8 +362,8 @@ impl ToolOrchestrator { let initial_attempt = SandboxAttempt { sandbox: initial_sandbox, sandbox_requested, - permissions: &permissions, - exec_server_permissions: permission_profile, + permissions: initial_permissions, + exec_server_permissions: escalated_profile.as_ref().unwrap_or(permission_profile), enforce_managed_network: managed_network_active, manager: &sandbox_manager, sandbox_cwd: &sandbox_policy_cwd, @@ -449,14 +513,14 @@ impl ToolOrchestrator { let retry_sandbox_requested = !unsandboxed_allowed && sandbox_manager.should_sandbox( - &permissions, + initial_permissions, sandbox_preference, managed_network_active, ); let retry_sandbox = if retry_sandbox_requested && !executor_managed_process_sandbox { sandbox_manager.select_initial( - &permissions, + initial_permissions, sandbox_preference, windows_sandbox_type, managed_network_active, @@ -472,8 +536,10 @@ impl ToolOrchestrator { let retry_attempt = SandboxAttempt { sandbox: retry_sandbox, sandbox_requested: retry_sandbox_requested, - permissions: &permissions, - exec_server_permissions: permission_profile, + permissions: initial_permissions, + exec_server_permissions: escalated_profile + .as_ref() + .unwrap_or(permission_profile), enforce_managed_network: managed_network_active, manager: &sandbox_manager, sandbox_cwd: &sandbox_policy_cwd, diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index 8e7c256b566e..c6ef29c606e7 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -14,6 +14,7 @@ use crate::sandboxing::ExecOptions; use crate::sandboxing::ExecServerEnvConfig; use crate::sandboxing::SandboxPermissions; use crate::session::turn_context::TurnEnvironment; +use crate::shell::ShellInvocation; use crate::shell::ShellType; use crate::shell_snapshot::ShellSnapshotSandbox; use crate::shell_snapshot::snapshot_read_permissions; @@ -58,7 +59,6 @@ use codex_tools::UnifiedExecShellMode; use codex_utils_path_uri::PathUri; use std::collections::HashMap; use std::io; -use std::path::PathBuf; use std::time::Duration; use tokio_util::sync::CancellationToken; @@ -74,7 +74,7 @@ const REMOTE_NETWORK_POLICY_DECISION_MARGIN: Duration = Duration::from_secs(10); #[derive(Clone, Debug)] pub struct UnifiedExecRequest { pub command: Vec, - pub shell_type: ShellType, + pub(crate) shell: ShellInvocation, pub hook_command: String, pub process_id: i32, pub cwd: PathUri, @@ -271,6 +271,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt ctx: &ToolCtx, ) -> Result { let base_command = &req.command; + let requested_shell = &req.shell.shell; let windows_sandbox_proxy_settings_mode = ctx.session.windows_sandbox_proxy_settings_mode; let session_shell = ctx.session.user_shell(); let environment_shell = req @@ -306,19 +307,11 @@ impl<'a> ToolRuntime for UnifiedExecRunt "credential brokerage does not yet support shell_zsh_fork; disable shell_zsh_fork to use the broker".to_string(), )); } - let requested_shell = (credential_broker_available - && (req.shell_type != environment_shell.shell_type - || base_command.first().is_some_and(|path| { - path != environment_shell.shell_path.to_string_lossy().as_ref() - }))) - .then(|| { - base_command.first().map(|path| crate::shell::Shell { - shell_type: req.shell_type, - shell_path: PathBuf::from(path), - }) - }) - .flatten(); - let shell = requested_shell.as_ref().unwrap_or(environment_shell); + let shell = if credential_broker_available { + requested_shell + } else { + environment_shell + }; let shell_snapshot = if environment_is_remote || credential_broker_available && launch_sandbox_permissions.requires_escalated_permissions() @@ -348,7 +341,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt let shell_snapshot_location = shell_snapshot.as_ref().map(|snapshot| snapshot.path()); let mut env = exec_env_for_sandbox_permissions(&req.env, launch_sandbox_permissions); let snapshot_credential_context = if let Some(snapshot) = shell_snapshot.as_ref() - && (managed_network.is_some() || base_command.get(1).is_some_and(|flag| flag == "-lc")) + && (managed_network.is_some() || req.shell.is_posix_login()) { Some( snapshot @@ -576,7 +569,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt // preparation is a no-op for PowerShell on other platforms. let command = prepare_powershell_command_for_windows_sandbox( &command, - Some(&req.shell_type), + Some(&requested_shell.shell_type), attempt.sandbox_requested, executor_windows_sandbox_selection( attempt.windows_sandbox_type, @@ -585,7 +578,7 @@ impl<'a> ToolRuntime for UnifiedExecRunt ), environment_is_remote, ); - let command = if matches!(req.shell_type, ShellType::PowerShell) { + let command = if matches!(requested_shell.shell_type, ShellType::PowerShell) { prefix_powershell_script_with_utf8(&command) } else { command @@ -629,6 +622,17 @@ impl<'a> ToolRuntime for UnifiedExecRunt req.additional_permissions.as_ref(), internal_permissions.as_ref(), ); + let baseline_file_system = req + .turn_environment + .permission_profile() + .file_system_sandbox_policy(); + let permissions = if baseline_file_system.has_denied_read_restrictions() + && attempt.exec_server_permissions.file_system_sandbox_policy() != baseline_file_system + { + permissions.with_filesystem_escalation() + } else { + permissions + }; if let UnifiedExecShellMode::ZshFork(zsh_fork_config) = &self.shell_mode { let command = build_unified_exec_sandbox_command( @@ -750,6 +754,7 @@ mod tests { use crate::config::PermissionProfileSnapshot; use crate::environment_selection::EnvironmentConfigOrigin; use crate::exec::DEFAULT_EXEC_COMMAND_TIMEOUT_MS; + use crate::shell::Shell; use crate::tools::sandboxing::ToolRuntime; use codex_exec_server::Environment; use codex_exec_server::LOCAL_ENVIRONMENT_ID; @@ -853,7 +858,13 @@ mod tests { let runtime = UnifiedExecRuntime::new(&manager, UnifiedExecShellMode::Direct); let request = UnifiedExecRequest { command: vec!["pwd".to_string()], - shell_type: ShellType::Sh, + shell: ShellInvocation { + shell: Shell { + shell_type: ShellType::Sh, + shell_path: "sh".into(), + }, + use_login_shell: false, + }, hook_command: "pwd".to_string(), process_id: 1000, cwd: cwd.into(), @@ -956,7 +967,13 @@ mod tests { .expect("current dir is absolute"); UnifiedExecRequest { command: vec!["zsh".to_string(), "-c".to_string(), "echo hi".to_string()], - shell_type: ShellType::Zsh, + shell: ShellInvocation { + shell: Shell { + shell_type: ShellType::Zsh, + shell_path: "zsh".into(), + }, + use_login_shell: false, + }, hook_command: "echo hi".to_string(), process_id: 1000, cwd: cwd.clone().into(), diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index 85185947d940..bf1fd2d79e6a 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -233,19 +233,36 @@ pub(crate) fn default_exec_approval_requirement( #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SandboxOverride { NoOverride, + EscalatedSandboxWithRestrictions, BypassSandboxFirstAttempt, } +impl SandboxOverride { + pub(crate) fn ensure_native_sandbox(self, sandbox: SandboxType) -> Result<(), ToolError> { + if self == Self::EscalatedSandboxWithRestrictions && sandbox == SandboxType::None { + return Err(ToolError::Rejected( + "command escalation with denied reads requires an available filesystem sandbox" + .to_string(), + )); + } + Ok(()) + } +} + pub(crate) fn sandbox_override_for_first_attempt( sandbox_permissions: SandboxPermissions, exec_approval_requirement: &ExecApprovalRequirement, file_system_sandbox_policy: &FileSystemSandboxPolicy, + already_approved: bool, ) -> SandboxOverride { - // Deny-read restrictions are part of the active permission policy. Running - // without a filesystem sandbox would discard them, even if the command was - // otherwise approved by rules or explicit escalation. + // Only actual approval of an explicit escalation may widen the filesystem; + // a command allow rule does not authorize removing filesystem restrictions. if !unsandboxed_execution_allowed(file_system_sandbox_policy) { - return SandboxOverride::NoOverride; + return if sandbox_permissions.requires_escalated_permissions() && already_approved { + SandboxOverride::EscalatedSandboxWithRestrictions + } else { + SandboxOverride::NoOverride + }; } // ExecPolicy `Allow` can intentionally imply full trust (Skip + bypass_sandbox=true), diff --git a/codex-rs/core/src/tools/sandboxing_tests.rs b/codex-rs/core/src/tools/sandboxing_tests.rs index 0142440f6c95..95731415baf4 100644 --- a/codex-rs/core/src/tools/sandboxing_tests.rs +++ b/codex-rs/core/src/tools/sandboxing_tests.rs @@ -163,6 +163,7 @@ fn additional_permissions_allow_bypass_sandbox_first_attempt_when_execpolicy_ski proposed_execpolicy_amendment: None, }, &FileSystemSandboxPolicy::default(), + /*already_approved*/ false, ), SandboxOverride::BypassSandboxFirstAttempt ); @@ -178,13 +179,14 @@ fn guardian_bypasses_sandbox_for_explicit_escalation_on_first_attempt() { proposed_execpolicy_amendment: None, }, &FileSystemSandboxPolicy::default(), + /*already_approved*/ true, ), SandboxOverride::BypassSandboxFirstAttempt ); } #[test] -fn deny_read_blocks_explicit_escalation_and_policy_bypass() { +fn deny_read_preserves_the_sandbox_for_explicit_escalation_and_blocks_policy_bypass() { let file_system_policy = FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { path: FileSystemPath::GlobPattern { pattern: "**/*.env".to_string(), @@ -201,11 +203,16 @@ fn deny_read_blocks_explicit_escalation_and_policy_bypass() { proposed_execpolicy_amendment: None, }, &file_system_policy, + /*already_approved*/ true, ), - SandboxOverride::NoOverride, - "explicit escalation would drop deny-read filesystem policy, so keep the first attempt sandboxed", + SandboxOverride::EscalatedSandboxWithRestrictions, + "explicit escalation must widen the first attempt without dropping the sandbox", ); assert!(!unsandboxed_execution_allowed(&file_system_policy)); + assert!(matches!( + SandboxOverride::EscalatedSandboxWithRestrictions.ensure_native_sandbox(SandboxType::None), + Err(ToolError::Rejected(reason)) if reason.contains("requires an available filesystem sandbox"), + )); assert_eq!( sandbox_permissions_preserving_denied_reads( SandboxPermissions::RequireEscalated, @@ -235,6 +242,7 @@ fn deny_read_blocks_explicit_escalation_and_policy_bypass() { proposed_execpolicy_amendment: None, }, &file_system_policy, + /*already_approved*/ true, ), SandboxOverride::NoOverride, "exec-policy allow rules would drop deny-read filesystem policy, so keep the first attempt sandboxed", diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 3645306c27c8..28af9b02290f 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -43,7 +43,7 @@ use crate::session::session::Session; use crate::session::step_context::StepContext; use crate::session::turn_context::TurnContext; use crate::session::turn_context::TurnEnvironment; -use crate::shell::ShellType; +use crate::shell::ShellInvocation; use crate::tools::network_approval::DeferredNetworkApproval; use codex_core_plugins::PluginMetricsSidecar; @@ -113,7 +113,7 @@ impl UnifiedExecContext { #[derive(Debug)] pub(crate) struct ExecCommandRequest { pub command: Vec, - pub shell_type: ShellType, + pub shell: ShellInvocation, pub hook_command: String, pub process_id: i32, pub yield_time_ms: u64, diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 3dc3e97d9124..565194441f2a 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -1505,7 +1505,7 @@ impl UnifiedExecProcessManager { .await; let req = UnifiedExecToolRequest { command: request.command.clone(), - shell_type: request.shell_type, + shell: request.shell.clone(), hook_command: request.hook_command.clone(), process_id: request.process_id, cwd, diff --git a/codex-rs/core/src/unified_exec/process_manager_tests.rs b/codex-rs/core/src/unified_exec/process_manager_tests.rs index 7edc743ce422..ce56681e68b4 100644 --- a/codex-rs/core/src/unified_exec/process_manager_tests.rs +++ b/codex-rs/core/src/unified_exec/process_manager_tests.rs @@ -1,4 +1,7 @@ use super::*; +use crate::shell::Shell; +use crate::shell::ShellInvocation; +use crate::shell::ShellType; use crate::unified_exec::clamp_yield_time; use codex_network_proxy::ManagedNetworkSandboxContext; use core_test_support::assert_regex_match; @@ -459,7 +462,13 @@ async fn failed_initial_end_for_unstored_process_uses_fallback_output() { "-lc".to_string(), "echo before".to_string(), ], - shell_type: crate::shell::ShellType::Sh, + shell: ShellInvocation { + shell: Shell { + shell_type: ShellType::Sh, + shell_path: "sh".into(), + }, + use_login_shell: true, + }, hook_command: "echo before".to_string(), process_id: 123, yield_time_ms: 1000, diff --git a/codex-rs/core/src/unified_exec/shell_snapshot.rs b/codex-rs/core/src/unified_exec/shell_snapshot.rs index dc79ea461947..8deffbaba9a2 100644 --- a/codex-rs/core/src/unified_exec/shell_snapshot.rs +++ b/codex-rs/core/src/unified_exec/shell_snapshot.rs @@ -179,15 +179,12 @@ pub(super) fn shell_snapshot_request( || !request.turn_environment.shell_snapshot_v2_supported || request.turn_environment.selection.cwd != *cwd || !matches!(request.shell_mode, UnifiedExecShellMode::Direct) - || !matches!( - request.shell_type, - ShellType::Bash | ShellType::Zsh | ShellType::Sh - ) - || request.command.get(1).is_none_or(|flag| flag != "-lc") + || !request.shell.is_posix_login() { return None; } + let shell = &request.shell.shell; Some(ShellSnapshotRequest { scope_id: format!( "{}:{}", @@ -195,8 +192,8 @@ pub(super) fn shell_snapshot_request( request.turn_environment.selection.environment_id ), shell: ShellInfo { - name: request.shell_type.name().to_string(), - path: request.command.first()?.clone(), + name: shell.name().to_string(), + path: shell.shell_path.to_string_lossy().into_owned(), }, }) } diff --git a/codex-rs/core/src/unified_exec/stdin_approval.rs b/codex-rs/core/src/unified_exec/stdin_approval.rs index cef5a1331c85..646ea8ec9be9 100644 --- a/codex-rs/core/src/unified_exec/stdin_approval.rs +++ b/codex-rs/core/src/unified_exec/stdin_approval.rs @@ -33,6 +33,7 @@ pub(crate) struct TerminalPermissions { policy: TerminalPolicy, sandbox_source: TerminalSandboxSource, launch_permissions: SandboxPermissions, + filesystem_escalated: bool, additional_permissions: Option, internal_permissions: Option, } @@ -100,14 +101,19 @@ impl TerminalPermissions { merge_permission_profiles(additional_permissions, internal_permissions), ), sandbox_source, - // A bypass is a property of the successful attempt, not a difference - // between settings: a full-access environment can still bypass a proxy. + // Retain the successful attempt's authority independently from its baseline. launch_permissions, + filesystem_escalated: false, additional_permissions: additional_permissions.cloned(), internal_permissions: internal_permissions.cloned(), } } + pub(crate) fn with_filesystem_escalation(mut self) -> Self { + self.filesystem_escalated = true; + self + } + /// Compares launch policy with current policy including retained grants, /// without reading turn state or touching a process. fn review_requirement( @@ -129,15 +135,17 @@ impl TerminalPermissions { } // Runtime-internal grants are part of an ordinary launch, so only permissions // beyond the baseline plus those grants need a fresh stdin approval. - Ok(if bypassed || &self.policy != current { - SandboxPermissions::RequireEscalated - } else if self.policy.sandbox.permissions - == effective_permission_profile(baseline, self.internal_permissions.as_ref()) - { - SandboxPermissions::UseDefault - } else { - SandboxPermissions::WithAdditionalPermissions - }) + Ok( + if bypassed || self.filesystem_escalated || &self.policy != current { + SandboxPermissions::RequireEscalated + } else if self.policy.sandbox.permissions + == effective_permission_profile(baseline, self.internal_permissions.as_ref()) + { + SandboxPermissions::UseDefault + } else { + SandboxPermissions::WithAdditionalPermissions + }, + ) } fn approval_reason( @@ -146,6 +154,8 @@ impl TerminalPermissions { ) -> Result { let authority = if self.launch_permissions.requires_escalated_permissions() { "This terminal was launched outside the sandbox, bypassing any managed network proxy." + } else if self.filesystem_escalated { + "This terminal has approved broader filesystem access; denied reads are still enforced." } else if self.policy.sandbox.permissions == PermissionProfile::Disabled { "This terminal runs without a filesystem sandbox." } else { diff --git a/codex-rs/core/src/unified_exec/stdin_approval_tests.rs b/codex-rs/core/src/unified_exec/stdin_approval_tests.rs index 69a701c5a1ee..170edefa49a2 100644 --- a/codex-rs/core/src/unified_exec/stdin_approval_tests.rs +++ b/codex-rs/core/src/unified_exec/stdin_approval_tests.rs @@ -29,6 +29,7 @@ fn terminal_permissions(profile: &PermissionProfile) -> TerminalPermissions { }, sandbox_source: TerminalSandboxSource::Native, launch_permissions: SandboxPermissions::UseDefault, + filesystem_escalated: false, additional_permissions: None, internal_permissions: None, } diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index fd229e37d733..8d3668f7dabf 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -3802,6 +3802,102 @@ touch {outside_path:?} Ok(()) } +#[cfg(unix)] +#[test_case(Some(ReviewDecision::Approved); "approved")] +#[test_case(Some(ReviewDecision::denied("rejected by user")); "denied")] +#[test_case(None; "allow_rule_skips_approval_without_widening")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn explicit_escalation_with_denied_reads(decision: Option) -> Result<()> { + skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + if decision.is_none() { + let rules = home.path().join("rules"); + fs::create_dir(&rules)?; + fs::write( + rules.join("default.rules"), + r#"prefix_rule(pattern=["/bin/sh", "escalation.sh"], decision="allow")"#, + )?; + } + let outside_marker = home.path().join("escalated-marker"); + let quoted_marker = shlex::try_join([outside_marker.to_string_lossy().as_ref()])?; + let approval_policy = AskForApproval::OnRequest; + let mut builder = test_codex() + .with_home(home) + .with_config(move |config| { + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config.features.enable(Feature::UnifiedExec).unwrap(); + config.features.disable(Feature::ShellZshFork).unwrap(); + let mut file_system = + restrictive_workspace_write_profile().file_system_sandbox_policy(); + file_system.entries.push(FileSystemSandboxEntry::new( + config.cwd.join("secret.txt").into(), + FileSystemAccessMode::Deny, + )); + config + .permissions + .set_permission_profile(PermissionProfile::from_runtime_permissions( + &file_system, + NetworkSandboxPolicy::Restricted, + )) + .unwrap(); + }) + .with_workspace_setup(move |cwd, _fs| async move { + fs::write(cwd.join("secret.txt"), "secret")?; + fs::write( + cwd.join("escalation.sh"), + format!( + "printf ran > ran || exit 10; (printf outside > {quoted_marker}) 2> errors; \ + if cat secret.txt > errors 2>&1; then exit 11; fi; \ + grep -q secret.txt errors || exit 12; printf 'protected\\n'" + ), + )?; + Ok(()) + }); + let test = builder.build(&server).await?; + let workspace_marker = test.cwd.path().join("ran"); + let command = "/bin/sh escalation.sh"; + let call_id = "explicit-deny-read-escalation"; + let event = exec_command_event( + call_id, + command, + Some(10_000), + SandboxPermissions::RequireEscalated, + /*justification*/ None, + )?; + mount_sse_once(&server, sse(vec![event, ev_completed("run")])).await; + let response = mount_sse_once(&server, sse(vec![ev_completed("done")])).await; + submit_turn_preserving_active_permission_profile(&test, "run the command", approval_policy) + .await?; + let approved = decision == Some(ReviewDecision::Approved); + let rejected = decision.is_some() && !approved; + if let Some(decision) = decision { + let request = expect_exec_approval(&test, command).await; + assert!(!outside_marker.exists() && !workspace_marker.exists()); + test.codex + .submit(Op::ExecApproval { + id: request.effective_approval_id(), + turn_id: None, + decision, + }) + .await?; + } + wait_for_completion_without_approval(&test).await; + let result = parse_result(&response.single_request().function_call_output(call_id)); + if rejected { + assert!(result.stdout.contains("rejected by user"), "{result:?}"); + assert!(!workspace_marker.exists()); + } else { + assert_eq!( + (result.exit_code, result.stdout.as_str()), + (Some(0), "protected\n") + ); + assert_eq!(fs::read_to_string(workspace_marker)?, "ran"); + } + assert_eq!(outside_marker.exists(), approved); + Ok(()) +} + /// Verifies that zsh-fork applies an inner script's allow rule even when the /// model invokes an outer wrapper, and that the escalated script retains the /// named profile needed to reconstruct the original sandbox remotely without diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 4094ed06f3f3..eb0802902a20 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2330,11 +2330,11 @@ async fn result_metadata_preserves_results_within_request_budget( "payload": "l".repeat(31 * 1024), "openai/resource_access": resource_access, }), - serde_json::json!({ "payload": "m".repeat(20 * 1024) }), - serde_json::json!({ "payload": "m".repeat(20 * 1024) }), - serde_json::json!({ "payload": "m".repeat(20 * 1024) }), - serde_json::json!({ "payload": "m".repeat(20 * 1024) }), - serde_json::json!({ "payload": "m".repeat(20 * 1024) }), + serde_json::json!({ "payload": "m".repeat(450 * 1024) }), + serde_json::json!({ "payload": "m".repeat(450 * 1024) }), + serde_json::json!({ "payload": "m".repeat(450 * 1024) }), + serde_json::json!({ "payload": "m".repeat(450 * 1024) }), + serde_json::json!({ "payload": "m".repeat(450 * 1024) }), serde_json::json!({ "payload": "o".repeat(40 * 1024), "openai/resource_access": { @@ -2349,8 +2349,8 @@ async fn result_metadata_preserves_results_within_request_budget( .iter() .map(|metadata| serde_json::to_vec(metadata).unwrap().len()) .collect::>(); - assert!(metadata_sizes.iter().sum::() > 128 * 1024); - assert!(metadata_sizes.iter().sum::() < 1024 * 1024); + assert!(metadata_sizes.iter().sum::() > 2 * 1024 * 1024); + assert!(metadata_sizes.iter().sum::() < 15 * 1024 * 1024); assert!(serde_json::to_vec(&result_metadata[6]["openai/resource_access"])?.len() > 32 * 1024); for (arguments, metadata) in arguments.iter().zip(&result_metadata) { let result = serde_json::json!({ @@ -2488,7 +2488,7 @@ async fn result_metadata_preserves_results_within_request_budget( .iter() .map(codex_protocol::models::executed_tool_call_metadata_bytes) .sum::() - <= 2 * 1024 * 1024 + > 2 * 1024 * 1024 ); let captured = serde_json::to_value(captured)?; for (input, expected_metadata) in [ diff --git a/codex-rs/core/tests/suite/compact_program_tests.rs b/codex-rs/core/tests/suite/compact_program_tests.rs index d65c36660d3d..3a357df4104c 100644 --- a/codex-rs/core/tests/suite/compact_program_tests.rs +++ b/codex-rs/core/tests/suite/compact_program_tests.rs @@ -254,6 +254,30 @@ async fn model_switch_program_pair( Arc::clone(&resumed.codex) } }; + if matches!(history, History::ApiKeyResume) { + // Resuming a ChatGPT turn does not enable the API-key rollout features. + let error = submit_pair(&thread, next_model, Some(CyberAccessProgram::DaybreakRed)) + .await + .expect_err("API-key selections require the rollout features"); + assert!( + error + .to_string() + .contains("Cyber access programs are disabled for this API-key session.") + ); + thread.shutdown_and_wait().await?; + assert_eq!( + server + .received_requests() + .await + .expect("recorded fixture requests") + .iter() + .filter(|request| request.method == "POST" && request.url.path() == "/v1/responses") + .count(), + 1, + "only the original ChatGPT turn should reach inference" + ); + return Ok(()); + } let summary = if local { ev_assistant_message("summary", "summary of surviving history") } else { @@ -277,17 +301,7 @@ async fn model_switch_program_pair( submit_pair(&thread, next_model, Some(CyberAccessProgram::DaybreakRed)).await?; thread.shutdown_and_wait().await?; let requests = requests.requests(); - let authorized = !matches!(history, History::ApiKeyResume); - let old_program = if authorized { - expected_program - } else { - Value::Null - }; - let new_program = if authorized { - json!({"cyber": "daybreak_red"}) - } else { - Value::Null - }; + let new_program = json!({"cyber": "daybreak_red"}); let actual = requests .iter() .map(|request| { @@ -302,7 +316,7 @@ async fn model_switch_program_pair( } following.push(json!([next_model, new_program])); assert_eq!(&actual[1..], following.as_slice()); - assert_eq!(actual[0], json!([previous_model, old_program])); + assert_eq!(actual[0], json!([previous_model, expected_program])); if !local { assert_eq!(requests[0].inputs_of_type("compaction_trigger").len(), 1); } diff --git a/codex-rs/core/tests/suite/cyber_access_program.rs b/codex-rs/core/tests/suite/cyber_access_program.rs index e5021c0f0317..1d7ee9adaecf 100644 --- a/codex-rs/core/tests/suite/cyber_access_program.rs +++ b/codex-rs/core/tests/suite/cyber_access_program.rs @@ -159,10 +159,10 @@ async fn recover_turn_restores_cyber_access_program_without_making_it_sticky() - } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn cyber_access_program_omits_api_key_and_spoofed_custom_provider() -> Result<()> { +async fn cyber_access_program_omits_spoofed_custom_providers() -> Result<()> { core_test_support::skip_if_no_network!(Ok(())); for (auth, provider_id) in [ - (CodexAuth::from_api_key("test-key"), "openai"), + (CodexAuth::from_api_key("test-key"), "custom"), (CodexAuth::create_dummy_chatgpt_auth_for_testing(), "custom"), ] { let server = responses::start_mock_server().await; diff --git a/codex-rs/core/tests/suite/direct_tool_metadata.rs b/codex-rs/core/tests/suite/direct_tool_metadata.rs index 65c03e3b023f..22ab2c4a2597 100644 --- a/codex-rs/core/tests/suite/direct_tool_metadata.rs +++ b/codex-rs/core/tests/suite/direct_tool_metadata.rs @@ -1,6 +1,9 @@ //! Direct-call metadata coverage, including malformed calls and metadata budgets. use anyhow::Result; +use codex_core::ForkSnapshot; +use codex_core::StartThreadOptions; +use codex_core::TurnInputRequest; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_model_provider::RemoteCompactionSupport; @@ -23,6 +26,7 @@ use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::configure_search_capable_model; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; @@ -495,7 +499,6 @@ async fn direct_function_and_tool_search_mark_complete_attempts( ) -> Result<()> { skip_if_no_network!(Ok(())); - let request_budget = 2 * 1024 * 1024; let server = start_mock_server().await; let mut builder = test_codex().with_config(move |config| { configure_search_capable_model(config); @@ -620,25 +623,27 @@ async fn direct_function_and_tool_search_mark_complete_attempts( }) ); } - assert!(metadata_bytes <= request_budget); if budget_calls > 0 { assert!(metadata_bytes > 128 * 1024); } Ok(()) } +#[test_case(false; "ungranted_provider")] +#[test_case(true; "granted_provider")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn direct_metadata_limit_retains_marker_without_sending_it_to_custom_provider() -> Result<()> -{ +async fn direct_metadata_limit_respects_provider_support( + include_internal_metadata: bool, +) -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; let apps = AppsTestServer::mount(&server).await?; - let raw_metadata = json!({"openai/resource_access": {"payload": "x".repeat(600 * 1024)}}); - let raw_bytes = serde_json::to_vec(&raw_metadata)?.len(); + fn metadata_for_query(query: &str) -> Value { + json!({"openai/resource_access": {"payload": "x".repeat(600 * 1024), "query": query}}) + } + let raw_bytes = serde_json::to_vec(&metadata_for_query("first"))?.len(); assert!(raw_bytes < 1024 * 1024); assert!(raw_bytes * 2 > 1024 * 1024); - assert!(raw_bytes * 2 < 2 * 1024 * 1024); - let fixture_metadata = raw_metadata.clone(); Mock::given(method("POST")) .and(path_regex("^/api/codex/ps/mcp/?$")) .and(body_partial_json(json!({"method": "tools/call"}))) @@ -651,7 +656,7 @@ async fn direct_metadata_limit_retains_marker_without_sending_it_to_custom_provi "jsonrpc": "2.0", "id": body["id"], "result": { "content": [{"type": "text", "text": format!("result for {query}")}], - "_meta": fixture_metadata, + "_meta": metadata_for_query(query), "isError": false } })) @@ -664,35 +669,56 @@ async fn direct_metadata_limit_retains_marker_without_sending_it_to_custom_provi .respond_with(ResponseTemplate::new(/*s*/ 200)) .mount(&server) .await; - let responses = mount_sse_sequence( - &server, - vec![ - sse(vec![ - ev_response_created("resp-1"), - ev_function_call_with_namespace( - "first", - SEARCH_CALENDAR_NAMESPACE, - SEARCH_CALENDAR_LIST_TOOL, - r#"{"query":"first"}"#, - ), - ev_completed("resp-1"), - ]), - sse(vec![ - ev_response_created("resp-2"), + let mut events = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call_with_namespace( + "first", + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_LIST_TOOL, + r#"{"query":"first"}"#, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_function_call_with_namespace( + "second", + SEARCH_CALENDAR_NAMESPACE, + SEARCH_CALENDAR_LIST_TOOL, + r#"{"query":"second"}"#, + ), + ev_completed("resp-2"), + ]), + sse(vec![ev_response_created("resp-3"), ev_completed("resp-3")]), + ]; + if include_internal_metadata { + for query in ["fork", "resume"] { + events.push(sse(vec![ + ev_response_created(&format!("resp-{query}")), ev_function_call_with_namespace( - "second", + query, SEARCH_CALENDAR_NAMESPACE, SEARCH_CALENDAR_LIST_TOOL, - r#"{"query":"second"}"#, + &json!({"query": query}).to_string(), ), - ev_completed("resp-2"), - ]), - sse(vec![ev_response_created("resp-3"), ev_completed("resp-3")]), - ], - ) - .await; - let mut builder = search_capable_apps_builder(apps.chatgpt_base_url).with_config(|config| { - config.model_provider.include_internal_metadata = false; + ev_completed(&format!("resp-{query}")), + ])); + events.push(sse(vec![ + ev_response_created(&format!("resp-{query}-done")), + ev_completed(&format!("resp-{query}-done")), + ])); + } + } + let responses = mount_sse_sequence(&server, events).await; + let mock_url = server.uri(); + let configure_metadata = move |config: &mut codex_core::config::Config| { + assert_eq!(config.chatgpt_base_url, mock_url); + assert_eq!( + config.model_provider.base_url.as_deref(), + Some(format!("{mock_url}/v1").as_str()) + ); + config.model_provider.include_internal_metadata = include_internal_metadata; config.analytics_enabled = Some(true); config .features @@ -706,7 +732,9 @@ async fn direct_metadata_limit_retains_marker_without_sending_it_to_custom_provi .features .disable(Feature::CodeModeOnly) .expect("disable code-mode-only tools"); - }); + }; + let mut builder = search_capable_apps_builder(apps.chatgpt_base_url.clone()) + .with_config(configure_metadata.clone()); // build() always selects the local test environment; all network destinations are the mock. let test = builder.build(&server).await?; assert!(test.codex.analytics_enabled()); @@ -762,18 +790,20 @@ async fn direct_metadata_limit_retains_marker_without_sending_it_to_custom_provi recorded_metadata["executed_tool_calls"][0]["arguments"], json!({"query": query}) ); - // An ungranted custom Responses provider must not receive internal result metadata. - assert!( - wire_metadata["executed_tool_calls"][0] - .get("tool_result_metadata") - .is_none() - ); + let wire_result_metadata = + wire_metadata["executed_tool_calls"][0].get("tool_result_metadata"); + if include_internal_metadata { + assert_eq!(wire_result_metadata, Some(&metadata_for_query(query))); + } else { + // An ungranted provider must not receive internal result metadata. + assert!(wire_result_metadata.is_none()); + } } let first_metadata = tool_call_metadata(recorded_outputs[0].clone()); let second_metadata = tool_call_metadata(recorded_outputs[1].clone()); assert_eq!( first_metadata["executed_tool_calls"][0]["tool_result_metadata"], - raw_metadata + metadata_for_query("first") ); let marker = second_metadata["executed_tool_calls"][0]["tool_result_metadata"] .as_str() @@ -784,5 +814,117 @@ async fn direct_metadata_limit_retains_marker_without_sending_it_to_custom_provi .and_then(|value| value.parse::().ok()) .expect("marker should report omitted bytes"); assert!(overage > 0); + if !include_internal_metadata { + return Ok(()); + } + + let rollout_path = test.codex.rollout_path().expect("rollout path"); + test.codex.shutdown_and_wait().await?; + let rollout = tokio::fs::read_to_string(&rollout_path).await?; + let rollout = rollout + .lines() + .map(serde_json::from_str::) + .collect::>>()?; + for (call_id, expected) in [("first", &first_metadata), ("second", &second_metadata)] { + let item = rollout + .iter() + .find(|line| { + line["type"] == "response_item" + && line["payload"]["type"] == "function_call_output" + && line["payload"]["call_id"] == call_id + }) + .expect("persisted tool output"); + assert_eq!(tool_call_metadata(item["payload"].clone()), *expected); + } + let forked = test + .thread_manager + .fork_legacy_thread( + ForkSnapshot::Interrupted, + StartThreadOptions::new(test.config.clone()), + rollout_path.clone(), + ) + .await?; + wait_for_mcp_server(&forked.thread, CODEX_APPS_MCP_SERVER_NAME).await?; + forked + .thread + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "List events in the fork.".to_string(), + text_elements: Vec::new(), + }])) + .await?; + wait_for_event(&forked.thread, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + forked + .thread + .shutdown_and_wait() + .await + .expect("shutdown fork"); + + let resumed = search_capable_apps_builder(apps.chatgpt_base_url) + .with_config(configure_metadata) + .resume(&server, test.home.clone(), rollout_path) + .await + .expect("resume original thread"); + wait_for_mcp_server(&resumed.codex, CODEX_APPS_MCP_SERVER_NAME) + .await + .expect("start resumed MCP server"); + resumed + .submit_text_turn("List events after resume.") + .await + .expect("run resumed turn"); + resumed + .codex + .shutdown_and_wait() + .await + .expect("shutdown resumed thread"); + + let requests = responses.requests(); + assert_eq!(requests.len(), 7); + for request in &requests[3..] { + for original in [&first_output, &second_output] { + let call_id = original["call_id"].as_str().expect("call ID"); + let output = request.function_call_output(call_id); + assert_eq!(output["output"], original["output"]); + let passthrough = output.get("internal_chat_message_metadata_passthrough"); + assert!( + passthrough + .and_then(|meta| meta.get("executed_tool_calls")) + .is_none() + ); + assert!( + passthrough + .and_then(|meta| meta.get("tool_calls_complete")) + .is_none() + ); + } + } + for (request, query) in [(&requests[4], "fork"), (&requests[6], "resume")] { + let output = request.function_call_output(query); + assert!( + output["output"] + .to_string() + .contains(&format!("result for {query}")) + ); + let metadata = tool_call_metadata(output); + assert_eq!(metadata["tool_calls_complete"], true); + assert_eq!( + metadata["executed_tool_calls"].as_array().map(Vec::len), + Some(1) + ); + assert_eq!( + metadata["executed_tool_calls"][0]["name"], + format!("{SEARCH_CALENDAR_NAMESPACE}{SEARCH_CALENDAR_LIST_TOOL}") + ); + assert_eq!( + metadata["executed_tool_calls"][0]["arguments"], + json!({"query": query}) + ); + assert_eq!( + metadata["executed_tool_calls"][0]["tool_result_metadata"], + metadata_for_query(query) + ); + } Ok(()) } diff --git a/codex-rs/core/tests/suite/mcp_storage_telemetry_tests.rs b/codex-rs/core/tests/suite/mcp_storage_telemetry_tests.rs new file mode 100644 index 000000000000..71a970115002 --- /dev/null +++ b/codex-rs/core/tests/suite/mcp_storage_telemetry_tests.rs @@ -0,0 +1,177 @@ +//! Exercises thread-owned storage attribution through turn-time MCP authentication recovery. + +use anyhow::Result; +use codex_config::McpServerConfig; +use codex_config::types::OAuthCredentialsStoreMode; +use codex_core::StartThreadOptions; +use codex_core::TurnInputRequest; +use codex_history::InitialHistory; +use codex_history::RolloutItem; +use codex_otel::MetricsClient; +use codex_otel::MetricsConfig; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::SessionMeta; +use codex_protocol::protocol::SessionMetaLine; +use codex_protocol::user_input::UserInput; +use core_test_support::responses; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::MetricData; +use pretty_assertions::assert_eq; +use serde_json::json; +use std::collections::BTreeMap; +use std::sync::Arc; +use wiremock::Mock; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +fn storage_loads(metrics: &MetricsClient) -> Result> { + let mut counts = BTreeMap::new(); + for metric in metrics + .snapshot()? + .scope_metrics() + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + { + if metric.name() == "codex.auth_storage.operation" + && let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() + { + for point in sum.data_points() { + let tags: BTreeMap<_, _> = point + .attributes() + .map(|tag| (tag.key.to_string(), tag.value.to_string())) + .collect(); + if tags.get("credential_kind").map(String::as_str) == Some("mcp") + && tags.get("operation").map(String::as_str) == Some("load") + { + *counts.entry(tags["originator"].clone()).or_default() += point.value(); + } + } + } + } + Ok(counts) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn recovery_without_a_snapshot_keeps_each_threads_originator() -> Result<()> { + skip_if_no_network!(Ok(())); + // Isolate global metrics and CODEX_HOME from other integration tests. + if std::env::var_os("CODEX_MCP_TELEMETRY_TEST_CHILD").is_none() { + let home = tempfile::tempdir()?; + let output = tokio::process::Command::new(std::env::current_exe()?) + .args(["--exact", "suite::rmcp_client::storage_telemetry_tests::recovery_without_a_snapshot_keeps_each_threads_originator", "--nocapture"]) + .env("CODEX_MCP_TELEMETRY_TEST_CHILD", "1") + .env("CODEX_HOME", home.path()) + .output().await?; + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return Ok(()); + } + let server = responses::start_mock_server().await; + let mcp = responses::start_mock_server().await; + Mock::given(method("POST")) + .and(path("/mcp")) + .respond_with(ResponseTemplate::new(/*s*/ 401).append_header("www-authenticate", "Bearer")) + .mount(&mcp) + .await; + Mock::given(method("GET")) + .and(path("/.well-known/oauth-authorization-server/mcp")) + .respond_with(ResponseTemplate::new(/*s*/ 200).set_body_json(json!({ + "authorization_endpoint": format!("{}/authorize", mcp.uri()), + "token_endpoint": format!("{}/token", mcp.uri()), + }))) + .mount(&mcp) + .await; + let home = Arc::new(tempfile::tempdir()?); + // The OAuth endpoint remains on the host when the test uses a remote executor. + let fixture = test_codex() + .with_home(home) + .build_with_remote_and_local_env(&server) + .await?; + let server_config: McpServerConfig = + serde_json::from_value(json!({"url": format!("{}/mcp", mcp.uri())}))?; + let mut config = fixture.config.clone(); + config.mcp_oauth_credentials_store_mode = OAuthCredentialsStoreMode::File; + config + .mcp_servers + .set([("reauth".into(), server_config)].into())?; + let metrics = codex_otel::install_global_metrics(MetricsClient::new( + MetricsConfig::in_memory("test", "test", "1", Default::default()).with_runtime_reader(), + )?); + let originators = ["codex_desktop", "codex_vscode"]; + let mut threads = Vec::new(); + for originator in originators { + let thread = fixture + .thread_manager + .start_thread(StartThreadOptions { + initial_history: InitialHistory::Forked(vec![RolloutItem::SessionMeta( + SessionMetaLine { + meta: SessionMeta { + originator: originator.into(), + ..Default::default() + }, + git: None, + }, + )]), + environments: Some(fixture.codex.environment_selections().await), + ..StartThreadOptions::new(config.clone()) + }) + .await? + .thread; + let EventMsg::McpStartupComplete(startup) = wait_for_event(&thread, |event| { + matches!(event, EventMsg::McpStartupComplete(_)) + }) + .await + else { + unreachable!(); + }; + assert_eq!( + startup + .failed + .iter() + .map(|failure| failure.server.as_str()) + .collect::>(), + vec!["reauth"] + ); + threads.push(thread); + responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg", "Sign in to continue."), + responses::ev_completed("resp"), + ]), + ) + .await; + } + // The runtime reader exports deltas; drain startup observations before recovery. + let _ = storage_loads(&metrics)?; + futures::future::try_join_all(threads.iter().map(|thread| async { + thread + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "Continue".into(), + text_elements: Vec::new(), + }])) + .await?; + wait_for_event(thread, |event| matches!(event, EventMsg::TurnComplete(_))).await; + Ok::<_, anyhow::Error>(()) + })) + .await?; + let mut after = storage_loads(&metrics)?; + after.retain(|_, count| *count > 0); + assert_eq!( + after.keys().map(String::as_str).collect::>(), + originators + ); + for thread in threads { + thread.shutdown_and_wait().await?; + } + fixture.codex.shutdown_and_wait().await?; + metrics.shutdown()?; + Ok(()) +} diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 2833288bb678..5400f0f4efa2 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -132,6 +132,8 @@ use wiremock::MockServer; #[path = "mcp_oauth_refresh_tests.rs"] mod oauth_refresh_tests; +#[path = "mcp_storage_telemetry_tests.rs"] +mod storage_telemetry_tests; static OPENAI_PNG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAD0AAAA9CAYAAAAeYmHpAAAE6klEQVR4Aeyau44UVxCGx1fZsmRLlm3Zoe0XcGQ5cUiCCIgJeS9CHgAhMkISQnIuGQgJEkBcxLW+nqnZ6uqqc+nuWRC7q/P3qetf9e+MtOwyX25O4Nep6JPyop++0qev9HrfgZ+F6r2DuB/vHOrt/UIkqdDHYvujOW6fO7h/CNEI+a5jc+pBR8uy0jVFsziYu5HtfSUk+Io34q921hLNctFSX0gwww+S8wce8K1LfCU+cYW4888aov8NxqvQILUPPReLOrm6zyLxa4i+6VZuFbJo8d1MOHZm+7VUtB/aIvhPWc/3SWg49JcwFLlHxuXKjtyloo+YNhuW3VS+WPBuUEMvCFKjEDVgFBQHXrnazpqiSxNZCkQ1kYiozsbm9Oz7l4i2Il7vGccGNWAc3XosDrZe/9P3ZnMmzHNEQw4smf8RQ87XEAMsC7Az0Au+dgXerfH4+sHvEc0SYGic8WBBUGqFH2gN7yDrazy7m2pbRTeRmU3+MjZmr1h6LJgPbGy23SI6GlYT0brQ71IY8Us4PNQCm+zepSbaD2BY9xCaAsD9IIj/IzFmKMSdHHonwdZATbTnYREf6/VZGER98N9yCWIvXQwXDoDdhZJoT8jwLnJXDB9w4Sb3e6nK5ndzlkTLnP3JBu4LKkbrYrU69gCVceV0JvpyuW1xlsUVngzhwMetn/XamtTORF9IO5YnWNiyeF9zCAfqR3fUW+vZZKLtgP+ts8BmQRBREAdRDhH3o8QuRh/YucNFz2BEjxbRN6LGzphfKmvP6v6QhqIQyZ8XNJ0W0X83MR1PEcJBNO2KC2Z1TW/v244scp9FwRViZxIOBF0Lctk7ZVSavdLvRlV1hz/ysUi9sr8CIcB3nvWBwA93ykTz18eAYxQ6N/K2DkPA1lv3iXCwmDUT7YkjIby9siXueIJj9H+pzSqJ9oIuJWTUgSSt4WO7o/9GGg0viR4VinNRUDoIj34xoCd6pxD3aK3zfdbnx5v1J3ZNNEJsE0sBG7N27ReDrJc4sFxz7dI/ZAbOmmiKvHBitQXpAdR6+F7v+/ol/tOouUV01EeMZQF2BoQDn6dP4XNr+j9GZEtEK1/L8pFw7bd3a53tsTa7WD+054jOFmPg1XBKPQgnqFfmFcy32ZRvjmiIIQTYFvyDxQ8nH8WIwwGwlyDjDznnilYyFr6njrlZwsKkBpO59A7OwgdzPEWRm+G+oeb7IfyNuzjEEVLrOVxJsxvxwF8kmCM6I2QYmJunz4u4TrADpfl7mlbRTWQ7VmrBzh3+C9f6Grc3YoGN9dg/SXFthpRsT6vobfXRs2VBlgBHXVMLHjDNbIZv1sZ9+X3hB09cXdH1JKViyG0+W9bWZDa/r2f9zAFR71sTzGpMSWz2iI4YssWjWo3REy1MDGjdwe5e0dFSiAC1JakBvu4/CUS8Eh6dqHdU0Or0ioY3W5ClSqDXAy7/6SRfgw8vt4I+tbvvNtFT2kVDhY5+IGb1rCqYaXNF08vSALsXCPmt0kQNqJT1p5eI1mkIV/BxCY1z85lOzeFbPBQHURkkPTlwTYK9gTVE25l84IbFFN+YJDHjdpn0gq6mrHht0dkcjbM4UL9283O5p77GN+SPW/QwVB4IUYg7Or+Kp7naR6qktP98LNF2UxWo9yObPIT9KYg+hK4i56no4rfnM0qeyFf6AwAAAP//trwR3wAAAAZJREFUAwBZ0sR75itw5gAAAABJRU5ErkJggg=="; diff --git a/codex-rs/core/tests/suite/unified_exec_process_events.rs b/codex-rs/core/tests/suite/unified_exec_process_events.rs index ff85b4fba468..39d8db419093 100644 --- a/codex-rs/core/tests/suite/unified_exec_process_events.rs +++ b/codex-rs/core/tests/suite/unified_exec_process_events.rs @@ -4,6 +4,7 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_config::test_support::CloudConfigBundleFixture; use codex_core::TurnInputRequest; +use codex_exec_server::ExecParams; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::ModeKind; @@ -13,8 +14,11 @@ use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::FileSystemSandboxPolicyContext; use codex_protocol::permissions::FileSystemSpecialPath; use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_protocol::permissions::ReadDenyMatcher; +use codex_protocol::permissions::project_roots_glob_pattern; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EnvironmentConfigState; use codex_protocol::protocol::EventMsg; @@ -44,6 +48,9 @@ use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; use std::fs; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use std::time::Duration; use test_case::test_case; use tokio::net::TcpListener; @@ -65,6 +72,11 @@ const REPLAY_RETAINED_OUTPUT_SEQ: u64 = 800; enum PushedExecScenario { Complete, DirectDenied, + DenyWin, + DenyWinUnchanged, + DenyOldLinux, + DenyOldLinuxWithRoot(FileSystemAccessMode), + DenyLinux, ElevatedPowerShell, RejectedLongWindowsDangerousCommand, SandboxedInterceptedPatch, @@ -156,18 +168,34 @@ async fn respond_environment_info( ) { let shell = if matches!( scenario, - PushedExecScenario::ElevatedPowerShell + PushedExecScenario::DenyWin + | PushedExecScenario::DenyWinUnchanged + | PushedExecScenario::ElevatedPowerShell | PushedExecScenario::RejectedLongWindowsDangerousCommand ) { json!({ "name": "powershell", "path": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" }) } else { json!({ "name": "zsh", "path": "/bin/zsh" }) }; - let platform_os = matches!( + let platform_os = match scenario { + PushedExecScenario::DenyWin + | PushedExecScenario::DenyWinUnchanged + | PushedExecScenario::RejectedLongWindowsDangerousCommand => Some("windows"), + PushedExecScenario::DenyOldLinux + | PushedExecScenario::DenyOldLinuxWithRoot(_) + | PushedExecScenario::DenyLinux => Some("linux"), + _ => None, + }; + let mut capabilities = json!({ "networkProxyLaunch": true }); + if matches!( scenario, - PushedExecScenario::RejectedLongWindowsDangerousCommand - ) - .then_some("windows"); + PushedExecScenario::DenyOldLinuxWithRoot(_) | PushedExecScenario::DenyLinux + ) { + capabilities["linuxRootWritePreservesDevices"] = json!(true); + } + if matches!(scenario, PushedExecScenario::DenyLinux) { + capabilities["linuxApprovedRootWritePreservesRestrictions"] = json!(true); + } send_exec_server_json( websocket, json!({ @@ -175,7 +203,7 @@ async fn respond_environment_info( "result": { "shell": shell, "platformOs": platform_os, - "capabilities": { "networkProxyLaunch": true } + "capabilities": capabilities, } }), ) @@ -185,6 +213,7 @@ async fn respond_environment_info( async fn serve_exec_with_pushed_events( listener: TcpListener, scenario: PushedExecScenario, + process_started: Arc, ) -> PushedExecServerResult { let unrestricted_patch = matches!( scenario, @@ -198,7 +227,10 @@ async fn serve_exec_with_pushed_events( // The runtime may still be finishing local setup before its first tool call. let request = read_exec_server_json(&mut websocket, STARTUP_TIMEOUT).await; match request["method"].as_str() { - Some("process/start") => break request, + Some("process/start") => { + process_started.store(true, Ordering::SeqCst); + break request; + } Some("environment/info") => { respond_environment_info(&mut websocket, &request["id"], scenario).await; } @@ -372,7 +404,13 @@ async fn serve_exec_with_pushed_events( .await; match scenario { - PushedExecScenario::Complete | PushedExecScenario::ElevatedPowerShell => { + PushedExecScenario::Complete + | PushedExecScenario::DenyWin + | PushedExecScenario::DenyWinUnchanged + | PushedExecScenario::DenyOldLinux + | PushedExecScenario::DenyOldLinuxWithRoot(_) + | PushedExecScenario::DenyLinux + | PushedExecScenario::ElevatedPowerShell => { let encoded_output = BASE64_STANDARD.encode(COMPLETE_OUTPUT); for message in [ json!({ @@ -475,8 +513,13 @@ async fn serve_exec_with_pushed_events( "failure": null, "sandboxDenied": true, }), - PushedExecScenario::ElevatedPowerShell => { - panic!("elevated remote PowerShell must not read a remote process") + PushedExecScenario::DenyWin + | PushedExecScenario::DenyWinUnchanged + | PushedExecScenario::DenyOldLinux + | PushedExecScenario::DenyOldLinuxWithRoot(_) + | PushedExecScenario::DenyLinux + | PushedExecScenario::ElevatedPowerShell => { + panic!("completed remote PowerShell must not read a remote process") } PushedExecScenario::RejectedLongWindowsDangerousCommand => { panic!("dangerous command must not read a remote process") @@ -571,6 +614,12 @@ async fn serve_exec_with_pushed_events( #[test_case(PushedExecScenario::Complete, ManagedNetworkScenario::Disabled, false ; "disabled_managed_network_omits_executor_proxy_launch")] #[cfg_attr(not(windows), test_case(PushedExecScenario::Complete, ManagedNetworkScenario::Enabled { policy_callbacks: true }, true ; "foreign_windows_managed_network_preserves_approval_registration"))] #[cfg_attr(not(windows), test_case(PushedExecScenario::Complete, ManagedNetworkScenario::None, true ; "foreign_windows_workspace_sandbox"))] +#[test_case(PushedExecScenario::DenyWin, ManagedNetworkScenario::None, true ; "old_windows_deny_read_allows_approved_escalation")] +#[test_case(PushedExecScenario::DenyWinUnchanged, ManagedNetworkScenario::None, true ; "old_windows_deny_read_keeps_unmaterializable_policy_unchanged")] +#[cfg_attr(not(windows), test_case(PushedExecScenario::DenyOldLinux, ManagedNetworkScenario::None, false ; "old_linux_deny_read_rejects_approved_escalation"))] +#[cfg_attr(not(windows), test_case(PushedExecScenario::DenyOldLinuxWithRoot(FileSystemAccessMode::Deny), ManagedNetworkScenario::None, false ; "old_linux_deny_read_keeps_root_deny_unchanged"))] +#[cfg_attr(not(windows), test_case(PushedExecScenario::DenyOldLinuxWithRoot(FileSystemAccessMode::Write), ManagedNetworkScenario::None, false ; "device_only_linux_deny_read_rejects_unchanged_writable_root_metadata"))] +#[cfg_attr(not(windows), test_case(PushedExecScenario::DenyLinux, ManagedNetworkScenario::None, false ; "linux_deny_read_allows_approved_escalation"))] #[test_case(PushedExecScenario::ElevatedPowerShell, ManagedNetworkScenario::None, true ; "windows_elevated_powershell_disables_profile")] #[cfg_attr(not(windows), test_case(PushedExecScenario::RejectedLongWindowsDangerousCommand, ManagedNetworkScenario::None, true ; "remote_windows_dangerous_command_rejection_is_bounded"))] #[cfg_attr(not(windows), test_case(PushedExecScenario::SandboxedInterceptedPatch, ManagedNetworkScenario::None, true ; "foreign_windows_intercepted_patch_is_sandboxed"))] @@ -593,9 +642,27 @@ async fn exec_command_consumes_pushed_remote_process_events( policy_callbacks: true } ); + let deny_read = matches!( + scenario, + PushedExecScenario::DenyWin + | PushedExecScenario::DenyWinUnchanged + | PushedExecScenario::DenyOldLinux + | PushedExecScenario::DenyOldLinuxWithRoot(_) + | PushedExecScenario::DenyLinux + ); let listener = TcpListener::bind("127.0.0.1:0").await?; let server = start_mock_server().await; let tool_call = match scenario { + _ if deny_read => ev_function_call( + CALL_ID, + "exec_command", + &json!({ + "cmd": "codex-deny-read-escalation-test", + "sandbox_permissions": "require_escalated", + "justification": "Can I run outside the workspace?", + }) + .to_string(), + ), PushedExecScenario::SandboxedDirectPatch | PushedExecScenario::SandboxedDirectPatchDenied | PushedExecScenario::SandboxedDirectPatchRetry => ev_apply_patch_custom_tool_call( @@ -642,7 +709,12 @@ async fn exec_command_consumes_pushed_remote_process_events( ) .await; let exec_server_url = format!("ws://{}", listener.local_addr()?); - let exec_server = tokio::spawn(serve_exec_with_pushed_events(listener, scenario)); + let process_started = Arc::new(AtomicBool::new(false)); + let exec_server = tokio::spawn(serve_exec_with_pushed_events( + listener, + scenario, + Arc::clone(&process_started), + )); let mut builder = test_codex().with_exec_server_url(exec_server_url); if managed_network_configured { let cloud_config_bundle = match managed_network { @@ -705,7 +777,8 @@ timeout = 900 } let mut builder = builder.with_config(move |config| { config.project_doc_max_bytes = 0; - if matches!(scenario, PushedExecScenario::ElevatedPowerShell) { + if matches!(scenario, PushedExecScenario::ElevatedPowerShell) || (foreign_cwd && deny_read) + { config.set_windows_elevated_sandbox_enabled(/*value*/ true); } if managed_network_configured { @@ -719,6 +792,45 @@ timeout = 900 let turn_permission_profile = if managed_network_configured { test.session_configured.permission_profile.clone() + } else if deny_read { + let mut filesystem = FileSystemSandboxPolicy::read_only(); + if let PushedExecScenario::DenyOldLinuxWithRoot(access) = scenario { + filesystem.entries[0].access = access; + } + filesystem.entries.extend( + [ + FileSystemPath::Special { + value: FileSystemSpecialPath::project_roots(Some("secret.txt".into())), + }, + FileSystemPath::GlobPattern { + pattern: project_roots_glob_pattern("*.token".as_ref()), + }, + ] + .map(|path| FileSystemSandboxEntry::new(path, FileSystemAccessMode::Deny)), + ); + if matches!(scenario, PushedExecScenario::DenyWinUnchanged) { + filesystem.entries.push(FileSystemSandboxEntry::new( + FileSystemPath::Special { + value: FileSystemSpecialPath::Tmpdir, + }, + FileSystemAccessMode::Deny, + )); + } + if matches!( + scenario, + PushedExecScenario::DenyOldLinuxWithRoot(FileSystemAccessMode::Write) + ) { + let cwd = PathUri::parse("file:///workspace")?; + let context = FileSystemSandboxPolicyContext { + cwd: &cwd, + workspace_roots: std::slice::from_ref(&cwd), + user_home_dir: None, + temporary_directories: None, + }; + filesystem = filesystem.for_approved_command(&context); + assert_eq!(filesystem.for_approved_command(&context), filesystem); + } + PermissionProfile::from_runtime_permissions(&filesystem, NetworkSandboxPolicy::Enabled) } else if matches!(scenario, PushedExecScenario::FullDiskInterceptedPatch) { PermissionProfile::from_runtime_permissions( &FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry::new( @@ -757,7 +869,7 @@ timeout = 900 PermissionProfile::Disabled }; let (sandbox_policy, permission_profile) = - turn_permission_fields(turn_permission_profile, test.config.cwd.as_path()); + turn_permission_fields(turn_permission_profile.clone(), test.config.cwd.as_path()); test.codex .start_or_steer_turn( TurnInputRequest::user_input(vec![UserInput::Text { @@ -765,18 +877,26 @@ timeout = 900 text_elements: Vec::new(), }]) .with_thread_settings(ThreadSettingsOverrides { - environments: foreign_cwd.then(|| { - let cwd = PathUri::parse("file:///C:/workspace").expect("valid Windows cwd"); + environments: (foreign_cwd || deny_read).then(|| { + let cwd = PathUri::parse(if foreign_cwd { + "file:///C:/workspace" + } else { + "file:///workspace" + }) + .expect("valid executor cwd"); + let mut workspace_roots = vec![cwd.clone()]; + if foreign_cwd { + workspace_roots.push( + PathUri::parse("file:///D:/other-workspace") + .expect("valid Windows workspace root"), + ); + } TurnEnvironmentSelections::new( test.config.cwd.clone(), vec![TurnEnvironmentSelection { environment_id: codex_exec_server::REMOTE_ENVIRONMENT_ID.to_string(), - cwd: cwd.clone(), - workspace_roots: vec![ - cwd, - PathUri::parse("file:///D:/other-workspace") - .expect("valid Windows workspace root"), - ], + cwd, + workspace_roots, config: EnvironmentConfigState::FromThread, }], ) @@ -788,6 +908,7 @@ timeout = 900 PushedExecScenario::SandboxedDirectPatchDenied | PushedExecScenario::SandboxedDirectPatchRetry ) + || deny_read { AskForApproval::OnRequest } else { @@ -820,6 +941,16 @@ timeout = 900 EventMsg::ExecCommandBegin(event) if event.call_id == CALL_ID => { saw_exec_command_begin = true; } + EventMsg::ExecApprovalRequest(approval) if deny_read => { + assert_eq!(approval.call_id, CALL_ID); + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::Approved, + }) + .await?; + } EventMsg::ApplyPatchApprovalRequest(approval) if matches!( scenario, @@ -849,6 +980,24 @@ timeout = 900 } } } + if matches!( + scenario, + PushedExecScenario::DenyOldLinux + | PushedExecScenario::DenyOldLinuxWithRoot(FileSystemAccessMode::Write) + ) { + let request = response_mock.last_request().context("upgrade response")?; + let (output, _) = request + .function_call_output_content_and_success(CALL_ID) + .context("upgrade output")?; + assert!( + output + .expect("upgrade rejection should include output") + .contains("upgrade the exec-server") + ); + assert!(!process_started.load(Ordering::SeqCst)); + exec_server.abort(); + return Ok(()); + } if matches!( scenario, PushedExecScenario::RejectedLongWindowsDangerousCommand @@ -998,7 +1147,7 @@ timeout = 900 "toolCallId": CALL_ID, }), ); - if foreign_cwd && !managed_network_configured { + if foreign_cwd && !managed_network_configured && !deny_read { let permissions = response_mock.requests()[0] .message_input_texts("developer") .join("\n") @@ -1036,10 +1185,40 @@ timeout = 900 .is_some_and(|argv| argv.iter().any(|arg| arg == "-NoProfile")), "elevated remote PowerShell must not load a user profile" ); + } else if deny_read { + assert_eq!(params["sandbox"]["windowsSandboxLevel"], "elevated"); } else { assert_eq!(params["sandbox"]["windowsSandboxLevel"], "restricted-token"); } } + if deny_read { + let params = &exec_server_result.process_start["params"]; + let params: ExecParams = serde_json::from_value(params.clone())?; + let sandbox = params.sandbox.context("sandbox with denied reads")?; + let context = sandbox.policy_context(); + let policy = sandbox.permissions.file_system_sandbox_policy(); + let policy = policy.materialize_project_roots_with_path_uris(context.workspace_roots); + if matches!( + scenario, + PushedExecScenario::DenyWinUnchanged | PushedExecScenario::DenyOldLinuxWithRoot(_) + ) { + let baseline = turn_permission_profile.file_system_sandbox_policy(); + assert_eq!( + policy, + baseline.materialize_project_roots_with_path_uris(context.workspace_roots) + ); + assert!(!policy.can_write_path(&context.cwd.join("../outside")?, &context)); + } else { + let matcher = + ReadDenyMatcher::from_context(&policy, &context).context("denied reads")?; + assert!(policy.can_write_path(&context.cwd.join("../outside")?, &context)); + for root in context.workspace_roots { + for path in ["secret.txt", "private.token"] { + assert!(matcher.is_read_denied_uri(&root.join(path)?, &context)); + } + } + } + } if managed_network_enabled { let params = &exec_server_result.process_start["params"]; assert_eq!(params["enforceManagedNetwork"], true); @@ -1078,7 +1257,13 @@ timeout = 900 let output = output.context("exec_command output should contain text")?; let process_read_requests = exec_server_result.process_read_requests; match scenario { - PushedExecScenario::Complete | PushedExecScenario::ElevatedPowerShell => { + PushedExecScenario::Complete + | PushedExecScenario::DenyWin + | PushedExecScenario::DenyWinUnchanged + | PushedExecScenario::DenyOldLinux + | PushedExecScenario::DenyOldLinuxWithRoot(_) + | PushedExecScenario::DenyLinux + | PushedExecScenario::ElevatedPowerShell => { assert_ne!(success, Some(false)); assert!(saw_exec_command_begin); assert!(output.contains("Process exited with code 0")); diff --git a/codex-rs/core/tests/suite/unified_exec_stdin_approval.rs b/codex-rs/core/tests/suite/unified_exec_stdin_approval.rs index a8e769e1fdc2..6df66e35ffa6 100644 --- a/codex-rs/core/tests/suite/unified_exec_stdin_approval.rs +++ b/codex-rs/core/tests/suite/unified_exec_stdin_approval.rs @@ -9,6 +9,10 @@ use codex_features::Feature; use codex_protocol::approvals::ExecApprovalKind; use codex_protocol::models::AdditionalPermissionProfile; use codex_protocol::models::PermissionProfile; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GranularApprovalConfig; @@ -192,6 +196,123 @@ async fn stdin_reviews_retained_grants_after_turn_permissions_expire() -> Result Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stdin_reviews_escalated_terminal_with_denied_reads_before_delivering_input() -> Result<()> +{ + skip_if_target_windows!(Ok(()), "uses a POSIX interactive shell"); + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + let harness = TestCodexHarness::with_auto_env_builder(test_codex().with_config(|config| { + config.features.enable(Feature::WriteStdinApproval).unwrap(); + config + .features + .enable(Feature::ExecPermissionApprovals) + .unwrap(); + config.features.enable(Feature::UnifiedExec).unwrap(); + config.features.disable(Feature::ShellZshFork).unwrap(); + config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); + let mut file_system = PermissionProfile::read_only().file_system_sandbox_policy(); + file_system.entries.push(FileSystemSandboxEntry::new( + FileSystemPath::GlobPattern { + pattern: "**/secret.env".into(), + }, + FileSystemAccessMode::Deny, + )); + config + .permissions + .set_permission_profile(PermissionProfile::from_runtime_permissions( + &file_system, + NetworkSandboxPolicy::Restricted, + )) + .unwrap(); + })) + .await?; + let codex = &harness.test().codex; + harness.write_file("secret.env", "secret").await?; + let _open = mount_function_call_agent_response( + harness.server(), + "open", + &json!({"cmd":"/bin/bash --noprofile --norc", "tty":true, "yield_time_ms":200, "sandbox_permissions":"require_escalated", "justification":"open an escalated terminal"}).to_string(), + "exec_command", + ).await; + codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "open terminal".into(), + text_elements: Vec::new(), + }])) + .await?; + let launch = wait_for_event_match(codex, |event| match event { + EventMsg::ExecApprovalRequest(request) => Some(request.clone()), + _ => None, + }) + .await; + assert_eq!(launch.kind, ExecApprovalKind::Command); + codex + .submit(Op::ExecApproval { + id: launch.effective_approval_id(), + turn_id: Some(launch.turn_id), + decision: ReviewDecision::Approved, + }) + .await?; + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let _writes = mount_sse_sequence(harness.server(), vec![ + tool_response("denied", "write_stdin", json!({"session_id":1000,"chars":"printf denied > denied-marker\n"})), + tool_response("allowed", "write_stdin", json!({"session_id":1000,"chars":"if test -e denied-marker || cat secret.env; then exit 12; fi; printf approved > approved-marker; exit\n"})), + sse(vec![ev_completed("done")]), + ]).await; + codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "send input".into(), + text_elements: Vec::new(), + }])) + .await?; + for (id, decision) in [ + ("denied", ReviewDecision::denied("blocked input")), + ("allowed", ReviewDecision::Approved), + ] { + let request = wait_for_event_match(codex, |event| match event { + EventMsg::ExecApprovalRequest(request) => Some(request.clone()), + _ => None, + }) + .await; + assert_eq!( + ( + request.kind, + request.call_id.as_str(), + request.effective_approval_id().as_str() + ), + (ExecApprovalKind::WriteStdin, "open", id) + ); + assert!(!harness.path_exists("denied-marker").await?); + assert!(!harness.path_exists("approved-marker").await?); + codex + .submit(Op::ExecApproval { + id: request.effective_approval_id(), + turn_id: Some(request.turn_id), + decision, + }) + .await?; + } + let mut process_end = None; + let mut turn_completed = false; + while process_end.is_none() || !turn_completed { + match wait_for_event(codex, |_| true).await { + EventMsg::ExecCommandEnd(event) if event.call_id == "open" => process_end = Some(event), + EventMsg::TurnComplete(_) => turn_completed = true, + EventMsg::ExecApprovalRequest(request) => { + anyhow::bail!("unexpected additional stdin approval: {request:?}") + } + _ => {} + } + } + let process_end = process_end.expect("retained terminal should finish"); + assert_eq!(process_end.exit_code, 0); + assert!(!harness.path_exists("denied-marker").await?); + assert_eq!(harness.read_file_text("approved-marker").await?, "approved"); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn strict_stdin_review_reaches_guardian_with_sandbox_prompts_disabled() -> Result<()> { skip_if_target_windows!(Ok(()), "uses a POSIX interactive shell"); diff --git a/codex-rs/core/tests/suite/windows_sandbox.rs b/codex-rs/core/tests/suite/windows_sandbox.rs index b9d7b085a886..ccbabef97f74 100644 --- a/codex-rs/core/tests/suite/windows_sandbox.rs +++ b/codex-rs/core/tests/suite/windows_sandbox.rs @@ -1,4 +1,7 @@ use anyhow::Context; +use codex_config::types::ApprovalsReviewer; +use codex_core::TurnInputRequest; +use codex_core::config::Constrained; use codex_core::exec::ExecCapturePolicy; use codex_core::exec::ExecParams; use codex_core::exec::process_exec_tool_call; @@ -17,6 +20,12 @@ use codex_protocol::permissions::FileSystemSandboxEntry; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::FileSystemSpecialPath; use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::ReviewDecision; +use codex_protocol::user_input::UserInput; +use codex_windows_sandbox_test_support::WindowsSandboxAccountTestGuard; use core_test_support::PathExt; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; @@ -26,6 +35,8 @@ use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::test_codex::TestCodexHarness; use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use core_test_support::wait_for_event_with_timeout; use pretty_assertions::assert_eq; use serde_json::json; use serial_test::serial; @@ -34,6 +45,7 @@ use std::ffi::OsString; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::time::Duration; use tempfile::TempDir; struct EnvVarGuard { @@ -77,6 +89,7 @@ impl TestCodexHome { } fn codex_home_for_windows_sandbox_test(name: &str) -> anyhow::Result { + // Only tests that use the elevated accounts also take the shared account guard. if let Some(test_tmpdir) = std::env::var_os("TEST_TMPDIR") { // The elevated backend provisions machine-local sandbox users. Bazel // retries run in the same Windows VM, so keep CODEX_HOME stable within @@ -181,6 +194,7 @@ fn assert_managed_deny_probe(output: &std::process::Output, launch: usize) -> an #[test] #[serial(codex_home)] fn windows_sandbox_cli_preserves_managed_deny_reads_across_launches() -> anyhow::Result<()> { + let _account_guard = WindowsSandboxAccountTestGuard::acquire()?; let codex_home = codex_home_for_windows_sandbox_test("windows-cli-managed-deny-read-codex-home")?; @@ -391,6 +405,7 @@ async fn windows_restricted_token_rejects_exact_and_glob_deny_read_policy() -> a #[tokio::test] #[serial(codex_home)] async fn windows_elevated_does_not_create_missing_workspace_metadata() -> anyhow::Result<()> { + let _account_guard = WindowsSandboxAccountTestGuard::acquire()?; let codex_home = codex_home_for_windows_sandbox_test("windows-elevated-missing-metadata-codex-home")?; let _codex_home_guard = EnvVarGuard::set("CODEX_HOME", codex_home.path().as_os_str()); @@ -516,6 +531,7 @@ $rules = foreach ($name in @('codex_sandbox_offline_block_inbound', 'codex_sandb #[tokio::test] #[serial(codex_home)] async fn windows_elevated_enforces_deny_read_and_protects_setup_marker() -> anyhow::Result<()> { + let _account_guard = WindowsSandboxAccountTestGuard::acquire()?; let codex_home = codex_home_for_windows_sandbox_test("windows-elevated-deny-read-codex-home")?; let _codex_home_guard = EnvVarGuard::set("CODEX_HOME", codex_home.path().as_os_str()); stage_windows_sandbox_helpers()?; @@ -656,6 +672,7 @@ async fn windows_elevated_enforces_deny_read_and_protects_setup_marker() -> anyh #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial(codex_home)] async fn windows_elevated_unified_exec_enforces_large_recursive_deny_reads() -> anyhow::Result<()> { + let _account_guard = WindowsSandboxAccountTestGuard::acquire()?; let codex_home = codex_home_for_windows_sandbox_test("windows-elevated-tool-runtime-deny-read-codex-home")?; let _codex_home_guard = EnvVarGuard::set("CODEX_HOME", codex_home.path().as_os_str()); @@ -825,3 +842,155 @@ async fn windows_elevated_unified_exec_enforces_large_recursive_deny_reads() -> Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(codex_home)] +async fn windows_elevated_approved_git_pull_preserves_deny_read() -> anyhow::Result<()> { + use EventMsg::ExecApprovalRequest; + use EventMsg::ExecCommandEnd; + use EventMsg::TurnComplete; + + let _account_guard = WindowsSandboxAccountTestGuard::acquire()?; + let codex_home = codex_home_for_windows_sandbox_test("windows-elevated-git-pull-deny-read")?; + let _codex_home_guard = EnvVarGuard::set("CODEX_HOME", codex_home.path().as_os_str()); + stage_windows_sandbox_helpers()?; + let home = dunce::canonicalize(codex_home.path())?.abs(); + let builder = test_codex() + .with_windows_cmd_shell() + .with_config(move |config| { + config.codex_home = home; + config.set_windows_elevated_sandbox_enabled(true); + config.prefer_mxc = false; + config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); + config.approvals_reviewer = ApprovalsReviewer::User; + config.features.enable(Feature::UnifiedExec).unwrap(); + let mut filesystem = FileSystemSandboxPolicy::workspace_write( + &[], + /*exclude_tmpdir_env_var*/ true, + /*exclude_slash_tmp*/ true, + ); + filesystem.entries.extend( + [ + config.cwd.join("exact-secret.txt").into(), + FileSystemPath::GlobPattern { + pattern: "**/*.env".into(), + }, + ] + .map(|path| FileSystemSandboxEntry::new(path, FileSystemAccessMode::Deny)), + ); + config + .permissions + .set_permission_profile(PermissionProfile::from_runtime_permissions( + &filesystem, + NetworkSandboxPolicy::Restricted, + )) + .unwrap(); + }) + .with_workspace_setup(|cwd, _fs| async move { + std::fs::write(cwd.join("exact-secret.txt"), "exact secret")?; + std::fs::write(cwd.join("secret.env"), "glob secret")?; + Ok(()) + }); + let harness = TestCodexHarness::with_builder(builder).await?; + let git = |cwd: &Path, args: &[&str]| -> anyhow::Result { + let output = Command::new("git") + .args(["-c", "commit.gpgsign=false"]) + .current_dir(cwd) + .args(args) + .output()?; + anyhow::ensure!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(String::from_utf8(output.stdout)?.trim().to_string()) + }; + let upstream = TempDir::new()?; + let upstream = upstream.path(); + let upstream_path = upstream.to_str().context("UTF-8 upstream path")?; + let cwd = harness.cwd(); + let read = |name| std::fs::read_to_string(cwd.join(name)); + let init = ["init", "--initial-branch=main"]; + let commit = ["commit", "-am", "fixture"]; + git(upstream, &init)?; + git(upstream, &["config", "user.name", "Codex Test"])?; + git(upstream, &["config", "user.email", "codex@test.com"])?; + std::fs::write(upstream.join("tracked.txt"), "before\n")?; + git(upstream, &["add", "tracked.txt"])?; + git(upstream, &commit)?; + git(cwd, &init)?; + git(cwd, &["pull", "-q", "--ff-only", upstream_path, "main"])?; + let fetch_head = cwd.join(".git/FETCH_HEAD"); + std::fs::remove_file(&fetch_head)?; + std::fs::write(upstream.join("tracked.txt"), "after\n")?; + git(upstream, &commit)?; + let expected = git(upstream, &["rev-parse", "HEAD"])?; + let bundle = cwd.join("upstream.bundle"); + let bundle_path = bundle.to_str().context("UTF-8 bundle path")?; + git(upstream, &["bundle", "create", bundle_path, "--all"])?; + + let args = json!({"cmd": concat!( + "git pull --quiet --ff-only .\\upstream.bundle main || exit /b 10 & ", + "type tracked.txt >NUL || exit /b 11 & ", + "type exact-secret.txt >NUL 2>NUL && exit /b 12 & ", + "type secret.env >NUL 2>NUL && exit /b 13 & ", + "echo git-and-denials-ok>git-and-denials-ok.txt || exit /b 14 & exit /b 0" + ), "sandbox_permissions": "require_escalated", + "justification": "Can I pull the fixture repository?", + "yield_time_ms": 30_000, "tty": false, "login": false, + }); + let call_id = "git-pull"; + core_test_support::responses::mount_function_call_agent_response( + harness.server(), + call_id, + &args.to_string(), + "exec_command", + ) + .await; + let codex = &harness.test().codex; + codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "pull the fixture repository".into(), + text_elements: Vec::new(), + }])) + .await?; + let approval = wait_for_event(codex, |event| { + matches!(event, ExecApprovalRequest(_) | TurnComplete(_)) + }) + .await; + let ExecApprovalRequest(approval) = approval else { + anyhow::bail!("expected approval before git ran: {approval:?}"); + }; + assert_eq!(approval.call_id, call_id); + assert!(!fetch_head.exists()); + assert!(!cwd.join("git-and-denials-ok.txt").exists()); + assert_eq!(read("tracked.txt")?.trim(), "before"); + codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::Approved, + }) + .await?; + let mut process_end = None; + let mut turn_completed = false; + while process_end.is_none() || !turn_completed { + match wait_for_event_with_timeout(codex, |_| true, Duration::from_secs(120)).await { + ExecCommandEnd(event) if event.call_id == call_id => process_end = Some(event), + TurnComplete(_) => turn_completed = true, + ExecApprovalRequest(request) => { + anyhow::bail!("unexpected additional Git approval: {request:?}") + } + _ => {} + } + } + let result = process_end.expect("approved Git process should finish"); + let output = &result.aggregated_output; + assert_eq!(result.exit_code, 0, "sandbox output: {output}"); + assert_eq!(read("git-and-denials-ok.txt")?.trim(), "git-and-denials-ok"); + assert!(read(".git/FETCH_HEAD")?.starts_with(&expected)); + assert_eq!(read("tracked.txt")?.trim(), "after"); + assert_eq!(read("exact-secret.txt")?, "exact secret"); + assert_eq!(read("secret.env")?, "glob secret"); + Ok(()) +} diff --git a/codex-rs/exec-server-protocol/Cargo.toml b/codex-rs/exec-server-protocol/Cargo.toml index 62841ed97314..eda1add2726e 100644 --- a/codex-rs/exec-server-protocol/Cargo.toml +++ b/codex-rs/exec-server-protocol/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] base64 = { workspace = true } codex-file-system = { workspace = true } +codex-install-context = { workspace = true } codex-network-proxy = { workspace = true } codex-protocol = { workspace = true } codex-shell-command = { workspace = true } diff --git a/codex-rs/exec-server-protocol/src/protocol.rs b/codex-rs/exec-server-protocol/src/protocol.rs index 84b266432fa0..97402d5e1260 100644 --- a/codex-rs/exec-server-protocol/src/protocol.rs +++ b/codex-rs/exec-server-protocol/src/protocol.rs @@ -6,6 +6,7 @@ use codex_file_system::FileSystemSandboxContext; pub use codex_file_system::WalkOptions; pub use codex_file_system::WalkOutcome; use codex_file_system::WireFileSystemSandboxContext; +use codex_install_context::InstallContext; use codex_network_proxy::ManagedNetworkSandboxContext; use codex_network_proxy::RemoteNetworkProxyLaunchConfig; use codex_protocol::ThreadId; @@ -116,6 +117,9 @@ pub struct EnvironmentInfo { /// Operating system reported by the executor; absent for legacy exec-servers. #[serde(default, skip_serializing_if = "Option::is_none")] pub platform_os: Option, + /// Executor directories to prepend to `PATH` when missing, in priority order. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub prepend_path_dirs: Vec, /// Executor-local default directories for resolving `:tmpdir`, when reported. /// On Windows, a command's `TEMP` or `TMP` overrides take precedence. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -252,6 +256,13 @@ impl EnvironmentInfo { cwd: cwd.and_then(|cwd| PathUri::from_host_native_path(cwd).ok()), user_home_dir: PathUri::from_host_native_path("~").ok(), platform_os: Some(std::env::consts::OS.to_string()), + prepend_path_dirs: InstallContext::current() + .package_layout + .as_ref() + .and_then(|layout| layout.path_dir.as_ref()) + .into_iter() + .map(PathUri::from_abs_path) + .collect(), temporary_directories: Some(temporary_directories), temp_dir, capabilities: EnvironmentCapabilities { @@ -1372,6 +1383,7 @@ mod tests { cwd: None, user_home_dir: None, platform_os: None, + prepend_path_dirs: Vec::new(), temporary_directories: None, temp_dir: None, capabilities: EnvironmentCapabilities::default(), @@ -1447,6 +1459,7 @@ mod tests { "cwd": null, "userHomeDir": "file:///C:/Users/remote", "platformOs": "windows", + "prependPathDirs": ["file:///C:/tools/bin", "file:///D:/tools/bin"], "temporaryDirectories": ["file:///C:/Temp", "file:///D:/Temp"], "capabilities": { "networkProxyLaunch": false, diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index 3c78a5021c49..25500aaaf5fb 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -95,6 +95,7 @@ anyhow = { workspace = true } codex-exec-server-test-support = { workspace = true } codex-test-binary-support = { workspace = true } codex-utils-cargo-bin = { workspace = true } +codex-windows-sandbox-test-support = { workspace = true } ctor = { workspace = true } http = { workspace = true } opentelemetry_sdk = { workspace = true } diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index 538bb214fc19..1192a5b8f366 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -840,11 +840,25 @@ impl ExecServerClient { // TODO: Remove after app-server migrates off this call. pub async fn force_environment_info(&self) -> Result { let rpc_client = self.rpc_client().await?; - self.map_rpc_call_result( - rpc_client - .call_with_timeout(ENVIRONMENT_INFO_METHOD, &(), ENVIRONMENT_INFO_TIMEOUT) - .await, + // Bound sending as well as receiving: a stuck transport can fill the outbound queue. + let result = timeout( + ENVIRONMENT_INFO_TIMEOUT, + rpc_client.call(ENVIRONMENT_INFO_METHOD, &()), ) + .await; + match result { + Ok(result) => self.map_rpc_call_result(result), + Err(_) => { + let error = ExecServerError::from(RpcCallError::TimedOut { + method: ENVIRONMENT_INFO_METHOD.to_string(), + timeout: ENVIRONMENT_INFO_TIMEOUT, + }); + // Retire only the connection we probed; recovery ignores a stale client. + rpc_client.close_transport().await; + self.inner.request_recovery(rpc_client, error.to_string()); + Err(error) + } + } } pub async fn read_environment_config( @@ -2661,8 +2675,10 @@ mod tests { Ok(()) } + #[test_case::test_case(false; "socket_closed")] + #[test_case::test_case(true; "health_check_timed_out")] #[tokio::test] - async fn remote_websocket_client_resumes_session() { + async fn remote_websocket_client_resumes_session(health_check_timeout: bool) { let listener = TcpListener::bind("127.0.0.1:0") .await .expect("listener should bind"); @@ -2680,7 +2696,22 @@ mod tests { /*expected_resume_session_id*/ None, ) .await; - first.close(None).await.expect("websocket should close"); + if health_check_timeout { + let JSONRPCMessage::Request(request) = read_jsonrpc_websocket(&mut first).await + else { + panic!("expected environment info request"); + }; + assert_eq!(request.method, "environment/info"); + // Keep the socket open without answering. The client must close it itself. + while let Some(Ok(message)) = first.next().await { + assert!(matches!(message, Message::Ping(_) | Message::Close(_))); + if matches!(message, Message::Close(_)) { + break; + } + } + } else { + first.close(None).await.expect("websocket should close"); + } let mut resumed = accept_websocket(&listener).await; complete_websocket_initialize( @@ -2703,6 +2734,13 @@ mod tests { HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), ); let stable_client = client.get().await.expect("client should connect"); + if health_check_timeout { + let error = stable_client + .force_environment_info() + .await + .expect_err("unanswered health check should time out"); + assert!(error.to_string().contains("timed out")); + } timeout(Duration::from_secs(1), resumed_rx) .await .expect("session resume should not time out") diff --git a/codex-rs/exec-server/tests/file_system_windows.rs b/codex-rs/exec-server/tests/file_system_windows.rs index 1b9830a1ce2b..8193b570ea08 100644 --- a/codex-rs/exec-server/tests/file_system_windows.rs +++ b/codex-rs/exec-server/tests/file_system_windows.rs @@ -33,6 +33,7 @@ use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::SandboxPolicy; use codex_sandboxing::SandboxType; use codex_utils_path_uri::PathUri; +use codex_windows_sandbox_test_support::WindowsSandboxAccountTestGuard; use futures::TryStreamExt; use pretty_assertions::assert_eq; use test_case::test_case; @@ -456,6 +457,7 @@ async fn file_system_remote_fs_helper_respects_windows_sandbox_write_policy( async fn file_system_elevated_relative_read_denial_uses_policy_cwd( implementation: FileSystemImplementation, ) -> Result<()> { + let _account_guard = WindowsSandboxAccountTestGuard::acquire()?; // Both implementations re-enter this test binary; the elevated backend finds its helpers // next to that binary, while Cargo and Bazel provide them separately. let test_exe = std::env::current_exe()?; diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index db35cd4d448a..7f61ef3aee9a 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -96,6 +96,8 @@ pub enum Feature { AnalyticsPlanHistory, /// Discover model catalogs for OpenAI API-key authentication. ApiKeyModelDiscovery, + /// Forward explicit programs with builtin OpenAI API keys. + ApiKeyCyberAccessPrograms, /// Deprecated no-op; use `tui.fullscreen_transcript` instead. TranscriptV2, // Stable. @@ -168,6 +170,8 @@ pub enum Feature { UseLegacyLandlock, /// Experimental shell snapshotting. ShellSnapshot, + /// Restore bundled tools to PATH after Codex's login shell starts. + LoginShellPackagePath, /// Expose the selected PowerShell execution host's bounded major/minor version. PowerShellShellVersion, /// Keep policy-filtered shell snapshots entirely in executor memory. @@ -1018,6 +1022,16 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::LoginShellPackagePath, + key: "login_shell_package_path", + stage: Stage::Experimental { + name: "Bundled tools in login shells", + menu_description: "Keep bundled tools such as ripgrep available when login shell startup resets PATH.", + announcement: "", + }, + default_enabled: false, + }, FeatureSpec { id: Feature::PowerShellShellVersion, key: "powershell_shell_version", @@ -1288,6 +1302,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::ApiKeyCyberAccessPrograms, + key: "api_key_cyber_access_programs", + stage: Stage::Stable, + default_enabled: false, + }, FeatureSpec { id: Feature::EnableRequestCompression, key: "enable_request_compression", diff --git a/codex-rs/hooks/src/engine/discovery.rs b/codex-rs/hooks/src/engine/discovery.rs index 5003dbd24f08..0c5a28ef35ef 100644 --- a/codex-rs/hooks/src/engine/discovery.rs +++ b/codex-rs/hooks/src/engine/discovery.rs @@ -29,8 +29,8 @@ use super::HookListEntry; use super::HookListEntryHandler; use super::dispatcher::hook_event_name_label; use crate::config_rules::hook_states_from_stack; +use crate::engine::HookMatcher; use crate::events::common::matcher_pattern_for_event; -use crate::events::common::validate_matcher_pattern; use crate::events::session_end::SESSION_END_DEFAULT_TIMEOUT_SEC; use crate::events::session_end::SESSION_END_MAX_TIMEOUT_SEC; use crate::output_spill::AdditionalContextLimit; @@ -486,20 +486,22 @@ fn append_matcher_groups( ) { for (group_index, group) in groups.into_iter().enumerate() { let matcher = matcher_pattern_for_event(event_name, group.matcher.as_deref()); - if let Some(matcher) = matcher - && let Err(err) = validate_matcher_pattern(matcher) - { - let warning = format!( - "invalid matcher {matcher:?} in {}: {err}", - source.path.display() - ); - if group.hooks.is_empty() { - warnings.push(warning); - } else { - source.record_load_failure(warning, warnings); + let compiled_matcher = match matcher.map(HookMatcher::new).transpose() { + Ok(matcher) => matcher, + Err(err) => { + let matcher = matcher.unwrap_or_default(); + let warning = format!( + "invalid matcher {matcher:?} in {}: {err}", + source.path.display() + ); + if group.hooks.is_empty() { + warnings.push(warning); + } else { + source.record_load_failure(warning, warnings); + } + continue; } - continue; - } + }; for (handler_index, handler) in group.hooks.iter().cloned().enumerate() { let normalized = match handler { HookHandlerConfig::Command { @@ -720,7 +722,7 @@ fn append_matcher_groups( handlers.push(ConfiguredHandler { builtin, event_name, - matcher: matcher.map(ToOwned::to_owned), + matcher: compiled_matcher.clone(), timeout_sec, status_message, additional_context_limit: AdditionalContextLimit::from_config( @@ -862,6 +864,7 @@ fn hook_source_for_requirement_source(source: Option<&RequirementSource>) -> Hoo #[cfg(test)] mod tests { + use super::HookMatcher; use codex_config::ConfigLayerEntry; use codex_config::ConfigLayerSource; use codex_config::HookEventsToml; @@ -1296,7 +1299,7 @@ mod tests { vec![ConfiguredHandler { builtin: false, event_name: HookEventName::PreToolUse, - matcher: Some("^Bash$".to_string()), + matcher: Some(HookMatcher::new("^Bash$").expect("valid matcher")), timeout_sec: 600, status_message: None, additional_context_limit: Default::default(), @@ -1366,7 +1369,7 @@ mod tests { assert_eq!( handlers .iter() - .map(|handler| handler.matcher.as_deref()) + .map(|handler| handler.matcher.as_ref().map(HookMatcher::as_str)) .collect::>(), vec![Some("other"), Some("other")] ); @@ -1446,7 +1449,7 @@ mod tests { .iter() .map(|handler| ( handler.timeout_sec, - handler.matcher.as_deref(), + handler.matcher.as_ref().map(HookMatcher::as_str), handler.execution_mode() )) .collect::>(), @@ -1558,7 +1561,10 @@ mod tests { assert_eq!(warnings, Vec::::new()); assert_eq!(handlers.len(), 1); - assert_eq!(handlers[0].matcher.as_deref(), Some("*")); + assert_eq!( + handlers[0].matcher.as_ref().map(HookMatcher::as_str), + Some("*") + ); } #[test] @@ -1582,7 +1588,10 @@ mod tests { assert_eq!(warnings, Vec::::new()); assert_eq!(handlers.len(), 1); assert_eq!(handlers[0].event_name, HookEventName::PostToolUse); - assert_eq!(handlers[0].matcher.as_deref(), Some("Edit|Write")); + assert_eq!( + handlers[0].matcher.as_ref().map(HookMatcher::as_str), + Some("Edit|Write") + ); } #[test] diff --git a/codex-rs/hooks/src/engine/dispatcher.rs b/codex-rs/hooks/src/engine/dispatcher.rs index c5ee0d2ce1da..b34118d6f2b8 100644 --- a/codex-rs/hooks/src/engine/dispatcher.rs +++ b/codex-rs/hooks/src/engine/dispatcher.rs @@ -60,11 +60,11 @@ pub(crate) fn select_handlers_for_matcher_inputs( | HookEventName::PreCompact | HookEventName::PostCompact => { if matcher_inputs.is_empty() { - matches_matcher(handler.matcher.as_deref(), /*input*/ None) + matches_matcher(handler.matcher.as_ref(), /*input*/ None) } else { matcher_inputs .iter() - .any(|input| matches_matcher(handler.matcher.as_deref(), Some(input))) + .any(|input| matches_matcher(handler.matcher.as_ref(), Some(input))) } } HookEventName::UserPromptSubmit | HookEventName::Stop | HookEventName::Interrupt => { @@ -352,7 +352,8 @@ mod tests { ConfiguredHandler { builtin: false, event_name, - matcher: matcher.map(str::to_owned), + matcher: matcher + .map(|pattern| crate::engine::HookMatcher::new(pattern).expect("valid matcher")), timeout_sec: 5, status_message: None, additional_context_limit: Default::default(), @@ -609,7 +610,7 @@ mod tests { ), make_handler( HookEventName::UserPromptSubmit, - Some("["), + Some("^unmatched$"), "echo second", /*display_order*/ 1, ), diff --git a/codex-rs/hooks/src/engine/matcher.rs b/codex-rs/hooks/src/engine/matcher.rs new file mode 100644 index 000000000000..3cd95aacd1e1 --- /dev/null +++ b/codex-rs/hooks/src/engine/matcher.rs @@ -0,0 +1,49 @@ +//! Compiled hook matchers retain their original spelling for configuration identity. +//! Regex parsing happens during discovery; dispatch only evaluates admitted matchers. + +#[derive(Debug, Clone)] +pub(crate) enum HookMatcher { + All(String), + Exact(String), + Regex(regex::Regex), +} + +impl HookMatcher { + pub(crate) fn new(pattern: &str) -> Result { + if pattern.is_empty() || pattern == "*" { + Ok(Self::All(pattern.to_string())) + } else if pattern + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '|') + { + Ok(Self::Exact(pattern.to_string())) + } else { + regex::Regex::new(pattern).map(Self::Regex) + } + } + + pub(crate) fn as_str(&self) -> &str { + match self { + Self::All(pattern) | Self::Exact(pattern) => pattern, + Self::Regex(regex) => regex.as_str(), + } + } + + pub(crate) fn matches(&self, input: Option<&str>) -> bool { + match self { + Self::All(_) => true, + Self::Exact(pattern) => { + input.is_some_and(|input| pattern.split('|').any(|candidate| candidate == input)) + } + Self::Regex(regex) => input.is_some_and(|input| regex.is_match(input)), + } + } +} + +impl PartialEq for HookMatcher { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Eq for HookMatcher {} diff --git a/codex-rs/hooks/src/engine/mod.rs b/codex-rs/hooks/src/engine/mod.rs index deaf29679a8c..a007f3b1a459 100644 --- a/codex-rs/hooks/src/engine/mod.rs +++ b/codex-rs/hooks/src/engine/mod.rs @@ -1,10 +1,13 @@ pub(crate) mod command_runner; pub(crate) mod discovery; pub(crate) mod dispatcher; +mod matcher; pub(crate) mod mcp_runner; pub(crate) mod output_parser; pub(crate) mod schema_loader; +pub(crate) use matcher::HookMatcher; + use crate::events::compact::PostCompactRequest; use crate::events::compact::PreCompactOutcome; use crate::events::compact::PreCompactRequest; @@ -60,7 +63,7 @@ pub(crate) struct ConfiguredHandler { /// Internally admitted cleanup hook, enabled independently of per-hook state. pub builtin: bool, pub event_name: codex_protocol::protocol::HookEventName, - pub matcher: Option, + pub matcher: Option, pub timeout_sec: u64, pub status_message: Option, pub additional_context_limit: AdditionalContextLimit, diff --git a/codex-rs/hooks/src/events/common.rs b/codex-rs/hooks/src/events/common.rs index 7b4c4dcab31e..c6e1a1d324cd 100644 --- a/codex-rs/hooks/src/events/common.rs +++ b/codex-rs/hooks/src/events/common.rs @@ -127,28 +127,11 @@ pub(crate) fn matcher_pattern_for_event( } } -pub(crate) fn validate_matcher_pattern(matcher: &str) -> Result<(), regex::Error> { - if is_match_all_matcher(matcher) || is_exact_matcher(matcher) { - return Ok(()); - } - regex::Regex::new(matcher).map(|_| ()) -} - -pub(crate) fn matches_matcher(matcher: Option<&str>, input: Option<&str>) -> bool { - match matcher { - None => true, - Some(matcher) if is_match_all_matcher(matcher) => true, - Some(matcher) if is_exact_matcher(matcher) => input - .map(|input| matcher.split('|').any(|candidate| candidate == input)) - .unwrap_or(false), - Some(matcher) => input - .and_then(|input| { - regex::Regex::new(matcher) - .ok() - .map(|regex| regex.is_match(input)) - }) - .unwrap_or(false), - } +pub(crate) fn matches_matcher( + matcher: Option<&crate::engine::HookMatcher>, + input: Option<&str>, +) -> bool { + matcher.is_none_or(|matcher| matcher.matches(input)) } pub(crate) fn matcher_inputs<'a>( @@ -162,16 +145,6 @@ pub(crate) fn matcher_inputs<'a>( .collect() } -fn is_match_all_matcher(matcher: &str) -> bool { - matcher.is_empty() || matcher == "*" -} - -fn is_exact_matcher(matcher: &str) -> bool { - matcher - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '|') -} - #[cfg(test)] mod tests { use codex_protocol::protocol::HookEventName; @@ -179,7 +152,7 @@ mod tests { use super::matcher_pattern_for_event; use super::matches_matcher; - use super::validate_matcher_pattern; + use crate::engine::HookMatcher; #[test] fn matcher_omitted_matches_all_occurrences() { @@ -189,75 +162,110 @@ mod tests { #[test] fn matcher_star_matches_all_occurrences() { - assert!(matches_matcher(Some("*"), Some("Bash"))); - assert!(matches_matcher(Some("*"), Some("Edit"))); - assert_eq!(validate_matcher_pattern("*"), Ok(())); + assert!(matches_matcher( + Some(&HookMatcher::new("*").expect("valid matcher")), + Some("Bash") + )); + assert!(matches_matcher( + Some(&HookMatcher::new("*").expect("valid matcher")), + Some("Edit") + )); + assert_eq!(HookMatcher::new("*").map(|_| ()), Ok(())); } #[test] fn matcher_empty_string_matches_all_occurrences() { - assert!(matches_matcher(Some(""), Some("Bash"))); - assert!(matches_matcher(Some(""), Some("SessionStart"))); - assert_eq!(validate_matcher_pattern(""), Ok(())); + assert!(matches_matcher( + Some(&HookMatcher::new("").expect("valid matcher")), + Some("Bash") + )); + assert!(matches_matcher( + Some(&HookMatcher::new("").expect("valid matcher")), + Some("SessionStart") + )); + assert_eq!(HookMatcher::new("").map(|_| ()), Ok(())); } #[test] fn exact_matcher_supports_pipe_alternatives() { - assert!(matches_matcher(Some("Edit|Write"), Some("Edit"))); - assert!(matches_matcher(Some("Edit|Write"), Some("Write"))); - assert!(!matches_matcher(Some("Edit|Write"), Some("Bash"))); - assert_eq!(validate_matcher_pattern("Edit|Write"), Ok(())); + assert!(matches_matcher( + Some(&HookMatcher::new("Edit|Write").expect("valid matcher")), + Some("Edit") + )); + assert!(matches_matcher( + Some(&HookMatcher::new("Edit|Write").expect("valid matcher")), + Some("Write") + )); + assert!(!matches_matcher( + Some(&HookMatcher::new("Edit|Write").expect("valid matcher")), + Some("Bash") + )); + assert_eq!(HookMatcher::new("Edit|Write").map(|_| ()), Ok(())); } #[test] fn literal_matcher_uses_exact_matching() { - assert!(matches_matcher(Some("Bash"), Some("Bash"))); - assert!(!matches_matcher(Some("Bash"), Some("BashOutput"))); assert!(matches_matcher( - Some("mcp__memory__create_entities"), + Some(&HookMatcher::new("Bash").expect("valid matcher")), + Some("Bash") + )); + assert!(!matches_matcher( + Some(&HookMatcher::new("Bash").expect("valid matcher")), + Some("BashOutput") + )); + assert!(matches_matcher( + Some(&HookMatcher::new("mcp__memory__create_entities").expect("valid matcher")), Some("mcp__memory__create_entities") )); assert!(!matches_matcher( - Some("mcp__memory"), + Some(&HookMatcher::new("mcp__memory").expect("valid matcher")), Some("mcp__memory__create_entities") )); - assert_eq!(validate_matcher_pattern("mcp__memory"), Ok(())); + assert_eq!(HookMatcher::new("mcp__memory").map(|_| ()), Ok(())); } #[test] fn matcher_uses_regex_when_it_contains_regex_characters() { - assert!(matches_matcher(Some("^Bash"), Some("BashOutput"))); - assert_eq!(validate_matcher_pattern("^Bash"), Ok(())); + assert!(matches_matcher( + Some(&HookMatcher::new("^Bash").expect("valid matcher")), + Some("BashOutput") + )); + assert_eq!(HookMatcher::new("^Bash").map(|_| ()), Ok(())); } #[test] fn mcp_matchers_support_regex_wildcards() { assert!(matches_matcher( - Some("mcp__memory__.*"), + Some(&HookMatcher::new("mcp__memory__.*").expect("valid matcher")), Some("mcp__memory__create_entities") )); assert!(matches_matcher( - Some("mcp__.*__write.*"), + Some(&HookMatcher::new("mcp__.*__write.*").expect("valid matcher")), Some("mcp__filesystem__write_file") )); assert!(!matches_matcher( - Some("mcp__.*__write.*"), + Some(&HookMatcher::new("mcp__.*__write.*").expect("valid matcher")), Some("mcp__filesystem__read_file") )); - assert_eq!(validate_matcher_pattern("mcp__memory__.*"), Ok(())); + assert_eq!(HookMatcher::new("mcp__memory__.*").map(|_| ()), Ok(())); } #[test] fn matcher_supports_anchored_regexes() { - assert!(matches_matcher(Some("^Bash$"), Some("Bash"))); - assert!(!matches_matcher(Some("^Bash$"), Some("BashOutput"))); - assert_eq!(validate_matcher_pattern("^Bash$"), Ok(())); + assert!(matches_matcher( + Some(&HookMatcher::new("^Bash$").expect("valid matcher")), + Some("Bash") + )); + assert!(!matches_matcher( + Some(&HookMatcher::new("^Bash$").expect("valid matcher")), + Some("BashOutput") + )); + assert_eq!(HookMatcher::new("^Bash$").map(|_| ()), Ok(())); } #[test] fn invalid_regex_is_rejected() { - assert!(validate_matcher_pattern("[").is_err()); - assert!(!matches_matcher(Some("["), Some("Bash"))); + assert!(HookMatcher::new("[").is_err()); } #[test] diff --git a/codex-rs/hooks/src/events/post_tool_use.rs b/codex-rs/hooks/src/events/post_tool_use.rs index aeb0ed20a7a1..0ca0f6c673d8 100644 --- a/codex-rs/hooks/src/events/post_tool_use.rs +++ b/codex-rs/hooks/src/events/post_tool_use.rs @@ -591,7 +591,7 @@ mod tests { ConfiguredHandler { builtin: false, event_name: HookEventName::PostToolUse, - matcher: Some("^Bash$".to_string()), + matcher: Some(crate::engine::HookMatcher::new("^Bash$").expect("valid matcher")), timeout_sec: 5, status_message: Some("running post tool use hook".to_string()), additional_context_limit: Default::default(), diff --git a/codex-rs/hooks/src/events/pre_tool_use.rs b/codex-rs/hooks/src/events/pre_tool_use.rs index c1baebd4f01f..e57b823a1c86 100644 --- a/codex-rs/hooks/src/events/pre_tool_use.rs +++ b/codex-rs/hooks/src/events/pre_tool_use.rs @@ -775,7 +775,7 @@ mod tests { ConfiguredHandler { builtin: false, event_name: HookEventName::PreToolUse, - matcher: Some("^Bash$".to_string()), + matcher: Some(crate::engine::HookMatcher::new("^Bash$").expect("valid matcher")), timeout_sec: 5, status_message: None, additional_context_limit: Default::default(), diff --git a/codex-rs/hooks/src/events/session_end_tests.rs b/codex-rs/hooks/src/events/session_end_tests.rs index 0c100ee4a4c6..05fa107d4fa9 100644 --- a/codex-rs/hooks/src/events/session_end_tests.rs +++ b/codex-rs/hooks/src/events/session_end_tests.rs @@ -62,7 +62,8 @@ fn handler(matcher: Option<&str>) -> ConfiguredHandler { ConfiguredHandler { builtin: false, event_name: HookEventName::SessionEnd, - matcher: matcher.map(str::to_string), + matcher: matcher + .map(|pattern| crate::engine::HookMatcher::new(pattern).expect("valid matcher")), timeout_sec: 2, status_message: None, additional_context_limit: Default::default(), diff --git a/codex-rs/keyring-store/src/error_display_tests.rs b/codex-rs/keyring-store/src/error_display_tests.rs new file mode 100644 index 000000000000..a7ea7f244f7b --- /dev/null +++ b/codex-rs/keyring-store/src/error_display_tests.rs @@ -0,0 +1,51 @@ +//! Ensures credential payloads cannot escape through diagnostic error formatting. + +use super::*; +use keyring::credential::CredentialApi; +use pretty_assertions::assert_eq; + +#[test] +fn payload_bearing_errors_redact_credentials_and_end_the_source_chain() { + let credential = keyring::mock::MockCredential::default(); + credential.set_password("private-credential-value").unwrap(); + for (error, message) in [ + ( + KeyringError::Ambiguous(vec![Box::new(credential) as Box]), + "Entry is matched by 1 credentials", + ), + ( + KeyringError::BadEncoding(b"private-credential-value".to_vec()), + "Data is not UTF-8 encoded", + ), + ] { + let error = CredentialStoreError::new(error); + assert_eq!( + ( + error.to_string(), + error.message(), + format!("{error:?}"), + error.source().is_none() + ), + ( + message.to_string(), + message.to_string(), + message.to_string(), + true + ), + ); + } +} + +#[test] +fn debug_preserves_native_error_details() { + let error = CredentialStoreError::new(KeyringError::PlatformFailure(Box::new( + std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "permission denied by keyring", + ), + ))); + assert_eq!( + format!("{error:?}"), + "Platform secure storage failure: permission denied by keyring", + ); +} diff --git a/codex-rs/keyring-store/src/lib.rs b/codex-rs/keyring-store/src/lib.rs index 3b0a1c414fe2..bd38bc6ecd75 100644 --- a/codex-rs/keyring-store/src/lib.rs +++ b/codex-rs/keyring-store/src/lib.rs @@ -5,7 +5,6 @@ use std::fmt; use std::fmt::Debug; use tracing::trace; -#[derive(Debug)] pub enum CredentialStoreError { Other(KeyringError), } @@ -16,9 +15,7 @@ impl CredentialStoreError { } pub fn message(&self) -> String { - match self { - Self::Other(error) => error.to_string(), - } + self.to_string() } pub fn into_error(self) -> KeyringError { @@ -31,19 +28,34 @@ impl CredentialStoreError { impl fmt::Display for CredentialStoreError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Other(KeyringError::Ambiguous(items)) => { + write!(f, "Entry is matched by {} credentials", items.len()) + } Self::Other(error) => write!(f, "{error}"), } } } +impl fmt::Debug for CredentialStoreError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + impl Error for CredentialStoreError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { + // Do not expose payload-bearing variants to error-chain formatters. + Self::Other(KeyringError::Ambiguous(_) | KeyringError::BadEncoding(_)) => None, Self::Other(error) => Some(error), } } } +#[cfg(test)] +#[path = "error_display_tests.rs"] +mod error_display_tests; + mod error_kind; impl From for std::io::Error { diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index 8772f46b5867..7762770be8c9 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] codex-utils-process = { workspace = true } +anyhow = { workspace = true } base64 = { workspace = true } chrono = { workspace = true, features = ["serde"] } codex-agent-identity = { workspace = true } @@ -44,7 +45,7 @@ urlencoding = { workspace = true } webbrowser = { workspace = true } [dev-dependencies] -anyhow = { workspace = true } +opentelemetry_sdk = { workspace = true, features = ["testing"] } core_test_support = { workspace = true } jsonwebtoken = { workspace = true } keyring = { workspace = true } diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 14cfee92212f..cdc9c8d35603 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -976,8 +976,8 @@ fn read_non_empty_env_var(key: &str) -> Option { .filter(|value| !value.is_empty()) } -/// Delete the auth.json file inside `codex_home` if it exists. Returns `Ok(true)` -/// if a file was removed, `Ok(false)` if no auth file was present. +/// Delete credentials for `codex_home` through the selected storage backend. +/// Returns `Ok(true)` if credentials were removed, `Ok(false)` if none were present. pub fn logout( codex_home: &Path, auth_credentials_store_mode: AuthCredentialsStoreMode, @@ -1018,7 +1018,7 @@ pub async fn logout_with_revoke( ) } -/// Writes an `auth.json` that contains only the API key. +/// Store API-key credentials through the selected storage backend. pub fn login_with_api_key( codex_home: &Path, api_key: &str, @@ -1043,7 +1043,8 @@ pub fn login_with_api_key( ) } -/// Writes an `auth.json` that contains only the access token. +/// Validate and store personal-access-token or Agent Identity credentials through +/// the selected storage backend. pub async fn login_with_access_token( codex_home: &Path, access_token: &str, @@ -1454,7 +1455,7 @@ fn logout_with_message( ); let error_message = match removal_result { Ok(_) => message, - Err(err) => format!("{message}. Failed to remove auth.json: {err}"), + Err(err) => format!("{message}. Failed to remove stored credentials: {err}"), }; Err(std::io::Error::other(error_message)) } @@ -2031,12 +2032,12 @@ impl UnauthorizedRecovery { } } -/// Central manager providing a single source of truth for auth.json derived +/// Central manager providing a single source of truth for stored /// authentication data. It loads once (or on preference change) and then /// hands out cloned `CodexAuth` values so the rest of the program has a /// consistent snapshot. /// -/// External modifications to `auth.json` will NOT be observed until +/// External modifications to stored credentials will NOT be observed until /// `reload()` is called explicitly. This matches the design goal of avoiding /// different parts of the program seeing inconsistent auth data mid‑run. pub struct AuthManager { @@ -2135,10 +2136,7 @@ fn default_agent_identity_authapi_base_url() -> Option { impl AuthManager { /// Returns the application policy associated with this account owner. pub fn application_network_policy(&self) -> codex_http_client::NetworkPolicy { - self.auth_route_config - .http_client_factory() - .network_policy() - .clone() + self.auth_route_config.application_network_policy().clone() } /// Creates content clients bound to the account currently owned by this manager. @@ -2636,8 +2634,7 @@ impl AuthManager { auth_changed_for_refresh && !same_owner(previous, new_auth.as_ref()); if owner_changed { self.auth_route_config - .http_client_factory() - .network_policy() + .application_network_policy() .invalidate(); } if auth_changed_for_refresh { @@ -2939,10 +2936,10 @@ impl AuthManager { result } - /// Log out by deleting the on‑disk auth.json (if present). Returns Ok(true) - /// if a file was removed, Ok(false) if no auth file existed. On success, - /// reloads the in‑memory auth cache so callers immediately observe the - /// unauthenticated state. + /// Log out by deleting credentials from the selected storage backend and the + /// ephemeral store. Returns `Ok(true)` if credentials were removed, `Ok(false)` + /// if none were present. On success, reloads the in-memory auth cache so + /// callers immediately observe the unauthenticated state. pub async fn logout(&self) -> std::io::Result { self.ensure_logout_allowed()?; let removed = logout_all_stores( @@ -2950,7 +2947,7 @@ impl AuthManager { self.auth_credentials_store_mode, self.keyring_backend_kind, )?; - // Always reload to clear any cached auth (even if file absent). + // Always reload to clear cached auth, even if no stored credentials were present. self.clear_external_auth(); self.reload().await; Ok(removed) @@ -2970,7 +2967,7 @@ impl AuthManager { self.auth_credentials_store_mode, self.keyring_backend_kind, )?; - // Always reload to clear any cached auth (even if file absent). + // Always reload to clear cached auth, even if no stored credentials were present. self.clear_external_auth(); self.reload().await; Ok(result) diff --git a/codex-rs/login/src/auth/storage.rs b/codex-rs/login/src/auth/storage.rs index 8a9d75f44876..4dccbc9138e8 100644 --- a/codex-rs/login/src/auth/storage.rs +++ b/codex-rs/login/src/auth/storage.rs @@ -1,5 +1,7 @@ use chrono::DateTime; use chrono::Utc; +use codex_otel::auth_storage::Operation; +use codex_otel::auth_storage::Store; use serde::Deserialize; use serde::Serialize; use sha2::Digest; @@ -18,6 +20,12 @@ use std::sync::Arc; use std::sync::Mutex; use tracing::warn; +#[path = "storage_error.rs"] +mod storage_error; + +#[path = "storage_telemetry.rs"] +mod storage_telemetry; + use super::BedrockAccessKeysAuth; use super::BedrockApiKeyAuth; use crate::token_data::TokenData; @@ -251,44 +259,50 @@ fn compute_store_key(codex_home: &Path) -> std::io::Result { #[derive(Clone, Debug)] struct DirectKeyringAuthStorage { codex_home: PathBuf, + mode: AuthCredentialsStoreMode, keyring_store: Arc, } impl DirectKeyringAuthStorage { - fn new(codex_home: PathBuf, keyring_store: Arc) -> Self { + fn new( + codex_home: PathBuf, + keyring_store: Arc, + mode: AuthCredentialsStoreMode, + ) -> Self { Self { codex_home, keyring_store, + mode, } } fn load_from_keyring(&self, key: &str) -> std::io::Result> { - match self.keyring_store.load(KEYRING_SERVICE, key) { - Ok(Some(serialized)) => serde_json::from_str(&serialized).map(Some).map_err(|err| { - std::io::Error::other(format!( - "failed to deserialize CLI auth from keyring: {err}" - )) + match self + .keyring_store + .load(KEYRING_SERVICE, key) + .map_err(std::io::Error::from) + .map_err(|error| { + storage_error::with_context("failed to load CLI auth from keyring", error) + })? { + Some(serialized) => serde_json::from_str(&serialized).map(Some).map_err(|err| { + storage_error::with_context("failed to deserialize CLI auth from keyring", err) }), - Ok(None) => Ok(None), - Err(error) => Err(std::io::Error::other(format!( - "failed to load CLI auth from keyring: {}", - error.message() - ))), + None => Ok(None), } } fn save_to_keyring(&self, key: &str, value: &str) -> std::io::Result<()> { - match self.keyring_store.save(KEYRING_SERVICE, key, value) { - Ok(()) => Ok(()), - Err(error) => { - let message = format!( - "failed to write OAuth tokens to keyring: {}", - error.message() - ); - warn!("{message}"); - Err(std::io::Error::other(message)) - } - } + self.keyring_store + .save(KEYRING_SERVICE, key, value) + .map_err(std::io::Error::from) + .map_err(|error| { + let error = + storage_error::with_context("failed to write OAuth tokens to keyring", error); + if self.mode == AuthCredentialsStoreMode::Keyring { + warn!("{error}"); + } + error + }) } } @@ -303,9 +317,17 @@ impl AuthStorageBackend for DirectKeyringAuthStorage { // Simpler error mapping per style: prefer method reference over closure let serialized = serde_json::to_string(auth).map_err(std::io::Error::other)?; self.save_to_keyring(&key, &serialized)?; - if let Err(err) = delete_file_if_exists(&self.codex_home) { + let mut telemetry = storage_telemetry::telemetry( + self.mode, + AuthKeyringBackendKind::Direct, + Operation::Cleanup, + ); + let result = delete_file_if_exists(&self.codex_home); + telemetry.record_delete_attempt(Store::File, &result); + if let Err(err) = &result { warn!("failed to remove CLI auth fallback file: {err}"); } + drop(telemetry); Ok(()) } @@ -314,8 +336,9 @@ impl AuthStorageBackend for DirectKeyringAuthStorage { let keyring_removed = self .keyring_store .delete(KEYRING_SERVICE, &key) - .map_err(|err| { - std::io::Error::other(format!("failed to delete auth from keyring: {err}")) + .map_err(std::io::Error::from) + .map_err(|error| { + storage_error::with_context("failed to delete auth from keyring", error) })?; let file_removed = delete_file_if_exists(&self.codex_home)?; Ok(keyring_removed || file_removed) @@ -325,6 +348,7 @@ impl AuthStorageBackend for DirectKeyringAuthStorage { #[derive(Clone)] struct SecretsKeyringAuthStorage { codex_home: PathBuf, + mode: AuthCredentialsStoreMode, direct_storage: DirectKeyringAuthStorage, secrets_manager: SecretsManager, } @@ -338,9 +362,13 @@ impl Debug for SecretsKeyringAuthStorage { } impl SecretsKeyringAuthStorage { - fn new(codex_home: PathBuf, keyring_store: Arc) -> Self { + fn new( + codex_home: PathBuf, + keyring_store: Arc, + mode: AuthCredentialsStoreMode, + ) -> Self { let direct_storage = - DirectKeyringAuthStorage::new(codex_home.clone(), Arc::clone(&keyring_store)); + DirectKeyringAuthStorage::new(codex_home.clone(), Arc::clone(&keyring_store), mode); let secrets_manager = SecretsManager::new_with_keyring_store_and_namespace( codex_home.clone(), SecretsBackendKind::Local, @@ -349,6 +377,7 @@ impl SecretsKeyringAuthStorage { ); Self { codex_home, + mode, direct_storage, secrets_manager, } @@ -360,15 +389,17 @@ impl AuthStorageBackend for SecretsKeyringAuthStorage { match self .secrets_manager .get(&SecretScope::Global, &CODEX_AUTH_SECRET_NAME) - .map_err(|err| { - std::io::Error::other(format!( - "failed to load CLI auth from encrypted auth storage: {err}" - )) + .map_err(|error| { + storage_error::with_context( + "failed to load CLI auth from encrypted auth storage", + error, + ) })? { Some(serialized) => serde_json::from_str(&serialized).map(Some).map_err(|err| { - std::io::Error::other(format!( - "failed to deserialize CLI auth from encrypted auth storage: {err}" - )) + storage_error::with_context( + "failed to deserialize CLI auth from encrypted auth storage", + err, + ) }), None => Ok(None), } @@ -378,15 +409,27 @@ impl AuthStorageBackend for SecretsKeyringAuthStorage { let serialized = serde_json::to_string(auth).map_err(std::io::Error::other)?; self.secrets_manager .set(&SecretScope::Global, &CODEX_AUTH_SECRET_NAME, &serialized) - .map_err(|err| { - let message = - format!("failed to write OAuth tokens to encrypted auth storage: {err}"); - warn!("{message}"); - std::io::Error::other(message) + .map_err(|error| { + let error = storage_error::with_context( + "failed to write OAuth tokens to encrypted auth storage", + error, + ); + if self.mode == AuthCredentialsStoreMode::Keyring { + warn!("{error}"); + } + error })?; - if let Err(err) = delete_file_if_exists(&self.codex_home) { + let mut telemetry = storage_telemetry::telemetry( + self.mode, + AuthKeyringBackendKind::Secrets, + Operation::Cleanup, + ); + let result = delete_file_if_exists(&self.codex_home); + telemetry.record_delete_attempt(Store::File, &result); + if let Err(err) = &result { warn!("failed to remove CLI auth fallback file: {err}"); } + drop(telemetry); Ok(()) } @@ -394,10 +437,11 @@ impl AuthStorageBackend for SecretsKeyringAuthStorage { let keyring_removed = self .secrets_manager .delete(&SecretScope::Global, &CODEX_AUTH_SECRET_NAME) - .map_err(|err| { - std::io::Error::other(format!( - "failed to delete auth from encrypted auth storage: {err}" - )) + .map_err(|error| { + storage_error::with_context( + "failed to delete auth from encrypted auth storage", + error, + ) })?; let file_removed = delete_file_if_exists(&self.codex_home)?; let direct_removed = self.direct_storage.delete()?; @@ -409,6 +453,7 @@ impl AuthStorageBackend for SecretsKeyringAuthStorage { struct AutoAuthStorage { keyring_storage: Arc, file_storage: Arc, + keyring_backend_kind: AuthKeyringBackendKind, } impl AutoAuthStorage { @@ -422,37 +467,76 @@ impl AutoAuthStorage { codex_home.clone(), keyring_store, keyring_backend_kind, + AuthCredentialsStoreMode::Auto, ), file_storage: Arc::new(FileAuthStorage::new(codex_home)), + keyring_backend_kind, } } } impl AuthStorageBackend for AutoAuthStorage { fn load(&self) -> std::io::Result> { - match self.keyring_storage.load() { + let mut telemetry = storage_telemetry::telemetry( + AuthCredentialsStoreMode::Auto, + self.keyring_backend_kind, + Operation::Load, + ); + let result = self.keyring_storage.load(); + telemetry.record_load_attempt( + storage_telemetry::keyring_store(self.keyring_backend_kind), + &result, + ); + match result { Ok(Some(auth)) => Ok(Some(auth)), - Ok(None) => self.file_storage.load(), + Ok(None) => { + let result = self.file_storage.load(); + telemetry.record_load_attempt(Store::File, &result); + result + } Err(err) => { warn!("failed to load CLI auth from keyring, falling back to file storage: {err}"); - self.file_storage.load() + telemetry.record_secure_error(&err); + let result = self.file_storage.load(); + telemetry.record_load_attempt(Store::File, &result); + result } } } fn save(&self, auth: &AuthDotJson) -> std::io::Result<()> { - match self.keyring_storage.save(auth) { + let mut telemetry = storage_telemetry::telemetry( + AuthCredentialsStoreMode::Auto, + self.keyring_backend_kind, + Operation::Save, + ); + let result = self.keyring_storage.save(auth); + telemetry.record_save_attempt( + storage_telemetry::keyring_store(self.keyring_backend_kind), + &result, + ); + match result { Ok(()) => Ok(()), Err(err) => { warn!("failed to save auth to keyring, falling back to file storage: {err}"); - self.file_storage.save(auth) + telemetry.record_secure_error(&err); + let result = self.file_storage.save(auth); + telemetry.record_save_attempt(Store::File, &result); + result } } } fn delete(&self) -> std::io::Result { // Keyring storage will delete from disk as well - self.keyring_storage.delete() + let mut telemetry = storage_telemetry::telemetry( + AuthCredentialsStoreMode::Auto, + self.keyring_backend_kind, + Operation::Delete, + ); + let result = self.keyring_storage.delete(); + telemetry.record_delete_attempt(Store::Multiple, &result); + result } } @@ -514,10 +598,10 @@ fn create_auth_storage_with_store( keyring_store: Arc, keyring_backend_kind: AuthKeyringBackendKind, ) -> Arc { - match mode { + let storage: Arc = match mode { AuthCredentialsStoreMode::File => Arc::new(FileAuthStorage::new(codex_home)), AuthCredentialsStoreMode::Keyring => { - create_keyring_auth_storage(codex_home, keyring_store, keyring_backend_kind) + create_keyring_auth_storage(codex_home, keyring_store, keyring_backend_kind, mode) } AuthCredentialsStoreMode::Auto => Arc::new(AutoAuthStorage::new( codex_home, @@ -525,21 +609,27 @@ fn create_auth_storage_with_store( keyring_backend_kind, )), AuthCredentialsStoreMode::Ephemeral => Arc::new(EphemeralAuthStorage::new(codex_home)), - } + }; + storage_telemetry::observe(storage, mode, keyring_backend_kind) } fn create_keyring_auth_storage( codex_home: PathBuf, keyring_store: Arc, keyring_backend_kind: AuthKeyringBackendKind, + mode: AuthCredentialsStoreMode, ) -> Arc { match keyring_backend_kind { - AuthKeyringBackendKind::Direct => { - Arc::new(DirectKeyringAuthStorage::new(codex_home, keyring_store)) - } - AuthKeyringBackendKind::Secrets => { - Arc::new(SecretsKeyringAuthStorage::new(codex_home, keyring_store)) - } + AuthKeyringBackendKind::Direct => Arc::new(DirectKeyringAuthStorage::new( + codex_home, + keyring_store, + mode, + )), + AuthKeyringBackendKind::Secrets => Arc::new(SecretsKeyringAuthStorage::new( + codex_home, + keyring_store, + mode, + )), } } diff --git a/codex-rs/login/src/auth/storage_error.rs b/codex-rs/login/src/auth/storage_error.rs new file mode 100644 index 000000000000..b25daa02470f --- /dev/null +++ b/codex-rs/login/src/auth/storage_error.rs @@ -0,0 +1,49 @@ +//! Removes credential JSON data errors while preserving native storage diagnostics and typed causes. + +#[derive(thiserror::Error)] +struct CredentialStorageError { + context: &'static str, + #[source] + source: anyhow::Error, +} + +impl std::fmt::Display for CredentialStorageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {:#}", self.context, self.source) + } +} + +impl std::fmt::Debug for CredentialStorageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } +} + +pub(super) fn with_context( + context: &'static str, + source: impl Into, +) -> std::io::Error { + let mut source = source.into(); + let mut cause: Option<&(dyn std::error::Error + 'static)> = Some(source.as_ref()); + while let Some(error) = cause { + if let Some(error) = error.downcast_ref::() + && error.is_data() + { + // Discard the entire chain: serde errors and their context may contain credentials. + source = anyhow::anyhow!( + "invalid credential JSON ({:?} at line {} column {})", + error.classify(), + error.line(), + error.column(), + ); + break; + } + cause = if let Some(error) = error.downcast_ref::() { + // io::Error::source() skips its contained error. + error.get_ref().map(|error| error as &dyn std::error::Error) + } else { + error.source() + }; + } + std::io::Error::other(CredentialStorageError { context, source }) +} diff --git a/codex-rs/login/src/auth/storage_error_tests.rs b/codex-rs/login/src/auth/storage_error_tests.rs new file mode 100644 index 000000000000..8b44b7f6d2c3 --- /dev/null +++ b/codex-rs/login/src/auth/storage_error_tests.rs @@ -0,0 +1,146 @@ +//! Checks that storage diagnostics preserve useful causes without exposing credential values. + +use super::*; +use pretty_assertions::assert_eq; + +#[test] +fn secure_storage_errors_display_the_underlying_cause() -> anyhow::Result<()> { + for kind in [ + AuthKeyringBackendKind::Direct, + AuthKeyringBackendKind::Secrets, + ] { + for (error_kind, cause) in [ + ( + std::io::ErrorKind::PermissionDenied, + "permission denied by keyring", + ), + (std::io::ErrorKind::NotConnected, "keyring unavailable"), + ] { + let home = tempdir()?; + let keyring = MockKeyringStore::default(); + let storage = create_auth_storage_with_store( + home.path().to_path_buf(), + AuthCredentialsStoreMode::Keyring, + Arc::new(keyring.clone()), + kind, + ); + let auth = auth_with_prefix("private credential value"); + storage.save(&auth)?; + let key = match kind { + AuthKeyringBackendKind::Direct => compute_store_key(home.path())?, + AuthKeyringBackendKind::Secrets => { + compute_keyring_account(home.path(), LocalSecretsNamespace::CodexAuth) + } + }; + for operation in [Operation::Load, Operation::Save, Operation::Delete] { + keyring.set_error( + &key, + KeyringError::PlatformFailure(Box::new(std::io::Error::new(error_kind, cause))), + ); + let error = match operation { + Operation::Load => storage.load().unwrap_err(), + Operation::Save => storage.save(&auth).unwrap_err(), + Operation::Delete => storage.delete().unwrap_err(), + Operation::Cleanup | Operation::RefreshPersist => unreachable!(), + }; + let message = error.to_string(); + assert!(message.starts_with("failed to "), "{message}"); + assert!(message.contains(cause), "{message}"); + assert!(!message.contains("private credential value")); + let debug = format!("{error:?}"); + assert!(debug.contains(cause), "{debug}"); + assert!(!debug.contains("private credential value")); + } + } + } + Ok(()) +} + +#[test] +fn json_syntax_and_eof_errors_preserve_diagnostics() { + for (input, category) in [ + ( + r#"{"OPENAI_API_KEY" "private-credential-value"}"#, + serde_json::error::Category::Syntax, + ), + ( + r#"{"OPENAI_API_KEY":"private-credential-value""#, + serde_json::error::Category::Eof, + ), + ] { + let source = serde_json::from_str::(input).unwrap_err(); + assert_eq!(source.classify(), category); + let expected = format!("failed to load auth: decrypted auth JSON: {source}"); + let error = storage_error::with_context( + "failed to load auth", + anyhow::Error::new(source).context("decrypted auth JSON"), + ); + let inner = error.get_ref().unwrap(); + assert_eq!( + (error.to_string(), format!("{inner:?}")), + (expected.clone(), expected) + ); + assert!(!format!("{error:?}").contains("private-credential-value")); + } +} + +#[test] +fn json_data_errors_redact_credential_values() { + enum Wrapper { + Direct, + AnyhowContext, + Io, + Nested, + } + + for wrapper in [ + Wrapper::Direct, + Wrapper::AnyhowContext, + Wrapper::Io, + Wrapper::Nested, + ] { + let source = + serde_json::from_str::(r#"{"auth_mode":"private-credential-value"}"#) + .unwrap_err(); + assert!(source.to_string().contains("private-credential-value")); + let expected = format!( + "failed to load auth: invalid credential JSON (Data at line {} column {})", + source.line(), + source.column(), + ); + let source = match wrapper { + Wrapper::Direct => anyhow::Error::new(source), + Wrapper::AnyhowContext => anyhow::Error::new(source).context("decrypted auth JSON"), + Wrapper::Io => anyhow::Error::new(std::io::Error::other(source)), + Wrapper::Nested => anyhow::Error::new(std::io::Error::other( + anyhow::Error::new(std::io::Error::other(source)) + .context("private-credential-value"), + )) + .context("decrypted auth JSON"), + }; + let error = storage_error::with_context("failed to load auth", source); + let inner = error.get_ref().unwrap(); + assert_eq!( + (error.to_string(), format!("{inner:?}")), + (expected.clone(), expected) + ); + assert!(!format!("{error:?}").contains("private-credential-value")); + + let error = anyhow::Error::new(error).context("credential storage failed"); + for formatted in [format!("{error:#}"), format!("{error:?}")] { + assert!( + !formatted.contains("private-credential-value"), + "{formatted}" + ); + } + for cause in error.chain() { + assert!(cause.downcast_ref::().is_none()); + for formatted in [cause.to_string(), format!("{cause:?}")] { + assert!( + !formatted.contains("private-credential-value"), + "{formatted}" + ); + } + } + } +} diff --git a/codex-rs/login/src/auth/storage_policy_tests.rs b/codex-rs/login/src/auth/storage_policy_tests.rs new file mode 100644 index 000000000000..0d389b756c97 --- /dev/null +++ b/codex-rs/login/src/auth/storage_policy_tests.rs @@ -0,0 +1,443 @@ +//! Exercises configured storage policy and its emitted metadata. + +use super::*; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::MetricData; +use pretty_assertions::assert_eq; + +#[test] +fn storage_metrics_distinguish_explicit_file_from_secure_failure_fallback() -> anyhow::Result<()> { + if std::env::var_os("CODEX_AUTH_STORAGE_TEST_CHILD").is_none() { + let output = std::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + std::thread::current().name().unwrap(), + "--nocapture", + ]) + .env("CODEX_AUTH_STORAGE_TEST_CHILD", "1") + .output()?; + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return Ok(()); + } + use std::collections::BTreeMap; + let metrics = codex_otel::install_global_metrics(codex_otel::MetricsClient::new( + codex_otel::MetricsConfig::in_memory("test", "test", "1", Default::default()) + .with_runtime_reader(), + )?); + let mut expected = BTreeMap::new(); + for kind in [ + AuthKeyringBackendKind::Direct, + AuthKeyringBackendKind::Secrets, + ] { + for (scenario, mode, operation, actual, secure, outcome, reason, error) in [ + ( + "success", + AuthCredentialsStoreMode::Auto, + "save", + storage_telemetry::keyring_store(kind).as_ref(), + "success", + "success", + "none", + "none", + ), + ( + "cleanup_failed", + AuthCredentialsStoreMode::Auto, + "save", + storage_telemetry::keyring_store(kind).as_ref(), + "success", + "success", + "none", + "none", + ), + ( + "success", + AuthCredentialsStoreMode::Keyring, + "save", + storage_telemetry::keyring_store(kind).as_ref(), + "success", + "success", + "none", + "none", + ), + ( + "cleanup_failed", + AuthCredentialsStoreMode::Keyring, + "save", + storage_telemetry::keyring_store(kind).as_ref(), + "success", + "success", + "none", + "none", + ), + ( + "denied", + AuthCredentialsStoreMode::Auto, + "save", + "file", + "error", + "success", + "secure_error", + "access_denied", + ), + ( + "explicit", + AuthCredentialsStoreMode::File, + "save", + "file", + "not_attempted", + "success", + "none", + "none", + ), + ( + "missing", + AuthCredentialsStoreMode::Auto, + "load", + "file", + "not_found", + "success", + "secure_entry_missing", + "none", + ), + ( + "failed_file", + AuthCredentialsStoreMode::Auto, + "load", + "file", + "error", + "error", + "secure_error", + "access_denied", + ), + ] { + let home = tempdir()?; + let keyring = MockKeyringStore::default(); + let key = match kind { + AuthKeyringBackendKind::Direct => compute_store_key(home.path())?, + AuthKeyringBackendKind::Secrets => { + compute_keyring_account(home.path(), LocalSecretsNamespace::CodexAuth) + } + }; + let auth = auth_with_prefix("private credential value"); + if operation == "load" || scenario == "success" { + FileAuthStorage::new(home.path().to_path_buf()).save(&auth)?; + } + if scenario == "cleanup_failed" { + std::fs::create_dir(get_auth_file(home.path()))?; + } + if scenario == "failed_file" { + std::fs::write(get_auth_file(home.path()), "invalid private file contents")?; + } + if error == "access_denied" { + keyring.set_error( + &key, + KeyringError::PlatformFailure(Box::new(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "private account path error", + ))), + ); + // Secrets loads consult the keyring only when an encrypted file exists. + if operation == "load" && kind == AuthKeyringBackendKind::Secrets { + let manager = SecretsManager::new_with_keyring_store_and_namespace( + home.path().to_path_buf(), + SecretsBackendKind::Local, + Arc::new(MockKeyringStore::default()), + LocalSecretsNamespace::CodexAuth, + ); + manager.set(&SecretScope::Global, &CODEX_AUTH_SECRET_NAME, "{}")?; + } + } + let storage = create_auth_storage_with_store( + home.path().to_path_buf(), + mode, + Arc::new(keyring), + kind, + ); + if operation == "load" { + let result = storage.load(); + if outcome == "error" { + assert!(result.is_err()); + } else { + assert_eq!(result?, Some(auth.clone())); + } + } else { + storage.save(&auth)?; + } + let store_mode = match mode { + AuthCredentialsStoreMode::Auto => "auto", + AuthCredentialsStoreMode::Keyring => "keyring", + AuthCredentialsStoreMode::File => "file", + AuthCredentialsStoreMode::Ephemeral => "ephemeral", + }; + let tags: BTreeMap = BTreeMap::from([ + ("credential_kind".into(), "codex".into()), + ("store_mode".into(), store_mode.into()), + ( + "selected_store".into(), + storage_telemetry::keyring_store(kind).as_ref().into(), + ), + ("actual_store".into(), actual.into()), + ("operation".into(), operation.into()), + ("secure_outcome".into(), secure.into()), + ("outcome".into(), outcome.into()), + ("fallback_reason".into(), reason.into()), + ("secure_error".into(), error.into()), + ("storage_phase".into(), "policy".into()), + ("originator".into(), "none".into()), + ]); + if secure == "success" { + let mut cleanup_tags = tags.clone(); + cleanup_tags.extend([ + ("actual_store".into(), "file".into()), + ("operation".into(), "cleanup".into()), + ("secure_outcome".into(), "not_attempted".into()), + ( + "outcome".into(), + if scenario == "cleanup_failed" { + "error" + } else { + "success" + } + .into(), + ), + ]); + *expected.entry(cleanup_tags).or_insert(0) += 1; + } + *expected.entry(tags).or_insert(0) += 1; + } + } + let snapshot = metrics.snapshot()?; + let mut actual = Vec::new(); + for metric in snapshot + .scope_metrics() + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + { + if metric.name() == "codex.auth_storage.operation" + && let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() + { + for point in sum.data_points() { + let tags: BTreeMap = point + .attributes() + .map(|a| (a.key.to_string(), a.value.to_string())) + .collect(); + actual.push((point.value(), tags)); + } + } + } + let mut expected: Vec<_> = expected + .into_iter() + .map(|(tags, count)| (count, tags)) + .collect(); + actual.sort(); + expected.sort(); + assert_eq!(actual, expected); + metrics.shutdown()?; + Ok(()) +} + +#[test] +fn auto_auth_storage_load_prefers_keyring_value() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = AutoAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthKeyringBackendKind::Secrets, + ); + let keyring_auth = auth_with_prefix("keyring"); + seed_secrets_backend_with_auth(&mock_keyring, codex_home.path(), &keyring_auth)?; + + let file_auth = auth_with_prefix("file"); + storage.file_storage.save(&file_auth)?; + + let loaded = storage.load()?; + assert_eq!(loaded, Some(keyring_auth)); + Ok(()) +} + +#[test] +fn auto_auth_storage_load_uses_file_when_keyring_empty() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = AutoAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring), + AuthKeyringBackendKind::Secrets, + ); + + let expected = auth_with_prefix("file-only"); + storage.file_storage.save(&expected)?; + + let loaded = storage.load()?; + assert_eq!(loaded, Some(expected)); + Ok(()) +} + +#[test] +fn auto_auth_storage_load_falls_back_when_keyring_errors() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = AutoAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthKeyringBackendKind::Secrets, + ); + let key = compute_keyring_account(codex_home.path(), LocalSecretsNamespace::CodexAuth); + + let encrypted = auth_with_prefix("encrypted"); + seed_secrets_backend_with_auth(&mock_keyring, codex_home.path(), &encrypted)?; + mock_keyring.set_error(&key, KeyringError::Invalid("error".into(), "load".into())); + + let expected = auth_with_prefix("fallback"); + storage.file_storage.save(&expected)?; + + let loaded = storage.load()?; + assert_eq!(loaded, Some(expected)); + Ok(()) +} + +#[test] +fn auto_auth_storage_save_prefers_keyring() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = AutoAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthKeyringBackendKind::Secrets, + ); + let stale = auth_with_prefix("stale"); + storage.file_storage.save(&stale)?; + + let expected = auth_with_prefix("to-save"); + storage.save(&expected)?; + + assert_keyring_saved_auth_and_removed_fallback(&mock_keyring, codex_home.path(), &expected)?; + Ok(()) +} + +#[test] +fn auto_auth_storage_save_falls_back_when_keyring_errors() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = AutoAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthKeyringBackendKind::Secrets, + ); + let key = compute_keyring_account(codex_home.path(), LocalSecretsNamespace::CodexAuth); + mock_keyring.set_error(&key, KeyringError::Invalid("error".into(), "save".into())); + + let auth = auth_with_prefix("fallback"); + storage.save(&auth)?; + + let auth_file = get_auth_file(codex_home.path()); + assert!( + auth_file.exists(), + "fallback auth.json should be created when keyring save fails" + ); + let saved = storage + .file_storage + .load()? + .context("fallback auth should exist")?; + assert_eq!(saved, auth); + assert!( + mock_keyring.saved_value(&key).is_none(), + "keyring should not contain value when save fails" + ); + Ok(()) +} + +#[test] +fn auto_auth_storage_delete_removes_keyring_and_file() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = AutoAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthKeyringBackendKind::Secrets, + ); + let auth = auth_with_prefix("to-delete"); + let auth_file = seed_secrets_backend_and_fallback_auth_file_for_delete( + &mock_keyring, + codex_home.path(), + &auth, + )?; + + let removed = storage.delete()?; + + assert!(removed, "delete should report removal"); + assert_eq!(storage.load()?, None, "encrypted auth should be removed"); + assert!( + !auth_file.exists(), + "fallback auth.json should be removed after delete" + ); + Ok(()) +} + +#[test] +fn direct_keyring_auth_storage_saves_legacy_keyring_entry() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = DirectKeyringAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, + ); + let auth_file = get_auth_file(codex_home.path()); + std::fs::write(&auth_file, "stale")?; + let auth = auth_with_prefix("direct"); + + storage.save(&auth)?; + + let legacy_key = compute_store_key(codex_home.path())?; + let saved_value = mock_keyring + .saved_value(&legacy_key) + .context("direct keyring auth entry should exist")?; + assert_eq!(saved_value, serde_json::to_string(&auth)?); + assert!(!encrypted_auth_file(codex_home.path()).exists()); + assert!( + !auth_file.exists(), + "fallback auth.json should be removed after keyring save" + ); + assert_eq!(storage.load()?, Some(auth)); + Ok(()) +} + +#[test] +fn secrets_keyring_auth_storage_save_persists_and_removes_fallback_file() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let mock_keyring = MockKeyringStore::default(); + let storage = SecretsKeyringAuthStorage::new( + codex_home.path().to_path_buf(), + Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, + ); + let auth_file = get_auth_file(codex_home.path()); + std::fs::write(&auth_file, "stale")?; + let auth = AuthDotJson { + auth_mode: Some(AuthMode::Chatgpt), + openai_api_key: None, + tokens: Some(TokenData { + id_token: Default::default(), + access_token: "access".to_string(), + refresh_token: "refresh".to_string(), + account_id: Some("account".to_string()), + }), + last_refresh: Some(Utc::now()), + agent_identity: None, + personal_access_token: None, + bedrock_api_key: None, + bedrock_access_keys: None, + }; + + storage.save(&auth)?; + + assert_keyring_saved_auth_and_removed_fallback(&mock_keyring, codex_home.path(), &auth)?; + Ok(()) +} diff --git a/codex-rs/login/src/auth/storage_telemetry.rs b/codex-rs/login/src/auth/storage_telemetry.rs new file mode 100644 index 000000000000..9fb46399817d --- /dev/null +++ b/codex-rs/login/src/auth/storage_telemetry.rs @@ -0,0 +1,108 @@ +//! Observes logical Codex credential operations without inspecting authentication material. + +use super::AuthDotJson; +use super::AuthStorageBackend; +use codex_config::types::AuthCredentialsStoreMode; +use codex_config::types::AuthKeyringBackendKind; +use codex_otel::auth_storage::AuthStorageOriginator; +use codex_otel::auth_storage::CredentialKind; +use codex_otel::auth_storage::Operation; +use codex_otel::auth_storage::StorageTelemetry; +use codex_otel::auth_storage::Store; +use codex_otel::auth_storage::StoreMode; +use std::sync::Arc; + +pub(super) fn keyring_store(kind: AuthKeyringBackendKind) -> Store { + match kind { + AuthKeyringBackendKind::Direct => Store::DirectKeyring, + AuthKeyringBackendKind::Secrets => Store::Secrets, + } +} + +pub(super) fn telemetry( + mode: AuthCredentialsStoreMode, + kind: AuthKeyringBackendKind, + operation: Operation, +) -> StorageTelemetry { + let mode = match mode { + AuthCredentialsStoreMode::File => StoreMode::File, + AuthCredentialsStoreMode::Auto => StoreMode::Auto, + AuthCredentialsStoreMode::Keyring => StoreMode::Keyring, + AuthCredentialsStoreMode::Ephemeral => StoreMode::Ephemeral, + }; + StorageTelemetry::new( + CredentialKind::Codex, + mode, + keyring_store(kind), + operation, + AuthStorageOriginator::current(), + ) +} + +pub(super) fn observe( + inner: Arc, + mode: AuthCredentialsStoreMode, + kind: AuthKeyringBackendKind, +) -> Arc { + // Auto records both attempts together so file fallback cannot hide a secure-store failure. + if mode == AuthCredentialsStoreMode::Auto { + return inner; + } + Arc::new(ObservedStorage { inner, mode, kind }) +} + +#[derive(Debug)] +struct ObservedStorage { + inner: Arc, + mode: AuthCredentialsStoreMode, + kind: AuthKeyringBackendKind, +} + +impl ObservedStorage { + fn store(&self) -> Store { + match self.mode { + AuthCredentialsStoreMode::File => Store::File, + AuthCredentialsStoreMode::Ephemeral => Store::Ephemeral, + AuthCredentialsStoreMode::Keyring | AuthCredentialsStoreMode::Auto => { + keyring_store(self.kind) + } + } + } +} + +impl AuthStorageBackend for ObservedStorage { + fn load(&self) -> std::io::Result> { + let mut telemetry = telemetry(self.mode, self.kind, Operation::Load); + let result = self.inner.load(); + telemetry.record_load_attempt(self.store(), &result); + if self.mode == AuthCredentialsStoreMode::Keyring + && let Err(error) = &result + { + telemetry.record_secure_error(error); + } + result + } + + fn save(&self, auth: &AuthDotJson) -> std::io::Result<()> { + let mut telemetry = telemetry(self.mode, self.kind, Operation::Save); + let result = self.inner.save(auth); + telemetry.record_save_attempt(self.store(), &result); + if self.mode == AuthCredentialsStoreMode::Keyring + && let Err(error) = &result + { + telemetry.record_secure_error(error); + } + result + } + + fn delete(&self) -> std::io::Result { + let store = match self.mode { + AuthCredentialsStoreMode::File | AuthCredentialsStoreMode::Ephemeral => self.store(), + AuthCredentialsStoreMode::Auto | AuthCredentialsStoreMode::Keyring => Store::Multiple, + }; + let mut telemetry = telemetry(self.mode, self.kind, Operation::Delete); + let result = self.inner.delete(); + telemetry.record_delete_attempt(store, &result); + result + } +} diff --git a/codex-rs/login/src/auth/storage_tests.rs b/codex-rs/login/src/auth/storage_tests.rs index 79c51c871e7d..01c2e6f575e1 100644 --- a/codex-rs/login/src/auth/storage_tests.rs +++ b/codex-rs/login/src/auth/storage_tests.rs @@ -458,6 +458,7 @@ fn secrets_keyring_auth_storage_load_returns_deserialized_auth() -> anyhow::Resu let storage = SecretsKeyringAuthStorage::new( codex_home.path().to_path_buf(), Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, ); let expected = AuthDotJson { auth_mode: Some(AuthMode::ApiKey), @@ -486,34 +487,6 @@ fn keyring_auth_storage_compute_store_key_for_home_directory() -> anyhow::Result Ok(()) } -#[test] -fn direct_keyring_auth_storage_saves_legacy_keyring_entry() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = DirectKeyringAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - ); - let auth_file = get_auth_file(codex_home.path()); - std::fs::write(&auth_file, "stale")?; - let auth = auth_with_prefix("direct"); - - storage.save(&auth)?; - - let legacy_key = compute_store_key(codex_home.path())?; - let saved_value = mock_keyring - .saved_value(&legacy_key) - .context("direct keyring auth entry should exist")?; - assert_eq!(saved_value, serde_json::to_string(&auth)?); - assert!(!encrypted_auth_file(codex_home.path()).exists()); - assert!( - !auth_file.exists(), - "fallback auth.json should be removed after keyring save" - ); - assert_eq!(storage.load()?, Some(auth)); - Ok(()) -} - #[test] fn direct_keyring_auth_storage_delete_removes_keyring_and_file() -> anyhow::Result<()> { let codex_home = tempdir()?; @@ -521,6 +494,7 @@ fn direct_keyring_auth_storage_delete_removes_keyring_and_file() -> anyhow::Resu let storage = DirectKeyringAuthStorage::new( codex_home.path().to_path_buf(), Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, ); let auth = auth_with_prefix("direct-delete"); storage.save(&auth)?; @@ -586,38 +560,6 @@ fn factory_uses_secrets_backend_only_when_requested() -> anyhow::Result<()> { Ok(()) } -#[test] -fn secrets_keyring_auth_storage_save_persists_and_removes_fallback_file() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = SecretsKeyringAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - ); - let auth_file = get_auth_file(codex_home.path()); - std::fs::write(&auth_file, "stale")?; - let auth = AuthDotJson { - auth_mode: Some(AuthMode::Chatgpt), - openai_api_key: None, - tokens: Some(TokenData { - id_token: Default::default(), - access_token: "access".to_string(), - refresh_token: "refresh".to_string(), - account_id: Some("account".to_string()), - }), - last_refresh: Some(Utc::now()), - agent_identity: None, - personal_access_token: None, - bedrock_api_key: None, - bedrock_access_keys: None, - }; - - storage.save(&auth)?; - - assert_keyring_saved_auth_and_removed_fallback(&mock_keyring, codex_home.path(), &auth)?; - Ok(()) -} - #[test] fn secrets_keyring_auth_storage_delete_removes_keyring_and_file() -> anyhow::Result<()> { let codex_home = tempdir()?; @@ -625,6 +567,7 @@ fn secrets_keyring_auth_storage_delete_removes_keyring_and_file() -> anyhow::Res let storage = SecretsKeyringAuthStorage::new( codex_home.path().to_path_buf(), Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, ); let auth = auth_with_prefix("to-delete"); let auth_file = seed_secrets_backend_and_fallback_auth_file_for_delete( @@ -651,11 +594,13 @@ fn secrets_keyring_auth_storage_delete_removes_legacy_direct_keyring_entry() -> let direct_storage = DirectKeyringAuthStorage::new( codex_home.path().to_path_buf(), Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, ); direct_storage.save(&auth_with_prefix("legacy-direct"))?; let storage = SecretsKeyringAuthStorage::new( codex_home.path().to_path_buf(), Arc::new(mock_keyring.clone()), + AuthCredentialsStoreMode::Keyring, ); let auth = auth_with_prefix("to-delete"); let auth_file = seed_secrets_backend_and_fallback_auth_file_for_delete( @@ -680,141 +625,8 @@ fn secrets_keyring_auth_storage_delete_removes_legacy_direct_keyring_entry() -> Ok(()) } -#[test] -fn auto_auth_storage_load_prefers_keyring_value() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = AutoAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - AuthKeyringBackendKind::Secrets, - ); - let keyring_auth = auth_with_prefix("keyring"); - seed_secrets_backend_with_auth(&mock_keyring, codex_home.path(), &keyring_auth)?; +#[path = "storage_policy_tests.rs"] +mod policy; - let file_auth = auth_with_prefix("file"); - storage.file_storage.save(&file_auth)?; - - let loaded = storage.load()?; - assert_eq!(loaded, Some(keyring_auth)); - Ok(()) -} - -#[test] -fn auto_auth_storage_load_uses_file_when_keyring_empty() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = AutoAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring), - AuthKeyringBackendKind::Secrets, - ); - - let expected = auth_with_prefix("file-only"); - storage.file_storage.save(&expected)?; - - let loaded = storage.load()?; - assert_eq!(loaded, Some(expected)); - Ok(()) -} - -#[test] -fn auto_auth_storage_load_falls_back_when_keyring_errors() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = AutoAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - AuthKeyringBackendKind::Secrets, - ); - let key = compute_keyring_account(codex_home.path(), LocalSecretsNamespace::CodexAuth); - - let encrypted = auth_with_prefix("encrypted"); - seed_secrets_backend_with_auth(&mock_keyring, codex_home.path(), &encrypted)?; - mock_keyring.set_error(&key, KeyringError::Invalid("error".into(), "load".into())); - - let expected = auth_with_prefix("fallback"); - storage.file_storage.save(&expected)?; - - let loaded = storage.load()?; - assert_eq!(loaded, Some(expected)); - Ok(()) -} - -#[test] -fn auto_auth_storage_save_prefers_keyring() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = AutoAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - AuthKeyringBackendKind::Secrets, - ); - let stale = auth_with_prefix("stale"); - storage.file_storage.save(&stale)?; - - let expected = auth_with_prefix("to-save"); - storage.save(&expected)?; - - assert_keyring_saved_auth_and_removed_fallback(&mock_keyring, codex_home.path(), &expected)?; - Ok(()) -} - -#[test] -fn auto_auth_storage_save_falls_back_when_keyring_errors() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = AutoAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - AuthKeyringBackendKind::Secrets, - ); - let key = compute_keyring_account(codex_home.path(), LocalSecretsNamespace::CodexAuth); - mock_keyring.set_error(&key, KeyringError::Invalid("error".into(), "save".into())); - - let auth = auth_with_prefix("fallback"); - storage.save(&auth)?; - - let auth_file = get_auth_file(codex_home.path()); - assert!( - auth_file.exists(), - "fallback auth.json should be created when keyring save fails" - ); - let saved = storage - .file_storage - .load()? - .context("fallback auth should exist")?; - assert_eq!(saved, auth); - assert!( - mock_keyring.saved_value(&key).is_none(), - "keyring should not contain value when save fails" - ); - Ok(()) -} - -#[test] -fn auto_auth_storage_delete_removes_keyring_and_file() -> anyhow::Result<()> { - let codex_home = tempdir()?; - let mock_keyring = MockKeyringStore::default(); - let storage = AutoAuthStorage::new( - codex_home.path().to_path_buf(), - Arc::new(mock_keyring.clone()), - AuthKeyringBackendKind::Secrets, - ); - let auth = auth_with_prefix("to-delete"); - let auth_file = seed_secrets_backend_and_fallback_auth_file_for_delete( - &mock_keyring, - codex_home.path(), - &auth, - )?; - - let removed = storage.delete()?; - - assert!(removed, "delete should report removal"); - assert_eq!(storage.load()?, None, "encrypted auth should be removed"); - assert!( - !auth_file.exists(), - "fallback auth.json should be removed after delete" - ); - Ok(()) -} +#[path = "storage_error_tests.rs"] +mod errors; diff --git a/codex-rs/login/src/outbound_proxy.rs b/codex-rs/login/src/outbound_proxy.rs index 430b99d453fa..89fca57e089c 100644 --- a/codex-rs/login/src/outbound_proxy.rs +++ b/codex-rs/login/src/outbound_proxy.rs @@ -1,4 +1,5 @@ use codex_http_client::HttpClientFactory; +use codex_http_client::NetworkPolicy; /// Auth-layer adapter around client-owned proxy policy. /// @@ -7,6 +8,7 @@ use codex_http_client::HttpClientFactory; #[derive(Debug, Clone, PartialEq, Eq)] pub struct AuthRouteConfig { http_client_factory: HttpClientFactory, + application_network_policy: NetworkPolicy, local_bootstrap_factory: Option, } @@ -14,11 +16,19 @@ impl AuthRouteConfig { /// Adapts an application-resolved HTTP client factory for auth requests. pub fn from_http_client_factory(http_client_factory: HttpClientFactory) -> Self { Self { + application_network_policy: http_client_factory.network_policy().clone(), http_client_factory, local_bootstrap_factory: None, } } + /// Selects the account-owned policy revoked when the authenticated identity changes. + /// Bootstrap discovery can use separate local rules until this policy is loaded. + pub fn with_application_network_policy(mut self, policy: NetworkPolicy) -> Self { + self.application_network_policy = policy; + self + } + /// Installs the configuration owner's local-only policy for auth discovery. /// Only auth-owned endpoint constructors can access this factory. pub fn with_local_bootstrap_factory(mut self, factory: HttpClientFactory) -> Self { @@ -43,4 +53,8 @@ impl AuthRouteConfig { pub fn http_client_factory(&self) -> &HttpClientFactory { &self.http_client_factory } + + pub(crate) fn application_network_policy(&self) -> &NetworkPolicy { + &self.application_network_policy + } } diff --git a/codex-rs/login/src/server.rs b/codex-rs/login/src/server.rs index d9dcb321563f..d65c95287c3b 100644 --- a/codex-rs/login/src/server.rs +++ b/codex-rs/login/src/server.rs @@ -61,6 +61,7 @@ use codex_http_client::HttpClient; use codex_http_client::HttpClientBuilder; use codex_http_client::HttpClientFactory; use codex_http_client::OutboundProxyPolicy; +use codex_otel::auth_storage::AuthStorageOriginator; use codex_protocol::auth::AuthMode; use codex_utils_template::Template; use serde_json::Value as JsonValue; @@ -226,7 +227,7 @@ pub fn run_login_server(opts: ServerOptions) -> io::Result { let server_handle = { let shutdown_notify = shutdown_notify.clone(); let server = server; - tokio::spawn(async move { + let task = async move { let mut callback_result = LoginCallbackResult::default(); let result = loop { tokio::select! { @@ -311,7 +312,8 @@ pub fn run_login_server(opts: ServerOptions) -> io::Result { // running `server.recv()` in a loop exits cleanly. server.unblock(); result - }) + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)) }; Ok(LoginServer { @@ -839,38 +841,46 @@ pub(crate) async fn persist_tokens_async( ) -> io::Result<()> { // Reuse existing synchronous logic but run it off the async runtime. let codex_home = codex_home.to_path_buf(); - tokio::task::spawn_blocking(move || { - let mut tokens = TokenData { - id_token: parse_chatgpt_jwt_claims(&id_token).map_err(io::Error::other)?, - access_token, - refresh_token, - account_id: None, - }; - if let Some(acc) = jwt_auth_claims(&id_token) - .get("chatgpt_account_id") - .and_then(|v| v.as_str()) - { - tokens.account_id = Some(acc.to_string()); - } - let auth = AuthDotJson { - auth_mode: Some(AuthMode::Chatgpt), - openai_api_key: api_key, - tokens: Some(tokens), - last_refresh: Some(Utc::now()), - agent_identity: None, - personal_access_token: None, - bedrock_api_key: None, - bedrock_access_keys: None, - }; - save_auth( - &codex_home, - &auth, - auth_credentials_store_mode, - keyring_backend_kind, - ) - }) - .await - .map_err(|e| io::Error::other(format!("persist task failed: {e}")))? + // Blocking tasks don't inherit task-local context. + // Preserve the initiating client for credential-storage metrics. + let originator = AuthStorageOriginator::current(); + + let persist_task = tokio::task::spawn_blocking(move || { + originator.sync_scope(|| { + let mut tokens = TokenData { + id_token: parse_chatgpt_jwt_claims(&id_token).map_err(io::Error::other)?, + access_token, + refresh_token, + account_id: None, + }; + if let Some(acc) = jwt_auth_claims(&id_token) + .get("chatgpt_account_id") + .and_then(|v| v.as_str()) + { + tokens.account_id = Some(acc.to_string()); + } + let auth = AuthDotJson { + auth_mode: Some(AuthMode::Chatgpt), + openai_api_key: api_key, + tokens: Some(tokens), + last_refresh: Some(Utc::now()), + agent_identity: None, + personal_access_token: None, + bedrock_api_key: None, + bedrock_access_keys: None, + }; + save_auth( + &codex_home, + &auth, + auth_credentials_store_mode, + keyring_backend_kind, + ) + }) + }); + + persist_task + .await + .map_err(|e| io::Error::other(format!("persist task failed: {e}")))? } /// Validates the ID token against an optional workspace restriction. diff --git a/codex-rs/model-provider/src/amazon_bedrock/runtime_catalog_tests.rs b/codex-rs/model-provider/src/amazon_bedrock/runtime_catalog_tests.rs index f769d034810f..2b3e4e2d4a49 100644 --- a/codex-rs/model-provider/src/amazon_bedrock/runtime_catalog_tests.rs +++ b/codex-rs/model-provider/src/amazon_bedrock/runtime_catalog_tests.rs @@ -76,7 +76,7 @@ fn runtime_catalog_disables_web_search_without_overriding_review_models() { "global.openai.gpt-6-sol", None, false, - Some(MultiAgentVersion::V1), + Some(MultiAgentVersion::V2), ), ( "global.openai.gpt-6-luna", @@ -118,7 +118,7 @@ fn runtime_catalog_disables_web_search_without_overriding_review_models() { "us.openai.gpt-6-sol", None, false, - Some(MultiAgentVersion::V1), + Some(MultiAgentVersion::V2), ), ( "us.openai.gpt-6-luna", diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index 0ca5117c8caf..914ed5260c95 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -25,6 +25,7 @@ codex-utils-image = { workspace = true } codex-utils-path-uri = { workspace = true } codex-utils-redacted-string = { workspace = true } codex-utils-string = { workspace = true } +derive_more = { workspace = true, features = ["debug"] } encoding_rs = { workspace = true } gix-url = { workspace = true } globset = { workspace = true } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 318a4faa3df0..bb29f84985f5 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -49,9 +49,8 @@ pub use executed_tool_calls::ToolResultMetadata; pub use executed_tool_calls::ToolResultSource; pub use executed_tool_calls::ToolResultSources; pub use executed_tool_calls::bound_executed_tool_calls_for_message; -pub use executed_tool_calls::bound_executed_tool_calls_for_prompt; -pub use executed_tool_calls::bound_executed_tool_calls_for_prompt_prioritizing_recent; pub use executed_tool_calls::executed_tool_call_metadata_bytes; +pub use executed_tool_calls::normalize_executed_tool_call_arguments; pub use item_metadata::ContentItemKind; /// Controls the per-command sandbox override requested by a shell-like tool call. @@ -874,10 +873,11 @@ pub enum ResponseInputItem { }, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] +#[derive(derive_more::Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ContentItem { InputText { + #[debug("{:?} <{} bytes>", &text[..text.floor_char_boundary(/*index*/ 512)], text.len())] text: String, }, InputImage { diff --git a/codex-rs/protocol/src/models/executed_tool_calls.rs b/codex-rs/protocol/src/models/executed_tool_calls.rs index 676c0919d8ff..5bd758d75cb7 100644 --- a/codex-rs/protocol/src/models/executed_tool_calls.rs +++ b/codex-rs/protocol/src/models/executed_tool_calls.rs @@ -13,8 +13,6 @@ const MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES: usize = 8 * 1024; const MAX_TOOL_RESULT_SOURCES: usize = 32; /// Maximum UTF-8 bytes for each source's `type` and `id` separately, not the source list. pub const MAX_TOOL_RESULT_SOURCE_FIELD_BYTES: usize = 128; -/// Maximum serialized warehouse-only attempted-tool metadata in one request. -const MAX_EXECUTED_TOOL_CALL_METADATA_BYTES: usize = 2 * 1024 * 1024; const RESOURCE_ACCESS_METADATA_KEY: &str = "openai/resource_access"; const EXECUTED_TOOL_CALL_METADATA_FIELD_BYTES: usize = b"\"executed_tool_calls\":".len(); const INTERNAL_CHAT_MESSAGE_METADATA_PASSTHROUGH_FIELD_BYTES: usize = @@ -75,23 +73,12 @@ impl InternalChatMessageMetadataPassthrough { } } -/// Bounds attempted-tool metadata fairly across the complete serialized request. -pub fn bound_executed_tool_calls_for_prompt(items: &mut [ResponseItem]) { - bound_executed_tool_calls_for_prompt_with_priority(items, /*prioritize_recent*/ false); -} - -/// Bounds retained history without letting older calls displace the newest calls. -pub fn bound_executed_tool_calls_for_prompt_prioritizing_recent(items: &mut [ResponseItem]) { - items.reverse(); - bound_executed_tool_calls_for_prompt_with_priority(items, /*prioritize_recent*/ true); - items.reverse(); -} - -fn bound_executed_tool_calls_for_prompt_with_priority( - items: &mut [ResponseItem], - prioritize_recent: bool, -) { +/// Bounds recorded arguments and clears completion when any call in a cell is truncated. +/// Returns cells whose arguments were newly truncated. Ordinary tool-call arguments and +/// outputs are not changed. +pub fn normalize_executed_tool_call_arguments(items: &mut [ResponseItem]) -> HashSet { let mut damaged_cells = HashSet::new(); + let mut newly_truncated_cells = HashSet::new(); for item in items.iter_mut() { let Some(metadata) = item .internal_chat_message_metadata_passthrough_mut() @@ -100,6 +87,7 @@ fn bound_executed_tool_calls_for_prompt_with_priority( continue; }; let mut truncated = false; + let mut newly_truncated = false; for call in metadata.executed_tool_calls.iter_mut().flatten() { let argument_bytes = serde_json::to_vec(&call.arguments) .map(|bytes| bytes.len()) @@ -111,6 +99,7 @@ fn bound_executed_tool_calls_for_prompt_with_priority( MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES, /*omitted_calls*/ None, ); + newly_truncated = true; } truncated |= call.truncation().is_some(); } @@ -118,15 +107,12 @@ fn bound_executed_tool_calls_for_prompt_with_priority( metadata.tool_calls_complete = None; damaged_cells.extend(metadata.cell_id.clone()); } + if newly_truncated { + newly_truncated_cells.extend(metadata.cell_id.clone()); + } } clear_damaged_cell_completeness(items, &damaged_cells); - - bound_executed_tool_calls_with_metadata_budget( - items, - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES, - prioritize_recent, - /*whole_message*/ false, - ); + newly_truncated_cells } /// Bounds optional observations to the space left in the actual outgoing message. @@ -134,20 +120,6 @@ fn bound_executed_tool_calls_for_prompt_with_priority( pub fn bound_executed_tool_calls_for_message( items: &mut [ResponseItem], max_metadata_bytes: usize, -) { - bound_executed_tool_calls_with_metadata_budget( - items, - max_metadata_bytes, - /*prioritize_recent*/ false, - /*whole_message*/ true, - ); -} - -fn bound_executed_tool_calls_with_metadata_budget( - items: &mut [ResponseItem], - max_metadata_bytes: usize, - prioritize_recent: bool, - whole_message: bool, ) { let mut damaged_cells = HashSet::new(); @@ -155,19 +127,12 @@ fn bound_executed_tool_calls_with_metadata_budget( if total_metadata_bytes <= max_metadata_bytes { return; } - let mut remaining_bytes = shed_generic_result_metadata( - items, - max_metadata_bytes, - prioritize_recent, - whole_message, - total_metadata_bytes, - ); - - // At the message limit, shed optional sources and recorded arguments before - // resource-access evidence. The prompt budget keeps its existing order. - if whole_message && metadata_bytes(items) > max_metadata_bytes { - let overage_bytes = - shed_result_sources(items, max_metadata_bytes, /*whole_message*/ true); + let mut remaining_bytes = + shed_generic_result_metadata(items, max_metadata_bytes, total_metadata_bytes); + + // Shed optional sources and recorded arguments before resource-access evidence. + if metadata_bytes(items) > max_metadata_bytes { + let overage_bytes = shed_result_sources(items, max_metadata_bytes); if overage_bytes > 0 { truncate_call_arguments_to_fit( items, @@ -178,13 +143,7 @@ fn bound_executed_tool_calls_with_metadata_budget( } remaining_bytes = metadata_bytes(items); } - shed_remaining_result_metadata( - items, - max_metadata_bytes, - prioritize_recent, - whole_message, - remaining_bytes, - ); + shed_remaining_result_metadata(items, max_metadata_bytes, remaining_bytes); // Recheck the serialized request rather than relying on the incremental size accounting. if metadata_bytes(items) <= max_metadata_bytes { @@ -198,21 +157,7 @@ fn bound_executed_tool_calls_with_metadata_budget( if metadata_bytes(items) <= max_metadata_bytes { return; } - // The whole-message path already handled sources and arguments before resources. - if !whole_message { - shed_result_sources(items, max_metadata_bytes, /*whole_message*/ false); - } - - if metadata_bytes(items) <= max_metadata_bytes { - return; - } - - distribute_remaining_budget( - items, - max_metadata_bytes, - prioritize_recent, - &mut damaged_cells, - ); + distribute_remaining_budget(items, max_metadata_bytes, &mut damaged_cells); clear_damaged_cell_completeness(items, &damaged_cells); } @@ -224,23 +169,13 @@ fn metadata_bytes(items: &[ResponseItem]) -> usize { type ResultMetadataEntry<'a> = (usize, usize, usize, &'a mut ToolResultMetadata); -fn result_metadata_by_size( - items: &mut [ResponseItem], - prioritize_recent: bool, -) -> Vec> { +fn result_metadata_by_size(items: &mut [ResponseItem]) -> Vec> { let mut result_metadata = Vec::new(); for (item_index, item) in items.iter_mut().enumerate() { if let Some(metadata) = item .internal_chat_message_metadata_passthrough_mut() .and_then(Option::as_mut) { - // The retained-history entry point has already reversed the items. Break size ties - // by evicting older outputs, while keeping the original call order within an output. - let eviction_order = if prioritize_recent { - usize::MAX - item_index - } else { - item_index - }; for (call_index, call) in metadata .executed_tool_calls .iter_mut() @@ -252,7 +187,7 @@ fn result_metadata_by_size( .map_or(usize::MAX, |value| value.len()); result_metadata.push(( bytes, - eviction_order, + item_index, call_index, &mut call.tool_result_metadata, )); @@ -269,12 +204,10 @@ fn result_metadata_by_size( fn shed_generic_result_metadata( items: &mut [ResponseItem], max_metadata_bytes: usize, - prioritize_recent: bool, - whole_message: bool, mut total_metadata_bytes: usize, ) -> usize { // Shed the largest values first so one large result does not discard unrelated small results. - let mut result_metadata = result_metadata_by_size(items, prioritize_recent); + let mut result_metadata = result_metadata_by_size(items); for (bytes, _, _, metadata) in &mut result_metadata { if total_metadata_bytes <= max_metadata_bytes { break; @@ -294,19 +227,17 @@ fn shed_generic_result_metadata( } // Tiny generic values and markers can be cheaper than replacing them with a // new marker. Remove their complete fields before sacrificing resource evidence. - if whole_message { - for (bytes, _, _, metadata) in &mut result_metadata { - if total_metadata_bytes <= max_metadata_bytes { - break; - } - if metadata.retain_resource_access() { - continue; - } - **metadata = ToolResultMetadata::default(); - total_metadata_bytes = total_metadata_bytes - .saturating_sub(bytes.saturating_add(b",\"tool_result_metadata\":".len())); - *bytes = 0; + for (bytes, _, _, metadata) in &mut result_metadata { + if total_metadata_bytes <= max_metadata_bytes { + break; + } + if metadata.retain_resource_access() { + continue; } + **metadata = ToolResultMetadata::default(); + total_metadata_bytes = total_metadata_bytes + .saturating_sub(bytes.saturating_add(b",\"tool_result_metadata\":".len())); + *bytes = 0; } total_metadata_bytes } @@ -314,11 +245,9 @@ fn shed_generic_result_metadata( fn shed_remaining_result_metadata( items: &mut [ResponseItem], max_metadata_bytes: usize, - prioritize_recent: bool, - whole_message: bool, mut total_metadata_bytes: usize, ) { - let mut result_metadata = result_metadata_by_size(items, prioritize_recent); + let mut result_metadata = result_metadata_by_size(items); // Resource evidence is optional too; it must not displace the call inventory. for (bytes, _, _, metadata) in &mut result_metadata { if total_metadata_bytes <= max_metadata_bytes { @@ -333,9 +262,8 @@ fn shed_remaining_result_metadata( // Use the updated sizes: markers should be removed before smaller provider metadata. result_metadata.sort_by_key(|(bytes, order, call_index, metadata)| { ( - whole_message && !metadata.is_omitted_due_to_size_limit(), - std::cmp::Reverse(*bytes), !metadata.is_omitted_due_to_size_limit(), + std::cmp::Reverse(*bytes), *order, *call_index, ) @@ -354,11 +282,7 @@ fn shed_remaining_result_metadata( } } -fn shed_result_sources( - items: &mut [ResponseItem], - max_metadata_bytes: usize, - whole_message: bool, -) -> usize { +fn shed_result_sources(items: &mut [ResponseItem], max_metadata_bytes: usize) -> usize { // Source evidence is optional; dropping it must not discard calls or their completion proof. let mut overage_bytes = metadata_bytes(items).saturating_sub(max_metadata_bytes); for item in items.iter_mut() { @@ -367,7 +291,7 @@ fn shed_result_sources( .and_then(Option::as_mut) { for call in metadata.executed_tool_calls.iter_mut().flatten() { - if whole_message && overage_bytes == 0 { + if overage_bytes == 0 { break; } if let Some(sources) = call.tool_result_sources.take() { @@ -439,7 +363,6 @@ fn truncate_call_arguments_to_fit( fn distribute_remaining_budget( items: &mut [ResponseItem], max_metadata_bytes: usize, - prioritize_recent: bool, damaged_cells: &mut HashSet, ) { let mut remaining_items = items @@ -452,11 +375,7 @@ fn distribute_remaining_budget( if item_bytes == 0 { continue; } - let item_budget = if prioritize_recent { - remaining_bytes - } else { - remaining_bytes / remaining_items - }; + let item_budget = remaining_bytes / remaining_items; if item_bytes > item_budget { // Remember the cell before a too-small share removes its metadata entirely. damaged_cells.extend( diff --git a/codex-rs/protocol/src/models/executed_tool_calls_tests.rs b/codex-rs/protocol/src/models/executed_tool_calls_tests.rs index 9195a693afc9..422db7f82e63 100644 --- a/codex-rs/protocol/src/models/executed_tool_calls_tests.rs +++ b/codex-rs/protocol/src/models/executed_tool_calls_tests.rs @@ -88,253 +88,20 @@ fn result_metadata_comparison_tracks_raw_values_and_call_bindings() { } #[test] -fn executed_tool_call_prompt_budget_includes_metadata_fields() -> Result<()> { - let metadata_bytes = |items: &[ResponseItem]| -> Result { - items.iter().try_fold(0_usize, |bytes, item| { - let with_metadata = serde_json::to_vec(item)?.len(); - let mut without_metadata = item.clone(); - without_metadata.clear_executed_tool_calls(); - let without_metadata = serde_json::to_vec(&without_metadata)?.len(); - Ok(bytes + with_metadata.saturating_sub(without_metadata)) - }) - }; - - for with_turn_id in [true, false] { - let mut items = (0..4_000) - .map(|index| { - let mut item = output(&index.to_string()); - *item - .internal_chat_message_metadata_passthrough_mut() - .unwrap() = with_turn_id.then(|| passthrough_metadata("turn-1")); - item.append_executed_tool_calls(vec![ExecutedToolCall::new( - String::new(), - serde_json::Value::Null, - )]); - item.set_tool_call_cell_id(&format!("cell-\"\\{}", "x".repeat(512))); - item.mark_tool_calls_complete(); - item - }) - .collect::>(); - - assert!(metadata_bytes(&items)? > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - bound_executed_tool_calls_for_prompt(&mut items); - assert!(metadata_bytes(&items)? <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - for metadata in items - .iter() - .filter_map(ResponseItem::executed_tool_call_metadata) - { - assert_eq!(metadata.tool_calls_complete, None); - } - - let calls = items - .iter() - .filter_map(ResponseItem::executed_tool_call_metadata) - .filter_map(|metadata| metadata.executed_tool_calls.as_ref()) - .flatten() - .collect::>(); - // Shares smaller than a local marker omit that output; surviving calls stay unchanged. - assert!(!calls.is_empty() && calls.len() < 4_000); - assert!(calls.iter().all(|call| { - **call == ExecutedToolCall::new(String::new(), serde_json::Value::Null) - })); - - let bounded_items = items.clone(); - bound_executed_tool_calls_for_prompt(&mut items); - assert_eq!(items, bounded_items); - - let oversized_names = - ["\0", "é"].map(|name| name.repeat(MAX_EXECUTED_TOOL_CALL_METADATA_BYTES)); - let mut oversized_items = oversized_names - .iter() - .enumerate() - .map(|(index, name)| { - let mut item = output(&index.to_string()); - *item - .internal_chat_message_metadata_passthrough_mut() - .unwrap() = with_turn_id.then(|| passthrough_metadata("turn-1")); - item.append_executed_tool_calls(vec![ExecutedToolCall::new( - name.clone(), - serde_json::Value::Null, - )]); - item.set_tool_call_cell_id("cell-\"\\"); - item - }) - .collect::>(); - - bound_executed_tool_calls_for_prompt(&mut oversized_items); - assert!(metadata_bytes(&oversized_items)? <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - let calls = oversized_items - .iter() - .filter_map(ResponseItem::executed_tool_call_metadata) - .filter_map(|metadata| metadata.executed_tool_calls.as_ref()) - .flatten() - .collect::>(); - assert_eq!(calls.len(), 2); - for ((item, call), oversized_name) in - oversized_items.iter().zip(calls).zip(&oversized_names) - { - assert_eq!( - item.executed_tool_call_metadata() - .unwrap() - .cell_id - .as_deref(), - Some("cell-\"\\"), - ); - assert!(call.name.len() < oversized_name.len()); - assert!(oversized_name.starts_with(&call.name)); - let truncation = call.truncation().expect("trusted omission marker"); - assert_eq!(truncation.omitted_calls, None); - assert_eq!(truncation.original_name_bytes, Some(oversized_name.len())); - assert_eq!( - serde_json::to_value(&call.arguments)?["_codex_executed_tool_call_truncated"]["original_name_bytes"], - serde_json::json!(oversized_name.len()), - ); - } - - let bounded_items = oversized_items.clone(); - bound_executed_tool_calls_for_prompt(&mut oversized_items); - assert_eq!(oversized_items, bounded_items); - } - - let arguments = serde_json::json!({ "payload": "x".repeat(7 * 1024) }); - let argument_bytes = serde_json::to_vec(&arguments)?.len(); - assert!(argument_bytes < MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES); - let call_count = MAX_EXECUTED_TOOL_CALL_METADATA_BYTES / argument_bytes + 1; - assert!(call_count * argument_bytes > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - let mut items = ["A", "wait-A", "B"].map(|call_id| { - let mut item = output(call_id); - item.mark_tool_calls_complete(); - item - }); - for item in &mut items[..2] { - item.set_tool_call_cell_id("cell-A"); - } - let calls = vec![ExecutedToolCall::new("test_tool".to_string(), arguments); call_count]; - items[0].append_executed_tool_calls(calls.clone()); - items[2].append_executed_tool_calls(vec![ExecutedToolCall::new( - "test_tool".to_string(), - serde_json::json!({}), - )]); - assert!(metadata_bytes(&items)? > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - for bound in [ - bound_executed_tool_calls_for_prompt, - bound_executed_tool_calls_for_prompt_prioritizing_recent, - ] { - let mut bounded = items.clone(); - bound(&mut bounded); - assert!(metadata_bytes(&bounded)? <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - assert_eq!(bounded[2], items[2]); - let metadata = bounded[0].executed_tool_call_metadata().unwrap(); - assert_eq!(metadata.tool_calls_complete, None); - let recorded = metadata.executed_tool_calls.as_ref().unwrap(); - assert_eq!(recorded.len(), 1); - assert_eq!(recorded[0].name, "test_tool"); - let truncation = recorded[0].truncation().unwrap(); - assert_eq!(truncation.original_bytes, argument_bytes); - assert_eq!(truncation.omitted_calls, Some(call_count - 1)); - assert_eq!( - bounded[1] - .executed_tool_call_metadata() - .unwrap() - .tool_calls_complete, - None, - ); - bounded[0].append_executed_tool_calls(calls.clone()); - assert!(metadata_bytes(&bounded)? > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - bound(&mut bounded); - assert!(metadata_bytes(&bounded)? <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - assert_eq!(bounded[2], items[2]); - assert_eq!( - first_executed_tool_call(&mut bounded[0]) - .unwrap() - .truncation() - .unwrap() - .omitted_calls, - Some(2 * call_count - 1), - ); - } - - let mut items = ["exec", "wait"].map(|call_id| { - let mut item = output(call_id); - item.set_tool_call_cell_id("cell-\"\\"); - item - }); - items[0].append_executed_tool_calls(vec![ - ExecutedToolCall::new( - "test_tool".to_string(), - serde_json::json!({ "payload": "x".repeat(MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES - 256) }), - ); - 16 - ]); - items[1].mark_tool_calls_complete(); - // Leave room for calls, but not their sources, without exceeding the argument limit. - let padding_bytes = MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - metadata_bytes(&items)? - 1024; - let calls = items[0] - .ensure_tool_call_metadata() - .unwrap() - .executed_tool_calls - .as_mut() - .unwrap(); - let call_count = calls.len(); - for (index, call) in calls.iter_mut().enumerate() { - call.name.push_str( - &"x".repeat( - padding_bytes / call_count + usize::from(index < padding_bytes % call_count), - ), - ); - } - let expected = items.clone(); - assert!( - first_executed_tool_call(&mut items[0]) - .expect("recorded call should exist") - .set_tool_result_sources(ToolResultSources::new( - (0..MAX_TOOL_RESULT_SOURCES) - .map(|index| ToolResultSource { - r#type: "document".to_string(), - id: format!( - "R{index:0width$}", - width = MAX_TOOL_RESULT_SOURCE_FIELD_BYTES - 1 - ), - }) - .collect(), - )) - ); - assert!(metadata_bytes(&items)? > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - bound_executed_tool_calls_for_prompt(&mut items); - assert_eq!(items, expected); - - // Empty waits can carry only the marker; its bytes still count toward the budget. +fn metadata_byte_count_matches_serialized_fields() -> Result<()> { for metadata in [None, Some(passthrough_metadata("turn-1"))] { let mut item = output("wait"); *item .internal_chat_message_metadata_passthrough_mut() .unwrap() = metadata; - let without_marker = item.clone(); - item.set_tool_call_cell_id(&format!("cell-\"\\{}", "x".repeat(256))); + let without_calls = item.clone(); + item.set_tool_call_cell_id("cell"); item.mark_tool_calls_complete(); assert_eq!( - metadata_bytes(std::slice::from_ref(&item))?, executed_tool_call_metadata_bytes(&item), + serde_json::to_vec(&item)?.len() - serde_json::to_vec(&without_calls)?.len(), ); - let mut items = vec![item; 8_000]; - assert!(metadata_bytes(&items)? > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - bound_executed_tool_calls_for_prompt_prioritizing_recent(&mut items); - assert!(metadata_bytes(&items)? <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - let retained = items - .iter() - .filter(|item| executed_tool_call_metadata_bytes(item) > 0) - .count(); - assert!(retained > 0 && retained < 8_000); - for item in &mut items { - assert!( - item.executed_tool_call_metadata() - .is_none_or(|metadata| metadata.tool_calls_complete.is_none()) - ); - item.clear_executed_tool_calls(); - } - assert_eq!(items, vec![without_marker; 8_000]); } - Ok(()) } @@ -365,7 +132,7 @@ fn model_arguments_cannot_forge_executed_tool_call_truncation() -> Result<()> { forged_marker.clone(), )]); - bound_executed_tool_calls_for_prompt(std::slice::from_mut(&mut item)); + normalize_executed_tool_call_arguments(std::slice::from_mut(&mut item)); let call = item .executed_tool_call_metadata() .and_then(|metadata| metadata.executed_tool_calls.as_ref()) @@ -410,7 +177,7 @@ fn tool_call_completeness_is_host_only_and_fail_closed() -> Result<()> { let mut item = output("call-1"); item.set_tool_call_cell_id("cell-1"); item.mark_tool_calls_complete(); - bound_executed_tool_calls_for_prompt(std::slice::from_mut(&mut item)); + normalize_executed_tool_call_arguments(std::slice::from_mut(&mut item)); assert_eq!( serde_json::to_value(&item)?["internal_chat_message_metadata_passthrough"], serde_json::json!({ @@ -421,15 +188,21 @@ fn tool_call_completeness_is_host_only_and_fail_closed() -> Result<()> { item.clear_executed_tool_calls(); assert!(item.executed_tool_call_metadata().is_none()); - for call in [ - ExecutedToolCall::new( - "test_tool".to_string(), - serde_json::json!({ "payload": "x".repeat(MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES + 1) }), + for (call, newly_truncated) in [ + ( + ExecutedToolCall::new( + "test_tool".to_string(), + serde_json::json!({ "payload": "x".repeat(MAX_EXECUTED_TOOL_CALL_ARGUMENT_BYTES + 1) }), + ), + true, ), - ExecutedToolCall::truncated( - "test_tool".to_string(), - /*original_bytes*/ 9_000, - /*max_bytes*/ 0, + ( + ExecutedToolCall::truncated( + "test_tool".to_string(), + /*original_bytes*/ 9_000, + /*max_bytes*/ 0, + ), + false, ), ] { for same_cell in [false, true] { @@ -441,7 +214,12 @@ fn tool_call_completeness_is_host_only_and_fail_closed() -> Result<()> { } item.mark_tool_calls_complete(); } - bound_executed_tool_calls_for_prompt(&mut items); + let changed_cells = normalize_executed_tool_call_arguments(&mut items); + assert_eq!( + changed_cells.contains("cell-1"), + same_cell && newly_truncated + ); + assert!(normalize_executed_tool_call_arguments(&mut items).is_empty()); assert_eq!( items.map(|item| item .executed_tool_call_metadata() @@ -817,77 +595,9 @@ fn message_budget_preserves_names_and_turn_metadata_and_invalidates_shared_cell_ } } -#[test] -fn raw_result_metadata_is_shed_before_sources_calls_or_completion() -> Result<()> { - // Keep each result below its capture limit while filling the larger request budget. - let name = format!( - "test_tool{}", - "x".repeat(MAX_EXECUTED_TOOL_CALL_METADATA_BYTES / 8 - 16 * 1024) - ); - let mut call = ExecutedToolCall::new(name, serde_json::json!({})); - call.set_tool_result_sources(ToolResultSources::new(vec![ToolResultSource { - r#type: "test_resource".to_string(), - id: "R1".to_string(), - }])); - let mut items = [ - "first", - "second", - "third", - "fourth", - "fifth", - "sixth", - "seventh", - "without-metadata", - ] - .map(|call_id| { - let mut item = ResponseItem::from(ResponseInputItem::FunctionCallOutput { - call_id: call_id.to_string(), - output: FunctionCallOutputPayload::from_text("unchanged output".to_string()), - }); - item.append_executed_tool_calls(vec![call.clone()]); - item.set_tool_call_cell_id(call_id); - item.mark_tool_calls_complete(); - item - }); - let metadata = ToolResultMetadata::new(&serde_json::json!({ "large": "x".repeat(20_000) })); - for item in &mut items[..7] { - first_executed_tool_call(item) - .unwrap() - .set_tool_result_metadata(metadata.clone()); - } - let original_bytes = items - .iter() - .map(executed_tool_call_metadata_bytes) - .sum::(); - let mut expected = items.clone(); - first_executed_tool_call(&mut expected[0]) - .unwrap() - .set_tool_result_metadata(ToolResultMetadata::omitted_due_to_size_limit( - original_bytes - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES, - )); - assert!(original_bytes > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); - for bound in [ - bound_executed_tool_calls_for_prompt, - bound_executed_tool_calls_for_prompt_prioritizing_recent, - ] { - let mut bounded = items.clone(); - bound(&mut bounded); - assert_eq!(bounded, expected); - assert!( - bounded - .iter() - .map(executed_tool_call_metadata_bytes) - .sum::() - <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - ); - bound(&mut bounded); - assert_eq!(bounded, expected); - } - Ok(()) -} - #[test] fn result_metadata_shedding_keeps_smaller_results_within_one_output() { + let budget = 2 * 1024 * 1024; let resource_only = serde_json::json!({ "openai/resource_access": { "resource_coverage": "complete", "resources": [] }, }); @@ -928,7 +638,7 @@ fn result_metadata_shedding_keeps_smaller_results_within_one_output() { serde_json::json!({ "padding": "x".repeat(17_000) }); 6 ]); - let name_padding = (MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - 128 * 1024) / metadata.len(); + let name_padding = (budget - 128 * 1024) / metadata.len(); let mut item = output("exec"); item.append_executed_tool_calls( metadata @@ -950,10 +660,10 @@ fn result_metadata_shedding_keeps_smaller_results_within_one_output() { ); item.set_tool_call_cell_id("exec"); item.mark_tool_calls_complete(); - assert!(executed_tool_call_metadata_bytes(&item) > MAX_EXECUTED_TOOL_CALL_METADATA_BYTES); + assert!(executed_tool_call_metadata_bytes(&item) > budget); let replacement = replacement.unwrap_or_else(|| { ToolResultMetadata::omitted_due_to_size_limit( - executed_tool_call_metadata_bytes(&item) - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES, + executed_tool_call_metadata_bytes(&item) - budget, ) }); let mut expected = item.clone(); @@ -964,162 +674,11 @@ fn result_metadata_shedding_keeps_smaller_results_within_one_output() { .as_mut() .unwrap()[reduced_index] .set_tool_result_metadata(replacement); - for bound in [ - bound_executed_tool_calls_for_prompt, - bound_executed_tool_calls_for_prompt_prioritizing_recent, - |items: &mut [ResponseItem]| { - bound_executed_tool_calls_for_message(items, MAX_EXECUTED_TOOL_CALL_METADATA_BYTES) - }, - ] { - let mut bounded = item.clone(); - bound(std::slice::from_mut(&mut bounded)); - assert_eq!(bounded, expected); - assert!( - executed_tool_call_metadata_bytes(&bounded) - <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - ); - bound(std::slice::from_mut(&mut bounded)); - assert_eq!(bounded, expected); - } - } -} - -#[test] -fn result_metadata_shedding_removes_only_markers_needed_to_fit() { - let mut item = output("exec"); - let mut call = ExecutedToolCall::new("test_tool".to_string(), serde_json::json!({})); - call.set_tool_result_sources(ToolResultSources::new(vec![ToolResultSource { - r#type: "test_resource".to_string(), - id: "R1".to_string(), - }])); - let mut calls = vec![call; 20]; - let omitted = ToolResultMetadata::omitted_due_to_size_limit(/*overage_bytes*/ 1); - let marker_bytes = serde_json::to_vec(&omitted).unwrap().len(); - // A real object tied with a marker must survive even though it is older. - let equal_sized = ToolResultMetadata::new(&serde_json::json!({ - "value": "x".repeat(marker_bytes - r#"{"value":""}"#.len()), - })); - assert_eq!( - serde_json::to_vec(&equal_sized).unwrap().len(), - marker_bytes - ); - calls[0].set_tool_result_metadata(equal_sized); - for index in [1, 3] { - calls[index].set_tool_result_metadata(omitted.clone()); - } - calls[2].set_tool_result_metadata(ToolResultMetadata::new(&serde_json::json!({}))); - item.append_executed_tool_calls(calls); - item.set_tool_call_cell_id("exec"); - item.mark_tool_calls_complete(); - let remaining_bytes = - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES + 1 - executed_tool_call_metadata_bytes(&item); - let calls = item - .ensure_tool_call_metadata() - .unwrap() - .executed_tool_calls - .as_mut() - .unwrap(); - let call_count = calls.len(); - for (index, call) in calls.iter_mut().enumerate() { - // Pad names so the per-argument limit cannot preempt result-metadata shedding. - call.name.push_str(&"x".repeat( - remaining_bytes / call_count + usize::from(index < remaining_bytes % call_count), - )); - } - assert_eq!( - executed_tool_call_metadata_bytes(&item), - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES + 1 - ); - let mut expected = item.clone(); - expected - .ensure_tool_call_metadata() - .unwrap() - .executed_tool_calls - .as_mut() - .unwrap()[1] - .set_tool_result_metadata(ToolResultMetadata::default()); - let tied_markers = item.clone(); - // The new marker has a larger overage value and therefore more bytes. Remove - // that marker first, while tied smaller markers still precede provider data. - item.ensure_tool_call_metadata() - .unwrap() - .executed_tool_calls - .as_mut() - .unwrap()[3] - .set_tool_result_metadata(ToolResultMetadata::new(&serde_json::json!({ - "large": "x".repeat(4 * 1024), - }))); - let mut larger_marker_expected = item.clone(); - larger_marker_expected - .ensure_tool_call_metadata() - .unwrap() - .executed_tool_calls - .as_mut() - .unwrap()[3] - .set_tool_result_metadata(ToolResultMetadata::default()); - for (item, expected) in [(tied_markers, expected), (item, larger_marker_expected)] { - for bound in [ - bound_executed_tool_calls_for_prompt, - bound_executed_tool_calls_for_prompt_prioritizing_recent, - ] { - let mut bounded = item.clone(); - bound(std::slice::from_mut(&mut bounded)); - assert_eq!(bounded, expected); - assert!( - executed_tool_call_metadata_bytes(&bounded) - <= MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - ); - bound(std::slice::from_mut(&mut bounded)); - assert_eq!(bounded, expected); - } - } -} - -#[test] -fn result_metadata_markers_do_not_displace_existing_evidence() { - let mut item = ResponseItem::from(ResponseInputItem::FunctionCallOutput { - call_id: "exec".to_string(), - output: FunctionCallOutputPayload::from_text("unchanged output".to_string()), - }); - let mut call = ExecutedToolCall::new("test_tool".to_string(), serde_json::json!({})); - call.set_tool_result_sources(ToolResultSources::new(vec![ToolResultSource { - r#type: "test_resource".to_string(), - id: "R1".to_string(), - }])); - item.append_executed_tool_calls(vec![call; 20]); - item.set_tool_call_cell_id("exec"); - item.mark_tool_calls_complete(); - let remaining_bytes = - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - executed_tool_call_metadata_bytes(&item); - let calls = item - .ensure_tool_call_metadata() - .unwrap() - .executed_tool_calls - .as_mut() - .unwrap(); - let call_count = calls.len(); - for (index, call) in calls.iter_mut().enumerate() { - // Fill the budget exactly without increasing arguments past their individual limit. - call.name.push_str(&"x".repeat( - remaining_bytes / call_count + usize::from(index < remaining_bytes % call_count), - )); - } - assert_eq!( - executed_tool_call_metadata_bytes(&item), - MAX_EXECUTED_TOOL_CALL_METADATA_BYTES - ); - let expected = item.clone(); - for metadata in [ - serde_json::json!({ "too-large": "x".repeat(32 * 1024) }), - serde_json::json!({ - "openai/resource_access": { "resource_coverage": "complete", "resources": [] }, - }), - ] { - let mut item = expected.clone(); - first_executed_tool_call(&mut item) - .unwrap() - .set_tool_result_metadata(ToolResultMetadata::new(&metadata)); - bound_executed_tool_calls_for_prompt(std::slice::from_mut(&mut item)); - assert_eq!(item, expected); + let mut bounded = item.clone(); + bound_executed_tool_calls_for_message(std::slice::from_mut(&mut bounded), budget); + assert_eq!(bounded, expected); + assert!(executed_tool_call_metadata_bytes(&bounded) <= budget); + bound_executed_tool_calls_for_message(std::slice::from_mut(&mut bounded), budget); + assert_eq!(bounded, expected); } } diff --git a/codex-rs/protocol/src/turn_input.rs b/codex-rs/protocol/src/turn_input.rs index e7c17ead2fcd..cb27cd4f9639 100644 --- a/codex-rs/protocol/src/turn_input.rs +++ b/codex-rs/protocol/src/turn_input.rs @@ -142,7 +142,7 @@ pub enum TurnInputMode { Steer { expected_turn_id: String }, } -/// Requested cyber treatment for a ChatGPT-authenticated Codex turn. +/// Requested cyber treatment for an OpenAI model turn. /// Authorization and model-tier restrictions remain server-owned. #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "snake_case")] diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index 8f5f87c00a79..9e28c5ec3d9b 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -11,10 +11,11 @@ pub const MAX_USER_INPUT_TEXT_CHARS: usize = 1 << 20; /// User input #[non_exhaustive] -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, TS, JsonSchema)] +#[derive(derive_more::Debug, Clone, Deserialize, Serialize, PartialEq, TS, JsonSchema)] #[serde(tag = "type", rename_all = "snake_case")] pub enum UserInput { Text { + #[debug("{:?} <{} bytes>", &text[..text.floor_char_boundary(/*index*/ 512)], text.len())] text: String, /// UI-defined spans within `text` that should be treated as special elements. /// These are byte ranges into the UTF-8 `text` buffer and are used to render diff --git a/codex-rs/rmcp-client/Cargo.toml b/codex-rs/rmcp-client/Cargo.toml index b133ddaf46f4..671bfd9ecc97 100644 --- a/codex-rs/rmcp-client/Cargo.toml +++ b/codex-rs/rmcp-client/Cargo.toml @@ -69,6 +69,7 @@ webbrowser = { workspace = true } which = { workspace = true } [dev-dependencies] +opentelemetry_sdk = { workspace = true, features = ["testing"] } codex-http-client = { workspace = true } codex-utils-cargo-bin = { workspace = true } pretty_assertions = { workspace = true } @@ -94,10 +95,14 @@ windows-sys = { version = "0.52", features = ["Win32_Storage_FileSystem"] } [target.'cfg(any(target_os = "freebsd", target_os = "openbsd"))'.dependencies] keyring = { workspace = true, features = ["sync-secret-service"] } -# This test is compiled through `#[path]` inside the inline `oauth::tests` module. Cargo-shear -# cannot resolve that nested module path and otherwise reports the linked file as unlinked. +# These tests are compiled through `#[path]` inside the inline `oauth::tests` module. Cargo-shear +# cannot resolve those nested module paths and otherwise reports the linked files as unlinked. [package.metadata.cargo-shear] -ignored-paths = ["src/oauth/tests/persistor_tests.rs", "src/oauth/tests/credential_store_tests.rs"] +ignored-paths = [ + "src/oauth/tests/persistor_tests.rs", + "src/oauth/tests/credential_store_tests.rs", + "src/oauth/tests/telemetry_tests.rs", +] [lib] doctest = false diff --git a/codex-rs/rmcp-client/src/ema_identity_tests.rs b/codex-rs/rmcp-client/src/ema_identity_tests.rs index 53c7ef497ad4..56222f744dca 100644 --- a/codex-rs/rmcp-client/src/ema_identity_tests.rs +++ b/codex-rs/rmcp-client/src/ema_identity_tests.rs @@ -233,7 +233,7 @@ impl KeyringStore for GatedKeyringStore { async fn refresh_subject_reread_is_cancellable_and_releases_guard_on_failure() -> Result<()> { let _home = TempCodexHome::new(); let (_server, issuer) = discovery().await; - let store = ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct); + let store = ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct); let stored = credentials(&issuer, "user", /*expires_at*/ 0); let snapshot = StoredOAuthCredentialSnapshot::new(stored.clone(), store); for outcome in [ @@ -312,7 +312,7 @@ async fn refresh_subject_reread_is_cancellable_and_releases_guard_on_failure() - async fn refresh_subject_rereads_pinned_credentials_after_id_token_expiry() -> Result<()> { let _home = TempCodexHome::new(); let (_server, issuer) = discovery().await; - let store = ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct); + let store = ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct); let stored = credentials(&issuer, "user", /*expires_at*/ 0); let snapshot = StoredOAuthCredentialSnapshot::new(stored.clone(), store); let keyring = MockKeyringStore::default(); @@ -341,7 +341,7 @@ async fn refresh_subject_rejects_removed_replaced_or_missing_credentials() -> Re let _home = TempCodexHome::new(); let (server, issuer) = discovery().await; let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); - let store = ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct); + let store = ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct); let original = credentials(&issuer, "user", /*expires_at*/ 0); for change in [ "deleted", @@ -357,7 +357,7 @@ async fn refresh_subject_rejects_removed_replaced_or_missing_credentials() -> Re let snapshot = StoredOAuthCredentialSnapshot::new( original.clone(), if change == "file" { - ResolvedOAuthCredentialStore::File + ResolvedOAuthCredentialStore::file() } else { store }, diff --git a/codex-rs/rmcp-client/src/oauth.rs b/codex-rs/rmcp-client/src/oauth.rs index c4fa7a1f9550..f3127d399ad1 100644 --- a/codex-rs/rmcp-client/src/oauth.rs +++ b/codex-rs/rmcp-client/src/oauth.rs @@ -25,6 +25,7 @@ mod refresh_transaction; mod resolved_store; mod runtime; mod store_lock; +mod telemetry; #[cfg(test)] #[path = "oauth/test_support.rs"] @@ -35,6 +36,8 @@ use anyhow::Error; use anyhow::Result; use codex_config::types::AuthKeyringBackendKind; use codex_config::types::OAuthCredentialsStoreMode; +use codex_otel::auth_storage::Operation; +use codex_otel::auth_storage::Store; use codex_secrets::LocalSecretsNamespace; use codex_secrets::SecretName; use codex_secrets::SecretScope; @@ -214,7 +217,7 @@ impl StoredOAuthCredentialSnapshot { store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, ) -> Result> { - if self.store == ResolvedOAuthCredentialStore::File + if self.store == ResolvedOAuthCredentialStore::file() && store_mode == OAuthCredentialsStoreMode::Auto { return Self::for_runtime_refresh( @@ -227,7 +230,15 @@ impl StoredOAuthCredentialSnapshot { .map(|snapshot| snapshot.map(|snapshot| snapshot.credentials)); } - let credentials = match self.store.try_load(&DefaultKeyringStore, server_name, url) { + let mut observation = telemetry::resolved(self.store, Operation::Load); + let result = self.store.try_load(&DefaultKeyringStore, server_name, url); + observation.record_load_attempt(telemetry::store(self.store), &result); + if matches!(self.store.backend, resolved_store::Backend::Keyring(_)) + && let Err(error) = &result + { + telemetry::record_secure_error(&mut observation, error.as_ref()); + } + let credentials = match result { Ok(credentials) => credentials, Err(error) if oauth_store_is_contended(&error) => return Ok(None), Err(error) => return Err(error), @@ -393,7 +404,7 @@ fn load_oauth_tokens_from_direct_keyring( Ok(Some(tokens)) } Ok(None) => Ok(None), - Err(error) => Err(Error::new(error.into_error())), + Err(error) => Err(Error::new(std::io::Error::from(error))), } } @@ -440,7 +451,7 @@ enum OAuthKeyringLoadError { #[error(transparent)] StoreLock(#[from] OAuthStoreLockFailure), /// The selected keyring backend itself was unavailable or its data was invalid. - #[error(transparent)] + #[error("{0}")] Backend(#[from] anyhow::Error), } @@ -463,6 +474,7 @@ pub(crate) fn save_oauth_tokens_with_lock_held( keyring_backend_kind: AuthKeyringBackendKind, ) -> Result<()> { let keyring_store = DefaultKeyringStore; + let mut observation = telemetry::policy(store_mode, keyring_backend_kind, Operation::Save); match store_mode { OAuthCredentialsStoreMode::Auto => save_oauth_tokens_with_keyring_with_fallback_to_file( &keyring_store, @@ -470,13 +482,25 @@ pub(crate) fn save_oauth_tokens_with_lock_held( server_name, tokens, ), - OAuthCredentialsStoreMode::File => save_oauth_tokens_to_file(tokens), - OAuthCredentialsStoreMode::Keyring => save_oauth_tokens_with_keyring_and_cleanup_file( - &keyring_store, - keyring_backend_kind, - server_name, - tokens, - ), + OAuthCredentialsStoreMode::File => { + let result = save_oauth_tokens_to_file(tokens); + observation.record_save_attempt(Store::File, &result); + result + } + OAuthCredentialsStoreMode::Keyring => { + let result = save_oauth_tokens_with_keyring_and_cleanup_file( + &keyring_store, + store_mode, + keyring_backend_kind, + server_name, + tokens, + ); + observation.record_save_attempt(telemetry::keyring(keyring_backend_kind), &result); + if let Err(error) = &result { + telemetry::record_secure_error(&mut observation, error.as_ref()); + } + result + } } } @@ -506,17 +530,10 @@ fn save_oauth_tokens_to_direct_keyring( let serialized = serde_json::to_string(tokens).context("failed to serialize OAuth tokens")?; let key = compute_store_key(server_name, &tokens.url)?; - match keyring_store.save(KEYRING_SERVICE, &key, &serialized) { - Ok(()) => Ok(()), - Err(error) => { - let message = format!( - "failed to write OAuth tokens to keyring: {}", - error.message() - ); - warn!("{message}"); - Err(Error::new(error.into_error()).context(message)) - } - } + keyring_store + .save(KEYRING_SERVICE, &key, &serialized) + .map_err(std::io::Error::from) + .context("failed to write OAuth tokens to keyring") } /// Saves one credential while holding the Secrets aggregate-store lock across the mutation. @@ -558,20 +575,20 @@ fn save_oauth_tokens_to_secrets_keyring_with_lock_held( keyring_store: &K, + store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, server_name: &str, tokens: &StoredOAuthTokens, ) -> Result<()> { save_oauth_tokens_with_keyring(keyring_store, keyring_backend_kind, server_name, tokens)?; let key = compute_store_key(server_name, &tokens.url)?; - if let Err(error) = delete_oauth_tokens_from_file(&key) { - warn!( - server_name, - keyring_backend = ?keyring_backend_kind, - error = %error, - "failed to remove OAuth tokens from fallback storage" - ); + let mut observation = telemetry::policy(store_mode, keyring_backend_kind, Operation::Cleanup); + let result = delete_oauth_tokens_from_file(&key); + observation.record_delete_attempt(Store::File, &result); + if result.is_err() { + warn!("failed to remove OAuth tokens from fallback storage"); } + drop(observation); Ok(()) } @@ -581,22 +598,34 @@ fn save_oauth_tokens_with_keyring_with_fallback_to_file Result<()> { - match save_oauth_tokens_with_keyring_and_cleanup_file( + let mut observation = telemetry::policy( + OAuthCredentialsStoreMode::Auto, + keyring_backend_kind, + Operation::Save, + ); + let result = save_oauth_tokens_with_keyring_and_cleanup_file( keyring_store, + OAuthCredentialsStoreMode::Auto, keyring_backend_kind, server_name, tokens, - ) { + ); + observation.record_save_attempt(telemetry::keyring(keyring_backend_kind), &result); + match result { Ok(()) => Ok(()), // As on load, a store lock failure is a coordination failure rather than evidence that // the keyring backend is unavailable. Falling back could leave a newer File token hidden // behind a stale Secrets entry. - Err(error) if error.downcast_ref::().is_some() => Err(error), + Err(error) if error.downcast_ref::().is_some() => { + telemetry::record_secure_error(&mut observation, error.as_ref()); + Err(error) + } Err(error) => { let message = error.to_string(); - warn!("falling back to file storage for OAuth tokens: {message}"); - save_oauth_tokens_to_file(tokens) - .with_context(|| format!("failed to write OAuth tokens to keyring: {message}")) + telemetry::record_secure_error(&mut observation, error.as_ref()); + let result = save_oauth_tokens_to_file(tokens); + observation.record_save_attempt(Store::File, &result); + result.with_context(|| format!("failed to write OAuth tokens to keyring: {message}")) } } } @@ -636,25 +665,28 @@ fn delete_oauth_tokens_from_keyring_and_file( server_name: &str, url: &str, ) -> Result { - let key = compute_store_key(server_name, url)?; - let keyring_result = - delete_oauth_tokens_from_keyring(keyring_store, keyring_backend_kind, server_name, url); - let keyring_removed = match keyring_result { - Ok(removed) => removed, - Err(error) => { - let message = error.to_string(); - warn!("failed to delete OAuth tokens from keyring: {message}"); - match store_mode { - OAuthCredentialsStoreMode::Auto | OAuthCredentialsStoreMode::Keyring => { - return Err(error).context("failed to delete OAuth tokens from keyring"); + let mut observation = telemetry::policy(store_mode, keyring_backend_kind, Operation::Delete); + let result = compute_store_key(server_name, url).and_then(|key| { + let keyring_result = + delete_oauth_tokens_from_keyring(keyring_store, keyring_backend_kind, server_name, url); + let keyring_removed = match keyring_result { + Ok(removed) => removed, + Err(error) => { + warn!("failed to delete OAuth tokens from keyring"); + match store_mode { + OAuthCredentialsStoreMode::Auto | OAuthCredentialsStoreMode::Keyring => { + return Err(error).context("failed to delete OAuth tokens from keyring"); + } + OAuthCredentialsStoreMode::File => false, } - OAuthCredentialsStoreMode::File => false, } - } - }; + }; - let file_removed = delete_oauth_tokens_from_file(&key)?; - Ok(keyring_removed || file_removed) + let file_removed = delete_oauth_tokens_from_file(&key)?; + Ok(keyring_removed || file_removed) + }); + observation.record_delete_attempt(Store::Multiple, &result); + result } fn delete_oauth_tokens_from_keyring( @@ -685,7 +717,7 @@ fn delete_oauth_tokens_from_direct_keyring( let key = compute_store_key(server_name, url)?; keyring_store .delete(KEYRING_SERVICE, &key) - .map_err(|error| Error::new(error.into_error())) + .map_err(|error| Error::new(std::io::Error::from(error))) } fn delete_oauth_tokens_from_secrets_keyring( @@ -786,7 +818,7 @@ impl OAuthPersistor { expires_at, }; if last_credentials.as_ref() != Some(&stored) { - self.inner.credential_store.save( + self.inner.credential_store.save_with_refresh_telemetry( &DefaultKeyringStore, &self.inner.server_name, &stored, @@ -1160,6 +1192,8 @@ mod tests { mod credential_store_tests; #[path = "persistor_tests.rs"] mod persistor_tests; + #[path = "telemetry_tests.rs"] + mod telemetry_tests; use super::test_support::TempCodexHome; @@ -1224,7 +1258,7 @@ mod tests { .expect("tokens should load from keyring"); assert_eq!( resolved.store, - ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct) + ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct) ); assert_tokens_match_without_expiry(&resolved.tokens, &expected); Ok(()) @@ -1247,7 +1281,7 @@ mod tests { AuthKeyringBackendKind::Direct, )? .expect("tokens should load from fallback"); - assert_eq!(resolved.store, ResolvedOAuthCredentialStore::File); + assert_eq!(resolved.store, ResolvedOAuthCredentialStore::file()); assert_tokens_match_without_expiry(&resolved.tokens, &expected); Ok(()) } @@ -1271,7 +1305,7 @@ mod tests { AuthKeyringBackendKind::Direct, )? .expect("tokens should load from fallback"); - assert_eq!(resolved.store, ResolvedOAuthCredentialStore::File); + assert_eq!(resolved.store, ResolvedOAuthCredentialStore::file()); assert_tokens_match_without_expiry(&resolved.tokens, &expected); Ok(()) } @@ -1298,7 +1332,7 @@ mod tests { )?; assert_eq!(fs::read(fallback_path)?, fallback_before); - let loaded = ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct) + let loaded = ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct) .load(&store, &keyring_tokens.server_name, &keyring_tokens.url)? .expect("tokens should load from the selected keyring store"); assert_tokens_match_without_expiry(&loaded, &keyring_tokens); diff --git a/codex-rs/rmcp-client/src/oauth/credential_store.rs b/codex-rs/rmcp-client/src/oauth/credential_store.rs index 2d79fe403b68..c191e6936a48 100644 --- a/codex-rs/rmcp-client/src/oauth/credential_store.rs +++ b/codex-rs/rmcp-client/src/oauth/credential_store.rs @@ -243,9 +243,11 @@ impl CredentialStore for OAuthCredentialStore .map_err(credential_store_error)?; let tokens = tokio::task::spawn_blocking(move || -> Result { let _guard = guard; - inner - .store - .save(&inner.keyring, &inner.server_name, &tokens)?; + inner.store.save_with_refresh_telemetry( + &inner.keyring, + &inner.server_name, + &tokens, + )?; Ok(tokens) }) .await diff --git a/codex-rs/rmcp-client/src/oauth/ema_identity.rs b/codex-rs/rmcp-client/src/oauth/ema_identity.rs index 6f804a245f63..bf4ea0d17a33 100644 --- a/codex-rs/rmcp-client/src/oauth/ema_identity.rs +++ b/codex-rs/rmcp-client/src/oauth/ema_identity.rs @@ -45,7 +45,7 @@ impl StoredOAuthCredentialSnapshot { &self, keyring_store: &K, ) -> Result { - if !matches!(self.store, ResolvedOAuthCredentialStore::Keyring(_)) { + if self.store == ResolvedOAuthCredentialStore::file() { bail!("enterprise IdP credentials require keyring storage"); } let previous = &self.credentials; diff --git a/codex-rs/rmcp-client/src/oauth/ema_identity_tests.rs b/codex-rs/rmcp-client/src/oauth/ema_identity_tests.rs index c66721f3959b..bd804998d220 100644 --- a/codex-rs/rmcp-client/src/oauth/ema_identity_tests.rs +++ b/codex-rs/rmcp-client/src/oauth/ema_identity_tests.rs @@ -25,7 +25,7 @@ async fn keyring_failure_does_not_reuse_the_pinned_refresh_token() -> Result<()> let _lock = RefreshCredentialLock::acquire_for_server(&tokens.server_name, &tokens.url).await?; let snapshot = StoredOAuthCredentialSnapshot::new( tokens, - ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct), + ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct), ); let keyring = MockKeyringStore::default(); keyring.set_error( @@ -64,7 +64,7 @@ fn ordinary_oauth_names_cannot_alias_enterprise_credential_keys() -> Result<()> ); let keyring = MockKeyringStore::default(); - let store = ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct); + let store = ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct); let enterprise_tokens: StoredOAuthTokens = serde_json::from_value(json!({ "server_name": enterprise_name, "url": issuer, "issuer": issuer, "client_id": "idp-client", "token_response": { diff --git a/codex-rs/rmcp-client/src/oauth/refresh_transaction.rs b/codex-rs/rmcp-client/src/oauth/refresh_transaction.rs index e5341ca73b97..641f930c9265 100644 --- a/codex-rs/rmcp-client/src/oauth/refresh_transaction.rs +++ b/codex-rs/rmcp-client/src/oauth/refresh_transaction.rs @@ -246,13 +246,13 @@ impl OAuthPersistor { // TODO: Add a bounded persistence retry only if telemetry shows this is common; never // silently switch stores or continue with an unpersisted credential. debug!("persisting refreshed MCP OAuth credentials to the resolved store"); - if let Err(error) = - self.inner - .credential_store - .save(keyring_store, &self.inner.server_name, &refreshed) - { + let result = self.inner.credential_store.save_with_refresh_telemetry( + keyring_store, + &self.inner.server_name, + &refreshed, + ); + if let Err(error) = result { warn!( - error = %error, "failed to persist refreshed MCP OAuth credentials; returning the error and restoring the previous in-process credentials" ); install_tokens_in_manager(&mut guard, &latest, self.inner.oauth_config.as_ref()) diff --git a/codex-rs/rmcp-client/src/oauth/resolved_store.rs b/codex-rs/rmcp-client/src/oauth/resolved_store.rs index 904f5d967056..bdd29b58f94e 100644 --- a/codex-rs/rmcp-client/src/oauth/resolved_store.rs +++ b/codex-rs/rmcp-client/src/oauth/resolved_store.rs @@ -1,11 +1,15 @@ //! Resolves the configured MCP OAuth store and pins that concrete source for one client lifecycle. +//! Retains the originating client for later background storage observations. +use super::telemetry; use anyhow::Context; use anyhow::Result; use codex_config::types::AuthKeyringBackendKind; use codex_config::types::OAuthCredentialsStoreMode; use codex_keyring_store::KeyringStore; -use tracing::warn; +use codex_otel::auth_storage::AuthStorageOriginator; +use codex_otel::auth_storage::Operation; +use codex_otel::auth_storage::Store; use super::OAuthKeyringLoadError; use super::OAuthStore; @@ -29,13 +33,57 @@ use super::save_oauth_tokens_with_keyring; /// client that loaded credentials from one store must reread, refresh, persist, and remove only /// through that store. A mid-lifecycle backend failure is unexpected and must return an error /// rather than falling back to another possibly stale refresh token. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ResolvedOAuthCredentialStore { + pub(super) backend: Backend, + pub(super) mode: OAuthCredentialsStoreMode, + pub(super) kind: AuthKeyringBackendKind, + pub(super) originator: AuthStorageOriginator, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ResolvedOAuthCredentialStore { +pub(super) enum Backend { File, Keyring(AuthKeyringBackendKind), } +// Telemetry metadata must not change credential identity or trigger a reconnect. +impl PartialEq for ResolvedOAuthCredentialStore { + fn eq(&self, other: &Self) -> bool { + self.backend == other.backend + } +} +impl Eq for ResolvedOAuthCredentialStore {} + impl ResolvedOAuthCredentialStore { + pub(crate) fn file() -> Self { + Self { + backend: Backend::File, + mode: OAuthCredentialsStoreMode::File, + kind: AuthKeyringBackendKind::Direct, + originator: AuthStorageOriginator::current(), + } + } + + pub(crate) fn keyring(kind: AuthKeyringBackendKind) -> Self { + Self { + backend: Backend::Keyring(kind), + mode: OAuthCredentialsStoreMode::Keyring, + kind, + originator: AuthStorageOriginator::current(), + } + } + + fn with_policy( + mut self, + mode: OAuthCredentialsStoreMode, + kind: AuthKeyringBackendKind, + ) -> Self { + self.mode = mode; + self.kind = kind; + self + } + /// Loads credentials only from this already-resolved authority. /// /// Unlike `resolve_oauth_tokens_from_store_policy`, this never evaluates configured @@ -46,10 +94,11 @@ impl ResolvedOAuthCredentialStore { server_name: &str, url: &str, ) -> Result> { - match self { - Self::File => load_oauth_tokens_from_file(server_name, url) + let mut observation = telemetry::resolved(self, Operation::Load); + let result = match self.backend { + Backend::File => load_oauth_tokens_from_file(server_name, url) .context("failed to reread OAuth tokens from resolved file storage"), - Self::Keyring(keyring_backend_kind) => load_oauth_tokens_from_keyring( + Backend::Keyring(keyring_backend_kind) => load_oauth_tokens_from_keyring( keyring_store, keyring_backend_kind, server_name, @@ -59,7 +108,14 @@ impl ResolvedOAuthCredentialStore { .context( "failed to reread OAuth tokens from resolved keyring storage; refusing file fallback", ), + }; + observation.record_load_attempt(telemetry::store(self), &result); + if matches!(self.backend, Backend::Keyring(_)) + && let Err(error) = &result + { + telemetry::record_secure_error(&mut observation, error.as_ref()); } + result } /// Reads the selected authority without waiting for its aggregate-store lock. @@ -69,16 +125,18 @@ impl ResolvedOAuthCredentialStore { server_name: &str, url: &str, ) -> Result> { - match self { - Self::File => { + match self.backend { + Backend::File => { let _store_lock = OAuthStoreLock::try_acquire_for_read(OAuthStore::File)?; load_oauth_tokens_from_file_with_lock_held(server_name, url) .context("failed to probe OAuth tokens from resolved file storage") } - Self::Keyring(AuthKeyringBackendKind::Direct) => { - self.load(keyring_store, server_name, url) + Backend::Keyring(AuthKeyringBackendKind::Direct) => { + load_oauth_tokens_from_keyring(keyring_store, AuthKeyringBackendKind::Direct, server_name, url) + .map_err(anyhow::Error::from) + .context("failed to reread OAuth tokens from resolved keyring storage; refusing file fallback") } - Self::Keyring(AuthKeyringBackendKind::Secrets) => { + Backend::Keyring(AuthKeyringBackendKind::Secrets) => { let _store_lock = OAuthStoreLock::try_acquire_for_read(OAuthStore::Secrets)?; load_oauth_tokens_from_secrets_keyring_with_lock_held( keyring_store, @@ -97,15 +155,41 @@ impl ResolvedOAuthCredentialStore { server_name: &str, tokens: &StoredOAuthTokens, ) -> Result<()> { - match self { - Self::File => save_oauth_tokens_to_file(tokens), - Self::Keyring(keyring_backend_kind) => save_oauth_tokens_with_keyring( + let mut observation = telemetry::resolved(self, Operation::Save); + let result = match self.backend { + Backend::File => save_oauth_tokens_to_file(tokens), + Backend::Keyring(keyring_backend_kind) => save_oauth_tokens_with_keyring( keyring_store, keyring_backend_kind, server_name, tokens, ), + }; + observation.record_save_attempt(telemetry::store(self), &result); + if matches!(self.backend, Backend::Keyring(_)) + && let Err(error) = &result + { + telemetry::record_secure_error(&mut observation, error.as_ref()); } + result + } + + /// Records the refresh-persistence milestone in addition to the underlying pinned save. + pub(crate) fn save_with_refresh_telemetry( + self, + keyring_store: &K, + server_name: &str, + tokens: &StoredOAuthTokens, + ) -> Result<()> { + let mut observation = telemetry::resolved(self, Operation::RefreshPersist); + let result = self.save(keyring_store, server_name, tokens); + observation.record_save_attempt(telemetry::store(self), &result); + if matches!(self.backend, Backend::Keyring(_)) + && let Err(error) = &result + { + telemetry::record_secure_error(&mut observation, error.as_ref()); + } + result } /// Deletes credentials only from this already-resolved authority. @@ -115,18 +199,24 @@ impl ResolvedOAuthCredentialStore { server_name: &str, url: &str, ) -> Result { - match self { - Self::File => { - let key = compute_store_key(server_name, url)?; - delete_oauth_tokens_from_file(&key) - } - Self::Keyring(AuthKeyringBackendKind::Direct) => { + let mut observation = telemetry::resolved(self, Operation::Delete); + let result = match self.backend { + Backend::File => compute_store_key(server_name, url) + .and_then(|key| delete_oauth_tokens_from_file(&key)), + Backend::Keyring(AuthKeyringBackendKind::Direct) => { delete_oauth_tokens_from_direct_keyring(keyring_store, server_name, url) } - Self::Keyring(AuthKeyringBackendKind::Secrets) => { + Backend::Keyring(AuthKeyringBackendKind::Secrets) => { delete_oauth_tokens_from_secrets_keyring(keyring_store, server_name, url) } + }; + observation.record_delete_attempt(telemetry::store(self), &result); + if matches!(self.backend, Backend::Keyring(_)) + && let Err(error) = &result + { + telemetry::record_secure_error(&mut observation, error.as_ref()); } + result } } @@ -143,6 +233,7 @@ pub(crate) fn resolve_oauth_tokens_from_store_policy Result> { + let mut observation = telemetry::policy(store_mode, keyring_backend_kind, Operation::Load); match store_mode { OAuthCredentialsStoreMode::Auto => { // Auto remains keyring-first at lifecycle startup. The returned source is then pinned @@ -152,59 +243,80 @@ pub(crate) fn resolve_oauth_tokens_from_store_policy Ok(Some(ResolvedOAuthTokens { tokens, - store: ResolvedOAuthCredentialStore::Keyring(keyring_backend_kind), + store: ResolvedOAuthCredentialStore::keyring(keyring_backend_kind) + .with_policy(store_mode, keyring_backend_kind), })), - Ok(None) => Ok( - load_oauth_tokens_from_file(server_name, url)?.map(|tokens| { - ResolvedOAuthTokens { - tokens, - store: ResolvedOAuthCredentialStore::File, - } - }), - ), + Ok(None) => { + let result = load_oauth_tokens_from_file(server_name, url); + observation.record_load_attempt(Store::File, &result); + Ok(result?.map(|tokens| ResolvedOAuthTokens { + tokens, + store: ResolvedOAuthCredentialStore::file() + .with_policy(store_mode, keyring_backend_kind), + })) + } // Auto may fall back when the keyring backend is unavailable, but a Secrets // aggregate-lock failure means authority may be changing. Consulting File in // that state could replay credentials hidden behind a newer Secrets entry. - Err(OAuthKeyringLoadError::StoreLock(error)) => Err(error.into()), + Err(OAuthKeyringLoadError::StoreLock(error)) => { + telemetry::record_secure_error(&mut observation, &error); + Err(error.into()) + } Err(error) => { - warn!("failed to read OAuth tokens from keyring: {error}"); - Ok(load_oauth_tokens_from_file(server_name, url) + telemetry::record_secure_error(&mut observation, &error); + let result = load_oauth_tokens_from_file(server_name, url); + observation.record_load_attempt(Store::File, &result); + Ok(result .with_context(|| { format!("failed to read OAuth tokens from keyring: {error}") })? .map(|tokens| ResolvedOAuthTokens { tokens, - store: ResolvedOAuthCredentialStore::File, + store: ResolvedOAuthCredentialStore::file() + .with_policy(store_mode, keyring_backend_kind), })) } } } - OAuthCredentialsStoreMode::File => Ok(load_oauth_tokens_from_file(server_name, url)?.map( - |tokens| ResolvedOAuthTokens { + OAuthCredentialsStoreMode::File => { + let result = load_oauth_tokens_from_file(server_name, url); + observation.record_load_attempt(Store::File, &result); + Ok(result?.map(|tokens| ResolvedOAuthTokens { tokens, - store: ResolvedOAuthCredentialStore::File, - }, - )), - OAuthCredentialsStoreMode::Keyring => Ok(load_oauth_tokens_from_keyring( - keyring_store, - keyring_backend_kind, - server_name, - url, - ) - .map_err(anyhow::Error::from) - .context("failed to read OAuth tokens from keyring")? - .map(|tokens| ResolvedOAuthTokens { - tokens, - store: ResolvedOAuthCredentialStore::Keyring(keyring_backend_kind), - })), + store: ResolvedOAuthCredentialStore::file() + .with_policy(store_mode, keyring_backend_kind), + })) + } + OAuthCredentialsStoreMode::Keyring => { + let result = load_oauth_tokens_from_keyring( + keyring_store, + keyring_backend_kind, + server_name, + url, + ); + observation.record_load_attempt(telemetry::keyring(keyring_backend_kind), &result); + if let Err(error) = &result { + telemetry::record_secure_error(&mut observation, error); + } + Ok(result + .map_err(anyhow::Error::from) + .context("failed to read OAuth tokens from keyring")? + .map(|tokens| ResolvedOAuthTokens { + tokens, + store: ResolvedOAuthCredentialStore::keyring(keyring_backend_kind) + .with_policy(store_mode, keyring_backend_kind), + })) + } } } @@ -215,24 +327,34 @@ pub(crate) fn try_resolve_oauth_tokens_from_store_policy Result> { - let load = |store: ResolvedOAuthCredentialStore| { - store - .try_load(keyring_store, server_name, url) - .map(|tokens| tokens.map(|tokens| ResolvedOAuthTokens { tokens, store })) + let mut observation = telemetry::policy(store_mode, keyring_backend_kind, Operation::Load); + let mut load = |store: ResolvedOAuthCredentialStore| { + let result = store.try_load(keyring_store, server_name, url); + observation.record_load_attempt(telemetry::store(store), &result); + if matches!(store.backend, Backend::Keyring(_)) + && let Err(error) = &result + { + telemetry::record_secure_error(&mut observation, error.as_ref()); + } + result.map(|tokens| tokens.map(|tokens| ResolvedOAuthTokens { tokens, store })) }; - let keyring = ResolvedOAuthCredentialStore::Keyring(keyring_backend_kind); + let keyring = ResolvedOAuthCredentialStore::keyring(keyring_backend_kind) + .with_policy(store_mode, keyring_backend_kind); match store_mode { - OAuthCredentialsStoreMode::File => load(ResolvedOAuthCredentialStore::File), + OAuthCredentialsStoreMode::File => { + load(ResolvedOAuthCredentialStore::file().with_policy(store_mode, keyring_backend_kind)) + } OAuthCredentialsStoreMode::Keyring => load(keyring), OAuthCredentialsStoreMode::Auto => match load(keyring) { Ok(Some(tokens)) => Ok(Some(tokens)), - Ok(None) => load(ResolvedOAuthCredentialStore::File), + Ok(None) => load( + ResolvedOAuthCredentialStore::file().with_policy(store_mode, keyring_backend_kind), + ), Err(error) if error.downcast_ref::().is_some() => Err(error), - Err(error) => { - warn!("failed to read OAuth tokens from keyring: {error}"); - load(ResolvedOAuthCredentialStore::File) - .with_context(|| format!("failed to read OAuth tokens from keyring: {error}")) - } + Err(error) => load( + ResolvedOAuthCredentialStore::file().with_policy(store_mode, keyring_backend_kind), + ) + .with_context(|| format!("failed to read OAuth tokens from keyring: {error}")), }, } } diff --git a/codex-rs/rmcp-client/src/oauth/telemetry.rs b/codex-rs/rmcp-client/src/oauth/telemetry.rs new file mode 100644 index 000000000000..cbfa47aeb74e --- /dev/null +++ b/codex-rs/rmcp-client/src/oauth/telemetry.rs @@ -0,0 +1,97 @@ +//! Maps MCP storage policy, pinned stores, and typed failures to credential-free metric dimensions. + +use super::OAuthKeyringLoadError; +use super::OAuthStoreLockFailure; +use super::ResolvedOAuthCredentialStore; +use super::resolved_store::Backend; +use codex_config::types::AuthKeyringBackendKind; +use codex_config::types::OAuthCredentialsStoreMode; +use codex_otel::auth_storage::AuthStorageOriginator; +use codex_otel::auth_storage::CredentialKind; +use codex_otel::auth_storage::Operation; +use codex_otel::auth_storage::StoragePhase; +use codex_otel::auth_storage::StorageTelemetry; +use codex_otel::auth_storage::Store; +use codex_otel::auth_storage::StoreMode; +use std::error::Error; +use std::io::ErrorKind; + +/// Preserve aggregate-lock categories even when a transparent load error hides their source. +pub(super) fn record_secure_error( + observation: &mut StorageTelemetry, + error: &(dyn Error + 'static), +) { + let mut cause = Some(error); + while let Some(current) = cause { + let lock = match current.downcast_ref::() { + Some(OAuthKeyringLoadError::StoreLock(lock)) => Some(lock), + Some(OAuthKeyringLoadError::Backend(_)) | None => { + current.downcast_ref::() + } + }; + if let Some(OAuthStoreLockFailure::Timeout { + acquire_timeout, .. + }) = lock + { + let kind = if acquire_timeout.is_zero() { + ErrorKind::WouldBlock + } else { + ErrorKind::TimedOut + }; + observation.record_secure_error(&std::io::Error::from(kind)); + return; + } + cause = current.source(); + } + observation.record_secure_error(error); +} + +pub(super) fn keyring(kind: AuthKeyringBackendKind) -> Store { + match kind { + AuthKeyringBackendKind::Direct => Store::DirectKeyring, + AuthKeyringBackendKind::Secrets => Store::Secrets, + } +} + +pub(super) fn store(store: ResolvedOAuthCredentialStore) -> Store { + match store.backend { + Backend::File => Store::File, + Backend::Keyring(kind) => keyring(kind), + } +} + +pub(super) fn policy( + mode: OAuthCredentialsStoreMode, + kind: AuthKeyringBackendKind, + operation: Operation, +) -> StorageTelemetry { + StorageTelemetry::new( + CredentialKind::Mcp, + store_mode(mode), + keyring(kind), + operation, + AuthStorageOriginator::current(), + ) +} + +pub(super) fn resolved( + source: ResolvedOAuthCredentialStore, + operation: Operation, +) -> StorageTelemetry { + StorageTelemetry::new( + CredentialKind::Mcp, + store_mode(source.mode), + keyring(source.kind), + operation, + source.originator, + ) + .with_phase(StoragePhase::Pinned) +} + +fn store_mode(mode: OAuthCredentialsStoreMode) -> StoreMode { + match mode { + OAuthCredentialsStoreMode::File => StoreMode::File, + OAuthCredentialsStoreMode::Auto => StoreMode::Auto, + OAuthCredentialsStoreMode::Keyring => StoreMode::Keyring, + } +} diff --git a/codex-rs/rmcp-client/src/oauth/tests/credential_store_tests.rs b/codex-rs/rmcp-client/src/oauth/tests/credential_store_tests.rs index 86882fc46490..575ed835cdb0 100644 --- a/codex-rs/rmcp-client/src/oauth/tests/credential_store_tests.rs +++ b/codex-rs/rmcp-client/src/oauth/tests/credential_store_tests.rs @@ -36,7 +36,7 @@ async fn mutations_require_and_retain_the_transaction_guard() -> Result<()> { save_oauth_tokens_to_file(&initial)?; let store = OAuthCredentialStore::new( initial.clone(), - ResolvedOAuthCredentialStore::File, + ResolvedOAuthCredentialStore::file(), MockKeyringStore::default(), /*oauth_config*/ None, ); @@ -97,7 +97,7 @@ async fn save_publishes_only_persisted_credentials() -> Result<()> { let _env = TempCodexHome::new(); let initial = sample_tokens(); let keyring = MockKeyringStore::default(); - let authority = ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct); + let authority = ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct); authority.save(&keyring, &initial.server_name, &initial)?; let mut fallback = initial.clone(); fallback @@ -167,7 +167,7 @@ async fn pinned_read_failure_does_not_adopt_fallback_credentials() -> Result<()> keyring.set_error(&key, KeyringError::Invalid("test".into(), "load".into())); let store = OAuthCredentialStore::new( initial.clone(), - ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct), + ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct), keyring, /*oauth_config*/ None, ); @@ -194,7 +194,7 @@ async fn replacement_or_removal_does_not_acknowledge_a_new_runtime_snapshot() -> save_oauth_tokens_to_file(&initial)?; let store = OAuthCredentialStore::new( initial.clone(), - ResolvedOAuthCredentialStore::File, + ResolvedOAuthCredentialStore::file(), MockKeyringStore::default(), /*oauth_config*/ None, ); @@ -238,7 +238,7 @@ async fn storage_roundtrip_preserves_absolute_and_unknown_expiry() -> Result<()> save_oauth_tokens_to_file(&initial)?; let store = OAuthCredentialStore::new( initial.clone(), - ResolvedOAuthCredentialStore::File, + ResolvedOAuthCredentialStore::file(), MockKeyringStore::default(), /*oauth_config*/ None, ); diff --git a/codex-rs/rmcp-client/src/oauth/tests/persistor_tests.rs b/codex-rs/rmcp-client/src/oauth/tests/persistor_tests.rs index 2951504e5953..e938bcec488a 100644 --- a/codex-rs/rmcp-client/src/oauth/tests/persistor_tests.rs +++ b/codex-rs/rmcp-client/src/oauth/tests/persistor_tests.rs @@ -179,7 +179,7 @@ async fn coordinated_401_refresh_rereads_and_persists_before_retry() -> Result<( let mut manager = authorization_manager_for(&initial).await?; manager.set_credential_store(OAuthCredentialStore::new( initial.clone(), - ResolvedOAuthCredentialStore::File, + ResolvedOAuthCredentialStore::file(), DefaultKeyringStore, /*oauth_config*/ None, )); @@ -396,7 +396,7 @@ async fn resolved_keyring_read_error_preserves_in_memory_credentials() -> Result initial.server_name.clone(), initial.url.clone(), Arc::clone(&manager), - ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct), + ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct), Some(initial.clone()), /*oauth_config*/ None, ); @@ -875,7 +875,7 @@ async fn coordinated_manager_for( let mut manager = authorization_manager_for(tokens).await?; let store = OAuthCredentialStore::new( tokens.clone(), - ResolvedOAuthCredentialStore::File, + ResolvedOAuthCredentialStore::file(), DefaultKeyringStore, /*oauth_config*/ None, ); @@ -888,13 +888,13 @@ async fn persistor_for(tokens: &StoredOAuthTokens) -> Result { tokens.server_name.clone(), tokens.url.clone(), Arc::new(TokioMutex::new(authorization_manager_for(tokens).await?)), - ResolvedOAuthCredentialStore::File, + ResolvedOAuthCredentialStore::file(), Some(tokens.clone()), /*oauth_config*/ None, )) } -async fn test_context() -> Result<(TempCodexHome, MockServer, StoredOAuthTokens)> { +pub(super) async fn test_context() -> Result<(TempCodexHome, MockServer, StoredOAuthTokens)> { let env = TempCodexHome::new(); let server = MockServer::start().await; mount_oauth_metadata(&server).await; @@ -902,7 +902,9 @@ async fn test_context() -> Result<(TempCodexHome, MockServer, StoredOAuthTokens) Ok((env, server, tokens)) } -async fn authorization_manager_for(tokens: &StoredOAuthTokens) -> Result { +pub(super) async fn authorization_manager_for( + tokens: &StoredOAuthTokens, +) -> Result { let oauth_http_client = Arc::new(OAuthHttpClientAdapter::new( Arc::new(RouteAwareHttpClient::new(HttpClientFactory::new( OutboundProxyPolicy::ReqwestDefault, diff --git a/codex-rs/rmcp-client/src/oauth/tests/store_lock_tests.rs b/codex-rs/rmcp-client/src/oauth/tests/store_lock_tests.rs index b8428f5e5e97..8213c5ce4252 100644 --- a/codex-rs/rmcp-client/src/oauth/tests/store_lock_tests.rs +++ b/codex-rs/rmcp-client/src/oauth/tests/store_lock_tests.rs @@ -176,7 +176,7 @@ fn legacy_rmcp_oauth_keyring_credentials_remain_readable() -> Result<()> { assert_eq!( resolved.store, - crate::oauth::ResolvedOAuthCredentialStore::Keyring(AuthKeyringBackendKind::Direct) + crate::oauth::ResolvedOAuthCredentialStore::keyring(AuthKeyringBackendKind::Direct) ); assert_tokens_match_without_expiry(&resolved.tokens, &expected); assert!(crate::oauth::oauth_tokens_are_usable(&resolved.tokens)); diff --git a/codex-rs/rmcp-client/src/oauth/tests/telemetry_tests.rs b/codex-rs/rmcp-client/src/oauth/tests/telemetry_tests.rs new file mode 100644 index 000000000000..a3af6f2a4723 --- /dev/null +++ b/codex-rs/rmcp-client/src/oauth/tests/telemetry_tests.rs @@ -0,0 +1,460 @@ +//! Exercises emitted metrics through MCP storage policy and pinned-store operations. + +use super::persistor_tests::authorization_manager_for; +use super::persistor_tests::test_context; +use super::*; +use codex_otel::auth_storage::AuthStorageOriginator; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::MetricData; +use pretty_assertions::assert_eq; +use std::collections::BTreeMap; + +type MetricRecord = (String, u64, BTreeMap); + +#[test] +fn policy_and_pinned_operations_keep_distinct_outcomes_and_originator() -> Result<()> { + if isolated_process()? { + return Ok(()); + } + let _env = TempCodexHome::new(); + keyring::set_default_credential_builder(keyring::mock::default_credential_builder()); + let metrics = test_metrics()?; + let store = MockKeyringStore::default(); + let tokens = sample_tokens(); + let key = compute_store_key(&tokens.server_name, &tokens.url)?; + store.set_error( + &key, + KeyringError::PlatformFailure(std::io::Error::from(std::io::ErrorKind::Unsupported).into()), + ); + save_oauth_tokens_with_keyring_with_fallback_to_file( + &store, + AuthKeyringBackendKind::Direct, + &tokens.server_name, + &tokens, + )?; + store.save(KEYRING_SERVICE, &key, &serde_json::to_string(&tokens)?)?; + let resolved = AuthStorageOriginator::from_client_name("codex_vscode") + .sync_scope(|| { + resolve_oauth_tokens_from_store_policy( + &store, + &tokens.server_name, + &tokens.url, + OAuthCredentialsStoreMode::Auto, + AuthKeyringBackendKind::Direct, + ) + })? + .unwrap(); + let snapshot = StoredOAuthCredentialSnapshot::new(resolved.tokens, resolved.store); + store.set_error( + &key, + KeyringError::PlatformFailure(std::io::Error::from(std::io::ErrorKind::TimedOut).into()), + ); + // Pinned operations retain the original Auto policy and client on another thread. + std::thread::scope(|scope| { + scope + .spawn(|| { + // The SDK mock is empty; this probe must emit exactly one pinned observation. + assert_eq!( + snapshot.reload( + &tokens.server_name, + &tokens.url, + OAuthCredentialsStoreMode::Auto, + AuthKeyringBackendKind::Direct, + )?, + None + ); + assert!( + resolved + .store + .save(&store, &tokens.server_name, &tokens) + .is_err() + ); + Ok::<_, anyhow::Error>(()) + }) + .join() + .unwrap() + })?; + let mut expected = vec![ + expected_metric( + /*count*/ 1, + &[ + ("operation", "save"), + ("actual_store", "file"), + ("secure_outcome", "error"), + ("fallback_reason", "secure_error"), + ("secure_error", "unsupported"), + ("originator", "none"), + ], + ), + expected_metric(/*count*/ 1, &[]), + expected_metric( + /*count*/ 1, + &[ + ("storage_phase", "pinned"), + ("secure_outcome", "not_found"), + ("outcome", "not_found"), + ], + ), + expected_metric( + /*count*/ 1, + &[ + ("storage_phase", "pinned"), + ("operation", "save"), + ("secure_outcome", "error"), + ("outcome", "error"), + ("secure_error", "timeout"), + ], + ), + ]; + expected.sort(); + assert_eq!(operation_metrics(&metrics)?, expected); + metrics.shutdown()?; + Ok(()) +} + +fn isolated_process() -> Result { + if std::env::var_os("CODEX_AUTH_STORAGE_TEST_CHILD").is_none() { + let output = std::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + std::thread::current().name().unwrap(), + "--nocapture", + ]) + .env("CODEX_AUTH_STORAGE_TEST_CHILD", "1") + .output()?; + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return Ok(true); + } + Ok(false) +} + +fn operation_metrics(metrics: &codex_otel::MetricsClient) -> Result> { + let snapshot = metrics.snapshot()?; + let mut actual = Vec::new(); + for metric in snapshot + .scope_metrics() + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + { + if matches!( + metric.name(), + "codex.auth_storage.operation" | "codex.auth_storage.refresh_persist" + ) && let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() + { + for point in sum.data_points() { + actual.push(( + metric.name().to_string(), + point.value(), + point + .attributes() + .map(|a| (a.key.to_string(), a.value.to_string())) + .collect::>(), + )); + } + } + } + actual.sort(); + Ok(actual) +} + +fn expected_metric(count: u64, tags: &[(&str, &str)]) -> MetricRecord { + let mut attributes = BTreeMap::from([ + ("credential_kind", "mcp"), + ("store_mode", "auto"), + ("selected_store", "direct_keyring"), + ("actual_store", "direct_keyring"), + ("operation", "load"), + ("secure_outcome", "success"), + ("outcome", "success"), + ("fallback_reason", "none"), + ("secure_error", "none"), + ("storage_phase", "policy"), + ("originator", "codex_vscode"), + ]); + attributes.extend(tags.iter().copied()); + ( + if attributes["operation"] == "refresh_persist" { + "codex.auth_storage.refresh_persist" + } else { + "codex.auth_storage.operation" + } + .into(), + count, + attributes + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + ) +} + +fn test_metrics() -> Result { + Ok(codex_otel::install_global_metrics( + codex_otel::MetricsClient::new( + codex_otel::MetricsConfig::in_memory("test", "test", "1", Default::default()) + .with_runtime_reader(), + )?, + )) +} + +#[test] +fn cleanup_failure_retains_policy_and_remains_best_effort() -> Result<()> { + if isolated_process()? { + return Ok(()); + } + let _env = TempCodexHome::new(); + let metrics = test_metrics()?; + let tokens = sample_tokens(); + let key = compute_store_key(&tokens.server_name, &tokens.url)?; + std::fs::write(fallback_file_path()?, "{")?; + let mut expected = Vec::new(); + AuthStorageOriginator::from_client_name("codex_vscode").sync_scope(|| -> Result<()> { + for (mode, mode_tag) in [ + (OAuthCredentialsStoreMode::Auto, "auto"), + (OAuthCredentialsStoreMode::Keyring, "keyring"), + ] { + let store = MockKeyringStore::default(); + save_oauth_tokens_with_keyring_and_cleanup_file( + &store, + mode, + AuthKeyringBackendKind::Direct, + &tokens.server_name, + &tokens, + )?; + assert_eq!( + store.saved_value(&key), + Some(serde_json::to_string(&tokens)?) + ); + expected.push(expected_metric( + /*count*/ 1, + &[ + ("store_mode", mode_tag), + ("operation", "cleanup"), + ("actual_store", "file"), + ("secure_outcome", "not_attempted"), + ("outcome", "error"), + ], + )); + } + Ok(()) + })?; + expected.sort(); + assert_eq!(operation_metrics(&metrics)?, expected); + metrics.shutdown()?; + Ok(()) +} + +#[test] +fn lock_errors_retain_typed_categories_through_wrappers() -> Result<()> { + if isolated_process()? { + return Ok(()); + } + let metrics = test_metrics()?; + let mut expected = Vec::new(); + AuthStorageOriginator::from_client_name("codex_vscode").sync_scope(|| { + for (duration, category) in [ + (Duration::ZERO, "locked"), + (Duration::from_secs(/*secs*/ 1), "timeout"), + ] { + for wrapped in [false, true] { + let error = OAuthStoreLockFailure::Timeout { + store: OAuthStore::Secrets, + path: "private-lock-path".into(), + acquire_timeout: duration, + }; + let error = if wrapped { + Error::from(OAuthKeyringLoadError::StoreLock(error)).context("private context") + } else { + Error::from(error) + }; + let result: Result> = Err(error); + let mut observation = telemetry::policy( + OAuthCredentialsStoreMode::Auto, + AuthKeyringBackendKind::Secrets, + Operation::Load, + ); + observation.record_load_attempt(Store::Secrets, &result); + telemetry::record_secure_error( + &mut observation, + result.as_ref().unwrap_err().as_ref(), + ); + } + expected.push(expected_metric( + /*count*/ 2, + &[ + ("selected_store", "secrets"), + ("actual_store", "secrets"), + ("secure_outcome", "error"), + ("outcome", "error"), + ("secure_error", category), + ], + )); + } + }); + expected.sort(); + assert_eq!(operation_metrics(&metrics)?, expected); + metrics.shutdown()?; + Ok(()) +} + +#[tokio::test(flavor = "current_thread")] +async fn coordinated_refresh_persistence_records_success_and_failure() -> Result<()> { + use rmcp::transport::auth::CredentialStore; + + if isolated_process()? { + return Ok(()); + } + let _env = TempCodexHome::new(); + let metrics = test_metrics()?; + for fail_save in [false, true] { + let keyring = MockKeyringStore::default(); + let tokens = sample_tokens(); + let key = compute_store_key(&tokens.server_name, &tokens.url)?; + keyring.save(KEYRING_SERVICE, &key, &serde_json::to_string(&tokens)?)?; + let resolved = AuthStorageOriginator::from_client_name("codex_vscode") + .sync_scope(|| { + resolve_oauth_tokens_from_store_policy( + &keyring, + &tokens.server_name, + &tokens.url, + OAuthCredentialsStoreMode::Auto, + AuthKeyringBackendKind::Direct, + ) + })? + .unwrap(); + let store = OAuthCredentialStore::new( + tokens, + resolved.store, + keyring.clone(), + /*oauth_config*/ None, + ); + let _guard = store.acquire_transaction_guard().await?; + let mut credentials = store.load().await?.unwrap(); + credentials + .token_response + .as_mut() + .unwrap() + .set_access_token(AccessToken::new("refreshed-access".into())); + if fail_save { + keyring.set_error( + &key, + KeyringError::PlatformFailure( + std::io::Error::from(std::io::ErrorKind::TimedOut).into(), + ), + ); + } + let _ = operation_metrics(&metrics)?; + assert_eq!(store.save(credentials).await.is_err(), fail_save); + let outcome = if fail_save { "error" } else { "success" }; + let expected = ["save", "refresh_persist"].map(|operation| { + expected_metric( + /*count*/ 1, + &[ + ("operation", operation), + ("storage_phase", "pinned"), + ("secure_outcome", outcome), + ("outcome", outcome), + ("secure_error", if fail_save { "timeout" } else { "none" }), + ], + ) + }); + assert_eq!(operation_metrics(&metrics)?, expected); + } + metrics.shutdown()?; + Ok(()) +} + +#[expect( + clippy::await_holding_invalid_type, + reason = "The legacy authorization manager serializes refresh through its Tokio mutex" +)] +#[tokio::test(flavor = "current_thread")] +async fn legacy_refresh_persistence_records_each_write_once() -> Result<()> { + use wiremock::Mock; + use wiremock::ResponseTemplate; + use wiremock::matchers::method; + use wiremock::matchers::path; + + if isolated_process()? { + return Ok(()); + } + let metrics = test_metrics()?; + for proactive in [false, true] { + for fail_save in [false, true] { + let (_env, server, initial) = test_context().await?; + save_oauth_tokens_to_file(&initial)?; + let file = fallback_file_path()?; + Mock::given(method("POST")) + .and(path("/oauth/token")) + .respond_with(move |_: &wiremock::Request| { + if fail_save { + std::fs::write(&file, "{").unwrap(); + } + ResponseTemplate::new(/*s*/ 200).set_body_json(serde_json::json!({ + "access_token": "refreshed-access", "token_type": "Bearer", + "refresh_token": "rotated-refresh", "expires_in": 3600, + })) + }) + .expect(1) + .mount(&server) + .await; + let manager = Arc::new(Mutex::new(authorization_manager_for(&initial).await?)); + let authority = AuthStorageOriginator::from_client_name("codex_vscode") + .sync_scope(ResolvedOAuthCredentialStore::file); + let persistor = OAuthPersistor::new( + initial.server_name.clone(), + initial.url.clone(), + Arc::clone(&manager), + authority, + Some(initial), + /*oauth_config*/ None, + ); + let result = if proactive { + persistor.refresh_if_needed().await + } else { + manager.lock().await.refresh_token().await?; + persistor.persist_if_needed().await + }; + assert_eq!(result.is_err(), fail_save); + let mut expected = Vec::new(); + for operation in if proactive { + vec!["load", "save", "refresh_persist"] + } else { + vec!["save", "refresh_persist"] + } { + expected.push(expected_metric( + /*count*/ 1, + &[ + ("operation", operation), + ("store_mode", "file"), + ("actual_store", "file"), + ("storage_phase", "pinned"), + ("secure_outcome", "not_attempted"), + ( + "outcome", + if fail_save && operation != "load" { + "error" + } else { + "success" + }, + ), + ], + )); + } + expected.sort(); + assert_eq!(operation_metrics(&metrics)?, expected); + if !fail_save { + // The proactive path installs the persisted response; the reactive path updates + // its snapshot. Neither should emit another milestone without another write. + persistor.persist_if_needed().await?; + assert_eq!(operation_metrics(&metrics)?, Vec::new()); + } + server.verify().await; + } + } + metrics.shutdown()?; + Ok(()) +} diff --git a/codex-rs/rmcp-client/src/perform_oauth_login.rs b/codex-rs/rmcp-client/src/perform_oauth_login.rs index a271ed342e09..a9053f6b7f48 100644 --- a/codex-rs/rmcp-client/src/perform_oauth_login.rs +++ b/codex-rs/rmcp-client/src/perform_oauth_login.rs @@ -11,6 +11,7 @@ use anyhow::anyhow; use anyhow::bail; use codex_config::McpServerOAuthConfig; use codex_exec_server::HttpClient; +use codex_otel::auth_storage::AuthStorageOriginator; use rmcp::transport::AuthorizationManager; use rmcp::transport::AuthorizationSession; use rmcp::transport::auth::AuthorizationMetadata; @@ -846,14 +847,15 @@ impl OauthLoginFlow { let server_name = self.server_name.clone(); let (tx, rx) = oneshot::channel(); - tokio::spawn(async move { + let task = async move { let result = self.finish(/*emit_browser_url*/ false).await; if let Err(err) = &result { eprintln!("Failed to complete OAuth login for '{server_name}': {err:#}"); } let _ = tx.send(result); - }); + }; + tokio::spawn(AuthStorageOriginator::current().scope(task)); rx } diff --git a/codex-rs/rmcp-client/src/rmcp_client.rs b/codex-rs/rmcp-client/src/rmcp_client.rs index b8817d2ebadc..39dc2cdda454 100644 --- a/codex-rs/rmcp-client/src/rmcp_client.rs +++ b/codex-rs/rmcp-client/src/rmcp_client.rs @@ -21,6 +21,7 @@ use codex_config::types::AuthKeyringBackendKind; use codex_config::types::McpServerEnvVar; use codex_exec_server::HttpClient; use codex_keyring_store::DefaultKeyringStore; +use codex_otel::auth_storage::AuthStorageOriginator; use futures::FutureExt; use futures::future::BoxFuture; use http::HeaderMap; @@ -168,6 +169,7 @@ enum TransportRecipe { store_mode: OAuthCredentialsStoreMode, keyring_backend_kind: AuthKeyringBackendKind, pinned_credential_store: Arc>, + originator: AuthStorageOriginator, http_client: Arc, auth_provider: Option, redirect_mode: StreamableHttpRedirectMode, @@ -608,6 +610,7 @@ impl RmcpClient { store_mode, keyring_backend_kind, pinned_credential_store: Arc::new(OnceLock::new()), + originator: AuthStorageOriginator::current(), http_client, auth_provider, redirect_mode, @@ -1113,6 +1116,7 @@ impl RmcpClient { store_mode, keyring_backend_kind, pinned_credential_store, + originator, http_client, auth_provider, redirect_mode, @@ -1148,13 +1152,18 @@ impl RmcpClient { let oauth_keyring_backend_kind = *keyring_backend_kind; let pinned_credential_store = Arc::clone(pinned_credential_store); - tokio::task::spawn_blocking(move || -> Result> { + // Recovery can run after the startup scope ends, even when startup found + // no credentials and therefore could not pin a store with this attribution. + let originator = *originator; + let load = move || -> Result> { if let Some(store) = pinned_credential_store.get().copied() { // Rebuilds reread the source selected during first construction. Only // initial construction below evaluates configured store policy. return store .load(&DefaultKeyringStore, &oauth_server_name, &oauth_url) - .map(|tokens| tokens.map(|tokens| ResolvedOAuthTokens { tokens, store })); + .map(|tokens| { + tokens.map(|tokens| ResolvedOAuthTokens { tokens, store }) + }); } match resolve_oauth_tokens_from_store_policy( @@ -1183,9 +1192,12 @@ impl RmcpClient { Ok(None) } } - }) - .await - .map_err(|error| anyhow!("OAuth credential loading task failed: {error}"))?? + }; + tokio::task::spawn_blocking(move || originator.sync_scope(load)) + .await + .map_err(|error| { + anyhow!("OAuth credential loading task failed: {error}") + })?? } else { None }; diff --git a/codex-rs/rmcp-client/tests/streamable_http_recovery.rs b/codex-rs/rmcp-client/tests/streamable_http_recovery.rs index 817587a9b76a..8ac200fe0356 100644 --- a/codex-rs/rmcp-client/tests/streamable_http_recovery.rs +++ b/codex-rs/rmcp-client/tests/streamable_http_recovery.rs @@ -1,5 +1,8 @@ mod streamable_http_test_support; +#[path = "streamable_http_recovery/telemetry_tests.rs"] +mod telemetry_tests; + use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; diff --git a/codex-rs/rmcp-client/tests/streamable_http_recovery/telemetry_tests.rs b/codex-rs/rmcp-client/tests/streamable_http_recovery/telemetry_tests.rs new file mode 100644 index 000000000000..71171c5f73f5 --- /dev/null +++ b/codex-rs/rmcp-client/tests/streamable_http_recovery/telemetry_tests.rs @@ -0,0 +1,152 @@ +//! Checks attribution when a session rebuild discovers credentials after anonymous startup. + +use super::streamable_http_test_support::arm_session_post_failure; +use super::streamable_http_test_support::call_echo_tool; +use super::streamable_http_test_support::expected_echo_result; +use super::streamable_http_test_support::initialize_client; +use super::streamable_http_test_support::spawn_streamable_http_server; +use codex_config::types::AuthKeyringBackendKind; +use codex_config::types::OAuthCredentialsStoreMode; +use codex_exec_server::Environment; +use codex_otel::MetricsClient; +use codex_otel::MetricsConfig; +use codex_otel::auth_storage::AuthStorageOriginator; +use codex_rmcp_client::RmcpClient; +use codex_rmcp_client::StoredOAuthTokens; +use codex_rmcp_client::WrappedOAuthTokenResponse; +use codex_rmcp_client::save_oauth_tokens; +use oauth2::AccessToken; +use oauth2::basic::BasicTokenType; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::MetricData; +use pretty_assertions::assert_eq; +use rmcp::transport::auth::OAuthTokenResponse; +use std::collections::BTreeMap; + +#[tokio::test] +async fn recovery_retains_originator_without_initial_credentials() -> anyhow::Result<()> { + if std::env::var_os("CODEX_MCP_RECOVERY_TELEMETRY_TEST_CHILD").is_none() { + let home = tempfile::tempdir()?; + let output = tokio::process::Command::new(std::env::current_exe()?) + .args([ + "--exact", + "telemetry_tests::recovery_retains_originator_without_initial_credentials", + "--nocapture", + ]) + .env("CODEX_MCP_RECOVERY_TELEMETRY_TEST_CHILD", "1") + .env("CODEX_HOME", home.path()) + .output() + .await?; + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return Ok(()); + } + let metrics = codex_otel::install_global_metrics(MetricsClient::new( + MetricsConfig::in_memory("test", "test", "1", Default::default()).with_runtime_reader(), + )?); + let (_server, base_url) = spawn_streamable_http_server().await?; + let url = format!("{base_url}/mcp"); + let client = AuthStorageOriginator::from_client_name("codex_vscode") + .scope(async { + let client = RmcpClient::new_streamable_http_client( + "recovery-telemetry", + &url, + /*bearer_token*/ None, + /*http_headers*/ None, + /*env_http_headers*/ None, + OAuthCredentialsStoreMode::File, + AuthKeyringBackendKind::Direct, + Environment::default_for_tests().get_http_client(), + /*auth_provider*/ None, + ) + .await?; + initialize_client(&client).await?; + anyhow::Ok(client) + }) + .await?; + assert_eq!( + call_echo_tool(&client, "warmup").await?, + expected_echo_result("warmup") + ); + + for recovery in 0..3 { + if recovery == 1 { + // Credentials arrive after startup and the first anonymous recovery. + save_oauth_tokens( + "recovery-telemetry", + &StoredOAuthTokens { + server_name: "recovery-telemetry".into(), + url: url.clone(), + issuer: Some(url.clone()), + client_id: "client-id".into(), + token_response: WrappedOAuthTokenResponse(OAuthTokenResponse::new( + AccessToken::new("saved-access-token".into()), + BasicTokenType::Bearer, + Default::default(), + )), + expires_at: None, + }, + OAuthCredentialsStoreMode::File, + AuthKeyringBackendKind::Direct, + ) + .await?; + } + arm_session_post_failure( + &base_url, + /*status*/ 404, + /*remaining*/ 1, + /*www_authenticate_headers*/ &[], + ) + .await?; + assert_eq!( + call_echo_tool(&client, "recovered").await?, + expected_echo_result("recovered") + ); + } + + let snapshot = metrics.snapshot()?; + let mut loads = Vec::new(); + for metric in snapshot + .scope_metrics() + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + { + if metric.name() == "codex.auth_storage.operation" + && let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() + { + for point in sum.data_points() { + let tags: BTreeMap<_, _> = point + .attributes() + .map(|tag| (tag.key.to_string(), tag.value.to_string())) + .collect(); + if tags["credential_kind"] == "mcp" && tags["operation"] == "load" { + loads.push(( + tags["originator"].clone(), + tags["storage_phase"].clone(), + tags["outcome"].clone(), + point.value(), + )); + } + } + } + } + loads.sort(); + assert_eq!( + loads, + vec![ + ("codex_vscode".into(), "pinned".into(), "success".into(), 1), + ( + "codex_vscode".into(), + "policy".into(), + "not_found".into(), + 2 + ), + ("codex_vscode".into(), "policy".into(), "success".into(), 1), + ] + ); + metrics.shutdown()?; + Ok(()) +} diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index 0361312cabba..6b4cc42c4392 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -74,6 +74,8 @@ pub fn default_filter() -> Targets { .with_target("codex_otel.log_only", LevelFilter::OFF) .with_target("codex_otel.trace_safe", LevelFilter::OFF) .with_target("rmcp", LevelFilter::INFO) + .with_target("tokio_graceful::guard", LevelFilter::DEBUG) + .with_target("tokio_graceful::trigger", LevelFilter::DEBUG) .with_target("codex_api::responses_websocket_timing", LevelFilter::OFF) .with_target("codex_core::post_sampling_token_estimate", LevelFilter::OFF) // Full model request bodies and streamed response payloads overwhelm the diff --git a/codex-rs/state/src/log_db_filter_tests.rs b/codex-rs/state/src/log_db_filter_tests.rs index d628dd834a27..3f04e1049cdf 100644 --- a/codex-rs/state/src/log_db_filter_tests.rs +++ b/codex-rs/state/src/log_db_filter_tests.rs @@ -49,6 +49,11 @@ async fn sqlite_sink_filters_noisy_targets_without_dropping_useful_diagnostics() tracing::warn!(target: "h2::proto::streams", "retained-http2-warning"); tracing::debug!(target: "rmcp::transport", "dropped-rmcp-debug"); tracing::info!(target: "rmcp::transport", "retained-rmcp-info"); + tracing::trace!(target: "tokio_graceful::guard", "dropped-guard-trace"); + tracing::debug!(target: "tokio_graceful::guard", "retained-guard-debug"); + tracing::trace!(target: "tokio_graceful::trigger", "dropped-trigger-trace"); + tracing::debug!(target: "tokio_graceful::trigger", "retained-trigger-debug"); + tracing::trace!(target: "tokio_graceful::shutdown", "retained-shutdown-trace"); tracing::debug!( target: "codex_rmcp_client::oauth", "dropped-codex-rmcp-client-debug" @@ -118,6 +123,21 @@ async fn sqlite_sink_filters_noisy_targets_without_dropping_useful_diagnostics() ), ("WARN", "h2::proto::streams", Some("retained-http2-warning")), ("INFO", "rmcp::transport", Some("retained-rmcp-info")), + ( + "DEBUG", + "tokio_graceful::guard", + Some("retained-guard-debug") + ), + ( + "DEBUG", + "tokio_graceful::trigger", + Some("retained-trigger-debug") + ), + ( + "TRACE", + "tokio_graceful::shutdown", + Some("retained-shutdown-trace") + ), ( "INFO", "codex_rmcp_client::oauth", diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 2adc20c5b491..3cf16222264d 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -43,6 +43,7 @@ mod backfill; mod external_agent_config_imports; mod goals; mod logs; +mod logs_maintenance; mod memories; mod memory_versions; mod projects; @@ -276,12 +277,7 @@ impl StateRuntime { runtime.close().await; return Err(err); } - if let Err(err) = runtime.run_logs_startup_maintenance().await { - warn!( - "failed to run startup maintenance for logs db at {}: {err}", - logs_path.display(), - ); - } + runtime.start_periodic_logs_maintenance(std::time::Duration::from_secs(30 * 60)); Ok(runtime) } diff --git a/codex-rs/state/src/runtime/logs.rs b/codex-rs/state/src/runtime/logs.rs index 0cdc91124c56..9b331b491abe 100644 --- a/codex-rs/state/src/runtime/logs.rs +++ b/codex-rs/state/src/runtime/logs.rs @@ -1,7 +1,5 @@ use super::*; -const LOG_RETENTION_DAYS: i64 = 10; - impl StateRuntime { pub async fn insert_log(&self, entry: &LogEntry) -> anyhow::Result<()> { self.insert_logs(std::slice::from_ref(entry)).await @@ -280,30 +278,6 @@ WHERE id IN ( Ok(()) } - pub(crate) async fn delete_logs_before(&self, cutoff_ts: i64) -> anyhow::Result { - let result = sqlx::query("DELETE FROM logs WHERE ts < ?") - .bind(cutoff_ts) - .execute(self.logs_pool.as_ref()) - .await?; - Ok(result.rows_affected()) - } - - pub(crate) async fn run_logs_startup_maintenance(&self) -> anyhow::Result<()> { - let Some(cutoff) = - Utc::now().checked_sub_signed(chrono::Duration::days(LOG_RETENTION_DAYS)) - else { - return Ok(()); - }; - self.delete_logs_before(cutoff.timestamp()).await?; - // Startup cleanup should not wait behind or block foreground work. - // PASSIVE checkpoints copy whatever is immediately available and skip - // frames that would require waiting on active readers or writers. - sqlx::query("PRAGMA wal_checkpoint(PASSIVE)") - .execute(self.logs_pool.as_ref()) - .await?; - Ok(()) - } - /// Query logs with optional filters. pub async fn query_logs(&self, query: &LogQuery) -> anyhow::Result> { let mut builder = QueryBuilder::::new( @@ -568,6 +542,7 @@ mod tests { #[tokio::test] async fn insert_logs_use_dedicated_log_database() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -578,7 +553,7 @@ mod tests { runtime .insert_logs(&[LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -753,6 +728,7 @@ mod tests { #[tokio::test] async fn query_logs_with_search_matches_rendered_body_substring() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -764,7 +740,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1_700_000_001, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -777,7 +753,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 1_700_000_002, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -809,6 +785,7 @@ mod tests { #[tokio::test] async fn query_logs_filters_level_set_without_rewriting_stored_level() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -820,7 +797,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "TRACE".to_string(), target: "cli".to_string(), @@ -833,7 +810,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -846,7 +823,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "warn".to_string(), target: "cli".to_string(), @@ -859,7 +836,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 4, + ts: now + 4, ts_nanos: 0, level: "ERROR".to_string(), target: "cli".to_string(), @@ -897,6 +874,7 @@ mod tests { #[tokio::test] async fn insert_logs_prunes_old_rows_when_thread_exceeds_size_limit() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -909,7 +887,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -922,7 +900,7 @@ mod tests { module_path: Some("mod".to_string()), }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -947,13 +925,14 @@ mod tests { .expect("query thread logs"); assert_eq!(rows.len(), 1); - assert_eq!(rows[0].ts, 2); + assert_eq!(rows[0].ts, now + 2); let _ = tokio::fs::remove_dir_all(codex_home).await; } #[tokio::test] async fn insert_logs_prunes_single_thread_row_when_it_exceeds_size_limit() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -965,7 +944,7 @@ mod tests { let eleven_mebibytes = "d".repeat(11 * 1024 * 1024); runtime .insert_logs(&[LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -995,6 +974,7 @@ mod tests { #[tokio::test] async fn insert_logs_prunes_threadless_rows_per_process_uuid_only() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1007,7 +987,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1020,7 +1000,7 @@ mod tests { module_path: Some("mod".to_string()), }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1033,7 +1013,7 @@ mod tests { module_path: Some("mod".to_string()), }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1060,13 +1040,14 @@ mod tests { let mut timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); timestamps.sort_unstable(); - assert_eq!(timestamps, vec![2, 3]); + assert_eq!(timestamps, vec![now + 2, now + 3]); let _ = tokio::fs::remove_dir_all(codex_home).await; } #[tokio::test] async fn insert_logs_prunes_single_threadless_process_row_when_it_exceeds_size_limit() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1078,7 +1059,7 @@ mod tests { let eleven_mebibytes = "e".repeat(11 * 1024 * 1024); runtime .insert_logs(&[LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1108,6 +1089,7 @@ mod tests { #[tokio::test] async fn insert_logs_prunes_threadless_rows_with_null_process_uuid() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1120,7 +1102,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1133,7 +1115,7 @@ mod tests { module_path: Some("mod".to_string()), }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1146,7 +1128,7 @@ mod tests { module_path: Some("mod".to_string()), }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1172,13 +1154,14 @@ mod tests { let mut timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); timestamps.sort_unstable(); - assert_eq!(timestamps, vec![2, 3]); + assert_eq!(timestamps, vec![now + 2, now + 3]); let _ = tokio::fs::remove_dir_all(codex_home).await; } #[tokio::test] async fn insert_logs_prunes_single_threadless_null_process_row_when_it_exceeds_limit() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1190,7 +1173,7 @@ mod tests { let eleven_mebibytes = "f".repeat(11 * 1024 * 1024); runtime .insert_logs(&[LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1220,6 +1203,7 @@ mod tests { #[tokio::test] async fn insert_logs_prunes_old_rows_when_thread_exceeds_row_limit() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1230,7 +1214,7 @@ mod tests { let entries: Vec = (1..=1_001) .map(|ts| LogEntry { - ts, + ts: now + ts, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1258,14 +1242,15 @@ mod tests { let timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); assert_eq!(timestamps.len(), 1_000); - assert_eq!(timestamps.first().copied(), Some(2)); - assert_eq!(timestamps.last().copied(), Some(1_001)); + assert_eq!(timestamps.first().copied(), Some(now + 2)); + assert_eq!(timestamps.last().copied(), Some(now + 1_001)); let _ = tokio::fs::remove_dir_all(codex_home).await; } #[tokio::test] async fn insert_logs_prunes_old_threadless_rows_when_process_exceeds_row_limit() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1276,7 +1261,7 @@ mod tests { let entries: Vec = (1..=1_001) .map(|ts| LogEntry { - ts, + ts: now + ts, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1308,14 +1293,15 @@ mod tests { .map(|row| row.ts) .collect(); assert_eq!(timestamps.len(), 1_000); - assert_eq!(timestamps.first().copied(), Some(2)); - assert_eq!(timestamps.last().copied(), Some(1_001)); + assert_eq!(timestamps.first().copied(), Some(now + 2)); + assert_eq!(timestamps.last().copied(), Some(now + 1_001)); let _ = tokio::fs::remove_dir_all(codex_home).await; } #[tokio::test] async fn insert_logs_prunes_old_threadless_null_process_rows_when_row_limit_exceeded() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1326,7 +1312,7 @@ mod tests { let entries: Vec = (1..=1_001) .map(|ts| LogEntry { - ts, + ts: now + ts, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1358,14 +1344,15 @@ mod tests { .map(|row| row.ts) .collect(); assert_eq!(timestamps.len(), 1_000); - assert_eq!(timestamps.first().copied(), Some(2)); - assert_eq!(timestamps.last().copied(), Some(1_001)); + assert_eq!(timestamps.first().copied(), Some(now + 2)); + assert_eq!(timestamps.last().copied(), Some(now + 1_001)); let _ = tokio::fs::remove_dir_all(codex_home).await; } #[tokio::test] async fn query_feedback_logs_returns_newest_lines_within_limit_in_order() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1377,7 +1364,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1390,7 +1377,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1403,7 +1390,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1427,9 +1414,14 @@ mod tests { assert_eq!( String::from_utf8(bytes).expect("valid utf-8"), [ - format_feedback_log_line(/*ts*/ 1, /*ts_nanos*/ 0, "INFO", "alpha"), - format_feedback_log_line(/*ts*/ 2, /*ts_nanos*/ 0, "INFO", "bravo"), - format_feedback_log_line(/*ts*/ 3, /*ts_nanos*/ 0, "INFO", "charlie"), + format_feedback_log_line(/*ts*/ now + 1, /*ts_nanos*/ 0, "INFO", "alpha"), + format_feedback_log_line(/*ts*/ now + 2, /*ts_nanos*/ 0, "INFO", "bravo"), + format_feedback_log_line( + /*ts*/ now + 3, + /*ts_nanos*/ 0, + "INFO", + "charlie" + ), ] .concat() ); @@ -1439,6 +1431,7 @@ mod tests { #[tokio::test] async fn query_feedback_logs_excludes_oversized_newest_row() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1451,7 +1444,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1464,7 +1457,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1492,6 +1485,7 @@ mod tests { #[tokio::test] async fn query_feedback_logs_includes_threadless_rows_from_same_process() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1503,7 +1497,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1516,7 +1510,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1529,7 +1523,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1542,7 +1536,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 4, + ts: now + 4, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1567,19 +1561,19 @@ mod tests { String::from_utf8(bytes).expect("valid utf-8"), [ format_feedback_log_line( - /*ts*/ 1, + /*ts*/ now + 1, /*ts_nanos*/ 0, "INFO", "threadless-before" ), format_feedback_log_line( - /*ts*/ 2, + /*ts*/ now + 2, /*ts_nanos*/ 0, "INFO", "thread-scoped" ), format_feedback_log_line( - /*ts*/ 3, + /*ts*/ now + 3, /*ts_nanos*/ 0, "INFO", "threadless-after" @@ -1593,6 +1587,7 @@ mod tests { #[tokio::test] async fn query_feedback_logs_excludes_threadless_rows_from_prior_processes() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1604,7 +1599,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1617,7 +1612,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1630,7 +1625,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1643,7 +1638,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 4, + ts: now + 4, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1668,19 +1663,19 @@ mod tests { String::from_utf8(bytes).expect("valid utf-8"), [ format_feedback_log_line( - /*ts*/ 2, + /*ts*/ now + 2, /*ts_nanos*/ 0, "INFO", "old-process-thread" ), format_feedback_log_line( - /*ts*/ 3, + /*ts*/ now + 3, /*ts_nanos*/ 0, "INFO", "new-process-thread" ), format_feedback_log_line( - /*ts*/ 4, + /*ts*/ now + 4, /*ts_nanos*/ 0, "INFO", "new-process-threadless" @@ -1694,6 +1689,7 @@ mod tests { #[tokio::test] async fn query_feedback_logs_keeps_newest_suffix_across_thread_and_threadless_logs() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1714,7 +1710,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1727,7 +1723,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1740,7 +1736,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1772,6 +1768,7 @@ mod tests { #[tokio::test] async fn query_feedback_logs_for_threads_merges_requested_threads_and_threadless_rows() { + let now = chrono::Utc::now().timestamp(); let codex_home = unique_temp_dir(); let runtime = StateRuntime::init( crate::SqliteConfig::new_for_testing(codex_home.as_path().abs()), @@ -1783,7 +1780,7 @@ mod tests { runtime .insert_logs(&[ LogEntry { - ts: 1, + ts: now + 1, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1796,7 +1793,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 2, + ts: now + 2, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1809,7 +1806,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 3, + ts: now + 3, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1822,7 +1819,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 4, + ts: now + 4, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1835,7 +1832,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 5, + ts: now + 5, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1848,7 +1845,7 @@ mod tests { module_path: None, }, LogEntry { - ts: 6, + ts: now + 6, ts_nanos: 0, level: "INFO".to_string(), target: "cli".to_string(), @@ -1872,16 +1869,26 @@ mod tests { assert_eq!( String::from_utf8(bytes).expect("valid utf-8"), [ - format_feedback_log_line(/*ts*/ 1, /*ts_nanos*/ 0, "INFO", "thread-1"), - format_feedback_log_line(/*ts*/ 2, /*ts_nanos*/ 0, "INFO", "thread-2"), format_feedback_log_line( - /*ts*/ 3, + /*ts*/ now + 1, + /*ts_nanos*/ 0, + "INFO", + "thread-1" + ), + format_feedback_log_line( + /*ts*/ now + 2, + /*ts_nanos*/ 0, + "INFO", + "thread-2" + ), + format_feedback_log_line( + /*ts*/ now + 3, /*ts_nanos*/ 0, "INFO", "threadless-proc-1" ), format_feedback_log_line( - /*ts*/ 4, + /*ts*/ now + 4, /*ts_nanos*/ 0, "INFO", "threadless-proc-2" diff --git a/codex-rs/state/src/runtime/logs_maintenance.rs b/codex-rs/state/src/runtime/logs_maintenance.rs new file mode 100644 index 000000000000..7d7879730c5c --- /dev/null +++ b/codex-rs/state/src/runtime/logs_maintenance.rs @@ -0,0 +1,84 @@ +//! Prune diagnostic logs in the background immediately, then on a timer. +//! SQLite keeps freed pages for reuse. + +use super::StateRuntime; +use chrono::Utc; +use sqlx::SqlitePool; +use std::sync::Arc; +use std::time::Duration; + +const LOG_RETENTION_SECONDS: i64 = 10 * 24 * 60 * 60; // 10 days +const LOG_DATABASE_BUDGET_BYTES: i64 = 64 * 1024 * 1024; // 64 MiB + +impl StateRuntime { + pub(super) fn start_periodic_logs_maintenance(self: &Arc, period: Duration) { + let runtime = Arc::downgrade(self); + tokio::spawn(async move { + let mut interval = tokio::time::interval_at( + /*start*/ tokio::time::Instant::now(), + /*period*/ period, + ); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut first_sweep = true; + loop { + // Hold no strong runtime reference while waiting. + interval.tick().await; + let Some(runtime) = runtime.upgrade() else { + break; + }; + if runtime.logs_pool.is_closed() { + break; + } + if let Err(err) = prune_by_age_and_size( + &runtime.logs_pool, + Utc::now().timestamp(), + LOG_DATABASE_BUDGET_BYTES, + ) + .await + { + tracing::warn!("failed to prune diagnostic logs: {err}"); + } + if first_sweep { + first_sweep = false; + // Preserve the startup checkpoint without waiting for readers or writers. + if let Err(err) = sqlx::query("PRAGMA wal_checkpoint(PASSIVE)") + .execute(runtime.logs_pool.as_ref()) + .await + { + tracing::warn!("failed to checkpoint diagnostic logs: {err}"); + } + } + } + }); + } +} + +async fn prune_by_age_and_size( + pool: &SqlitePool, + now: i64, + budget_bytes: i64, +) -> anyhow::Result<()> { + let mut retention_seconds = LOG_RETENTION_SECONDS; + loop { + sqlx::query("DELETE FROM logs WHERE ts < ?") + .bind(now.saturating_sub(retention_seconds)) + .execute(pool) + .await?; + // Exclude free pages: SQLite reuses them without shrinking the file. + let occupied_bytes: i64 = sqlx::query_scalar( + "SELECT (page_count - freelist_count) * page_size \ + FROM pragma_page_count(), pragma_freelist_count(), pragma_page_size()", + ) + .fetch_one(pool) + .await?; + if occupied_bytes <= budget_bytes || retention_seconds == 1 { + return Ok(()); + } + // Keep the newest second even if it exceeds the budget. + retention_seconds = (retention_seconds / 2).max(1); + } +} + +#[cfg(test)] +#[path = "logs_maintenance_tests.rs"] +mod tests; diff --git a/codex-rs/state/src/runtime/logs_maintenance_tests.rs b/codex-rs/state/src/runtime/logs_maintenance_tests.rs new file mode 100644 index 000000000000..1ca3939b1ab2 --- /dev/null +++ b/codex-rs/state/src/runtime/logs_maintenance_tests.rs @@ -0,0 +1,166 @@ +//! Exercise age/size pruning and automatic cleanup of an idle runtime. + +use super::LOG_RETENTION_SECONDS; +use super::StateRuntime; +use super::prune_by_age_and_size; +use crate::SqliteConfig; +use crate::migrations::LOGS_MIGRATOR; +use chrono::Utc; +use codex_utils_absolute_path::test_support::PathExt; +use pretty_assertions::assert_eq; +use sqlx::SqlitePool; +use std::path::PathBuf; +use std::time::Duration; + +const NOW: i64 = 2_000_000_000; +const DAY: i64 = 24 * 60 * 60; + +async fn database_with_logs(ages: &[i64]) -> (SqlitePool, PathBuf) { + let directory = super::super::test_support::unique_temp_dir(); + tokio::fs::create_dir_all(&directory).await.unwrap(); + let pool = SqliteConfig::new_for_testing(directory.as_path().abs()) + .open_read_write_pool(&directory.join("logs.sqlite")) + .await + .unwrap(); + LOGS_MIGRATOR.run(&pool).await.unwrap(); + for age in ages { + sqlx::query( + "INSERT INTO logs (ts, ts_nanos, level, target, feedback_log_body) \ + VALUES (?, 0, 'INFO', 'retention-test', ?)", + ) + .bind(NOW - age) + .bind("x".repeat(128 * 1024)) + .execute(&pool) + .await + .unwrap(); + } + (pool, directory) +} + +async fn retained_ages(pool: &SqlitePool) -> Vec { + sqlx::query_scalar("SELECT ? - ts FROM logs ORDER BY ts DESC") + .bind(NOW) + .fetch_all(pool) + .await + .unwrap() +} + +#[tokio::test] +async fn keeps_ten_day_boundary_when_under_budget() { + let (pool, directory) = + database_with_logs(&[0, LOG_RETENTION_SECONDS, LOG_RETENTION_SECONDS + 1]).await; + prune_by_age_and_size(&pool, NOW, /*budget_bytes*/ 1024 * 1024) + .await + .unwrap(); + assert_eq!(retained_ages(&pool).await, vec![0, LOG_RETENTION_SECONDS]); + pool.close().await; + tokio::fs::remove_dir_all(directory).await.unwrap(); +} + +#[tokio::test] +async fn halves_again_when_first_halving_deletes_nothing() { + let (pool, directory) = database_with_logs(&[DAY, 2 * DAY, 4 * DAY]).await; + // Three payloads exceed this budget; two fit, including table/index pages. + prune_by_age_and_size(&pool, NOW, /*budget_bytes*/ 350 * 1024) + .await + .unwrap(); + assert_eq!(retained_ages(&pool).await, vec![DAY, 2 * DAY]); + // A second cleanup must ignore free pages left in the file. + let allocated: i64 = sqlx::query_scalar( + "SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert!(allocated > 350 * 1024); + prune_by_age_and_size(&pool, NOW, /*budget_bytes*/ 350 * 1024) + .await + .unwrap(); + assert_eq!(retained_ages(&pool).await, vec![DAY, 2 * DAY]); + pool.close().await; + tokio::fs::remove_dir_all(directory).await.unwrap(); +} + +#[tokio::test] +async fn stops_at_one_second_when_recent_rows_exceed_budget() { + let (pool, directory) = database_with_logs(&[-1, 0, 1, 2, DAY]).await; + prune_by_age_and_size(&pool, NOW, /*budget_bytes*/ 1) + .await + .unwrap(); + assert_eq!(retained_ages(&pool).await, vec![-1, 0, 1]); + pool.close().await; + tokio::fs::remove_dir_all(directory).await.unwrap(); +} + +#[tokio::test] +async fn periodic_cleanup_prunes_without_new_log_writes_or_restart() { + let directory = super::super::test_support::unique_temp_dir(); + let runtime = StateRuntime::init( + SqliteConfig::new_for_testing(directory.as_path().abs()), + "test-provider".to_string(), + ) + .await + .unwrap(); + let now = Utc::now().timestamp(); + for timestamp in [now - 11 * DAY, now] { + sqlx::query("INSERT INTO logs (ts, ts_nanos, level, target) VALUES (?, 0, 'INFO', 'test')") + .bind(timestamp) + .execute(runtime.logs_pool.as_ref()) + .await + .unwrap(); + } + runtime.start_periodic_logs_maintenance(Duration::from_millis(10)); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let timestamps: Vec = sqlx::query_scalar("SELECT ts FROM logs ORDER BY ts") + .fetch_all(runtime.logs_pool.as_ref()) + .await + .unwrap(); + if timestamps == vec![now] { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("periodic cleanup should remove expired logs while idle"); + runtime.close().await; + tokio::fs::remove_dir_all(directory).await.unwrap(); +} + +#[tokio::test] +async fn startup_cleanup_runs_without_waiting_for_the_period() { + let directory = super::super::test_support::unique_temp_dir(); + tokio::fs::create_dir_all(&directory).await.unwrap(); + let sqlite = SqliteConfig::new_for_testing(directory.as_path().abs()); + let pool = sqlite + .open_read_write_pool(&sqlite.logs_db_path()) + .await + .unwrap(); + LOGS_MIGRATOR.run(&pool).await.unwrap(); + sqlx::query("INSERT INTO logs (ts, ts_nanos, level, target) VALUES (0, 0, 'INFO', 'test')") + .execute(&pool) + .await + .unwrap(); + pool.close().await; + + let runtime = StateRuntime::init(sqlite, "test-provider".to_string()) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM logs") + .fetch_one(runtime.logs_pool.as_ref()) + .await + .unwrap(); + if count == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("startup cleanup should run before the 30-minute interval"); + runtime.close().await; + tokio::fs::remove_dir_all(directory).await.unwrap(); +} diff --git a/codex-rs/tui/src/app/empty_state_animation_tests.rs b/codex-rs/tui/src/app/empty_state_animation_tests.rs index 6647d1bd0840..a1a9a66c1cc6 100644 --- a/codex-rs/tui/src/app/empty_state_animation_tests.rs +++ b/codex-rs/tui/src/app/empty_state_animation_tests.rs @@ -181,14 +181,6 @@ async fn empty_state_animation_preserves_header_cursor_and_footer() -> Result<() let size = Size::new(/*width*/ 120, /*height*/ 44); let mut tui = crate::tui::test_support::make_test_tui()?; tui.set_owned_screen(/*owned*/ true)?; - app.chat_widget - .empty_state_animation - .borrow() - .greeting - .set(crate::empty_state_animation::Greeting { - phrase: "Pull up a prompt.", - }) - .expect("initial greeting"); app.queue_clear_ui_header(&mut tui); app.transcript_cells .push(Arc::new(history_cell::StartupWarningsCell::mcp( @@ -209,7 +201,6 @@ async fn empty_state_animation_preserves_header_cursor_and_footer() -> Result<() let after = crate::custom_terminal::test_support::last_rendered_buffer(&tui.terminal); assert!(has_blossom(&tui)); assert_eq!(tui.terminal.last_known_cursor_pos, cursor); - assert!(text(&before).contains("Pull up a prompt.")); assert_eq!( &after.content[after.index_of(/*x*/ 0, before_bottom.y)..], &before.content[before.index_of(/*x*/ 0, before_bottom.y)..] @@ -222,14 +213,6 @@ async fn empty_state_animation_preserves_header_cursor_and_footer() -> Result<() text(&before) ) ); - // Editing, clearing, and resizing must never choose another phrase. - let phrase_pos = after - .content - .iter() - .position(|cell| cell.symbol() == "P") - .expect("the phrase is visible above the composer"); - assert_eq!(after.content[phrase_pos].fg, crate::style::accent_color()); - let short = Size::new(/*width*/ 120, /*height*/ 12); draw(&mut app, &mut tui, short)?; assert!(!has_blossom(&tui)); @@ -242,12 +225,6 @@ async fn empty_state_animation_preserves_header_cursor_and_footer() -> Result<() app.chat_widget.apply_external_edit(String::new()); draw(&mut app, &mut tui, size)?; assert!(has_blossom(&tui)); - assert!( - text(crate::custom_terminal::test_support::last_rendered_buffer( - &tui.terminal - )) - .contains("Pull up a prompt.") - ); assert_eq!(app.transcript_cells.len(), history_len); app.local_settings.tui.animations = false; draw(&mut app, &mut tui, size)?; @@ -269,7 +246,7 @@ async fn empty_state_animation_preserves_header_cursor_and_footer() -> Result<() } #[tokio::test] -async fn submitting_a_draft_keeps_greeting_and_dismisses_logo_even_after_clear() -> Result<()> { +async fn submitting_a_draft_dismisses_logo_even_after_clear() -> Result<()> { let (mut app, _events, _ops) = crate::app::tests::make_test_app_with_channels().await; app.local_settings.tui.animations = true; let size = Size::new(/*width*/ 120, /*height*/ 44); @@ -282,20 +259,12 @@ async fn submitting_a_draft_keeps_greeting_and_dismisses_logo_even_after_clear() .start_fresh(); draw(&mut app, &mut tui, size)?; assert!(has_blossom(&tui)); - let greeting = *app - .chat_widget - .empty_state_animation - .borrow() - .greeting - .get() - .expect("choose a greeting for the fresh thread"); app.chat_widget .apply_external_edit("first prompt".to_string()); draw(&mut app, &mut tui, size)?; let drafting = crate::custom_terminal::test_support::last_rendered_buffer(&tui.terminal); assert!(!has_blossom(&tui)); assert!(text(drafting).contains("first prompt")); - assert!(text(drafting).contains(greeting.phrase)); app.chat_widget.apply_external_edit(String::new()); draw(&mut app, &mut tui, size)?; assert!(has_blossom(&tui)); @@ -305,14 +274,9 @@ async fn submitting_a_draft_keeps_greeting_and_dismisses_logo_even_after_clear() .handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); draw(&mut app, &mut tui, size)?; assert!(!has_blossom(&tui)); - let submitted = crate::custom_terminal::test_support::last_rendered_buffer(&tui.terminal); - assert_eq!(text(submitted).matches(greeting.phrase).count(), 1); - // The header is also stable when it scrolls out and back into the viewport. let short = Size::new(/*width*/ 120, /*height*/ 8); draw(&mut app, &mut tui, short)?; draw(&mut app, &mut tui, size)?; - let restored = crate::custom_terminal::test_support::last_rendered_buffer(&tui.terminal); - assert!(text(restored).contains(greeting.phrase)); assert!(!has_blossom(&tui)); app.reset_transcript_state_after_clear(); draw(&mut app, &mut tui, size)?; diff --git a/codex-rs/tui/src/app/history_ui.rs b/codex-rs/tui/src/app/history_ui.rs index 52e7e0203ed8..f0b72ac3106d 100644 --- a/codex-rs/tui/src/app/history_ui.rs +++ b/codex-rs/tui/src/app/history_ui.rs @@ -316,18 +316,13 @@ impl App { &self, version: &'static str, ) -> history_cell::SessionHeaderHistoryCell { - let mut header = history_cell::SessionHeaderHistoryCell::new( + history_cell::SessionHeaderHistoryCell::new( self.chat_widget.model_display_name().to_string(), self.chat_widget.current_reasoning_effort(), self.config.cwd.to_path_buf(), version, ) - .with_yolo_mode(history_cell::is_yolo_mode(&self.config)); - history_cell::set_session_greeting( - &mut header, - &self.chat_widget.empty_state_animation.borrow().greeting, - ); - header + .with_yolo_mode(history_cell::is_yolo_mode(&self.config)) } pub(super) fn clear_ui_header_lines(&self, width: u16) -> Vec> { diff --git a/codex-rs/tui/src/app/session_lifecycle.rs b/codex-rs/tui/src/app/session_lifecycle.rs index 545577d8403e..4723dec35306 100644 --- a/codex-rs/tui/src/app/session_lifecycle.rs +++ b/codex-rs/tui/src/app/session_lifecycle.rs @@ -1108,7 +1108,7 @@ impl App { ThreadAttachPresentation::Fresh | ThreadAttachPresentation::FreshWithDraft ) { self.chat_widget.mark_fresh_task_for_sparkle(&started); - // FreshWithDraft inherits its provisional greeting and replay at the handoff. + // FreshWithDraft inherits its provisional replay at the handoff. if matches!(presentation, ThreadAttachPresentation::Fresh) { self.chat_widget .empty_state_animation diff --git a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__empty_state_animation_tests__fresh_thread_header.snap b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__empty_state_animation_tests__fresh_thread_header.snap index 1261499117df..789d6e4bda53 100644 --- a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__empty_state_animation_tests__fresh_thread_header.snap +++ b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__empty_state_animation_tests__fresh_thread_header.snap @@ -7,7 +7,7 @@ enabled: >_ OpenAI Codex (v0.0.0) /tmp/project - Pull up a prompt. + @@ -53,7 +53,7 @@ disabled: >_ OpenAI Codex (v0.0.0) /tmp/project - Pull up a prompt. + diff --git a/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs b/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs index 5512cdb0e82b..0162d1e1a655 100644 --- a/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs +++ b/codex-rs/tui/src/app/tests/background_task_defaults_tests.rs @@ -86,16 +86,6 @@ async fn command_center_new_keeps_startup_draft_visible_through_handoff() -> Res app.chat_widget.composer_text_with_pending(), "draft during startup" ); - assert_eq!( - app.chat_widget - .empty_state_animation - .borrow() - .greeting - .get() - .unwrap() - .phrase, - "Pull up a prompt." - ); while let Ok(event) = events.try_recv() { Box::pin(app.handle_event(&mut tui, &mut server, event)).await?; diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__startup_frame_tests__startup_tip_in_transcript.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__startup_frame_tests__startup_tip_in_transcript.snap index 8ed7c18cff6d..d45000ee78fe 100644 --- a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__startup_frame_tests__startup_tip_in_transcript.snap +++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__startup_frame_tests__startup_tip_in_transcript.snap @@ -1,6 +1,5 @@ --- source: tui/src/app/tests/startup_frame_tests.rs -assertion_line: 72 expression: "normalize_snapshot_paths(normalize_agent_center_snapshot(snapshots.join(\"\\n\\n\")))" --- 80 columns @@ -8,8 +7,6 @@ expression: "normalize_snapshot_paths(normalize_agent_center_snapshot(snapshots. >_ OpenAI Codex (v0.0.0) /tmp/project - Pull up a prompt. - • Conversation continues here. @@ -20,6 +17,8 @@ expression: "normalize_snapshot_paths(normalize_agent_center_snapshot(snapshots. + + › Ask Codex to do anything GPT-5.6-Sol default · /tmp/project @@ -30,8 +29,6 @@ expression: "normalize_snapshot_paths(normalize_agent_center_snapshot(snapshots. >_ OpenAI Codex (v0.0.0) /tmp/project - Pull up a prompt. - • Conversation continues here. @@ -42,6 +39,8 @@ expression: "normalize_snapshot_paths(normalize_agent_center_snapshot(snapshots. + + › Ask Codex to do anything GPT-5.6-Sol default · /tmp/pr… diff --git a/codex-rs/tui/src/app/tests/startup_frame_tests.rs b/codex-rs/tui/src/app/tests/startup_frame_tests.rs index c836d543eebb..46cf092e35c5 100644 --- a/codex-rs/tui/src/app/tests/startup_frame_tests.rs +++ b/codex-rs/tui/src/app/tests/startup_frame_tests.rs @@ -21,14 +21,8 @@ async fn owned_startup_hides_tip_in_transcript() -> Result<()> { let mut app = crate::app::test_support::make_test_app().await; app.local_settings.tui.show_tooltips = true; app.local_settings.tui.animations = false; - let greeting = Arc::new(std::sync::OnceLock::new()); - greeting - .set(crate::empty_state_animation::Greeting { - phrase: "Pull up a prompt.", - }) - .unwrap(); let session = test_thread_session(ThreadId::new(), app.config.cwd.to_path_buf()); - let mut session_info = new_session_info( + let session_info = new_session_info( &app.config, &app.local_settings, &session.model, @@ -38,7 +32,6 @@ async fn owned_startup_hides_tip_in_transcript() -> Result<()> { Some("Use /mcp to list configured MCP tools.".into()), /*auth_plan*/ None, ); - history_cell::set_session_greeting(&mut session_info, &greeting); app.transcript_cells = vec![ Arc::new(session_info), Arc::new(AgentMessageCell::new( @@ -80,9 +73,8 @@ async fn owned_startup_hides_tip_in_transcript() -> Result<()> { .map(ToString::to_string) .collect::>() .join("\n"); - assert!(text.contains("Pull up a prompt.")); assert!(text.contains("Use /mcp")); - assert!(!text.contains("model:")); + assert_eq!(text.contains("model:"), mode == HistoryRenderMode::Raw); assert!(!text.contains('╭')); } app.transcript_view.begin_search(); diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 72722559fcac..0988bf7cd154 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -114,6 +114,7 @@ //! //! # Submission and Prompt Expansion //! +//! Multiline pastes continue blockquote prefixes. //! `Enter` submits immediately. `Tab` requests queuing while a task is running; if no task is //! running, `Tab` submits just like Enter so input is never dropped. //! Vim Replace shares Insert's composer actions; only textarea editing differs. @@ -562,6 +563,7 @@ pub(crate) struct ChatComposerConfig { pub(crate) shell_commands_enabled: bool, /// Whether pasting a file path can attach local images. pub(crate) image_paste_enabled: bool, + pub(crate) blockquote_paste_enabled: bool, /// Strip leading and trailing whitespace from submissions. pub(crate) trim_submission: bool, /// Embedded editors reset Vim only when their owner accepts the answer. @@ -575,6 +577,7 @@ impl Default for ChatComposerConfig { slash_commands_enabled: true, shell_commands_enabled: true, image_paste_enabled: true, + blockquote_paste_enabled: true, trim_submission: true, reset_vim_on_submission: true, } @@ -582,16 +585,14 @@ impl Default for ChatComposerConfig { } impl ChatComposerConfig { - /// A minimal preset for plain-text inputs embedded in other surfaces. - /// - /// This disables popups, slash and shell commands, and image-path attachment behavior - /// so the composer behaves like a simple notes field. + /// Text answers support Markdown, without popups, commands, or image attachments. pub(crate) const fn plain_text() -> Self { Self { popups_enabled: false, slash_commands_enabled: false, shell_commands_enabled: false, image_paste_enabled: false, + blockquote_paste_enabled: true, trim_submission: true, reset_vim_on_submission: true, } diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/blockquote_paste_tests.rs b/codex-rs/tui/src/bottom_pane/chat_composer/blockquote_paste_tests.rs new file mode 100644 index 000000000000..fcd393739ab0 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/chat_composer/blockquote_paste_tests.rs @@ -0,0 +1,144 @@ +use super::super::tests::new_test_composer; +use super::super::tests::snapshot_composer_state_with_width; +use super::super::*; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; +use pretty_assertions::assert_eq; + +#[test] +fn blockquote_paste_continues_current_line_prefix() { + for (draft, pasted, expected) in [ + ("> ", "first\nsecond", "> first\n> second\n\n"), + ("intro\n> ", "α\r\n\r\nβ\r", "intro\n> α\n> \n> β\n> \n\n"), + ( + "> existing ", + "first\nsecond", + "> existing first\n> second\n\n", + ), + ("text > ", "first\nsecond", "text > first\nsecond"), + (">", "first\nsecond", ">first\nsecond"), + ( + "> quote\nplain ", + "first\nsecond", + "> quote\nplain first\nsecond", + ), + ("> ", "single", "> single"), + ("!> ", "out\npwd", "!> out\npwd"), + (" !echo hi\n> ", "out\npwd", " !echo hi\n> out\npwd"), + ] { + let (mut composer, _rx) = new_test_composer(); + composer.insert_str(draft); + composer.handle_paste(pasted.to_string()); + assert_eq!(composer.current_text(), expected); + } +} + +#[test] +fn blockquote_paste_uses_cursor_line_and_preserves_suffix() { + let (mut composer, _rx) = new_test_composer(); + composer.insert_str("intro\n> suffix\nend"); + composer.draft.textarea.set_cursor("intro\n> ".len()); + composer.handle_paste("first\nsecond".to_string()); + assert_eq!( + composer.draft.textarea.text(), + "intro\n> first\n> second\n\nsuffix\nend" + ); + composer.insert_str("next "); + assert_eq!( + composer.current_text(), + "intro\n> first\n> second\n\nnext suffix\nend" + ); +} + +#[test] +fn blockquote_paste_large_expands_quoted_text_on_submit() { + let (mut composer, _rx) = new_test_composer(); + composer.insert_str("> "); + let first = "x".repeat(LARGE_PASTE_CHAR_THRESHOLD); + composer.handle_paste(format!("{first}\nsecond")); + let quoted = format!("{first}\n> second"); + let placeholder = format!("[Pasted Content {} chars]", quoted.chars().count()); + assert_eq!( + composer.draft.textarea.text(), + format!("> {placeholder}\n\n") + ); + assert_eq!( + composer.draft.pending_pastes, + vec![(placeholder, quoted.clone())] + ); + composer.insert_str("Next block"); + let (result, _) = composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + let InputResult::Submitted { text, .. } = result else { + panic!("expected Submitted"); + }; + assert_eq!(text, format!("> {quoted}\n\nNext block")); +} + +#[test] +fn blockquote_paste_rendered() { + snapshot_composer_state_with_width( + "blockquote_paste", + /*width*/ 60, + /*enhanced_keys_supported*/ false, + |composer| { + composer.insert_str("Please explain:\n> "); + composer.handle_paste("first line\n\nlast line".to_string()); + composer.insert_str("Next block"); + }, + ); +} + +#[test] +fn blockquote_paste_replacing_selected_prefix_stays_literal() { + let (mut composer, _rx) = new_test_composer(); + composer.insert_str("> selected"); + let area = Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ 80, /*height*/ 10, + ); + composer.render(area, &mut Buffer::empty(area)); + let (x, y) = composer.cursor_pos(area).unwrap(); + for (kind, column) in [ + ( + MouseEventKind::Down(crossterm::event::MouseButton::Left), + x - 10, + ), + (MouseEventKind::Drag(crossterm::event::MouseButton::Left), x), + ] { + let event = MouseEvent { + kind, + column, + row: y, + modifiers: KeyModifiers::NONE, + }; + assert!(composer.prepare_mouse(event)); + composer.render(area, &mut Buffer::empty(area)); + assert!(composer.handle_mouse(event)); + } + composer.handle_paste("first\nsecond".to_string()); + assert_eq!(composer.current_text(), "first\nsecond"); +} + +#[test] +fn blockquote_paste_embedded_answer_starts_next_block() { + let (mut composer, _rx) = new_test_composer(); + composer.config = ChatComposerConfig::plain_text(); + composer.insert_str("> "); + composer.handle_paste("first\nsecond".to_string()); + composer.insert_str("Next block"); + assert_eq!(composer.current_text(), "> first\n> second\n\nNext block"); +} + +#[test] +fn blockquote_paste_embedded_answer_rendered() { + snapshot_composer_state_with_width( + "blockquote_paste_embedded_answer", + /*width*/ 60, + /*enhanced_keys_supported*/ false, + |composer| { + composer.config = ChatComposerConfig::plain_text(); + composer.insert_str("> "); + composer.handle_paste("first line\n\nlast line".to_string()); + composer.insert_str("Next block"); + }, + ); +} diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/paste_input.rs b/codex-rs/tui/src/bottom_pane/chat_composer/paste_input.rs index 48ea632c1c35..eecc926631d3 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer/paste_input.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer/paste_input.rs @@ -1,20 +1,29 @@ //! Classify and integrate composer pastes and capture raw paste tabs before completion or submission. +//! Multiline pastes on a line beginning with `> ` continue that prefix after every newline, +//! including blank and trailing lines. This happens before large pastes are collapsed, so their +//! expanded text retains the quoting. Two unquoted newlines after the inserted text or placeholder +//! leave the cursor in the next Markdown block, including in embedded answer fields. +//! Provisional startup input, search queries, and shell input remain literal. +//! Paste classification and insertion share one textarea edit target, including selection replacement. use super::*; use crate::bottom_pane::paste_burst::FlushResult; +use crate::bottom_pane::textarea::EditTarget; impl ChatComposer { pub(crate) fn insert_str(&mut self, text: &str) { + self.insert_str_at_target(self.draft.textarea.edit_target(), text); + } + + fn insert_str_at_target(&mut self, target: EditTarget, text: &str) { if !text.is_empty() && self.sparkle.draft.get() == sparkle::SparkleDraft::Untouched { self.dismiss_sparkle(); } let started_vim_edit = self.begin_direct_vim_edit(); - let elements_before = self - .draft - .textarea - .mouse_selection_range() - .map(|_| self.draft.textarea.element_payloads()); - self.draft.textarea.insert_str(text); + let elements_before = target + .replaces_text() + .then(|| self.draft.textarea.element_payloads()); + self.draft.textarea.insert_str_at_target(target, text); if let Some(elements_before) = elements_before { self.reconcile_deleted_elements(elements_before); } @@ -130,10 +139,24 @@ impl ChatComposer { } let started_vim_edit = self.begin_direct_vim_edit(); let elements_before = self.draft.textarea.element_payloads(); + let target = self.draft.textarea.edit_target(); + let before_cursor = &self.draft.textarea.text()[..target.start()]; + let current_line = before_cursor.rsplit('\n').next().unwrap_or_default(); + let is_blockquote = self.config.blockquote_paste_enabled + && !self.is_bang_shell_command() + && current_line.starts_with("> ") + && pasted.contains('\n'); + let pasted = if is_blockquote { + std::borrow::Cow::Owned(pasted.replace('\n', "\n> ")) + } else { + pasted + }; let char_count = pasted.chars().count(); if char_count > LARGE_PASTE_CHAR_THRESHOLD { let placeholder = self.next_large_paste_placeholder(char_count); - self.draft.textarea.insert_element(&placeholder); + self.draft + .textarea + .insert_element_at_target(target, &placeholder); self.draft .pending_pastes .push((placeholder, pasted.into_owned())); @@ -144,7 +167,11 @@ impl ChatComposer { let cursor = self.draft.textarea.cursor(); self.draft.textarea.insert_str_at(cursor, " "); } else { - self.insert_str(&pasted); + self.insert_str_at_target(target, &pasted); + } + if is_blockquote { + let textarea = &mut self.draft.textarea; + textarea.insert_str_at(textarea.cursor(), "\n\n"); } self.draft.paste_burst.clear_after_explicit_paste(); self.reconcile_deleted_elements(elements_before); @@ -191,3 +218,7 @@ impl ChatComposer { false } } + +#[cfg(test)] +#[path = "blockquote_paste_tests.rs"] +mod blockquote_paste_tests; diff --git a/codex-rs/tui/src/bottom_pane/experimental_features_view_tests.rs b/codex-rs/tui/src/bottom_pane/experimental_features_view_tests.rs index 38110e635857..6c103f5f6105 100644 --- a/codex-rs/tui/src/bottom_pane/experimental_features_view_tests.rs +++ b/codex-rs/tui/src/bottom_pane/experimental_features_view_tests.rs @@ -47,25 +47,30 @@ fn experimental_features_keep_following_choice_at_wrap_boundary() { insta::assert_snapshot!(snapshots.join("\n\n")); } +/// Experimental settings show each feature's label, description, and default state. #[test] -fn experimental_features_analytics_plan_history() { - let feature = Feature::AnalyticsPlanHistory; - let stage = feature.stage(); - let (app_tx, _app_rx) = tokio::sync::mpsc::unbounded_channel(); - let view = ExperimentalFeaturesView::new( - vec![ExperimentalFeatureItem { - key: feature.key().to_string(), - name: stage.experimental_menu_name().unwrap().to_string(), - description: stage.experimental_menu_description().unwrap().to_string(), - enabled: feature.default_enabled(), - writable: true, - }], - ThreadId::new(), - /*catalog_rx*/ None, - AppEventSender::new(app_tx), - crate::keymap::RuntimeKeymap::defaults().list, - ); - snapshot_view("experimental_features_analytics_plan_history", &view); +fn experimental_features_display_metadata() { + for feature in [ + Feature::AnalyticsPlanHistory, + Feature::LoginShellPackagePath, + ] { + let stage = feature.stage(); + let (app_tx, _app_rx) = tokio::sync::mpsc::unbounded_channel(); + let view = ExperimentalFeaturesView::new( + vec![ExperimentalFeatureItem { + key: feature.key().to_string(), + name: stage.experimental_menu_name().unwrap().to_string(), + description: stage.experimental_menu_description().unwrap().to_string(), + enabled: feature.default_enabled(), + writable: true, + }], + ThreadId::new(), + /*catalog_rx*/ None, + AppEventSender::new(app_tx), + crate::keymap::RuntimeKeymap::defaults().list, + ); + snapshot_view(&format!("experimental_features_{}", feature.key()), &view); + } } fn server_feature(name: &str) -> ExperimentalFeature { diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__blockquote_paste.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__blockquote_paste.snap new file mode 100644 index 000000000000..69bbc367af1e --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__blockquote_paste.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/bottom_pane/chat_composer.rs +assertion_line: 5302 +expression: terminal.backend() +--- +" " +"› Please explain: " +" > first line " +" > " +" > last line " +" " +" Next block " +" " +" 100% context left " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__blockquote_paste_embedded_answer.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__blockquote_paste_embedded_answer.snap new file mode 100644 index 000000000000..e53f1d94ce6b --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__blockquote_paste_embedded_answer.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/bottom_pane/chat_composer.rs +assertion_line: 5302 +expression: terminal.backend() +--- +" " +"› > first line " +" > " +" > last line " +" " +" Next block " +" " +" " +" 100% context left " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__experimental_features_view__tests__experimental_features_login_shell_package_path.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__experimental_features_view__tests__experimental_features_login_shell_package_path.snap new file mode 100644 index 000000000000..3cb14de54905 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__experimental_features_view__tests__experimental_features_login_shell_package_path.snap @@ -0,0 +1,16 @@ +--- +source: tui/src/bottom_pane/experimental_features_view_tests.rs +expression: buffer_text(&buffer) +--- + + Experimental features + Checked features are configured on. Some experimental features take effect + only in new tasks or after restarting the Codex server. + + +› [ ] Bundled tools in login shells Keep bundled tools such as ripgrep + available when login shell startup resets + PATH. + + + space toggle · enter save · esc save/close diff --git a/codex-rs/tui/src/bottom_pane/textarea.rs b/codex-rs/tui/src/bottom_pane/textarea.rs index 60bd8e7e7068..51f988b83069 100644 --- a/codex-rs/tui/src/bottom_pane/textarea.rs +++ b/codex-rs/tui/src/bottom_pane/textarea.rs @@ -16,6 +16,7 @@ //! Visible web URLs carry their complete terminal hyperlink destination across wrapped rows; //! masked rendering never exposes hyperlink destinations. //! Mouse selection uses those same visual rows and keeps graphemes and elements atomic. +//! The editing module resolves replacement targets shared by insertion and paste-context inspection. use crate::key_hint::KeyBindingListExt; use crate::key_hint::is_altgr; @@ -50,12 +51,14 @@ use std::ops::Range; use std::sync::Arc; use unicode_segmentation::UnicodeSegmentation; +mod editing; mod hyperlinks; mod mouse; mod vim; mod vim_commands; mod vim_search; mod wrapping; + use self::vim::VimMode; use self::vim::VimMotion; use self::vim::VimOperator; @@ -67,6 +70,7 @@ use self::vim_commands::VimCommandState; use self::vim_commands::VimEditTarget; use self::vim_commands::VimInsertPosition; pub(crate) use self::vim_commands::VimPersistentState; +pub(crate) use editing::EditTarget; const WORD_SEPARATORS: &str = "`~!@#$%^&*()-=+[{]}\\|;:'\",.<>/?"; @@ -441,16 +445,6 @@ impl TextArea { &self.text } - pub fn insert_str(&mut self, text: &str) { - let replaced_selection = !text.is_empty() && self.delete_mouse_selection(); - self.record_vim_inserted_text(text); - if self.is_vim_replace_mode() && !replaced_selection { - self.replace_vim_text(text); - } else { - self.insert_str_at(self.cursor_pos, text); - } - } - pub fn insert_str_at(&mut self, pos: usize, text: &str) { self.mouse_selection = None; self.clear_vim_replace_recovery(); @@ -1723,17 +1717,6 @@ impl TextArea { true } - pub fn insert_element(&mut self, text: &str) -> u64 { - self.delete_mouse_selection(); - let start = self.clamp_pos_for_insertion(self.cursor_pos); - self.insert_str_at(start, text); - let end = start + text.len(); - let id = self.add_element(start..end); - // Place cursor at end of inserted element - self.set_cursor(end); - id - } - fn add_element(&mut self, range: Range) -> u64 { let id = self.next_element_id(); self.elements.push(TextElement { id, range }); diff --git a/codex-rs/tui/src/bottom_pane/textarea/editing.rs b/codex-rs/tui/src/bottom_pane/textarea/editing.rs new file mode 100644 index 000000000000..6c97ea69ddfc --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/textarea/editing.rs @@ -0,0 +1,81 @@ +//! Resolve the replacement range once for context-sensitive edits, then consume it on insertion. +//! Targets respect character and atomic-element boundaries and must be used before another edit +//! or cursor/selection change. Text insertion retains Vim Replace and repeat policies; atomic +//! elements always replace the target literally. + +use super::*; + +/// The selection to replace, or an empty range at the insertion cursor. +/// Vim Replace without a selection computes its overwrite span as it inserts each grapheme. +#[derive(Debug)] +pub(crate) struct EditTarget { + range: Range, +} + +impl EditTarget { + pub(crate) fn start(&self) -> usize { + self.range.start + } + + pub(crate) fn replaces_text(&self) -> bool { + !self.range.is_empty() + } +} + +impl TextArea { + pub(crate) fn edit_target(&self) -> EditTarget { + let range = self.mouse_selection_range().map_or_else( + || { + let cursor = self.clamp_pos_for_insertion(self.cursor_pos); + cursor..cursor + }, + |range| self.expand_range_to_element_boundaries(range), + ); + EditTarget { range } + } + + pub fn insert_str(&mut self, text: &str) { + self.insert_str_at_target(self.edit_target(), text); + } + + pub(crate) fn insert_str_at_target(&mut self, target: EditTarget, text: &str) { + // Empty text must not delete a selection or create a Vim replacement step. + if text.is_empty() { + if !self.is_vim_replace_mode() { + self.insert_str_at(self.cursor_pos, text); + } + return; + } + let replaces_selection = target.replaces_text(); + if replaces_selection { + // Pointer-selected edits cannot be replayed as keyboard-relative Vim commands. + self.vim_commands = VimCommandState::default(); + } + self.record_vim_inserted_text(text); + if self.is_vim_replace_mode() && !replaces_selection { + self.replace_vim_text(text); + } else { + self.replace_range(target.range, text); + } + } + + pub fn insert_element(&mut self, text: &str) -> u64 { + self.insert_element_at_target(self.edit_target(), text) + } + + pub(crate) fn insert_element_at_target(&mut self, target: EditTarget, text: &str) -> u64 { + let start = target.start(); + if target.replaces_text() { + self.vim_commands = VimCommandState::default(); + } + self.replace_range(target.range, text); + let end = start + text.len(); + let id = self.add_element(start..end); + self.set_cursor(end); + id + } +} + +#[cfg(test)] +#[path = "editing_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/bottom_pane/textarea/editing_tests.rs b/codex-rs/tui/src/bottom_pane/textarea/editing_tests.rs new file mode 100644 index 000000000000..8df7f7234d14 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/textarea/editing_tests.rs @@ -0,0 +1,74 @@ +use super::*; +use crossterm::event::MouseButton; +use crossterm::event::MouseEvent; +use crossterm::event::MouseEventKind; +use pretty_assertions::assert_eq; + +#[test] +fn target_replaces_forward_and_backward_selections() { + for (anchor, cursor) in [(1, 4), (4, 1)] { + let mut textarea = TextArea::new(); + textarea.insert_str("abcdef"); + let area = Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ 20, /*height*/ 2, + ); + let mut state = TextAreaState::default(); + StatefulWidgetRef::render_ref(&&textarea, area, &mut Buffer::empty(area), &mut state); + for (kind, column) in [ + (MouseEventKind::Down(MouseButton::Left), anchor), + (MouseEventKind::Drag(MouseButton::Left), cursor), + ] { + assert!(textarea.handle_mouse( + MouseEvent { + kind, + column, + row: 0, + modifiers: KeyModifiers::NONE + }, + state, + )); + } + let target = textarea.edit_target(); + assert_eq!(&textarea.text()[..target.start()], "a"); + textarea.insert_str_at_target(target, "界"); + assert_eq!((textarea.text(), textarea.cursor()), ("a界ef", "a界".len())); + assert_eq!(textarea.mouse_selection_range(), None); + } +} + +#[test] +fn target_element_insertion_preserves_adjacent_elements() { + let mut textarea = TextArea::new(); + textarea.insert_str("before "); + textarea.insert_element("[old]"); + textarea.insert_str(" after"); + textarea.set_cursor("before ".len()); + let target = textarea.edit_target(); + textarea.insert_element_at_target(target, "[new]"); + assert_eq!( + ( + textarea.text(), + textarea.cursor(), + textarea.element_payloads() + ), + ( + "before [new][old] after", + "before [new]".len(), + vec!["[new]".to_string(), "[old]".to_string()] + ), + ); +} + +#[test] +fn target_text_preserves_vim_replace_and_backspace_recovery() { + let mut textarea = TextArea::new(); + textarea.insert_str("abc"); + textarea.set_cursor(/*pos*/ 0); + textarea.set_vim_enabled(/*enabled*/ true); + textarea.input(KeyCode::Char('R').into()); + let target = textarea.edit_target(); + textarea.insert_str_at_target(target, "界"); + assert_eq!((textarea.text(), textarea.cursor()), ("界bc", "界".len())); + textarea.input(KeyCode::Backspace.into()); + assert_eq!((textarea.text(), textarea.cursor()), ("abc", 0)); +} diff --git a/codex-rs/tui/src/bottom_pane/textarea/mouse.rs b/codex-rs/tui/src/bottom_pane/textarea/mouse.rs index 00c2c2adfedb..ccbe221a6779 100644 --- a/codex-rs/tui/src/bottom_pane/textarea/mouse.rs +++ b/codex-rs/tui/src/bottom_pane/textarea/mouse.rs @@ -63,16 +63,6 @@ impl TextArea { (!range.is_empty()).then_some(range) } - pub(super) fn delete_mouse_selection(&mut self) -> bool { - let Some(range) = self.mouse_selection_range() else { - return false; - }; - // A pointer-selected edit cannot be replayed as a keyboard-relative Vim command. - self.vim_commands = VimCommandState::default(); - self.replace_range(range, ""); - true - } - pub(crate) fn handle_mouse(&mut self, event: MouseEvent, state: TextAreaState) -> bool { let area = self.rendered_area.get(); let dragging = self.mouse_selection.as_ref().is_some_and(|s| s.dragging); diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index fb4575ab0db4..35a684b3bb76 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -64,8 +64,7 @@ impl ChatWidget { }; let empty_state_animation = crate::empty_state_animation::EmptyStateAnimation::default(); - let mut header = Self::placeholder_session_header_cell(&config); - history_cell::set_session_greeting(header.as_mut(), &empty_state_animation.greeting); + let header = Self::placeholder_session_header_cell(&config); let active_cell = Some(header); let current_cwd = Some(config.cwd.to_path_buf()); diff --git a/codex-rs/tui/src/chatwidget/session_flow.rs b/codex-rs/tui/src/chatwidget/session_flow.rs index e03b2d5679be..f1e33dd29dbc 100644 --- a/codex-rs/tui/src/chatwidget/session_flow.rs +++ b/codex-rs/tui/src/chatwidget/session_flow.rs @@ -201,7 +201,7 @@ impl ChatWidget { let model_for_header = self.current_model().to_string(); if display == SessionConfiguredDisplay::Normal { let startup_tooltip_override = self.startup_tooltip_override.take(); - let mut session_info_cell = history_cell::new_session_info( + let session_info_cell = history_cell::new_session_info( &self.config, &self.local_settings, &model_for_header, @@ -211,10 +211,6 @@ impl ChatWidget { startup_tooltip_override, self.plan_type, ); - history_cell::set_session_greeting( - &mut session_info_cell, - &self.empty_state_animation.borrow().greeting, - ); self.apply_session_info_cell(session_info_cell); } else if self .transcript diff --git a/codex-rs/tui/src/empty_state_animation.rs b/codex-rs/tui/src/empty_state_animation.rs index 1d57c8761814..8003d2df7faf 100644 --- a/codex-rs/tui/src/empty_state_animation.rs +++ b/codex-rs/tui/src/empty_state_animation.rs @@ -2,14 +2,12 @@ //! Visible time pauses while hidden. Fresh conversations start settled and can replay on a click. mod geometry; -mod greetings; mod lighting; mod paths; mod policy; mod renderer; mod sequence; -use std::sync::Arc; use std::sync::OnceLock; use std::time::Duration; use std::time::Instant; @@ -24,7 +22,6 @@ use ratatui::style::Color; use crate::motion::MotionMode; use crate::terminal_palette; -pub(crate) use greetings::Greeting; use lighting::Lighting; pub(crate) use policy::Presentation; pub(crate) use policy::is_startup_cell; @@ -45,9 +42,6 @@ pub(crate) enum ComposerState { #[derive(Default)] pub(crate) struct EmptyStateAnimation { eligible: bool, - // Initialized once for fresh threads; never initialized for a resumed or forked thread. - // Headers retain the selection after the temporary blossom is dismissed. - pub(crate) greeting: Arc>, spin_elapsed: Duration, last_frame: Option, fade_elapsed: Duration, @@ -67,7 +61,6 @@ impl EmptyStateAnimation { pub(crate) fn start_fresh(&mut self) { self.cancel_replay(); self.eligible = true; - self.greeting.get_or_init(Greeting::choose); self.spin_elapsed = Duration::ZERO; self.last_frame = None; self.fade_elapsed = Duration::ZERO; @@ -75,16 +68,12 @@ impl EmptyStateAnimation { self.opacity = 1.0; } - /// Keep the provisional pose and phrase in headers already bound to the live thread. + /// Keep the provisional pose when handing off to the live thread. pub(crate) fn continue_from(&mut self, source: &mut Self) { let mut previous = std::mem::take(source); if !previous.is_eligible() { previous.start_fresh(); } - if let Some(greeting) = previous.greeting.get() { - let _ = self.greeting.set(*greeting); - } - previous.greeting = Arc::clone(&self.greeting); *self = previous; } diff --git a/codex-rs/tui/src/empty_state_animation/greetings.rs b/codex-rs/tui/src/empty_state_animation/greetings.rs deleted file mode 100644 index 6dce8f65bd0a..000000000000 --- a/codex-rs/tui/src/empty_state_animation/greetings.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Startup phrases retained from the approved greeting list (PR #10248). -//! One selection is shared by the provisional, live, and committed session headers. - -use rand::Rng; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Greeting { - pub(crate) phrase: &'static str, -} - -impl Greeting { - pub(super) fn choose() -> Self { - Self { - phrase: GREETINGS[rand::rng().random_range(0..GREETINGS.len())], - } - } -} - -const GREETINGS: &[&str] = &[ - "Whoa, fancy meeting you here!", - "Look who’s at the keyboard.", - "Hello, you. Got an idea?", - "Nice to see you in these parts.", - "Pull up a prompt.", - "What brings you to this corner of the terminal?", - "Hey there. Big plans or little fixes?", - "What are we getting into today?", - "Got something you want to try?", - "What’s today’s little adventure?", - "Anything interesting on the docket?", - "Take your time. The cursor can wait.", - "A half-formed idea will do.", - "Come on in. There’s room for an idea.", - "You, me, and a blinking cursor.", - "It takes two to tango. What’s our first step?", - "You bring the idea. I’ll bring the brackets.", - "Your move, teammate.", - "What are we cooking up?", - "Shall we turn “what if” into something?", - "What’s the first thing on our napkin sketch?", - "Bring your unfinished thoughts.", - "You set the direction. We’ll work out the steps.", - "Ah, the terminal. A classic meeting spot.", - "Nice place you’ve got here. Very monospace.", - "Welcome to our little rectangle of possibility.", - "A cursor blinks. The plot thickens.", - "The prompt is yours.", - "Fancy a little quality terminal time?", - "How’s life between the brackets?", - "Shall we give this cursor a purpose?", - "A fresh prompt. An open question.", - "Your keyboard has entered the chat.", - "Hello, world. Hello, you.", - "Welcome to the blinking edge of possibility.", - "Any loose ends? Semicolons count.", - "Shall we make this terminal earn its keep?", - "Got an itch to fix a thing?", - "Show me the bit that’s being weird.", - "Greetings, fellow tinkerer.", - "Well, well, well. An idea approaches.", - "What’s today’s side quest?", - "Is this a plan day or a poke-around day?", - "Practical, peculiar, or a little of both?", - "Rough sketches welcome.", - "Shall we see where this goes?", - "All right. What have you got?", - "Back for another round?", - "Same terminal, new possibilities.", - "Welcome back. Familiar territory or a fresh adventure?", - "Follow the white cursor.", - "Welcome to the command line, Neo.", - "A glitch in the Matrix, or just a missing semicolon?", - "How deep does this codebase go?", - "Speak, friend, and enter a prompt.", - "One prompt to begin the journey.", - "May the source be with you.", - "These might be the bugs you’re looking for.", - "A new prompt awakens.", - "Forty-two is an answer. What’s the question?", - "It’s dangerous to code alone. Take a prompt.", - "The code must flow.", - "You rang? Metaphorically. You typed.", - "Hello again, carbon-based collaborator.", - "This looks like the start of a perfectly reasonable rabbit hole.", - "Shall we turn “huh?” into “aha!”?", - "Well, this terminal just got interesting.", - "The source is strong with this one.", - "There’s a perfectly good prompt with your name on it.", - "Here for a quick fix or the extended edition?", - "What’s the latest from your side of the keyboard?", - "Got a minute and a mildly unreasonable idea?", - "A hunch is a perfectly respectable starting point.", - "Hello again. What’s the plot this time?", - "Shall we make a little something out of nothing?", - "Welcome to the neighborhood. Lots of characters here.", - "Nice terminal. Does it come in widescreen?", - "Your cursor called. It wants a plot.", - "Shall we put some verbs after that cursor?", - "Welcome. There’s no dress code, just code.", - "This terminal has excellent conversational potential.", - "Got an idea that won’t stay on the napkin?", - "What are we building in this episode?", - "Bring a question. Bonus points if it’s a weird one.", - "What are we poking with a metaphorical stick?", - "Shall we make the thing that makes the other thing easier?", - "Welcome to the part where the idea gets interesting.", - "A long time ago, in a directory not so far away…", -]; diff --git a/codex-rs/tui/src/history_cell/session.rs b/codex-rs/tui/src/history_cell/session.rs index 11cce0ecf1e5..48295d4b85ce 100644 --- a/codex-rs/tui/src/history_cell/session.rs +++ b/codex-rs/tui/src/history_cell/session.rs @@ -1,10 +1,6 @@ //! Session headers, onboarding guidance, and transcript cards. -use std::sync::Arc; -use std::sync::OnceLock; - use super::*; -use crate::empty_state_animation::Greeting; use crate::line_truncation::line_width; use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; use crate::style::accent_color; @@ -110,17 +106,6 @@ impl HistoryCell for SessionNoticeCell { #[derive(Debug)] pub struct SessionInfoCell(CompositeHistoryCell); -/// Bind provisional and configured banners to the thread's chosen greeting. -pub(crate) fn set_session_greeting(cell: &mut dyn HistoryCell, greeting: &Arc>) { - if let Some(header) = cell.as_any_mut().downcast_mut::() { - header.greeting = Arc::clone(greeting); - } else if let Some(info) = cell.as_any_mut().downcast_mut::() { - for part in &mut info.0.parts { - set_session_greeting(part.as_mut(), greeting); - } - } -} - /// Fullscreen transcript presentation omits tips; scrollback retains the original cells. pub(crate) fn fullscreen_session_lines( cell: &dyn HistoryCell, @@ -292,7 +277,6 @@ pub(crate) struct SessionHeaderHistoryCell { reasoning_effort: Option, directory: PathBuf, yolo_mode: bool, - greeting: Arc>, } impl SessionHeaderHistoryCell { @@ -308,7 +292,6 @@ impl SessionHeaderHistoryCell { reasoning_effort, directory, yolo_mode: false, - greeting: Default::default(), } } @@ -371,13 +354,6 @@ impl HistoryCell for SessionHeaderHistoryCell { "YOLO mode".magenta().bold(), ])); } - if let Some(greeting) = self.greeting.get() { - // The tip/help that follows has its own normal composite separator. - lines.extend([ - Line::default(), - Line::from(vec![" ".into(), greeting.phrase.fg(accent_color())]), - ]); - } lines .into_iter() .map(|line| truncate_line_with_ellipsis_if_overflow(line, width)) @@ -385,13 +361,6 @@ impl HistoryCell for SessionHeaderHistoryCell { } fn raw_lines(&self) -> Vec> { - if self.greeting.get().is_some() { - return self - .display_lines(u16::MAX) - .into_iter() - .map(|line| Line::from(line.to_string())) - .collect(); - } let mut lines = vec![ Line::from(format!("OpenAI Codex (v{})", self.version)), Line::from(format!( diff --git a/codex-rs/tui/src/onboarding/auth.rs b/codex-rs/tui/src/onboarding/auth.rs index 29022d525beb..2cdc98ca68e4 100644 --- a/codex-rs/tui/src/onboarding/auth.rs +++ b/codex-rs/tui/src/onboarding/auth.rs @@ -704,7 +704,7 @@ impl AuthModeWidget { "Use your own OpenAI API key for usage-based billing".bold(), ]), "".into(), - " Paste or type your API key below. It will be stored locally in auth.json.".into(), + " Paste or type your API key below.".into(), "".into(), ]; if state.prepopulated_from_env { @@ -1277,6 +1277,29 @@ mod tests { (widget, codex_home) } + #[tokio::test] + async fn api_key_entry_snapshots() { + let (mut widget, _tmp) = widget_forced_chatgpt().await; + widget.auth_config.forced_login_method = None; + *widget.sign_in_state.write().unwrap() = + SignInState::ApiKeyEntry(ApiKeyInputState::default()); + + for (name, width) in [("api_key_entry", 80), ("api_key_entry_narrow", 40)] { + let height = 14; + let area = Rect::new(/*x*/ 0, /*y*/ 0, width, height); + let mut terminal = crate::custom_terminal::Terminal::with_options( + crate::test_backend::VT100Backend::new(width, height), + ) + .expect("terminal"); + terminal.set_viewport_area(area); + terminal + .draw(|frame| widget.render_ref(area, frame.buffer_mut())) + .expect("draw"); + + insta::assert_snapshot!(name, terminal.backend().to_string()); + } + } + #[tokio::test] async fn api_key_flow_disabled_when_chatgpt_forced() { let (mut widget, _tmp) = widget_forced_chatgpt().await; diff --git a/codex-rs/tui/src/onboarding/snapshots/codex_tui__onboarding__auth__tests__api_key_entry.snap b/codex-rs/tui/src/onboarding/snapshots/codex_tui__onboarding__auth__tests__api_key_entry.snap new file mode 100644 index 000000000000..376f264c5f20 --- /dev/null +++ b/codex-rs/tui/src/onboarding/snapshots/codex_tui__onboarding__auth__tests__api_key_entry.snap @@ -0,0 +1,15 @@ +--- +source: tui/src/onboarding/auth.rs +expression: terminal.backend().to_string() +--- +> Use your own OpenAI API key for usage-based billing + + Paste or type your API key below. + + + +╭API key───────────────────────────────────────────────────────────────────────╮ +│Paste or type your API key │ +╰──────────────────────────────────────────────────────────────────────────────╯ + Press enter to save + Press esc to go back diff --git a/codex-rs/tui/src/onboarding/snapshots/codex_tui__onboarding__auth__tests__api_key_entry_narrow.snap b/codex-rs/tui/src/onboarding/snapshots/codex_tui__onboarding__auth__tests__api_key_entry_narrow.snap new file mode 100644 index 000000000000..499103514f6e --- /dev/null +++ b/codex-rs/tui/src/onboarding/snapshots/codex_tui__onboarding__auth__tests__api_key_entry_narrow.snap @@ -0,0 +1,15 @@ +--- +source: tui/src/onboarding/auth.rs +expression: terminal.backend().to_string() +--- +> Use your own OpenAI API key for +usage-based billing + + Paste or type your API key below. + + +╭API key───────────────────────────────╮ +│Paste or type your API key │ +╰──────────────────────────────────────╯ + Press enter to save + Press esc to go back diff --git a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__layout__tests__owned_startup_layout.snap b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__layout__tests__owned_startup_layout.snap index 957e7539d5fa..6065eb840688 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__layout__tests__owned_startup_layout.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__layout__tests__owned_startup_layout.snap @@ -7,7 +7,7 @@ cursor=Some((13, 12)) >_ OpenAI Codex (v) loading - Pull up a prompt. + diff --git a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_fresh_session_transitions.snap b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_fresh_session_transitions.snap index fb019c0e5810..a01b3b6d9d21 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_fresh_session_transitions.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_fresh_session_transitions.snap @@ -1,6 +1,5 @@ --- source: tui/src/startup_draft_tests.rs -assertion_line: 285 expression: "snapshots.join(\"\\n---\\n\")" --- resume lookup fallback: @@ -8,8 +7,6 @@ before: >_ OpenAI Codex (v) loading - - Pull up a prompt. Resuming session… › draft while loading @@ -20,8 +17,6 @@ after: >_ OpenAI Codex (v) loading - Pull up a prompt. - › draft while loading @@ -32,8 +27,6 @@ before: >_ OpenAI Codex (v) loading - - Pull up a prompt. Forking session… › draft while loading @@ -44,8 +37,6 @@ after: >_ OpenAI Codex (v) loading - Pull up a prompt. - › draft while loading @@ -59,8 +50,6 @@ after: >_ OpenAI Codex (v) loading - Pull up a prompt. - › Ask Codex to do anything @@ -74,8 +63,6 @@ after: >_ OpenAI Codex (v) loading - Pull up a prompt. - › Ask Codex to do anything diff --git a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_full_frames.snap b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_full_frames.snap index 13af775d7e38..fc4ea3030117 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_full_frames.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_full_frames.snap @@ -1,6 +1,5 @@ --- source: tui/src/startup_draft_tests.rs -assertion_line: 154 expression: "snapshots.join(\"\\n---\\n\")" --- empty (48 columns): @@ -8,8 +7,6 @@ empty (48 columns): >_ OpenAI Codex (v) loading - Pull up a prompt. - › Ask Codex to do anything @@ -40,8 +37,6 @@ multiline (48 columns): >_ OpenAI Codex (v) loading - Pull up a prompt. - › first startup line second startup line @@ -53,8 +48,6 @@ narrow (18 columns): >_ OpenAI Codex… loading - Pull up a promp… - › first startup line diff --git a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_onboarding_transition.snap b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_onboarding_transition.snap index 681cfd873b88..74e7b7d1f08c 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_onboarding_transition.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__startup_draft_onboarding_transition.snap @@ -1,6 +1,5 @@ --- source: tui/src/startup_draft_tests.rs -assertion_line: 802 expression: frames --- before onboarding: hidden (0x0 viewport) @@ -10,8 +9,6 @@ after onboarding: >_ OpenAI Codex (v) loading - Pull up a prompt. - › Ask Codex to do anything diff --git a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__terminal_app_ssh_startup.snap b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__terminal_app_ssh_startup.snap index 2ec0e1e64159..ae2fe6147157 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__terminal_app_ssh_startup.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__startup_draft__tests__terminal_app_ssh_startup.snap @@ -1,6 +1,5 @@ --- source: tui/src/startup_draft_tests.rs -assertion_line: 194 expression: "frames.join(\"\\n---\\n\").replace(crate::version::CODEX_CLI_VERSION, \"\")" --- terminal_app_over_ssh=false, owned=true @@ -8,7 +7,7 @@ terminal_app_over_ssh=false, owned=true >_ OpenAI Codex (v) loading - Pull up a prompt. + @@ -26,8 +25,6 @@ terminal_app_over_ssh=true, owned=false >_ OpenAI Codex (v) loading - Pull up a prompt. - › Ask Codex to do anything diff --git a/codex-rs/tui/src/startup_draft.rs b/codex-rs/tui/src/startup_draft.rs index 96ffcecd709c..7af33c518533 100644 --- a/codex-rs/tui/src/startup_draft.rs +++ b/codex-rs/tui/src/startup_draft.rs @@ -202,8 +202,7 @@ impl StartupDraftPump { ) { blossom.start_fresh(); } - let mut header = startup_session_header(/*config*/ None); - history_cell::set_session_greeting(header.as_mut(), &blossom.greeting); + let header = startup_session_header(/*config*/ None); Self { header, blossom: std::cell::RefCell::new(blossom), @@ -262,7 +261,6 @@ impl StartupDraftPump { local_settings.tui.animations && local_settings.tui.effects.welcome, ); self.header = startup_session_header(Some(config)); - history_cell::set_session_greeting(self.header.as_mut(), &self.blossom.borrow().greeting); self.bottom_pane.set_status_line_enabled( local_settings .tui @@ -530,7 +528,11 @@ fn startup_draft_bottom_pane( effects: Default::default(), skills: None, }, - ChatComposerConfig::plain_text(), + ChatComposerConfig { + // Keep partial pastes literal until the startup draft is handed off. + blockquote_paste_enabled: false, + ..ChatComposerConfig::plain_text() + }, ); bottom_pane.set_context_window_pending(/*pending*/ true); bottom_pane diff --git a/codex-rs/tui/src/startup_draft_tests.rs b/codex-rs/tui/src/startup_draft_tests.rs index baaa0ebdfeff..db2476edf3dc 100644 --- a/codex-rs/tui/src/startup_draft_tests.rs +++ b/codex-rs/tui/src/startup_draft_tests.rs @@ -29,15 +29,8 @@ where { let (tx, rx) = unbounded_channel(); let mut blossom = crate::empty_state_animation::EmptyStateAnimation::default(); - blossom - .greeting - .set(crate::empty_state_animation::Greeting { - phrase: "Pull up a prompt.", - }) - .expect("initial greeting"); blossom.start_fresh(); - let mut header = startup_session_header(/*config*/ None); - crate::history_cell::set_session_greeting(header.as_mut(), &blossom.greeting); + let header = startup_session_header(/*config*/ None); StartupDraftPump { header, blossom: std::cell::RefCell::new(blossom), @@ -100,13 +93,6 @@ fn startup_draft_renders_full_empty_and_multiline_composer_frames() { ] { let mut pump = startup_test_pump(std::iter::empty()); pump.session_action = session_action; - // Resume and fork have no fresh-thread greeting. Exercise the real initial state. - if matches!( - session_action, - StartupDraftSessionAction::Resume | StartupDraftSessionAction::Fork - ) { - pump.header = startup_session_header(/*config*/ None); - } pump.bottom_pane .set_composer_text(text.to_string(), Vec::new(), Vec::new()); let renderable = @@ -314,15 +300,15 @@ async fn startup_draft_hydrates_its_header_without_moving_the_composer() { startup_draft_renderable(&pump.header, &pump.bottom_pane, pump.session_action) .desired_height(width); - assert_eq!(pump.header.raw_lines()[2].to_string().trim(), "loading"); + assert_eq!(pump.header.raw_lines()[2].to_string(), "directory: loading"); pump.apply_config(&config); let expected_directory = crate::history_cell::SessionHeaderHistoryCell::format_directory_inner( config.cwd.as_path(), /*max_width*/ None, ); assert_eq!( - pump.header.raw_lines()[2].to_string().trim(), - expected_directory + pump.header.raw_lines()[2].to_string(), + format!("directory: {expected_directory}") ); assert_eq!( startup_draft_renderable(&pump.header, &pump.bottom_pane, pump.session_action) diff --git a/codex-rs/tui/src/startup_preflight.rs b/codex-rs/tui/src/startup_preflight.rs index 7741319b76cd..cd284e401bfb 100644 --- a/codex-rs/tui/src/startup_preflight.rs +++ b/codex-rs/tui/src/startup_preflight.rs @@ -45,6 +45,10 @@ pub(super) fn should_delay_startup_composer_for_first_login( match std::fs::metadata(codex_home) { Ok(metadata) if !metadata.is_dir() => return false, Ok(_) => { + // With file auth as the default, auth.json is a hint of existing credentials. + // An existing config.toml may select keyring storage, which need not create + // auth.json. Keep the composer visible when any of these files exists; + // the normal account lookup decides whether sign-in is required. for state_file in ["auth.json", "config.toml", "environments.toml"] { if !matches!(codex_home.join(state_file).try_exists(), Ok(false)) { return false; diff --git a/codex-rs/utils/path-uri/src/api_path_string.rs b/codex-rs/utils/path-uri/src/api_path_string.rs index 3b9def4ba40d..b2ecdf0aed55 100644 --- a/codex-rs/utils/path-uri/src/api_path_string.rs +++ b/codex-rs/utils/path-uri/src/api_path_string.rs @@ -162,9 +162,11 @@ impl LegacyAppPathString { /// Infers the path convention of an absolute API path from its spelling. /// - /// Relative paths and ambiguous spellings return `None`. In particular, - /// slash-prefixed paths are treated as POSIX even when they could also be - /// interpreted as slash-delimited Windows UNC paths. + /// Two leading separators select Windows UNC or namespace syntax, including + /// forward and mixed slashes, independently of the current host. This favors + /// UNC paths over ambiguous double-slash POSIX paths. Call [`Self::to_path_uri`] + /// with an explicit POSIX convention to preserve that interpretation. + /// Relative paths return `None`; inferred prefixes still require validation. pub fn infer_absolute_path_convention(&self) -> Option { let bytes = self.0.as_bytes(); let has_windows_drive_root = matches!( @@ -172,7 +174,12 @@ impl LegacyAppPathString { [drive, b':', separator, ..] if drive.is_ascii_alphabetic() && is_windows_separator_byte(*separator) ); - if has_windows_drive_root || self.0.starts_with(r"\\") { + let has_windows_unc_root = matches!( + bytes, + [first, second, ..] + if is_windows_separator_byte(*first) && is_windows_separator_byte(*second) + ); + if has_windows_drive_root || has_windows_unc_root { Some(PathConvention::Windows) } else if self.0.starts_with('/') { Some(PathConvention::Posix) diff --git a/codex-rs/utils/path-uri/src/api_path_string_tests.rs b/codex-rs/utils/path-uri/src/api_path_string_tests.rs index d5a621d29079..b115a9006b73 100644 --- a/codex-rs/utils/path-uri/src/api_path_string_tests.rs +++ b/codex-rs/utils/path-uri/src/api_path_string_tests.rs @@ -418,7 +418,7 @@ fn infers_absolute_path_conventions_from_api_text() { (r"\\.\COM1", Some(PathConvention::Windows)), ("/workspace/file.rs", Some(PathConvention::Posix)), ("/C:/workspace/file.rs", Some(PathConvention::Posix)), - ("//server/share/file.rs", Some(PathConvention::Posix)), + ("//server/share/file.rs", Some(PathConvention::Windows)), ("", None), (".", None), ("subdir/file.rs", None), @@ -657,3 +657,62 @@ fn serializes_and_deserializes_as_a_string() { rendered ); } + +#[test] +fn inferred_unc_paths_accept_forward_and_mixed_slashes_on_every_host() { + for raw in [ + "//server/share/project", + r"//server/share\project", + r"/\server/share/project", + r"\/server/share/project", + r"\\server\share\project", + "//wsl.localhost/Ubuntu/home/user/project", + "//127.0.0.1/c$/project", + "//LOCALHOST/c$/project", + "//?/C:/project", + ] { + let path = LegacyAppPathString::from_string(raw); + let expected = path.to_path_uri(PathConvention::Windows).expect(raw); + let inferred = path.to_inferred_path_uri().expect(raw); + assert_eq!( + (inferred.clone(), inferred.infer_path_convention()), + (expected, Some(PathConvention::Windows)), + "{raw}" + ); + let serialized = serde_json::to_string(&inferred).expect("serialize URI"); + assert_eq!( + serde_json::from_str::(&serialized).expect("deserialize URI"), + inferred, + "{raw}" + ); + } + let slash = LegacyAppPathString::from_string("//server/share/project") + .to_inferred_path_uri() + .unwrap(); + assert_eq!( + slash, + PathUri::parse("file://server/share/project").unwrap() + ); +} + +#[test] +fn double_slash_posix_paths_require_an_explicit_convention() { + let path = LegacyAppPathString::from_string("//server/share/project"); + assert_eq!( + path.to_path_uri(PathConvention::Posix).unwrap(), + PathUri::parse("file:///server/share/project").unwrap() + ); + let cwd = PathUri::parse("file:///workspace").unwrap(); + assert!(matches!( + path.resolve_against(&cwd, /*user_home_dir*/ None), + Err(LegacyAppPathStringError::MismatchedConvention { .. }) + )); + for raw in ["//", "//server", "//server/", "///server/share"] { + assert!( + LegacyAppPathString::from_string(raw) + .to_inferred_path_uri() + .is_none(), + "{raw}" + ); + } +} diff --git a/codex-rs/utils/path-uri/src/lib.rs b/codex-rs/utils/path-uri/src/lib.rs index c35142fd9a3c..7a42363dd1ad 100644 --- a/codex-rs/utils/path-uri/src/lib.rs +++ b/codex-rs/utils/path-uri/src/lib.rs @@ -810,7 +810,8 @@ fn native_path_segments_start_with( } fn infer_opaque_path_convention(path_bytes: &[u8]) -> Option { - if path_bytes.starts_with(b"/") { + // A UTF-16LE Windows slash starts with `/\0`, unlike a POSIX root. + if path_bytes.starts_with(b"/") && !path_bytes.starts_with(b"/\0") { return Some(PathConvention::Posix); } if !path_bytes.len().is_multiple_of(2) { @@ -824,7 +825,8 @@ fn infer_opaque_path_convention(path_bytes: &[u8]) -> Option { let second = path_wide.next()?; let has_drive = u8::try_from(first).is_ok_and(|drive| drive.is_ascii_alphabetic()) && second == u16::from(b':'); - let has_unc_prefix = first == u16::from(b'\\') && second == u16::from(b'\\'); + let is_separator = |character| character == u16::from(b'\\') || character == u16::from(b'/'); + let has_unc_prefix = is_separator(first) && is_separator(second); (has_drive || has_unc_prefix).then_some(PathConvention::Windows) } diff --git a/codex-rs/utils/path-uri/src/tests.rs b/codex-rs/utils/path-uri/src/tests.rs index fab23b0c467f..49dbd2781fb5 100644 --- a/codex-rs/utils/path-uri/src/tests.rs +++ b/codex-rs/utils/path-uri/src/tests.rs @@ -532,6 +532,38 @@ fn file_uri_round_trips_literal_percent_characters() { assert_eq!(uri.basename(), Some("file".to_string())); } +#[cfg(windows)] +#[test] +fn host_windows_paths_round_trip_through_file_uris() { + for native in [ + r"C:\temp\test-file.txt", + r"C:/temp/test-file.txt", + r"C:\temp/test-file.txt", + r"\\127.0.0.1\c$\temp\test-file.txt", + r"//127.0.0.1/c$/temp/test-file.txt", + r"\\LOCALHOST\c$\temp\test-file.txt", + r"//LOCALHOST/c$/temp/test-file.txt", + r"\\.\C:\temp\test-file.txt", + r"\\?\C:\temp\test-file.txt", + r"\\.\UNC\LOCALHOST\c$\temp\test-file.txt", + r"\\?\UNC\server\share\temp\test-file.txt", + r"\\.\COM1", + r"\\?\Volume{00000000-0000-0000-0000-000000000000}\file.txt", + ] { + let expected = + codex_utils_absolute_path::AbsolutePathBuf::from_absolute_path_checked(native) + .expect(native); + let uri = PathUri::from_host_native_path(native).expect(native); + let restored = PathUri::parse(&uri.to_string()).expect("parse URI"); + assert_eq!( + restored.infer_path_convention(), + Some(PathConvention::Windows), + "{native}" + ); + assert_eq!(restored.to_abs_path().expect(native), expected, "{native}"); + } +} + #[test] #[cfg(windows)] fn file_uri_round_trips_windows_unc_paths() { @@ -957,6 +989,93 @@ fn join_normalizes_absolute_parent_segments() { } } +#[test] +fn absolute_windows_paths_round_trip() { + for (native, expected) in [ + (r"c:\temp\test-file.txt", r"C:\temp\test-file.txt"), + (r"C:/temp/test-file.txt", r"C:\temp\test-file.txt"), + (r"C:\temp/test-file.txt", r"C:\temp\test-file.txt"), + ( + r"\\127.0.0.1\c$\temp\test-file.txt", + r"\\127.0.0.1\c$\temp\test-file.txt", + ), + ( + r"//127.0.0.1/c$/temp/test-file.txt", + r"\\127.0.0.1\c$\temp\test-file.txt", + ), + ( + r"\\server/share\temp/test-file.txt", + r"\\server\share\temp\test-file.txt", + ), + ( + r"\\LOCALHOST\c$\temp\test-file.txt", + r"\\LOCALHOST\c$\temp\test-file.txt", + ), + ( + r"//LOCALHOST/c$/temp/test-file.txt", + r"//LOCALHOST/c$/temp/test-file.txt", + ), + ( + r"/\LOCALHOST/c$/temp/test-file.txt", + r"/\LOCALHOST/c$/temp/test-file.txt", + ), + ( + r"\/LOCALHOST/c$/temp/test-file.txt", + r"\/LOCALHOST/c$/temp/test-file.txt", + ), + (r"\\.\c:\temp\test-file.txt", r"C:\temp\test-file.txt"), + (r"\\?\c:\temp\test-file.txt", r"C:\temp\test-file.txt"), + ( + r"\\.\UNC\server\c$\temp\test-file.txt", + r"\\server\c$\temp\test-file.txt", + ), + ( + r"\\?\UNC\server\c$\temp\test-file.txt", + r"\\server\c$\temp\test-file.txt", + ), + ( + r"\\.\UNC\LOCALHOST\c$\temp\test-file.txt", + r"\\.\UNC\LOCALHOST\c$\temp\test-file.txt", + ), + (r"\\.\COM1", r"\\.\COM1"), + ( + r"\\?\Volume{00000000-0000-0000-0000-000000000000}\file.txt", + r"\\?\Volume{00000000-0000-0000-0000-000000000000}\file.txt", + ), + (r"//./COM1", r"//./COM1"), + (r"//./C:/temp/test-file.txt", r"//./C:/temp/test-file.txt"), + (r"//?/C:/temp/test-file.txt", r"//?/C:/temp/test-file.txt"), + ( + r"//./UNC/server/share/file.txt", + r"//./UNC/server/share/file.txt", + ), + ( + r"//?/UNC/server/share/file.txt", + r"//?/UNC/server/share/file.txt", + ), + ( + r"C:/a b/日本語/100%#file.txt:stream", + r"C:\a b\日本語\100%#file.txt:stream", + ), + ] { + let uri = LegacyAppPathString::from_string(native) + .to_path_uri(PathConvention::Windows) + .expect(native); + let serialized = serde_json::to_string(&uri).expect("serialize URI"); + let restored: PathUri = serde_json::from_str(&serialized).expect("deserialize URI"); + assert_eq!( + ( + restored.infer_path_convention(), + restored.inferred_native_path_string() + ), + (Some(PathConvention::Windows), expected.to_string()), + "{native}" + ); + let base = PathUri::parse("file:///D:/workspace").expect("base URI"); + assert_eq!(base.join(native).expect(native), restored, "{native}"); + } +} + #[test] fn windows_namespace_normalization_preserves_opaque_paths() { let base = PathUri::parse("file:///C:/workspace").expect("valid Windows base URI"); @@ -1059,6 +1178,24 @@ fn join_replaces_windows_absolute_path() { ); } +#[test] +fn windows_relative_paths_accept_both_separators() { + let base = PathUri::parse("file:///C:/workspace/src").expect("base URI"); + for (native, expected) in [ + (r"..\tests/file.rs", "file:///C:/workspace/tests/file.rs"), + (r"../tests\file.rs", "file:///C:/workspace/tests/file.rs"), + (r"/temp/file.rs", "file:///C:/temp/file.rs"), + (r"\temp/file.rs", "file:///C:/temp/file.rs"), + (r"c:tests/file.rs", "file:///C:/workspace/src/tests/file.rs"), + ] { + assert_eq!( + base.join(native).expect(native), + PathUri::parse(expected).unwrap() + ); + } + assert!(base.join("D:tests/file.rs").is_err()); +} + #[test] fn join_windows_root_relative_path_preserves_drive_or_share() { for (base, path, expected) in [ diff --git a/codex-rs/windows-sandbox-rs/BUILD.bazel b/codex-rs/windows-sandbox-rs/BUILD.bazel index 50255d45406f..8531cb4d664c 100644 --- a/codex-rs/windows-sandbox-rs/BUILD.bazel +++ b/codex-rs/windows-sandbox-rs/BUILD.bazel @@ -55,8 +55,4 @@ codex_rust_crate( ":codex-command-runner", ":codex-windows-sandbox-setup", ], - # Setup rotates machine-wide account passwords. The process-local test lock - # cannot prevent exec-server's elevated filesystem tests from rotating them - # between this crate's setup and logon. Windows tests execute locally in CI. - test_tags = ["exclusive-if-local"], ) diff --git a/codex-rs/windows-sandbox-rs/Cargo.toml b/codex-rs/windows-sandbox-rs/Cargo.toml index 65332f3506bf..356991b1ca87 100644 --- a/codex-rs/windows-sandbox-rs/Cargo.toml +++ b/codex-rs/windows-sandbox-rs/Cargo.toml @@ -102,4 +102,5 @@ features = [ version = "0.52" [dev-dependencies] +codex-windows-sandbox-test-support = { workspace = true } pretty_assertions = { workspace = true } diff --git a/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs b/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs index f35ee56b3821..4520a51a88f8 100644 --- a/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs +++ b/codex-rs/windows-sandbox-rs/src/unified_exec/tests.rs @@ -16,6 +16,7 @@ use codex_protocol::models::PermissionProfile; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_pty::ProcessDriver; use codex_utils_pty::ProcessSignal; +use codex_windows_sandbox_test_support::WindowsSandboxAccountTestGuard; use pretty_assertions::assert_eq; use std::collections::HashMap; use std::fs; @@ -302,6 +303,8 @@ fn legacy_non_tty_cmd_emits_output() { #[test] fn elevated_non_tty_cmd_forwards_env_output_and_exit() { + let _account_guard = + WindowsSandboxAccountTestGuard::acquire().expect("lock Windows sandbox test accounts"); let _guard = legacy_process_test_guard(); let runtime = current_thread_runtime(); runtime.block_on(async move { diff --git a/codex-rs/windows-sandbox-rs/tests/support/BUILD.bazel b/codex-rs/windows-sandbox-rs/tests/support/BUILD.bazel new file mode 100644 index 000000000000..dea523964c5b --- /dev/null +++ b/codex-rs/windows-sandbox-rs/tests/support/BUILD.bazel @@ -0,0 +1,8 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "support", + crate_name = "codex_windows_sandbox_test_support", + # This support library has no tests of its own. + test_tags = ["manual"], +) diff --git a/codex-rs/windows-sandbox-rs/tests/support/Cargo.toml b/codex-rs/windows-sandbox-rs/tests/support/Cargo.toml new file mode 100644 index 000000000000..6cfb1fb78e85 --- /dev/null +++ b/codex-rs/windows-sandbox-rs/tests/support/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "codex-windows-sandbox-test-support" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +test = false +doctest = false + +[lints] +workspace = true + +[target.'cfg(windows)'.dependencies] +dirs = { workspace = true } diff --git a/codex-rs/windows-sandbox-rs/tests/support/src/lib.rs b/codex-rs/windows-sandbox-rs/tests/support/src/lib.rs new file mode 100644 index 000000000000..7c69fd1fee49 --- /dev/null +++ b/codex-rs/windows-sandbox-rs/tests/support/src/lib.rs @@ -0,0 +1,49 @@ +//! Coordinates tests that share the runner's Windows sandbox accounts across processes. + +#![cfg(windows)] + +use std::fs::File; +use std::fs::OpenOptions; +use std::io; + +/// Serializes tests that provision or use the runner's shared Windows sandbox accounts. +/// Hold this from before setup until all sandbox commands and cleanup have finished. +#[must_use = "the account lock is released when the guard is dropped"] +pub struct WindowsSandboxAccountTestGuard { + _file: File, +} + +impl WindowsSandboxAccountTestGuard { + /// Acquires the same file lock across all test processes run by this Windows user. + pub fn acquire() -> io::Result { + let directory = dirs::data_local_dir().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "Windows local application data directory is unavailable", + ) + })?; + // Bazel uses per-test temporary directories, so the lock lives in the runner user's + // Windows known folder instead. It is separate from the production setup lock. + let path = directory.join("codex-windows-sandbox-integration-test.lock"); + // Leave the file in place: deleting it would let processes lock different files at this path. + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path) + .map_err(|error| { + io::Error::new( + error.kind(), + format!("open Windows sandbox test lock: {error}"), + ) + })?; + file.lock().map_err(|error| { + io::Error::new( + error.kind(), + format!("lock Windows sandbox test accounts: {error}"), + ) + })?; + Ok(Self { _file: file }) + } +}