[pull] main from openai:main

codex-rs/app-server/src/lib.rs

@@ -78,6 +78,7 @@ use tracing::info;
 use tracing::warn;
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::Layer;
+use tracing_subscriber::fmt::format::FmtSpan;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::registry::Registry;
 use tracing_subscriber::util::SubscriberInitExt;
@@ -164,6 +165,10 @@ enum LogFormat {
 
 type StderrLogLayer = Box<dyn Layer<Registry> + Send + Sync + 'static>;
 
+fn stderr_span_events() -> FmtSpan {
+    FmtSpan::NEW | FmtSpan::CLOSE
+}
+
 /// Control-plane messages from the processor/transport side to the outbound router task.
 ///
 /// `run_main_with_transport_options` uses two loops/tasks:
@@ -703,16 +708,19 @@ pub async fn run_main_with_transport_options(
     // Install a simple subscriber so `tracing` output is visible. Users can
     // control the log level with `RUST_LOG` and switch to JSON logs with
     // `LOG_FORMAT=json`.
+    // SQLx enters the caller's span for each command. Skip enter/exit records
+    // that can block its worker on stderr while holding a write transaction.
+    // Preserve span boundaries, busy/idle timings, and explicit events.
     let stderr_fmt: StderrLogLayer = match log_format_from_env() {
         LogFormat::Json => tracing_subscriber::fmt::layer()
             .json()
             .with_writer(std::io::stderr)
-            .with_span_events(tracing_subscriber::fmt::format::FmtSpan::FULL)
+            .with_span_events(stderr_span_events())
             .with_filter(EnvFilter::from_default_env())
             .boxed(),
         LogFormat::Default => tracing_subscriber::fmt::layer()
             .with_writer(std::io::stderr)
-            .with_span_events(tracing_subscriber::fmt::format::FmtSpan::FULL)
+            .with_span_events(stderr_span_events())
             .with_filter(EnvFilter::from_default_env())
             .boxed(),
     };
@@ -1524,6 +1532,10 @@ fn analytics_rpc_transport(transport: &AppServerTransport) -> AppServerRpcTransp
     }
 }
 
+#[cfg(test)]
+#[path = "stderr_logging_tests.rs"]
+mod stderr_logging_tests;
+
 #[cfg(test)]
 mod tests {
     use super::LogFormat;

codex-rs/app-server/src/stderr_logging_tests.rs

@@ -0,0 +1,124 @@
+//! Regressions for SQLite progress and stderr span diagnostics.
+
+use crate::LogFormat;
+use crate::StderrLogLayer;
+use crate::stderr_span_events;
+use anyhow::Context;
+use anyhow::Result;
+use codex_state::SqliteConfig;
+use codex_utils_absolute_path::test_support::PathExt;
+use pretty_assertions::assert_eq;
+use std::io::Read;
+use std::sync::Mutex;
+use std::time::Duration;
+use tempfile::TempDir;
+use test_case::test_case;
+use tokio::sync::oneshot;
+use tokio::time::timeout;
+use tracing_subscriber::EnvFilter;
+use tracing_subscriber::Layer;
+use tracing_subscriber::layer::SubscriberExt;
+
+#[test_case(LogFormat::Json; "json")]
+#[test_case(LogFormat::Default; "text")]
+#[tokio::test]
+async fn sqlite_transaction_completes_with_undrained_stderr(format: LogFormat) -> Result<()> {
+    let dir = TempDir::new()?;
+    let config = SqliteConfig::new_for_testing(dir.path().abs());
+    let path = dir.path().join("logging.sqlite");
+    let writer_pool = config.open_read_write_pool(&path).await?;
+    sqlx::query("CREATE TABLE writes (id INTEGER PRIMARY KEY, value INTEGER NOT NULL)")
+        .execute(&writer_pool)
+        .await?;
+    // Open both pools before tracing so this isolates logging during a transaction.
+    let contender_pool = config.open_read_write_pool(&path).await?;
+    let (mut reader, writer) = std::io::pipe()?;
+    let fmt = tracing_subscriber::fmt::layer()
+        .with_writer(Mutex::new(writer))
+        .with_span_events(stderr_span_events());
+    let filter = EnvFilter::new("off,codex_app_server::stderr_logging_tests=info");
+    let fmt: StderrLogLayer = match format {
+        LogFormat::Json => fmt.json().with_filter(filter).boxed(),
+        LogFormat::Default => fmt.with_filter(filter).boxed(),
+    };
+    let subscriber = tracing_subscriber::registry().with(fmt);
+    let (locked_tx, locked_rx) = oneshot::channel();
+    let (finished_tx, finished_rx) = oneshot::channel();
+
+    // A synchronous stderr write can block a runtime thread as well as SQLx's
+    // worker. Keep the timeout on a separate thread so regressions cannot hang it.
+    let worker = std::thread::spawn(move || {
+        let result = tracing::subscriber::with_default(subscriber, || -> Result<()> {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()?;
+            let span = tracing::info_span!("sqlite_transaction");
+            span.in_scope(|| {
+                runtime.block_on(async {
+                    let mut transaction = writer_pool.begin().await?;
+                    sqlx::query("INSERT INTO writes VALUES (1, 0)")
+                        .execute(&mut *transaction)
+                        .await?;
+                    let _ = locked_tx.send(());
+                    // Enough commands to fill an unread OS pipe if every entry
+                    // and exit of the caller's span produces a log record.
+                    for _ in 0..4096 {
+                        sqlx::query("UPDATE writes SET value = value + 1 WHERE id = 1")
+                            .execute(&mut *transaction)
+                            .await?;
+                    }
+                    transaction.commit().await?;
+                    tracing::info!("transaction committed");
+                    tracing::warn!("explicit warning");
+                    writer_pool.close().await;
+                    Ok(())
+                })
+            })
+        });
+        let _ = finished_tx.send(result);
+    });
+
+    let progress = timeout(Duration::from_secs(/*secs*/ 15), async {
+        locked_rx.await?;
+        let write = async {
+            sqlx::query("INSERT INTO writes VALUES (2, 0)")
+                .execute(&contender_pool)
+                .await
+                .context("competing writer could not acquire the SQLite lock")
+        };
+        tokio::try_join!(write, async { finished_rx.await? })?;
+        Ok::<_, anyhow::Error>(())
+    })
+    .await;
+
+    // Always drain before reporting a failure, releasing a worker blocked in
+    // tracing if enter/exit logging is accidentally restored.
+    let logs = tokio::task::spawn_blocking(move || {
+        let mut logs = String::new();
+        reader.read_to_string(&mut logs)?;
+        Ok::<_, std::io::Error>(logs)
+    })
+    .await??;
+    worker.join().expect("SQLite worker panicked");
+    progress.context("SQLite transaction stalled with undrained stderr")??;
+    let rows: Vec<(i64, i64)> = sqlx::query_as("SELECT id, value FROM writes ORDER BY id")
+        .fetch_all(&contender_pool)
+        .await?;
+    contender_pool.close().await;
+    assert_eq!(rows, vec![(1, 4096), (2, 0)]);
+
+    let lines: Vec<_> = logs.lines().collect();
+    assert_eq!(lines.len(), 4, "{logs}");
+    for (line, message) in
+        lines
+            .iter()
+            .zip(["new", "transaction committed", "explicit warning", "close"])
+    {
+        assert!(line.contains(message), "{logs}");
+    }
+    assert!(lines[2].contains("WARN"), "{logs}");
+    for field in ["time.busy", "time.idle"] {
+        assert!(lines[3].contains(field), "{logs}");
+    }
+    Ok(())
+}

codex-rs/cli/src/debug_sandbox.rs

@@ -413,7 +413,11 @@ async fn run_command_under_sandbox(
             // for the unsandboxed shell that resumes when Codex exits.
             match args.as_mut_slice() {
                 [flag, policy, ..] if flag.as_str() == "-p" => {
-                    policy.push_str("\n(deny file-ioctl (ioctl-command TIOCSTI))");
+                    // Older macOS policy compilers do not define the TIOCSTI symbol.
+                    policy.push_str(&format!(
+                        "\n(deny file-ioctl (ioctl-command {}))",
+                        libc::TIOCSTI
+                    ));
                 }
                 _ => anyhow::bail!("Seatbelt command is missing its generated policy"),
             }

codex-rs/config/src/config_toml.rs

@@ -3,6 +3,7 @@
 use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::num::NonZeroU64;
+use std::num::NonZeroUsize;
 use std::path::Path;
 
 use crate::HooksToml;
@@ -579,6 +580,12 @@ pub struct AutoReviewToml {
     pub extra_policy: Option<String>,
     /// Experimental full Guardian prompt template containing the tenant policy placeholder.
     pub experimental_policy_template: Option<String>,
+    /// Experimental replacement for the history-retrieval instructions when history tools
+    /// and Apps are enabled. Omitted or blank values use the built-in prompt.
+    pub experimental_conversation_history_prompt: Option<String>,
+    /// Maximum estimated tokens per Guardian history-tool response, before the standard
+    /// serialization allowance. Defaults to 4,000; stricter parent tool limits still apply.
+    pub conversation_history_max_output_tokens: Option<NonZeroUsize>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)]

codex-rs/core/config.schema.json

@@ -398,6 +398,16 @@
           ],
           "description": "Controls whether circuit-breaker interruptions include a structured error. Strict mode writes structured errors that older clients may not recognize when reading shared history. Defaults to `default`."
         },
+        "conversation_history_max_output_tokens": {
+          "description": "Maximum estimated tokens per Guardian history-tool response, before the standard serialization allowance. Defaults to 4,000; stricter parent tool limits still apply.",
+          "format": "uint",
+          "minimum": 1.0,
+          "type": "integer"
+        },
+        "experimental_conversation_history_prompt": {
+          "description": "Experimental replacement for the history-retrieval instructions when history tools and Apps are enabled. Omitted or blank values use the built-in prompt.",
+          "type": "string"
+        },
         "experimental_policy_template": {
           "description": "Experimental full Guardian prompt template containing the tenant policy placeholder.",
           "type": "string"
@@ -875,6 +885,9 @@
             "guardian_approval": {
               "type": "boolean"
             },
+            "guardian_conversation_history_tools": {
+              "type": "boolean"
+            },
             "guardian_enhanced_node_repl_transcripts": {
               "type": "boolean"
             },
@@ -887,6 +900,9 @@
             "guardian_reuse_parent_compaction": {
               "type": "boolean"
             },
+            "guardian_root_handoff_context": {
+              "type": "boolean"
+            },
             "guardianv2": {
               "$ref": "#/definitions/FeatureToml_for_GuardianV2ConfigToml"
             },
@@ -6905,6 +6921,9 @@
         "guardian_approval": {
           "type": "boolean"
         },
+        "guardian_conversation_history_tools": {
+          "type": "boolean"
+        },
         "guardian_enhanced_node_repl_transcripts": {
           "type": "boolean"
         },
@@ -6917,6 +6936,9 @@
         "guardian_reuse_parent_compaction": {
           "type": "boolean"
         },
+        "guardian_root_handoff_context": {
+          "type": "boolean"
+        },
         "guardianv2": {
           "$ref": "#/definitions/FeatureToml_for_GuardianV2ConfigToml"
         },

codex-rs/core/src/agent/control.rs

@@ -71,6 +71,7 @@ mod interrupt;
 mod legacy;
 mod residency;
 mod resume;
+mod root_handoff;
 mod runtime;
 mod runtime_context;
 mod sender_context;