From edf5456eab5f91c6636a007e5f93b51bf4129950 Mon Sep 17 00:00:00 2001 From: rick12345 Date: Fri, 13 Nov 2026 17:18:46 +0800 Subject: [PATCH] fix(agent): preserve NUL characters in archived tool output Generated-by: OpenAI Codex --- plugins/agent/facts/src/archive/source.rs | 68 +++++++++++++++++++++++ plugins/agent/facts/src/read/source.rs | 12 +++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/plugins/agent/facts/src/archive/source.rs b/plugins/agent/facts/src/archive/source.rs index 8093358adb..693d4b72fa 100644 --- a/plugins/agent/facts/src/archive/source.rs +++ b/plugins/agent/facts/src/archive/source.rs @@ -303,3 +303,71 @@ async fn scan_source( fn invalid(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, message) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::prompt_source::source::{Allocate, ChargedBuffer, SourceBytes}; + use serde_json::json; + + struct Budget; + + impl SourceBudget for Budget { + fn allocate(&self, minimum: usize) -> Allocate<'_> { + Box::pin(async move { Ok(ChargedBuffer::new(vec![0; minimum], ())) }) + } + } + + #[tokio::test] + async fn archived_shell_output_with_nul_can_be_paged() { + let stdout = format!("{}\nlast\0line", "前\0后\n".repeat(1_000)); + let projection = JsonSource::new(SourceBytes::Inline( + serde_json::to_vec(&json!({ + "kind": "json", + "value": { + "kind": "terminal", + "status": "completed", + "exitCode": 0, + "output": { + "mode": "pipes", + "stdout": stdout, + "stderr": "", + "stdoutTruncated": false, + "stderrTruncated": false, + "redacted": false + } + } + })) + .unwrap() + .into(), + )); + let budget: Arc = Arc::new(Budget); + let stop = CancellationToken::new(); + let first = ArchivedPlaceholder::prepare_source( + "original-event".into(), + "shell-call".into(), + "Shell".into(), + projection.clone(), + budget.clone(), + &stop, + ) + .await + .unwrap() + .expect("long shell output needs an archive") + .page; + let next = first.next.expect("long output has a continuation"); + let second = read::projection_page( + projection, + "Shell", + &next.resolve().unwrap(), + MAX_PAGE_CHARS, + budget, + &stop, + ) + .await + .unwrap(); + assert!(!first.partial_line); + assert!(second.next.is_none()); + assert_eq!(format!("{}\n{}", first.content, second.content), stdout); + } +} diff --git a/plugins/agent/facts/src/read/source.rs b/plugins/agent/facts/src/read/source.rs index 7fb45fdb29..0ec94ff3c1 100644 --- a/plugins/agent/facts/src/read/source.rs +++ b/plugins/agent/facts/src/read/source.rs @@ -66,7 +66,9 @@ async fn page( .transpose() .map_err(io::Error::other)? .map_or(0, |value| value.encode_utf16().count() + 12); - let mut page = Utf8Page::new(request, max_chars.saturating_sub(overhead)); + // These parts are serialized JSON or decoded JSON strings, so a NUL + // belongs to the tool result and is escaped again when the page is encoded. + let mut page = Utf8Page::with_nul_policy(request, max_chars.saturating_sub(overhead), false); let mut previous = false; for bytes in parts { let mut cursor = bytes.open(budget.clone()); @@ -380,12 +382,18 @@ async fn shell( pub struct Utf8Page<'a> { scan: TextPageScan<'a>, pending: Vec, + reject_nul: bool, } impl<'a> Utf8Page<'a> { pub fn new(request: &'a ReadRequest, budget: usize) -> Self { + Self::with_nul_policy(request, budget, true) + } + + fn with_nul_policy(request: &'a ReadRequest, budget: usize, reject_nul: bool) -> Self { Self { scan: request.scan_with_budget(budget), pending: Vec::new(), + reject_nul, } } pub fn push(&mut self, bytes: &[u8]) -> io::Result<()> { @@ -400,7 +408,7 @@ impl<'a> Utf8Page<'a> { } }; let text = std::str::from_utf8(&self.pending[..end]).map_err(io::Error::other)?; - if text.contains('\0') { + if self.reject_nul && text.contains('\0') { return Err(io::Error::other( "Read cannot decode this binary content as text", ));