From 6aa0078fce83dd187f2d6f76a9cf406604334784 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 15:17:46 +0500 Subject: [PATCH 01/18] feat(debug-files): --type elf accepts directories of .so files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Walk directories recursively for .so/.so.dbg/.so.sym (+ --extension), reading libraries in place instead of requiring a pre-built zip. Multiple paths (dirs and/or zips) are collected first so the one-per-build-id preference spans all of them. Each zip gets its own extraction dir; an unreadable directory root fails instead of exiting 0; a missing path is exit 10. Closes #69 πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- README.md | 2 +- src/cli/debug_files.rs | 178 ++++++++++++++++++++-------------------- src/symbols/elf.rs | 160 ++++++++++++++++++++++++++++++++++-- src/symbols/mod.rs | 4 +- tests/elf_upload.rs | 180 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 425 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 0980d3e..495165f 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The upload flow itself. ProGuard, Rust, ELF, dSYM, PDB, sourcemap, and Unity IL2 #### Android NDK β€” `--type elf` -Takes AGP's pre-built `native-debug-symbols.zip` (a directory is rejected; the Gradle plugin zips the intermediates folder first). Every library inside is uploaded as its own symbol, keyed by its GNU build-id β€” `--uuid` (the SDK's `BUILD_UUID`) is still required but only correlates logs. Built-in entry names are `.so`, `.so.dbg` (`ndk.debugSymbolLevel = 'FULL'`) and `.so.sym` (`'SYMBOL_TABLE'`, function names only β€” no `file:line`). A library without a build-id is warned about and skipped; two entries with the same build-id upload once, preferring the one with DWARF. The server dedups on the build-id, so switching a library from `SYMBOL_TABLE` to `FULL` needs `--force` to replace the stored symbols. +Takes one or more paths: AGP's pre-built `native-debug-symbols.zip`, and/or directories, which are walked recursively for libraries (e.g. `build/intermediates/merged_native_libs/`) and read in place β€” nothing is re-zipped. Every library found is uploaded as its own symbol, keyed by its GNU build-id β€” `--uuid` (the SDK's `BUILD_UUID`) is still required but only correlates logs. Built-in entry names are `.so`, `.so.dbg` (`ndk.debugSymbolLevel = 'FULL'`) and `.so.sym` (`'SYMBOL_TABLE'`, function names only β€” no `file:line`). A library without a build-id is warned about and skipped; two entries with the same build-id upload once (across all the given paths), preferring the one with DWARF. The server dedups on the build-id, so switching a library from `SYMBOL_TABLE` to `FULL` needs `--force` to replace the stored symbols. #### Unity IL2CPP β€” `--type il2cpp-linemap` diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index 4d9dd29..3fc10fd 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -17,7 +17,9 @@ const DEFAULT_ENDPOINT: &str = "https://api.bugsee.com"; pub enum DebugFilesCommand { /// Discover, package, and upload debug information files from one or more paths. Upload { - /// One or more directories or files to scan. + /// One or more directories or files to scan. For `--type elf`, a directory + /// is walked recursively for native libraries (e.g. AGP's + /// `merged_native_libs`), and a file is a `native-debug-symbols.zip`. #[arg(required = true)] paths: Vec, @@ -141,8 +143,9 @@ pub enum DebugFilesCommand { /// such as `.so.sym` work. It only widens the NAME match: each type's content check /// still applies to what it picks up. /// - /// Per type: `elf` β€” entries in the native-debug-symbols zip (built in: `.so`, - /// `.so.dbg`, `.so.sym`; each still needs a GNU build-id). `proguard` β€” files in a + /// Per type: `elf` β€” files in a scanned directory or entries in the + /// native-debug-symbols zip (built in: `.so`, `.so.dbg`, `.so.sym`; each still + /// needs a GNU build-id). `proguard` β€” files in a /// scanned directory (built in: `mapping*.txt`). `sourcemaps` β€” files in a scanned /// directory (built in: `.map`). `pdb` β€” files in a scanned directory, compared /// case-insensitively (built in: `.pdb`; each still needs the MSF container @@ -1078,15 +1081,9 @@ pub(crate) fn discover_dsyms(paths: &[PathBuf], extra: &ExtraSuffixes) -> Vec/out` directory -/// before invoking the CLI). +/// Upload native libraries from one or more inputs: AGP's +/// `native-debug-symbols.zip` and/or directories walked recursively. Each +/// library becomes its own symbol document keyed by its GNU build-id. #[allow(clippy::too_many_arguments)] /// Max concurrent per-`.so` register+upload pipelines. A native archive can hold /// dozens of libraries across ABIs; uploading them serially would dominate CI @@ -1151,18 +1148,6 @@ async fn run_elf_upload( if paths.is_empty() { return Err(input_not_found("no input paths supplied")); } - for p in paths { - if !p.is_file() { - // Directory-walk + re-pack is out of scope β€” the Gradle plugin - // pre-zips the intermediate-folder case before invoking the CLI. - return Err(input_invalid(format!( - "--type elf expects a pre-built zip file (typically AGP's \ - native-debug-symbols.zip); got {}", - p.display() - ))); - } - } - // `build_uuid` (the SDK's runtime BUILD_UUID) is NO LONGER the native // symbol identity β€” each `.so` is keyed by its OWN GNU build-id (one file β†’ // one symbol document β†’ one S3 object). The BUILD_UUID belongs to the @@ -1194,10 +1179,42 @@ async fn run_elf_upload( let mut full_already_existed = 0u32; let mut skipped_no_build_id = 0u32; - for archive in paths { - tracing::info!(path = %archive.display(), "processing native-debug-symbols archive"); - let work_dir = tempfile::tempdir()?; - let libs = elf::extract_libs(archive, work_dir.path(), extra)?; + // Collect from EVERY input first so the one-per-build-id preference spans + // paths (e.g. the same library under two variant folders), then upload once. + let work_dir = tempfile::tempdir()?; + let mut libs: Vec = Vec::new(); + for (n, input) in paths.iter().enumerate() { + tracing::info!(path = %input.display(), "scanning for native libraries"); + // Own extraction dir per input: zip entries are staged as `_`, + // and indices restart at 0 in every archive. + let input_dir = work_dir.path().join(n.to_string()); + std::fs::create_dir_all(&input_dir)?; + let found = elf::collect_libs(input, &input_dir, extra).map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + input_not_found(format!("{} does not exist", input.display())) + } else if input.is_dir() { + input_invalid(format!("cannot scan {}: {e}", input.display())) + } else { + input_invalid(format!( + "--type elf expects a directory or a native-debug-symbols zip; \ + cannot read {}: {e}", + input.display() + )) + } + })?; + if found.is_empty() { + // Nothing even matched by name β€” the one case `--extension` fixes. + // Ahead of the dry-run exit: a dry run is how a caller checks this. + tracing::warn!( + path = %input.display(), + "no files named .so / .so.dbg / .so.sym (or an --extension suffix) β€” \ + nothing to upload. If your toolchain emits another suffix, pass it \ + with --extension" + ); + } + libs.extend(found); + } + { let libs = elf::keep_richest_per_build_id(libs); let total = libs.len(); @@ -1220,61 +1237,45 @@ async fn run_elf_upload( tracing::info!( libraries = total, uploadable = uploadable.len(), - "extracted native libraries" + "collected native libraries" ); if total == 0 { - // Nothing even matched by name β€” the one case `--extension` fixes. - // Ahead of the dry-run exit: a dry run is how a caller checks this. - tracing::warn!( - archive = %archive.display(), - "no entries named .so / .so.dbg / .so.sym (or an --extension suffix) β€” \ - nothing to upload. If your toolchain emits another suffix, pass it \ - with --extension" - ); - continue; - } - if dry_run { + // Already warned per input above. + } else if dry_run { tracing::info!( - "dry-run: would register + upload {} libraries from {}", - uploadable.len(), - archive.display() + "dry-run: would register + upload {} libraries", + uploadable.len() ); - continue; - } - if uploadable.is_empty() { - tracing::warn!( - archive = %archive.display(), - "no native libraries with a GNU build-id β€” nothing to upload" - ); - continue; - } - - let client = client.as_ref().expect("client constructed when !dry_run"); - - // Run each `.so`'s pack β†’ register β†’ dedup-or-PUT pipeline concurrently. - let outcomes: Vec<(bool, anyhow::Result)> = - futures_util::stream::iter(uploadable) - .map(|lib| { - let client = client.clone(); - let work = work_dir.path().to_path_buf(); - async move { - let symbol_table_only = lib.name.ends_with(".so.sym"); - let outcome = upload_one_so(&client, ctx, &lib, &work).await; - (symbol_table_only, outcome) - } - }) - .buffer_unordered(ELF_UPLOAD_CONCURRENCY) - .collect() - .await; - - for (symbol_table_only, outcome) in outcomes { - match outcome? { - presigned::Outcome::Uploaded => uploaded += 1, - presigned::Outcome::AlreadyExists => { - already_existed += 1; - if !symbol_table_only { - full_already_existed += 1; + } else if uploadable.is_empty() { + tracing::warn!("no native libraries with a GNU build-id β€” nothing to upload"); + } else { + let client = client.as_ref().expect("client constructed when !dry_run"); + + // Run each `.so`'s pack β†’ register β†’ dedup-or-PUT pipeline concurrently. + let outcomes: Vec<(bool, anyhow::Result)> = + futures_util::stream::iter(uploadable.into_iter().enumerate()) + .map(|(i, lib)| { + let client = client.clone(); + let work = work_dir.path().to_path_buf(); + async move { + let symbol_table_only = lib.name.ends_with(".so.sym"); + let outcome = upload_one_so(&client, ctx, &lib, i, &work).await; + (symbol_table_only, outcome) + } + }) + .buffer_unordered(ELF_UPLOAD_CONCURRENCY) + .collect() + .await; + + for (symbol_table_only, outcome) in outcomes { + match outcome? { + presigned::Outcome::Uploaded => uploaded += 1, + presigned::Outcome::AlreadyExists => { + already_existed += 1; + if !symbol_table_only { + full_already_existed += 1; + } } } } @@ -1314,6 +1315,7 @@ async fn upload_one_so( client: &reqwest::Client, ctx: ElfUploadCtx<'_>, lib: &elf::ElfLib, + index: usize, work_dir: &Path, ) -> anyhow::Result { let build_id = lib @@ -1325,18 +1327,14 @@ async fn upload_one_so( .and_then(|n| n.to_str()) .unwrap_or("lib.so"); - // Name the staging ZIP after the extracted `.so`'s on-disk filename, which - // `elf::extract_libs` already made unique with an index prefix - // (`_`). Keying it on `build_id` instead would collide under - // the concurrent `buffer_unordered` upload if two libraries shared a - // build-id β€” two tasks would `File::create` the same path and interleave - // writes into a corrupt ZIP with the wrong SHA-1. - let zip_stem = lib - .path - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("lib.so"); - let zip_path = work_dir.join(format!("{zip_stem}.zip")); + // Name the staging ZIP by the caller's per-library index, unique within the + // run. Keying it on `build_id` would collide under the concurrent + // `buffer_unordered` upload if two libraries shared a build-id, and the + // basename alone collides across ABIs (a directory scan reads libraries in + // place, so `lib/arm64-v8a/libfoo.so` and `lib/x86_64/libfoo.so` both end + // in `libfoo.so`) β€” two tasks would `File::create` the same path and + // interleave writes into a corrupt ZIP with the wrong SHA-1. + let zip_path = work_dir.join(format!("{index}_{entry_name}.zip")); compress::pack_single_entry(&lib.path, entry_name, &zip_path, ctx.strategy)?; let hash = elf::sha1_hex_of_file(&zip_path)?; diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index 7a3150c..0b05c70 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -2,7 +2,7 @@ //! //! The caller hands the CLI an already-packaged `native-debug-symbols.zip` //! (the artifact AGP writes under `build/outputs/native-debug-symbols/ -//! /`). We open it and, for EACH `.so` (or the `.so.dbg` / `.so.sym` +//! /`) or a directory of libraries. We open/walk it and, for EACH `.so` (or the `.so.dbg` / `.so.sym` //! variants, depending on `ndk.debugSymbolLevel`), read the GNU build-id //! (`code_id`) via `symbolic-debuginfo` β€” the SAME crate family (major 13) the //! worker uses, so the identifier is byte-identical producer↔consumer. @@ -21,9 +21,9 @@ //! matched at crash time, so it is warned about and skipped β€” never faked with //! the BUILD_UUID. //! -//! Out of scope: walking a loose directory of unstripped `.so`s (AGP's -//! intermediate-folder case) β€” the Gradle plugin pre-zips that before invoking -//! the CLI. +//! The input is either that zip or a directory, walked recursively for the same +//! entry names (AGP's `merged_native_libs` intermediates folder) β€” libraries +//! are read in place, so nothing is re-compressed just to be unpacked again. use sha1::{Digest as _, Sha1}; use std::io::Read; @@ -42,7 +42,7 @@ pub fn sha1_hex_of_file(path: &Path) -> std::io::Result { Ok(sha1_hex(&std::fs::read(path)?)) } -/// One native library extracted from an AGP `native-debug-symbols.zip`. +/// One native library found in an AGP `native-debug-symbols.zip` or a scanned directory. /// /// Each `.so` is its OWN symbol: one file β†’ one symbol document β†’ one S3 /// object, keyed by its own GNU build-id. (`images[]` within a single document @@ -59,7 +59,8 @@ pub struct ElfLib { /// Object architecture (e.g. `arm64`); diagnostic only β€” the worker /// reconciles the canonical arch from the ELF during processing. pub arch: String, - /// Temp file holding the extracted `.so` bytes, packed into its own upload. + /// File holding the library bytes, packed into its own upload: a temp copy + /// extracted from a zip, or the original file when scanned from a directory. pub path: PathBuf, /// How much this file can symbolicate: (DWARF debug info, symbol table, /// byte size), compared in that order by [`keep_richest_per_build_id`]. @@ -122,6 +123,79 @@ pub fn extract_libs( Ok(libs) } +/// Collect native libraries from `input`: a directory is walked recursively +/// (see [`scan_dir`]), anything else is read as a `native-debug-symbols.zip` +/// and extracted into `out_dir` (see [`extract_libs`]). +pub fn collect_libs( + input: &Path, + out_dir: &Path, + extra: &ExtraSuffixes, +) -> std::io::Result> { + if input.is_dir() { + scan_dir(input, extra) + } else { + extract_libs(input, out_dir, extra) + } +} + +/// Walk `dir` recursively for native libraries (same name rules as the zip +/// entries), reading each one's identity in place. `name` is the path relative +/// to `dir`. Symlinks are followed (loops are skipped); unreadable entries are +/// skipped with a warning rather than failing the whole scan. Sorted by path so +/// the "first entry wins" tie-break in [`keep_richest_per_build_id`] is stable. +pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result> { + let mut libs = Vec::new(); + for entry in walkdir::WalkDir::new(dir) + .follow_links(true) + .sort_by_file_name() + { + let entry = match entry { + Ok(e) => e, + // An unreadable root means nothing was scanned: fail, never exit 0 with + // a misleading "pass --extension" hint. + Err(e) if e.depth() == 0 => return Err(e.into()), + Err(e) => { + tracing::warn!(error = %e, "skipping unreadable path while scanning for native libraries"); + continue; + } + }; + if !entry.file_type().is_file() { + continue; + } + let file_name = entry.file_name().to_string_lossy(); + if !(is_native_lib_entry(&file_name) || extra.matches(&file_name)) { + continue; + } + let bytes = match std::fs::read(entry.path()) { + Ok(b) => b, + Err(e) => { + tracing::warn!(path = %entry.path().display(), error = %e, "skipping unreadable native library"); + continue; + } + }; + let ElfIdentity { + build_id, + arch, + has_debug_info, + has_symbols, + } = parse_elf_identity(&bytes); + let name = entry + .path() + .strip_prefix(dir) + .unwrap_or(entry.path()) + .to_string_lossy() + .replace('\\', "/"); + libs.push(ElfLib { + name, + build_id, + arch, + path: entry.path().to_path_buf(), + richness: (has_debug_info, has_symbols, bytes.len() as u64), + }); + } + Ok(libs) +} + /// Whether a `native-debug-symbols.zip` entry is a native library to upload. /// The suffix depends on AGP's `ndk.debugSymbolLevel`: `FULL` packs unstripped /// `.so` (occasionally `.so.dbg`), while `SYMBOL_TABLE` packs `.so.sym` β€” an @@ -359,6 +433,80 @@ mod tests { assert_eq!(std::fs::read(&libs[0].path).unwrap(), b"not a real elf"); } + fn fixture_bytes() -> Vec { + std::fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"), + ) + .unwrap() + } + + #[test] + fn scan_dir_walks_recursively_in_place_and_skips_other_files() { + let dir = tempfile::tempdir().unwrap(); + let bytes = fixture_bytes(); + for rel in [ + "lib/arm64-v8a/libsymbol1.so", + "lib/x86_64/libsymbol1.so.sym", + "lib/x86_64/readme.txt", + ] { + let p = dir.path().join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, &bytes).unwrap(); + } + std::fs::write(dir.path().join("lib/x86_64/libbare.so"), b"not elf").unwrap(); + + let libs = scan_dir(dir.path(), &ExtraSuffixes::default()).unwrap(); + assert_eq!( + names(&libs), + [ + "lib/arm64-v8a/libsymbol1.so", + "lib/x86_64/libbare.so", + "lib/x86_64/libsymbol1.so.sym" + ] + ); + assert_eq!( + libs[0].build_id.as_deref(), + Some("bca64abfec40dbb631bb8f1c37414472") + ); + assert_eq!(libs[1].build_id, None, "non-ELF yields no build-id"); + assert_eq!( + libs[0].path, + dir.path().join("lib/arm64-v8a/libsymbol1.so"), + "read in place, not copied" + ); + + let extra = ExtraSuffixes::parse(&["txt".to_string()]).unwrap(); + assert_eq!(scan_dir(dir.path(), &extra).unwrap().len(), 4); + } + + #[test] + fn collect_libs_dispatches_on_directory_vs_zip() { + let dir = tempfile::tempdir().unwrap(); + let bytes = fixture_bytes(); + let libdir = dir.path().join("libs"); + std::fs::create_dir_all(&libdir).unwrap(); + std::fs::write(libdir.join("libsymbol1.so"), &bytes).unwrap(); + let zip_path = dir.path().join("n.zip"); + { + let mut zw = zip::ZipWriter::new(std::fs::File::create(&zip_path).unwrap()); + zw.start_file("arm64-v8a/libsymbol1.so", SimpleFileOptions::default()) + .unwrap(); + zw.write_all(&bytes).unwrap(); + zw.finish().unwrap(); + } + let out = dir.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let x = ExtraSuffixes::default(); + assert_eq!( + names(&collect_libs(&libdir, &out, &x).unwrap()), + ["libsymbol1.so"] + ); + assert_eq!( + names(&collect_libs(&zip_path, &out, &x).unwrap()), + ["arm64-v8a/libsymbol1.so"] + ); + } + fn lib(name: &str, build_id: Option<&str>, richness: (bool, bool, u64)) -> ElfLib { ElfLib { name: name.to_string(), diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 75838a9..d30ef4c 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -2,8 +2,8 @@ //! //! Submodules handle format-specific concerns: //! - `proguard` β€” R8 / ProGuard mapping files (v0.1) -//! - `elf` β€” NDK native-debug-symbols zip pass-through -//! (v0.1 pass-through; per-`.so` build-id extraction TBD) +//! - `elf` β€” NDK native libraries (a native-debug-symbols zip or a +//! directory), per-`.so` build-id keyed //! - `dsym` β€” Apple Mach-O dSYM bundles (v0.1: single bundle; //! multi-bundle directory walk + BCSymbolMap support TBD) //! - `pdb` β€” Windows PDB (MSF container); identity is the debug id diff --git a/tests/elf_upload.rs b/tests/elf_upload.rs index 7791c98..168429e 100644 --- a/tests/elf_upload.rs +++ b/tests/elf_upload.rs @@ -325,3 +325,183 @@ fn elf_dry_run_with_no_matching_entries_suggests_extension() { .success() .stderr(predicates::str::contains("pass it with --extension")); } + +fn elf_args<'a>(endpoint: &'a str, inputs: &'a [String]) -> Vec<&'a str> { + let mut a = vec![ + "--endpoint", + endpoint, + "--app-token", + "TKN", + "debug-files", + "upload", + "--type", + "elf", + "--version", + "1.0", + "--build", + "1", + "--uuid", + "00000000-0000-0000-0000-000000000000", + ]; + a.extend(inputs.iter().map(String::as_str)); + a +} + +/// A directory (AGP's `merged_native_libs` layout) is walked recursively. The +/// same library under two ABI folders, plus a zip of it, shares ONE build-id, +/// so exactly one POST + PUT go out; the non-library file is ignored. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn elf_upload_walks_a_directory_and_dedups_by_build_id_across_paths() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"); + let root = tmp.path().join("merged_native_libs/release/out/lib"); + for abi in ["arm64-v8a", "x86_64"] { + std::fs::create_dir_all(root.join(abi)).unwrap(); + std::fs::copy(&fixture, root.join(abi).join("libsymbol1.so")).unwrap(); + } + std::fs::write(root.join("x86_64/notes.txt"), b"ignored").unwrap(); + let zip_path = pack_native_zip(tmp.path(), "arm64-v8a/libsymbol1.so"); + + let put_url = format!("{}/put/sym", server.uri()); + Mock::given(method("POST")) + .and(wm_path("/apps/TKN/symbols")) + .and(body_string_contains(FIXTURE_BUILD_ID)) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"code": 0, "endpoint": put_url})), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + let endpoint = server.uri(); + let inputs = vec![ + tmp.path() + .join("merged_native_libs") + .to_string_lossy() + .into_owned(), + zip_path.to_string_lossy().into_owned(), + ]; + tokio::task::spawn_blocking(move || { + common::cli() + .args(elf_args(&endpoint, &inputs)) + .assert() + .success(); + }) + .await + .unwrap(); +} + +/// An input path that exists as neither a directory nor a readable zip still +/// fails (it is not silently skipped), with a message naming it. +#[test] +fn elf_upload_rejects_a_missing_path() { + let tmp = tempfile::tempdir().unwrap(); + let missing = tmp.path().join("nope").to_string_lossy().into_owned(); + let inputs = vec![missing]; + common::cli() + .args(elf_args("http://127.0.0.1:1", &inputs)) + .assert() + .code(10) + .stderr(predicates::str::contains("nope")); +} + +/// Two zips whose entries share a basename and archive index but carry DIFFERENT +/// build-ids must each upload their own bytes (extraction must not collide). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn elf_upload_keeps_same_named_entries_of_different_zips_apart() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"); + let a = pack_native_zip(tmp.path(), "arm64-v8a/libsymbol1.so"); + let a = { + let d = tmp.path().join("a"); + std::fs::create_dir_all(&d).unwrap(); + let t = d.join("a.zip"); + std::fs::rename(a, &t).unwrap(); + t + }; + // Same fixture with the build-id's first byte flipped (found by value). + let mut other = std::fs::read(&fixture).unwrap(); + let id = hex::decode(FIXTURE_BUILD_ID).unwrap(); + let pos = other.windows(id.len()).position(|w| w == id).unwrap(); + other[pos] ^= 0xff; + let other_id = hex::encode(&other[pos..pos + id.len()]); + let b = tmp.path().join("b.zip"); + { + let mut zw = zip::ZipWriter::new(std::fs::File::create(&b).unwrap()); + zw.start_file( + "x86_64/libsymbol1.so", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + zw.write_all(&other).unwrap(); + zw.finish().unwrap(); + } + + let put_url = format!("{}/put/sym", server.uri()); + Mock::given(method("POST")) + .and(wm_path("/apps/TKN/symbols")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"code": 0, "endpoint": put_url})), + ) + .expect(2) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(2) + .mount(&server) + .await; + + let endpoint = server.uri(); + let inputs = vec![ + a.to_string_lossy().into_owned(), + b.to_string_lossy().into_owned(), + ]; + tokio::task::spawn_blocking(move || { + common::cli() + .args(elf_args(&endpoint, &inputs)) + .assert() + .success(); + }) + .await + .unwrap(); + + let reqs = server.received_requests().await.unwrap(); + let posts: Vec = reqs + .iter() + .filter(|r| r.method.as_str() == "POST") + .map(|r| String::from_utf8_lossy(&r.body).into_owned()) + .collect(); + assert!(posts.iter().any(|p| p.contains(FIXTURE_BUILD_ID))); + assert!(posts.iter().any(|p| p.contains(&other_id))); + let puts = reqs.iter().filter(|r| r.method.as_str() == "PUT").count(); + assert_eq!(puts, 2); +} + +/// An unreadable/nonexistent directory root must not exit 0. +#[cfg(unix)] +#[test] +fn elf_upload_fails_on_an_unreadable_directory() { + use std::os::unix::fs::PermissionsExt; + let tmp = tempfile::tempdir().unwrap(); + let locked = tmp.path().join("locked"); + std::fs::create_dir_all(&locked).unwrap(); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap(); + // Root bypasses permission checks; nothing to assert then. + let readable = std::fs::read_dir(&locked).is_ok(); + let inputs = vec![locked.to_string_lossy().into_owned()]; + let assert = common::cli() + .args(elf_args("http://127.0.0.1:1", &inputs)) + .assert(); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap(); + if !readable { + assert.failure(); + } +} From 7ace775f9131a51fb48ead06464f1b0b91ab8142 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 15:28:44 +0500 Subject: [PATCH 02/18] fix(debug-files): empty ELF directory exits 10; don't descend dir symlinks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review on #70: a directory with no matching libraries is a miswired path, not success (exit 10; empty zip still warns). The walk no longer follows directory symlinks (file symlinks are still read). Add changelog entry and tests. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 7 +++++++ src/cli/debug_files.rs | 10 ++++++++++ src/symbols/elf.rs | 9 ++++++--- tests/elf_upload.rs | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68f73ab..1a585bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- **`debug-files upload --type elf` accepts directories.** One or more directories (and/or + `native-debug-symbols.zip` files) are scanned; directories are walked recursively for `.so` / + `.so.dbg` / `.so.sym` (+ `--extension`) and read in place, so a Gradle build can point it at + `merged_native_libs/` without re-zipping. One upload per GNU build-id across all + paths. A directory with no matching libraries exits 10; an empty zip still only warns. See #69. + ## [0.7.13] - 2026-10-07 ### Added diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index 3fc10fd..dacb029 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -1202,6 +1202,16 @@ async fn run_elf_upload( )) } })?; + if found.is_empty() && input.is_dir() { + // A directory is the build's own output: empty usually means a + // miswired path or a task that ran before the libraries were merged, + // and exit 0 would let the build go green with no native symbols. + // (An empty ZIP stays a warning, as it always was.) + return Err(input_not_found(format!( + "no .so / .so.dbg / .so.sym files (or --extension suffix) under {}", + input.display() + ))); + } if found.is_empty() { // Nothing even matched by name β€” the one case `--extension` fixes. // Ahead of the dry-run exit: a dry run is how a caller checks this. diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index 0b05c70..5e9b809 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -140,13 +140,13 @@ pub fn collect_libs( /// Walk `dir` recursively for native libraries (same name rules as the zip /// entries), reading each one's identity in place. `name` is the path relative -/// to `dir`. Symlinks are followed (loops are skipped); unreadable entries are +/// to `dir`. Directory symlinks are not descended, file symlinks are read; unreadable entries are /// skipped with a warning rather than failing the whole scan. Sorted by path so /// the "first entry wins" tie-break in [`keep_richest_per_build_id`] is stable. pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result> { let mut libs = Vec::new(); for entry in walkdir::WalkDir::new(dir) - .follow_links(true) + .follow_links(false) .sort_by_file_name() { let entry = match entry { @@ -159,7 +159,10 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result Date: Wed, 7 Oct 2026 16:04:41 +0500 Subject: [PATCH 03/18] fix(debug-files): fail ELF dir scan on any I/O error; harden tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address adversarial review on #70: - any I/O error while scanning a directory (unreadable subdir/library, dangling link) now fails with exit 11 instead of silently uploading a partial set - the symlink test used same-build-id copies so dedup masked it; the escaped library now has its own id (verified to fail if links are followed) - pin exit 11 for unreadable inputs; hints for a zip-in-directory and a bare .so; NotFound only maps to exit 10 when the input itself is missing - e2e harness gets a directory flow; changelog/help/README updated; remove a leftover block scope πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 10 ++- README.md | 2 +- scripts/e2e_flows.py | 17 +++++ src/cli/debug_files.rs | 138 +++++++++++++++++++++++------------------ src/symbols/elf.rs | 30 +++------ tests/elf_upload.rs | 108 +++++++++++++++++++++++++++----- 6 files changed, 207 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a585bd..a517bc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). `native-debug-symbols.zip` files) are scanned; directories are walked recursively for `.so` / `.so.dbg` / `.so.sym` (+ `--extension`) and read in place, so a Gradle build can point it at `merged_native_libs/` without re-zipping. One upload per GNU build-id across all - paths. A directory with no matching libraries exits 10; an empty zip still only warns. See #69. + paths. A directory with no matching libraries exits 10; an empty zip still only warns. Directory + symlinks are not descended (file symlinks are read), and any I/O error while scanning (an + unreadable subdirectory or library, a dangling link) fails with exit 11 rather than uploading a + partial set. See #69. + +### Changed +- **`--type elf` collects every input before uploading anything.** A corrupt second zip now aborts + (exit 11) before the first zip uploads, instead of after. A path that does not exist now exits + 10 (input not found) rather than 11; both are in the no-fallback range. ## [0.7.13] - 2026-10-07 diff --git a/README.md b/README.md index 495165f..2c36897 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The upload flow itself. ProGuard, Rust, ELF, dSYM, PDB, sourcemap, and Unity IL2 #### Android NDK β€” `--type elf` -Takes one or more paths: AGP's pre-built `native-debug-symbols.zip`, and/or directories, which are walked recursively for libraries (e.g. `build/intermediates/merged_native_libs/`) and read in place β€” nothing is re-zipped. Every library found is uploaded as its own symbol, keyed by its GNU build-id β€” `--uuid` (the SDK's `BUILD_UUID`) is still required but only correlates logs. Built-in entry names are `.so`, `.so.dbg` (`ndk.debugSymbolLevel = 'FULL'`) and `.so.sym` (`'SYMBOL_TABLE'`, function names only β€” no `file:line`). A library without a build-id is warned about and skipped; two entries with the same build-id upload once (across all the given paths), preferring the one with DWARF. The server dedups on the build-id, so switching a library from `SYMBOL_TABLE` to `FULL` needs `--force` to replace the stored symbols. +Takes one or more paths: AGP's pre-built `native-debug-symbols.zip`, and/or directories, which are walked recursively for libraries (e.g. `build/intermediates/merged_native_libs/`) and read in place β€” nothing is re-zipped. Directory symlinks are not descended; an unreadable entry fails the run (exit 11) and a directory with no libraries exits 10. Every library found is uploaded as its own symbol, keyed by its GNU build-id β€” `--uuid` (the SDK's `BUILD_UUID`) is still required but only correlates logs. Built-in entry names are `.so`, `.so.dbg` (`ndk.debugSymbolLevel = 'FULL'`) and `.so.sym` (`'SYMBOL_TABLE'`, function names only β€” no `file:line`). A library without a build-id is warned about and skipped; two entries with the same build-id upload once (across all the given paths), preferring the one with DWARF. The server dedups on the build-id, so switching a library from `SYMBOL_TABLE` to `FULL` needs `--force` to replace the stored symbols. #### Unity IL2CPP β€” `--type il2cpp-linemap` diff --git a/scripts/e2e_flows.py b/scripts/e2e_flows.py index 2e586e0..4cd6e83 100644 --- a/scripts/e2e_flows.py +++ b/scripts/e2e_flows.py @@ -241,6 +241,12 @@ def make_fixtures(fix): import zipfile as zf with zf.ZipFile(os.path.join(fix, "native-debug-symbols.zip"), "w") as z: z.writestr("arm64-v8a/libfoo.so", b"\x7fELF" + b"\x02\x01\x01\x00" + b"\x00" * 256) + # AGP's merged_native_libs layout: a directory of unstripped libraries (a REAL + # ELF with a GNU build-id, so the flow registers + uploads one symbol). + libdir = os.path.join(fix, "native_libs", "lib", "arm64-v8a") + os.makedirs(libdir, exist_ok=True) + shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", + "tests", "fixtures", "elf", "libsymbol1.so"), libdir) # A Rust MSVC build drops the PDB into target// next to unrelated # files, so the flow points at a directory, not the file. tgt = os.path.join(fix, "target", "release") @@ -488,6 +494,17 @@ def main(): results["elf"] = run(binpath, "elf", ["debug-files", "upload", "--type", "elf", os.path.join(fix, "native-debug-symbols.zip"), "--uuid", "11111111-2222-3333-4444-555555555555"] + v) + results["elf_dir"] = run(binpath, "elf_dir", ["debug-files", "upload", "--type", "elf", + os.path.join(fix, "native_libs"), + "--uuid", "11111111-2222-3333-4444-555555555555"] + v) + try: + posted = json.load(open(cappath("elf_dir__symbols_post.json"))) + ok = (posted.get("uuid") == "bca64abfec40dbb631bb8f1c37414472" + and STATE["puts"].get("elf_dir") == 1) + except Exception as e: + print(" [FAIL] could not verify the elf_dir flow:", e) + ok = False + results["elf_dir_keyed_by_build_id"] = ok results["pdb"] = run(binpath, "pdb", ["debug-files", "upload", "--type", "pdb", os.path.join(fix, "target", "release")] + v) if os.path.isdir(os.path.join(fix, "App.dSYM")): diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index dacb029..f5ebf75 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -19,7 +19,9 @@ pub enum DebugFilesCommand { Upload { /// One or more directories or files to scan. For `--type elf`, a directory /// is walked recursively for native libraries (e.g. AGP's - /// `merged_native_libs`), and a file is a `native-debug-symbols.zip`. + /// `merged_native_libs`; directory symlinks are not descended, an I/O error + /// fails the run, and no match exits 10), and a file is a + /// `native-debug-symbols.zip`. #[arg(required = true)] paths: Vec, @@ -1190,14 +1192,19 @@ async fn run_elf_upload( let input_dir = work_dir.path().join(n.to_string()); std::fs::create_dir_all(&input_dir)?; let found = elf::collect_libs(input, &input_dir, extra).map_err(|e| { - if e.kind() == std::io::ErrorKind::NotFound { + if e.kind() == std::io::ErrorKind::NotFound && !input.exists() { input_not_found(format!("{} does not exist", input.display())) } else if input.is_dir() { input_invalid(format!("cannot scan {}: {e}", input.display())) } else { + let hint = if input.extension().is_some_and(|x| x == "so") { + " (a single library is not accepted: pass its directory)" + } else { + "" + }; input_invalid(format!( "--type elf expects a directory or a native-debug-symbols zip; \ - cannot read {}: {e}", + cannot read {}: {e}{hint}", input.display() )) } @@ -1207,8 +1214,16 @@ async fn run_elf_upload( // miswired path or a task that ran before the libraries were merged, // and exit 0 would let the build go green with no native symbols. // (An empty ZIP stays a warning, as it always was.) + let has_zip = std::fs::read_dir(input).is_ok_and(|mut d| { + d.any(|e| e.is_ok_and(|e| e.path().extension().is_some_and(|x| x == "zip"))) + }); + let hint = if has_zip { + " (it contains a .zip: zips inside a directory are not opened; pass the zip itself)" + } else { + "" + }; return Err(input_not_found(format!( - "no .so / .so.dbg / .so.sym files (or --extension suffix) under {}", + "no .so / .so.dbg / .so.sym files (or --extension suffix) under {}{hint}", input.display() ))); } @@ -1224,68 +1239,67 @@ async fn run_elf_upload( } libs.extend(found); } - { - let libs = elf::keep_richest_per_build_id(libs); - let total = libs.len(); - - // A `.so` with no GNU build-id can never be matched at crash time, so - // warn + skip rather than fake an identity (never the build UUID). - let mut uploadable: Vec = Vec::new(); - for lib in libs { - if lib.build_id.is_some() { - uploadable.push(lib); - } else { - skipped_no_build_id += 1; - tracing::warn!( - lib = %lib.name, - arch = %lib.arch, - "native library has no GNU build-id; skipping β€” it cannot be \ - symbolicated. Build the library with -Wl,--build-id." - ); - } + + let libs = elf::keep_richest_per_build_id(libs); + let total = libs.len(); + + // A `.so` with no GNU build-id can never be matched at crash time, so + // warn + skip rather than fake an identity (never the build UUID). + let mut uploadable: Vec = Vec::new(); + for lib in libs { + if lib.build_id.is_some() { + uploadable.push(lib); + } else { + skipped_no_build_id += 1; + tracing::warn!( + lib = %lib.name, + arch = %lib.arch, + "native library has no GNU build-id; skipping β€” it cannot be \ + symbolicated. Build the library with -Wl,--build-id." + ); } + } + tracing::info!( + libraries = total, + uploadable = uploadable.len(), + "collected native libraries" + ); + + if total == 0 { + // Already warned per input above. + } else if dry_run { tracing::info!( - libraries = total, - uploadable = uploadable.len(), - "collected native libraries" + "dry-run: would register + upload {} libraries", + uploadable.len() ); + } else if uploadable.is_empty() { + tracing::warn!("no native libraries with a GNU build-id β€” nothing to upload"); + } else { + let client = client.as_ref().expect("client constructed when !dry_run"); - if total == 0 { - // Already warned per input above. - } else if dry_run { - tracing::info!( - "dry-run: would register + upload {} libraries", - uploadable.len() - ); - } else if uploadable.is_empty() { - tracing::warn!("no native libraries with a GNU build-id β€” nothing to upload"); - } else { - let client = client.as_ref().expect("client constructed when !dry_run"); - - // Run each `.so`'s pack β†’ register β†’ dedup-or-PUT pipeline concurrently. - let outcomes: Vec<(bool, anyhow::Result)> = - futures_util::stream::iter(uploadable.into_iter().enumerate()) - .map(|(i, lib)| { - let client = client.clone(); - let work = work_dir.path().to_path_buf(); - async move { - let symbol_table_only = lib.name.ends_with(".so.sym"); - let outcome = upload_one_so(&client, ctx, &lib, i, &work).await; - (symbol_table_only, outcome) - } - }) - .buffer_unordered(ELF_UPLOAD_CONCURRENCY) - .collect() - .await; - - for (symbol_table_only, outcome) in outcomes { - match outcome? { - presigned::Outcome::Uploaded => uploaded += 1, - presigned::Outcome::AlreadyExists => { - already_existed += 1; - if !symbol_table_only { - full_already_existed += 1; - } + // Run each `.so`'s pack β†’ register β†’ dedup-or-PUT pipeline concurrently. + let outcomes: Vec<(bool, anyhow::Result)> = + futures_util::stream::iter(uploadable.into_iter().enumerate()) + .map(|(i, lib)| { + let client = client.clone(); + let work = work_dir.path().to_path_buf(); + async move { + let symbol_table_only = lib.name.ends_with(".so.sym"); + let outcome = upload_one_so(&client, ctx, &lib, i, &work).await; + (symbol_table_only, outcome) + } + }) + .buffer_unordered(ELF_UPLOAD_CONCURRENCY) + .collect() + .await; + + for (symbol_table_only, outcome) in outcomes { + match outcome? { + presigned::Outcome::Uploaded => uploaded += 1, + presigned::Outcome::AlreadyExists => { + already_existed += 1; + if !symbol_table_only { + full_already_existed += 1; } } } diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index 5e9b809..57a3dec 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -140,25 +140,19 @@ pub fn collect_libs( /// Walk `dir` recursively for native libraries (same name rules as the zip /// entries), reading each one's identity in place. `name` is the path relative -/// to `dir`. Directory symlinks are not descended, file symlinks are read; unreadable entries are -/// skipped with a warning rather than failing the whole scan. Sorted by path so -/// the "first entry wins" tie-break in [`keep_richest_per_build_id`] is stable. +/// to `dir`. Directory symlinks are not descended, file symlinks are read. Any +/// I/O error (unreadable subdirectory, unreadable library, dangling link) fails +/// the scan: skipping would upload a partial set (a whole ABI missing) and let +/// the build go green, the same way the zip path fails on a read error. Sorted +/// by path so the "first entry wins" tie-break in [`keep_richest_per_build_id`] +/// is stable. pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result> { let mut libs = Vec::new(); for entry in walkdir::WalkDir::new(dir) .follow_links(false) .sort_by_file_name() { - let entry = match entry { - Ok(e) => e, - // An unreadable root means nothing was scanned: fail, never exit 0 with - // a misleading "pass --extension" hint. - Err(e) if e.depth() == 0 => return Err(e.into()), - Err(e) => { - tracing::warn!(error = %e, "skipping unreadable path while scanning for native libraries"); - continue; - } - }; + let entry = entry?; // Directory symlinks are never descended (a walk must stay under the // root); a symlink to a FILE is a candidate and `read` follows it. let ft = entry.file_type(); @@ -169,13 +163,9 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result b, - Err(e) => { - tracing::warn!(path = %entry.path().display(), error = %e, "skipping unreadable native library"); - continue; - } - }; + let bytes = std::fs::read(entry.path()).map_err(|e| { + std::io::Error::new(e.kind(), format!("{}: {e}", entry.path().display())) + })?; let ElfIdentity { build_id, arch, diff --git a/tests/elf_upload.rs b/tests/elf_upload.rs index 54a1628..bbe7954 100644 --- a/tests/elf_upload.rs +++ b/tests/elf_upload.rs @@ -486,24 +486,87 @@ async fn elf_upload_keeps_same_named_entries_of_different_zips_apart() { } /// An unreadable/nonexistent directory root must not exit 0. +/// Any I/O error while scanning a directory fails the run (exit 11): skipping +/// would upload a partial set β€” a whole ABI missing β€” behind a green build. #[cfg(unix)] #[test] -fn elf_upload_fails_on_an_unreadable_directory() { +fn elf_upload_fails_on_unreadable_directory_content() { use std::os::unix::fs::PermissionsExt; + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"); + let lock = |p: &Path| { + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o000)).unwrap(); + // Root bypasses permission checks; nothing to assert then. + std::fs::read_dir(p).is_err() && std::fs::read(p).is_err() + }; + let unlock = |p: &Path| { + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)).unwrap(); + }; + let run = |root: &Path| { + let inputs = vec![root.to_string_lossy().into_owned()]; + common::cli() + .args(elf_args("http://127.0.0.1:1", &inputs)) + .assert() + .code(11) + }; + + // unreadable root let tmp = tempfile::tempdir().unwrap(); - let locked = tmp.path().join("locked"); - std::fs::create_dir_all(&locked).unwrap(); - std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap(); - // Root bypasses permission checks; nothing to assert then. - let readable = std::fs::read_dir(&locked).is_ok(); - let inputs = vec![locked.to_string_lossy().into_owned()]; - let assert = common::cli() - .args(elf_args("http://127.0.0.1:1", &inputs)) - .assert(); - std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap(); - if !readable { - assert.failure(); + let root = tmp.path().join("locked"); + std::fs::create_dir_all(&root).unwrap(); + if lock(&root) { + run(&root); } + unlock(&root); + + // a good library plus an unreadable ABI subdirectory + let root = tmp.path().join("r2"); + std::fs::create_dir_all(root.join("x86_64")).unwrap(); + std::fs::copy(&fixture, root.join("libok.so")).unwrap(); + if lock(&root.join("x86_64")) { + run(&root); + } + unlock(&root.join("x86_64")); + + // a good library plus an unreadable library file + let root = tmp.path().join("r3"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::copy(&fixture, root.join("libok.so")).unwrap(); + std::fs::copy(&fixture, root.join("libbad.so")).unwrap(); + if lock(&root.join("libbad.so")) { + run(&root); + } + unlock(&root.join("libbad.so")); + + // a dangling symlink named like a library + let root = tmp.path().join("r4"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::copy(&fixture, root.join("libok.so")).unwrap(); + std::os::unix::fs::symlink(tmp.path().join("gone"), root.join("libdangling.so")).unwrap(); + run(&root); +} + +/// A directory holding only a zip, and a bare `.so`, get pointed at the right input. +#[test] +fn elf_upload_hints_for_a_zip_in_a_directory_and_a_bare_so() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("native-debug-symbols"); + std::fs::create_dir_all(&dir).unwrap(); + pack_native_zip(&dir, "arm64-v8a/libsymbol1.so"); + let inputs = vec![dir.to_string_lossy().into_owned()]; + common::cli() + .args(elf_args("http://127.0.0.1:1", &inputs)) + .assert() + .code(10) + .stderr(predicates::str::contains("pass the zip itself")); + + let so = tmp.path().join("libx.so"); + std::fs::write(&so, b"x").unwrap(); + let inputs = vec![so.to_string_lossy().into_owned()]; + common::cli() + .args(elf_args("http://127.0.0.1:1", &inputs)) + .assert() + .code(11) + .stderr(predicates::str::contains("pass its directory")); } /// A directory that exists but holds no libraries is a miswired path / too-early @@ -519,7 +582,20 @@ fn elf_upload_empty_directory_is_input_not_found() { .stderr(predicates::str::contains("no .so")); } +/// The fixture with its GNU build-id's first byte flipped: same ELF, different identity. +fn fixture_with_other_build_id() -> (Vec, String) { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"); + let mut bytes = std::fs::read(&fixture).unwrap(); + let id = hex::decode(FIXTURE_BUILD_ID).unwrap(); + let pos = bytes.windows(id.len()).position(|w| w == id).unwrap(); + bytes[pos] ^= 0xff; + let other = hex::encode(&bytes[pos..pos + id.len()]); + (bytes, other) +} + /// Directory symlinks are not descended; a symlink to a library file is read. +/// The escaped library has its OWN build-id, so following the link would make it +/// 2 libraries (same-id copies would be collapsed by dedup and prove nothing). #[cfg(unix)] #[test] fn elf_dry_run_does_not_descend_directory_symlinks_but_reads_file_symlinks() { @@ -527,7 +603,11 @@ fn elf_dry_run_does_not_descend_directory_symlinks_but_reads_file_symlinks() { let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"); let outside = tmp.path().join("outside"); std::fs::create_dir_all(&outside).unwrap(); - std::fs::copy(&fixture, outside.join("libescaped.so")).unwrap(); + std::fs::write( + outside.join("libescaped.so"), + fixture_with_other_build_id().0, + ) + .unwrap(); let root = tmp.path().join("root"); std::fs::create_dir_all(&root).unwrap(); std::os::unix::fs::symlink(&outside, root.join("linkdir")).unwrap(); From d7655c556ec00a8f1786c0bac1e0748ed8b3b36e Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 16:35:37 +0500 Subject: [PATCH 04/18] perf(upload): stream the presigned PUT body and the SHA-1 instead of reading whole files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit put_payload read the packed archive into memory and cloned it for every retry attempt, and sha1_hex_of_file read it whole again; with up to 6 uploads in flight that held ~3x the archive size each. Both now stream in 64 KiB chunks (the PUT keeps an explicit Content-Length, which a presigned S3 PUT requires, and re-opens the file per attempt). Peak RSS, 4 x 100 MB libraries: 564 MB -> 143 MB. Shared by every presigned upload (elf, dsym, pdb, rust). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- src/symbols/elf.rs | 42 +++++++++--- src/upload/presigned.rs | 142 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 173 insertions(+), 11 deletions(-) diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index 57a3dec..ca6e630 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -32,14 +32,21 @@ use symbolic_debuginfo::Archive; use super::suffix::ExtraSuffixes; -fn sha1_hex(bytes: &[u8]) -> String { - let digest: [u8; 20] = Sha1::digest(bytes).into(); - hex::encode(digest) -} - /// SHA-1 hex of a file's bytes β€” the wire `hash` for a per-`.so` upload. pub fn sha1_hex_of_file(path: &Path) -> std::io::Result { - Ok(sha1_hex(&std::fs::read(path)?)) + // Streamed: the hashed file is the packed archive, which can be tens of MB. + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha1::new(); + let mut buf = [0u8; 64 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + let digest: [u8; 20] = hasher.finalize().into(); + Ok(hex::encode(digest)) } /// One native library found in an AGP `native-debug-symbols.zip` or a scanned directory. @@ -163,9 +170,10 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result = (0..300_001u32).map(|i| (i % 251) as u8).collect(); + std::fs::write(&path, &data).unwrap(); + let expected: [u8; 20] = Sha1::digest(&data).into(); + assert_eq!(sha1_hex_of_file(&path).unwrap(), hex::encode(expected)); + std::fs::write(&path, b"").unwrap(); + assert_eq!( + sha1_hex_of_file(&path).unwrap(), + "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "SHA-1 of the empty input" + ); + } + #[test] fn parse_elf_identity_on_non_elf_returns_none_unknown() { let id = parse_elf_identity(b"this is not an ELF file"); diff --git a/src/upload/presigned.rs b/src/upload/presigned.rs index 848dcb2..5a2c119 100644 --- a/src/upload/presigned.rs +++ b/src/upload/presigned.rs @@ -228,11 +228,20 @@ pub async fn put_payload( payload: &Path, ) -> Result<()> { tracing::debug!(url = %http::redact_url(presigned_url), "PUT payload"); - let payload_bytes = tokio::fs::read(payload).await?; + // Stream the payload from disk: reading it whole (and cloning it per retry + // attempt) held two-plus copies of the archive in memory per upload. The + // length is sent explicitly because a streamed body would otherwise go out + // chunked, which a presigned S3 PUT rejects. + let len = tokio::fs::metadata(payload).await?.len(); // The PUT is idempotent (overwrite of the same key) β€” retry on transport - // AND retriable status. + // AND retriable status. Each attempt re-opens the file from the start. let put_resp = http::send_with_retry(policy, "symbol PUT", true, || { - client.put(presigned_url).body(payload_bytes.clone()) + client + .put(presigned_url) + .header(reqwest::header::CONTENT_LENGTH, len) + .body(reqwest::Body::wrap_stream(file_chunks( + payload.to_path_buf(), + ))) }) .await?; @@ -248,6 +257,38 @@ pub async fn put_payload( Ok(()) } +/// The file's bytes in 64 KiB chunks. The file is opened lazily on first poll so +/// an open error surfaces as a failed (retriable) request, not a panic. +fn file_chunks( + path: std::path::PathBuf, +) -> impl futures_util::Stream>> + Send + 'static { + use tokio::io::AsyncReadExt; + enum St { + Unopened(std::path::PathBuf), + Open(tokio::fs::File), + Done, + } + futures_util::stream::unfold(St::Unopened(path), |st| async move { + let mut file = match st { + St::Unopened(p) => match tokio::fs::File::open(p).await { + Ok(f) => f, + Err(e) => return Some((Err(e), St::Done)), + }, + St::Open(f) => f, + St::Done => return None, + }; + let mut buf = vec![0u8; 64 * 1024]; + match file.read(&mut buf).await { + Ok(0) => None, + Ok(n) => { + buf.truncate(n); + Some((Ok(buf), St::Open(file))) + } + Err(e) => Some((Err(e), St::Done)), + } + }) +} + /// Run the two-stage presigned upload for a single symbol artifact /// ([`register`] then [`put_payload`]). The payload is always produced up /// front; callers that want to skip producing it when the server already has @@ -535,6 +576,101 @@ mod tests { assert!(matches!(reg, Registration::AlreadyExists), "got {reg:?}"); } + /// A payload bigger than the 64 KiB stream chunk and not a multiple of it. + fn patterned_payload(len: usize) -> (tempfile::NamedTempFile, Vec) { + use std::io::Write; + let data: Vec = (0..len).map(|i| (i % 251) as u8).collect(); + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(&data).unwrap(); + (f, data) + } + + /// The PUT streams the file but must still go out with an exact + /// `Content-Length` (a presigned S3 PUT rejects a chunked body) and every byte. + #[tokio::test] + async fn put_payload_streams_the_whole_file_with_an_exact_content_length() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/put")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + let (payload, data) = patterned_payload(300_001); + let client = http::build_client().unwrap(); + put_payload( + &client, + RetryPolicy::none(), + &format!("{}/put", server.uri()), + payload.path(), + ) + .await + .unwrap(); + + let reqs = server.received_requests().await.unwrap(); + assert_eq!(reqs.len(), 1); + assert_eq!( + reqs[0] + .headers + .get("content-length") + .map(|v| v.to_str().unwrap().to_owned()), + Some(data.len().to_string()) + ); + assert!( + reqs[0].headers.get("transfer-encoding").is_none(), + "must not be chunked" + ); + assert_eq!(reqs[0].body, data); + } + + /// Each retry re-opens the file: the second attempt must carry the full body, + /// not whatever a shared cursor had left over. + #[tokio::test] + async fn put_payload_retry_resends_the_complete_body() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (payload, data) = patterned_payload(200_000); + let client = http::build_client().unwrap(); + put_payload( + &client, + RetryPolicy::fast(3), + &format!("{}/put", server.uri()), + payload.path(), + ) + .await + .unwrap(); + + let reqs = server.received_requests().await.unwrap(); + assert_eq!(reqs.len(), 2, "one 503 then one success"); + for r in &reqs { + assert_eq!(r.body, data, "every attempt sends the whole payload"); + } + } + + /// A payload that vanished is a reported error, not a panic or a hang. + #[tokio::test] + async fn put_payload_reports_a_missing_payload_file() { + let server = MockServer::start().await; + let client = http::build_client().unwrap(); + let err = put_payload( + &client, + RetryPolicy::none(), + &format!("{}/put", server.uri()), + Path::new("/nonexistent/payload.zip"), + ) + .await + .unwrap_err(); + assert!(!err.to_string().is_empty()); + } + #[tokio::test] async fn upload_skips_the_put_when_the_server_reports_the_nested_duplicate() { let server = MockServer::start().await; From 20a7eb35b45a765810aa2d37b560b591d820009c Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 16:35:51 +0500 Subject: [PATCH 05/18] perf(elf): memory-map libraries for the identity scan instead of reading them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scan_dir read every library whole just to extract its build-id, arch and richness, all of which live in the headers/notes. Map the file read-only instead (memmap2 was already in the tree via symbolic-debuginfo): the largest-file transient disappears. An empty file, which cannot be mapped, is reported as having no build-id rather than failing the scan. Peak RSS, 4 x 100 MB libraries: 143 MB -> 36 MB; --dry-run 112 MB -> 5 MB. Separate commit because it is the only step needing `unsafe`. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- Cargo.lock | 1 + Cargo.toml | 1 + src/symbols/elf.rs | 31 ++++++++++++++++++++++++++++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bb9b0a..8b2ff33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,6 +215,7 @@ dependencies = [ "hex", "libc", "md-5 0.11.0", + "memmap2", "plist", "predicates", "regex", diff --git a/Cargo.toml b/Cargo.toml index c5d7a7d..fc15b36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,6 +153,7 @@ sha1 = "0.11" sha2 = "0.11" md-5 = "0.11" hex = "0.4" +memmap2 = "0.9" # File discovery walkdir = "2" diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index ca6e630..2c3a316 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -173,13 +173,27 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result std::io::Result Date: Wed, 7 Oct 2026 16:36:05 +0500 Subject: [PATCH 06/18] docs(changelog): note the streaming/mmap memory reduction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a517bc0..3eea103 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - **`--type elf` collects every input before uploading anything.** A corrupt second zip now aborts (exit 11) before the first zip uploads, instead of after. A path that does not exist now exits 10 (input not found) rather than 11; both are in the no-fallback range. +- **Presigned symbol uploads no longer hold the whole archive in memory.** The PUT body and the + SHA-1 are streamed from disk in 64 KiB chunks (the PUT keeps an exact `Content-Length` and + re-opens the file per retry), and the ELF identity scan memory-maps libraries instead of reading + them. Peak RSS for four 100 MB libraries dropped from 564 MB to 36 MB. Wire format unchanged. ## [0.7.13] - 2026-10-07 From 7b454ac55836488c3feb8db556d9dcedbcda2519 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 16:41:19 +0500 Subject: [PATCH 07/18] perf: stop loading whole files into memory across the upload/identify paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit of every std::fs::read / read_to_string / tokio::fs::read outside tests: - upload build artefact PUT (IPA/APK/AAB, hundreds of MB) and the build-info PUT now stream from disk via a shared http::file_body (+ exact Content-Length), instead of reading whole and cloning per retry - ProGuard identify: MD5 + SHA-1 in one streamed pass (was whole-file read) - dSYM / PDB / Rust ELF / `xcode` IPA Mach-O / `dsym` identity reads use a shared symbols::mapped::map_file (the single unsafe, empty-file safe) instead of copying 100+ MB binaries to read a UUID from the headers - ELF zip input: stream entries to disk, then map for identity - source maps: streamed SHA-1/size; read_debug_id maps the file (non-UTF-8 still an I/O error, exit 10) Peak RSS: upload build (300 MB artefact + 200 MB mapping) 661 -> 32 MB; 200 MB ProGuard mapping 241 -> 32 MB; ELF zip of 4 x 100 MB 463 -> 36 MB; 89 MB source map 274 -> 183 MB (JSON tree remains). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 8 +++++ src/cli/debug_files.rs | 2 +- src/cli/dsym.rs | 2 +- src/cli/xcode_ipa.rs | 2 +- src/inject/mod.rs | 9 +++-- src/symbols/dsym.rs | 2 +- src/symbols/elf.rs | 49 ++++++++++++-------------- src/symbols/mapped.rs | 52 ++++++++++++++++++++++++++++ src/symbols/mod.rs | 1 + src/symbols/pdb.rs | 2 +- src/symbols/proguard.rs | 51 +++++++++++++++++++++++---- src/symbols/rust.rs | 2 +- src/symbols/sourcemap.rs | 55 ++++++++++++++++++++++++----- src/upload/build.rs | 74 ++++++++++++++++++++++++++++++++++++++-- src/upload/build_info.rs | 10 +++--- src/upload/http.rs | 46 +++++++++++++++++++++++++ src/upload/presigned.rs | 38 ++------------------- 17 files changed, 312 insertions(+), 93 deletions(-) create mode 100644 src/symbols/mapped.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eea103..3a1065e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,14 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). SHA-1 are streamed from disk in 64 KiB chunks (the PUT keeps an exact `Content-Length` and re-opens the file per retry), and the ELF identity scan memory-maps libraries instead of reading them. Peak RSS for four 100 MB libraries dropped from 564 MB to 36 MB. Wire format unchanged. +- **The rest of the whole-file reads are gone too.** The build artefact PUT (`upload build`, an + IPA/APK/AAB) and the build-info PUT stream from disk; ProGuard identification hashes in one + streamed pass; dSYM, PDB, Rust and `xcode` identity reads memory-map the file instead of copying + it; zip-input extraction streams entries to disk; source-map identification streams its hash. + Peak RSS: `upload build` with a 300 MB artefact + 200 MB mapping 661 MB -> 32 MB, a 200 MB + ProGuard mapping 241 MB -> 32 MB, an ELF zip of four 100 MB libraries 463 MB -> 36 MB. Wire + format and exit codes unchanged. + ## [0.7.13] - 2026-10-07 diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index f5ebf75..b1d91a2 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -1952,7 +1952,7 @@ fn stripped_copy_without_sources( // Typed, not a bare `?` into anyhow: an I/O failure on the map must classify the way // `sourcemap::identify` classifies one on the SAME file moments earlier β€” exit 10, which // integrators are documented not to fall back on β€” rather than exit 1 ("unexpected"). - let bytes = std::fs::read(map_path).map_err(crate::error::Error::Io)?; + let bytes = crate::symbols::mapped::map_file(map_path).map_err(crate::error::Error::Io)?; let Ok(serde_json::Value::Object(mut map)) = serde_json::from_slice(&bytes) else { return Ok(None); }; diff --git a/src/cli/dsym.rs b/src/cli/dsym.rs index 336fc33..7152667 100644 --- a/src/cli/dsym.rs +++ b/src/cli/dsym.rs @@ -155,7 +155,7 @@ fn push_slices_from_macho(path: &Path, out: &mut Vec) { return; } } - let data = match std::fs::read(path) { + let data = match crate::symbols::mapped::map_file(path) { Ok(d) => d, Err(_) => return, }; diff --git a/src/cli/xcode_ipa.rs b/src/cli/xcode_ipa.rs index 2d004b9..435a5a5 100644 --- a/src/cli/xcode_ipa.rs +++ b/src/cli/xcode_ipa.rs @@ -232,7 +232,7 @@ pub fn main_executable_uuid(app_path: &Path) -> Option { return None; } - let data = std::fs::read(&binary_path).ok()?; + let data = crate::symbols::mapped::map_file(&binary_path).ok()?; let archive = Archive::parse(&data).ok()?; let mut slices: Vec<(String, Uuid)> = Vec::new(); for obj in archive.objects() { diff --git a/src/inject/mod.rs b/src/inject/mod.rs index 1486ad6..1cd40c9 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -564,8 +564,13 @@ fn write_map_debug_id( /// / legacy `uuid`, in that precedence. Mirrors the worker's /// `symbolfiles/sourcemap.py:parse`. pub fn read_debug_id(map_path: &Path) -> Result> { - let raw = std::fs::read_to_string(map_path)?; - let value: serde_json::Value = serde_json::from_str(&raw).map_err(|e| { + // Mapped rather than read into a String: the JSON tree below is the only + // copy this needs. UTF-8 is checked up front so a non-UTF-8 map still fails + // as an I/O `InvalidData` error, exactly as `read_to_string` did. + let bytes = crate::symbols::mapped::map_file(map_path)?; + let raw = std::str::from_utf8(&bytes) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| { Error::InputInvalid(format!( "sourcemap is not valid JSON: {e} ({})", map_path.display() diff --git a/src/symbols/dsym.rs b/src/symbols/dsym.rs index f2abf5d..3f073f3 100644 --- a/src/symbols/dsym.rs +++ b/src/symbols/dsym.rs @@ -79,7 +79,7 @@ pub fn identify(dsym_path: &Path) -> Result { continue; } let bin_path = entry.path(); - let data = std::fs::read(&bin_path)?; + let data = super::mapped::map_file(&bin_path)?; let archive = Archive::parse(&data).map_err(|e| { Error::InputInvalid(format!( "failed to parse Mach-O at {}: {}", diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index 2c3a316..f4fd182 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -101,23 +101,29 @@ pub fn extract_libs( continue; } - let mut bytes = Vec::new(); - entry.read_to_end(&mut bytes)?; - let ElfIdentity { - build_id, - arch, - has_debug_info, - has_symbols, - } = parse_elf_identity(&bytes); - let richness = (has_debug_info, has_symbols, bytes.len() as u64); - let base = Path::new(&name) .file_name() .and_then(|n| n.to_str()) .unwrap_or("lib.so"); // Entries across ABIs can share a basename β€” prefix with the index. let out_path = out_dir.join(format!("{i}_{base}")); - std::fs::write(&out_path, &bytes)?; + // Stream the entry to disk, then read its identity from the file: an + // inflated unstripped library is 100+ MB and was held whole in memory. + { + let mut out = std::fs::File::create(&out_path)?; + std::io::copy(&mut entry, &mut out)?; + } + let ElfIdentity { + build_id, + arch, + has_debug_info, + has_symbols, + } = parse_elf_identity(&super::mapped::map_file(&out_path)?); + let richness = ( + has_debug_info, + has_symbols, + std::fs::metadata(&out_path)?.len(), + ); libs.push(ElfLib { name, @@ -173,27 +179,16 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result); + +impl Deref for Mapped { + type Target = [u8]; + fn deref(&self) -> &[u8] { + self.0.as_deref().unwrap_or(&[]) + } +} + +/// Map `path` read-only. +/// +/// Keep the result scoped to the parse and drop it before the file is +/// rewritten or removed (Windows refuses to replace a mapped file). +pub fn map_file(path: &Path) -> std::io::Result { + let file = std::fs::File::open(path)?; + if file.metadata()?.len() == 0 { + return Ok(Mapped(None)); + } + // SAFETY: the map is read-only; the only hazard is another process + // truncating the file while it is being parsed, which the build output + // and symbol files scanned here do not do. + Ok(Mapped(Some(unsafe { memmap2::Mmap::map(&file) }?))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_file_contents_and_handles_an_empty_file() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("f"); + std::fs::write(&p, b"abc").unwrap(); + assert_eq!(&*map_file(&p).unwrap(), b"abc"); + std::fs::write(&p, b"").unwrap(); + assert!(map_file(&p).unwrap().is_empty()); + assert!(map_file(&dir.path().join("missing")).is_err()); + } +} diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index d30ef4c..a6919a5 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -26,6 +26,7 @@ pub mod dsym; pub mod elf; pub mod il2cpp_linemap; +pub mod mapped; pub mod pdb; pub mod proguard; pub mod rust; diff --git a/src/symbols/pdb.rs b/src/symbols/pdb.rs index 8f00f67..3afa6f3 100644 --- a/src/symbols/pdb.rs +++ b/src/symbols/pdb.rs @@ -81,7 +81,7 @@ pub fn identify(pdb_path: &Path) -> Result { ))); } - let data = std::fs::read(pdb_path)?; + let data = super::mapped::map_file(pdb_path)?; let archive = Archive::parse(&data).map_err(|e| { Error::InputInvalid(format!( "failed to parse PDB at {}: {}", diff --git a/src/symbols/proguard.rs b/src/symbols/proguard.rs index c9fb78b..67f3ff5 100644 --- a/src/symbols/proguard.rs +++ b/src/symbols/proguard.rs @@ -25,15 +25,33 @@ pub fn looks_like_mapping_filename(name: &str) -> bool { /// Read a file from disk and compute (debug-id, sha1 hex). /// -/// Streams the SHA1 chunk-at-a-time to keep memory flat for large mappings -/// (multi-megabyte mappings are common for big apps). MD5 still needs the -/// whole buffer because Java's `nameUUIDFromBytes` operates on contiguous -/// bytes; mappings rarely exceed 50 MB so this is acceptable. +/// Both digests are computed in ONE streamed pass (64 KiB buffer), so memory +/// stays flat however large the mapping is: Java's `nameUUIDFromBytes` is just +/// an MD5 of the bytes with the version/variant bits set, and MD5 is +/// incremental like SHA-1. pub fn identify(path: &Path) -> std::io::Result { - let bytes = std::fs::read(path)?; - Ok(identify_bytes(&bytes)) + use std::io::Read as _; + let mut file = std::fs::File::open(path)?; + let mut md5 = Md5::new(); + let mut sha1 = Sha1::new(); + let mut buf = [0u8; 64 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + md5.update(&buf[..n]); + sha1.update(&buf[..n]); + } + let md5_bytes: [u8; 16] = md5.finalize().into(); + let sha1_bytes: [u8; 20] = sha1.finalize().into(); + Ok(MappingIdentity { + debug_id: Builder::from_md5_bytes(md5_bytes).into_uuid(), + content_sha1_hex: hex::encode(sha1_bytes), + }) } +#[cfg(test)] pub fn identify_bytes(bytes: &[u8]) -> MappingIdentity { MappingIdentity { debug_id: java_name_uuid_from_bytes(bytes), @@ -51,6 +69,7 @@ pub struct MappingIdentity { pub content_sha1_hex: String, } +#[cfg(test)] /// Replicates Java `UUID.nameUUIDFromBytes(name)`: /// 1. MD5 of input bytes. /// 2. Set the version nibble of byte 6 to 3. @@ -62,6 +81,7 @@ pub fn java_name_uuid_from_bytes(name: &[u8]) -> Uuid { Builder::from_md5_bytes(md5_bytes).into_uuid() } +#[cfg(test)] fn sha1_hex(bytes: &[u8]) -> String { let digest: [u8; 20] = Sha1::digest(bytes).into(); hex::encode(digest) @@ -88,6 +108,25 @@ mod tests { ); } + #[test] + fn streamed_identify_equals_in_memory_identify_across_buffer_boundaries() { + // Bigger than the 64 KiB buffer and not a multiple of it: a chunking bug + // would change either digest relative to the one-shot computation. + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("mapping.txt"); + let data: Vec = (0..300_001u32).map(|i| (i % 251) as u8).collect(); + std::fs::write(&p, &data).unwrap(); + let streamed = identify(&p).unwrap(); + let whole = identify_bytes(&data); + assert_eq!(streamed.debug_id, whole.debug_id); + assert_eq!(streamed.content_sha1_hex, whole.content_sha1_hex); + std::fs::write(&p, b"").unwrap(); + assert_eq!( + identify(&p).unwrap().debug_id.to_string(), + "d41d8cd9-8f00-3204-a980-0998ecf8427e" + ); + } + #[test] fn sha1_hex_matches_reference() { assert_eq!(sha1_hex(b""), "da39a3ee5e6b4b0d3255bfef95601890afd80709"); diff --git a/src/symbols/rust.rs b/src/symbols/rust.rs index 39f0c3a..d5e740a 100644 --- a/src/symbols/rust.rs +++ b/src/symbols/rust.rs @@ -153,7 +153,7 @@ fn is_bundle_like(p: &Path, extra: &ExtraSuffixes) -> bool { /// on β€” and to the `code_id` the Rust SDK reports for that module at crash /// time. All three agreeing is what makes symbolication resolve. fn parse_elf(path: &Path) -> Option<(Option, String, bool)> { - let bytes = std::fs::read(path).ok()?; + let bytes = super::mapped::map_file(path).ok()?; let archive = Archive::parse(&bytes).ok()?; let obj = archive.objects().next()?.ok()?; Some(( diff --git a/src/symbols/sourcemap.rs b/src/symbols/sourcemap.rs index 99e7885..5f3337d 100644 --- a/src/symbols/sourcemap.rs +++ b/src/symbols/sourcemap.rs @@ -17,12 +17,28 @@ use crate::error::Result; use crate::inject; /// Read a `.map` file and derive its upload identity. +/// +/// The hash is streamed (64 KiB buffer) and the size comes from the file +/// itself, so identifying a map no longer copies it onto the heap. pub fn identify(path: &Path) -> Result { - let bytes = std::fs::read(path)?; + use std::io::Read as _; + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha1::new(); + let mut buf = [0u8; 64 * 1024]; + let mut size_bytes = 0u64; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + size_bytes += n as u64; + } + let digest: [u8; 20] = hasher.finalize().into(); Ok(SourcemapIdentity { debug_id: inject::read_debug_id(path)?, - content_sha1_hex: sha1_hex(&bytes), - size_bytes: bytes.len() as u64, + content_sha1_hex: hex::encode(digest), + size_bytes, }) } @@ -39,15 +55,38 @@ pub struct SourcemapIdentity { pub size_bytes: u64, } -fn sha1_hex(bytes: &[u8]) -> String { - let digest: [u8; 20] = Sha1::digest(bytes).into(); - hex::encode(digest) -} - #[cfg(test)] mod tests { use super::*; + #[test] + fn identify_streams_the_hash_and_size_across_buffer_boundaries() { + // Bigger than the 64 KiB read buffer and not a multiple of it. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("big.js.map"); + let body = format!( + r#"{{"version":3,"debug_id":"did-big","mappings":"{}"}}"#, + "A".repeat(300_001) + ); + std::fs::write(&path, &body).unwrap(); + let id = identify(&path).unwrap(); + let expected: [u8; 20] = Sha1::digest(body.as_bytes()).into(); + assert_eq!(id.content_sha1_hex, hex::encode(expected)); + assert_eq!(id.size_bytes, body.len() as u64); + assert_eq!(id.debug_id.as_deref(), Some("did-big")); + } + + #[test] + fn identify_still_reports_a_non_utf8_map_as_an_io_error() { + // `read_debug_id` used `read_to_string`, so invalid UTF-8 was an I/O + // `InvalidData` error (exit 10), not a JSON error (exit 11). The mapped + // read must keep that classification. + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bad.js.map"); + std::fs::write(&path, b"{\"debug_id\":\"\xff\xfe\"}").unwrap(); + assert!(matches!(identify(&path), Err(crate::error::Error::Io(_)))); + } + #[test] fn identify_reads_debug_id_and_hashes_bytes() { let dir = tempfile::tempdir().unwrap(); diff --git a/src/upload/build.rs b/src/upload/build.rs index f90d4f3..483fc83 100644 --- a/src/upload/build.rs +++ b/src/upload/build.rs @@ -184,13 +184,16 @@ pub async fn run(params: Params<'_>, policy: RetryPolicy) -> Result { } // PUT the artefact ZIP. Idempotent overwrite of the same S3 key, so a // retriable 5xx is safe to retry. - let body = tokio::fs::read(&zip_path).await?; - tracing::debug!(endpoint = %http::redact_url(®.artifact_endpoint), body_len = body.len(), "PUT artefact"); + // Streamed from disk: the artefact is an IPA/APK/AAB (hundreds of MB), and + // reading it whole and cloning it per attempt held two copies in memory. + let body_len = http::file_len(&zip_path).await?; + tracing::debug!(endpoint = %http::redact_url(®.artifact_endpoint), body_len, "PUT artefact"); let put = http::send_with_retry(policy, "artefact PUT", true, || { client .put(®.artifact_endpoint) .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body.clone()) + .header(reqwest::header::CONTENT_LENGTH, body_len) + .body(http::file_body(&zip_path)) }) .await?; if !put.status().is_success() { @@ -472,6 +475,71 @@ mod tests { assert_eq!(got, b"x -> y\n".repeat(50)); } + /// The artefact PUT streams the ZIP from disk: a large artefact (bigger than + /// the 64 KiB stream chunk) must arrive whole with an exact `Content-Length` + /// (no chunked body, which a presigned S3 PUT rejects), and a retried attempt + /// must re-send the complete body. + #[tokio::test] + async fn artefact_put_streams_with_exact_length_and_retries_whole() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let data: Vec = (0..300_001u32).map(|i| (i % 251) as u8).collect(); + let artifact = write(tmp.path(), "app.aab", &data); + let payload = write(tmp.path(), "p.json", br#"{"uuid":"abc","version":"1.0"}"#); + let put_url = format!("{}/artefact-put", server.uri()); + + Mock::given(method("POST")) + .and(path("/v2/apps/TKN/builds")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, "result": { "build_id": "b1", "endpoint": put_url } + }))) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path("/artefact-put")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path("/artefact-put")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + + let uri = server.uri(); + let params = base_params(&uri, &payload, &artifact); + run(params, RetryPolicy::fast(3)).await.unwrap(); + + let received = server.received_requests().await.unwrap(); + let puts: Vec<_> = received + .iter() + .filter(|r| r.url.path() == "/artefact-put") + .collect(); + assert_eq!(puts.len(), 2, "one 503 then one success"); + for put in &puts { + assert_eq!( + put.headers + .get("content-length") + .map(|v| v.to_str().unwrap().to_owned()), + Some(put.body.len().to_string()) + ); + assert!(put.headers.get("transfer-encoding").is_none()); + assert_eq!( + put.headers.get("content-type").unwrap(), + "application/octet-stream" + ); + let mut zip = ZipArchive::new(std::io::Cursor::new(put.body.clone())).unwrap(); + let mut got = Vec::new(); + zip.by_name("app.aab") + .unwrap() + .read_to_end(&mut got) + .unwrap(); + assert_eq!(got, data, "the artefact survives the stream intact"); + } + assert_eq!(puts[0].body, puts[1].body, "retry re-sends the same bytes"); + } + #[tokio::test] async fn uploads_build_info_bundle_from_same_registration() { let server = MockServer::start().await; diff --git a/src/upload/build_info.rs b/src/upload/build_info.rs index afe4e82..8deee9e 100644 --- a/src/upload/build_info.rs +++ b/src/upload/build_info.rs @@ -136,10 +136,9 @@ pub async fn run(params: Params<'_>, policy: RetryPolicy) -> Result { } }; - // Read the bundle into memory so retries can re-issue the body. The - // build-info bundle is small (typically < 5 MB compressed). - let body = tokio::fs::read(&zip_path).await?; - tracing::debug!(url = %http::redact_url(&presigned), body_len = body.len(), "PUT build-info bundle"); + // Streamed from disk like the other presigned PUTs (each retry re-opens it). + let body_len = http::file_len(&zip_path).await?; + tracing::debug!(url = %http::redact_url(&presigned), body_len, "PUT build-info bundle"); // The PUT to the presigned S3 URL is idempotent (an overwrite of the same // key), so retrying on a retriable 5xx is safe. // @@ -153,7 +152,8 @@ pub async fn run(params: Params<'_>, policy: RetryPolicy) -> Result { client .put(&presigned) .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body.clone()) + .header(reqwest::header::CONTENT_LENGTH, body_len) + .body(http::file_body(&zip_path)) }) .await?; diff --git a/src/upload/http.rs b/src/upload/http.rs index 56712e8..4bef074 100644 --- a/src/upload/http.rs +++ b/src/upload/http.rs @@ -215,6 +215,52 @@ pub fn truncate_for_log(s: &str, max: usize) -> String { format!("{}…", &s[..end]) } +/// A request body that streams `path` from disk in 64 KiB chunks. +/// +/// Reading the file whole (and cloning it for each retry attempt) holds one or +/// two full copies of an artefact in memory per upload. The file is opened +/// lazily on first poll so an open error surfaces as a failed (retriable) +/// request rather than a panic, and every attempt re-opens it from the start. +/// +/// A streamed body has no known length and would go out chunked, which a +/// presigned S3 PUT rejects: the caller MUST also set `Content-Length` +/// (see [`file_len`]). +pub fn file_body(path: &std::path::Path) -> reqwest::Body { + use tokio::io::AsyncReadExt; + enum St { + Unopened(std::path::PathBuf), + Open(tokio::fs::File), + Done, + } + reqwest::Body::wrap_stream(futures_util::stream::unfold( + St::Unopened(path.to_path_buf()), + |st| async move { + let mut file = match st { + St::Unopened(p) => match tokio::fs::File::open(p).await { + Ok(f) => f, + Err(e) => return Some((Err(e), St::Done)), + }, + St::Open(f) => f, + St::Done => return None, + }; + let mut buf = vec![0u8; 64 * 1024]; + match file.read(&mut buf).await { + Ok(0) => None, + Ok(n) => { + buf.truncate(n); + Some((Ok::<_, std::io::Error>(buf), St::Open(file))) + } + Err(e) => Some((Err(e), St::Done)), + } + }, + )) +} + +/// The byte length to send as `Content-Length` alongside [`file_body`]. +pub async fn file_len(path: &std::path::Path) -> std::io::Result { + Ok(tokio::fs::metadata(path).await?.len()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/upload/presigned.rs b/src/upload/presigned.rs index 5a2c119..63724d4 100644 --- a/src/upload/presigned.rs +++ b/src/upload/presigned.rs @@ -232,16 +232,14 @@ pub async fn put_payload( // attempt) held two-plus copies of the archive in memory per upload. The // length is sent explicitly because a streamed body would otherwise go out // chunked, which a presigned S3 PUT rejects. - let len = tokio::fs::metadata(payload).await?.len(); + let len = http::file_len(payload).await?; // The PUT is idempotent (overwrite of the same key) β€” retry on transport // AND retriable status. Each attempt re-opens the file from the start. let put_resp = http::send_with_retry(policy, "symbol PUT", true, || { client .put(presigned_url) .header(reqwest::header::CONTENT_LENGTH, len) - .body(reqwest::Body::wrap_stream(file_chunks( - payload.to_path_buf(), - ))) + .body(http::file_body(payload)) }) .await?; @@ -257,38 +255,6 @@ pub async fn put_payload( Ok(()) } -/// The file's bytes in 64 KiB chunks. The file is opened lazily on first poll so -/// an open error surfaces as a failed (retriable) request, not a panic. -fn file_chunks( - path: std::path::PathBuf, -) -> impl futures_util::Stream>> + Send + 'static { - use tokio::io::AsyncReadExt; - enum St { - Unopened(std::path::PathBuf), - Open(tokio::fs::File), - Done, - } - futures_util::stream::unfold(St::Unopened(path), |st| async move { - let mut file = match st { - St::Unopened(p) => match tokio::fs::File::open(p).await { - Ok(f) => f, - Err(e) => return Some((Err(e), St::Done)), - }, - St::Open(f) => f, - St::Done => return None, - }; - let mut buf = vec![0u8; 64 * 1024]; - match file.read(&mut buf).await { - Ok(0) => None, - Ok(n) => { - buf.truncate(n); - Some((Ok(buf), St::Open(file))) - } - Err(e) => Some((Err(e), St::Done)), - } - }) -} - /// Run the two-stage presigned upload for a single symbol artifact /// ([`register`] then [`put_payload`]). The payload is always produced up /// front; callers that want to skip producing it when the server already has From 530ba3cdc7626e06e9fab0dd1b741e96c2008fe2 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 16:56:22 +0500 Subject: [PATCH 08/18] perf(sourcemaps): stream map JSON with struson instead of a serde_json::Value tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source map is tens of MB (a huge `mappings` string plus whole sources in `sourcesContent`), and parsing it into a Value held ~2x the file just to read or change a few top-level keys. New inject::mapjson walks the document once with struson (syntax still validated end to end, trailing garbage rejected): - read_debug_id / restamp_id carry-check: top-level string keys only - --strip-sources-content: filtered copy, drops sourcesContent at the top level and inside nested sections[].map, everything else copied verbatim - inject's map rewrite: two passes (read ids, then stream-copy with the ids set) into a sibling temp file, copied over the original so permissions / symlinks survive - the inject debug-id (UUIDv5 over len||bundle||map) is hashed incrementally with the map streamed from disk; pinned bit-for-bit to Uuid::new_v5 by a test across buffer boundaries Peak RSS, 89 MB map: upload 183 -> 32 MB, --strip-sources-content 262 -> 32 MB, sourcemaps inject 298 -> 6 MB. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 9 ++ Cargo.lock | 56 ++++++++ Cargo.toml | 4 + src/cli/debug_files.rs | 91 +++++------- src/inject/mapjson.rs | 313 +++++++++++++++++++++++++++++++++++++++++ src/inject/mod.rs | 174 +++++++++++++++-------- 6 files changed, 533 insertions(+), 114 deletions(-) create mode 100644 src/inject/mapjson.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1065e..8899121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). Peak RSS: `upload build` with a 300 MB artefact + 200 MB mapping 661 MB -> 32 MB, a 200 MB ProGuard mapping 241 MB -> 32 MB, an ELF zip of four 100 MB libraries 463 MB -> 36 MB. Wire format and exit codes unchanged. +- **Source maps are streamed instead of parsed into a JSON tree.** Reading a map's debug-id, + stripping `sourcesContent` (`--strip-sources-content`, including nested indexed-map sections) and + `sourcemaps inject` rewriting a map's ids now walk the document with `struson`, and the inject + debug-id hash is computed incrementally instead of over a concatenated copy. Peak RSS for an + 89 MB map: upload 183 MB -> 32 MB, `--strip-sources-content` 262 MB -> 32 MB, `inject` 298 MB -> + 6 MB. Debug-ids are unchanged. Two byte-level differences: a stripped copy keeps the map's + original key order (it was alphabetical), and `inject` writes `debug_id` / `debugId` last. Both + are still valid maps; an invalid-UTF-8 map outside a string value now reports as invalid JSON + (exit 11) rather than I/O (exit 10). ## [0.7.13] - 2026-10-07 diff --git a/Cargo.lock b/Cargo.lock index 8b2ff33..3471bf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,7 @@ dependencies = [ "serde_json", "sha1", "sha2", + "struson", "symbolic-common", "symbolic-debuginfo", "tempfile", @@ -464,6 +465,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "duplicate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e92f10a49176cbffacaedabfaa11d51db1ea0f80a83c26e1873b43cd1742c24" +dependencies = [ + "heck", + "proc-macro2", + "proc-macro2-diagnostics", +] + [[package]] name = "elementtree" version = "1.2.3" @@ -1482,6 +1494,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", +] + [[package]] name = "quick-xml" version = "0.42.0" @@ -2000,6 +2024,38 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "struson" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8eaaf563e9de627dadbe66ff8b7ef8f065fc69844c90ed6acdc90bdc9f0571" +dependencies = [ + "duplicate", + "strum", + "thiserror 2.0.21", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index fc15b36..079fd2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -172,6 +172,10 @@ tempfile = "3" # [dependencies] table (NOT a cfg(target) table). self-replace = "1" +# Streaming JSON reader/writer: lets the source-map paths read or rewrite a few +# top-level keys of a tens-of-MB map without building a serde_json::Value tree. +struson = "0.7" + # Unix double-fork for `xcode post-action`'s background (daemon) mode. This is a # TARGET-specific table, so it MUST come AFTER the main [dependencies] table β€” # placed mid-table, TOML would capture every following key into the unix-only diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index b1d91a2..feb10a8 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -1917,23 +1917,6 @@ fn is_stylesheet_or_declaration_map(path: &Path, extra: &ExtraSuffixes) -> bool }) } -/// Remove every `sourcesContent` a source map can carry; `true` when one was there. -/// -/// Not just the top-level key: an INDEXED map (spec Β§Index-Map) keeps its source inside -/// `sections[i].map`, and those sections nest. Stripping only the top level left the source in the -/// upload while reporting success β€” the flag's whole promise, failing silently. -fn remove_sources_content(map: &mut serde_json::Map) -> bool { - let mut removed = map.remove("sourcesContent").is_some(); - if let Some(serde_json::Value::Array(sections)) = map.get_mut("sections") { - for section in sections { - if let Some(serde_json::Value::Object(inner)) = section.get_mut("map") { - removed |= remove_sources_content(inner); - } - } - } - removed -} - /// A copy of `map_path` with `sourcesContent` removed, plus its recomputed identity β€” or `None` /// when nothing needs stripping (flag off, no `sourcesContent`, or the map is not the JSON object we /// expect, which is a privacy preference's business to ignore rather than to fail over). @@ -1952,40 +1935,42 @@ fn stripped_copy_without_sources( // Typed, not a bare `?` into anyhow: an I/O failure on the map must classify the way // `sourcemap::identify` classifies one on the SAME file moments earlier β€” exit 10, which // integrators are documented not to fall back on β€” rather than exit 1 ("unexpected"). - let bytes = crate::symbols::mapped::map_file(map_path).map_err(crate::error::Error::Io)?; - let Ok(serde_json::Value::Object(mut map)) = serde_json::from_slice(&bytes) else { - return Ok(None); - }; - if !remove_sources_content(&mut map) { - return Ok(None); - } - let stripped = serde_json::to_vec(&map).map_err(|e| { - input_invalid(format!( - "{}: could not re-serialize the map without its sources: {e}", - map_path.display() - )) - })?; let name = map_path .file_name() .map(Path::new) .unwrap_or_else(|| Path::new("bundle.js.map")); let out = tmpdir.join(name); - std::fs::write(&out, &stripped).map_err(crate::error::Error::Io)?; + // Streamed input -> output: only the dropped `sourcesContent` values (the bulk of a + // map) are never held, and nothing else is either, so memory is flat however big the map. + let before_bytes = std::fs::metadata(map_path) + .map_err(crate::error::Error::Io)? + .len(); + let input = + std::io::BufReader::new(std::fs::File::open(map_path).map_err(crate::error::Error::Io)?); + let output = + std::io::BufWriter::new(std::fs::File::create(&out).map_err(crate::error::Error::Io)?); + match crate::inject::mapjson::strip_sources_content(input, output) { + Ok(true) => {} + // Nothing to strip, or not the JSON object we expect: the copy is moot. + Ok(false) | Err(crate::inject::mapjson::MapJsonError::Json(_)) => { + let _ = std::fs::remove_file(&out); + return Ok(None); + } + Err(crate::inject::mapjson::MapJsonError::Io(e)) => { + let _ = std::fs::remove_file(&out); + return Err(crate::error::Error::Io(e).into()); + } + } + // The identity describes what is UPLOADED (the stripped copy): its streamed hash and size. + // The debug-id is carried over untouched: it is the key, and re-deriving it here would + // break the pair with the bundle that `sourcemaps inject` stamped. + let identity = sourcemap::identify(&out)?; tracing::info!( path = %map_path.display(), - before_bytes = bytes.len(), - after_bytes = stripped.len(), + before_bytes, + after_bytes = identity.size_bytes, "stripped sourcesContent from the uploaded copy" ); - let identity = sourcemap::SourcemapIdentity { - debug_id: crate::inject::read_debug_id(&out)?, - content_sha1_hex: { - use sha1::Digest; - let digest: [u8; 20] = sha1::Sha1::digest(&stripped).into(); - hex::encode(digest) - }, - size_bytes: stripped.len() as u64, - }; Ok(Some((out, identity))) } @@ -3399,18 +3384,7 @@ mod sourcemap_upload_tests { } async fn uploaded_map_json(server: &MockServer) -> serde_json::Value { - let requests = server.received_requests().await.unwrap(); - let put = requests - .iter() - .find(|r| r.method.as_str() == "PUT") - .expect("no PUT"); - let mut archive = - zip::ZipArchive::new(std::io::Cursor::new(put.body.clone())).expect("not a zip"); - assert_eq!(archive.len(), 1, "one entry: just the map"); - let mut entry = archive.by_index(0).unwrap(); - let mut body = String::new(); - std::io::Read::read_to_string(&mut entry, &mut body).unwrap(); - serde_json::from_str(&body).expect("uploaded map is not JSON") + serde_json::from_slice(&uploaded_map_bytes(server).await).expect("uploaded map is not JSON") } /// `sourcesContent` embeds the ORIGINAL SOURCE in the map, which is how symbolication shows @@ -3495,14 +3469,17 @@ mod sourcemap_upload_tests { .body, ) .unwrap(); - let uploaded = uploaded_map_json(&server).await; + let uploaded = uploaded_map_bytes(&server).await; let expected = { use sha1::Digest; - let digest: [u8; 20] = - sha1::Sha1::digest(serde_json::to_vec(&uploaded).unwrap()).into(); + let digest: [u8; 20] = sha1::Sha1::digest(&uploaded).into(); hex::encode(digest) }; assert_eq!(post["hash"].as_str().unwrap(), expected); + assert!( + !String::from_utf8_lossy(&uploaded).contains("sourcesContent"), + "and the hash is of the STRIPPED bytes" + ); } /// Without the flag, the source rides along exactly as before β€” that is what makes the diff --git a/src/inject/mapjson.rs b/src/inject/mapjson.rs new file mode 100644 index 0000000..7d74c77 --- /dev/null +++ b/src/inject/mapjson.rs @@ -0,0 +1,313 @@ +//! Streaming reads and rewrites of source-map JSON. +//! +//! A source map is routinely tens of MB, mostly one huge `mappings` string and +//! an array of whole source files (`sourcesContent`). Parsing it into a +//! `serde_json::Value` held ~2x the file on the heap just to look at, or change, +//! a handful of top-level keys. Everything here walks the document once with +//! [`struson`] instead, so memory stays at the reader's small buffer plus the +//! one member name / id string being looked at β€” whatever the map's size. +//! +//! Syntax is still validated end to end: a skipped value is checked like a read +//! one, and trailing garbage after the top-level value is an error. + +use std::io::{Read, Write}; + +use struson::reader::{JsonReader, JsonStreamReader, ReaderError, ValueType}; +use struson::writer::{JsonStreamWriter, JsonWriter}; + +/// Why a streaming pass over a map failed. +#[derive(Debug)] +pub enum MapJsonError { + /// The input is not (this shape of) valid JSON. + Json(String), + /// Writing the rewritten copy failed. + Io(std::io::Error), +} + +impl std::fmt::Display for MapJsonError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MapJsonError::Json(m) => f.write_str(m), + MapJsonError::Io(e) => write!(f, "{e}"), + } + } +} + +impl From for MapJsonError { + fn from(e: ReaderError) -> Self { + match e { + // Invalid UTF-8 inside a string surfaces as an I/O `InvalidData` error; + // keep it one (as `read_to_string` reported it) rather than "bad JSON". + ReaderError::IoError { error, .. } => MapJsonError::Io(error), + other => MapJsonError::Json(other.to_string()), + } + } +} + +impl From for MapJsonError { + fn from(e: struson::reader::TransferError) -> Self { + match e { + struson::reader::TransferError::ReaderError(r) => r.into(), + struson::reader::TransferError::WriterError(w) => MapJsonError::Io(w), + } + } +} + +impl From for MapJsonError { + fn from(e: std::io::Error) -> Self { + MapJsonError::Io(e) + } +} + +type Res = std::result::Result; + +/// What [`top_level_strings`] found. +pub struct TopLevel { + /// Whether the document's top-level value is an object. + pub is_object: bool, + /// One entry per requested key, in order: its value when it is a string, + /// else `None` (absent, or any other JSON type). With a duplicated key the + /// LAST occurrence wins, as it does in a parsed `serde_json::Value`. + pub values: Vec>, +} + +/// Read the string values of the given top-level keys, validating the whole +/// document. A top-level value that is not an object yields `is_object: false` +/// and no values (it is still syntax-checked). +pub fn top_level_strings(input: R, keys: &[&str]) -> Res { + let mut r = JsonStreamReader::new(input); + let mut values: Vec> = vec![None; keys.len()]; + let is_object = r.peek()? == ValueType::Object; + if is_object { + r.begin_object()?; + while r.has_next()? { + let idx = { + let name = r.next_name()?; + keys.iter().position(|k| *k == name) + }; + match idx { + Some(i) if r.peek()? == ValueType::String => values[i] = Some(r.next_string()?), + Some(i) => { + values[i] = None; + r.skip_value()?; + } + None => r.skip_value()?, + } + } + r.end_object()?; + } else { + r.skip_value()?; + } + r.consume_trailing_whitespace()?; + Ok(TopLevel { is_object, values }) +} + +/// Copy `input` to `output`, dropping every `sourcesContent` β€” at the top level +/// AND inside the (nestable) `sections[i].map` of an indexed map. Returns +/// whether anything was dropped. Everything else is copied verbatim, in its +/// original order, without being held in memory. +/// +/// Errors with [`MapJsonError::Json`] when the input is not a JSON object. +pub fn strip_sources_content(input: R, output: W) -> Res { + let mut r = JsonStreamReader::new(input); + let mut w = JsonStreamWriter::new(output); + if r.peek()? != ValueType::Object { + return Err(MapJsonError::Json("not a JSON object".into())); + } + let removed = strip_object(&mut r, &mut w)?; + r.consume_trailing_whitespace()?; + w.finish_document()?; + Ok(removed) +} + +fn strip_object(r: &mut R, w: &mut W) -> Res { + let mut removed = false; + r.begin_object()?; + w.begin_object()?; + while r.has_next()? { + let name = r.next_name()?.to_owned(); + if name == "sourcesContent" { + r.skip_value()?; + removed = true; + continue; + } + w.name(&name)?; + if name == "sections" && r.peek()? == ValueType::Array { + r.begin_array()?; + w.begin_array()?; + while r.has_next()? { + if r.peek()? == ValueType::Object { + removed |= strip_section(r, w)?; + } else { + r.transfer_to(w)?; + } + } + r.end_array()?; + w.end_array()?; + } else { + r.transfer_to(w)?; + } + } + r.end_object()?; + w.end_object()?; + Ok(removed) +} + +/// One `sections[i]` object: only its `map` (when an object) is searched. +fn strip_section(r: &mut R, w: &mut W) -> Res { + let mut removed = false; + r.begin_object()?; + w.begin_object()?; + while r.has_next()? { + let name = r.next_name()?.to_owned(); + w.name(&name)?; + if name == "map" && r.peek()? == ValueType::Object { + removed |= strip_object(r, w)?; + } else { + r.transfer_to(w)?; + } + } + r.end_object()?; + w.end_object()?; + Ok(removed) +} + +/// Copy `input` to `output` with the top-level `debug_id` and `debugId` set to +/// `debug_id` (any existing ones are dropped and the pair is written last). +/// Everything else is copied verbatim and in order. +/// +/// Errors with [`MapJsonError::Json`] when the input is not a JSON object. +pub fn set_debug_ids(input: R, output: W, debug_id: &str) -> Res<()> { + let mut r = JsonStreamReader::new(input); + let mut w = JsonStreamWriter::new(output); + if r.peek()? != ValueType::Object { + return Err(MapJsonError::Json("not a JSON object".into())); + } + r.begin_object()?; + w.begin_object()?; + while r.has_next()? { + let name = r.next_name()?.to_owned(); + if name == "debug_id" || name == "debugId" { + r.skip_value()?; + continue; + } + w.name(&name)?; + r.transfer_to(&mut w)?; + } + r.end_object()?; + w.name("debug_id")?; + w.string_value(debug_id)?; + w.name("debugId")?; + w.string_value(debug_id)?; + w.end_object()?; + r.consume_trailing_whitespace()?; + w.finish_document()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn strip(s: &str) -> Res<(bool, String)> { + let mut out = Vec::new(); + let removed = strip_sources_content(s.as_bytes(), &mut out)?; + Ok((removed, String::from_utf8(out).unwrap())) + } + + #[test] + fn top_level_strings_picks_string_values_last_wins_and_ignores_other_types() { + let t = top_level_strings( + br#"{"debug_id": 5, "debugId": "camel", "uuid": "a", "uuid": "b", "x": {"debug_id": "nested"}}"#.as_slice(), + &["debug_id", "debugId", "uuid"], + ) + .unwrap(); + assert!(t.is_object); + assert_eq!( + t.values, + [None, Some("camel".into()), Some("b".into())], + "non-string -> None, duplicates -> last, nested keys ignored" + ); + // A later non-string occurrence replaces an earlier string, as in a Value. + let t = top_level_strings(br#"{"uuid": "a", "uuid": 1}"#.as_slice(), &["uuid"]).unwrap(); + assert_eq!(t.values, [None]); + } + + #[test] + fn top_level_strings_non_object_and_invalid_json() { + for ok in ["[]", "3", "\"s\"", "null"] { + let t = top_level_strings(ok.as_bytes(), &["uuid"]).unwrap(); + assert!(!t.is_object, "{ok}"); + assert_eq!(t.values, [None]); + } + for bad in [ + "", + "{", + r#"{"a":}"#, + r#"{"a":1} x"#, + r#"{"a":1}{"b":2}"#, + "[1,]", + ] { + assert!( + matches!( + top_level_strings(bad.as_bytes(), &["a"]), + Err(MapJsonError::Json(_)) + ), + "{bad:?} must be rejected" + ); + } + } + + #[test] + fn strip_removes_top_level_and_nested_section_sources_in_order() { + let (removed, out) = strip( + r#"{"version":3,"sourcesContent":["a"],"sections":[{"offset":{"line":0,"column":0},"map":{"sources":["x"],"sourcesContent":["secret"],"sections":[{"map":{"sourcesContent":["deep"],"names":[]}}]}},7],"mappings":"AAAA"}"#, + ) + .unwrap(); + assert!(removed); + assert_eq!( + out, + r#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"sources":["x"],"sections":[{"map":{"names":[]}}]}},7],"mappings":"AAAA"}"# + ); + } + + #[test] + fn strip_reports_nothing_removed_and_ignores_non_object_sections() { + let (removed, out) = + strip(r#"{"sections":"nope","map":{"sourcesContent":[1]},"a":1.50}"#).unwrap(); + assert!(!removed, "only sections[].map is searched"); + assert!( + out.contains(r#""a":1.50"#), + "numbers keep their text: {out}" + ); + } + + #[test] + fn strip_rejects_non_objects_and_bad_json() { + assert!(matches!(strip("[]"), Err(MapJsonError::Json(_)))); + assert!(matches!(strip("{"), Err(MapJsonError::Json(_)))); + assert!(matches!( + strip(r#"{"a":1} junk"#), + Err(MapJsonError::Json(_)) + )); + } + + #[test] + fn set_debug_ids_replaces_existing_pair_and_keeps_the_rest() { + let mut out = Vec::new(); + set_debug_ids( + br#"{"debug_id":"old","version":3,"debugId":"older","mappings":"AA"}"#.as_slice(), + &mut out, + "new-id", + ) + .unwrap(); + assert_eq!( + String::from_utf8(out).unwrap(), + r#"{"version":3,"mappings":"AA","debug_id":"new-id","debugId":"new-id"}"# + ); + assert!(matches!( + set_debug_ids(b"[]".as_slice(), Vec::new(), "x"), + Err(MapJsonError::Json(_)) + )); + } +} diff --git a/src/inject/mod.rs b/src/inject/mod.rs index 1cd40c9..a028762 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -23,6 +23,7 @@ use uuid::Uuid; use crate::error::{Error, Result}; +pub mod mapjson; pub mod sri; /// Fixed namespace for Bugsee sourcemap debug-ids β€” keeps UUIDv5 generation @@ -54,15 +55,48 @@ pub fn compute_debug_id(content: &[u8]) -> Uuid { /// server while the upload reported the new one as already there. The bundle's /// length is hashed ahead of it, so the boundary between the two inputs cannot /// shift without changing the id. +#[cfg(test)] pub fn compute_debug_id_with_map(bundle: &[u8], map: Option<&[u8]>) -> Uuid { let Some(map) = map else { return compute_debug_id(bundle); }; - let mut input = Vec::with_capacity(8 + bundle.len() + map.len()); - input.extend_from_slice(&(bundle.len() as u64).to_le_bytes()); - input.extend_from_slice(bundle); - input.extend_from_slice(map); - Uuid::new_v5(&DEBUG_ID_NAMESPACE, &input) + // Reading a slice cannot fail. + hash_with_map(bundle, map).expect("reading from a slice cannot fail") +} + +/// [`compute_debug_id_with_map`] with the map STREAMED from `map_path` instead of +/// held in memory: a map is tens of MB, and concatenating it with the bundle +/// into one buffer just to hash it was a second full copy. +pub fn compute_debug_id_with_map_file(bundle: &[u8], map_path: &Path) -> std::io::Result { + hash_with_map( + bundle, + std::io::BufReader::new(std::fs::File::open(map_path)?), + ) +} + +/// UUIDv5 over `len(bundle) as u64 LE || bundle || map`, computed incrementally. +/// +/// A v5 UUID is the first 16 bytes of SHA-1(namespace || name) with the +/// version/variant bits set, so feeding the pieces to the hasher one at a time +/// gives the same id as `Uuid::new_v5` over their concatenation. +fn hash_with_map(bundle: &[u8], mut map: impl std::io::Read) -> std::io::Result { + use sha1::{Digest as _, Sha1}; + let mut hasher = Sha1::new(); + hasher.update(DEBUG_ID_NAMESPACE.as_bytes()); + hasher.update((bundle.len() as u64).to_le_bytes()); + hasher.update(bundle); + let mut buf = [0u8; 64 * 1024]; + loop { + let n = map.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + let digest = hasher.finalize(); + let mut first16 = [0u8; 16]; + first16.copy_from_slice(&digest[..16]); + Ok(uuid::Builder::from_sha1_bytes(first16).into_uuid()) } /// The runtime stub appended to each JS bundle. On load it registers @@ -352,8 +386,10 @@ fn inject_one(js_path: &Path, dry_run: bool, stats: &mut InjectStats) -> Result< } }, None => { - let map_bytes = map_path.as_deref().map(std::fs::read).transpose()?; - let id = compute_debug_id_with_map(content.as_bytes(), map_bytes.as_deref()); + let id = match map_path.as_deref() { + Some(m) => compute_debug_id_with_map_file(content.as_bytes(), m)?, + None => compute_debug_id(content.as_bytes()), + }; if !dry_run { std::fs::write(js_path, format!("{content}{}", runtime_stub(&id)))?; } @@ -392,19 +428,18 @@ fn restamp_id<'a>( let Some(original) = content.strip_suffix(runtime_stub(&id).as_str()) else { return Ok(None); }; - let map_bytes = std::fs::read(map_path)?; + let file = std::fs::File::open(map_path)?; // A map that is not JSON is not re-keyed here: writing the id into it fails // next, and it must fail BEFORE the bundle is rewritten, as it always did. - let Ok(map) = serde_json::from_slice::(&map_bytes) else { + let Ok(top) = + mapjson::top_level_strings(std::io::BufReader::new(file), &["debug_id", "debugId"]) + else { return Ok(None); }; - let carries_id = ["debug_id", "debugId"] - .iter() - .any(|k| map.get(*k).and_then(serde_json::Value::as_str).is_some()); - if carries_id { + if top.values.iter().any(Option::is_some) { return Ok(None); } - let fresh = compute_debug_id_with_map(original.as_bytes(), Some(&map_bytes)); + let fresh = compute_debug_id_with_map_file(original.as_bytes(), map_path)?; Ok((fresh.to_string() != id).then_some((original, fresh))) } @@ -500,25 +535,15 @@ fn write_map_debug_id( freshly_computed: bool, dry_run: bool, ) -> Result { - let raw = std::fs::read_to_string(map_path)?; - let mut value: serde_json::Value = serde_json::from_str(&raw).map_err(|e| { - Error::InputInvalid(format!( - "sourcemap is not valid JSON: {e} ({})", - map_path.display() - )) - })?; - let obj = value.as_object_mut().ok_or_else(|| { - Error::InputInvalid(format!( + // Pass 1 (streamed, also validates the whole file): what ids does it carry? + let top = map_top_level(map_path, &["debug_id", "debugId"])?; + if !top.is_object { + return Err(Error::InputInvalid(format!( "sourcemap is not a JSON object: {}", map_path.display() - )) - })?; - let current = |k: &str| { - obj.get(k) - .and_then(serde_json::Value::as_str) - .map(str::to_string) - }; - let (snake, camel) = (current("debug_id"), current("debugId")); + ))); + } + let (snake, camel) = (top.values[0].clone(), top.values[1].clone()); match map_id_action( snake.as_deref(), camel.as_deref(), @@ -544,18 +569,28 @@ fn write_map_debug_id( return Ok(false); } } - obj.insert( - "debug_id".into(), - serde_json::Value::String(debug_id.to_string()), - ); - obj.insert( - "debugId".into(), - serde_json::Value::String(debug_id.to_string()), - ); if !dry_run { - let serialized = serde_json::to_string(&value) - .map_err(|e| Error::InputInvalid(format!("failed to serialize sourcemap JSON: {e}")))?; - std::fs::write(map_path, serialized)?; + // Pass 2: stream the map into a sibling temp file with the ids set, then + // copy it over the original (not rename: that would swap the inode and + // drop the permissions / symlink / hard-link the build gave the map). + let dir = map_path.parent().filter(|p| !p.as_os_str().is_empty()); + let tmp = match dir { + Some(d) => tempfile::NamedTempFile::new_in(d)?, + None => tempfile::NamedTempFile::new_in(".")?, + }; + { + let input = std::io::BufReader::new(std::fs::File::open(map_path)?); + let mut out = std::io::BufWriter::new(tmp.as_file()); + mapjson::set_debug_ids(input, &mut out, debug_id).map_err(|e| match e { + mapjson::MapJsonError::Io(e) => Error::Io(e), + mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( + "failed to rewrite sourcemap JSON: {m} ({})", + map_path.display() + )), + })?; + std::io::Write::flush(&mut out)?; + } + std::fs::copy(tmp.path(), map_path)?; } Ok(true) } @@ -564,22 +599,24 @@ fn write_map_debug_id( /// / legacy `uuid`, in that precedence. Mirrors the worker's /// `symbolfiles/sourcemap.py:parse`. pub fn read_debug_id(map_path: &Path) -> Result> { - // Mapped rather than read into a String: the JSON tree below is the only - // copy this needs. UTF-8 is checked up front so a non-UTF-8 map still fails - // as an I/O `InvalidData` error, exactly as `read_to_string` did. - let bytes = crate::symbols::mapped::map_file(map_path)?; - let raw = std::str::from_utf8(&bytes) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| { - Error::InputInvalid(format!( - "sourcemap is not valid JSON: {e} ({})", + // Walked as a stream: no copy of the map and no JSON tree, so a 100 MB map + // costs a few KB. Invalid UTF-8 inside a string stays an I/O `InvalidData` + // error, as it was with `read_to_string`. + let top = map_top_level(map_path, &["debug_id", "debugId", "uuid"])?; + Ok(top.values.into_iter().flatten().next()) +} + +/// Stream-read the string values of top-level `keys` of the map at `map_path`. +/// Not-JSON becomes `InputInvalid` (naming the file); an I/O failure stays `Io`. +fn map_top_level(map_path: &Path, keys: &[&str]) -> Result { + let file = std::io::BufReader::new(std::fs::File::open(map_path)?); + mapjson::top_level_strings(file, keys).map_err(|e| match e { + mapjson::MapJsonError::Io(e) => Error::Io(e), + mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( + "sourcemap is not valid JSON: {m} ({})", map_path.display() - )) - })?; - Ok(["debug_id", "debugId", "uuid"] - .iter() - .find_map(|k| value.get(*k).and_then(serde_json::Value::as_str)) - .map(str::to_string)) + )), + }) } #[cfg(test)] @@ -590,6 +627,29 @@ mod tests { /// RELATIVE root have to take turns, or they change the directory out from under each other. static CWD_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + #[test] + fn streamed_map_hash_equals_the_concatenated_uuid_v5_formula() { + // The id is a public key (it is embedded in shipped bundles): the + // incremental hash must reproduce `Uuid::new_v5` over `len || bundle || + // map` bit for bit, including across the 64 KiB read-buffer boundary. + let dir = tempfile::tempdir().unwrap(); + for map_len in [0usize, 1, 65_535, 65_536, 65_537, 300_001] { + let bundle = b"console.log('x')"; + let map: Vec = (0..map_len).map(|i| (i % 251) as u8).collect(); + let mut input = (bundle.len() as u64).to_le_bytes().to_vec(); + input.extend_from_slice(bundle); + input.extend_from_slice(&map); + let expected = Uuid::new_v5(&DEBUG_ID_NAMESPACE, &input); + let p = dir.path().join("m.map"); + std::fs::write(&p, &map).unwrap(); + assert_eq!( + compute_debug_id_with_map_file(bundle, &p).unwrap(), + expected + ); + assert_eq!(compute_debug_id_with_map(bundle, Some(&map)), expected); + } + } + #[test] fn is_contained_relative_rejects_traversal_and_absolute() { assert!(is_contained_relative("bundle.js.map")); From fed5bb265d71133b4fd4f0233d33cd5858ce6267 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:16:49 +0500 Subject: [PATCH 09/18] perf(inject): edit JS bundles in place; differential + on-disk tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sourcemaps inject` read the whole bundle and rewrote it via `format!("{content}{stub}")`: a full second copy plus the String. The bundle is now mapped read-only (zero-copy &str for the existing tail / suffix / contains checks), a plan is computed, the mapping is dropped (Windows refuses to truncate a mapped file), and the stub is appended - or, to re-key, our own trailing stub is cut off and the new one appended - with no copy of the bundle. Output bytes are identical. 150 MB bundle: 317 MB -> 2.4 MB dirty memory. Tests: - mapjson differential tests against the serde_json::Value logic it replaced (600 generated documents per operation, duplicate keys, truncation / trailing garbage rejected everywhere, a 200 000-case random-corruption sweep proving the stream accepts exactly what serde accepts and never launders invalid JSON into an upload, a 3 MB mappings string streamed through intact) - inject on-disk behaviour: exact bytes for awkward bundles (empty, CRLF, BOM, multibyte, one huge line), restamp cuts back to the original bytes, foreign ids only append, dry runs change no byte or mtime, permissions and symlinks survive on both files, read-only / non-UTF-8 bundle and unusable maps are errors that leave the files untouched with no temp files πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- src/inject/mapjson.rs | 350 +++++++++++++++++++++++++++++++ src/inject/mod.rs | 463 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 761 insertions(+), 52 deletions(-) diff --git a/src/inject/mapjson.rs b/src/inject/mapjson.rs index 7d74c77..8e859c4 100644 --- a/src/inject/mapjson.rs +++ b/src/inject/mapjson.rs @@ -311,3 +311,353 @@ mod tests { )); } } + +/// Differential tests: the streaming implementations must agree with the +/// `serde_json::Value`-tree logic they replaced, over many generated documents +/// (whitespace, escapes, surrogate pairs, exponent numbers, nested sections). +#[cfg(test)] +mod differential { + use super::*; + use serde_json::Value; + + /// Deterministic xorshift: reproducible failures without a `rand` dependency. + struct Rng(u64); + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + fn pick<'a>(&mut self, xs: &[&'a str]) -> &'a str { + xs[self.below(xs.len() as u64) as usize] + } + fn ws(&mut self) -> &'static str { + ["", "", " ", "\n", "\t ", "\r\n "][self.below(6) as usize] + } + } + + const KEYS: &[&str] = &[ + "version", + "sources", + "sourcesContent", + "mappings", + "debug_id", + "debugId", + "uuid", + "sections", + "map", + "names", + "file", + "offset", + "x\\u00e9y", + "k", + ]; + const STRINGS: &[&str] = &[ + r#""""#, + r#""a""#, + r#""h\u00e9llo""#, + "\"h\u{e9}llo \u{2603}\"", + r#""esc \n \" \\ \/ \t""#, + r#""\ud83d\ude00""#, + r#""AAAA;AACA,SAAS""#, + r#""11111111-1111-1111-1111-111111111111""#, + ]; + const NUMBERS: &[&str] = &[ + "0", + "-1", + "3", + "1.50", + "2e10", + "-0.5E-3", + "100000000000000000000", + ]; + + fn scalar(r: &mut Rng) -> String { + match r.below(5) { + 0 => "null".into(), + 1 => "true".into(), + 2 => "false".into(), + 3 => r.pick(NUMBERS).into(), + _ => r.pick(STRINGS).into(), + } + } + + fn value(r: &mut Rng, depth: u32) -> String { + if depth == 0 || r.below(3) == 0 { + return scalar(r); + } + if r.below(2) == 0 { + let n = r.below(4); + let items: Vec = (0..n).map(|_| value(r, depth - 1)).collect(); + let sep = format!("{},{}", r.ws(), r.ws()); + format!("[{}{}{}]", r.ws(), items.join(&sep), r.ws()) + } else { + object(r, depth - 1, false) + } + } + + /// An object over the map vocabulary. `maplike` biases towards the keys the + /// transforms care about and recurses into `sections[].map`. Keys are unique + /// within an object (duplicates have their own targeted test). + fn object(r: &mut Rng, depth: u32, maplike: bool) -> String { + let n = r.below(6); + let mut used = std::collections::HashSet::new(); + let mut members = Vec::new(); + for _ in 0..n { + let key = r.pick(KEYS); + if !used.insert(key) { + continue; + } + let v = match key { + "sections" if depth > 0 => { + let m = r.below(3); + let secs: Vec = (0..m) + .map(|_| { + let mut parts = vec![format!(r#""offset":{}"#, value(r, 1))]; + if r.below(4) != 0 { + parts.push(format!(r#""map":{}"#, object(r, depth - 1, true))); + } + if r.below(5) == 0 { + parts.push(format!(r#""url":{}"#, scalar(r))); + } + format!("{{{}}}", parts.join(",")) + }) + .collect(); + format!("[{}]", secs.join(",")) + } + "sourcesContent" if maplike || r.below(2) == 0 => { + format!(r#"["{}","src2"]"#, "x".repeat(r.below(200) as usize)) + } + _ => value(r, depth.saturating_sub(1)), + }; + members.push(format!("{}\"{}\"{}:{}{}", r.ws(), key, r.ws(), r.ws(), v)); + } + format!("{{{}{}}}", members.join(","), r.ws()) + } + + fn doc(r: &mut Rng) -> String { + format!("{}{}{}", r.ws(), object(r, 3, true), r.ws()) + } + + // --- the reference (old, tree-based) behaviour ------------------------------- + + fn ref_strip(map: &mut serde_json::Map) -> bool { + let mut removed = map.remove("sourcesContent").is_some(); + if let Some(Value::Array(sections)) = map.get_mut("sections") { + for section in sections { + if let Some(Value::Object(inner)) = section.get_mut("map") { + removed |= ref_strip(inner); + } + } + } + removed + } + + fn ref_ids(v: &Value, keys: &[&str]) -> Vec> { + keys.iter() + .map(|k| v.get(*k).and_then(Value::as_str).map(str::to_string)) + .collect() + } + + const CASES: u64 = 600; + + #[test] + fn strip_sources_content_agrees_with_the_tree_implementation() { + let mut r = Rng(0x9E37_79B9_7F4A_7C15); + let mut removed_some = 0; + for case in 0..CASES { + let text = doc(&mut r); + let mut expected: Value = serde_json::from_str(&text) + .unwrap_or_else(|e| panic!("generator produced invalid JSON ({e}): {text}")); + let expect_removed = ref_strip(expected.as_object_mut().unwrap()); + let mut out = Vec::new(); + let removed = strip_sources_content(text.as_bytes(), &mut out) + .unwrap_or_else(|e| panic!("case {case}: {e}\n{text}")); + let got: Value = serde_json::from_slice(&out).unwrap_or_else(|e| { + panic!( + "case {case}: output not JSON ({e}): {}", + String::from_utf8_lossy(&out) + ) + }); + assert_eq!(got, expected, "case {case}\ninput: {text}"); + assert_eq!(removed, expect_removed, "case {case}\ninput: {text}"); + removed_some += removed as u32; + } + assert!( + removed_some > 50, + "the generator must exercise stripping ({removed_some})" + ); + } + + #[test] + fn top_level_strings_agrees_with_the_tree_implementation() { + let keys = ["debug_id", "debugId", "uuid"]; + let mut r = Rng(0xD1B5_4A32_D192_ED03); + for case in 0..CASES { + let text = doc(&mut r); + let v: Value = serde_json::from_str(&text).unwrap(); + let got = top_level_strings(text.as_bytes(), &keys).unwrap(); + assert!(got.is_object); + assert_eq!(got.values, ref_ids(&v, &keys), "case {case}\ninput: {text}"); + } + } + + #[test] + fn set_debug_ids_agrees_with_the_tree_implementation() { + let mut r = Rng(0xA076_1D64_78BD_642F); + for case in 0..CASES { + let text = doc(&mut r); + let mut expected: Value = serde_json::from_str(&text).unwrap(); + let obj = expected.as_object_mut().unwrap(); + obj.insert("debug_id".into(), Value::String("new-id".into())); + obj.insert("debugId".into(), Value::String("new-id".into())); + let mut out = Vec::new(); + set_debug_ids(text.as_bytes(), &mut out, "new-id").unwrap(); + let got: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(got, expected, "case {case}\ninput: {text}"); + // Idempotent: applying it to its own output changes nothing semantically. + let mut again = Vec::new(); + set_debug_ids(out.as_slice(), &mut again, "new-id").unwrap(); + assert_eq!(serde_json::from_slice::(&again).unwrap(), expected); + } + } + + #[test] + fn duplicate_keys_resolve_like_a_parsed_value() { + // serde_json keeps the LAST of a duplicated key; so must the stream. + let text = r#"{"debug_id":"a","debug_id":"b","uuid":"u","uuid":7,"sourcesContent":["s"],"sourcesContent":["t"]}"#; + let v: Value = serde_json::from_str(text).unwrap(); + let keys = ["debug_id", "uuid"]; + assert_eq!( + top_level_strings(text.as_bytes(), &keys).unwrap().values, + ref_ids(&v, &keys) + ); + let mut out = Vec::new(); + assert!(strip_sources_content(text.as_bytes(), &mut out).unwrap()); + assert!(!String::from_utf8(out).unwrap().contains("sourcesContent")); + } + + #[test] + fn truncated_or_trailing_garbage_documents_are_rejected_everywhere() { + let mut r = Rng(0x1234_5678_9ABC_DEF1); + for case in 0..200 { + let text = doc(&mut r); + let text = text.trim_end(); + // Any strict prefix of a top-level object is invalid JSON. + let cut = 1 + r.below(text.len() as u64 - 1) as usize; + let mut cut = cut; + while !text.is_char_boundary(cut) { + cut -= 1; + } + let bad = &text[..cut]; + assert!( + serde_json::from_str::(bad).is_err(), + "case {case}: {bad}" + ); + assert!( + top_level_strings(bad.as_bytes(), &["uuid"]).is_err(), + "case {case}: {bad}" + ); + assert!( + strip_sources_content(bad.as_bytes(), Vec::new()).is_err(), + "case {case}: {bad}" + ); + assert!( + set_debug_ids(bad.as_bytes(), Vec::new(), "x").is_err(), + "case {case}: {bad}" + ); + // Trailing garbage after a complete document. + let junk = format!("{text} x"); + assert!( + top_level_strings(junk.as_bytes(), &["uuid"]).is_err(), + "case {case}" + ); + assert!( + strip_sources_content(junk.as_bytes(), Vec::new()).is_err(), + "case {case}" + ); + } + } + + #[test] + fn a_large_map_streams_through_with_its_big_values_intact() { + // Bigger than any internal buffer: a 3 MB mappings string must pass through + // `transfer_to` byte for byte while a 3 MB sourcesContent entry is dropped. + let mappings = "AAAA;".repeat(600_000); + let content = "z".repeat(3_000_000); + let text = format!( + r#"{{"version":3,"sourcesContent":["{content}"],"mappings":"{mappings}","debug_id":"d"}}"# + ); + let mut out = Vec::new(); + assert!(strip_sources_content(text.as_bytes(), &mut out).unwrap()); + let got: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(got["mappings"].as_str().unwrap(), mappings); + assert!(got.get("sourcesContent").is_none()); + assert_eq!(got["debug_id"], "d"); + } + + /// A number no double can hold (`1e999999`) is valid JSON grammar. Reading ids + /// skips it; copying it is refused as a JSON error β€” which the strip path treats + /// as "not the map we expect" and uploads the original, as the tree parser's + /// "number out of range" always led it to. + #[test] + fn out_of_range_numbers_are_skippable_but_not_copyable() { + let doc = br#"{"n":1e999999,"sourcesContent":["x"],"debug_id":"d"}"#; + assert_eq!( + top_level_strings(doc.as_slice(), &["debug_id"]) + .unwrap() + .values, + [Some("d".to_string())] + ); + assert!(matches!( + strip_sources_content(doc.as_slice(), Vec::new()), + Err(MapJsonError::Json(_)) + )); + assert!(matches!( + set_debug_ids(doc.as_slice(), Vec::new(), "x"), + Err(MapJsonError::Json(_)) + )); + } + + /// 200 000 randomly corrupted documents (byte replaced / deleted / inserted): + /// the stream accepts exactly what `serde_json` accepts, and everything it + /// copies out re-parses β€” it never launders invalid JSON into an upload. + #[test] + fn random_corruption_is_accepted_exactly_when_serde_accepts_it() { + let base = br#"{"version":3,"file":"a\u00e9.js","sources":["a.ts","b\"q"],"sourcesContent":["x\ny","\ud83d\ude00"],"names":["n1"],"mappings":"AAAA;AACA","x":[1,2.5e3,-0,true,null,{"k":"v"}]}"#; + let mut r = Rng(0x1234_5678_9ABC_DEF1); + for case in 0..200_000 { + let mut b = base.to_vec(); + for _ in 0..1 + r.below(2) { + let at = r.below(b.len() as u64) as usize; + match r.below(3) { + 0 => b[at] = r.next() as u8, + 1 => { + b.remove(at); + } + _ => b.insert(at, r.next() as u8), + } + } + let serde_ok = serde_json::from_slice::(&b).is_ok(); + let mut out = Vec::new(); + let streamed_ok = strip_sources_content(b.as_slice(), &mut out).is_ok(); + assert_eq!( + streamed_ok, + serde_ok, + "case {case} disagrees on {:?}", + String::from_utf8_lossy(&b) + ); + if streamed_ok { + assert!( + serde_json::from_slice::(&out).is_ok(), + "case {case}: output is not valid JSON: {}", + String::from_utf8_lossy(&out) + ); + } + } + } +} diff --git a/src/inject/mod.rs b/src/inject/mod.rs index a028762..d12e719 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -331,73 +331,146 @@ fn collect_targets( /// Mirrors `inject_one`'s decision, and `would_rewrite_agrees_with_what_inject_actually_writes` /// pins the two together so they cannot drift. fn would_rewrite(js_path: &Path) -> Result { - let content = std::fs::read_to_string(js_path)?; - let map_path = paired_map(js_path, &content); - Ok(match existing_debug_id(&content) { + let mapped = crate::symbols::mapped::map_file(js_path)?; + let content = bundle_text(&mapped)?; + let map_path = paired_map(js_path, content); + Ok(match existing_debug_id(content) { // A regenerated map re-keys the bundle; a foreign id still needs our runtime registration. Some(id) => { - restamp_id(&content, &id, map_path.as_deref())?.is_some() + restamp_id(content, &id, map_path.as_deref())?.is_some() || !content.contains(&runtime_registration(&id)) } None => true, }) } +/// A bundle's bytes as text. Invalid UTF-8 is an I/O `InvalidData` error, exactly +/// as `read_to_string` reported it. +fn bundle_text(bytes: &[u8]) -> Result<&str> { + std::str::from_utf8(bytes) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e).into()) +} + +/// How a bundle is to change. Bundles are only ever APPENDED to (or, to re-key, have +/// our own trailing stub swapped), so the change is expressed as a tail rather +/// than as a whole new file. +struct JsEdit { + /// Cut the file back to this many bytes first (dropping a stale stub of ours). + truncate_to: Option, + /// Then append this. + append: String, +} + +/// What `inject_one` decided for a bundle, for the stats and the log line. +enum JsOutcome { + Restamped { stale: String, fresh: Uuid }, + Registered { id: String }, + Already, + Injected { id: Uuid }, +} + +/// Apply `edit` to the bundle IN PLACE: no copy of the bundle is made or held, +/// and the file keeps its inode, permissions and any symlink (as `fs::write` +/// did, which also truncated and rewrote through the same path). +fn apply_js_edit(js_path: &Path, edit: &JsEdit) -> Result<()> { + use std::io::{Seek, SeekFrom, Write}; + let mut file = std::fs::OpenOptions::new().write(true).open(js_path)?; + if let Some(len) = edit.truncate_to { + file.set_len(len)?; + } + file.seek(SeekFrom::End(0))?; + file.write_all(edit.append.as_bytes())?; + Ok(()) +} + fn inject_one(js_path: &Path, dry_run: bool, stats: &mut InjectStats) -> Result<()> { - let content = std::fs::read_to_string(js_path)?; - let map_path = paired_map(js_path, &content); - - let (debug_id, freshly_computed) = match existing_debug_id(&content) { - Some(id) => match restamp_id(&content, &id, map_path.as_deref())? { - // Our stub, over a map regenerated with different content: re-key the - // bundle, or the upload dedups against the STALE map (webpack keeps a - // `[contenthash]` JS file it considers unchanged but re-emits its map). - Some((original, fresh)) => { - if !dry_run { - std::fs::write(js_path, format!("{original}{}", runtime_stub(&fresh)))?; - } - stats.js_restamped += 1; - tracing::info!( - path = %js_path.display(), - stale = %id, - debug_id = %fresh, - "re-stamped debug-id: the bundle's map was regenerated" - ); - (fresh.to_string(), true) - } - // Another tool's id (Rollup 4 `sourcemapDebugIds` writes its own comment): - // keep the id β€” its map already carries it β€” but add our runtime - // registration, without which the SDK cannot attach it to a crash frame. - None if !content.contains(&runtime_registration(&id)) => { - if !dry_run { - std::fs::write(js_path, format!("{content}{}", runtime_registration(&id)))?; + // Decide everything with the bundle mapped read-only (zero-copy `&str`), then + // DROP the mapping before touching the file: Windows refuses to truncate a + // file that has a mapped view open. + let (map_path, debug_id, freshly_computed, outcome, edit) = { + let mapped = crate::symbols::mapped::map_file(js_path)?; + let content = bundle_text(&mapped)?; + let map_path = paired_map(js_path, content); + + match existing_debug_id(content) { + Some(id) => match restamp_id(content, &id, map_path.as_deref())? { + // Our stub, over a map regenerated with different content: re-key the + // bundle, or the upload dedups against the STALE map (webpack keeps a + // `[contenthash]` JS file it considers unchanged but re-emits its map). + Some((original, fresh)) => ( + map_path, + fresh.to_string(), + true, + JsOutcome::Restamped { stale: id, fresh }, + Some(JsEdit { + truncate_to: Some(original.len() as u64), + append: runtime_stub(&fresh), + }), + ), + // Another tool's id (Rollup 4 `sourcemapDebugIds` writes its own comment): + // keep the id β€” its map already carries it β€” but add our runtime + // registration, without which the SDK cannot attach it to a crash frame. + None if !content.contains(&runtime_registration(&id)) => { + let edit = JsEdit { + truncate_to: None, + append: runtime_registration(&id), + }; + ( + map_path, + id.clone(), + false, + JsOutcome::Registered { id }, + Some(edit), + ) } - stats.js_registered += 1; - tracing::info!( - path = %js_path.display(), - debug_id = %id, - "registered an existing debug-id another tool wrote" - ); - (id, false) - } + None => (map_path, id, false, JsOutcome::Already, None), + }, None => { - stats.js_already += 1; - (id, false) - } - }, - None => { - let id = match map_path.as_deref() { - Some(m) => compute_debug_id_with_map_file(content.as_bytes(), m)?, - None => compute_debug_id(content.as_bytes()), - }; - if !dry_run { - std::fs::write(js_path, format!("{content}{}", runtime_stub(&id)))?; + let id = match map_path.as_deref() { + Some(m) => compute_debug_id_with_map_file(content.as_bytes(), m)?, + None => compute_debug_id(content.as_bytes()), + }; + ( + map_path, + id.to_string(), + true, + JsOutcome::Injected { id }, + Some(JsEdit { + truncate_to: None, + append: runtime_stub(&id), + }), + ) } + } + }; + + if let (Some(edit), false) = (&edit, dry_run) { + apply_js_edit(js_path, edit)?; + } + match outcome { + JsOutcome::Restamped { stale, fresh } => { + stats.js_restamped += 1; + tracing::info!( + path = %js_path.display(), + stale = %stale, + debug_id = %fresh, + "re-stamped debug-id: the bundle's map was regenerated" + ); + } + JsOutcome::Registered { id } => { + stats.js_registered += 1; + tracing::info!( + path = %js_path.display(), + debug_id = %id, + "registered an existing debug-id another tool wrote" + ); + } + JsOutcome::Already => stats.js_already += 1, + JsOutcome::Injected { id } => { stats.js_injected += 1; tracing::info!(path = %js_path.display(), debug_id = %id, "injected debug-id"); - (id.to_string(), true) } - }; + } if let Some(map_path) = map_path { if write_map_debug_id(&map_path, &debug_id, freshly_computed, dry_run)? { @@ -1764,3 +1837,289 @@ mod tests { assert_eq!(read_debug_id(&p).unwrap(), None); } } + +/// What `inject` does to files on disk: exact bytes, in-place edits, what survives +/// (permissions, symlinks), what must NOT change (dry runs, rejected input) and +/// what must not be left behind (temp files). +#[cfg(test)] +mod file_edits { + use super::*; + + fn inject_dir(dir: &Path) -> Result { + inject_paths(&[dir.to_path_buf()], &[], true, false) + } + + fn names(dir: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + v.sort(); + v + } + + const MAP: &str = r#"{"version":3,"sources":["a.ts"],"names":["x"],"mappings":"AAAA"}"#; + + /// Whatever the bundle ends with (or doesn't), injection is EXACTLY + /// `bundle || stub(id)` β€” nothing re-encoded, nothing normalised β€” and a second + /// run is a byte-for-byte no-op. + #[test] + fn appends_exactly_the_stub_to_awkward_bundles() { + let big = format!("{}\n", "var a=1;".repeat(30_000)); + let cases: Vec<(&str, String)> = vec![ + ("empty", String::new()), + ("no trailing newline", "a()".into()), + ("trailing newline", "a()\n".into()), + ("CRLF", "a()\r\nb()\r\n".into()), + ("multibyte", "var s = 'hΓ©llo β˜ƒ πŸ˜€';\n".into()), + ("BOM first", "\u{feff}a()\n".into()), + ("one huge line", big), + ]; + for (label, content) in cases { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + std::fs::write(&js, &content).unwrap(); + + inject_dir(dir.path()).unwrap(); + let id = compute_debug_id(content.as_bytes()); + assert_eq!( + std::fs::read(&js).unwrap(), + format!("{content}{}", runtime_stub(&id)).into_bytes(), + "{label}" + ); + + let once = std::fs::read(&js).unwrap(); + inject_dir(dir.path()).unwrap(); + assert_eq!(std::fs::read(&js).unwrap(), once, "{label}: not idempotent"); + } + } + + #[test] + fn with_a_map_the_id_covers_both_and_both_files_are_updated() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + let content = "a()\n//# sourceMappingURL=a.js.map\n"; + std::fs::write(&js, content).unwrap(); + std::fs::write(&map, MAP).unwrap(); + + let stats = inject_dir(dir.path()).unwrap(); + assert_eq!((stats.js_injected, stats.maps_updated), (1, 1)); + + let id = compute_debug_id_with_map(content.as_bytes(), Some(MAP.as_bytes())); + assert_eq!( + std::fs::read_to_string(&js).unwrap(), + format!("{content}{}", runtime_stub(&id)) + ); + let got: serde_json::Value = serde_json::from_slice(&std::fs::read(&map).unwrap()).unwrap(); + let mut expected: serde_json::Value = serde_json::from_str(MAP).unwrap(); + expected["debug_id"] = id.to_string().into(); + expected["debugId"] = id.to_string().into(); + assert_eq!(got, expected, "every other key survives the rewrite"); + assert_eq!(names(dir.path()), ["a.js", "a.js.map"], "no temp file left"); + } + + #[test] + fn a_map_with_unicode_and_escapes_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let map = dir.path().join("a.js.map"); + let original = r#"{"version":3,"sources":["src/hΓ©llo β˜ƒ.ts","q\"uote"],"sourcesContent":["line1\nline2\t\\ πŸ˜€"],"names":[],"mappings":"AAAA","x_google_ignoreList":[0],"n":1.50}"#; + std::fs::write(dir.path().join("a.js"), "a()\n").unwrap(); + std::fs::write(&map, original).unwrap(); + inject_dir(dir.path()).unwrap(); + + let got: serde_json::Value = serde_json::from_slice(&std::fs::read(&map).unwrap()).unwrap(); + let mut expected: serde_json::Value = serde_json::from_str(original).unwrap(); + let id = got["debug_id"].as_str().unwrap().to_string(); + expected["debug_id"] = id.clone().into(); + expected["debugId"] = id.into(); + assert_eq!(got, expected); + assert!( + std::fs::read_to_string(&map).unwrap().contains("1.50"), + "number text is kept, not round-tripped through f64" + ); + } + + /// Re-keying swaps ONLY our own trailing stub: the bundle is cut back to the + /// original bytes and the new stub appended β€” no remnant of the old id. + #[test] + fn restamp_cuts_back_to_the_original_bytes_before_appending() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + let content = "function f(){return 1}\n//# sourceMappingURL=a.js.map\n"; + std::fs::write(&js, content).unwrap(); + std::fs::write(&map, MAP).unwrap(); + inject_dir(dir.path()).unwrap(); + let first = std::fs::read_to_string(&js).unwrap(); + let old_id = existing_debug_id(&first).unwrap(); + + // The bundler regenerates the map with different content (and no ids). + let regenerated = r#"{"version":3,"sources":["b.ts"],"names":[],"mappings":"BBBB"}"#; + std::fs::write(&map, regenerated).unwrap(); + let stats = inject_dir(dir.path()).unwrap(); + assert_eq!(stats.js_restamped, 1); + + let new_id = compute_debug_id_with_map(content.as_bytes(), Some(regenerated.as_bytes())); + assert_ne!(new_id.to_string(), old_id); + let after = std::fs::read_to_string(&js).unwrap(); + assert_eq!(after, format!("{content}{}", runtime_stub(&new_id))); + assert!(!after.contains(&old_id), "no remnant of the stale id"); + assert!( + std::fs::read_to_string(&map) + .unwrap() + .contains(&new_id.to_string()), + "the regenerated map carries the new id" + ); + } + + /// A bundle another tool already stamped keeps its bytes and only GAINS our registration. + #[test] + fn registering_a_foreign_id_only_appends() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let id = "11111111-1111-1111-1111-111111111111"; + let content = format!("a()\n//# debugId={id}\n//# sourceMappingURL=a.js.map\n"); + std::fs::write(&js, &content).unwrap(); + std::fs::write( + dir.path().join("a.js.map"), + format!(r#"{{"version":3,"debug_id":"{id}","debugId":"{id}","mappings":""}}"#), + ) + .unwrap(); + let stats = inject_dir(dir.path()).unwrap(); + assert_eq!(stats.js_registered, 1); + assert_eq!( + std::fs::read_to_string(&js).unwrap(), + format!("{content}{}", runtime_registration(&id)) + ); + } + + #[test] + fn a_dry_run_changes_no_byte_and_no_mtime() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + std::fs::write(&js, "a()\n").unwrap(); + std::fs::write(&map, MAP).unwrap(); + let snap = |p: &Path| { + ( + std::fs::read(p).unwrap(), + std::fs::metadata(p).unwrap().modified().unwrap(), + ) + }; + let (js0, map0) = (snap(&js), snap(&map)); + let stats = inject_paths(&[dir.path().to_path_buf()], &[], true, true).unwrap(); + assert_eq!( + (stats.js_injected, stats.maps_updated), + (1, 1), + "it still reports" + ); + assert_eq!(snap(&js), js0); + assert_eq!(snap(&map), map0); + assert_eq!( + names(dir.path()), + ["a.js", "a.js.map"], + "dry run leaves no temp file" + ); + } + + #[test] + fn a_bundle_that_is_not_utf8_is_an_io_error_and_untouched() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + std::fs::write(&js, b"a()\xff\xfe\n").unwrap(); + let err = inject_dir(dir.path()).unwrap_err(); + assert!(matches!(err, Error::Io(_)), "{err:?}"); + assert_eq!(std::fs::read(&js).unwrap(), b"a()\xff\xfe\n"); + } + + /// A map that is not a JSON object cannot take an id: the run fails and the map + /// is left exactly as it was, with no temp file beside it. + #[test] + fn an_unusable_map_is_left_untouched() { + for (label, map) in [ + ("not json", "{not json"), + ("an array", "[]"), + ("truncated", r#"{"version":3,"mappings":"AA"#), + ("trailing junk", r#"{"version":3} x"#), + ] { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("a.js"), "a()\n").unwrap(); + let mp = dir.path().join("a.js.map"); + std::fs::write(&mp, map).unwrap(); + let err = inject_dir(dir.path()).unwrap_err(); + assert!(matches!(err, Error::InputInvalid(_)), "{label}: {err:?}"); + assert_eq!(std::fs::read_to_string(&mp).unwrap(), map, "{label}"); + assert_eq!(names(dir.path()), ["a.js", "a.js.map"], "{label}"); + } + } + + #[cfg(unix)] + mod unix { + use super::*; + use std::os::unix::fs::{symlink, PermissionsExt}; + + fn mode(p: &Path) -> u32 { + std::fs::metadata(p).unwrap().permissions().mode() & 0o777 + } + + #[test] + fn permissions_survive_on_both_files() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + std::fs::write(&js, "a()\n").unwrap(); + std::fs::write(&map, MAP).unwrap(); + std::fs::set_permissions(&js, std::fs::Permissions::from_mode(0o640)).unwrap(); + std::fs::set_permissions(&map, std::fs::Permissions::from_mode(0o600)).unwrap(); + inject_dir(dir.path()).unwrap(); + assert_eq!(mode(&js), 0o640); + assert_eq!(mode(&map), 0o600, "the map rewrite must not reset its mode"); + } + + #[test] + fn a_symlinked_bundle_and_map_stay_links_and_their_targets_change() { + let dir = tempfile::tempdir().unwrap(); + let real = tempfile::tempdir().unwrap(); + std::fs::write(real.path().join("real.js"), "a()\n").unwrap(); + std::fs::write(real.path().join("real.js.map"), MAP).unwrap(); + symlink(real.path().join("real.js"), dir.path().join("a.js")).unwrap(); + symlink(real.path().join("real.js.map"), dir.path().join("a.js.map")).unwrap(); + + inject_dir(dir.path()).unwrap(); + for l in ["a.js", "a.js.map"] { + assert!( + std::fs::symlink_metadata(dir.path().join(l)) + .unwrap() + .file_type() + .is_symlink(), + "{l} must still be a symlink" + ); + } + assert!(std::fs::read_to_string(real.path().join("real.js")) + .unwrap() + .contains("//# debugId=")); + assert!(std::fs::read_to_string(real.path().join("real.js.map")) + .unwrap() + .contains("\"debug_id\"")); + assert_eq!( + names(real.path()), + ["real.js", "real.js.map"], + "no temp file" + ); + } + + #[test] + fn a_read_only_bundle_is_an_error_and_stays_as_it_was() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + std::fs::write(&js, "a()\n").unwrap(); + std::fs::set_permissions(&js, std::fs::Permissions::from_mode(0o444)).unwrap(); + if std::fs::OpenOptions::new().write(true).open(&js).is_ok() { + return; // running as root: permissions are not enforced + } + assert!(matches!(inject_dir(dir.path()), Err(Error::Io(_)))); + assert_eq!(std::fs::read(&js).unwrap(), b"a()\n"); + } + } +} From 32cc797554c0fc4903217200d8d3316ecfe91f8f Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:16:49 +0500 Subject: [PATCH 10/18] test: file handling through the binary, packer/mmap/stream units, memory budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/file_handling.rs: exact bytes on disk after inject (idempotent, dry-run, stdout empty), large artefact PUT intact with exact Content-Length, directory libraries uploaded with their own bytes + build-ids, --strip-sources-content uploads a valid stripped copy and spares the original, ProGuard raw-byte identity (empty / CRLF / non-UTF-8 / large), zero-byte library in a scan - compress: every method round-trips sizes around the copy buffer with CRC verification, missing source is an error, a 12 MB source packs by streaming - symbols::mapped: directory, page-straddling sizes, permission denied - upload::http::file_body: every byte around the chunk size, missing file and a file shorter than its declared length fail the request - scripts/e2e_flows.py: peak-RSS budget (64 MB) for each large-input flow on ~96 MB inputs, measured in a throwaway wrapper process; 5 of 6 flows fail it on the pre-streaming binary (126-222 MB), so it has teeth. Skipped on Windows. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- scripts/e2e_flows.py | 129 ++++++++++ src/compress/mod.rs | 102 ++++++++ src/symbols/mapped.rs | 39 ++++ src/upload/http.rs | 75 ++++++ tests/file_handling.rs | 520 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 865 insertions(+) create mode 100644 tests/file_handling.rs diff --git a/scripts/e2e_flows.py b/scripts/e2e_flows.py index 4cd6e83..7534bc6 100644 --- a/scripts/e2e_flows.py +++ b/scripts/e2e_flows.py @@ -324,6 +324,132 @@ def run(binpath, flow, args, expect_code=0, expect_stderr=None): return ok +# --- Memory regression ------------------------------------------------------------------- +# +# The CLI handles artefacts and symbol files of hundreds of MB and must do it in +# bounded memory: stream, or map and look at headers, never hold a whole file (or +# two) on the heap. A regression to `fs::read` is invisible to every functional +# check, so run each large-input flow in a fresh process and bound its PEAK RSS. +# +# Inputs are ~96 MB; every flow must stay under MEM_LIMIT_MB. The budget is far above +# the streaming cost (~30-40 MB: runtime + zstd encoders) and far below any whole-file +# read (>= the input, doubled by a per-retry clone), so it is not flaky either way. +MEM_INPUT_MB = 96 +MEM_LIMIT_MB = 64 + +_RSS_WRAPPER = ( + "import resource, subprocess, sys\n" + "r = subprocess.run(sys.argv[1:], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n" + "print(r.returncode, resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss)\n" +) + + +def peak_rss_mb(argv, env): + """(exit code, peak RSS in MB) of `argv`, measured in a throwaway wrapper process so + RUSAGE_CHILDREN's maximum belongs to this command alone. None where RSS is unavailable.""" + if sys.platform == "win32": + return None + r = subprocess.run([sys.executable, "-c", _RSS_WRAPPER] + argv, capture_output=True, + text=True, env=env) + try: + rc, raw = r.stdout.split() + raw = int(raw) + except ValueError: + return None + # ru_maxrss is bytes on macOS, kilobytes on Linux. + return int(rc), raw / (1024 * 1024 if sys.platform == "darwin" else 1024) + + +def _write_big(path, total_mb, block): + with open(path, "wb") as f: + for _ in range(total_mb): + f.write(block()) + + +def memory_checks(binpath, fix, results): + if sys.platform == "win32": + print(" [SKIP] memory checks (no RSS accounting on Windows)") + return + mem = os.path.join(fix, "memory") + os.makedirs(mem, exist_ok=True) + env = dict(os.environ, BUGSEE_APP_TOKEN=TOKEN) + endpoint = f"http://127.0.0.1:{STATE['port']}" + base = [binpath, "--endpoint", endpoint, "--app-token", TOKEN] + mb = 1024 * 1024 + + # Incompressible artefact (stored in the upload zip, so the PUT is ~its size). + artefact = os.path.join(mem, "app.aab") + _write_big(artefact, MEM_INPUT_MB, lambda: os.urandom(mb)) + payload = os.path.join(mem, "p.json") + with open(payload, "w") as f: + json.dump({"uuid": "mem-build", "version": "1.0"}, f) + # A text mapping (compressible, like the real thing). + pg = os.path.join(mem, "pg") + os.makedirs(pg, exist_ok=True) + line = b"com.example.pkg.ClassNameNumber -> a.b:\n int fieldName -> a\n" + _write_big(os.path.join(pg, "mapping.txt"), MEM_INPUT_MB, lambda: line * (mb // len(line))) + # Native libraries: the real ELF fixture (it has a GNU build-id) padded to ~32 MB each, + # three ABIs with distinct build-ids, in AGP's merged_native_libs layout. + fixture = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tests", + "fixtures", "elf", "libsymbol1.so") + elf = open(fixture, "rb").read() + bid = bytes.fromhex("bca64abfec40dbb631bb8f1c37414472") + pos = elf.index(bid) + for i, abi in enumerate(["arm64-v8a", "armeabi-v7a", "x86_64"]): + d = os.path.join(mem, "libs", "lib", abi) + os.makedirs(d, exist_ok=True) + lib = bytearray(elf) + lib[pos] ^= i + 1 + with open(os.path.join(d, "libbig.so"), "wb") as f: + f.write(bytes(lib)) + for _ in range(MEM_INPUT_MB // 3): + f.write(os.urandom(mb)) + # A ~64 MB source map: a small `mappings` and a large `sourcesContent`. + maps = os.path.join(mem, "maps") + os.makedirs(maps, exist_ok=True) + with open(os.path.join(maps, "app.js.map"), "w") as f: + f.write('{"version":3,"debug_id":"44444444-4444-4444-4444-444444444444",' + '"sources":["a.ts"],"names":[],"mappings":"AAAA;","sourcesContent":["') + chunk = "x" * mb + for _ in range(MEM_INPUT_MB * 2 // 3): + f.write(chunk) + f.write('"]}') + # `sourcemaps inject` over the same map (the bundle stays small). + inj = os.path.join(mem, "inject") + os.makedirs(inj, exist_ok=True) + shutil.copy(os.path.join(maps, "app.js.map"), os.path.join(inj, "app.js.map")) + with open(os.path.join(inj, "app.js"), "w") as f: + f.write("console.log(1);\n") + + flows = [ + ("mem_artefact_put", base + ["upload", "build", "--payload-json", payload, + "--artifact", artefact]), + ("mem_proguard", base + ["debug-files", "upload", "--type", "proguard", + "--version", "1", "--build", "1", pg]), + ("mem_elf_directory", base + ["debug-files", "upload", "--type", "elf", + "--version", "1", "--build", "1", + "--uuid", "11111111-2222-3333-4444-555555555555", + os.path.join(mem, "libs")]), + ("mem_sourcemap", base + ["debug-files", "upload", "--type", "sourcemaps", + "--version", "1", "--build", "1", maps]), + ("mem_sourcemap_strip", base + ["debug-files", "upload", "--type", "sourcemaps", + "--strip-sources-content", "--version", "1", + "--build", "1", maps]), + ("mem_inject", [binpath, "sourcemaps", "inject", inj]), + ] + for flow, argv in flows: + STATE["flow"] = flow + got = peak_rss_mb(argv, env) + if got is None: + print(f" [SKIP] {flow} (no RSS available)") + continue + rc, rss = got + ok = rc == 0 and rss < MEM_LIMIT_MB + print(f" [{'PASS' if ok else 'FAIL'}] {flow}: exit {rc}, peak RSS {rss:.0f} MB " + f"(budget {MEM_LIMIT_MB} MB, input ~{MEM_INPUT_MB} MB)") + results[flow + "_bounded_memory"] = ok + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--bin", required=True, help="path to bugsee-cli[.exe]") @@ -597,6 +723,9 @@ def main(): print(" [FAIL] could not verify the register-only build:", e) results["build_register_only_ships_no_bytes"] = False + print("\n=== MEMORY ===") + memory_checks(binpath, fix, results) + srv.shutdown() if not a.keep: shutil.rmtree(work, ignore_errors=True) diff --git a/src/compress/mod.rs b/src/compress/mod.rs index 2aa5cc0..176e5a9 100644 --- a/src/compress/mod.rs +++ b/src/compress/mod.rs @@ -356,6 +356,108 @@ mod tests { } } + /// Sizes straddling the 64 KiB copy buffer (and empty): every entry must come + /// back byte-identical with a valid CRC, for each compression method. + #[test] + fn every_method_round_trips_sizes_around_the_copy_buffer() { + let dir = tempfile::tempdir().unwrap(); + for len in [0usize, 1, 65_535, 65_536, 65_537, 131_072, 300_001] { + // Patterned bytes: not a constant run (which would compress away any + // chunking bug) and not text. + let data: Vec = (0..len).map(|i| ((i * 31 + i / 251) % 256) as u8).collect(); + let src = dir.path().join("src.bin"); + std::fs::write(&src, &data).unwrap(); + for (label, entry, strategy) in [ + ( + "zstd", + ZipEntry::compressed("e.bin", &src), + Strategy::Zstd(3), + ), + ( + "deflate", + ZipEntry::compressed("e.bin", &src), + Strategy::Deflate, + ), + ("stored", ZipEntry::stored("e.bin", &src), Strategy::Zstd(3)), + ] { + let out = dir.path().join("out.zip"); + let size = pack_entries(&[entry], &out, strategy).unwrap(); + assert_eq!( + size, + std::fs::metadata(&out).unwrap().len(), + "{label} {len}" + ); + let mut archive = ZipArchive::new(File::open(&out).unwrap()).unwrap(); + let mut e = archive.by_name("e.bin").unwrap(); + let mut got = Vec::new(); + // `read_to_end` verifies the CRC-32 at EOF: corruption is an Err here. + e.read_to_end(&mut got).unwrap(); + assert_eq!(got, data, "{label} {len}"); + } + } + } + + #[test] + fn a_missing_source_is_an_error_not_an_empty_entry() { + let dir = tempfile::tempdir().unwrap(); + let out = dir.path().join("out.zip"); + let missing = dir.path().join("nope"); + let err = pack_entries( + &[ZipEntry::compressed("e", &missing)], + &out, + Strategy::default(), + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); + } + + /// The archive is built by streaming: the bytes of a source that is larger than + /// anything we would want in memory come out right, with a bounded buffer. + #[test] + fn a_large_source_packs_by_streaming() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("big.bin"); + { + let mut f = BufWriter::new(File::create(&src).unwrap()); + let block: Vec = (0..1_000_003u32).map(|i| (i % 253) as u8).collect(); + for _ in 0..12 { + f.write_all(&block).unwrap(); // ~12 MB + } + } + let out = dir.path().join("big.zip"); + pack_entries( + &[ZipEntry::stored("big.bin", &src)], + &out, + Strategy::default(), + ) + .unwrap(); + let mut archive = ZipArchive::new(File::open(&out).unwrap()).unwrap(); + let mut e = archive.by_name("big.bin").unwrap(); + assert_eq!(e.size(), 12_000_036); + use sha1::Digest; + let mut hasher_in = sha1::Sha1::new(); + let mut hasher_out = sha1::Sha1::new(); + { + let mut f = File::open(&src).unwrap(); + let mut buf = [0u8; 8192]; + loop { + let n = f.read(&mut buf).unwrap(); + if n == 0 { + break; + } + hasher_in.update(&buf[..n]); + } + loop { + let n = e.read(&mut buf).unwrap(); + if n == 0 { + break; + } + hasher_out.update(&buf[..n]); + } + assert_eq!(hasher_in.finalize(), hasher_out.finalize()); + } + } + #[test] fn resolve_strategy_applies_production_floor_and_flag_rules() { // Default is zstd at the documented default level. diff --git a/src/symbols/mapped.rs b/src/symbols/mapped.rs index 3027577..71a10a5 100644 --- a/src/symbols/mapped.rs +++ b/src/symbols/mapped.rs @@ -49,4 +49,43 @@ mod tests { assert!(map_file(&p).unwrap().is_empty()); assert!(map_file(&dir.path().join("missing")).is_err()); } + + #[test] + fn a_directory_is_an_error_not_a_panic_or_empty_bytes() { + // `File::open` succeeds on a directory on Unix; mapping it must not. + let dir = tempfile::tempdir().unwrap(); + assert!(map_file(dir.path()).is_err()); + } + + #[test] + fn large_and_page_straddling_files_map_whole() { + let dir = tempfile::tempdir().unwrap(); + for len in [4095usize, 4096, 4097, 1_000_003] { + let data: Vec = (0..len).map(|i| (i % 251) as u8).collect(); + let p = dir.path().join("f"); + std::fs::write(&p, &data).unwrap(); + let m = map_file(&p).unwrap(); + assert_eq!(m.len(), len); + assert_eq!(&*m, &data[..]); + } + } + + #[cfg(unix)] + #[test] + fn an_unreadable_file_is_a_permission_error() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("f"); + std::fs::write(&p, b"secret").unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o000)).unwrap(); + let readable = std::fs::File::open(&p).is_ok(); // root ignores modes + let r = map_file(&p); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o644)).unwrap(); + if !readable { + assert_eq!( + r.err().unwrap().kind(), + std::io::ErrorKind::PermissionDenied + ); + } + } } diff --git a/src/upload/http.rs b/src/upload/http.rs index 4bef074..f4f5ab2 100644 --- a/src/upload/http.rs +++ b/src/upload/http.rs @@ -267,6 +267,81 @@ mod tests { use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; + async fn put_file( + server: &MockServer, + path: &std::path::Path, + declared_len: u64, + ) -> reqwest::Result { + reqwest::Client::new() + .put(format!("{}/put", server.uri())) + .header(reqwest::header::CONTENT_LENGTH, declared_len) + .body(file_body(path)) + .send() + .await + } + + /// Sizes around the 64 KiB chunk: the streamed body is the file, byte for byte, + /// with the length the caller declared and no chunked encoding. + #[tokio::test] + async fn file_body_streams_every_byte_around_the_chunk_size() { + let dir = tempfile::tempdir().unwrap(); + for len in [0usize, 1, 65_535, 65_536, 65_537, 200_000] { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let data: Vec = (0..len).map(|i| (i % 251) as u8).collect(); + let p = dir.path().join("payload"); + std::fs::write(&p, &data).unwrap(); + let declared = file_len(&p).await.unwrap(); + assert_eq!(declared, len as u64); + + put_file(&server, &p, declared).await.unwrap(); + let req = &server.received_requests().await.unwrap()[0]; + assert_eq!(req.body, data, "len {len}"); + assert_eq!( + req.headers.get("content-length").unwrap().to_str().unwrap(), + len.to_string() + ); + assert!(req.headers.get("transfer-encoding").is_none(), "len {len}"); + } + } + + /// A payload that vanished (or cannot be read) fails the request instead of + /// sending an empty body that the server would store as a valid, empty symbol. + #[tokio::test] + async fn file_body_of_a_missing_file_fails_the_request() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let dir = tempfile::tempdir().unwrap(); + let result = put_file(&server, &dir.path().join("gone"), 10).await; + assert!(result.is_err(), "must not succeed with an empty body"); + } + + /// The file shrank after its length was read (a concurrent writer): the body + /// ends short of the declared Content-Length, which must surface as a failed + /// request, never as a "successful" upload of a truncated symbol. + #[tokio::test] + async fn file_body_shorter_than_its_declared_length_fails_the_request() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("payload"); + std::fs::write(&p, vec![7u8; 1000]).unwrap(); + let result = put_file(&server, &p, 5000).await; + assert!( + result.is_err(), + "a truncated body must not look like success" + ); + } + #[test] fn backoff_doubles_and_caps() { let p = RetryPolicy { diff --git a/tests/file_handling.rs b/tests/file_handling.rs new file mode 100644 index 0000000..bb2ad6a --- /dev/null +++ b/tests/file_handling.rs @@ -0,0 +1,520 @@ +//! File reading, processing and output, end to end through the COMPILED binary. +//! +//! The unit tests pin each piece; these pin what an integrator can observe: the +//! exact bytes left on disk, the exact bytes sent over the wire (including large +//! payloads that must stream), stdout staying reserved for structured output, and +//! exit codes for unusable input. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use serde_json::{json, Value}; +use sha1::{Digest, Sha1}; +use wiremock::matchers::{method, path as wm_path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +mod common; + +const FIXTURE_ELF: &str = "tests/fixtures/elf/libsymbol1.so"; +const FIXTURE_BUILD_ID: &str = "bca64abfec40dbb631bb8f1c37414472"; + +fn write(dir: &Path, rel: &str, bytes: &[u8]) -> PathBuf { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, bytes).unwrap(); + p +} + +fn sha1_hex(bytes: &[u8]) -> String { + hex::encode(Sha1::digest(bytes)) +} + +/// Patterned, non-constant bytes (a constant run would hide a chunking bug). +fn pattern(len: usize) -> Vec { + (0..len).map(|i| ((i * 31 + i / 251) % 256) as u8).collect() +} + +/// The entries of an uploaded ZIP, decompressed, in order. +fn zip_entries(zip_bytes: &[u8]) -> Vec<(String, Vec)> { + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(zip_bytes.to_vec())).unwrap(); + (0..archive.len()) + .map(|i| { + let mut e = archive.by_index(i).unwrap(); + let mut data = Vec::new(); + e.read_to_end(&mut data).unwrap(); // verifies the CRC + (e.name().to_string(), data) + }) + .collect() +} + +async fn requests(server: &MockServer, verb: &str) -> Vec { + server + .received_requests() + .await + .unwrap() + .into_iter() + .filter(|r| r.method.as_str() == verb) + .collect() +} + +// --------------------------------------------------------------------------- +// `sourcemaps inject`: bytes on disk, idempotence, stdout discipline, exit codes +// --------------------------------------------------------------------------- + +const MAP: &str = r#"{"version":3,"sources":["a.ts"],"names":[],"mappings":"AAAA"}"#; + +#[test] +fn inject_stamps_files_idempotently_and_keeps_stdout_empty() { + let tmp = tempfile::tempdir().unwrap(); + let js = write(tmp.path(), "dist/app.js", b"console.log(1);\n"); + let map = write(tmp.path(), "dist/app.js.map", MAP.as_bytes()); + let dist = tmp.path().join("dist"); + + let out = common::cli() + .args(["sourcemaps", "inject"]) + .arg(&dist) + .assert() + .success() + .stdout("") // stdout is reserved for structured output + .get_output() + .clone(); + assert!( + String::from_utf8_lossy(&out.stderr).contains("inject"), + "progress goes to stderr" + ); + + let stamped_js = std::fs::read_to_string(&js).unwrap(); + assert!( + stamped_js.starts_with("console.log(1);\n"), + "original bytes are a prefix" + ); + assert!(stamped_js.contains("//# debugId=")); + let map_json: Value = serde_json::from_slice(&std::fs::read(&map).unwrap()).unwrap(); + assert_eq!(map_json["debug_id"], map_json["debugId"]); + assert_eq!(map_json["mappings"], "AAAA"); + + // Second run: no byte changes at all. + let (js1, map1) = (std::fs::read(&js).unwrap(), std::fs::read(&map).unwrap()); + common::cli() + .args(["sourcemaps", "inject"]) + .arg(&dist) + .assert() + .success(); + assert_eq!(std::fs::read(&js).unwrap(), js1); + assert_eq!(std::fs::read(&map).unwrap(), map1); +} + +#[test] +fn inject_dry_run_reports_but_writes_nothing() { + let tmp = tempfile::tempdir().unwrap(); + let js = write(tmp.path(), "app.js", b"a()\n"); + let map = write(tmp.path(), "app.js.map", MAP.as_bytes()); + common::cli() + .args(["sourcemaps", "inject", "--dry-run"]) + .arg(tmp.path()) + .assert() + .success() + .stdout(""); + assert_eq!(std::fs::read(&js).unwrap(), b"a()\n"); + assert_eq!(std::fs::read(&map).unwrap(), MAP.as_bytes()); + let mut left: Vec<_> = std::fs::read_dir(tmp.path()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + assert_eq!(left, ["app.js", "app.js.map"]); +} + +#[test] +fn inject_rejects_an_unusable_map_and_leaves_it_alone() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "app.js", b"a()\n"); + let map = write(tmp.path(), "app.js.map", b"{ not json"); + common::cli() + .args(["sourcemaps", "inject"]) + .arg(tmp.path()) + .assert() + .code(11) // InputInvalid + .stderr(predicates::str::contains("not valid JSON")); + assert_eq!(std::fs::read(&map).unwrap(), b"{ not json"); +} + +// --------------------------------------------------------------------------- +// stdout discipline for `debug-files upload` +// --------------------------------------------------------------------------- + +#[test] +fn upload_dry_runs_write_nothing_to_stdout_for_every_file_type() { + let tmp = tempfile::tempdir().unwrap(); + let elf = Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE_ELF); + write( + tmp.path(), + "native/lib/arm64-v8a/libsymbol1.so", + &std::fs::read(elf).unwrap(), + ); + write(tmp.path(), "proguard/mapping.txt", b"a.B -> c.D:\n"); + write( + tmp.path(), + "maps/app.js.map", + br#"{"version":3,"debug_id":"22222222-2222-2222-2222-222222222222","mappings":"AAAA"}"#, + ); + for (kind, dir) in [ + ("elf", "native"), + ("proguard", "proguard"), + ("sourcemaps", "maps"), + ] { + common::cli() + .args([ + "debug-files", + "upload", + "--type", + kind, + "--version", + "1", + "--build", + "1", + ]) + .args([ + "--uuid", + "00000000-0000-0000-0000-000000000000", + "--dry-run", + ]) + .arg(tmp.path().join(dir)) + .assert() + .success() + .stdout(""); + } +} + +// --------------------------------------------------------------------------- +// What goes over the wire for large payloads +// --------------------------------------------------------------------------- + +/// The artefact is the biggest thing the CLI sends: it must arrive intact, with an +/// exact Content-Length (no chunked body), whatever its size relative to the stream +/// chunk β€” here 5 MB, STORED inside the upload ZIP. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_large_artefact_arrives_intact_with_an_exact_content_length() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let artefact = pattern(5 * 1024 * 1024 + 17); + let artifact_path = write(tmp.path(), "app.aab", &artefact); + let payload = write(tmp.path(), "p.json", br#"{"uuid":"abc","version":"1.0"}"#); + let put_url = format!("{}/artefact-put", server.uri()); + + Mock::given(method("POST")) + .and(wm_path("/v2/apps/TKN/builds")) + .respond_with( + ResponseTemplate::new(200).set_body_json( + json!({"ok": true, "result": {"build_id": "b1", "endpoint": put_url}}), + ), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(wm_path("/artefact-put")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + let endpoint = server.uri(); + tokio::task::spawn_blocking(move || { + common::cli() + .args([ + "--endpoint", + &endpoint, + "--app-token", + "TKN", + "upload", + "build", + ]) + .arg("--payload-json") + .arg(&payload) + .arg("--artifact") + .arg(&artifact_path) + .assert() + .success(); + }) + .await + .unwrap(); + + let put = &requests(&server, "PUT").await[0]; + assert_eq!( + put.headers.get("content-length").unwrap().to_str().unwrap(), + put.body.len().to_string() + ); + assert!(put.headers.get("transfer-encoding").is_none()); + let entries = zip_entries(&put.body); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, "app.aab"); + assert_eq!( + sha1_hex(&entries[0].1), + sha1_hex(&artefact), + "artefact bytes survive the stream" + ); +} + +/// A directory of large libraries: each upload ZIP holds exactly that library's +/// bytes, registered under that library's OWN build-id. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn directory_libraries_upload_with_their_own_bytes_and_build_ids() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let fixture = std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE_ELF)).unwrap(); + // Two libraries with different build-ids (flip the id's first byte), each padded + // past the 64 KiB stream chunk so the PUT spans several chunks. + let id = hex::decode(FIXTURE_BUILD_ID).unwrap(); + let pos = fixture.windows(id.len()).position(|w| w == id).unwrap(); + let mut other = fixture.clone(); + other[pos] ^= 0xff; + let other_id = hex::encode(&other[pos..pos + id.len()]); + let (mut a, mut b) = (fixture.clone(), other.clone()); + a.extend(pattern(700_000)); + b.extend(pattern(900_001)); + write(tmp.path(), "libs/lib/arm64-v8a/libfoo.so", &a); + write(tmp.path(), "libs/lib/x86_64/libfoo.so", &b); + + let put_url = format!("{}/put", server.uri()); + Mock::given(method("POST")) + .and(wm_path("/apps/TKN/symbols")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"code": 0, "endpoint": put_url})), + ) + .expect(2) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(2) + .mount(&server) + .await; + + let endpoint = server.uri(); + let libs = tmp.path().join("libs"); + tokio::task::spawn_blocking(move || { + common::cli() + .args(["--endpoint", &endpoint, "--app-token", "TKN"]) + .args([ + "debug-files", + "upload", + "--type", + "elf", + "--version", + "1", + "--build", + "1", + ]) + .args(["--uuid", "00000000-0000-0000-0000-000000000000"]) + .arg(&libs) + .assert() + .success(); + }) + .await + .unwrap(); + + let posts: Vec = requests(&server, "POST") + .await + .iter() + .map(|r| serde_json::from_slice(&r.body).unwrap()) + .collect(); + let ids: Vec<&str> = posts.iter().map(|p| p["uuid"].as_str().unwrap()).collect(); + assert!( + ids.contains(&FIXTURE_BUILD_ID) && ids.contains(&other_id.as_str()), + "{ids:?}" + ); + for p in &posts { + assert_eq!(p["format"], "elf"); + assert_eq!(p["transform"], "breakpad"); + } + // Every PUT body is a zip whose single entry is one of the two libraries, whole, + // and the `hash` registered for it is the SHA-1 of the ZIP that was uploaded. + let puts = requests(&server, "PUT").await; + let mut seen = Vec::new(); + for put in &puts { + let entries = zip_entries(&put.body); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, "libfoo.so"); + seen.push(sha1_hex(&entries[0].1)); + assert!( + posts.iter().any(|p| p["hash"] == sha1_hex(&put.body)), + "registered hash must describe the uploaded ZIP" + ); + assert_eq!( + put.headers.get("content-length").unwrap().to_str().unwrap(), + put.body.len().to_string() + ); + } + seen.sort(); + let mut want = vec![sha1_hex(&a), sha1_hex(&b)]; + want.sort(); + assert_eq!(seen, want); +} + +/// `--strip-sources-content`: what is uploaded has no source and is still a valid +/// map; what is on disk is untouched; the registered hash is of the uploaded bytes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn strip_sources_content_uploads_a_valid_stripped_copy_and_spares_the_original() { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + let original = json!({ + "version": 3, + "debug_id": "33333333-3333-3333-3333-333333333333", + "sources": ["a.ts", "b.ts"], + "sourcesContent": ["const secret = 1;\n", "x".repeat(300_000)], + "names": [], + "mappings": "AAAA;AACA", + }); + let bytes = serde_json::to_vec(&original).unwrap(); + write(tmp.path(), "maps/app.js.map", &bytes); + + let put_url = format!("{}/put", server.uri()); + Mock::given(method("POST")) + .and(wm_path("/apps/TKN/symbols")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"code": 0, "endpoint": put_url})), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + let endpoint = server.uri(); + let dir = tmp.path().join("maps"); + tokio::task::spawn_blocking(move || { + common::cli() + .args(["--endpoint", &endpoint, "--app-token", "TKN"]) + .args([ + "debug-files", + "upload", + "--type", + "sourcemaps", + "--strip-sources-content", + ]) + .args(["--version", "1", "--build", "1"]) + .arg(&dir) + .assert() + .success(); + }) + .await + .unwrap(); + + let entries = zip_entries(&requests(&server, "PUT").await[0].body); + assert_eq!(entries.len(), 1); + let uploaded: Value = serde_json::from_slice(&entries[0].1).unwrap(); + let mut expected = original.clone(); + expected.as_object_mut().unwrap().remove("sourcesContent"); + assert_eq!(uploaded, expected, "exactly the map minus its sources"); + + let post: Value = serde_json::from_slice(&requests(&server, "POST").await[0].body).unwrap(); + assert_eq!( + post["uuid"], "33333333-3333-3333-3333-333333333333", + "the key is carried over" + ); + assert_eq!( + post["hash"], + sha1_hex(&entries[0].1), + "hash describes what was uploaded" + ); + assert_eq!( + std::fs::read(tmp.path().join("maps/app.js.map")).unwrap(), + bytes, + "the user's own map is not rewritten" + ); +} + +/// A ProGuard mapping goes out as ONE zstd entry whose bytes are the file's β€” also +/// when the file is empty or not valid UTF-8 (identity is over raw bytes). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn proguard_mappings_round_trip_raw_bytes() { + for (label, content) in [ + ("empty", Vec::new()), + ("crlf", b"a.B -> c.D:\r\n int x -> a\r\n".to_vec()), + ("non-utf8", b"a.B -> c.D:\n\xff\xfe\x00\x01\n".to_vec()), + ("large", pattern(400_003)), + ] { + let server = MockServer::start().await; + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "pg/mapping.txt", &content); + let put_url = format!("{}/put", server.uri()); + Mock::given(method("POST")) + .and(wm_path("/apps/TKN/symbols")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"code": 0, "endpoint": put_url})), + ) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let endpoint = server.uri(); + let dir = tmp.path().join("pg"); + tokio::task::spawn_blocking(move || { + common::cli() + .args(["--endpoint", &endpoint, "--app-token", "TKN"]) + .args(["debug-files", "upload", "--type", "proguard"]) + .args(["--version", "1", "--build", "1"]) + .arg(&dir) + .assert() + .success(); + }) + .await + .unwrap(); + + let entries = zip_entries(&requests(&server, "PUT").await[0].body); + assert_eq!( + entries, + [("mapping.txt".to_string(), content.clone())], + "{label}" + ); + let post: Value = serde_json::from_slice(&requests(&server, "POST").await[0].body).unwrap(); + // Java `UUID.nameUUIDFromBytes`: MD5 of the raw bytes, version-3 / IETF bits. + let mut md5 = ::digest(&content); + md5[6] = (md5[6] & 0x0f) | 0x30; + md5[8] = (md5[8] & 0x3f) | 0x80; + assert_eq!( + post["uuid"].as_str().unwrap().replace('-', ""), + hex::encode(md5), + "{label}: debug-id is over the raw bytes" + ); + } +} + +/// An empty/zero-byte library inside a scanned directory must not crash the scan +/// (it cannot be memory-mapped): it is reported as having no build-id and skipped. +#[test] +fn a_zero_byte_library_in_a_directory_is_skipped_not_fatal() { + let tmp = tempfile::tempdir().unwrap(); + let fixture = std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE_ELF)).unwrap(); + write(tmp.path(), "libs/libok.so", &fixture); + write(tmp.path(), "libs/libempty.so", b""); + common::cli() + .args([ + "debug-files", + "upload", + "--type", + "elf", + "--version", + "1", + "--build", + "1", + ]) + .args([ + "--uuid", + "00000000-0000-0000-0000-000000000000", + "--dry-run", + ]) + .arg(tmp.path().join("libs")) + .assert() + .success() + .stderr(predicates::str::contains("no GNU build-id")) + .stderr(predicates::str::contains( + "would register + upload 1 libraries", + )); +} From c5754f0e558b47c7e67c1d8fe1b54420559ea930 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:16:49 +0500 Subject: [PATCH 11/18] test+fix: broken-file coverage (JSON, ELF, Mach-O/DWARF, PDB, zip, plist, manifests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New corruption tests found four real defects, fixed here: - a Mach-O without LC_UUID was reported / uploaded as the nil UUID (dsym uuid, debug-files upload --type dsym); nil slices are now skipped and a bundle with none is rejected (xcode upload-dsyms fails the build for it) - --type elf accepted any format symbolic knows, so a Mach-O named *.so became an "elf" symbol keyed by its Mach-O UUID; require the \x7fELF magic - build-env read-plist (and the xcode flows) aborted the process with SIGABRT on a plist nested a few tens of thousands of levels deep (recursive plist::Value); parse on a 256 MiB-stack thread, refuse plists over 1 MiB - ios-deps reported a nameless `library::` dependency for a mangled Podfile.lock line such as `- (2.0)` Tests: tests/broken_files.rs drives the binary over broken source maps (19 JSON shapes x upload / strip / inject, incl. 100k-deep nesting), damaged ELF libraries and zips, Mach-O / dSYM / PDB, plists, manifests, build payloads and artefacts; unit sweeps truncate / flip every ELF, Mach-O, PDB, xcactivitylog and .app input and assert the result is an error or the TRUE identity - never a panic and never a different id (the identity is the upload key). Verified against the pre-fix code: the new tests fail there for each defect above. Characterised, not changed: IL2CPP mappings JSON is packed unvalidated. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 20 + CLAUDE.md | 16 + src/cli/build_env.rs | 148 +++++++ src/cli/dsym.rs | 50 +++ src/cli/ios_deps.rs | 7 + src/cli/xcactivitylog.rs | 136 +++++++ src/cli/xcode_ipa.rs | 83 ++++ src/symbols/dsym.rs | 143 ++++++- src/symbols/elf.rs | 226 +++++++++++ src/symbols/pdb.rs | 76 ++++ tests/broken_files.rs | 824 +++++++++++++++++++++++++++++++++++++++ 11 files changed, 1725 insertions(+), 4 deletions(-) create mode 100644 tests/broken_files.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8899121..e09ad84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,26 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). original key order (it was alphabetical), and `inject` writes `debug_id` / `debugId` last. Both are still valid maps; an invalid-UTF-8 map outside a string value now reports as invalid JSON (exit 11) rather than I/O (exit 10). +- **`sourcemaps inject` edits bundles in place.** The debug-id stub is appended (or, to re-key, our + own trailing stub is swapped) without copying the bundle: a 150 MB bundle's dirty memory went + from 317 MB to 2.4 MB. Output bytes are identical, and permissions and symlinks survive as before. + +### Fixed +- **`dsym uuid` and `debug-files upload --type dsym` no longer report a Mach-O with no `LC_UUID` + as the nil UUID** (`00000000-0000-0000-0000-000000000000`). Such a slice can never match a crash + report and every one would collide on the same all-zero key, so it is skipped; a bundle with no + usable slice is rejected (and fails an `xcode upload-dsyms` build phase, as any unreadable + bundle does). The app-binary path already treated nil as absent. +- **`--type elf` only treats real ELF files as ELF.** `symbolic` identifies Mach-O, PE and PDB just + as readily, so a macOS-built `.so` found by a directory scan would have been uploaded as an "elf" + symbol keyed by its Mach-O UUID. A file without the `\x7fELF` magic now has no build-id and is + skipped with the usual warning. +- **`build-env read-plist` (and the `xcode` flows that read `Info.plist`) no longer crash on a + deeply nested plist.** Tens of thousands of nested arrays overflowed the stack and aborted the + process with SIGABRT; it now reads as `{}` like any other unusable plist. Plists over 1 MiB are + refused up front. +- **`ios-deps` no longer reports a nameless dependency** (`library::`) for a truncated or mangled + `Podfile.lock` line such as `- (2.0)`. ## [0.7.13] - 2026-10-07 diff --git a/CLAUDE.md b/CLAUDE.md index aa49a21..ae78fc3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -114,6 +114,13 @@ rule, or entry name requires a major version bump and a coordinated rollout. ZIP uses method 93 (Z_STANDARD). `--no-zstd` is diagnostic only. - Every upload request carries the `X-Bugsee-Uploader: cli` telemetry header (set in `upload::http`). +- Bounded memory: inputs are artefacts and symbol files of hundreds of MB, so never load a + whole file onto the heap (`fs::read`, `read_to_string`, `tokio::fs::read`, or a `.clone()` of a + body per retry). Hash and copy through a 64 KiB buffer; send request bodies with + `upload::http::file_body` + an explicit `Content-Length` (a presigned S3 PUT rejects chunked); + read identity out of headers with `symbols::mapped::map_file` (the crate's only `unsafe`); walk + JSON with `inject::mapjson` (struson), not a `serde_json::Value` tree. `scripts/e2e_flows.py` + enforces a peak-RSS budget on ~96 MB inputs for every large-input flow, so a regression fails CI. - Return a typed `error::Error` (or `anyhow` wrapping one) so `main`'s `classify` maps it to the right exit code. A bare `anyhow::anyhow!` falls through to exit 1 (structural) β€” use a typed variant whenever the code matters to a caller. @@ -125,6 +132,15 @@ rule, or entry name requires a major version bump and a coordinated rollout. response contract, not just "something happened". - `tests/` holds integration tests that exec the COMPILED binary (`assert_cmd`) and pin exit codes + stdout JSON. +- `tests/file_handling.rs` pins file handling through the compiled binary: exact bytes left on + disk, large payloads on the wire, stdout kept empty, exit codes for unusable input. The streaming + JSON code is also tested differentially against the `serde_json::Value` behaviour it replaced. +- `tests/broken_files.rs` pins what the binary does with BROKEN input (corrupt JSON / ELF / + Mach-O / PDB / zip / plist / manifests): never a crash, a stable exit code, nothing sent for + input that could not be identified, the user's files untouched. A new file-reading code path + needs a truncation and a garbage case there (or a unit-level sweep next to the parser, as + `symbols/elf.rs` and `symbols/dsym.rs` do: a damaged file may fail to identify but must never + identify as something else, because the identity is the upload key). - `scripts/e2e_flows.py` is a stdlib-only end-to-end harness that drives a real binary through every upload flow against a protocol-accurate mock server. CI runs it from-source on every push (`.github/workflows/e2e.yml`) and against the diff --git a/src/cli/build_env.rs b/src/cli/build_env.rs index 5f108ee..67d5449 100644 --- a/src/cli/build_env.rs +++ b/src/cli/build_env.rs @@ -321,6 +321,32 @@ fn env_truthy(value: Option<&String>) -> bool { /// failure, root that isn't a Dict). The Python callers treat /// absence as "field unknown" rather than failing. pub fn read_plist_to_json(path: &std::path::Path) -> serde_json::Map { + // `plist::Value` is built (and later dropped) recursively, one stack frame per + // nesting level. A malformed or hostile plist a few tens of thousands of levels + // deep overflowed the 8 MiB main-thread stack and ABORTED the process (SIGABRT, + // no error, no `{}`). Two bounds make that impossible, whatever the file holds: + // - an Info.plist is a few KB; refuse anything over `MAX_PLIST_BYTES`, which + // caps the nesting depth (a level costs at least ~6 bytes in either format); + // - parse on a thread whose stack is far larger than that depth can use. + // As a side effect a panic inside the parser is contained and reads as `{}` too. + const MAX_PLIST_BYTES: u64 = 1024 * 1024; + const PLIST_PARSE_STACK_BYTES: usize = 256 * 1024 * 1024; + + match std::fs::metadata(path) { + Ok(m) if m.len() <= MAX_PLIST_BYTES => {} + _ => return serde_json::Map::new(), + } + let path = path.to_path_buf(); + std::thread::Builder::new() + .name("plist-parse".into()) + .stack_size(PLIST_PARSE_STACK_BYTES) + .spawn(move || parse_plist_to_json(&path)) + .ok() + .and_then(|handle| handle.join().ok()) + .unwrap_or_default() +} + +fn parse_plist_to_json(path: &std::path::Path) -> serde_json::Map { let mut out = serde_json::Map::new(); let value = match plist::Value::from_file(path) { Ok(v) => v, @@ -597,6 +623,128 @@ mod tests { } } + // ── read_plist_to_json on broken / hostile files ────────────── + + /// A binary plist whose objects form one chain of `n` nested arrays (4-byte refs). + fn nested_bplist(n: u32) -> Vec { + let mut body = b"bplist00".to_vec(); + let mut offsets = Vec::new(); + for i in 0..n { + offsets.push(body.len() as u32); + if i + 1 < n { + body.push(0xa1); // array of 1 + body.extend_from_slice(&(i + 1).to_be_bytes()); + } else { + body.push(0x50); // empty string + } + } + let table = body.len() as u64; + for o in offsets { + body.extend_from_slice(&o.to_be_bytes()); + } + body.extend_from_slice(&[0; 6]); + body.extend_from_slice(&[4, 4]); // offset size, object-ref size + body.extend_from_slice(&(n as u64).to_be_bytes()); + body.extend_from_slice(&0u64.to_be_bytes()); // top object + body.extend_from_slice(&table.to_be_bytes()); + body + } + + fn plist_map(bytes: &[u8]) -> serde_json::Map { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("Info.plist"); + std::fs::write(&p, bytes).unwrap(); + read_plist_to_json(&p) + } + + /// Nesting deep enough to overflow the main-thread stack used to ABORT the whole + /// process. It must read as an ordinary unusable plist (`{}`), in both formats and + /// right up to the size cap β€” debug-build frames are the largest, so this is the + /// worst case the thread's stack has to absorb. + #[test] + fn a_plist_nested_past_any_stack_reads_as_empty_instead_of_crashing() { + let xml = |n: usize| { + [ + &b""[..], + &b"".repeat(n), + &b"".repeat(n), + b"", + ] + .concat() + }; + // Largest XML nesting that still fits under the 1 MiB size cap. + assert!(xml(69_000).len() < 1024 * 1024); + assert!(plist_map(&xml(69_000)).is_empty()); + assert!(plist_map(&xml(200_000)).is_empty(), "over the cap"); + // Binary: ~9 bytes per level, so ~110k levels fit under the cap. + assert!(nested_bplist(110_000).len() < 1024 * 1024); + assert!(plist_map(&nested_bplist(110_000)).is_empty()); + assert!( + plist_map(&nested_bplist(300_000)).is_empty(), + "over the cap" + ); + } + + #[test] + fn broken_plists_read_as_empty_and_valid_neighbours_still_read() { + let valid: &[u8] = br#"CFBundleNameAppN7"#; + let m = plist_map(valid); + assert_eq!(m["CFBundleName"], "App"); + assert_eq!(m["N"], "7"); + + let mut broken: Vec<(String, Vec)> = vec![ + ("empty".into(), Vec::new()), + ( + "garbage".into(), + (0..300u32).map(|i| (i * 37 % 251) as u8).collect(), + ), + ("bplist magic only".into(), b"bplist00".to_vec()), + ("bplist truncated".into(), b"bplist00\xd1\x01\x02".to_vec()), + ( + "array root".into(), + br#""#.to_vec(), + ), + ( + "no root".into(), + br#""#.to_vec(), + ), + ( + "not utf-8".into(), + b"\xff\xfex" + .to_vec(), + ), + ]; + // Every strict prefix of a valid XML plist is invalid: empty, never a panic + // and never a half-read dictionary. + for n in [0, 1, 20, 60, 100, valid.len() - 10, valid.len() - 1] { + broken.push((format!("xml truncated at {n}"), valid[..n].to_vec())); + } + for (label, bytes) in &broken { + let m = plist_map(bytes); + assert!(m.is_empty(), "{label}: {m:?}"); + } + } + + /// Entity declarations are not expanded (no billion-laughs blow-up, no external + /// fetch): a reference just reads as empty. + #[test] + fn xml_entities_are_not_expanded() { + let bomb = br#"]>k&c;"#; + assert_eq!(plist_map(bomb)["k"], ""); + let xxe = br#"]>k&x;"#; + assert_eq!(plist_map(xxe)["k"], ""); + } + + /// Something far larger than any Info.plist is refused up front, not parsed. + #[test] + fn an_oversized_plist_is_refused() { + let mut big = + br#"a"#.to_vec(); + big.extend(std::iter::repeat_n(b'x', 1024 * 1024)); + big.extend_from_slice(b""); + assert!(plist_map(&big).is_empty()); + } + // ── env_truthy β€” cross-language contract ───────────────────── #[test] diff --git a/src/cli/dsym.rs b/src/cli/dsym.rs index 7152667..3b13efb 100644 --- a/src/cli/dsym.rs +++ b/src/cli/dsym.rs @@ -164,6 +164,10 @@ fn push_slices_from_macho(path: &Path, out: &mut Vec) { Err(_) => return, }; for obj in archive.objects().flatten() { + // No LC_UUID reads as the nil UUID: not an identity, so not reported. + if obj.debug_id().is_nil() { + continue; + } out.push(DsymSliceView { uuid: format_uuid(obj.debug_id()), arch: obj.arch().name().to_string(), @@ -193,6 +197,52 @@ mod tests { use super::*; use tempfile::TempDir; + /// A Mach-O without an `LC_UUID` reads as the nil UUID: not an identity, so it + /// is not reported (an integrator would otherwise key a build on all zeros). + #[test] + fn extract_uuids_omits_nil_uuid_slices() { + let tmp = TempDir::new().unwrap(); + let good: [u8; 16] = [7; 16]; + let fat = crate::symbols::test_macho::fat_macho(&[ + (0x0100_000c, 0, good), + (0x0100_0007, 3, [0; 16]), + ]); + let p = tmp.path().join("fat"); + std::fs::write(&p, &fat).unwrap(); + let out = extract_uuids(&p); + assert_eq!( + out, + [format_uuid(DebugId::from_uuid(uuid::Uuid::from_bytes( + good + )))] + ); + + let only_nil = tmp.path().join("nil"); + std::fs::write( + &only_nil, + crate::symbols::test_macho::thin_macho(0x0100_000c, 0, [0; 16]), + ) + .unwrap(); + assert!(extract_uuids(&only_nil).is_empty()); + } + + /// Truncated Mach-O files never crash the extractor and never report a made-up UUID. + #[test] + fn extract_uuids_on_truncated_machos_is_empty_or_the_true_uuid() { + let tmp = TempDir::new().unwrap(); + let valid = crate::symbols::test_macho::thin_macho(0x0100_000c, 0, [9; 16]); + let true_uuid = format_uuid(DebugId::from_uuid(uuid::Uuid::from_bytes([9; 16]))); + let p = tmp.path().join("m"); + for len in 0..valid.len() { + std::fs::write(&p, &valid[..len]).unwrap(); + let out = extract_uuids(&p); + assert!( + out.is_empty() || out == [true_uuid.clone()], + "len {len}: {out:?}" + ); + } + } + #[test] fn extract_uuids_returns_empty_for_nonexistent_path() { let out = extract_uuids(Path::new("/no/such/file")); diff --git a/src/cli/ios_deps.rs b/src/cli/ios_deps.rs index 1deb240..27b8960 100644 --- a/src/cli/ios_deps.rs +++ b/src/cli/ios_deps.rs @@ -322,6 +322,13 @@ pub fn parse_podfile_lock(path: &Path) -> Vec { // Pod line: ` - Name (Version):` (2-space indent). if let Some(pod_body) = parse_pod_line(line) { let (name, version) = parse_name_version(pod_body); + if name.is_empty() { + // A truncated or mangled line (`- (2.0)`) names nothing; a nameless + // entry would put a bogus `library::` dependency in the graph. Its + // child lines have no owner either, so they are dropped with it. + current_pod = None; + continue; + } pods.insert( name.clone(), PodInfo { diff --git a/src/cli/xcactivitylog.rs b/src/cli/xcactivitylog.rs index 9964801..78d898d 100644 --- a/src/cli/xcactivitylog.rs +++ b/src/cli/xcactivitylog.rs @@ -1775,6 +1775,142 @@ mod tests { assert!(t.summary.is_none() && t.timeline.is_none()); } + // ── corrupted logs ───────────────────────────────────────────── + + /// A synthetic log with several sections, a blob and unicode titles. + fn sample_log() -> Vec { + build_log( + &[ + ("Ld App normal arm64", T0 + 1.0, T0 + 3.0), + ("CompileSwift normal arm64 hΓ©llo.swift", T0 + 0.5, T0 + 2.0), + ], + &[("Build App", T0, T0 + 5.0)], + &[], + true, + ) + } + + /// Cutting the stream anywhere is survivable: `None` or a parse of what is left, + /// never a panic and never a hang on a half-read length. + #[test] + fn every_truncation_of_a_log_is_survivable() { + let log = sample_log(); + assert!( + parse_stream(&log).is_some(), + "control: the whole log parses" + ); + for cut in 0..log.len() { + let r = std::panic::catch_unwind(|| parse_stream(&log[..cut])); + assert!(r.is_ok(), "parse_stream PANICKED on a {cut}-byte prefix"); + } + } + + /// Flip, replace and insert bytes all over the stream, concentrating on the + /// length-prefixed fields: a corrupted length (`99999999999`) must not turn into a + /// huge allocation or an out-of-bounds read. + #[test] + fn corrupted_bytes_never_panic_or_over_allocate() { + let log = sample_log(); + let mut state = 0xA076_1D64_78BD_642Fu64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let started = std::time::Instant::now(); + for round in 0..2000 { + let mut b = log.clone(); + for _ in 0..1 + next() % 4 { + let at = (next() % b.len() as u64) as usize; + match next() % 4 { + 0 => b[at] = next() as u8, + 1 => b[at] = b"0123456789^#%\"@"[(next() % 14) as usize], + 2 => { + b.remove(at); + } + _ => b.insert(at, next() as u8), + } + } + let r = std::panic::catch_unwind(|| parse_stream(&b)); + assert!( + r.is_ok(), + "parse_stream PANICKED on corruption round {round}" + ); + } + assert!( + started.elapsed() < std::time::Duration::from_secs(60), + "corrupted input must not make the tokenizer crawl" + ); + } + + #[test] + fn a_length_prefix_claiming_the_moon_is_rejected_cheaply() { + // `SLF0` header, then a string token claiming 2^60 bytes: must come back at once. + let mut data = b"SLF0".to_vec(); + data.extend_from_slice(b"99999999999999999999\""); + data.extend_from_slice(b"1152921504606846976\"x"); + let r = std::panic::catch_unwind(|| parse_stream(&data)); + assert!(r.is_ok()); + } + + /// Gzip-level damage: truncated stream, flipped payload byte, trailing junk, + /// empty file. All read as "no timings" (`None`), none panics. + #[test] + fn damaged_gzip_logs_parse_as_none() { + let td = tempfile::tempdir().unwrap(); + let path = write_gzipped_log(td.path(), &sample_log()); + let raw = std::fs::read(&path).unwrap(); + assert!( + parse_xcactivitylog(&path).is_some(), + "control: the intact log parses" + ); + let mut flipped = raw.clone(); + let mid = flipped.len() / 2; + flipped[mid] ^= 0xff; + let cases: Vec<(&str, Vec)> = vec![ + ("empty file", Vec::new()), + ("gzip magic only", vec![0x1f, 0x8b]), + ("truncated in half", raw[..raw.len() / 2].to_vec()), + ("trailer chopped", raw[..raw.len() - 6].to_vec()), + ("flipped payload byte", flipped), + ("not gzip", b"SLF0 plain, never compressed".to_vec()), + ]; + for (label, bytes) in cases { + let p = td.path().join("broken.xcactivitylog"); + std::fs::write(&p, bytes).unwrap(); + let r = std::panic::catch_unwind(|| parse_xcactivitylog(&p)); + assert!(r.is_ok(), "{label}: PANICKED"); + assert!(r.unwrap().is_none(), "{label}: must read as no timings"); + } + } + + /// A log that inflates far past the cap (a decompression bomb) is cut at the cap, + /// not inflated into memory. + #[test] + fn a_decompression_bomb_is_capped() { + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + let td = tempfile::tempdir().unwrap(); + let p = td.path().join("bomb.xcactivitylog"); + let mut enc = GzEncoder::new(std::fs::File::create(&p).unwrap(), Compression::best()); + enc.write_all(b"SLF0").unwrap(); + let zeros = vec![0u8; 1024 * 1024]; + for _ in 0..(super::MAX_DECOMPRESSED / (1024 * 1024) + 30) { + enc.write_all(&zeros).unwrap(); // ~50 MB of zeros, a few KB compressed + } + enc.finish().unwrap(); + assert!( + std::fs::metadata(&p).unwrap().len() < 200_000, + "the bomb really is tiny" + ); + // Returns promptly with no timings; the proof of the cap is that this does not + // hold ~50 MB (it reads at most MAX_DECOMPRESSED + 1 bytes). + let r = std::panic::catch_unwind(|| parse_xcactivitylog(&p)); + assert!(r.is_ok() && r.unwrap().is_none()); + } + #[test] fn resolve_swallows_bad_gzip() { // A `.xcactivitylog` that is not valid gzip β†’ parse returns None β†’ diff --git a/src/cli/xcode_ipa.rs b/src/cli/xcode_ipa.rs index 435a5a5..c2c88a0 100644 --- a/src/cli/xcode_ipa.rs +++ b/src/cli/xcode_ipa.rs @@ -273,6 +273,89 @@ mod tests { app } + /// The app's identity comes from two files that can each be damaged: a broken + /// Info.plist or executable means "no UUID" (the caller falls back to a random one + /// so the build record still lands), never a panic and never a made-up UUID. + #[test] + fn main_executable_uuid_is_none_for_every_broken_app() { + use crate::symbols::test_macho::{thin_macho, CPU_SUBTYPE_ARM64_ALL, CPU_TYPE_ARM64}; + let plist = |exe: &str| { + format!( + r#"CFBundleExecutable{exe}"# + ) + }; + let valid = thin_macho(CPU_TYPE_ARM64, CPU_SUBTYPE_ARM64_ALL, [7; 16]); + let nil = thin_macho(CPU_TYPE_ARM64, CPU_SUBTYPE_ARM64_ALL, [0; 16]); + + let build = |plist_bytes: &[u8], exe: Option<&[u8]>| { + let td = tempfile::tempdir().unwrap(); + let app = td.path().join("A.app"); + std::fs::create_dir_all(&app).unwrap(); + std::fs::write(app.join("Info.plist"), plist_bytes).unwrap(); + if let Some(b) = exe { + std::fs::write(app.join("MyApp"), b).unwrap(); + } + (td, app) + }; + + // Control: an intact app reports the real UUID. + let (_t, app) = build(plist("MyApp").as_bytes(), Some(&valid)); + assert_eq!(main_executable_uuid(&app).unwrap(), "07".repeat(16)); + + let mut cases: Vec<(String, Vec, Option>)> = vec![ + ("garbage plist".into(), vec![0xff; 100], Some(valid.clone())), + ("empty plist".into(), Vec::new(), Some(valid.clone())), + ( + "truncated plist".into(), + plist("MyApp").as_bytes()[..60].to_vec(), + Some(valid.clone()), + ), + ( + "plist without the key".into(), + br#""#.to_vec(), + Some(valid.clone()), + ), + ( + "executable missing".into(), + plist("MyApp").into_bytes(), + None, + ), + ( + "empty executable".into(), + plist("MyApp").into_bytes(), + Some(Vec::new()), + ), + ( + "garbage executable".into(), + plist("MyApp").into_bytes(), + Some(vec![0xab; 500]), + ), + ( + "executable with no LC_UUID".into(), + plist("MyApp").into_bytes(), + Some(nil), + ), + ( + "executable named by a path escape".into(), + plist("../outside").into_bytes(), + Some(valid.clone()), + ), + ]; + for n in 0..valid.len() { + cases.push(( + format!("executable cut at {n}"), + plist("MyApp").into_bytes(), + Some(valid[..n].to_vec()), + )); + } + for (label, plist_bytes, exe) in cases { + let (_t, app) = build(&plist_bytes, exe.as_deref()); + let got = std::panic::catch_unwind(|| main_executable_uuid(&app)) + .unwrap_or_else(|_| panic!("{label}: PANICKED")); + assert_eq!(got, None, "{label}"); + } + } + #[test] fn main_executable_uuid_reads_real_macho_uuid() { // Positive symbolic path for IPA extraction: the other diff --git a/src/symbols/dsym.rs b/src/symbols/dsym.rs index 3f073f3..a8694f1 100644 --- a/src/symbols/dsym.rs +++ b/src/symbols/dsym.rs @@ -73,6 +73,7 @@ pub fn identify(dsym_path: &Path) -> Result { } let mut slices = Vec::new(); + let mut saw_nil_uuid = false; for entry in std::fs::read_dir(&dwarf_dir)? { let entry = entry?; if !entry.file_type()?.is_file() { @@ -95,6 +96,19 @@ pub fn identify(dsym_path: &Path) -> Result { e, )) })?; + // A slice without an LC_UUID reads as the nil UUID. It can never match a + // crash report, and every such slice would collide on the same all-zero + // key on the server, so it is not a symbol: skip it (the same stance + // `xcode_ipa::select_preferred_uuid` takes for the app binary). + if obj.debug_id().is_nil() { + tracing::warn!( + path = %bin_path.display(), + arch = %obj.arch().name(), + "Mach-O slice has no LC_UUID (nil UUID); skipping it" + ); + saw_nil_uuid = true; + continue; + } slices.push(DsymSlice { uuid: obj.debug_id().to_string(), arch: obj.arch().name().to_string(), @@ -103,10 +117,17 @@ pub fn identify(dsym_path: &Path) -> Result { } if slices.is_empty() { - return Err(Error::InputInvalid(format!( - "dSYM bundle contains no Mach-O slices: {}", - dsym_path.display() - ))); + return Err(Error::InputInvalid(if saw_nil_uuid { + format!( + "no Mach-O slice in the dSYM bundle carries an LC_UUID: {}", + dsym_path.display() + ) + } else { + format!( + "dSYM bundle contains no Mach-O slices: {}", + dsym_path.display() + ) + })); } Ok(DsymIdentity { slices }) } @@ -163,6 +184,120 @@ mod tests { use std::fs; use std::io::Write; + // ── broken Mach-O / DWARF inputs ─────────────────────────────── + + const ARM64: u32 = 0x0100_000c; + const X86_64: u32 = 0x0100_0007; + const GOOD: [u8; 16] = [ + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, + 0x01, + ]; + + fn bundle_with(bytes: &[u8]) -> (tempfile::TempDir, PathBuf) { + let tmp = tempfile::tempdir().unwrap(); + let dsym = tmp.path().join("App.dSYM"); + let dwarf = dsym.join("Contents/Resources/DWARF"); + fs::create_dir_all(&dwarf).unwrap(); + fs::write(dwarf.join("App"), bytes).unwrap(); + (tmp, dsym) + } + + fn invalid(err: Error) -> String { + match err { + Error::InputInvalid(m) => m, + other => panic!("expected InputInvalid, got {other:?}"), + } + } + + /// A Mach-O with no `LC_UUID` reads as the nil UUID, which can never match a + /// crash report and would register every such dSYM under the same all-zero key. + /// It is not a symbol: the bundle is rejected, and says why. + #[test] + fn a_macho_without_a_uuid_is_rejected_not_registered_as_nil() { + let (_t, dsym) = bundle_with(&crate::symbols::test_macho::thin_macho(ARM64, 0, [0; 16])); + let msg = invalid(identify(&dsym).unwrap_err()); + assert!(msg.contains("LC_UUID"), "{msg}"); + } + + #[test] + fn a_nil_slice_in_a_fat_binary_is_skipped_and_the_real_ones_kept() { + let fat = crate::symbols::test_macho::fat_macho(&[(ARM64, 0, GOOD), (X86_64, 3, [0; 16])]); + let (_t, dsym) = bundle_with(&fat); + let id = identify(&dsym).unwrap(); + assert_eq!(id.slices.len(), 1, "{:?}", id.slices); + assert_eq!(id.slices[0].arch, "arm64"); + assert!(!id.slices[0].uuid.starts_with("00000000-0000-0000")); + } + + /// Whatever prefix of a valid Mach-O survives, `identify` returns an error or the + /// TRUE identity β€” never a panic and never a different UUID. + #[test] + fn every_truncation_of_a_macho_is_an_error_or_the_true_identity() { + let valid = crate::symbols::test_macho::thin_macho(ARM64, 0, GOOD); + let want = identify(&bundle_with(&valid).1).unwrap().slices; + assert_eq!(want.len(), 1); + for len in 0..valid.len() { + let (_t, dsym) = bundle_with(&valid[..len]); + if let Ok(id) = identify(&dsym) { + assert_eq!( + id.slices.iter().map(|s| &s.uuid).collect::>(), + want.iter().map(|s| &s.uuid).collect::>(), + "truncated to {len} bytes reported a different identity" + ); + } + } + } + + #[test] + fn corrupted_headers_never_panic_and_are_invalid_or_identified() { + let valid = crate::symbols::test_macho::fat_macho(&[(ARM64, 0, GOOD), (X86_64, 3, GOOD)]); + let mut cases: Vec<(String, Vec)> = vec![ + ("empty".into(), Vec::new()), + ("macho magic only".into(), vec![0xcf, 0xfa, 0xed, 0xfe]), + ("fat magic only".into(), vec![0xca, 0xfe, 0xba, 0xbe]), + ( + "fat claiming 2^32-1 slices".into(), + [ + &[0xca, 0xfe, 0xba, 0xbe, 0xff, 0xff, 0xff, 0xff][..], + &[0u8; 100], + ] + .concat(), + ), + ( + "garbage".into(), + (0..4000u32).map(|i| (i * 131 % 253) as u8).collect(), + ), + ("zeroed".into(), vec![0; valid.len()]), + ]; + // Every header byte replaced by 0x00 / 0xff / 0x7f / 0x80. + for i in 0..valid.len().min(96) { + for v in [0x00u8, 0xff, 0x7f, 0x80] { + let mut b = valid.clone(); + b[i] = v; + cases.push((format!("byte {i} = {v:#x}"), b)); + } + } + for (label, bytes) in cases { + let (_t, dsym) = bundle_with(&bytes); + // The assertion is "returns" β€” a panic fails the test; both outcomes are fine. + let _ = identify(&dsym) + .map(|id| id.slices.len()) + .map_err(|e| e.to_string()); + let _ = label; + } + } + + #[test] + fn a_subdirectory_or_empty_file_in_the_dwarf_folder_does_not_hide_a_good_slice() { + let (_t, dsym) = bundle_with(&crate::symbols::test_macho::thin_macho(ARM64, 0, GOOD)); + let dwarf = dsym.join("Contents/Resources/DWARF"); + fs::create_dir(dwarf.join("nested")).unwrap(); + fs::write(dwarf.join("junk"), b"").unwrap(); + // The empty file is not a Mach-O: that is an error for the bundle, loudly, + // rather than a silently partial identity. + assert!(identify(&dsym).is_err()); + } + #[test] fn rejects_plain_directory_without_dwarf_dir() { // Any NAME is accepted here (discovery owns the name policy, so a diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index f4fd182..dc20bfa 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -275,6 +275,12 @@ fn parse_elf_identity(bytes: &[u8]) -> ElfIdentity { has_debug_info: false, has_symbols: false, }; + // `symbolic` identifies Mach-O, PE and PDB just as readily, and their ids would + // read as a "build-id" here: a macOS `.so` or a stray `.pdb` named `*.so` would + // be registered as an ELF symbol under a foreign key. This uploader takes ELF. + if !bytes.starts_with(b"\x7fELF") { + return unknown(); + } let archive = match Archive::parse(bytes) { Ok(a) => a, Err(_) => return unknown(), @@ -546,6 +552,226 @@ mod tests { ); } + // ── corrupted / truncated ELF files ──────────────────────────── + + const TRUE_BUILD_ID: &str = "bca64abfec40dbb631bb8f1c37414472"; + + fn fixture() -> Vec { + std::fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"), + ) + .unwrap() + } + + /// Deterministic xorshift, so a failure names a reproducible byte. + fn xorshift(state: &mut u64) -> u64 { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + *state + } + + /// Run `parse_elf_identity`, turning a panic into a reportable failure. + fn parse_catching(bytes: &[u8], label: &str) -> ElfIdentity { + std::panic::catch_unwind(|| parse_elf_identity(bytes)) + .unwrap_or_else(|_| panic!("parse_elf_identity PANICKED on {label}")) + } + + /// The key of a symbol is its build-id, so the one thing a damaged file must never + /// do is report ANOTHER build-id: it would be uploaded under the wrong key and + /// look like a different library's symbols. Any prefix of the real library either + /// parses to no build-id or to the TRUE one β€” nothing in between. + #[test] + fn a_truncated_elf_has_no_build_id_or_the_true_one() { + let elf = fixture(); + let mut cuts: Vec = (0..=130).collect(); // every byte of the headers + let mut at = 131usize; + while at < elf.len() { + cuts.push(at); + at += at / 7 + 13; // geometric-ish sweep over the whole file + } + cuts.push(elf.len() - 1); + for cut in cuts { + let id = parse_catching(&elf[..cut], &format!("a {cut}-byte prefix")); + assert!( + id.build_id.is_none() || id.build_id.as_deref() == Some(TRUE_BUILD_ID), + "prefix of {cut} bytes reported build-id {:?}", + id.build_id + ); + } + // And the whole file still parses to the true identity (the sweep's control). + assert_eq!( + parse_catching(&elf, "the whole file").build_id.as_deref(), + Some(TRUE_BUILD_ID) + ); + } + + /// Damage to the ELF header and the section/program header tables β€” the + /// structures `symbolic` walks to find the build-id note and the debug sections β€” + /// must be survivable: no panic, whatever it then reports. + #[test] + fn corrupt_elf_headers_and_tables_never_panic() { + let elf = fixture(); + // e_shoff (0x28), e_phoff (0x20), e_shentsize/num/shstrndx (0x3a..0x40), e_phnum. + let fields: &[(usize, usize)] = &[ + (0x20, 8), + (0x28, 8), + (0x36, 2), + (0x38, 2), + (0x3a, 2), + (0x3c, 2), + (0x3e, 2), + ]; + let mut checked = 0; + for &(off, len) in fields { + for fill in [0x00u8, 0xff, 0x7f, 0x80, 0x01] { + let mut b = elf.clone(); + b[off..off + len].fill(fill); + parse_catching(&b, &format!("header field @{off:#x} = {fill:#x}")); + checked += 1; + } + } + // Every one of the first 64 header bytes flipped to a few values. + for i in 0..64 { + for v in [0x00u8, 0xff, 0x55] { + let mut b = elf.clone(); + b[i] = v; + parse_catching(&b, &format!("e_ident/header byte {i} = {v:#x}")); + checked += 1; + } + } + // The section header table is at e_shoff; flip bytes throughout it. + let shoff = u64::from_le_bytes(elf[0x28..0x30].try_into().unwrap()) as usize; + let shnum = u16::from_le_bytes(elf[0x3c..0x3e].try_into().unwrap()) as usize; + assert!(shoff > 0 && shnum > 0, "fixture has section headers"); + let table_end = (shoff + shnum * 64).min(elf.len()); + for i in (shoff..table_end).step_by(5) { + let mut b = elf.clone(); + b[i] ^= 0xff; + parse_catching(&b, &format!("section header byte {i} flipped")); + checked += 1; + } + assert!(checked > 100); + } + + #[test] + fn random_byte_flips_anywhere_never_panic() { + let elf = fixture(); + let mut state = 0x2545_F491_4F6C_DD1Du64; + for round in 0..300 { + let mut b = elf.clone(); + for _ in 0..1 + round % 8 { + let at = (xorshift(&mut state) % b.len() as u64) as usize; + b[at] = xorshift(&mut state) as u8; + } + parse_catching(&b, &format!("random corruption round {round}")); + } + } + + #[test] + fn non_elf_and_degenerate_inputs_have_no_identity() { + for (label, bytes) in [ + ("empty", Vec::new()), + ("one byte", vec![0x7f]), + ("magic only", b"\x7fELF".to_vec()), + ("zeros", vec![0; 4096]), + ("a text file", b"#!/bin/sh\necho hi\n".repeat(50)), + ("a ZIP", b"PK\x03\x04junk".repeat(40)), + ( + "a Mach-O", + crate::symbols::test_macho::thin_macho(0x0100_000c, 0, [3; 16]), + ), + ] { + let id = parse_catching(&bytes, label); + assert_eq!(id.build_id, None, "{label}"); + } + } + + /// A library that parses but carries no build-id (`-Wl,--build-id` was off) is + /// reported without one, so the uploader can warn and skip it. + #[test] + fn scan_dir_reports_damaged_libraries_without_a_build_id_instead_of_failing() { + let elf = fixture(); + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("libgood.so"), &elf).unwrap(); + std::fs::write(dir.path().join("libhalf.so"), &elf[..elf.len() / 2]).unwrap(); + std::fs::write(dir.path().join("libzero.so"), vec![0u8; 5000]).unwrap(); + std::fs::write(dir.path().join("libtext.so"), b"not an elf").unwrap(); + let libs = scan_dir(dir.path(), &ExtraSuffixes::default()).unwrap(); + let by_name = |n: &str| libs.iter().find(|l| l.name == n).unwrap(); + assert_eq!( + by_name("libgood.so").build_id.as_deref(), + Some(TRUE_BUILD_ID) + ); + for damaged in ["libhalf.so", "libzero.so", "libtext.so"] { + assert_eq!(by_name(damaged).build_id, None, "{damaged}"); + } + let kept = keep_richest_per_build_id(libs); + assert_eq!( + kept.len(), + 4, + "keyless libs pass through for the uploader to warn about" + ); + } + + /// A ZIP entry that fails its CRC (bit rot in `native-debug-symbols.zip`) is an + /// error for the archive, never a silently-wrong library. + #[test] + fn a_zip_entry_with_a_bad_crc_is_an_error() { + let elf = fixture(); + let dir = tempfile::tempdir().unwrap(); + let zip_path = dir.path().join("n.zip"); + { + let mut zw = zip::ZipWriter::new(std::fs::File::create(&zip_path).unwrap()); + zw.start_file( + "arm64-v8a/libfoo.so", + SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored), + ) + .unwrap(); + zw.write_all(&elf).unwrap(); + zw.finish().unwrap(); + } + let mut raw = std::fs::read(&zip_path).unwrap(); + let mid = raw.len() / 2; // inside the stored payload + raw[mid] ^= 0x5a; + std::fs::write(&zip_path, &raw).unwrap(); + let out = dir.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + assert!(extract_libs(&zip_path, &out, &ExtraSuffixes::default()).is_err()); + } + + #[test] + fn truncated_or_garbage_zips_are_errors() { + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.zip"); + { + let mut zw = zip::ZipWriter::new(std::fs::File::create(&good).unwrap()); + zw.start_file("a/libfoo.so", SimpleFileOptions::default()) + .unwrap(); + zw.write_all(&fixture()).unwrap(); + zw.finish().unwrap(); + } + let raw = std::fs::read(&good).unwrap(); + let out = dir.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + for (label, bytes) in [ + ("empty", Vec::new()), + ("half", raw[..raw.len() / 2].to_vec()), + ("no end-of-central-directory", raw[..raw.len() - 8].to_vec()), + ( + "garbage", + (0..5000u32).map(|i| (i * 7 % 251) as u8).collect(), + ), + ] { + let p = dir.path().join("bad.zip"); + std::fs::write(&p, bytes).unwrap(); + assert!( + extract_libs(&p, &out, &ExtraSuffixes::default()).is_err(), + "{label}" + ); + } + } + fn lib(name: &str, build_id: Option<&str>, richness: (bool, bool, u64)) -> ElfLib { ElfLib { name: name.to_string(), diff --git a/src/symbols/pdb.rs b/src/symbols/pdb.rs index 3afa6f3..1953cd5 100644 --- a/src/symbols/pdb.rs +++ b/src/symbols/pdb.rs @@ -240,6 +240,82 @@ mod tests { (dir, path) } + // ── corrupted / truncated PDBs ───────────────────────────────── + + /// A damaged PDB either fails to identify or identifies as ITSELF: never a + /// panic, and never a different debug id (it is the upload key). + #[test] + fn every_truncation_of_a_pdb_is_an_error_or_the_true_identity() { + let valid = synth_pdb(guid(), 7, 1, MACHINE_AMD64); + let want = "dfb8e43a-f242-3d73-a453-aeb6a777ef75-1"; + let mut cuts: Vec = (0..=200).collect(); + let mut at = 201usize; + while at < valid.len() { + cuts.push(at); + at += at / 9 + 17; + } + cuts.push(valid.len() - 1); + for cut in cuts { + let (_d, p) = write_temp(&valid[..cut]); + match identify(&p) { + Ok(id) => assert!( + id.slices.iter().all(|s| s.uuid == want), + "{cut}-byte prefix identified as {:?}", + id.slices + ), + Err(Error::InputInvalid(_)) => {} + Err(other) => panic!("{cut}-byte prefix: unexpected {other:?}"), + } + } + } + + #[test] + fn corrupted_msf_structures_never_panic() { + let valid = synth_pdb(guid(), 7, 1, MACHINE_AMD64); + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + // The superblock fields (page size, page count, directory size / location) + // set to hostile values: a zero or huge page size, absurd counts. + for off in (32..56).step_by(4) { + for v in [0u32, 1, 0xffff_ffff, 0x8000_0000, 4096 * 1024] { + let mut b = valid.clone(); + b[off..off + 4].copy_from_slice(&v.to_le_bytes()); + let (_d, p) = write_temp(&b); + let _ = identify(&p); + } + } + for _ in 0..300 { + let mut b = valid.clone(); + for _ in 0..1 + next() % 6 { + let at = (next() % b.len() as u64) as usize; + b[at] = next() as u8; + } + let (_d, p) = write_temp(&b); + let _ = identify(&p); + } + } + + #[test] + fn non_pdb_files_are_rejected_as_invalid() { + for (label, bytes) in [ + ("empty", Vec::new()), + ("msf magic only", super::MSF7_MAGIC.to_vec()), + ("zeros", vec![0u8; 8192]), + ("an ELF header", b"\x7fELF\x02\x01\x01".repeat(300)), + ] { + let (_d, p) = write_temp(&bytes); + assert!( + matches!(identify(&p), Err(Error::InputInvalid(_))), + "{label}" + ); + } + } + /// The upload key. This exact string is what the worker re-derives from the /// same bytes (`symbolfiles/pdb.py`, same `symbolic` major) and what a /// crashing Windows module resolves against β€” changing the formatting (to diff --git a/tests/broken_files.rs b/tests/broken_files.rs new file mode 100644 index 0000000..85ca2ad --- /dev/null +++ b/tests/broken_files.rs @@ -0,0 +1,824 @@ +//! How the compiled binary behaves on BROKEN input files: corrupted or truncated JSON, +//! ELF, Mach-O/DWARF, PDB, zip, plist and dependency manifests. +//! +//! The contract these pin, per flow: +//! - never a crash (no panic exit 101, no signal / abort); +//! - a stable exit code an integrator can branch on, and a message naming the file; +//! - nothing is sent to the server for input that could not be identified; +//! - the user's own files are left exactly as they were; +//! - stdout stays reserved for structured output. +//! +//! Where a flow deliberately TOLERATES a broken file (it warns and carries on), the test +//! says so, so loosening or tightening that is a conscious change. + +use std::path::{Path, PathBuf}; + +use serde_json::{json, Value}; +use wiremock::matchers::{method, path as wm_path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +mod common; + +const FIXTURE_ELF: &str = "tests/fixtures/elf/libsymbol1.so"; + +fn write(dir: &Path, rel: &str, bytes: &[u8]) -> PathBuf { + let p = dir.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, bytes).unwrap(); + p +} + +fn fixture_elf() -> Vec { + std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(FIXTURE_ELF)).unwrap() +} + +/// Exit code of a finished process, failing the test on a crash (panic = 101 or a signal). +fn code_of(out: &std::process::Output) -> i32 { + let code = out.status.code().unwrap_or(-1); + assert!( + code != 101 && code >= 0, + "the binary CRASHED (status {:?}):\n{}", + out.status, + String::from_utf8_lossy(&out.stderr) + ); + code +} + +fn run(args: &[&str]) -> std::process::Output { + common::cli().args(args).output().unwrap() +} + +fn stderr(out: &std::process::Output) -> String { + String::from_utf8_lossy(&out.stderr).into_owned() +} + +// --------------------------------------------------------------------------- +// Source-map JSON: upload, --strip-sources-content and inject, one table +// --------------------------------------------------------------------------- + +const VALID_MAP: &str = r#"{"version":3,"debug_id":"22222222-2222-2222-2222-222222222222","sources":["a"],"sourcesContent":["x"],"mappings":"AAAA"}"#; + +/// (label, map bytes, upload dry-run, strip dry-run, inject) exit codes. +/// +/// 11 = input invalid. Non-object but well-formed JSON (`[]`, `3`) is NOT a syntax +/// error: upload / strip dry runs tolerate it (there is no id to read), while +/// `inject` cannot write an id into it and says so. +fn json_cases() -> Vec<(&'static str, Vec, i32, i32, i32)> { + let deep_array = format!("{}{}", "[".repeat(100_000), "]".repeat(100_000)); + let deep_object = format!("{}1{}", r#"{"a":"#.repeat(50_000), "}".repeat(50_000)); + let bom = format!("\u{feff}{VALID_MAP}"); + let trailing_doc = format!("{VALID_MAP}{VALID_MAP}"); + let truncated = VALID_MAP[..40].to_string(); + let trailing_comma = format!("{},}}", &VALID_MAP[..VALID_MAP.len() - 1]); + let c = |label, bytes: &str, a, b, c| (label, bytes.as_bytes().to_vec(), a, b, c); + vec![ + c("empty", "", 11, 11, 11), + c("whitespace only", " \n\t ", 11, 11, 11), + c("truncated", &truncated, 11, 11, 11), + c("trailing comma", &trailing_comma, 11, 11, 11), + c("byte-order mark", &bom, 11, 11, 11), + c("two documents", &trailing_doc, 11, 11, 11), + c("NaN literal", r#"{"a":NaN}"#, 11, 11, 11), + c("bad escape", r#"{"a":"\x"}"#, 11, 11, 11), + c("lone surrogate", r#"{"debug_id":"\ud800"}"#, 11, 11, 11), + c( + "raw control char in a string", + "{\"a\":\"x\ty\"}", + 11, + 11, + 11, + ), + c("NUL byte in a string", "{\"a\":\"x\0y\"}", 11, 11, 11), + c("100k-deep arrays", &deep_array, 11, 11, 11), + c("50k-deep objects", &deep_object, 11, 11, 11), + c("array at the top level", "[]", 0, 0, 11), + c("number at the top level", "3", 0, 0, 11), + c("null at the top level", "null", 0, 0, 11), + c("string at the top level", "\"s\"", 0, 0, 11), + // A number no double holds is grammatical JSON: reading ids skips it, but + // inject cannot copy it faithfully, so it refuses. + c( + "out-of-range number", + r#"{"version":3,"n":1e999999,"mappings":""}"#, + 0, + 0, + 11, + ), + // Duplicate keys are legal; the last one wins, as in any JSON parser. + c( + "duplicate keys", + r#"{"debug_id":"a","debug_id":"b","mappings":""}"#, + 0, + 0, + 0, + ), + ] +} + +#[test] +fn broken_source_maps_never_crash_and_exit_with_stable_codes() { + for (label, bytes, want_upload, want_strip, want_inject) in json_cases() { + let tmp = tempfile::tempdir().unwrap(); + let map = write(tmp.path(), "m/app.js.map", &bytes); + let dir = tmp.path().join("m"); + let dir_s = dir.to_str().unwrap(); + let base = [ + "debug-files", + "upload", + "--type", + "sourcemaps", + "--version", + "1", + "--build", + "1", + "--dry-run", + ]; + + let up = common::cli().args(base).arg(&dir).output().unwrap(); + assert_eq!( + code_of(&up), + want_upload, + "upload: {label}\n{}", + stderr(&up) + ); + assert!( + up.stdout.is_empty(), + "upload: {label}: stdout must stay empty" + ); + + let strip = common::cli() + .args(base) + .arg("--strip-sources-content") + .arg(&dir) + .output() + .unwrap(); + assert_eq!( + code_of(&strip), + want_strip, + "strip: {label}\n{}", + stderr(&strip) + ); + + // Inject needs a bundle next to the map; it must not touch a map it rejects. + write(tmp.path(), "m/app.js", b"a()\n"); + let inj = run(&["sourcemaps", "inject", dir_s]); + assert_eq!( + code_of(&inj), + want_inject, + "inject: {label}\n{}", + stderr(&inj) + ); + if want_inject == 11 { + assert_eq!( + std::fs::read(&map).unwrap(), + bytes, + "inject rewrote a map it rejected: {label}" + ); + let left: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(left.len(), 2, "{label}: stray files left behind: {left:?}"); + } + if want_upload == 11 { + assert!( + stderr(&up).contains(map.to_str().unwrap()) || stderr(&up).contains("sourcemap"), + "{label}: the message must name the problem: {}", + stderr(&up) + ); + } + } +} + +/// A real (non-dry-run) upload of a map that cannot be identified registers NOTHING. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unidentifiable_source_map_sends_nothing_to_the_server() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"code": 0}))) + .expect(0) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + let tmp = tempfile::tempdir().unwrap(); + for (i, bytes) in [&b"{ not json"[..], b"[]", b"\"s\"", b"", b"{\"a\":"] + .iter() + .enumerate() + { + write(tmp.path(), &format!("m{i}/app.js.map"), bytes); + } + let endpoint = server.uri(); + let root = tmp.path().to_path_buf(); + tokio::task::spawn_blocking(move || { + for i in 0..5 { + let out = common::cli() + .args(["--endpoint", &endpoint, "--app-token", "TKN"]) + .args([ + "debug-files", + "upload", + "--type", + "sourcemaps", + "--version", + "1", + "--build", + "1", + ]) + .arg(root.join(format!("m{i}"))) + .output() + .unwrap(); + let code = code_of(&out); + assert_ne!(code, 0, "case m{i} must fail: {}", stderr(&out)); + } + }) + .await + .unwrap(); +} + +/// `--strip-sources-content` over a map it cannot parse uploads the ORIGINAL +/// (a privacy preference is not a reason to fail), but one that is valid JSON with +/// sources still has them removed. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn strip_over_an_unparseable_map_falls_back_to_the_original_bytes() { + let server = MockServer::start().await; + let put_url = format!("{}/put", server.uri()); + Mock::given(method("POST")) + .and(wm_path("/apps/TKN/symbols")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"code": 0, "endpoint": put_url})), + ) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let tmp = tempfile::tempdir().unwrap(); + // Valid JSON with a debug id (so it can be keyed) but a number the stream cannot + // copy: strip cannot produce a copy, so the original is uploaded. + let odd = br#"{"version":3,"debug_id":"55555555-5555-5555-5555-555555555555","n":1e999999,"sourcesContent":["secret"],"mappings":""}"#; + write(tmp.path(), "m/app.js.map", odd); + let endpoint = server.uri(); + let dir = tmp.path().join("m"); + tokio::task::spawn_blocking(move || { + let out = common::cli() + .args(["--endpoint", &endpoint, "--app-token", "TKN"]) + .args([ + "debug-files", + "upload", + "--type", + "sourcemaps", + "--strip-sources-content", + ]) + .args(["--version", "1", "--build", "1"]) + .arg(&dir) + .output() + .unwrap(); + assert_eq!(code_of(&out), 0, "{}", stderr(&out)); + }) + .await + .unwrap(); + let reqs = server.received_requests().await.unwrap(); + let put = reqs + .iter() + .find(|r| r.method.as_str() == "PUT") + .expect("uploaded"); + let mut archive = zip::ZipArchive::new(std::io::Cursor::new(put.body.clone())).unwrap(); + let mut body = Vec::new(); + std::io::Read::read_to_end(&mut archive.by_index(0).unwrap(), &mut body).unwrap(); + assert_eq!(body, odd.to_vec(), "the original bytes, untouched"); +} + +// --------------------------------------------------------------------------- +// ELF: damaged libraries in a directory, damaged zips +// --------------------------------------------------------------------------- + +fn elf_dry_run(input: &Path) -> std::process::Output { + common::cli() + .args([ + "debug-files", + "upload", + "--type", + "elf", + "--version", + "1", + "--build", + "1", + ]) + .args([ + "--uuid", + "00000000-0000-0000-0000-000000000000", + "--dry-run", + ]) + .arg(input) + .output() + .unwrap() +} + +/// A directory with one good library and every kind of damaged one: the damaged are +/// warned about and skipped, the good one is still uploaded, and the run succeeds. +#[test] +fn damaged_libraries_are_skipped_with_a_warning_and_the_good_one_survives() { + let elf = fixture_elf(); + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "libs/libgood.so", &elf); + write(tmp.path(), "libs/libempty.so", b""); + write(tmp.path(), "libs/libhalf.so", &elf[..elf.len() / 2]); + write(tmp.path(), "libs/libhead.so", &elf[..64]); + write(tmp.path(), "libs/libzero.so", &vec![0u8; 8192]); + write( + tmp.path(), + "libs/libtext.so", + b"#!/bin/sh\necho not a library\n", + ); + // A VALID Mach-O (it has a real UUID) named *.so, as a macOS-built module would be: + // not an ELF, so it must not become an "elf" symbol keyed by its Mach-O UUID. + let macho = thin_macho_with_uuid([0x5a; 16]); + write(tmp.path(), "libs/libmacho.so", &macho); + + let out = elf_dry_run(&tmp.path().join("libs")); + assert_eq!(code_of(&out), 0, "{}", stderr(&out)); + assert!(out.stdout.is_empty()); + let err = stderr(&out); + assert!(err.contains("would register + upload 1 libraries"), "{err}"); + assert_eq!(err.matches("no GNU build-id").count(), 6, "{err}"); +} + +/// If NOTHING in the directory is usable, that is a run with no symbols: it succeeds +/// (the zip contract) but says so. +#[test] +fn a_directory_of_only_damaged_libraries_uploads_nothing_and_says_so() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "libs/liba.so", b"junk"); + write(tmp.path(), "libs/libb.so", &[0u8; 100]); + let out = elf_dry_run(&tmp.path().join("libs")); + assert_eq!(code_of(&out), 0); + assert!( + stderr(&out).contains("would register + upload 0 libraries"), + "{}", + stderr(&out) + ); +} + +#[test] +fn broken_native_symbol_zips_are_input_invalid() { + let elf = fixture_elf(); + let tmp = tempfile::tempdir().unwrap(); + let good = write(tmp.path(), "good.zip", &[]); + { + let mut zw = zip::ZipWriter::new(std::fs::File::create(&good).unwrap()); + zw.start_file( + "arm64-v8a/libfoo.so", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + std::io::Write::write_all(&mut zw, &elf).unwrap(); + zw.finish().unwrap(); + } + let raw = std::fs::read(&good).unwrap(); + let mut crc = raw.clone(); + let mid = crc.len() / 2; + crc[mid] ^= 0x5a; + let cases: Vec<(&str, Vec)> = vec![ + ("empty file", Vec::new()), + ("truncated in half", raw[..raw.len() / 2].to_vec()), + ( + "end-of-central-directory missing", + raw[..raw.len() - 12].to_vec(), + ), + ( + "random bytes", + (0..4000u32).map(|i| (i * 7 % 251) as u8).collect(), + ), + ("a flipped byte in the payload", crc), + ]; + for (label, bytes) in cases { + let p = write(tmp.path(), "bad.zip", &bytes); + let out = elf_dry_run(&p); + assert_eq!(code_of(&out), 11, "{label}: {}", stderr(&out)); + assert!( + stderr(&out).contains("bad.zip"), + "{label}: names the file: {}", + stderr(&out) + ); + } +} + +// --------------------------------------------------------------------------- +// Mach-O / dSYM / PDB +// --------------------------------------------------------------------------- + +fn bundle_with(parent: &Path, name: &str, dwarf: &[u8]) -> PathBuf { + let b = parent.join(format!("{name}.dSYM")); + write(&b, &format!("Contents/Resources/DWARF/{name}"), dwarf); + b +} + +fn thin_macho_with_uuid(uuid: [u8; 16]) -> Vec { + let mut b = vec![0xcf, 0xfa, 0xed, 0xfe]; + b.extend_from_slice(&0x0100_000cu32.to_le_bytes()); // arm64 + b.extend_from_slice(&0u32.to_le_bytes()); + b.extend_from_slice(&0x0au32.to_le_bytes()); // MH_DSYM + b.extend_from_slice(&1u32.to_le_bytes()); + b.extend_from_slice(&24u32.to_le_bytes()); + b.extend_from_slice(&[0u8; 8]); + b.extend_from_slice(&0x1bu32.to_le_bytes()); // LC_UUID + b.extend_from_slice(&24u32.to_le_bytes()); + b.extend_from_slice(&uuid); + b +} + +#[test] +fn dsym_uuid_on_broken_machos_prints_an_empty_list_and_never_a_nil_uuid() { + let tmp = tempfile::tempdir().unwrap(); + let valid = thin_macho_with_uuid([0xAB; 16]); + let cases: Vec<(&str, Vec, Vec<&str>)> = vec![ + ( + "valid", + valid.clone(), + vec!["ABABABAB-ABAB-ABAB-ABAB-ABABABABABAB"], + ), + ("no LC_UUID (nil)", thin_macho_with_uuid([0; 16]), vec![]), + ("empty", Vec::new(), vec![]), + ("magic only", vec![0xcf, 0xfa, 0xed, 0xfe], vec![]), + ( + "fat header claiming 2^32 slices", + [ + &[0xca, 0xfe, 0xba, 0xbe, 0xff, 0xff, 0xff, 0xff][..], + &[0u8; 64], + ] + .concat(), + vec![], + ), + ( + "garbage", + (0..3000u32).map(|i| (i * 31 % 253) as u8).collect(), + vec![], + ), + ( + "valid, cut mid-command", + valid[..valid.len() - 5].to_vec(), + vec![], + ), + ]; + for (label, bytes, want) in cases { + for target in ["file", "bundle"] { + let bundle = bundle_with(tmp.path(), "App", &bytes); + let path = if target == "file" { + bundle.join("Contents/Resources/DWARF/App") + } else { + bundle + }; + let out = run(&["dsym", "uuid", path.to_str().unwrap()]); + assert_eq!(code_of(&out), 0, "{label}/{target}: {}", stderr(&out)); + let got: Vec = serde_json::from_slice(&out.stdout) + .unwrap_or_else(|e| panic!("{label}/{target}: stdout is not a JSON array ({e})")); + assert_eq!(got, want, "{label}/{target}"); + } + } +} + +#[test] +fn a_dsym_with_unreadable_dwarf_is_skipped_by_upload_but_fails_the_build_phase() { + let tmp = tempfile::tempdir().unwrap(); + for (label, bytes) in [ + ( + "garbage", + (0..3000u32) + .map(|i| (i * 31 % 253) as u8) + .collect::>(), + ), + ("empty", Vec::new()), + ("no LC_UUID", thin_macho_with_uuid([0; 16])), + ] { + let dsyms = tmp.path().join(label.replace(' ', "_")); + bundle_with(&dsyms, "App", &bytes); + + // `debug-files upload`: the bundle is warned about and skipped (exit 0, nothing sent). + let out = common::cli() + .args([ + "debug-files", + "upload", + "--type", + "dsym", + "--version", + "1", + "--build", + "1", + "--dry-run", + ]) + .arg(&dsyms) + .output() + .unwrap(); + assert_eq!(code_of(&out), 0, "{label}: {}", stderr(&out)); + assert!( + stderr(&out).contains("not a readable dSYM bundle"), + "{label}: {}", + stderr(&out) + ); + + // `xcode upload-dsyms`: symbols silently not uploaded is the failure it exists to + // prevent, so the same bundle FAILS the build phase (11). + let out = common::cli() + .args(["xcode", "upload-dsyms", "--no-background"]) + .env("DWARF_DSYM_FOLDER_PATH", &dsyms) + .env("BUGSEE_APP_TOKEN", "TKN") + .env("BUGSEE_ENDPOINT", "http://127.0.0.1:1") + .output() + .unwrap(); + assert_eq!(code_of(&out), 11, "{label}: {}", stderr(&out)); + assert!( + stderr(&out).contains("could not be read"), + "{label}: {}", + stderr(&out) + ); + } +} + +#[test] +fn broken_pdbs_are_not_identified_and_never_crash() { + let tmp = tempfile::tempdir().unwrap(); + let msf = b"Microsoft C/C++ MSF 7.00\r\n\x1aDS\0\0\0"; + for (label, bytes, want) in [ + ("empty", Vec::new(), 10), + ( + "random bytes", + (0..4000u32).map(|i| (i * 13 % 251) as u8).collect(), + 10, + ), + ("MSF magic only", msf.to_vec(), 11), + ( + "MSF magic + zeroed superblock", + [&msf[..], &[0u8; 64]].concat(), + 11, + ), + ( + "MSF magic + hostile superblock", + [&msf[..], &[0xffu8; 64]].concat(), + 11, + ), + ] { + let dir = tmp.path().join(label.replace([' ', '+'], "_")); + write(&dir, "a.pdb", &bytes); + let out = common::cli() + .args([ + "debug-files", + "upload", + "--type", + "pdb", + "--version", + "1", + "--build", + "1", + "--dry-run", + ]) + .arg(&dir) + .output() + .unwrap(); + assert_eq!(code_of(&out), want, "{label}: {}", stderr(&out)); + assert!(out.stdout.is_empty(), "{label}"); + } +} + +// --------------------------------------------------------------------------- +// Plists +// --------------------------------------------------------------------------- + +/// `build-env read-plist` reads `{}` for anything unusable, INCLUDING nesting deep +/// enough to overflow a stack (which used to abort the process with SIGABRT). +#[test] +fn read_plist_of_broken_files_is_an_empty_object_and_never_a_crash() { + let tmp = tempfile::tempdir().unwrap(); + let nested_xml = [ + &b""[..], + &b"".repeat(60_000), + &b"".repeat(60_000), + b"", + ] + .concat(); + for (label, bytes) in [ + ("empty", Vec::new()), + ( + "random bytes", + (0..300u32).map(|i| (i * 37 % 251) as u8).collect(), + ), + ( + "truncated XML", + b"CFBundle".to_vec(), + ), + ("bplist magic only", b"bplist00".to_vec()), + ( + "array root", + b"".to_vec(), + ), + ("60k-deep nesting", nested_xml), + ] { + let p = write(tmp.path(), "Info.plist", &bytes); + let out = run(&["build-env", "read-plist", p.to_str().unwrap()]); + assert_eq!(code_of(&out), 0, "{label}: {}", stderr(&out)); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "{}", "{label}"); + } +} + +// --------------------------------------------------------------------------- +// iOS dependency manifests +// --------------------------------------------------------------------------- + +fn ios_deps(pods: &[u8], pins: &[u8]) -> Value { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "Podfile.lock", pods); + write(tmp.path(), "Package.resolved", pins); + let out = common::cli() + .args(["ios-deps", "collect", "--project-root"]) + .arg(tmp.path()) + .output() + .unwrap(); + assert_eq!(code_of(&out), 0, "{}", stderr(&out)); + serde_json::from_slice(&out.stdout).expect("stdout is always the JSON document") +} + +/// Broken manifests degrade to fewer entries, never to a crash, a non-JSON stdout, or a +/// bogus nameless dependency in the graph. +#[test] +fn broken_dependency_manifests_still_yield_a_clean_json_document() { + let deep = [&b"["[..].repeat(100_000)[..], &b"]".repeat(100_000)].concat(); + let big_lock = [&b"PODS:\n"[..], &b" - Foo (1.0)\n".repeat(50_000)].concat(); + let cases: Vec<(&str, Vec, Vec)> = vec![ + ("empty", Vec::new(), Vec::new()), + ( + "random bytes", + (0..500u32).map(|i| (i * 37 % 251) as u8).collect(), + (0..500u32).map(|i| (i * 53 % 251) as u8).collect(), + ), + ( + "truncated", + b"PODS:\n - Foo (1.0\n - ".to_vec(), + br#"{"pins":[{"identity":"a","state":{"ver"#.to_vec(), + ), + ( + "not UTF-8", + b"PODS:\n - \xff\xfe (1.0)\n".to_vec(), + b"\xff".to_vec(), + ), + ( + "nameless pod lines", + b"PODS:\n - (1.0)\n - \n - Real (2.0)\n".to_vec(), + Vec::new(), + ), + ("100k-deep JSON", Vec::new(), deep), + ("50k pods", big_lock, Vec::new()), + ]; + for (label, pods, pins) in cases { + let doc = ios_deps(&pods, &pins); + let entries = doc["entries"] + .as_array() + .unwrap_or_else(|| panic!("{label}: no entries array")); + for e in entries { + let name = e["name"].as_str().unwrap_or(""); + assert!( + !name.is_empty(), + "{label}: nameless dependency in the graph: {e}" + ); + assert_ne!(e["id"], "library::", "{label}"); + } + } + // The survivable line of a mangled lockfile is still reported. + let doc = ios_deps(b"PODS:\n - (1.0)\n - \n - Real (2.0)\n", b""); + let names: Vec<&str> = doc["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["Real"]); +} + +// --------------------------------------------------------------------------- +// Build registration payloads and artefacts +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_broken_build_payload_fails_before_anything_is_sent() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"ok": true}))) + .expect(0) + .mount(&server) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + let tmp = tempfile::tempdir().unwrap(); + let artifact = write(tmp.path(), "app.aab", b"PK\x03\x04 fake"); + let endpoint = server.uri(); + let root = tmp.path().to_path_buf(); + tokio::task::spawn_blocking(move || { + for (label, payload) in [ + ("empty", &b""[..]), + ("truncated", br#"{"uuid":"a","vers"#), + ("an array", b"[]"), + ("null", b"null"), + ("binary", &[0xff, 0xfe, 0x00, 0x01]), + ] { + let p = write(&root, "p.json", payload); + let out = common::cli() + .args([ + "--endpoint", + &endpoint, + "--app-token", + "TKN", + "upload", + "build", + ]) + .arg("--payload-json") + .arg(&p) + .arg("--artifact") + .arg(&artifact) + .output() + .unwrap(); + assert_eq!(code_of(&out), 11, "{label}: {}", stderr(&out)); + assert!( + stderr(&out).contains("--payload-json"), + "{label}: {}", + stderr(&out) + ); + } + }) + .await + .unwrap(); +} + +#[test] +fn unusable_build_artifacts_are_input_not_found() { + let tmp = tempfile::tempdir().unwrap(); + let payload = write(tmp.path(), "p.json", br#"{"uuid":"a"}"#); + let dir = tmp.path().join("a_directory"); + std::fs::create_dir_all(&dir).unwrap(); + for (label, artifact) in [ + ("a directory", dir), + ("a missing file", tmp.path().join("nope.aab")), + ] { + let out = common::cli() + .args([ + "--endpoint", + "http://127.0.0.1:1", + "--app-token", + "TKN", + "upload", + "build", + "--dry-run", + ]) + .arg("--payload-json") + .arg(&payload) + .arg("--artifact") + .arg(&artifact) + .output() + .unwrap(); + assert_eq!(code_of(&out), 10, "{label}: {}", stderr(&out)); + } +} + +// --------------------------------------------------------------------------- +// IL2CPP line maps: a documented gap (characterisation) +// --------------------------------------------------------------------------- + +/// The IL2CPP mappings JSON is packed as-is and NOT validated: a corrupt file uploads +/// "successfully". This pins that on purpose β€” rejecting it would be a behaviour +/// change integrators must be told about β€” so a future validation lands deliberately. +#[test] +fn il2cpp_linemap_does_not_validate_the_mappings_json_today() { + let tmp = tempfile::tempdir().unwrap(); + for (label, bytes) in [ + ("empty", &b""[..]), + ("truncated", br#"{"a":"#), + ("binary", &[0xff, 0xfe, 0x00]), + ] { + let dir = tmp.path().join(label); + write(&dir, "LineNumberMappings.json", bytes); + let out = common::cli() + .args([ + "debug-files", + "upload", + "--type", + "il2cpp-linemap", + "--version", + "1", + "--build", + "1", + ]) + .args([ + "--uuid", + "11111111-2222-3333-4444-555555555555", + "--dry-run", + ]) + .arg(&dir) + .output() + .unwrap(); + assert_eq!(code_of(&out), 0, "{label}: {}", stderr(&out)); + } +} From 01e17d9c20b57cd5b8b8181776eea14f87821b60 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:26:04 +0500 Subject: [PATCH 12/18] test(elf): gate the unix-only fixture helper (dead code is an error on Windows CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The helper's only user is a #[cfg(unix)] test, so the Windows test jobs failed to compile under -D warnings. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- tests/elf_upload.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/elf_upload.rs b/tests/elf_upload.rs index bbe7954..62eeebd 100644 --- a/tests/elf_upload.rs +++ b/tests/elf_upload.rs @@ -583,6 +583,9 @@ fn elf_upload_empty_directory_is_input_not_found() { } /// The fixture with its GNU build-id's first byte flipped: same ELF, different identity. +/// (Unix-only because its sole user is: an unused helper is a hard error under CI's +/// `-D warnings` on Windows.) +#[cfg(unix)] fn fixture_with_other_build_id() -> (Vec, String) { let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/elf/libsymbol1.so"); let mut bytes = std::fs::read(&fixture).unwrap(); From 1ab176ab556b0f582243a22efcde2fcf23c62989 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:28:54 +0500 Subject: [PATCH 13/18] docs(elf): state the mmap contract - directories must not be written while scanned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (Cursor, #72): a library truncated or rewritten under its mapping is undefined behaviour (SIGBUS in practice). That is acceptable for build-finalized outputs, but the constraint is now stated where integrators read it: the SAFETY note, `--help` for `paths`, and the changelog. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 3 +++ src/cli/debug_files.rs | 4 +++- src/symbols/elf.rs | 8 +++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eea103..b0d127d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). SHA-1 are streamed from disk in 64 KiB chunks (the PUT keeps an exact `Content-Length` and re-opens the file per retry), and the ELF identity scan memory-maps libraries instead of reading them. Peak RSS for four 100 MB libraries dropped from 564 MB to 36 MB. Wire format unchanged. + The one new constraint: a directory passed to `--type elf` must not be written to while it is + scanned (a library truncated under its mapping is undefined behaviour, in practice SIGBUS), so + run it after the native build task has finished, as the Gradle task graph does. ## [0.7.13] - 2026-10-07 diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index f5ebf75..c931c20 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -21,7 +21,9 @@ pub enum DebugFilesCommand { /// is walked recursively for native libraries (e.g. AGP's /// `merged_native_libs`; directory symlinks are not descended, an I/O error /// fails the run, and no match exits 10), and a file is a - /// `native-debug-symbols.zip`. + /// `native-debug-symbols.zip`. The libraries are memory-mapped while scanned, so + /// they must be finished build outputs: do not run this while a linker is still + /// writing into the directory. #[arg(required = true)] paths: Vec, diff --git a/src/symbols/elf.rs b/src/symbols/elf.rs index 2c3a316..a445573 100644 --- a/src/symbols/elf.rs +++ b/src/symbols/elf.rs @@ -179,9 +179,11 @@ pub fn scan_dir(dir: &Path, extra: &ExtraSuffixes) -> std::io::Result Date: Wed, 7 Oct 2026 17:31:27 +0500 Subject: [PATCH 14/18] fix(inject): rewrite a map's content in place instead of fs::copy from a 0600 temp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (Cursor, #74): fs::copy carries the source file's metadata onto the destination, so every rewritten map was tightened to the temp file's 0o600 (and on Windows given FILE_ATTRIBUTE_TEMPORARY): unreadable by a deploy user afterwards. Write the temp file's content into the original through a truncating open instead, which keeps mode, inode, symlink and hard links. Pinned with a 0o644 map that is also hard-linked (verified to fail with fs::copy back). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- src/inject/mod.rs | 48 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/inject/mod.rs b/src/inject/mod.rs index a028762..d0b4d3f 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -570,9 +570,10 @@ fn write_map_debug_id( } } if !dry_run { - // Pass 2: stream the map into a sibling temp file with the ids set, then - // copy it over the original (not rename: that would swap the inode and - // drop the permissions / symlink / hard-link the build gave the map). + // Pass 2: stream the map into a sibling temp file with the ids set, then write + // its CONTENT into the original (not rename: that would swap the inode and + // drop the symlink / hard link the build gave the map; and not `fs::copy`: + // it also copies the temp file's metadata, i.e. its 0o600 mode, onto the map). let dir = map_path.parent().filter(|p| !p.as_os_str().is_empty()); let tmp = match dir { Some(d) => tempfile::NamedTempFile::new_in(d)?, @@ -590,7 +591,11 @@ fn write_map_debug_id( })?; std::io::Write::flush(&mut out)?; } - std::fs::copy(tmp.path(), map_path)?; + let mut dest = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(map_path)?; + std::io::copy(&mut std::fs::File::open(tmp.path())?, &mut dest)?; } Ok(true) } @@ -650,6 +655,41 @@ mod tests { } } + /// The rewrite must leave the map's mode, inode and links as the bundler made them. A + /// sibling temp file is 0o600, so any step that carries ITS metadata over (as + /// `fs::copy` does) silently tightens a 0o644 map to owner-only. Distinct modes on + /// the map and the bundle make that visible. + #[cfg(unix)] + #[test] + fn rewriting_a_map_keeps_its_mode_inode_and_hard_links() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + let link = dir.path().join("a.js.map.link"); + std::fs::write(&js, "a()\n").unwrap(); + std::fs::write(&map, r#"{"version":3,"mappings":"AAAA"}"#).unwrap(); + std::fs::hard_link(&map, &link).unwrap(); + std::fs::set_permissions(&map, std::fs::Permissions::from_mode(0o644)).unwrap(); + let ino = std::fs::metadata(&map).unwrap().ino(); + + inject_paths(&[dir.path().to_path_buf()], &[], true, false).unwrap(); + + let after = std::fs::metadata(&map).unwrap(); + assert_eq!( + after.permissions().mode() & 0o777, + 0o644, + "the mode must not be tightened" + ); + assert_eq!(after.ino(), ino, "same inode: rewritten in place"); + assert!(std::fs::read_to_string(&map).unwrap().contains("debug_id")); + assert_eq!( + std::fs::read(&link).unwrap(), + std::fs::read(&map).unwrap(), + "a hard link sees the new content" + ); + } + #[test] fn is_contained_relative_rejects_traversal_and_absolute() { assert!(is_contained_relative("bundle.js.map")); From 0f388c27aae832e7073f43ffd84d547dddf7d9bf Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:39:06 +0500 Subject: [PATCH 15/18] fix(upload): follow 307/308 on streamed PUTs ourselves; correct stale docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (Fable): reqwest follows a 307/308 only when it can replay the body, and a streamed body cannot be replayed, so any redirect on a symbol / artefact / build-info PUT became a hard failure (exit 30) where it used to be followed transparently. S3 answers a freshly created bucket's presigned URL with exactly such a temporary 307. http::put_file now owns the streaming PUT for all three call sites: retries like send_with_retry, re-issues the PUT (new stream from the file, same Content-Length and Content-Type) to Location on 307/308, up to 5 hops, and passes an unusable redirect through for the caller to report. Also: send_with_retry's doc no longer claims callers clone the body, and the mapped.rs note no longer calls its unsafe the crate's only one. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- src/symbols/mapped.rs | 5 +- src/upload/build.rs | 15 ++-- src/upload/build_info.rs | 15 ++-- src/upload/http.rs | 165 ++++++++++++++++++++++++++++++++++++++- src/upload/presigned.rs | 44 +++++++++-- 5 files changed, 218 insertions(+), 26 deletions(-) diff --git a/src/symbols/mapped.rs b/src/symbols/mapped.rs index 35ace55..0709ed8 100644 --- a/src/symbols/mapped.rs +++ b/src/symbols/mapped.rs @@ -3,8 +3,9 @@ //! Identifying a debug file (ELF build-id, Mach-O UUID, PDB GUID) touches only //! its headers, yet the files are routinely 100+ MB and `std::fs::read` copies //! every byte onto the heap. Mapping instead keeps the cost to the pages the -//! parser actually reads. This is the ONLY place the crate maps a file, so the -//! `unsafe` lives here. +//! parser actually reads. This is the ONLY place the crate memory-maps a file, so +//! that `unsafe` lives here (the crate has two other, unrelated `unsafe` uses: the +//! `fork` in `daemon.rs` and `gethostname` in `build_env.rs`). use std::ops::Deref; use std::path::Path; diff --git a/src/upload/build.rs b/src/upload/build.rs index 483fc83..b6f3ddb 100644 --- a/src/upload/build.rs +++ b/src/upload/build.rs @@ -188,13 +188,14 @@ pub async fn run(params: Params<'_>, policy: RetryPolicy) -> Result { // reading it whole and cloning it per attempt held two copies in memory. let body_len = http::file_len(&zip_path).await?; tracing::debug!(endpoint = %http::redact_url(®.artifact_endpoint), body_len, "PUT artefact"); - let put = http::send_with_retry(policy, "artefact PUT", true, || { - client - .put(®.artifact_endpoint) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .header(reqwest::header::CONTENT_LENGTH, body_len) - .body(http::file_body(&zip_path)) - }) + let put = http::put_file( + &client, + policy, + "artefact PUT", + ®.artifact_endpoint, + &zip_path, + Some("application/octet-stream"), + ) .await?; if !put.status().is_success() { let s = put.status().as_u16(); diff --git a/src/upload/build_info.rs b/src/upload/build_info.rs index 8deee9e..31a884f 100644 --- a/src/upload/build_info.rs +++ b/src/upload/build_info.rs @@ -148,13 +148,14 @@ pub async fn run(params: Params<'_>, policy: RetryPolicy) -> Result { // Omitting it makes S3 recompute a different signature β†’ 403 // SignatureDoesNotMatch. The artefact (`build.rs`) and chunk (`chunked.rs`) // PUTs set the same header for the same reason. - let put = http::send_with_retry(policy, "build-info PUT", true, || { - client - .put(&presigned) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .header(reqwest::header::CONTENT_LENGTH, body_len) - .body(http::file_body(&zip_path)) - }) + let put = http::put_file( + &client, + policy, + "build-info PUT", + &presigned, + &zip_path, + Some("application/octet-stream"), + ) .await?; let put_status = put.status(); diff --git a/src/upload/http.rs b/src/upload/http.rs index 4bef074..807d6f6 100644 --- a/src/upload/http.rs +++ b/src/upload/http.rs @@ -100,8 +100,8 @@ pub fn is_retriable_status(status: reqwest::StatusCode) -> bool { /// statuses. /// /// `make` is invoked once per attempt and MUST build an equivalent request each -/// time (callers clone the body β€” fine for the small JSON POSTs and in-memory -/// bundle PUTs this module serves). Returns the final response (which the caller +/// time (small JSON POSTs clone their body; a file PUT builds a fresh +/// [`file_body`] stream per attempt β€” see [`put_file`]). Returns the final response (which the caller /// inspects for success / parses) once it is a success, is non-retriable, or the /// attempts are exhausted; returns `UploadTransport` only when every attempt /// failed at the transport layer. @@ -261,12 +261,173 @@ pub async fn file_len(path: &std::path::Path) -> std::io::Result { Ok(tokio::fs::metadata(path).await?.len()) } +/// How many 307/308 hops [`put_file`] follows before giving up. +const MAX_PUT_REDIRECTS: usize = 5; + +/// PUT `path` to `url`, streaming it, retrying like [`send_with_retry`], and following +/// 307/308 redirects ITSELF. +/// +/// reqwest follows a 307/308 only when it can replay the request body, and a streamed +/// body cannot be replayed: it hands the 3xx back instead, which would fail the upload +/// (S3 answers a freshly created bucket's presigned URL with a temporary 307 to the +/// regional endpoint). Re-issuing the PUT to `Location` with a new stream from the +/// file is exactly what following it means. `content_type`, when set, is sent on every hop +/// (a presigned URL's signature may cover it). Returns the final response for the caller +/// to judge (success or not). +pub async fn put_file( + client: &reqwest::Client, + policy: RetryPolicy, + what: &str, + url: &str, + path: &std::path::Path, + content_type: Option<&str>, +) -> Result { + let len = file_len(path).await?; + let mut target = reqwest::Url::parse(url) + .map_err(|e| Error::UploadTransport(format!("{what}: invalid URL: {e}")))?; + for _ in 0..=MAX_PUT_REDIRECTS { + let resp = send_with_retry(policy, what, true, || { + let mut req = client + .put(target.clone()) + .header(reqwest::header::CONTENT_LENGTH, len) + .body(file_body(path)); + if let Some(ct) = content_type { + req = req.header(reqwest::header::CONTENT_TYPE, ct); + } + req + }) + .await?; + if !matches!(resp.status().as_u16(), 307 | 308) { + return Ok(resp); + } + let Some(next) = resp + .headers() + .get(reqwest::header::LOCATION) + .and_then(|v| v.to_str().ok()) + .and_then(|loc| target.join(loc).ok()) + else { + return Ok(resp); // a redirect with no usable Location: let the caller report it + }; + tracing::debug!( + from = %redact_url(target.as_str()), + to = %redact_url(next.as_str()), + status = resp.status().as_u16(), + "{what}: following redirect" + ); + target = next; + } + Err(Error::UploadTransport(format!( + "{what}: more than {MAX_PUT_REDIRECTS} redirects" + ))) +} + #[cfg(test)] mod tests { use super::*; use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; + // ── put_file: streaming PUT that follows 307/308 itself ────────────────── + + fn payload(len: usize) -> (tempfile::TempDir, std::path::PathBuf, Vec) { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("payload.zip"); + let data: Vec = (0..len).map(|i| (i % 251) as u8).collect(); + std::fs::write(&p, &data).unwrap(); + (dir, p, data) + } + + /// reqwest cannot replay a streamed body, so it hands a 307/308 back instead of + /// following it. `put_file` must follow it itself β€” with the WHOLE body again, the same + /// Content-Type, and a relative `Location` resolved against the current URL. + #[tokio::test] + async fn put_file_follows_307_and_308_resending_the_whole_body() { + for status in [307u16, 308] { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(wiremock::matchers::path("/first")) + .respond_with( + ResponseTemplate::new(status).insert_header("Location", "/regional/second"), + ) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(wiremock::matchers::path("/regional/second")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (_d, p, data) = payload(200_000); + let client = build_client().unwrap(); + let resp = put_file( + &client, + RetryPolicy::none(), + "test PUT", + &format!("{}/first", server.uri()), + &p, + Some("application/octet-stream"), + ) + .await + .unwrap(); + assert!(resp.status().is_success(), "{status}"); + let reqs = server.received_requests().await.unwrap(); + assert_eq!(reqs.len(), 2, "{status}: one redirect hop"); + for r in &reqs { + assert_eq!(r.body, data, "{status}: every hop carries the full body"); + assert_eq!( + r.headers.get("content-type").unwrap(), + "application/octet-stream" + ); + assert_eq!( + r.headers.get("content-length").unwrap().to_str().unwrap(), + data.len().to_string() + ); + } + } + } + + #[tokio::test] + async fn put_file_gives_up_on_a_redirect_loop_and_passes_through_an_unusable_redirect() { + // A loop: every hop redirects back. + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(307).insert_header("Location", "/again")) + .mount(&server) + .await; + let (_d, p, _) = payload(10); + let client = build_client().unwrap(); + let err = put_file( + &client, + RetryPolicy::none(), + "loop PUT", + &format!("{}/x", server.uri()), + &p, + None, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("redirects"), "{err}"); + assert!(server.received_requests().await.unwrap().len() <= MAX_PUT_REDIRECTS + 1); + + // A 307 with no Location is returned for the caller to report, not looped on. + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(307)) + .mount(&server) + .await; + let resp = put_file( + &client, + RetryPolicy::none(), + "bare PUT", + &format!("{}/x", server.uri()), + &p, + None, + ) + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 307); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + #[test] fn backoff_doubles_and_caps() { let p = RetryPolicy { diff --git a/src/upload/presigned.rs b/src/upload/presigned.rs index 63724d4..99711fb 100644 --- a/src/upload/presigned.rs +++ b/src/upload/presigned.rs @@ -232,16 +232,10 @@ pub async fn put_payload( // attempt) held two-plus copies of the archive in memory per upload. The // length is sent explicitly because a streamed body would otherwise go out // chunked, which a presigned S3 PUT rejects. - let len = http::file_len(payload).await?; // The PUT is idempotent (overwrite of the same key) β€” retry on transport // AND retriable status. Each attempt re-opens the file from the start. - let put_resp = http::send_with_retry(policy, "symbol PUT", true, || { - client - .put(presigned_url) - .header(reqwest::header::CONTENT_LENGTH, len) - .body(http::file_body(payload)) - }) - .await?; + let put_resp = + http::put_file(client, policy, "symbol PUT", presigned_url, payload, None).await?; let put_status = put_resp.status(); if !put_status.is_success() { @@ -622,6 +616,40 @@ mod tests { } /// A payload that vanished is a reported error, not a panic or a hang. + /// A freshly created S3 bucket answers the presigned URL with a temporary 307 to the + /// regional endpoint; the symbol PUT must follow it (a streamed body is not replayed by + /// reqwest on its own). + #[tokio::test] + async fn put_payload_follows_a_temporary_redirect() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/bucket/key")) + .respond_with( + ResponseTemplate::new(307) + .insert_header("Location", format!("{}/regional/key", server.uri())), + ) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path("/regional/key")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + let (payload, data) = patterned_payload(150_001); + let client = http::build_client().unwrap(); + put_payload( + &client, + RetryPolicy::none(), + &format!("{}/bucket/key", server.uri()), + payload.path(), + ) + .await + .unwrap(); + let reqs = server.received_requests().await.unwrap(); + assert_eq!(reqs.len(), 2); + assert_eq!(reqs[1].body, data); + } + #[tokio::test] async fn put_payload_reports_a_missing_payload_file() { let server = MockServer::start().await; From 3aa901345f5328c223ac6488cd1897fa0fb48f7d Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:40:14 +0500 Subject: [PATCH 16/18] fix(mapjson): copy any valid JSON number; anonymous scratch file; warn when strip is skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (Fable): - struson's default reader rejects exponents beyond +-99 and numbers over 100 chars (a guard for code that PARSES numbers). We only skip or copy their text, but transfer_to enforced it: a map with `1e100` made --strip-sources-content give up and upload the UNSTRIPPED map (sources included), exit 0, silently; inject exited 11. Reader now built with restrict_number_values = false (verified: the new test fails with it back on). - The strip fallback for an unusable map now logs a warning saying the upload is UNSTRIPPED instead of passing silently. - The map rewrite's scratch file is tempfile::tempfile() (anonymous, OS temp dir) instead of a sibling NamedTempFile: no write access to the map's directory needed (fs::write never needed it), and a killed run leaves no stray .tmpXXXX. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- src/cli/debug_files.rs | 15 ++++++++++++-- src/inject/mapjson.rs | 45 ++++++++++++++++++++++++++++++++++++++---- src/inject/mod.rs | 22 ++++++++++----------- 3 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index 16b7f1d..f7da7df 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -1953,8 +1953,19 @@ fn stripped_copy_without_sources( std::io::BufWriter::new(std::fs::File::create(&out).map_err(crate::error::Error::Io)?); match crate::inject::mapjson::strip_sources_content(input, output) { Ok(true) => {} - // Nothing to strip, or not the JSON object we expect: the copy is moot. - Ok(false) | Err(crate::inject::mapjson::MapJsonError::Json(_)) => { + // Nothing to strip: the copy is moot. + Ok(false) => { + let _ = std::fs::remove_file(&out); + return Ok(None); + } + // Not the JSON object we expect. A privacy preference is not a reason to fail the + // upload, but it must not fail SILENTLY either: say that the sources are NOT stripped. + Err(crate::inject::mapjson::MapJsonError::Json(why)) => { + tracing::warn!( + path = %map_path.display(), + reason = %why, + "--strip-sources-content could not process this map; uploading it UNSTRIPPED" + ); let _ = std::fs::remove_file(&out); return Ok(None); } diff --git a/src/inject/mapjson.rs b/src/inject/mapjson.rs index 7d74c77..ca8e50a 100644 --- a/src/inject/mapjson.rs +++ b/src/inject/mapjson.rs @@ -12,7 +12,7 @@ use std::io::{Read, Write}; -use struson::reader::{JsonReader, JsonStreamReader, ReaderError, ValueType}; +use struson::reader::{JsonReader, JsonStreamReader, ReaderError, ReaderSettings, ValueType}; use struson::writer::{JsonStreamWriter, JsonWriter}; /// Why a streaming pass over a map failed. @@ -61,6 +61,22 @@ impl From for MapJsonError { type Res = std::result::Result; +/// A reader that does NOT apply struson's default "restrict number values" (it rejects +/// an exponent beyond Β±99 and any number over 100 characters). That guard protects code +/// that PARSES numbers into big integers; this module never parses one, it only skips or +/// copies the number's text. Left on, valid JSON such as `1e100` made a copy fail, and +/// `--strip-sources-content` then fell back to uploading the unstripped map: the very +/// source the flag exists to keep off the wire. +fn reader(input: R) -> JsonStreamReader { + JsonStreamReader::new_custom( + input, + ReaderSettings { + restrict_number_values: false, + ..Default::default() + }, + ) +} + /// What [`top_level_strings`] found. pub struct TopLevel { /// Whether the document's top-level value is an object. @@ -75,7 +91,7 @@ pub struct TopLevel { /// document. A top-level value that is not an object yields `is_object: false` /// and no values (it is still syntax-checked). pub fn top_level_strings(input: R, keys: &[&str]) -> Res { - let mut r = JsonStreamReader::new(input); + let mut r = reader(input); let mut values: Vec> = vec![None; keys.len()]; let is_object = r.peek()? == ValueType::Object; if is_object { @@ -109,7 +125,7 @@ pub fn top_level_strings(input: R, keys: &[&str]) -> Res { /// /// Errors with [`MapJsonError::Json`] when the input is not a JSON object. pub fn strip_sources_content(input: R, output: W) -> Res { - let mut r = JsonStreamReader::new(input); + let mut r = reader(input); let mut w = JsonStreamWriter::new(output); if r.peek()? != ValueType::Object { return Err(MapJsonError::Json("not a JSON object".into())); @@ -178,7 +194,7 @@ fn strip_section(r: &mut R, w: &mut W) -> Res(input: R, output: W, debug_id: &str) -> Res<()> { - let mut r = JsonStreamReader::new(input); + let mut r = reader(input); let mut w = JsonStreamWriter::new(output); if r.peek()? != ValueType::Object { return Err(MapJsonError::Json("not a JSON object".into())); @@ -282,6 +298,27 @@ mod tests { ); } + /// Numbers are copied as TEXT, however unusual: a large exponent or a very long integer + /// is valid JSON, and refusing to copy it must never make a strip give up (which would + /// upload the sources it was asked to drop). + #[test] + fn unusual_numbers_are_copied_verbatim_and_sources_are_still_stripped() { + let long_int = "9".repeat(101); + for n in ["1e100", "1e-100", "-2.5E+150", "1e999999", long_int.as_str(), "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"] { + let doc = format!(r#"{{"n":{n},"sourcesContent":["secret"],"mappings":"AA"}}"#); + let (removed, out) = strip(&doc).unwrap_or_else(|e| panic!("{n}: {e}")); + assert!(removed, "{n}: the sources must still be stripped"); + assert_eq!(out, format!(r#"{{"n":{n},"mappings":"AA"}}"#), "{n}"); + let mut ids = Vec::new(); + set_debug_ids(doc.as_bytes(), &mut ids, "id").unwrap_or_else(|e| panic!("{n}: {e}")); + assert!(String::from_utf8(ids).unwrap().contains(&format!(r#""n":{n}"#)), "{n}"); + assert_eq!( + top_level_strings(doc.as_bytes(), &["mappings"]).unwrap().values, + [Some("AA".to_string())] + ); + } + } + #[test] fn strip_rejects_non_objects_and_bad_json() { assert!(matches!(strip("[]"), Err(MapJsonError::Json(_)))); diff --git a/src/inject/mod.rs b/src/inject/mod.rs index d0b4d3f..426dc67 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -570,18 +570,17 @@ fn write_map_debug_id( } } if !dry_run { - // Pass 2: stream the map into a sibling temp file with the ids set, then write - // its CONTENT into the original (not rename: that would swap the inode and - // drop the symlink / hard link the build gave the map; and not `fs::copy`: - // it also copies the temp file's metadata, i.e. its 0o600 mode, onto the map). - let dir = map_path.parent().filter(|p| !p.as_os_str().is_empty()); - let tmp = match dir { - Some(d) => tempfile::NamedTempFile::new_in(d)?, - None => tempfile::NamedTempFile::new_in(".")?, - }; + // Pass 2: stream the map into a scratch file with the ids set, then write its + // CONTENT into the original (not rename: that would swap the inode and drop the + // symlink / hard link the build gave the map; and not `fs::copy`: it also copies + // the scratch file's metadata, i.e. its 0o600 mode, onto the map). + // The scratch file is anonymous and lives in the OS temp dir, so rewriting a map + // needs no write access to the map's directory and a killed run leaves no stray + // `.tmpXXXX` beside it. + let mut scratch = tempfile::tempfile()?; { let input = std::io::BufReader::new(std::fs::File::open(map_path)?); - let mut out = std::io::BufWriter::new(tmp.as_file()); + let mut out = std::io::BufWriter::new(&scratch); mapjson::set_debug_ids(input, &mut out, debug_id).map_err(|e| match e { mapjson::MapJsonError::Io(e) => Error::Io(e), mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( @@ -595,7 +594,8 @@ fn write_map_debug_id( .write(true) .truncate(true) .open(map_path)?; - std::io::copy(&mut std::fs::File::open(tmp.path())?, &mut dest)?; + std::io::Seek::seek(&mut scratch, std::io::SeekFrom::Start(0))?; + std::io::copy(&mut scratch, &mut dest)?; } Ok(true) } From 0d99de4b2e3fded7a8aa95ce72e099aa2cc715e9 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:53:52 +0500 Subject: [PATCH 17/18] fix(inject): keep the map's scratch file on the map's own volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (Cursor, #74): the anonymous scratch lived in $TMPDIR, so rewriting an 89 MB map needed 89 MB free there; in CI containers /tmp is often a small tmpfs and inject failed (exit 10) although dist/ had room. tempfile_in() keeps it on the volume that already holds the map, falling back to the OS temp dir only when the directory refuses a file (read-only dir). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- src/inject/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/inject/mod.rs b/src/inject/mod.rs index 426dc67..862c233 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -574,10 +574,16 @@ fn write_map_debug_id( // CONTENT into the original (not rename: that would swap the inode and drop the // symlink / hard link the build gave the map; and not `fs::copy`: it also copies // the scratch file's metadata, i.e. its 0o600 mode, onto the map). - // The scratch file is anonymous and lives in the OS temp dir, so rewriting a map - // needs no write access to the map's directory and a killed run leaves no stray - // `.tmpXXXX` beside it. - let mut scratch = tempfile::tempfile()?; + // The scratch file is anonymous (a killed run leaves no stray `.tmpXXXX`). It lives + // beside the map, on the volume that already holds one copy of it: a tens-of-MB map + // must not need that much free space in `$TMPDIR`, which in CI containers is often a + // small tmpfs. Only when the directory refuses a file (it is read-only: editing the + // map in place never needed it writable) does it fall back to the OS temp dir. + let dir = map_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let mut scratch = tempfile::tempfile_in(dir).or_else(|_| tempfile::tempfile())?; { let input = std::io::BufReader::new(std::fs::File::open(map_path)?); let mut out = std::io::BufWriter::new(&scratch); From 75580203b1bf8ce604da6602454c2a9778c53ca7 Mon Sep 17 00:00:00 2001 From: Alexey Karimov Date: Wed, 7 Oct 2026 17:56:27 +0500 Subject: [PATCH 18/18] fix(inject): refuse an unusable map BEFORE touching the bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (Cursor, #75, repeated): inject_one stamped the bundle first and only then discovered the map was invalid / not an object / read-only, exiting 10/11 with a stamped bundle beside an un-keyed map; a retry would not repair it because restamp_id leaves our own stub alone. Preserving the old ordering was wrong. The map rewrite is now PLANNED before the first byte of the bundle changes: validated (a JSON object), staged in a scratch file beside the map, and the original opened for writing (a read-only map is refused here). Then the bundle is edited and the staged rewrite applied. Stats and the 'injected' log follow success. restamp_id also treats a non-object map as unusable, so a stamped bundle is not re-keyed for a map the rewrite would refuse. Tests (the first two fail with the old ordering): unusable maps leave BOTH files untouched (also at CLI level, with no 'injected' log line), a read-only map leaves the bundle unstamped, a regenerated unusable map does not restamp, and inject works with TMPDIR pointing nowhere. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) --- CHANGELOG.md | 2 + src/inject/mod.rs | 199 ++++++++++++++++++++++++++++++----------- tests/broken_files.rs | 5 ++ tests/file_handling.rs | 31 ++++++- 4 files changed, 184 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01c0639..9261995 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - **`sourcemaps inject` edits bundles in place.** The debug-id stub is appended (or, to re-key, our own trailing stub is swapped) without copying the bundle: a 150 MB bundle's dirty memory went from 317 MB to 2.4 MB. Output bytes are identical, and permissions and symlinks survive as before. + A map that cannot take the id (not a JSON object, invalid, read-only) is now refused BEFORE the + bundle is touched, so a failed run no longer leaves a stamped bundle beside an un-keyed map. ### Fixed - **`dsym uuid` and `debug-files upload --type dsym` no longer report a Mach-O with no `LC_UUID` diff --git a/src/inject/mod.rs b/src/inject/mod.rs index 7033cbf..a71ec16 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -444,9 +444,28 @@ fn inject_one(js_path: &Path, dry_run: bool, stats: &mut InjectStats) -> Result< } }; + // EVERYTHING that can fail on the map happens BEFORE the first byte of the bundle + // changes: the map is validated (a JSON object), its rewrite is staged in a scratch + // file, and the original is opened for writing (a read-only map is refused here). + // A map that cannot take the id then leaves the bundle exactly as the bundler wrote + // it, instead of a bundle stamped with an id its map never got (which a retry would + // not repair: `restamp_id` leaves our own stub alone). + let map_plan = match map_path.as_deref() { + Some(m) => Some(plan_map_rewrite(m, &debug_id, freshly_computed, dry_run)?), + None => None, + }; + if let (Some(edit), false) = (&edit, dry_run) { apply_js_edit(js_path, edit)?; } + let mut map_updated = false; + if let Some(MapPlan::Rewrite(rewrite)) = map_plan { + map_updated = true; + if let Some(rewrite) = rewrite { + apply_map_rewrite(rewrite)?; + } + } + match outcome { JsOutcome::Restamped { stale, fresh } => { stats.js_restamped += 1; @@ -471,11 +490,10 @@ fn inject_one(js_path: &Path, dry_run: bool, stats: &mut InjectStats) -> Result< tracing::info!(path = %js_path.display(), debug_id = %id, "injected debug-id"); } } - - if let Some(map_path) = map_path { - if write_map_debug_id(&map_path, &debug_id, freshly_computed, dry_run)? { - stats.maps_updated += 1; - tracing::debug!(path = %map_path.display(), debug_id = %debug_id, "wrote debug_id into map"); + if map_updated { + stats.maps_updated += 1; + if let Some(m) = map_path.as_deref() { + tracing::debug!(path = %m.display(), debug_id = %debug_id, "wrote debug_id into map"); } } Ok(()) @@ -509,7 +527,9 @@ fn restamp_id<'a>( else { return Ok(None); }; - if top.values.iter().any(Option::is_some) { + // A map the id cannot be written into (not a JSON object) is not re-keyed either: the + // rewrite would refuse it, and the bundle must not change when it does. + if !top.is_object || top.values.iter().any(Option::is_some) { return Ok(None); } let fresh = compute_debug_id_with_map_file(original.as_bytes(), map_path)?; @@ -600,14 +620,32 @@ fn map_id_action( } } -/// Make a `.map` JSON carry `debug_id` AND `debugId` equal to the bundle's id, -/// as [`map_id_action`] decides. Returns whether the map was changed. -fn write_map_debug_id( +/// A map rewrite staged in full but not yet applied (see [`plan_map_rewrite`]). +struct MapRewrite { + /// The original map, already OPENED for writing (so a read-only map fails at plan time). + dest: std::fs::File, + /// The complete new content. + scratch: std::fs::File, +} + +/// What the paired map needs. +enum MapPlan { + /// Already carries the right id, or must be left alone (shared with another bundle). + Unchanged, + /// To be rewritten. `None` in a dry run: validated and counted, never staged. + Rewrite(Option), +} + +/// Decide, and STAGE, making a `.map` JSON carry `debug_id` AND `debugId` equal to the +/// bundle's id, as [`map_id_action`] decides. Nothing on disk changes here: all the ways this +/// can fail (not a JSON object, invalid JSON, unreadable, not writable, no scratch space) +/// surface now, so the caller can refuse before touching the bundle. +fn plan_map_rewrite( map_path: &Path, debug_id: &str, freshly_computed: bool, dry_run: bool, -) -> Result { +) -> Result { // Pass 1 (streamed, also validates the whole file): what ids does it carry? let top = map_top_level(map_path, &["debug_id", "debugId"])?; if !top.is_object { @@ -623,7 +661,7 @@ fn write_map_debug_id( debug_id, freshly_computed, ) { - MapIdAction::Keep => return Ok(false), + MapIdAction::Keep => return Ok(MapPlan::Unchanged), MapIdAction::Fill => {} MapIdAction::Replace { stale } => tracing::warn!( path = %map_path.display(), @@ -639,44 +677,52 @@ fn write_map_debug_id( "source map carries a different debug id than its bundle, which already had one; left as is \ (is this map shared by more than one bundle?)" ); - return Ok(false); + return Ok(MapPlan::Unchanged); } } - if !dry_run { - // Pass 2: stream the map into a scratch file with the ids set, then write its - // CONTENT into the original (not rename: that would swap the inode and drop the - // symlink / hard link the build gave the map; and not `fs::copy`: it also copies - // the scratch file's metadata, i.e. its 0o600 mode, onto the map). - // The scratch file is anonymous (a killed run leaves no stray `.tmpXXXX`). It lives - // beside the map, on the volume that already holds one copy of it: a tens-of-MB map - // must not need that much free space in `$TMPDIR`, which in CI containers is often a - // small tmpfs. Only when the directory refuses a file (it is read-only: editing the - // map in place never needed it writable) does it fall back to the OS temp dir. - let dir = map_path - .parent() - .filter(|p| !p.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - let mut scratch = tempfile::tempfile_in(dir).or_else(|_| tempfile::tempfile())?; - { - let input = std::io::BufReader::new(std::fs::File::open(map_path)?); - let mut out = std::io::BufWriter::new(&scratch); - mapjson::set_debug_ids(input, &mut out, debug_id).map_err(|e| match e { - mapjson::MapJsonError::Io(e) => Error::Io(e), - mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( - "failed to rewrite sourcemap JSON: {m} ({})", - map_path.display() - )), - })?; - std::io::Write::flush(&mut out)?; - } - let mut dest = std::fs::OpenOptions::new() - .write(true) - .truncate(true) - .open(map_path)?; - std::io::Seek::seek(&mut scratch, std::io::SeekFrom::Start(0))?; - std::io::copy(&mut scratch, &mut dest)?; - } - Ok(true) + if dry_run { + return Ok(MapPlan::Rewrite(None)); + } + + // The original is opened for writing (NOT truncated yet) up front: a read-only map is + // an error now, not after the bundle was already stamped. + let dest = std::fs::OpenOptions::new().write(true).open(map_path)?; + + // Pass 2: stream the map into a scratch file with the ids set. Its CONTENT will later go + // into the original (not rename: that would swap the inode and drop the symlink / hard + // link the build gave the map; and not `fs::copy`: it also copies the scratch file's + // metadata, i.e. its 0o600 mode, onto the map). + // The scratch file is anonymous (a killed run leaves no stray `.tmpXXXX`). It lives + // beside the map, on the volume that already holds one copy of it: a tens-of-MB map + // must not need that much free space in `$TMPDIR`, which in CI containers is often a + // small tmpfs. Only when the directory refuses a file (it is read-only: editing the + // map in place never needed it writable) does it fall back to the OS temp dir. + let dir = map_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let scratch = tempfile::tempfile_in(dir).or_else(|_| tempfile::tempfile())?; + { + let input = std::io::BufReader::new(std::fs::File::open(map_path)?); + let mut out = std::io::BufWriter::new(&scratch); + mapjson::set_debug_ids(input, &mut out, debug_id).map_err(|e| match e { + mapjson::MapJsonError::Io(e) => Error::Io(e), + mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( + "failed to rewrite sourcemap JSON: {m} ({})", + map_path.display() + )), + })?; + std::io::Write::flush(&mut out)?; + } + Ok(MapPlan::Rewrite(Some(MapRewrite { dest, scratch }))) +} + +/// Write a staged rewrite into the original map: truncate it and copy the new content in. +fn apply_map_rewrite(mut rewrite: MapRewrite) -> Result<()> { + std::io::Seek::seek(&mut rewrite.scratch, std::io::SeekFrom::Start(0))?; + rewrite.dest.set_len(0)?; + std::io::copy(&mut rewrite.scratch, &mut rewrite.dest)?; + Ok(()) } /// Read the keying id from a `.map` for upload: `debug_id` (modern) / `debugId` @@ -2079,10 +2125,11 @@ mod file_edits { assert_eq!(std::fs::read(&js).unwrap(), b"a()\xff\xfe\n"); } - /// A map that is not a JSON object cannot take an id: the run fails and the map - /// is left exactly as it was, with no temp file beside it. + /// A map that is not a JSON object cannot take an id: the run fails and NOTHING is + /// touched. In particular the bundle must not be stamped first: it would then carry an id + /// its map never got, and a retry would not repair it (our own stub is left alone). #[test] - fn an_unusable_map_is_left_untouched() { + fn an_unusable_map_leaves_both_files_untouched() { for (label, map) in [ ("not json", "{not json"), ("an array", "[]"), @@ -2090,16 +2137,45 @@ mod file_edits { ("trailing junk", r#"{"version":3} x"#), ] { let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("a.js"), "a()\n").unwrap(); + let js = dir.path().join("a.js"); + std::fs::write(&js, "a()\n").unwrap(); let mp = dir.path().join("a.js.map"); std::fs::write(&mp, map).unwrap(); let err = inject_dir(dir.path()).unwrap_err(); assert!(matches!(err, Error::InputInvalid(_)), "{label}: {err:?}"); assert_eq!(std::fs::read_to_string(&mp).unwrap(), map, "{label}"); + assert_eq!( + std::fs::read_to_string(&js).unwrap(), + "a()\n", + "{label}: the bundle was stamped before the map was found unusable" + ); assert_eq!(names(dir.path()), ["a.js", "a.js.map"], "{label}"); } } + /// Same guarantee on a RE-run: a stamped bundle whose map was replaced by something + /// unusable is not re-keyed (that rewrite would be refused), so it is not changed either. + #[test] + fn an_unusable_regenerated_map_does_not_restamp_the_bundle() { + for (label, map) in [("an array", "[]"), ("not json", "{nope")] { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + std::fs::write(&js, "a()\n//# sourceMappingURL=a.js.map\n").unwrap(); + std::fs::write(dir.path().join("a.js.map"), MAP).unwrap(); + inject_dir(dir.path()).unwrap(); + let stamped = std::fs::read(&js).unwrap(); + + std::fs::write(dir.path().join("a.js.map"), map).unwrap(); + let err = inject_dir(dir.path()).unwrap_err(); + assert!(matches!(err, Error::InputInvalid(_)), "{label}: {err:?}"); + assert_eq!( + std::fs::read(&js).unwrap(), + stamped, + "{label}: bundle changed" + ); + } + } + #[cfg(unix)] mod unix { use super::*; @@ -2161,6 +2237,27 @@ mod file_edits { /// Rewriting a map needs no write access to its DIRECTORY (editing existing files in /// place never did): a read-only `dist/` holding writable files must still inject, and /// nothing may be created beside them. + /// A map that cannot be written (read-only) is refused BEFORE the bundle is stamped. + #[test] + fn a_read_only_map_is_an_error_and_the_bundle_stays_unstamped() { + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + std::fs::write(&js, "a()\n").unwrap(); + std::fs::write(&map, MAP).unwrap(); + std::fs::set_permissions(&map, std::fs::Permissions::from_mode(0o444)).unwrap(); + if std::fs::OpenOptions::new().write(true).open(&map).is_ok() { + return; // running as root: permissions are not enforced + } + assert!(matches!(inject_dir(dir.path()), Err(Error::Io(_)))); + assert_eq!( + std::fs::read(&js).unwrap(), + b"a()\n", + "the bundle was stamped" + ); + assert_eq!(std::fs::read_to_string(&map).unwrap(), MAP); + } + #[test] fn files_in_a_read_only_directory_are_still_rewritten() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/broken_files.rs b/tests/broken_files.rs index 9050312..2787dbd 100644 --- a/tests/broken_files.rs +++ b/tests/broken_files.rs @@ -181,6 +181,11 @@ fn broken_source_maps_never_crash_and_exit_with_stable_codes() { bytes, "inject rewrote a map it rejected: {label}" ); + assert_eq!( + std::fs::read(dir.join("app.js")).unwrap(), + b"a()\n", + "inject stamped the bundle although it rejected the map: {label}" + ); let left: Vec<_> = std::fs::read_dir(&dir) .unwrap() .map(|e| e.unwrap().file_name()) diff --git a/tests/file_handling.rs b/tests/file_handling.rs index bb2ad6a..05e1bf1 100644 --- a/tests/file_handling.rs +++ b/tests/file_handling.rs @@ -127,16 +127,43 @@ fn inject_dry_run_reports_but_writes_nothing() { #[test] fn inject_rejects_an_unusable_map_and_leaves_it_alone() { + use predicates::prelude::PredicateBooleanExt; let tmp = tempfile::tempdir().unwrap(); - write(tmp.path(), "app.js", b"a()\n"); + let js = write(tmp.path(), "app.js", b"a()\n"); let map = write(tmp.path(), "app.js.map", b"{ not json"); common::cli() .args(["sourcemaps", "inject"]) .arg(tmp.path()) .assert() .code(11) // InputInvalid - .stderr(predicates::str::contains("not valid JSON")); + .stderr(predicates::str::contains("not valid JSON")) + .stderr(predicates::str::contains("injected debug-id").not()); assert_eq!(std::fs::read(&map).unwrap(), b"{ not json"); + assert_eq!( + std::fs::read(&js).unwrap(), + b"a()\n", + "a failed inject must not leave a stamped bundle behind" + ); +} + +/// Rewriting a map must not depend on `$TMPDIR` having room (or existing): in CI containers +/// it is often a tiny tmpfs. The scratch file lives beside the map. +#[test] +fn inject_does_not_need_tmpdir() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "app.js", b"a()\n"); + let map = write(tmp.path(), "app.js.map", MAP.as_bytes()); + common::cli() + .args(["sourcemaps", "inject"]) + .arg(tmp.path()) + .env("TMPDIR", "/nonexistent/tmp") + .env("TMP", "/nonexistent/tmp") + .env("TEMP", "/nonexistent/tmp") + .assert() + .success(); + assert!(String::from_utf8(std::fs::read(&map).unwrap()) + .unwrap() + .contains("debug_id")); } // ---------------------------------------------------------------------------