diff --git a/CHANGELOG.md b/CHANGELOG.md index 96298e9..132d661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). Peak RSS: `upload build` with a 300 MB artefact + 200 MB mapping 661 MB -> 32 MB, a 200 MB ProGuard mapping 241 MB -> 32 MB, an ELF zip of four 100 MB libraries 463 MB -> 36 MB. Wire format and exit codes unchanged. +- **Source maps are streamed instead of parsed into a JSON tree.** Reading a map's debug-id, + stripping `sourcesContent` (`--strip-sources-content`, including nested indexed-map sections) and + `sourcemaps inject` rewriting a map's ids now walk the document with `struson`, and the inject + debug-id hash is computed incrementally instead of over a concatenated copy. Peak RSS for an + 89 MB map: upload 183 MB -> 32 MB, `--strip-sources-content` 262 MB -> 32 MB, `inject` 298 MB -> + 6 MB. Debug-ids are unchanged. Two byte-level differences: a stripped copy keeps the map's + original key order (it was alphabetical), and `inject` writes `debug_id` / `debugId` last. Both + are still valid maps; an invalid-UTF-8 map outside a string value now reports as invalid JSON + (exit 11) rather than I/O (exit 10). ## [0.7.13] - 2026-10-07 diff --git a/Cargo.lock b/Cargo.lock index 8b2ff33..3471bf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,7 @@ dependencies = [ "serde_json", "sha1", "sha2", + "struson", "symbolic-common", "symbolic-debuginfo", "tempfile", @@ -464,6 +465,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "duplicate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e92f10a49176cbffacaedabfaa11d51db1ea0f80a83c26e1873b43cd1742c24" +dependencies = [ + "heck", + "proc-macro2", + "proc-macro2-diagnostics", +] + [[package]] name = "elementtree" version = "1.2.3" @@ -1482,6 +1494,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", +] + [[package]] name = "quick-xml" version = "0.42.0" @@ -2000,6 +2024,38 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "struson" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8eaaf563e9de627dadbe66ff8b7ef8f065fc69844c90ed6acdc90bdc9f0571" +dependencies = [ + "duplicate", + "strum", + "thiserror 2.0.21", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index fc15b36..079fd2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -172,6 +172,10 @@ tempfile = "3" # [dependencies] table (NOT a cfg(target) table). self-replace = "1" +# Streaming JSON reader/writer: lets the source-map paths read or rewrite a few +# top-level keys of a tens-of-MB map without building a serde_json::Value tree. +struson = "0.7" + # Unix double-fork for `xcode post-action`'s background (daemon) mode. This is a # TARGET-specific table, so it MUST come AFTER the main [dependencies] table — # placed mid-table, TOML would capture every following key into the unix-only diff --git a/src/cli/debug_files.rs b/src/cli/debug_files.rs index 8449c61..f7da7df 100644 --- a/src/cli/debug_files.rs +++ b/src/cli/debug_files.rs @@ -1919,23 +1919,6 @@ fn is_stylesheet_or_declaration_map(path: &Path, extra: &ExtraSuffixes) -> bool }) } -/// Remove every `sourcesContent` a source map can carry; `true` when one was there. -/// -/// Not just the top-level key: an INDEXED map (spec §Index-Map) keeps its source inside -/// `sections[i].map`, and those sections nest. Stripping only the top level left the source in the -/// upload while reporting success — the flag's whole promise, failing silently. -fn remove_sources_content(map: &mut serde_json::Map) -> bool { - let mut removed = map.remove("sourcesContent").is_some(); - if let Some(serde_json::Value::Array(sections)) = map.get_mut("sections") { - for section in sections { - if let Some(serde_json::Value::Object(inner)) = section.get_mut("map") { - removed |= remove_sources_content(inner); - } - } - } - removed -} - /// A copy of `map_path` with `sourcesContent` removed, plus its recomputed identity — or `None` /// when nothing needs stripping (flag off, no `sourcesContent`, or the map is not the JSON object we /// expect, which is a privacy preference's business to ignore rather than to fail over). @@ -1954,40 +1937,53 @@ fn stripped_copy_without_sources( // Typed, not a bare `?` into anyhow: an I/O failure on the map must classify the way // `sourcemap::identify` classifies one on the SAME file moments earlier — exit 10, which // integrators are documented not to fall back on — rather than exit 1 ("unexpected"). - let bytes = crate::symbols::mapped::map_file(map_path).map_err(crate::error::Error::Io)?; - let Ok(serde_json::Value::Object(mut map)) = serde_json::from_slice(&bytes) else { - return Ok(None); - }; - if !remove_sources_content(&mut map) { - return Ok(None); - } - let stripped = serde_json::to_vec(&map).map_err(|e| { - input_invalid(format!( - "{}: could not re-serialize the map without its sources: {e}", - map_path.display() - )) - })?; let name = map_path .file_name() .map(Path::new) .unwrap_or_else(|| Path::new("bundle.js.map")); let out = tmpdir.join(name); - std::fs::write(&out, &stripped).map_err(crate::error::Error::Io)?; + // Streamed input -> output: only the dropped `sourcesContent` values (the bulk of a + // map) are never held, and nothing else is either, so memory is flat however big the map. + let before_bytes = std::fs::metadata(map_path) + .map_err(crate::error::Error::Io)? + .len(); + let input = + std::io::BufReader::new(std::fs::File::open(map_path).map_err(crate::error::Error::Io)?); + let output = + std::io::BufWriter::new(std::fs::File::create(&out).map_err(crate::error::Error::Io)?); + match crate::inject::mapjson::strip_sources_content(input, output) { + Ok(true) => {} + // Nothing to strip: the copy is moot. + Ok(false) => { + let _ = std::fs::remove_file(&out); + return Ok(None); + } + // Not the JSON object we expect. A privacy preference is not a reason to fail the + // upload, but it must not fail SILENTLY either: say that the sources are NOT stripped. + Err(crate::inject::mapjson::MapJsonError::Json(why)) => { + tracing::warn!( + path = %map_path.display(), + reason = %why, + "--strip-sources-content could not process this map; uploading it UNSTRIPPED" + ); + let _ = std::fs::remove_file(&out); + return Ok(None); + } + Err(crate::inject::mapjson::MapJsonError::Io(e)) => { + let _ = std::fs::remove_file(&out); + return Err(crate::error::Error::Io(e).into()); + } + } + // The identity describes what is UPLOADED (the stripped copy): its streamed hash and size. + // The debug-id is carried over untouched: it is the key, and re-deriving it here would + // break the pair with the bundle that `sourcemaps inject` stamped. + let identity = sourcemap::identify(&out)?; tracing::info!( path = %map_path.display(), - before_bytes = bytes.len(), - after_bytes = stripped.len(), + before_bytes, + after_bytes = identity.size_bytes, "stripped sourcesContent from the uploaded copy" ); - let identity = sourcemap::SourcemapIdentity { - debug_id: crate::inject::read_debug_id(&out)?, - content_sha1_hex: { - use sha1::Digest; - let digest: [u8; 20] = sha1::Sha1::digest(&stripped).into(); - hex::encode(digest) - }, - size_bytes: stripped.len() as u64, - }; Ok(Some((out, identity))) } @@ -3401,18 +3397,7 @@ mod sourcemap_upload_tests { } async fn uploaded_map_json(server: &MockServer) -> serde_json::Value { - let requests = server.received_requests().await.unwrap(); - let put = requests - .iter() - .find(|r| r.method.as_str() == "PUT") - .expect("no PUT"); - let mut archive = - zip::ZipArchive::new(std::io::Cursor::new(put.body.clone())).expect("not a zip"); - assert_eq!(archive.len(), 1, "one entry: just the map"); - let mut entry = archive.by_index(0).unwrap(); - let mut body = String::new(); - std::io::Read::read_to_string(&mut entry, &mut body).unwrap(); - serde_json::from_str(&body).expect("uploaded map is not JSON") + serde_json::from_slice(&uploaded_map_bytes(server).await).expect("uploaded map is not JSON") } /// `sourcesContent` embeds the ORIGINAL SOURCE in the map, which is how symbolication shows @@ -3497,14 +3482,17 @@ mod sourcemap_upload_tests { .body, ) .unwrap(); - let uploaded = uploaded_map_json(&server).await; + let uploaded = uploaded_map_bytes(&server).await; let expected = { use sha1::Digest; - let digest: [u8; 20] = - sha1::Sha1::digest(serde_json::to_vec(&uploaded).unwrap()).into(); + let digest: [u8; 20] = sha1::Sha1::digest(&uploaded).into(); hex::encode(digest) }; assert_eq!(post["hash"].as_str().unwrap(), expected); + assert!( + !String::from_utf8_lossy(&uploaded).contains("sourcesContent"), + "and the hash is of the STRIPPED bytes" + ); } /// Without the flag, the source rides along exactly as before — that is what makes the diff --git a/src/inject/mapjson.rs b/src/inject/mapjson.rs new file mode 100644 index 0000000..ca8e50a --- /dev/null +++ b/src/inject/mapjson.rs @@ -0,0 +1,350 @@ +//! Streaming reads and rewrites of source-map JSON. +//! +//! A source map is routinely tens of MB, mostly one huge `mappings` string and +//! an array of whole source files (`sourcesContent`). Parsing it into a +//! `serde_json::Value` held ~2x the file on the heap just to look at, or change, +//! a handful of top-level keys. Everything here walks the document once with +//! [`struson`] instead, so memory stays at the reader's small buffer plus the +//! one member name / id string being looked at — whatever the map's size. +//! +//! Syntax is still validated end to end: a skipped value is checked like a read +//! one, and trailing garbage after the top-level value is an error. + +use std::io::{Read, Write}; + +use struson::reader::{JsonReader, JsonStreamReader, ReaderError, ReaderSettings, ValueType}; +use struson::writer::{JsonStreamWriter, JsonWriter}; + +/// Why a streaming pass over a map failed. +#[derive(Debug)] +pub enum MapJsonError { + /// The input is not (this shape of) valid JSON. + Json(String), + /// Writing the rewritten copy failed. + Io(std::io::Error), +} + +impl std::fmt::Display for MapJsonError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MapJsonError::Json(m) => f.write_str(m), + MapJsonError::Io(e) => write!(f, "{e}"), + } + } +} + +impl From for MapJsonError { + fn from(e: ReaderError) -> Self { + match e { + // Invalid UTF-8 inside a string surfaces as an I/O `InvalidData` error; + // keep it one (as `read_to_string` reported it) rather than "bad JSON". + ReaderError::IoError { error, .. } => MapJsonError::Io(error), + other => MapJsonError::Json(other.to_string()), + } + } +} + +impl From for MapJsonError { + fn from(e: struson::reader::TransferError) -> Self { + match e { + struson::reader::TransferError::ReaderError(r) => r.into(), + struson::reader::TransferError::WriterError(w) => MapJsonError::Io(w), + } + } +} + +impl From for MapJsonError { + fn from(e: std::io::Error) -> Self { + MapJsonError::Io(e) + } +} + +type Res = std::result::Result; + +/// A reader that does NOT apply struson's default "restrict number values" (it rejects +/// an exponent beyond ±99 and any number over 100 characters). That guard protects code +/// that PARSES numbers into big integers; this module never parses one, it only skips or +/// copies the number's text. Left on, valid JSON such as `1e100` made a copy fail, and +/// `--strip-sources-content` then fell back to uploading the unstripped map: the very +/// source the flag exists to keep off the wire. +fn reader(input: R) -> JsonStreamReader { + JsonStreamReader::new_custom( + input, + ReaderSettings { + restrict_number_values: false, + ..Default::default() + }, + ) +} + +/// What [`top_level_strings`] found. +pub struct TopLevel { + /// Whether the document's top-level value is an object. + pub is_object: bool, + /// One entry per requested key, in order: its value when it is a string, + /// else `None` (absent, or any other JSON type). With a duplicated key the + /// LAST occurrence wins, as it does in a parsed `serde_json::Value`. + pub values: Vec>, +} + +/// Read the string values of the given top-level keys, validating the whole +/// document. A top-level value that is not an object yields `is_object: false` +/// and no values (it is still syntax-checked). +pub fn top_level_strings(input: R, keys: &[&str]) -> Res { + let mut r = reader(input); + let mut values: Vec> = vec![None; keys.len()]; + let is_object = r.peek()? == ValueType::Object; + if is_object { + r.begin_object()?; + while r.has_next()? { + let idx = { + let name = r.next_name()?; + keys.iter().position(|k| *k == name) + }; + match idx { + Some(i) if r.peek()? == ValueType::String => values[i] = Some(r.next_string()?), + Some(i) => { + values[i] = None; + r.skip_value()?; + } + None => r.skip_value()?, + } + } + r.end_object()?; + } else { + r.skip_value()?; + } + r.consume_trailing_whitespace()?; + Ok(TopLevel { is_object, values }) +} + +/// Copy `input` to `output`, dropping every `sourcesContent` — at the top level +/// AND inside the (nestable) `sections[i].map` of an indexed map. Returns +/// whether anything was dropped. Everything else is copied verbatim, in its +/// original order, without being held in memory. +/// +/// Errors with [`MapJsonError::Json`] when the input is not a JSON object. +pub fn strip_sources_content(input: R, output: W) -> Res { + let mut r = reader(input); + let mut w = JsonStreamWriter::new(output); + if r.peek()? != ValueType::Object { + return Err(MapJsonError::Json("not a JSON object".into())); + } + let removed = strip_object(&mut r, &mut w)?; + r.consume_trailing_whitespace()?; + w.finish_document()?; + Ok(removed) +} + +fn strip_object(r: &mut R, w: &mut W) -> Res { + let mut removed = false; + r.begin_object()?; + w.begin_object()?; + while r.has_next()? { + let name = r.next_name()?.to_owned(); + if name == "sourcesContent" { + r.skip_value()?; + removed = true; + continue; + } + w.name(&name)?; + if name == "sections" && r.peek()? == ValueType::Array { + r.begin_array()?; + w.begin_array()?; + while r.has_next()? { + if r.peek()? == ValueType::Object { + removed |= strip_section(r, w)?; + } else { + r.transfer_to(w)?; + } + } + r.end_array()?; + w.end_array()?; + } else { + r.transfer_to(w)?; + } + } + r.end_object()?; + w.end_object()?; + Ok(removed) +} + +/// One `sections[i]` object: only its `map` (when an object) is searched. +fn strip_section(r: &mut R, w: &mut W) -> Res { + let mut removed = false; + r.begin_object()?; + w.begin_object()?; + while r.has_next()? { + let name = r.next_name()?.to_owned(); + w.name(&name)?; + if name == "map" && r.peek()? == ValueType::Object { + removed |= strip_object(r, w)?; + } else { + r.transfer_to(w)?; + } + } + r.end_object()?; + w.end_object()?; + Ok(removed) +} + +/// Copy `input` to `output` with the top-level `debug_id` and `debugId` set to +/// `debug_id` (any existing ones are dropped and the pair is written last). +/// Everything else is copied verbatim and in order. +/// +/// Errors with [`MapJsonError::Json`] when the input is not a JSON object. +pub fn set_debug_ids(input: R, output: W, debug_id: &str) -> Res<()> { + let mut r = reader(input); + let mut w = JsonStreamWriter::new(output); + if r.peek()? != ValueType::Object { + return Err(MapJsonError::Json("not a JSON object".into())); + } + r.begin_object()?; + w.begin_object()?; + while r.has_next()? { + let name = r.next_name()?.to_owned(); + if name == "debug_id" || name == "debugId" { + r.skip_value()?; + continue; + } + w.name(&name)?; + r.transfer_to(&mut w)?; + } + r.end_object()?; + w.name("debug_id")?; + w.string_value(debug_id)?; + w.name("debugId")?; + w.string_value(debug_id)?; + w.end_object()?; + r.consume_trailing_whitespace()?; + w.finish_document()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn strip(s: &str) -> Res<(bool, String)> { + let mut out = Vec::new(); + let removed = strip_sources_content(s.as_bytes(), &mut out)?; + Ok((removed, String::from_utf8(out).unwrap())) + } + + #[test] + fn top_level_strings_picks_string_values_last_wins_and_ignores_other_types() { + let t = top_level_strings( + br#"{"debug_id": 5, "debugId": "camel", "uuid": "a", "uuid": "b", "x": {"debug_id": "nested"}}"#.as_slice(), + &["debug_id", "debugId", "uuid"], + ) + .unwrap(); + assert!(t.is_object); + assert_eq!( + t.values, + [None, Some("camel".into()), Some("b".into())], + "non-string -> None, duplicates -> last, nested keys ignored" + ); + // A later non-string occurrence replaces an earlier string, as in a Value. + let t = top_level_strings(br#"{"uuid": "a", "uuid": 1}"#.as_slice(), &["uuid"]).unwrap(); + assert_eq!(t.values, [None]); + } + + #[test] + fn top_level_strings_non_object_and_invalid_json() { + for ok in ["[]", "3", "\"s\"", "null"] { + let t = top_level_strings(ok.as_bytes(), &["uuid"]).unwrap(); + assert!(!t.is_object, "{ok}"); + assert_eq!(t.values, [None]); + } + for bad in [ + "", + "{", + r#"{"a":}"#, + r#"{"a":1} x"#, + r#"{"a":1}{"b":2}"#, + "[1,]", + ] { + assert!( + matches!( + top_level_strings(bad.as_bytes(), &["a"]), + Err(MapJsonError::Json(_)) + ), + "{bad:?} must be rejected" + ); + } + } + + #[test] + fn strip_removes_top_level_and_nested_section_sources_in_order() { + let (removed, out) = strip( + r#"{"version":3,"sourcesContent":["a"],"sections":[{"offset":{"line":0,"column":0},"map":{"sources":["x"],"sourcesContent":["secret"],"sections":[{"map":{"sourcesContent":["deep"],"names":[]}}]}},7],"mappings":"AAAA"}"#, + ) + .unwrap(); + assert!(removed); + assert_eq!( + out, + r#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"sources":["x"],"sections":[{"map":{"names":[]}}]}},7],"mappings":"AAAA"}"# + ); + } + + #[test] + fn strip_reports_nothing_removed_and_ignores_non_object_sections() { + let (removed, out) = + strip(r#"{"sections":"nope","map":{"sourcesContent":[1]},"a":1.50}"#).unwrap(); + assert!(!removed, "only sections[].map is searched"); + assert!( + out.contains(r#""a":1.50"#), + "numbers keep their text: {out}" + ); + } + + /// Numbers are copied as TEXT, however unusual: a large exponent or a very long integer + /// is valid JSON, and refusing to copy it must never make a strip give up (which would + /// upload the sources it was asked to drop). + #[test] + fn unusual_numbers_are_copied_verbatim_and_sources_are_still_stripped() { + let long_int = "9".repeat(101); + for n in ["1e100", "1e-100", "-2.5E+150", "1e999999", long_int.as_str(), "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"] { + let doc = format!(r#"{{"n":{n},"sourcesContent":["secret"],"mappings":"AA"}}"#); + let (removed, out) = strip(&doc).unwrap_or_else(|e| panic!("{n}: {e}")); + assert!(removed, "{n}: the sources must still be stripped"); + assert_eq!(out, format!(r#"{{"n":{n},"mappings":"AA"}}"#), "{n}"); + let mut ids = Vec::new(); + set_debug_ids(doc.as_bytes(), &mut ids, "id").unwrap_or_else(|e| panic!("{n}: {e}")); + assert!(String::from_utf8(ids).unwrap().contains(&format!(r#""n":{n}"#)), "{n}"); + assert_eq!( + top_level_strings(doc.as_bytes(), &["mappings"]).unwrap().values, + [Some("AA".to_string())] + ); + } + } + + #[test] + fn strip_rejects_non_objects_and_bad_json() { + assert!(matches!(strip("[]"), Err(MapJsonError::Json(_)))); + assert!(matches!(strip("{"), Err(MapJsonError::Json(_)))); + assert!(matches!( + strip(r#"{"a":1} junk"#), + Err(MapJsonError::Json(_)) + )); + } + + #[test] + fn set_debug_ids_replaces_existing_pair_and_keeps_the_rest() { + let mut out = Vec::new(); + set_debug_ids( + br#"{"debug_id":"old","version":3,"debugId":"older","mappings":"AA"}"#.as_slice(), + &mut out, + "new-id", + ) + .unwrap(); + assert_eq!( + String::from_utf8(out).unwrap(), + r#"{"version":3,"mappings":"AA","debug_id":"new-id","debugId":"new-id"}"# + ); + assert!(matches!( + set_debug_ids(b"[]".as_slice(), Vec::new(), "x"), + Err(MapJsonError::Json(_)) + )); + } +} diff --git a/src/inject/mod.rs b/src/inject/mod.rs index 1cd40c9..862c233 100644 --- a/src/inject/mod.rs +++ b/src/inject/mod.rs @@ -23,6 +23,7 @@ use uuid::Uuid; use crate::error::{Error, Result}; +pub mod mapjson; pub mod sri; /// Fixed namespace for Bugsee sourcemap debug-ids — keeps UUIDv5 generation @@ -54,15 +55,48 @@ pub fn compute_debug_id(content: &[u8]) -> Uuid { /// server while the upload reported the new one as already there. The bundle's /// length is hashed ahead of it, so the boundary between the two inputs cannot /// shift without changing the id. +#[cfg(test)] pub fn compute_debug_id_with_map(bundle: &[u8], map: Option<&[u8]>) -> Uuid { let Some(map) = map else { return compute_debug_id(bundle); }; - let mut input = Vec::with_capacity(8 + bundle.len() + map.len()); - input.extend_from_slice(&(bundle.len() as u64).to_le_bytes()); - input.extend_from_slice(bundle); - input.extend_from_slice(map); - Uuid::new_v5(&DEBUG_ID_NAMESPACE, &input) + // Reading a slice cannot fail. + hash_with_map(bundle, map).expect("reading from a slice cannot fail") +} + +/// [`compute_debug_id_with_map`] with the map STREAMED from `map_path` instead of +/// held in memory: a map is tens of MB, and concatenating it with the bundle +/// into one buffer just to hash it was a second full copy. +pub fn compute_debug_id_with_map_file(bundle: &[u8], map_path: &Path) -> std::io::Result { + hash_with_map( + bundle, + std::io::BufReader::new(std::fs::File::open(map_path)?), + ) +} + +/// UUIDv5 over `len(bundle) as u64 LE || bundle || map`, computed incrementally. +/// +/// A v5 UUID is the first 16 bytes of SHA-1(namespace || name) with the +/// version/variant bits set, so feeding the pieces to the hasher one at a time +/// gives the same id as `Uuid::new_v5` over their concatenation. +fn hash_with_map(bundle: &[u8], mut map: impl std::io::Read) -> std::io::Result { + use sha1::{Digest as _, Sha1}; + let mut hasher = Sha1::new(); + hasher.update(DEBUG_ID_NAMESPACE.as_bytes()); + hasher.update((bundle.len() as u64).to_le_bytes()); + hasher.update(bundle); + let mut buf = [0u8; 64 * 1024]; + loop { + let n = map.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + let digest = hasher.finalize(); + let mut first16 = [0u8; 16]; + first16.copy_from_slice(&digest[..16]); + Ok(uuid::Builder::from_sha1_bytes(first16).into_uuid()) } /// The runtime stub appended to each JS bundle. On load it registers @@ -352,8 +386,10 @@ fn inject_one(js_path: &Path, dry_run: bool, stats: &mut InjectStats) -> Result< } }, None => { - let map_bytes = map_path.as_deref().map(std::fs::read).transpose()?; - let id = compute_debug_id_with_map(content.as_bytes(), map_bytes.as_deref()); + let id = match map_path.as_deref() { + Some(m) => compute_debug_id_with_map_file(content.as_bytes(), m)?, + None => compute_debug_id(content.as_bytes()), + }; if !dry_run { std::fs::write(js_path, format!("{content}{}", runtime_stub(&id)))?; } @@ -392,19 +428,18 @@ fn restamp_id<'a>( let Some(original) = content.strip_suffix(runtime_stub(&id).as_str()) else { return Ok(None); }; - let map_bytes = std::fs::read(map_path)?; + let file = std::fs::File::open(map_path)?; // A map that is not JSON is not re-keyed here: writing the id into it fails // next, and it must fail BEFORE the bundle is rewritten, as it always did. - let Ok(map) = serde_json::from_slice::(&map_bytes) else { + let Ok(top) = + mapjson::top_level_strings(std::io::BufReader::new(file), &["debug_id", "debugId"]) + else { return Ok(None); }; - let carries_id = ["debug_id", "debugId"] - .iter() - .any(|k| map.get(*k).and_then(serde_json::Value::as_str).is_some()); - if carries_id { + if top.values.iter().any(Option::is_some) { return Ok(None); } - let fresh = compute_debug_id_with_map(original.as_bytes(), Some(&map_bytes)); + let fresh = compute_debug_id_with_map_file(original.as_bytes(), map_path)?; Ok((fresh.to_string() != id).then_some((original, fresh))) } @@ -500,25 +535,15 @@ fn write_map_debug_id( freshly_computed: bool, dry_run: bool, ) -> Result { - let raw = std::fs::read_to_string(map_path)?; - let mut value: serde_json::Value = serde_json::from_str(&raw).map_err(|e| { - Error::InputInvalid(format!( - "sourcemap is not valid JSON: {e} ({})", - map_path.display() - )) - })?; - let obj = value.as_object_mut().ok_or_else(|| { - Error::InputInvalid(format!( + // Pass 1 (streamed, also validates the whole file): what ids does it carry? + let top = map_top_level(map_path, &["debug_id", "debugId"])?; + if !top.is_object { + return Err(Error::InputInvalid(format!( "sourcemap is not a JSON object: {}", map_path.display() - )) - })?; - let current = |k: &str| { - obj.get(k) - .and_then(serde_json::Value::as_str) - .map(str::to_string) - }; - let (snake, camel) = (current("debug_id"), current("debugId")); + ))); + } + let (snake, camel) = (top.values[0].clone(), top.values[1].clone()); match map_id_action( snake.as_deref(), camel.as_deref(), @@ -544,18 +569,39 @@ fn write_map_debug_id( return Ok(false); } } - obj.insert( - "debug_id".into(), - serde_json::Value::String(debug_id.to_string()), - ); - obj.insert( - "debugId".into(), - serde_json::Value::String(debug_id.to_string()), - ); if !dry_run { - let serialized = serde_json::to_string(&value) - .map_err(|e| Error::InputInvalid(format!("failed to serialize sourcemap JSON: {e}")))?; - std::fs::write(map_path, serialized)?; + // Pass 2: stream the map into a scratch file with the ids set, then write its + // CONTENT into the original (not rename: that would swap the inode and drop the + // symlink / hard link the build gave the map; and not `fs::copy`: it also copies + // the scratch file's metadata, i.e. its 0o600 mode, onto the map). + // The scratch file is anonymous (a killed run leaves no stray `.tmpXXXX`). It lives + // beside the map, on the volume that already holds one copy of it: a tens-of-MB map + // must not need that much free space in `$TMPDIR`, which in CI containers is often a + // small tmpfs. Only when the directory refuses a file (it is read-only: editing the + // map in place never needed it writable) does it fall back to the OS temp dir. + let dir = map_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let mut scratch = tempfile::tempfile_in(dir).or_else(|_| tempfile::tempfile())?; + { + let input = std::io::BufReader::new(std::fs::File::open(map_path)?); + let mut out = std::io::BufWriter::new(&scratch); + mapjson::set_debug_ids(input, &mut out, debug_id).map_err(|e| match e { + mapjson::MapJsonError::Io(e) => Error::Io(e), + mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( + "failed to rewrite sourcemap JSON: {m} ({})", + map_path.display() + )), + })?; + std::io::Write::flush(&mut out)?; + } + let mut dest = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(map_path)?; + std::io::Seek::seek(&mut scratch, std::io::SeekFrom::Start(0))?; + std::io::copy(&mut scratch, &mut dest)?; } Ok(true) } @@ -564,22 +610,24 @@ fn write_map_debug_id( /// / legacy `uuid`, in that precedence. Mirrors the worker's /// `symbolfiles/sourcemap.py:parse`. pub fn read_debug_id(map_path: &Path) -> Result> { - // Mapped rather than read into a String: the JSON tree below is the only - // copy this needs. UTF-8 is checked up front so a non-UTF-8 map still fails - // as an I/O `InvalidData` error, exactly as `read_to_string` did. - let bytes = crate::symbols::mapped::map_file(map_path)?; - let raw = std::str::from_utf8(&bytes) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| { - Error::InputInvalid(format!( - "sourcemap is not valid JSON: {e} ({})", + // Walked as a stream: no copy of the map and no JSON tree, so a 100 MB map + // costs a few KB. Invalid UTF-8 inside a string stays an I/O `InvalidData` + // error, as it was with `read_to_string`. + let top = map_top_level(map_path, &["debug_id", "debugId", "uuid"])?; + Ok(top.values.into_iter().flatten().next()) +} + +/// Stream-read the string values of top-level `keys` of the map at `map_path`. +/// Not-JSON becomes `InputInvalid` (naming the file); an I/O failure stays `Io`. +fn map_top_level(map_path: &Path, keys: &[&str]) -> Result { + let file = std::io::BufReader::new(std::fs::File::open(map_path)?); + mapjson::top_level_strings(file, keys).map_err(|e| match e { + mapjson::MapJsonError::Io(e) => Error::Io(e), + mapjson::MapJsonError::Json(m) => Error::InputInvalid(format!( + "sourcemap is not valid JSON: {m} ({})", map_path.display() - )) - })?; - Ok(["debug_id", "debugId", "uuid"] - .iter() - .find_map(|k| value.get(*k).and_then(serde_json::Value::as_str)) - .map(str::to_string)) + )), + }) } #[cfg(test)] @@ -590,6 +638,64 @@ mod tests { /// RELATIVE root have to take turns, or they change the directory out from under each other. static CWD_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + #[test] + fn streamed_map_hash_equals_the_concatenated_uuid_v5_formula() { + // The id is a public key (it is embedded in shipped bundles): the + // incremental hash must reproduce `Uuid::new_v5` over `len || bundle || + // map` bit for bit, including across the 64 KiB read-buffer boundary. + let dir = tempfile::tempdir().unwrap(); + for map_len in [0usize, 1, 65_535, 65_536, 65_537, 300_001] { + let bundle = b"console.log('x')"; + let map: Vec = (0..map_len).map(|i| (i % 251) as u8).collect(); + let mut input = (bundle.len() as u64).to_le_bytes().to_vec(); + input.extend_from_slice(bundle); + input.extend_from_slice(&map); + let expected = Uuid::new_v5(&DEBUG_ID_NAMESPACE, &input); + let p = dir.path().join("m.map"); + std::fs::write(&p, &map).unwrap(); + assert_eq!( + compute_debug_id_with_map_file(bundle, &p).unwrap(), + expected + ); + assert_eq!(compute_debug_id_with_map(bundle, Some(&map)), expected); + } + } + + /// The rewrite must leave the map's mode, inode and links as the bundler made them. A + /// sibling temp file is 0o600, so any step that carries ITS metadata over (as + /// `fs::copy` does) silently tightens a 0o644 map to owner-only. Distinct modes on + /// the map and the bundle make that visible. + #[cfg(unix)] + #[test] + fn rewriting_a_map_keeps_its_mode_inode_and_hard_links() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let dir = tempfile::tempdir().unwrap(); + let js = dir.path().join("a.js"); + let map = dir.path().join("a.js.map"); + let link = dir.path().join("a.js.map.link"); + std::fs::write(&js, "a()\n").unwrap(); + std::fs::write(&map, r#"{"version":3,"mappings":"AAAA"}"#).unwrap(); + std::fs::hard_link(&map, &link).unwrap(); + std::fs::set_permissions(&map, std::fs::Permissions::from_mode(0o644)).unwrap(); + let ino = std::fs::metadata(&map).unwrap().ino(); + + inject_paths(&[dir.path().to_path_buf()], &[], true, false).unwrap(); + + let after = std::fs::metadata(&map).unwrap(); + assert_eq!( + after.permissions().mode() & 0o777, + 0o644, + "the mode must not be tightened" + ); + assert_eq!(after.ino(), ino, "same inode: rewritten in place"); + assert!(std::fs::read_to_string(&map).unwrap().contains("debug_id")); + assert_eq!( + std::fs::read(&link).unwrap(), + std::fs::read(&map).unwrap(), + "a hard link sees the new content" + ); + } + #[test] fn is_contained_relative_rejects_traversal_and_absolute() { assert!(is_contained_relative("bundle.js.map"));