feat(engine): the largest project follows the computer's memory (#1615)

crates/engine/src/cmd/items.rs

@@ -15,8 +15,17 @@ use super::{b, f, need_feature_layer, need_layer, ps};
 use crate::project::{StyleItem, Task, TaskStep};
 use crate::{CmdResult, CommandSpec as C, Project, Session};
 
-/// Largest item file read (512 MiB, like projects).
-const MAX_FILE: u64 = 512 << 20;
+/// Largest item file read: a project's ([`crate::project::max_bytes`]), for every kind of item
+/// (maps, layers, styles, tasks, packages), as each was bounded like a project before and an
+/// item file may be a whole project (Import Maps reads `.rgis` files).
+fn max_file() -> u64 {
+    crate::project::max_bytes()
+}
+
+/// What an item past [`max_file`] is told.
+fn too_large() -> String {
+    format!("the file is too large (maximum {})", crate::project::limit_said(max_file()))
+}
 
 pub fn commands() -> Vec<C> {
     vec![
@@ -368,10 +377,10 @@ pub(crate) fn read_text(p: &Value) -> Result<String, String> {
         return Ok(text);
     }
     match (ps(p, "text"), ps(p, "path")) {
-        (Some(t), _) if t.len() as u64 <= MAX_FILE => Ok(t.to_string()),
-        (Some(_), _) => Err("the file is too large".into()),
+        (Some(t), _) if t.len() as u64 <= max_file() => Ok(t.to_string()),
+        (Some(_), _) => Err(too_large()),
         (None, Some(path)) => {
-            let bytes = crate::project::read_file(Path::new(path), MAX_FILE)?;
+            let bytes = crate::project::read_file(Path::new(path), max_file())?;
             // Packages are zip archives holding the project.
             if bytes.starts_with(b"PK") {
                 return package_text(&bytes);
@@ -657,13 +666,13 @@ fn package_project(s: &mut Session) -> Result<Vec<u8>, String> {
         *m = embedded_map(m, &mut budget);
     }
     let name = p.name.clone();
-    let (text, report) = crate::durable::output_owned(p, |p| serde_json::to_string(p).map_err(|e| e.to_string()))?;
-    crate::durable::notify(s, &report);
-    let limit = crate::project::MAX_BYTES.min(rustgis_data::zip::MAX_TOTAL as u64);
-    if text.len() as u64 > limit {
-        return Err(format!("the project is too large to package (maximum {} MiB)", limit >> 20));
-    }
     let readme = format!("{} — a RustGIS project package.\nOpen it with RustGIS (Project › Open), or unzip it for {}.rgis.\n", name, name);
+    // The package as unzipped, the README with it, within what a zip and a project may hold.
+    let limit = crate::project::max_bytes().min(rustgis_data::zip::MAX_TOTAL as u64);
+    let room = limit.saturating_sub(readme.len() as u64);
+    let package_too_large = || format!("the project is too large to package (maximum {} MiB)", limit >> 20);
+    let (text, report) = crate::durable::output_owned(p, |p| crate::project::json_within(p, room)?.ok_or_else(package_too_large))?;
+    crate::durable::notify(s, &report);
     rustgis_data::zip::write(&[
         (format!("{}.{}", safe(&name), crate::project::EXTENSION), text.into_bytes()),
         ("README.txt".into(), readme.into_bytes()),
@@ -1617,8 +1626,8 @@ fn durable_bytes<T: crate::durable::Durable + serde::Serialize>(s: &mut Session,
 pub(crate) fn inline_text(p: &Value) -> Result<Option<String>, String> {
     if let Some(d) = ps(p, "data") {
         use base64::Engine;
-        if d.len() as u64 > MAX_FILE / 3 * 4 {
-            return Err("the file is too large".into());
+        if d.len() as u64 > max_file() / 3 * 4 {
+            return Err(too_large());
         }
         let bytes = base64::engine::general_purpose::STANDARD.decode(d.trim()).map_err(|_| "data must be base64".to_string())?;
         if bytes.starts_with(b"PK") {
@@ -1627,8 +1636,8 @@ pub(crate) fn inline_text(p: &Value) -> Result<Option<String>, String> {
         return String::from_utf8(bytes).map(Some).map_err(|_| "not a RustGIS file".to_string());
     }
     match ps(p, "text") {
-        Some(t) if t.len() as u64 <= MAX_FILE => Ok(Some(t.to_string())),
-        Some(_) => Err("the file is too large".into()),
+        Some(t) if t.len() as u64 <= max_file() => Ok(Some(t.to_string())),
+        Some(_) => Err(too_large()),
         None => Ok(None),
     }
 }

crates/engine/src/cmd/workflows.rs

@@ -386,6 +386,7 @@ fn environments(s: &mut Session, p: &Value) -> CmdResult {
         "densityCells": rustgis_analysis::max_density_cells(),
         "bufferBytes": rustgis_analysis::max_buffer_bytes(),
         "threadScratchBytes": rustgis_analysis::scratch_budget(),
+        "projectBytes": crate::project::max_bytes(),
     });
     Ok(json!({"extent": env.extent.map(|e| [e.min_x, e.min_y, e.max_x, e.max_y]), "cellSize": env.cell_size, "limits": limits}))
 }

crates/engine/src/peer.rs

@@ -477,13 +477,13 @@ impl Session {
     }
 }
 
-/// The same opening bound applies to peers; no destination is written on a formatting failure.
+/// A peer may be the web app, which opens at most [`BYTES_FLOOR`](crate::project::BYTES_FLOOR)
+/// whatever this computer saves: the text is never built past that. No destination is written on
+/// a formatting failure.
 fn text(p: &Project) -> Result<String, String> {
-    let text = serde_json::to_string(p).map_err(|e| e.to_string())?;
-    if text.len() as u64 > crate::project::MAX_BYTES {
-        return Err("the project is too large to exchange".into());
-    }
-    Ok(text)
+    let max = crate::project::BYTES_FLOOR;
+    crate::project::json_within(p, max)?
+        .ok_or_else(|| format!("the project is too large to exchange (maximum {} MiB, what the web app opens)", max >> 20))
 }
 
 #[cfg(test)]

crates/engine/src/project.rs

@@ -56,8 +56,122 @@ pub fn open_command(path: &str) -> &'static str {
 /// Saved-document compatibility version, independent of the RustGIS release version.
 /// Advance it when an older reader would lose meaningful saved information on resaving.
 pub const FORMAT: u32 = 1;
-/// Largest project file read or saved (512 MiB).
-pub const MAX_BYTES: u64 = 512 << 20;
+/// Largest project file read or saved in the browser, or on a computer whose memory isn't known
+/// (512 MiB): the floor of [`max_bytes`].
+pub const BYTES_FLOOR: u64 = 512 << 20;
+
+/// The most [`max_bytes`] grows to, however much memory the computer has (4 GiB).
+pub const BYTES_CEILING: u64 = 4 << 30;
+
+/// The memory an open project takes for each byte of its file, as measured opening projects of
+/// GeoJSON layers from the command line:
+/// - polygons of 40 vertices and 8 attributes: about 2 (337 MiB took 731 MiB; 674 MiB, 1.41 GiB;
+///   the app took 576 MiB more than empty to open and draw the first);
+/// - points with 12 short text attributes: about 4.4 (101 MiB took 449 MiB);
+/// - points with 30 one-letter attributes, about the most a byte of file costs: about 5.2
+///   (87 MiB took 453 MiB).
+///
+/// Planned at 5: at the limit, even the most measured takes little more than a quarter of the
+/// memory, and a project of the size in #1615 (659 MiB) opens on a computer sold as 16 GB, which
+/// reports a little less than 16 GiB.
+const MEMORY_PER_BYTE: u64 = 5;
+
+/// Largest project file read or saved on this computer (#1615): on the desktop as much as a
+/// quarter of its memory holds open at [`MEMORY_PER_BYTE`] a byte, from [`BYTES_FLOOR`] up to
+/// [`BYTES_CEILING`], as the other memory limits follow it (#1291 step 2); [`BYTES_FLOOR`] in the
+/// browser. Save and open use the same limit, so a project saved here opens here; one saved on a
+/// computer with more memory may need one like it.
+pub fn max_bytes() -> u64 {
+    #[cfg(test)]
+    if let Some(max) = TEST_MAX.with(std::cell::Cell::get) {
+        return max;
+    }
+    max_bytes_for(rustgis_data::memory::physical())
+}
+
+/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known). The formula is
+/// [`rustgis_data::memory::scaled`]'s, worked out here in `u64`, as that one counts in `usize`,
+/// which doesn't hold 4 GiB on a 32-bit build.
+fn max_bytes_for(memory: Option<u64>) -> u64 {
+    memory.map_or(BYTES_FLOOR, |m| (m / rustgis_data::memory::SHARE / MEMORY_PER_BYTE).clamp(BYTES_FLOOR, ceiling()))
+}
+
+/// The most [`max_bytes`] grows to in this build: [`BYTES_CEILING`] on a 64-bit build. A 32-bit
+/// build stays at [`BYTES_FLOOR`] whatever the memory: an open project takes 2 to 5 times its file
+/// in address space, which 4 GiB doesn't hold for a larger one.
+fn ceiling() -> u64 {
+    if usize::BITS >= 64 { BYTES_CEILING } else { BYTES_FLOOR }
+}
+
+/// The memory a 64-bit build needs for [`max_bytes`] to reach `bytes`: the inverse of
+/// [`max_bytes_for`] above the floor.
+fn memory_for(bytes: u64) -> u64 {
+    bytes.saturating_mul(rustgis_data::memory::SHARE).saturating_mul(MEMORY_PER_BYTE)
+}
+
+/// `value` as JSON text, or `None` as soon as it passes `max` bytes: the text of a project too
+/// large to keep is never built whole, only up to the limit.
+pub(crate) fn json_within<T: Serialize + ?Sized>(value: &T, max: u64) -> Result<Option<String>, String> {
+    struct Capped {
+        buf: Vec<u8>,
+        max: usize,
+        over: bool,
+    }
+    impl std::io::Write for Capped {
+        fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
+            if self.buf.len().saturating_add(b.len()) > self.max {
+                self.over = true;
+                return Err(std::io::Error::other("past the limit"));
+            }
+            // A machine short of memory gets an error, not an abort.
+            self.buf.try_reserve(b.len()).map_err(|_| std::io::Error::other("out of memory"))?;
+            self.buf.extend_from_slice(b);
+            Ok(b.len())
+        }
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+    let mut w = Capped { buf: Vec::new(), max: usize::try_from(max).unwrap_or(usize::MAX), over: false };
+    match serde_json::to_writer(&mut w, value) {
+        Ok(()) => String::from_utf8(w.buf).map(Some).map_err(|e| e.to_string()),
+        Err(_) if w.over => Ok(None),
+        Err(e) => Err(e.to_string()),
+    }
+}
+
+/// The limit as a refusal says it: in MiB and, where it follows the computer's memory, that it
+/// does, so a project refused here but opened elsewhere is understood.
+pub fn limit_said(max: u64) -> String {
+    // A 32-bit build keeps the floor whatever the memory: its limit doesn't follow it.
+    if rustgis_data::memory::physical().is_some() && ceiling() > BYTES_FLOOR {
+        format!("{} MiB on this computer, a limit that follows its memory", max >> 20)
+    } else {
+        format!("{} MiB", max >> 20)
+    }
+}
+
+#[cfg(test)]
+thread_local! {
+    /// A smaller [`max_bytes`] for the tests on this thread ([`with_max_bytes`]), so a test of
+    /// the limit doesn't build a project of gigabytes.
+    static TEST_MAX: std::cell::Cell<Option<u64>> = const { std::cell::Cell::new(None) };
+}
+
+/// Runs `f` with [`max_bytes`] at `max` on the calling thread only (work `f` hands to another
+/// thread sees the real limit); the limit is restored even if `f` panics.
+#[cfg(test)]
+pub(crate) fn with_max_bytes<T>(max: u64, f: impl FnOnce() -> T) -> T {
+    struct Restore;
+    impl Drop for Restore {
+        fn drop(&mut self) {
+            TEST_MAX.with(|m| m.set(None));
+        }
+    }
+    TEST_MAX.with(|m| m.set(Some(max)));
+    let _restore = Restore;
+    f()
+}
 
 /// Runtime capabilities never survive a project boundary, including an in-process save restore.
 /// `None` for an address that holds none, and for a wrapper that doesn't parse (kept as it is,
@@ -544,8 +658,9 @@ struct FileFormat {
 /// metadata is legacy content; arrays include legacy style files. Writers remain unchanged
 /// until a change needs a newer format, at which point it must declare that format.
 pub(crate) fn check_format(text: &str, kind: &str) -> Result<(), String> {
-    if text.len() as u64 > MAX_BYTES {
-        return Err(format!("the {kind} file is too large"));
+    let max = max_bytes();
+    if text.len() as u64 > max {
+        return Err(format!("the {kind} file is too large (maximum {})", limit_said(max)));
     }
     if text.trim_start().starts_with('[') {
         return Ok(());
@@ -1018,11 +1133,12 @@ impl Project {
     pub fn recovery_text_with_report(mut self) -> Result<(String, crate::durable::Report), String> {
         self.omit_pending_imports();
         let unrecorded = self.bound_address_losses(false);
-        let (text, mut report) = crate::durable::output_owned(self, project_text)?;
+        // Refused as a recovery copy, not as a save, and never built past the limit.
+        let (text, mut report) = crate::durable::output_owned(self, |p| {
+            let max = max_bytes();
+            json_within(p, max)?.ok_or_else(|| format!("the project is too large to keep a recovery copy (maximum {})", limit_said(max)))
+        })?;
         report.addresses = report.addresses.saturating_add(u32::try_from(unrecorded).unwrap_or(u32::MAX));
-        if text.len() as u64 > MAX_BYTES {
-            return Err("the project is too large to keep a recovery copy".into());
-        }
         Ok((text, report))
     }
 
@@ -1084,10 +1200,37 @@ impl Project {
         Ok(p)
     }
 
-    /// Opens a project file (a regular file within [`MAX_BYTES`]; see `read_file`), resolving
+    /// Opens a project file (a regular file within [`max_bytes`]; see `read_file`), resolving
     /// its relative paths against its folder.
     pub fn load(path: &Path) -> Result<Project, String> {
-        let bytes = read_file(path, MAX_BYTES)?;
+        let max = max_bytes();
+        // Past the limit: said with its size and the limit here, before anything is read.
+        if let Ok(meta) = std::fs::metadata(path)
+            && meta.is_file()
+            && meta.len() > max
+        {
+            // A computer with more memory opens it, up to the ceiling; past that none does.
+            // The memory a 64-bit build needs to open it (whole GiB, rounded up).
+            let needed = memory_for(meta.len()).div_ceil(1 << 30);
+            let more = if meta.len() > BYTES_CEILING {
+                format!(": no computer opens a project past {} MiB", BYTES_CEILING >> 20)
+            } else if meta.len() > ceiling() {
+                // Only a 32-bit build stops below the ceiling.
+                format!(": a 64-bit RustGIS on a computer with at least {needed} GiB of memory opens it")
+            } else if rustgis_data::memory::physical().is_some() {
+                format!(": a computer with at least {needed} GiB of memory opens it")
+            } else {
+                String::new()
+            };
+            return Err(format!(
+                "{} is {} MiB, more than a project can be (maximum {}){more}",
+                path.display(),
+                // Rounded up: a file a byte past 512 MiB isn't "512 MiB".
+                meta.len().div_ceil(1 << 20),
+                limit_said(max)
+            ));
+        }
+        let bytes = read_file(path, max)?;
         let text = String::from_utf8(bytes).map_err(|_| format!("{} isn't a RustGIS project", path.display()))?;
         let dir = std::path::absolute(path).ok().and_then(|p| p.parent().map(Path::to_path_buf));
         Self::read_json(&text, true, dir.as_deref(), true)
@@ -1353,11 +1496,73 @@ impl Durable for ModelGraph {
 
 /// Enforces the opening bound inside the formatter, before any destination is replaced.
 fn project_text(p: &Project) -> Result<String, String> {
-    let text = serde_json::to_string(p).map_err(|e| e.to_string())?;
-    if text.len() as u64 > MAX_BYTES {
-        return Err(format!("the project is too large to save (maximum {} MiB)", MAX_BYTES >> 20));
+    let max = max_bytes();
+    json_within(p, max)?.ok_or_else(|| format!("the project is too large to save (maximum {})", limit_said(max)))
+}
+
+#[cfg(test)]
+mod limit_tests {
+    use super::*;
+
+    const GIB: u64 = 1 << 30;
+
+    /// A quarter of the memory, at 5 bytes a byte of project: a twentieth of it, from 512 MiB to
+    /// 4 GiB; 512 MiB where the memory isn't known (the browser).
+    #[test]
+    fn the_project_limit_follows_the_memory() {
+        assert_eq!(max_bytes_for(None), BYTES_FLOOR);
+        assert_eq!(max_bytes_for(Some(8 * GIB)), BYTES_FLOOR, "never below the floor");
+        // Within this build's ceiling: a 32-bit build stays at the floor.
+        let within = |v: u64| v.clamp(BYTES_FLOOR, ceiling());
+        assert_eq!(max_bytes_for(Some(16 * GIB)), within(16 * GIB / 20));
+        assert_eq!(max_bytes_for(Some(64 * GIB)), within(64 * GIB / 20));
+        assert_eq!(max_bytes_for(Some(80 * GIB)), ceiling());
+        assert_eq!(max_bytes_for(Some(1024 * GIB)), ceiling(), "never above the ceiling");
+        assert_eq!(max_bytes_for(Some(u64::MAX)), ceiling(), "no overflow");
+        // The 659 MiB project of #1615 opens on a computer sold as 16 GB, which reports less (in a
+        // 64-bit build, as the app is).
+        if usize::BITS >= 64 {
+            assert!(max_bytes_for(Some(15 * GIB)) > 659 << 20);
+            assert_eq!(ceiling(), BYTES_CEILING);
+        } else {
+            assert_eq!(ceiling(), BYTES_FLOOR);
+        }
+        let max = max_bytes();
+        assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}");
+        // The memory named for a refused file is enough to open it, and not much more.
+        let size: u64 = 659 << 20;
+        let needed = memory_for(size).div_ceil(GIB) * GIB;
+        if usize::BITS >= 64 {
+            assert!(max_bytes_for(Some(needed)) >= size && max_bytes_for(Some(needed - GIB)) < size);
+        }
+        assert_eq!(needed / GIB, 13);
+    }
+
+    /// Writing stops at the limit: what comes after it is never serialized.
+    #[test]
+    fn serializing_stops_at_the_limit() {
+        struct Probe<'a>(&'a std::cell::Cell<bool>);
+        impl Serialize for Probe<'_> {
+            fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
+                self.0.set(true);
+                s.serialize_unit()
+            }
+        }
+        let visited = std::cell::Cell::new(false);
+        let v = ("x".repeat(100), Probe(&visited));
+        assert_eq!(json_within(&v, 50).ok(), Some(None));
+        assert!(!visited.get(), "the value after the limit was serialized");
+        assert!(json_within(&v, 1000).ok().flatten().is_some() && visited.get());
+    }
+
+    /// The text stops at the limit: whole within it, none past it, never built past it.
+    #[test]
+    fn json_is_never_built_past_the_limit() {
+        let v = serde_json::json!({"a": "x".repeat(100)});
+        let whole = serde_json::to_string(&v).unwrap_or_default();
+        assert_eq!(json_within(&v, whole.len() as u64).ok().flatten(), Some(whole.clone()));
+        assert_eq!(json_within(&v, whole.len() as u64 - 1).ok(), Some(None));
     }
-    Ok(text)
 }
 
 #[cfg(test)]

crates/engine/src/roles.rs

@@ -376,7 +376,7 @@ impl Durable for VecSteps {
 /// Reads documents at the existing reader bound, without normalization, trust or restore-cache
 /// access. Unchanged documents keep their exact text; only changed typed copies are serialized.
 fn document(kind: Document, v: &Value, depth: usize, report: &mut Report) -> Option<Value> {
-    let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::MAX_BYTES) else { return invalid(v, report) };
+    let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::max_bytes()) else { return invalid(v, report) };
     if serde_json::from_str::<DocumentFormat>(text).is_ok_and(|f| f.format > u64::from(crate::project::FORMAT)) {
         report.unchecked = report.unchecked.saturating_add(1);
         return None;