From 6697bfd0ce97ecc4daeca94b744965435694ce72 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Sun, 8 Nov 2026 16:20:53 +0100 Subject: [PATCH 1/5] feat(engine): the largest project follows the computer's memory (#1615) A project file was limited to 512 MiB, for saving (since #1364) and opening, whatever the computer: a project of many layers loaded whole couldn't be saved, and one saved by v0.5.0 past the limit couldn't be opened again. The limit now follows the computer's memory as the other memory limits do (#1291 step 2): - project::max_bytes(): on the desktop as much as a quarter of the memory holds open at 4 bytes a byte of project, from 512 MiB (the browser's, and where the memory isn't known) up to 4 GiB; about 1 GiB of project for every 16 GB. Measured: a 337 MiB project took 731 MiB to open from the command line, a 674 MiB one 1.41 GiB. - Save, open, the recovery copy, packages, peer exchange, saved role documents and item files (Import Maps reads projects) use it, so a project saved here opens here. - A refused project says its size and the limit, that the limit follows the memory, and that a computer with more memory opens it (or that none does, past 4 GiB); the recovery copy's refusal says its limit too. - gp.env reports it as projectBytes; the docs (Projects, the memory limits table) say it, and that packages stay within 1 GiB (256 MiB in the web app). - Tests: the limit at several memory sizes, a project just under the limit saved, opened and imported, one past it refused with both sizes; the #1364 test runs at a 1 MiB limit. --- crates/engine/src/cmd/items.rs | 28 ++++--- crates/engine/src/cmd/workflows.rs | 1 + crates/engine/src/peer.rs | 2 +- crates/engine/src/project.rs | 128 ++++++++++++++++++++++++++--- crates/engine/src/roles.rs | 2 +- crates/engine/src/tests.rs | 53 ++++++++++-- crates/ui-egui/src/autosave.rs | 8 +- docs/user-guide/analysis.md | 1 + docs/user-guide/workspace.md | 11 +++ 9 files changed, 203 insertions(+), 31 deletions(-) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 7bd3eb274..3574b2351 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -15,8 +15,16 @@ use super::{b, f, need_feature_layer, need_layer, ps}; use crate::project::{StyleItem, Task, TaskStep}; use crate::{CmdResult, CommandSpec as C, Project, Session}; -/// Largest item file read (512 MiB, like projects). -const MAX_FILE: u64 = 512 << 20; +/// Largest item file read: a project's ([`crate::project::max_bytes`]), as an item file may be +/// a whole project (Import Maps reads `.rgis` files). +fn max_file() -> u64 { + crate::project::max_bytes() +} + +/// What an item past [`max_file`] is told. +fn too_large() -> String { + format!("the file is too large (maximum {})", crate::project::limit_said(max_file())) +} pub fn commands() -> Vec { vec![ @@ -368,10 +376,10 @@ pub(crate) fn read_text(p: &Value) -> Result { return Ok(text); } match (ps(p, "text"), ps(p, "path")) { - (Some(t), _) if t.len() as u64 <= MAX_FILE => Ok(t.to_string()), - (Some(_), _) => Err("the file is too large".into()), + (Some(t), _) if t.len() as u64 <= max_file() => Ok(t.to_string()), + (Some(_), _) => Err(too_large()), (None, Some(path)) => { - let bytes = crate::project::read_file(Path::new(path), MAX_FILE)?; + let bytes = crate::project::read_file(Path::new(path), max_file())?; // Packages are zip archives holding the project. if bytes.starts_with(b"PK") { return package_text(&bytes); @@ -659,7 +667,7 @@ fn package_project(s: &mut Session) -> Result, String> { let name = p.name.clone(); let (text, report) = crate::durable::output_owned(p, |p| serde_json::to_string(p).map_err(|e| e.to_string()))?; crate::durable::notify(s, &report); - let limit = crate::project::MAX_BYTES.min(rustgis_data::zip::MAX_TOTAL as u64); + let limit = crate::project::max_bytes().min(rustgis_data::zip::MAX_TOTAL as u64); if text.len() as u64 > limit { return Err(format!("the project is too large to package (maximum {} MiB)", limit >> 20)); } @@ -1617,8 +1625,8 @@ fn durable_bytes(s: &mut Session, pub(crate) fn inline_text(p: &Value) -> Result, String> { if let Some(d) = ps(p, "data") { use base64::Engine; - if d.len() as u64 > MAX_FILE / 3 * 4 { - return Err("the file is too large".into()); + if d.len() as u64 > max_file() / 3 * 4 { + return Err(too_large()); } let bytes = base64::engine::general_purpose::STANDARD.decode(d.trim()).map_err(|_| "data must be base64".to_string())?; if bytes.starts_with(b"PK") { @@ -1627,8 +1635,8 @@ pub(crate) fn inline_text(p: &Value) -> Result, String> { return String::from_utf8(bytes).map(Some).map_err(|_| "not a RustGIS file".to_string()); } match ps(p, "text") { - Some(t) if t.len() as u64 <= MAX_FILE => Ok(Some(t.to_string())), - Some(_) => Err("the file is too large".into()), + Some(t) if t.len() as u64 <= max_file() => Ok(Some(t.to_string())), + Some(_) => Err(too_large()), None => Ok(None), } } diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 134a54393..b537f8a29 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -386,6 +386,7 @@ fn environments(s: &mut Session, p: &Value) -> CmdResult { "densityCells": rustgis_analysis::max_density_cells(), "bufferBytes": rustgis_analysis::max_buffer_bytes(), "threadScratchBytes": rustgis_analysis::scratch_budget(), + "projectBytes": crate::project::max_bytes(), }); Ok(json!({"extent": env.extent.map(|e| [e.min_x, e.min_y, e.max_x, e.max_y]), "cellSize": env.cell_size, "limits": limits})) } diff --git a/crates/engine/src/peer.rs b/crates/engine/src/peer.rs index c26f6199a..f575021fe 100644 --- a/crates/engine/src/peer.rs +++ b/crates/engine/src/peer.rs @@ -480,7 +480,7 @@ impl Session { /// The same opening bound applies to peers; no destination is written on a formatting failure. fn text(p: &Project) -> Result { let text = serde_json::to_string(p).map_err(|e| e.to_string())?; - if text.len() as u64 > crate::project::MAX_BYTES { + if text.len() as u64 > crate::project::max_bytes() { return Err("the project is too large to exchange".into()); } Ok(text) diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 8c7a4c5da..877c1f1f8 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -56,8 +56,69 @@ pub fn open_command(path: &str) -> &'static str { /// Saved-document compatibility version, independent of the RustGIS release version. /// Advance it when an older reader would lose meaningful saved information on resaving. pub const FORMAT: u32 = 1; -/// Largest project file read or saved (512 MiB). -pub const MAX_BYTES: u64 = 512 << 20; +/// Largest project file read or saved in the browser, or on a computer whose memory isn't known +/// (512 MiB): the floor of [`max_bytes`]. +pub const BYTES_FLOOR: u64 = 512 << 20; + +/// The most [`max_bytes`] grows to, however much memory the computer has (4 GiB). +pub const BYTES_CEILING: u64 = 4 << 30; + +/// The memory an open project takes for each byte of its file. Measured at about 2: a 337 MiB +/// project of four GeoJSON layers (240,000 polygons) took 731 MiB to open in the command line, and +/// 576 MiB more than an empty app to open and draw. Planned at 4, for the recovery copy written +/// while it is open and for data heavier to hold than to write. +const MEMORY_PER_BYTE: u64 = 4; + +/// Largest project file read or saved on this computer (#1615): on the desktop as much as a +/// quarter of its memory holds open at [`MEMORY_PER_BYTE`] a byte, from [`BYTES_FLOOR`] up to +/// [`BYTES_CEILING`], as the other memory limits follow it (#1291 step 2); [`BYTES_FLOOR`] in the +/// browser. Save and open use the same limit, so a project saved here opens here; one saved on a +/// computer with more memory may need one like it. +pub fn max_bytes() -> u64 { + #[cfg(test)] + if let Some(max) = TEST_MAX.with(std::cell::Cell::get) { + return max; + } + max_bytes_for(rustgis_data::memory::physical()) +} + +/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known). +fn max_bytes_for(memory: Option) -> u64 { + let (floor, ceiling) = (usize::try_from(BYTES_FLOOR).unwrap_or(usize::MAX), usize::try_from(BYTES_CEILING).unwrap_or(usize::MAX)); + rustgis_data::memory::scaled(memory, floor, ceiling, MEMORY_PER_BYTE) as u64 +} + +/// The limit as a refusal says it: in MiB and, where it follows the computer's memory, that it +/// does, so a project refused here but opened elsewhere is understood. +pub fn limit_said(max: u64) -> String { + if rustgis_data::memory::physical().is_some() { + format!("{} MiB on this computer, a limit that follows its memory", max >> 20) + } else { + format!("{} MiB", max >> 20) + } +} + +#[cfg(test)] +thread_local! { + /// A smaller [`max_bytes`] for the tests on this thread ([`with_max_bytes`]), so a test of + /// the limit doesn't build a project of gigabytes. + static TEST_MAX: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Runs `f` with [`max_bytes`] at `max` on the calling thread only (work `f` hands to another +/// thread sees the real limit); the limit is restored even if `f` panics. +#[cfg(test)] +pub(crate) fn with_max_bytes(max: u64, f: impl FnOnce() -> T) -> T { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + TEST_MAX.with(|m| m.set(None)); + } + } + TEST_MAX.with(|m| m.set(Some(max))); + let _restore = Restore; + f() +} /// Runtime capabilities never survive a project boundary, including an in-process save restore. /// `None` for an address that holds none, and for a wrapper that doesn't parse (kept as it is, @@ -544,8 +605,9 @@ struct FileFormat { /// metadata is legacy content; arrays include legacy style files. Writers remain unchanged /// until a change needs a newer format, at which point it must declare that format. pub(crate) fn check_format(text: &str, kind: &str) -> Result<(), String> { - if text.len() as u64 > MAX_BYTES { - return Err(format!("the {kind} file is too large")); + let max = max_bytes(); + if text.len() as u64 > max { + return Err(format!("the {kind} file is too large (maximum {})", limit_said(max))); } if text.trim_start().starts_with('[') { return Ok(()); @@ -1020,8 +1082,9 @@ impl Project { let unrecorded = self.bound_address_losses(false); let (text, mut report) = crate::durable::output_owned(self, project_text)?; report.addresses = report.addresses.saturating_add(u32::try_from(unrecorded).unwrap_or(u32::MAX)); - if text.len() as u64 > MAX_BYTES { - return Err("the project is too large to keep a recovery copy".into()); + let max = max_bytes(); + if text.len() as u64 > max { + return Err(format!("the project is too large to keep a recovery copy (maximum {})", limit_said(max))); } Ok((text, report)) } @@ -1084,10 +1147,31 @@ impl Project { Ok(p) } - /// Opens a project file (a regular file within [`MAX_BYTES`]; see `read_file`), resolving + /// Opens a project file (a regular file within [`max_bytes`]; see `read_file`), resolving /// its relative paths against its folder. pub fn load(path: &Path) -> Result { - let bytes = read_file(path, MAX_BYTES)?; + let max = max_bytes(); + // Past the limit: said with its size and the limit here, before anything is read. + if let Ok(meta) = std::fs::metadata(path) + && meta.is_file() + && meta.len() > max + { + // A computer with more memory opens it, up to the ceiling; past that none does. + let more = if meta.len() > BYTES_CEILING { + format!(": no computer opens a project past {} MiB", BYTES_CEILING >> 20) + } else if rustgis_data::memory::physical().is_some() { + ": a computer with more memory opens it".to_string() + } else { + String::new() + }; + return Err(format!( + "{} is {} MiB, more than a project can be (maximum {}){more}", + path.display(), + meta.len() >> 20, + limit_said(max) + )); + } + let bytes = read_file(path, max)?; let text = String::from_utf8(bytes).map_err(|_| format!("{} isn't a RustGIS project", path.display()))?; let dir = std::path::absolute(path).ok().and_then(|p| p.parent().map(Path::to_path_buf)); Self::read_json(&text, true, dir.as_deref(), true) @@ -1354,12 +1438,36 @@ impl Durable for ModelGraph { /// Enforces the opening bound inside the formatter, before any destination is replaced. fn project_text(p: &Project) -> Result { let text = serde_json::to_string(p).map_err(|e| e.to_string())?; - if text.len() as u64 > MAX_BYTES { - return Err(format!("the project is too large to save (maximum {} MiB)", MAX_BYTES >> 20)); + let max = max_bytes(); + if text.len() as u64 > max { + return Err(format!("the project is too large to save (maximum {})", limit_said(max))); } Ok(text) } +#[cfg(test)] +mod limit_tests { + use super::*; + + const GIB: u64 = 1 << 30; + + /// A quarter of the memory, at 4 bytes a byte of project: a sixteenth of it, from 512 MiB to + /// 4 GiB; 512 MiB where the memory isn't known (the browser). + #[test] + fn the_project_limit_follows_the_memory() { + assert_eq!(max_bytes_for(None), BYTES_FLOOR); + assert_eq!(max_bytes_for(Some(4 * GIB)), BYTES_FLOOR, "never below the floor"); + assert_eq!(max_bytes_for(Some(8 * GIB)), 512 << 20); + assert_eq!(max_bytes_for(Some(16 * GIB)), GIB); + assert_eq!(max_bytes_for(Some(64 * GIB)), 4 * GIB); + assert_eq!(max_bytes_for(Some(1024 * GIB)), BYTES_CEILING, "never above the ceiling"); + // The 659 MiB project of #1615 opens on a computer with 12 GiB. + assert!(max_bytes_for(Some(12 * GIB)) > 659 << 20); + let max = max_bytes(); + assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}"); + } +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod login_binding_tests { diff --git a/crates/engine/src/roles.rs b/crates/engine/src/roles.rs index cc3897628..62173f88a 100644 --- a/crates/engine/src/roles.rs +++ b/crates/engine/src/roles.rs @@ -376,7 +376,7 @@ impl Durable for VecSteps { /// Reads documents at the existing reader bound, without normalization, trust or restore-cache /// access. Unchanged documents keep their exact text; only changed typed copies are serialized. fn document(kind: Document, v: &Value, depth: usize, report: &mut Report) -> Option { - let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::MAX_BYTES) else { return invalid(v, report) }; + let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::max_bytes()) else { return invalid(v, report) }; if serde_json::from_str::(text).is_ok_and(|f| f.format > u64::from(crate::project::FORMAT)) { report.unchecked = report.unchecked.saturating_add(1); return None; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 82bbe3928..f7f1ac08a 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -2113,12 +2113,15 @@ fn oversized_project_save_preserves_file_and_live_state() { s.run("project.rename", &json!({"name": "Unsaved work"})).unwrap(); let source = dir.join("roads.geojson").display().to_string(); s.project.maps[0].layers[0].dataset.source = Some(source.clone()); - // A single large value reaches the real limit without a large data fixture on disk. - s.project.thumbnail = Some("x".repeat(crate::project::MAX_BYTES as usize)); - let error = s.run("project.saveAs", &json!({"path": path})).unwrap_err(); - assert!(error.contains("too large"), "{error}"); - let error = s.run("project.package", &json!({"path": package})).unwrap_err(); - assert!(error.contains("too large"), "{error}"); + // A single large value reaches the limit without a large data fixture on disk; the limit is + // held at 1 MiB here, as it follows this computer's memory (#1615). + let max = 1 << 20; + s.project.thumbnail = Some("x".repeat(max as usize)); + let (error, package_error) = crate::project::with_max_bytes(max, || { + (s.run("project.saveAs", &json!({"path": path})).unwrap_err(), s.run("project.package", &json!({"path": package})).unwrap_err()) + }); + assert!(error.contains("too large to save (maximum 1 MiB"), "{error}"); + assert!(package_error.contains("too large"), "{package_error}"); assert_eq!(std::fs::read(&path).unwrap(), original); assert_eq!(std::fs::read(&package).unwrap(), original_package); assert_eq!(s.project.name, "Unsaved work"); @@ -2132,6 +2135,43 @@ fn oversized_project_save_preserves_file_and_live_state() { std::fs::remove_dir_all(&dir).unwrap(); } +/// The project limit follows the computer's memory (#1615): a project as large as the limit saves +/// and opens again on the same computer, and one past it is refused before it is read, saying its +/// size and the limit. +#[test] +fn a_project_within_the_limit_saves_and_opens_and_one_past_it_says_both_sizes() { + let dir = std::env::temp_dir().join(format!("rustgis-project-limit-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("near.rgis"); + let max: u64 = 2 << 20; + let mut s = world(); + let small = { + s.run("project.saveAs", &json!({"path": path})).unwrap(); + std::fs::metadata(&path).unwrap().len() + }; + // Just under the limit: written and read back whole, the same limit both ways. + s.project.thumbnail = Some("x".repeat((max - small - 1024) as usize)); + crate::project::with_max_bytes(max, || { + s.run("project.saveAs", &json!({"path": path})).unwrap(); + assert!(std::fs::metadata(&path).unwrap().len() <= max); + assert_eq!(Project::load(&path).unwrap().thumbnail.map(|t| t.len()), s.project.thumbnail.as_ref().map(|t| t.len())); + }); + // Past it (as a project saved on a computer with more memory): refused, saying both sizes. + let big = dir.join("big.rgis"); + std::fs::write(&big, vec![b' '; (3 << 20) as usize]).unwrap(); + let e = crate::project::with_max_bytes(max, || Project::load(&big).unwrap_err()); + assert!(e.contains("is 3 MiB, more than a project can be (maximum 2 MiB"), "{e}"); + let e = crate::project::with_max_bytes(max, || crate::project::check_format(&" ".repeat(3 << 20), "project").unwrap_err()); + assert!(e.contains("too large (maximum 2 MiB"), "{e}"); + // Import Maps reads a project file under the same limit: the one saved above, not the one past it. + crate::project::with_max_bytes(max, || { + assert!(crate::cmd::items::read_text(&json!({"path": path})).is_ok()); + let e = crate::cmd::items::read_text(&json!({"path": big})).unwrap_err(); + assert!(e.contains("too large"), "{e}"); + }); + std::fs::remove_dir_all(&dir).ok(); +} + #[test] fn project_extension_is_rgis_and_legacy_rgisx_opens() { let mut s = world(); @@ -8266,6 +8306,7 @@ fn analysis_workflows_models_scripts_and_raster_functions() { assert!(limit("pagedFeatures") >= rustgis_data::ogc::PAGED_FEATURES_FLOOR, "{env}"); assert!(limit("densityCells") >= rustgis_analysis::DENSITY_CELLS_FLOOR, "{env}"); assert!(limit("bufferBytes") >= 512 << 20 && limit("threadScratchBytes") >= 256 << 20, "{env}"); + assert!(limit("projectBytes") as u64 == crate::project::max_bytes() && limit("projectBytes") >= 512 << 20, "{env}"); // Network route along the rivers: two points on the same river. let rivers = s.map().layer_by_name("Rivers").unwrap(); let (a, b2) = match rivers.dataset.features.iter().filter_map(|f| f.geometry.as_ref()).find_map(|g| match g { diff --git a/crates/ui-egui/src/autosave.rs b/crates/ui-egui/src/autosave.rs index b56ab561d..37b0ac56c 100644 --- a/crates/ui-egui/src/autosave.rs +++ b/crates/ui-egui/src/autosave.rs @@ -728,11 +728,13 @@ mod folder { fn read_pending(&self) -> Result { let path = self.list().first().map(|(p, _)| p.clone()).ok_or("there is no unsaved work to recover")?; let mut bytes = Vec::new(); + let max = rustgis_engine::project::max_bytes() + MAX_HEADER; std::fs::File::open(&path) - .and_then(|f| f.take(rustgis_engine::project::MAX_BYTES + MAX_HEADER + 1).read_to_end(&mut bytes)) + .and_then(|f| f.take(max + 1).read_to_end(&mut bytes)) .map_err(|e| format!("{}: {e}", path.display()))?; - if bytes.len() as u64 > rustgis_engine::project::MAX_BYTES + MAX_HEADER { - return Err("the recovery copy is too large".into()); + if bytes.len() as u64 > max { + let limit = rustgis_engine::project::limit_said(max - MAX_HEADER); + return Err(format!("the recovery copy is too large (maximum {limit})")); } // The description line goes in place: a copy near the size limit isn't held twice. let mut text = String::from_utf8(bytes).map_err(|_| "the recovery copy isn't text".to_string())?; diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index ec6623878..5e3dd0217 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -1207,6 +1207,7 @@ below the web app's fixed limits and never above a ceiling. | `densityCells`: cells of a Kernel Density raster | 16 million | 16 bytes a cell | 512 million | | `bufferBytes`: memory for Buffer's output and Select Layer By Location's buffered selectors | 512 MiB (128 MiB in the web app) | the bytes themselves | 8 GiB | | `threadScratchBytes`: working memory a tool's threads take together, beyond one thread's | 256 MiB | a sixteenth of memory, on top of the quarter | 4 GiB | +| `projectBytes`: the largest project file saved or opened (see [Projects](workspace.md#projects)) | 512 MiB | 4 bytes for each byte of the file (about 1 GiB for every 16 GB) | 4 GiB | So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index c3aedf98d..dbadce534 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -11,6 +11,17 @@ streamed layers reopen their sources, and file-backed rasters reopen their files sources accessible when moving a project. See [Project items](data.md#project-items) for templates, packages and files for individual items. +A project file holds the features of every layer loaded whole (GeoJSON, KML/KMZ, GPX, and +Shapefiles of up to 100,000 features), so a project of many such layers grows large. How large a +project can be saved and opened follows the computer's memory: on the desktop, as much as a +quarter of it holds open (about 1 GiB of project for every 16 GB), from 512 MiB up to 4 GiB; in +the web app, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened +again, and Open refuses a larger file, saying its size and the limit. A project saved on a +computer with more memory may need one like it to open. A project package (`.rgispkg`) stays +within 1 GiB (256 MiB in the web app), as zip archives do here. `gp.env` reports the limit as +`projectBytes`. Data kept in a GeoPackage, FlatGeobuf or GeoParquet file is read in place and +stays out of the project file. + **Open** also accepts a project package (`.rgispkg`), as do drag-and-drop and the desktop command line. A package opens as a new, unsaved project; Save writes a plain `.rgis` file. Opening or creating a project clears undo history; saving does not. From 845c024a6ce07ae386ea4b847507da327c165aba Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Sun, 8 Nov 2026 18:09:21 +0100 Subject: [PATCH 2/5] fix(engine): plan the project limit for attribute-heavy data and never build its text past it Review points on #1639: - Measured attribute-heavy projects: points with 12 short text attributes take about 4.4 bytes of memory a byte of file, with 30 one-letter attributes about 5.2. The limit now plans for 6 (about 680 MiB of project for every 16 GB), above the most measured. - A project's text is written into a buffer that stops at the limit (json_within), so one too large to save, package or exchange is never built whole first. - Peer exchange stays within what the web app opens (512 MiB), whatever this computer saves; a package counts its README within the zip's limit. - The limit is worked out in u64, the same on a 32-bit build. - The recovery copy's refusal is translatable (i18n::fmt); the docs say which files are read in place (more than 100,000 features). --- crates/engine/src/cmd/items.rs | 12 ++--- crates/engine/src/peer.rs | 12 ++--- crates/engine/src/project.rs | 82 +++++++++++++++++++++++++--------- crates/ui-egui/src/autosave.rs | 2 +- docs/user-guide/analysis.md | 2 +- docs/user-guide/workspace.md | 7 +-- 6 files changed, 79 insertions(+), 38 deletions(-) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 3574b2351..09ed99516 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -665,13 +665,13 @@ fn package_project(s: &mut Session) -> Result, String> { *m = embedded_map(m, &mut budget); } let name = p.name.clone(); - let (text, report) = crate::durable::output_owned(p, |p| serde_json::to_string(p).map_err(|e| e.to_string()))?; - crate::durable::notify(s, &report); - let limit = crate::project::max_bytes().min(rustgis_data::zip::MAX_TOTAL as u64); - if text.len() as u64 > limit { - return Err(format!("the project is too large to package (maximum {} MiB)", limit >> 20)); - } let readme = format!("{} — a RustGIS project package.\nOpen it with RustGIS (Project › Open), or unzip it for {}.rgis.\n", name, name); + // The package as unzipped, the README with it, within what a zip and a project may hold. + let limit = crate::project::max_bytes().min(rustgis_data::zip::MAX_TOTAL as u64); + let room = limit.saturating_sub(readme.len() as u64); + let too_large = || format!("the project is too large to package (maximum {} MiB)", limit >> 20); + let (text, report) = crate::durable::output_owned(p, |p| crate::project::json_within(p, room)?.ok_or_else(too_large))?; + crate::durable::notify(s, &report); rustgis_data::zip::write(&[ (format!("{}.{}", safe(&name), crate::project::EXTENSION), text.into_bytes()), ("README.txt".into(), readme.into_bytes()), diff --git a/crates/engine/src/peer.rs b/crates/engine/src/peer.rs index f575021fe..f075e32e2 100644 --- a/crates/engine/src/peer.rs +++ b/crates/engine/src/peer.rs @@ -477,13 +477,13 @@ impl Session { } } -/// The same opening bound applies to peers; no destination is written on a formatting failure. +/// A peer may be the web app, which opens at most [`BYTES_FLOOR`](crate::project::BYTES_FLOOR) +/// whatever this computer saves: the text is never built past that. No destination is written on +/// a formatting failure. fn text(p: &Project) -> Result { - let text = serde_json::to_string(p).map_err(|e| e.to_string())?; - if text.len() as u64 > crate::project::max_bytes() { - return Err("the project is too large to exchange".into()); - } - Ok(text) + let max = crate::project::BYTES_FLOOR; + crate::project::json_within(p, max)? + .ok_or_else(|| format!("the project is too large to exchange (maximum {} MiB, what the web app opens)", max >> 20)) } #[cfg(test)] diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 877c1f1f8..61f9d137d 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -63,11 +63,16 @@ pub const BYTES_FLOOR: u64 = 512 << 20; /// The most [`max_bytes`] grows to, however much memory the computer has (4 GiB). pub const BYTES_CEILING: u64 = 4 << 30; -/// The memory an open project takes for each byte of its file. Measured at about 2: a 337 MiB -/// project of four GeoJSON layers (240,000 polygons) took 731 MiB to open in the command line, and -/// 576 MiB more than an empty app to open and draw. Planned at 4, for the recovery copy written -/// while it is open and for data heavier to hold than to write. -const MEMORY_PER_BYTE: u64 = 4; +/// The memory an open project takes for each byte of its file, as measured opening projects of +/// GeoJSON layers from the command line: +/// - polygons of 40 vertices and 8 attributes: about 2 (337 MiB took 731 MiB; 674 MiB, 1.41 GiB; +/// the app took 576 MiB more than empty to open and draw the first); +/// - points with 12 short text attributes: about 4.4 (101 MiB took 449 MiB); +/// - points with 30 one-letter attributes, about the most a byte of file costs: about 5.2 +/// (87 MiB took 453 MiB). +/// +/// Planned at 6, above the most measured. +const MEMORY_PER_BYTE: u64 = 6; /// Largest project file read or saved on this computer (#1615): on the desktop as much as a /// quarter of its memory holds open at [`MEMORY_PER_BYTE`] a byte, from [`BYTES_FLOOR`] up to @@ -82,10 +87,39 @@ pub fn max_bytes() -> u64 { max_bytes_for(rustgis_data::memory::physical()) } -/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known). +/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known), as +/// [`rustgis_data::memory::scaled`] works it out, in `u64` so it is the same on a 32-bit build. fn max_bytes_for(memory: Option) -> u64 { - let (floor, ceiling) = (usize::try_from(BYTES_FLOOR).unwrap_or(usize::MAX), usize::try_from(BYTES_CEILING).unwrap_or(usize::MAX)); - rustgis_data::memory::scaled(memory, floor, ceiling, MEMORY_PER_BYTE) as u64 + memory.map_or(BYTES_FLOOR, |m| (m / rustgis_data::memory::SHARE / MEMORY_PER_BYTE).clamp(BYTES_FLOOR, BYTES_CEILING)) +} + +/// `value` as JSON text, or `None` as soon as it passes `max` bytes: the text of a project too +/// large to keep is never built whole, only up to the limit. +pub(crate) fn json_within(value: &T, max: u64) -> Result, String> { + struct Capped { + buf: Vec, + max: usize, + over: bool, + } + impl std::io::Write for Capped { + fn write(&mut self, b: &[u8]) -> std::io::Result { + if self.buf.len().saturating_add(b.len()) > self.max { + self.over = true; + return Err(std::io::Error::other("past the limit")); + } + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut w = Capped { buf: Vec::new(), max: usize::try_from(max).unwrap_or(usize::MAX), over: false }; + match serde_json::to_writer(&mut w, value) { + Ok(()) => String::from_utf8(w.buf).map(Some).map_err(|e| e.to_string()), + Err(_) if w.over => Ok(None), + Err(e) => Err(e.to_string()), + } } /// The limit as a refusal says it: in MiB and, where it follows the computer's memory, that it @@ -1437,12 +1471,8 @@ impl Durable for ModelGraph { /// Enforces the opening bound inside the formatter, before any destination is replaced. fn project_text(p: &Project) -> Result { - let text = serde_json::to_string(p).map_err(|e| e.to_string())?; let max = max_bytes(); - if text.len() as u64 > max { - return Err(format!("the project is too large to save (maximum {})", limit_said(max))); - } - Ok(text) + json_within(p, max)?.ok_or_else(|| format!("the project is too large to save (maximum {})", limit_said(max))) } #[cfg(test)] @@ -1451,21 +1481,31 @@ mod limit_tests { const GIB: u64 = 1 << 30; - /// A quarter of the memory, at 4 bytes a byte of project: a sixteenth of it, from 512 MiB to - /// 4 GiB; 512 MiB where the memory isn't known (the browser). + /// A quarter of the memory, at 6 bytes a byte of project: a twenty-fourth of it, from 512 MiB + /// to 4 GiB; 512 MiB where the memory isn't known (the browser). #[test] fn the_project_limit_follows_the_memory() { assert_eq!(max_bytes_for(None), BYTES_FLOOR); - assert_eq!(max_bytes_for(Some(4 * GIB)), BYTES_FLOOR, "never below the floor"); - assert_eq!(max_bytes_for(Some(8 * GIB)), 512 << 20); - assert_eq!(max_bytes_for(Some(16 * GIB)), GIB); - assert_eq!(max_bytes_for(Some(64 * GIB)), 4 * GIB); + assert_eq!(max_bytes_for(Some(8 * GIB)), BYTES_FLOOR, "never below the floor"); + assert_eq!(max_bytes_for(Some(16 * GIB)), 16 * GIB / 24); + assert_eq!(max_bytes_for(Some(64 * GIB)), 64 * GIB / 24); + assert_eq!(max_bytes_for(Some(96 * GIB)), 4 * GIB); assert_eq!(max_bytes_for(Some(1024 * GIB)), BYTES_CEILING, "never above the ceiling"); - // The 659 MiB project of #1615 opens on a computer with 12 GiB. - assert!(max_bytes_for(Some(12 * GIB)) > 659 << 20); + assert_eq!(max_bytes_for(Some(u64::MAX)), BYTES_CEILING, "no overflow"); + // The 659 MiB project of #1615 opens on a computer with 16 GiB. + assert!(max_bytes_for(Some(16 * GIB)) > 659 << 20); let max = max_bytes(); assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}"); } + + /// The text stops at the limit: whole within it, none past it, never built past it. + #[test] + fn json_is_never_built_past_the_limit() { + let v = serde_json::json!({"a": "x".repeat(100)}); + let whole = serde_json::to_string(&v).unwrap_or_default(); + assert_eq!(json_within(&v, whole.len() as u64).ok().flatten(), Some(whole.clone())); + assert_eq!(json_within(&v, whole.len() as u64 - 1).ok(), Some(None)); + } } #[cfg(test)] diff --git a/crates/ui-egui/src/autosave.rs b/crates/ui-egui/src/autosave.rs index 37b0ac56c..ab6d312c9 100644 --- a/crates/ui-egui/src/autosave.rs +++ b/crates/ui-egui/src/autosave.rs @@ -734,7 +734,7 @@ mod folder { .map_err(|e| format!("{}: {e}", path.display()))?; if bytes.len() as u64 > max { let limit = rustgis_engine::project::limit_said(max - MAX_HEADER); - return Err(format!("the recovery copy is too large (maximum {limit})")); + return Err(crate::i18n::fmt("the recovery copy is too large (maximum {limit})", &[("limit", &limit)])); } // The description line goes in place: a copy near the size limit isn't held twice. let mut text = String::from_utf8(bytes).map_err(|_| "the recovery copy isn't text".to_string())?; diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 5e3dd0217..ddcb5be9d 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -1207,7 +1207,7 @@ below the web app's fixed limits and never above a ceiling. | `densityCells`: cells of a Kernel Density raster | 16 million | 16 bytes a cell | 512 million | | `bufferBytes`: memory for Buffer's output and Select Layer By Location's buffered selectors | 512 MiB (128 MiB in the web app) | the bytes themselves | 8 GiB | | `threadScratchBytes`: working memory a tool's threads take together, beyond one thread's | 256 MiB | a sixteenth of memory, on top of the quarter | 4 GiB | -| `projectBytes`: the largest project file saved or opened (see [Projects](workspace.md#projects)) | 512 MiB | 4 bytes for each byte of the file (about 1 GiB for every 16 GB) | 4 GiB | +| `projectBytes`: the largest project file saved or opened (see [Projects](workspace.md#projects)) | 512 MiB | 6 bytes for each byte of the file (about 680 MiB for every 16 GB) | 4 GiB | So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index dbadce534..4d62d5c6f 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -14,13 +14,14 @@ templates, packages and files for individual items. A project file holds the features of every layer loaded whole (GeoJSON, KML/KMZ, GPX, and Shapefiles of up to 100,000 features), so a project of many such layers grows large. How large a project can be saved and opened follows the computer's memory: on the desktop, as much as a -quarter of it holds open (about 1 GiB of project for every 16 GB), from 512 MiB up to 4 GiB; in +quarter of it holds open (about 680 MiB of project for every 16 GB), from 512 MiB up to 4 GiB; in the web app, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened again, and Open refuses a larger file, saying its size and the limit. A project saved on a computer with more memory may need one like it to open. A project package (`.rgispkg`) stays within 1 GiB (256 MiB in the web app), as zip archives do here. `gp.env` reports the limit as -`projectBytes`. Data kept in a GeoPackage, FlatGeobuf or GeoParquet file is read in place and -stays out of the project file. +`projectBytes`. A GeoPackage, FlatGeobuf, GeoParquet or Shapefile of more than 100,000 features +(or any, added with `"stream": true`) is read in place and stays out of the project file; a +smaller one is loaded whole. **Open** also accepts a project package (`.rgispkg`), as do drag-and-drop and the desktop command line. A package opens as a new, unsaved project; Save writes a plain `.rgis` file. From ddb390076da01aa79019c4a69277fd153780cac3 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Sun, 8 Nov 2026 19:04:08 +0100 Subject: [PATCH 3/5] fix(engine): a 16 GB computer opens #1615's project; a 32-bit build's limit fits its buffers Review points on #1639: - The limit plans for 5 bytes of memory a byte of project (the most measured is 5.2): a computer sold as 16 GB reports a little under 16 GiB, and at 6 it refused the 659 MiB project of #1615. About 800 MiB for every 16 GB now. - On a 32-bit build the ceiling stays within what a buffer can grow to (half of isize::MAX), and a buffer that can't grow is an error, not an abort. - A refused file's size is rounded up, so one a byte past 512 MiB isn't "512 MiB". - The recovery copy's refusal is one translatable message with its limit in MiB. - A test proves serializing stops at the limit (a value after it is never visited); the item reader's comment says every kind of item follows the project limit. --- crates/engine/src/cmd/items.rs | 5 +-- crates/engine/src/project.rs | 56 ++++++++++++++++++++++++++-------- crates/ui-egui/src/autosave.rs | 4 +-- docs/user-guide/analysis.md | 2 +- docs/user-guide/workspace.md | 2 +- 5 files changed, 50 insertions(+), 19 deletions(-) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 09ed99516..7a191c767 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -15,8 +15,9 @@ use super::{b, f, need_feature_layer, need_layer, ps}; use crate::project::{StyleItem, Task, TaskStep}; use crate::{CmdResult, CommandSpec as C, Project, Session}; -/// Largest item file read: a project's ([`crate::project::max_bytes`]), as an item file may be -/// a whole project (Import Maps reads `.rgis` files). +/// Largest item file read: a project's ([`crate::project::max_bytes`]), for every kind of item +/// (maps, layers, styles, tasks, packages), as each was bounded like a project before and an +/// item file may be a whole project (Import Maps reads `.rgis` files). fn max_file() -> u64 { crate::project::max_bytes() } diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 61f9d137d..34e542f46 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -71,8 +71,10 @@ pub const BYTES_CEILING: u64 = 4 << 30; /// - points with 30 one-letter attributes, about the most a byte of file costs: about 5.2 /// (87 MiB took 453 MiB). /// -/// Planned at 6, above the most measured. -const MEMORY_PER_BYTE: u64 = 6; +/// Planned at 5: at the limit, even the most measured takes little more than a quarter of the +/// memory, and a project of the size in #1615 (659 MiB) opens on a computer sold as 16 GB, which +/// reports a little less than 16 GiB. +const MEMORY_PER_BYTE: u64 = 5; /// Largest project file read or saved on this computer (#1615): on the desktop as much as a /// quarter of its memory holds open at [`MEMORY_PER_BYTE`] a byte, from [`BYTES_FLOOR`] up to @@ -90,7 +92,13 @@ pub fn max_bytes() -> u64 { /// [`max_bytes`] on a computer with `memory` bytes (`None`: not known), as /// [`rustgis_data::memory::scaled`] works it out, in `u64` so it is the same on a 32-bit build. fn max_bytes_for(memory: Option) -> u64 { - memory.map_or(BYTES_FLOOR, |m| (m / rustgis_data::memory::SHARE / MEMORY_PER_BYTE).clamp(BYTES_FLOOR, BYTES_CEILING)) + memory.map_or(BYTES_FLOOR, |m| (m / rustgis_data::memory::SHARE / MEMORY_PER_BYTE).clamp(BYTES_FLOOR, ceiling())) +} + +/// [`BYTES_CEILING`], within what a buffer can hold here: on a 32-bit build a buffer holds at +/// most `isize::MAX` bytes and grows by doubling, so a project's text stops at half of that. +fn ceiling() -> u64 { + BYTES_CEILING.min(u64::try_from(isize::MAX).unwrap_or(u64::MAX) / 2).max(BYTES_FLOOR) } /// `value` as JSON text, or `None` as soon as it passes `max` bytes: the text of a project too @@ -107,6 +115,8 @@ pub(crate) fn json_within(value: &T, max: u64) -> Result< self.over = true; return Err(std::io::Error::other("past the limit")); } + // A machine short of memory gets an error, not an abort. + self.buf.try_reserve(b.len()).map_err(|_| std::io::Error::other("out of memory"))?; self.buf.extend_from_slice(b); Ok(b.len()) } @@ -1201,7 +1211,8 @@ impl Project { return Err(format!( "{} is {} MiB, more than a project can be (maximum {}){more}", path.display(), - meta.len() >> 20, + // Rounded up: a file a byte past 512 MiB isn't "512 MiB". + meta.len().div_ceil(1 << 20), limit_said(max) )); } @@ -1481,23 +1492,42 @@ mod limit_tests { const GIB: u64 = 1 << 30; - /// A quarter of the memory, at 6 bytes a byte of project: a twenty-fourth of it, from 512 MiB - /// to 4 GiB; 512 MiB where the memory isn't known (the browser). + /// A quarter of the memory, at 5 bytes a byte of project: a twentieth of it, from 512 MiB to + /// 4 GiB; 512 MiB where the memory isn't known (the browser). #[test] fn the_project_limit_follows_the_memory() { assert_eq!(max_bytes_for(None), BYTES_FLOOR); assert_eq!(max_bytes_for(Some(8 * GIB)), BYTES_FLOOR, "never below the floor"); - assert_eq!(max_bytes_for(Some(16 * GIB)), 16 * GIB / 24); - assert_eq!(max_bytes_for(Some(64 * GIB)), 64 * GIB / 24); - assert_eq!(max_bytes_for(Some(96 * GIB)), 4 * GIB); - assert_eq!(max_bytes_for(Some(1024 * GIB)), BYTES_CEILING, "never above the ceiling"); - assert_eq!(max_bytes_for(Some(u64::MAX)), BYTES_CEILING, "no overflow"); - // The 659 MiB project of #1615 opens on a computer with 16 GiB. - assert!(max_bytes_for(Some(16 * GIB)) > 659 << 20); + assert_eq!(max_bytes_for(Some(16 * GIB)), 16 * GIB / 20); + assert_eq!(max_bytes_for(Some(64 * GIB)), 64 * GIB / 20); + assert_eq!(max_bytes_for(Some(80 * GIB)), ceiling()); + assert_eq!(max_bytes_for(Some(1024 * GIB)), ceiling(), "never above the ceiling"); + assert_eq!(max_bytes_for(Some(u64::MAX)), ceiling(), "no overflow"); + // The 659 MiB project of #1615 opens on a computer sold as 16 GB, which reports less. + assert!(max_bytes_for(Some(15 * GIB)) > 659 << 20); + // A 32-bit build stops where its buffers can grow to. + assert!(ceiling() <= u64::try_from(isize::MAX).unwrap_or(u64::MAX) / 2 || ceiling() == BYTES_FLOOR); let max = max_bytes(); assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}"); } + /// Writing stops at the limit: what comes after it is never serialized. + #[test] + fn serializing_stops_at_the_limit() { + struct Probe<'a>(&'a std::cell::Cell); + impl Serialize for Probe<'_> { + fn serialize(&self, s: S) -> Result { + self.0.set(true); + s.serialize_unit() + } + } + let visited = std::cell::Cell::new(false); + let v = ("x".repeat(100), Probe(&visited)); + assert_eq!(json_within(&v, 50).ok(), Some(None)); + assert!(!visited.get(), "the value after the limit was serialized"); + assert!(json_within(&v, 1000).ok().flatten().is_some() && visited.get()); + } + /// The text stops at the limit: whole within it, none past it, never built past it. #[test] fn json_is_never_built_past_the_limit() { diff --git a/crates/ui-egui/src/autosave.rs b/crates/ui-egui/src/autosave.rs index ab6d312c9..31553ddc2 100644 --- a/crates/ui-egui/src/autosave.rs +++ b/crates/ui-egui/src/autosave.rs @@ -733,8 +733,8 @@ mod folder { .and_then(|f| f.take(max + 1).read_to_end(&mut bytes)) .map_err(|e| format!("{}: {e}", path.display()))?; if bytes.len() as u64 > max { - let limit = rustgis_engine::project::limit_said(max - MAX_HEADER); - return Err(crate::i18n::fmt("the recovery copy is too large (maximum {limit})", &[("limit", &limit)])); + let mib = ((max - MAX_HEADER) >> 20).to_string(); + return Err(crate::i18n::fmt("the recovery copy is too large (maximum {mib} MiB on this computer)", &[("mib", &mib)])); } // The description line goes in place: a copy near the size limit isn't held twice. let mut text = String::from_utf8(bytes).map_err(|_| "the recovery copy isn't text".to_string())?; diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index ddcb5be9d..b55a7d121 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -1207,7 +1207,7 @@ below the web app's fixed limits and never above a ceiling. | `densityCells`: cells of a Kernel Density raster | 16 million | 16 bytes a cell | 512 million | | `bufferBytes`: memory for Buffer's output and Select Layer By Location's buffered selectors | 512 MiB (128 MiB in the web app) | the bytes themselves | 8 GiB | | `threadScratchBytes`: working memory a tool's threads take together, beyond one thread's | 256 MiB | a sixteenth of memory, on top of the quarter | 4 GiB | -| `projectBytes`: the largest project file saved or opened (see [Projects](workspace.md#projects)) | 512 MiB | 6 bytes for each byte of the file (about 680 MiB for every 16 GB) | 4 GiB | +| `projectBytes`: the largest project file saved or opened (see [Projects](workspace.md#projects)) | 512 MiB | 5 bytes for each byte of the file (about 800 MiB for every 16 GB) | 4 GiB | So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index 4d62d5c6f..d702cb281 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -14,7 +14,7 @@ templates, packages and files for individual items. A project file holds the features of every layer loaded whole (GeoJSON, KML/KMZ, GPX, and Shapefiles of up to 100,000 features), so a project of many such layers grows large. How large a project can be saved and opened follows the computer's memory: on the desktop, as much as a -quarter of it holds open (about 680 MiB of project for every 16 GB), from 512 MiB up to 4 GiB; in +quarter of it holds open (about 800 MiB of project for every 16 GB), from 512 MiB up to 4 GiB; in the web app, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened again, and Open refuses a larger file, saying its size and the limit. A project saved on a computer with more memory may need one like it to open. A project package (`.rgispkg`) stays From be1a8b678ea91a295c9cc2eedaf3ed8512e11ee0 Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Sun, 8 Nov 2026 19:37:53 +0100 Subject: [PATCH 4/5] fix(engine): a 32-bit build keeps the 512 MiB project limit Review points on #1639: - An open project takes 2 to 5 times its file in address space, which a 32-bit build doesn't hold past about 512 MiB: such a build keeps the floor whatever the memory, as before this change, rather than about 1 GiB. - A file a 32-bit build refuses but a 64-bit one opens says so ("a 64-bit RustGIS on a computer with more memory opens it"); past 4 GiB, that no computer does. - The limit test expects each value within this build's ceiling. - The formula's comment says why it is worked out here rather than by memory::scaled (which counts in usize); the package's refusal closure no longer shadows too_large. - Docs: a 32-bit build keeps 512 MiB. --- crates/engine/src/cmd/items.rs | 4 ++-- crates/engine/src/project.rs | 33 ++++++++++++++++++++++----------- docs/user-guide/workspace.md | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 7a191c767..9ff956694 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -670,8 +670,8 @@ fn package_project(s: &mut Session) -> Result, String> { // The package as unzipped, the README with it, within what a zip and a project may hold. let limit = crate::project::max_bytes().min(rustgis_data::zip::MAX_TOTAL as u64); let room = limit.saturating_sub(readme.len() as u64); - let too_large = || format!("the project is too large to package (maximum {} MiB)", limit >> 20); - let (text, report) = crate::durable::output_owned(p, |p| crate::project::json_within(p, room)?.ok_or_else(too_large))?; + let package_too_large = || format!("the project is too large to package (maximum {} MiB)", limit >> 20); + let (text, report) = crate::durable::output_owned(p, |p| crate::project::json_within(p, room)?.ok_or_else(package_too_large))?; crate::durable::notify(s, &report); rustgis_data::zip::write(&[ (format!("{}.{}", safe(&name), crate::project::EXTENSION), text.into_bytes()), diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 34e542f46..203f693bf 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -89,16 +89,18 @@ pub fn max_bytes() -> u64 { max_bytes_for(rustgis_data::memory::physical()) } -/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known), as -/// [`rustgis_data::memory::scaled`] works it out, in `u64` so it is the same on a 32-bit build. +/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known). The formula is +/// [`rustgis_data::memory::scaled`]'s, worked out here in `u64`, as that one counts in `usize`, +/// which doesn't hold 4 GiB on a 32-bit build. fn max_bytes_for(memory: Option) -> u64 { memory.map_or(BYTES_FLOOR, |m| (m / rustgis_data::memory::SHARE / MEMORY_PER_BYTE).clamp(BYTES_FLOOR, ceiling())) } -/// [`BYTES_CEILING`], within what a buffer can hold here: on a 32-bit build a buffer holds at -/// most `isize::MAX` bytes and grows by doubling, so a project's text stops at half of that. +/// The most [`max_bytes`] grows to in this build: [`BYTES_CEILING`] on a 64-bit build. A 32-bit +/// build stays at [`BYTES_FLOOR`] whatever the memory: an open project takes 2 to 5 times its file +/// in address space, which 4 GiB doesn't hold for a larger one. fn ceiling() -> u64 { - BYTES_CEILING.min(u64::try_from(isize::MAX).unwrap_or(u64::MAX) / 2).max(BYTES_FLOOR) + if usize::BITS >= 64 { BYTES_CEILING } else { BYTES_FLOOR } } /// `value` as JSON text, or `None` as soon as it passes `max` bytes: the text of a project too @@ -1203,6 +1205,9 @@ impl Project { // A computer with more memory opens it, up to the ceiling; past that none does. let more = if meta.len() > BYTES_CEILING { format!(": no computer opens a project past {} MiB", BYTES_CEILING >> 20) + } else if meta.len() > ceiling() { + // Only a 32-bit build stops below the ceiling. + ": a 64-bit RustGIS on a computer with more memory opens it".to_string() } else if rustgis_data::memory::physical().is_some() { ": a computer with more memory opens it".to_string() } else { @@ -1498,15 +1503,21 @@ mod limit_tests { fn the_project_limit_follows_the_memory() { assert_eq!(max_bytes_for(None), BYTES_FLOOR); assert_eq!(max_bytes_for(Some(8 * GIB)), BYTES_FLOOR, "never below the floor"); - assert_eq!(max_bytes_for(Some(16 * GIB)), 16 * GIB / 20); - assert_eq!(max_bytes_for(Some(64 * GIB)), 64 * GIB / 20); + // Within this build's ceiling: a 32-bit build stays at the floor. + let within = |v: u64| v.clamp(BYTES_FLOOR, ceiling()); + assert_eq!(max_bytes_for(Some(16 * GIB)), within(16 * GIB / 20)); + assert_eq!(max_bytes_for(Some(64 * GIB)), within(64 * GIB / 20)); assert_eq!(max_bytes_for(Some(80 * GIB)), ceiling()); assert_eq!(max_bytes_for(Some(1024 * GIB)), ceiling(), "never above the ceiling"); assert_eq!(max_bytes_for(Some(u64::MAX)), ceiling(), "no overflow"); - // The 659 MiB project of #1615 opens on a computer sold as 16 GB, which reports less. - assert!(max_bytes_for(Some(15 * GIB)) > 659 << 20); - // A 32-bit build stops where its buffers can grow to. - assert!(ceiling() <= u64::try_from(isize::MAX).unwrap_or(u64::MAX) / 2 || ceiling() == BYTES_FLOOR); + // The 659 MiB project of #1615 opens on a computer sold as 16 GB, which reports less (in a + // 64-bit build, as the app is). + if usize::BITS >= 64 { + assert!(max_bytes_for(Some(15 * GIB)) > 659 << 20); + assert_eq!(ceiling(), BYTES_CEILING); + } else { + assert_eq!(ceiling(), BYTES_FLOOR); + } let max = max_bytes(); assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}"); } diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index d702cb281..5480f63a2 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -15,7 +15,7 @@ A project file holds the features of every layer loaded whole (GeoJSON, KML/KMZ, Shapefiles of up to 100,000 features), so a project of many such layers grows large. How large a project can be saved and opened follows the computer's memory: on the desktop, as much as a quarter of it holds open (about 800 MiB of project for every 16 GB), from 512 MiB up to 4 GiB; in -the web app, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened +the web app and a 32-bit build, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened again, and Open refuses a larger file, saying its size and the limit. A project saved on a computer with more memory may need one like it to open. A project package (`.rgispkg`) stays within 1 GiB (256 MiB in the web app), as zip archives do here. `gp.env` reports the limit as From 4969545adc03f9a204513a54f3bc40cc8f16792c Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Sun, 8 Nov 2026 20:12:26 +0100 Subject: [PATCH 5/5] fix(engine): a refused project names the memory that opens it Review points on #1639: - A file past the limit is told how much memory a computer needs to open it ("a computer with at least 13 GiB of memory opens it" for 659 MiB), not just "more". - On a 32-bit build, whose limit stays at 512 MiB, refusals no longer say the limit follows the memory. - The recovery copy is refused as one, not with the save message: its size check moved into the writer, where it can fire. - Docs: a package's 1 GiB (256 MiB in the web app) is what it holds unzipped. --- crates/engine/src/project.rs | 32 ++++++++++++++++++++++++-------- docs/user-guide/workspace.md | 8 +++++--- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 203f693bf..100be3ce2 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -103,6 +103,12 @@ fn ceiling() -> u64 { if usize::BITS >= 64 { BYTES_CEILING } else { BYTES_FLOOR } } +/// The memory a 64-bit build needs for [`max_bytes`] to reach `bytes`: the inverse of +/// [`max_bytes_for`] above the floor. +fn memory_for(bytes: u64) -> u64 { + bytes.saturating_mul(rustgis_data::memory::SHARE).saturating_mul(MEMORY_PER_BYTE) +} + /// `value` as JSON text, or `None` as soon as it passes `max` bytes: the text of a project too /// large to keep is never built whole, only up to the limit. pub(crate) fn json_within(value: &T, max: u64) -> Result, String> { @@ -137,7 +143,8 @@ pub(crate) fn json_within(value: &T, max: u64) -> Result< /// The limit as a refusal says it: in MiB and, where it follows the computer's memory, that it /// does, so a project refused here but opened elsewhere is understood. pub fn limit_said(max: u64) -> String { - if rustgis_data::memory::physical().is_some() { + // A 32-bit build keeps the floor whatever the memory: its limit doesn't follow it. + if rustgis_data::memory::physical().is_some() && ceiling() > BYTES_FLOOR { format!("{} MiB on this computer, a limit that follows its memory", max >> 20) } else { format!("{} MiB", max >> 20) @@ -1126,12 +1133,12 @@ impl Project { pub fn recovery_text_with_report(mut self) -> Result<(String, crate::durable::Report), String> { self.omit_pending_imports(); let unrecorded = self.bound_address_losses(false); - let (text, mut report) = crate::durable::output_owned(self, project_text)?; + // Refused as a recovery copy, not as a save, and never built past the limit. + let (text, mut report) = crate::durable::output_owned(self, |p| { + let max = max_bytes(); + json_within(p, max)?.ok_or_else(|| format!("the project is too large to keep a recovery copy (maximum {})", limit_said(max))) + })?; report.addresses = report.addresses.saturating_add(u32::try_from(unrecorded).unwrap_or(u32::MAX)); - let max = max_bytes(); - if text.len() as u64 > max { - return Err(format!("the project is too large to keep a recovery copy (maximum {})", limit_said(max))); - } Ok((text, report)) } @@ -1203,13 +1210,15 @@ impl Project { && meta.len() > max { // A computer with more memory opens it, up to the ceiling; past that none does. + // The memory a 64-bit build needs to open it (whole GiB, rounded up). + let needed = memory_for(meta.len()).div_ceil(1 << 30); let more = if meta.len() > BYTES_CEILING { format!(": no computer opens a project past {} MiB", BYTES_CEILING >> 20) } else if meta.len() > ceiling() { // Only a 32-bit build stops below the ceiling. - ": a 64-bit RustGIS on a computer with more memory opens it".to_string() + format!(": a 64-bit RustGIS on a computer with at least {needed} GiB of memory opens it") } else if rustgis_data::memory::physical().is_some() { - ": a computer with more memory opens it".to_string() + format!(": a computer with at least {needed} GiB of memory opens it") } else { String::new() }; @@ -1520,6 +1529,13 @@ mod limit_tests { } let max = max_bytes(); assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}"); + // The memory named for a refused file is enough to open it, and not much more. + let size: u64 = 659 << 20; + let needed = memory_for(size).div_ceil(GIB) * GIB; + if usize::BITS >= 64 { + assert!(max_bytes_for(Some(needed)) >= size && max_bytes_for(Some(needed - GIB)) < size); + } + assert_eq!(needed / GIB, 13); } /// Writing stops at the limit: what comes after it is never serialized. diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index 5480f63a2..831ea6e88 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -16,9 +16,11 @@ Shapefiles of up to 100,000 features), so a project of many such layers grows la project can be saved and opened follows the computer's memory: on the desktop, as much as a quarter of it holds open (about 800 MiB of project for every 16 GB), from 512 MiB up to 4 GiB; in the web app and a 32-bit build, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened -again, and Open refuses a larger file, saying its size and the limit. A project saved on a -computer with more memory may need one like it to open. A project package (`.rgispkg`) stays -within 1 GiB (256 MiB in the web app), as zip archives do here. `gp.env` reports the limit as +again, and Open refuses a larger file, saying its size, the limit, and how much memory a computer +needs to open it. A project saved on a +computer with more memory may need one like it to open. A project package (`.rgispkg`) holds +at most 1 GiB once unzipped (256 MiB in the web app), its project and its files together, as zip +archives do here; the package file itself is smaller. `gp.env` reports the limit as `projectBytes`. A GeoPackage, FlatGeobuf, GeoParquet or Shapefile of more than 100,000 features (or any, added with `"stream": true`) is read in place and stays out of the project file; a smaller one is loaded whole.