diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 7bd3eb274..9ff956694 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -15,8 +15,17 @@ use super::{b, f, need_feature_layer, need_layer, ps}; use crate::project::{StyleItem, Task, TaskStep}; use crate::{CmdResult, CommandSpec as C, Project, Session}; -/// Largest item file read (512 MiB, like projects). -const MAX_FILE: u64 = 512 << 20; +/// Largest item file read: a project's ([`crate::project::max_bytes`]), for every kind of item +/// (maps, layers, styles, tasks, packages), as each was bounded like a project before and an +/// item file may be a whole project (Import Maps reads `.rgis` files). +fn max_file() -> u64 { + crate::project::max_bytes() +} + +/// What an item past [`max_file`] is told. +fn too_large() -> String { + format!("the file is too large (maximum {})", crate::project::limit_said(max_file())) +} pub fn commands() -> Vec { vec![ @@ -368,10 +377,10 @@ pub(crate) fn read_text(p: &Value) -> Result { return Ok(text); } match (ps(p, "text"), ps(p, "path")) { - (Some(t), _) if t.len() as u64 <= MAX_FILE => Ok(t.to_string()), - (Some(_), _) => Err("the file is too large".into()), + (Some(t), _) if t.len() as u64 <= max_file() => Ok(t.to_string()), + (Some(_), _) => Err(too_large()), (None, Some(path)) => { - let bytes = crate::project::read_file(Path::new(path), MAX_FILE)?; + let bytes = crate::project::read_file(Path::new(path), max_file())?; // Packages are zip archives holding the project. if bytes.starts_with(b"PK") { return package_text(&bytes); @@ -657,13 +666,13 @@ fn package_project(s: &mut Session) -> Result, String> { *m = embedded_map(m, &mut budget); } let name = p.name.clone(); - let (text, report) = crate::durable::output_owned(p, |p| serde_json::to_string(p).map_err(|e| e.to_string()))?; - crate::durable::notify(s, &report); - let limit = crate::project::MAX_BYTES.min(rustgis_data::zip::MAX_TOTAL as u64); - if text.len() as u64 > limit { - return Err(format!("the project is too large to package (maximum {} MiB)", limit >> 20)); - } let readme = format!("{} — a RustGIS project package.\nOpen it with RustGIS (Project › Open), or unzip it for {}.rgis.\n", name, name); + // The package as unzipped, the README with it, within what a zip and a project may hold. + let limit = crate::project::max_bytes().min(rustgis_data::zip::MAX_TOTAL as u64); + let room = limit.saturating_sub(readme.len() as u64); + let package_too_large = || format!("the project is too large to package (maximum {} MiB)", limit >> 20); + let (text, report) = crate::durable::output_owned(p, |p| crate::project::json_within(p, room)?.ok_or_else(package_too_large))?; + crate::durable::notify(s, &report); rustgis_data::zip::write(&[ (format!("{}.{}", safe(&name), crate::project::EXTENSION), text.into_bytes()), ("README.txt".into(), readme.into_bytes()), @@ -1617,8 +1626,8 @@ fn durable_bytes(s: &mut Session, pub(crate) fn inline_text(p: &Value) -> Result, String> { if let Some(d) = ps(p, "data") { use base64::Engine; - if d.len() as u64 > MAX_FILE / 3 * 4 { - return Err("the file is too large".into()); + if d.len() as u64 > max_file() / 3 * 4 { + return Err(too_large()); } let bytes = base64::engine::general_purpose::STANDARD.decode(d.trim()).map_err(|_| "data must be base64".to_string())?; if bytes.starts_with(b"PK") { @@ -1627,8 +1636,8 @@ pub(crate) fn inline_text(p: &Value) -> Result, String> { return String::from_utf8(bytes).map(Some).map_err(|_| "not a RustGIS file".to_string()); } match ps(p, "text") { - Some(t) if t.len() as u64 <= MAX_FILE => Ok(Some(t.to_string())), - Some(_) => Err("the file is too large".into()), + Some(t) if t.len() as u64 <= max_file() => Ok(Some(t.to_string())), + Some(_) => Err(too_large()), None => Ok(None), } } diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index 134a54393..b537f8a29 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -386,6 +386,7 @@ fn environments(s: &mut Session, p: &Value) -> CmdResult { "densityCells": rustgis_analysis::max_density_cells(), "bufferBytes": rustgis_analysis::max_buffer_bytes(), "threadScratchBytes": rustgis_analysis::scratch_budget(), + "projectBytes": crate::project::max_bytes(), }); Ok(json!({"extent": env.extent.map(|e| [e.min_x, e.min_y, e.max_x, e.max_y]), "cellSize": env.cell_size, "limits": limits})) } diff --git a/crates/engine/src/peer.rs b/crates/engine/src/peer.rs index c26f6199a..f075e32e2 100644 --- a/crates/engine/src/peer.rs +++ b/crates/engine/src/peer.rs @@ -477,13 +477,13 @@ impl Session { } } -/// The same opening bound applies to peers; no destination is written on a formatting failure. +/// A peer may be the web app, which opens at most [`BYTES_FLOOR`](crate::project::BYTES_FLOOR) +/// whatever this computer saves: the text is never built past that. No destination is written on +/// a formatting failure. fn text(p: &Project) -> Result { - let text = serde_json::to_string(p).map_err(|e| e.to_string())?; - if text.len() as u64 > crate::project::MAX_BYTES { - return Err("the project is too large to exchange".into()); - } - Ok(text) + let max = crate::project::BYTES_FLOOR; + crate::project::json_within(p, max)? + .ok_or_else(|| format!("the project is too large to exchange (maximum {} MiB, what the web app opens)", max >> 20)) } #[cfg(test)] diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 8c7a4c5da..100be3ce2 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -56,8 +56,122 @@ pub fn open_command(path: &str) -> &'static str { /// Saved-document compatibility version, independent of the RustGIS release version. /// Advance it when an older reader would lose meaningful saved information on resaving. pub const FORMAT: u32 = 1; -/// Largest project file read or saved (512 MiB). -pub const MAX_BYTES: u64 = 512 << 20; +/// Largest project file read or saved in the browser, or on a computer whose memory isn't known +/// (512 MiB): the floor of [`max_bytes`]. +pub const BYTES_FLOOR: u64 = 512 << 20; + +/// The most [`max_bytes`] grows to, however much memory the computer has (4 GiB). +pub const BYTES_CEILING: u64 = 4 << 30; + +/// The memory an open project takes for each byte of its file, as measured opening projects of +/// GeoJSON layers from the command line: +/// - polygons of 40 vertices and 8 attributes: about 2 (337 MiB took 731 MiB; 674 MiB, 1.41 GiB; +/// the app took 576 MiB more than empty to open and draw the first); +/// - points with 12 short text attributes: about 4.4 (101 MiB took 449 MiB); +/// - points with 30 one-letter attributes, about the most a byte of file costs: about 5.2 +/// (87 MiB took 453 MiB). +/// +/// Planned at 5: at the limit, even the most measured takes little more than a quarter of the +/// memory, and a project of the size in #1615 (659 MiB) opens on a computer sold as 16 GB, which +/// reports a little less than 16 GiB. +const MEMORY_PER_BYTE: u64 = 5; + +/// Largest project file read or saved on this computer (#1615): on the desktop as much as a +/// quarter of its memory holds open at [`MEMORY_PER_BYTE`] a byte, from [`BYTES_FLOOR`] up to +/// [`BYTES_CEILING`], as the other memory limits follow it (#1291 step 2); [`BYTES_FLOOR`] in the +/// browser. Save and open use the same limit, so a project saved here opens here; one saved on a +/// computer with more memory may need one like it. +pub fn max_bytes() -> u64 { + #[cfg(test)] + if let Some(max) = TEST_MAX.with(std::cell::Cell::get) { + return max; + } + max_bytes_for(rustgis_data::memory::physical()) +} + +/// [`max_bytes`] on a computer with `memory` bytes (`None`: not known). The formula is +/// [`rustgis_data::memory::scaled`]'s, worked out here in `u64`, as that one counts in `usize`, +/// which doesn't hold 4 GiB on a 32-bit build. +fn max_bytes_for(memory: Option) -> u64 { + memory.map_or(BYTES_FLOOR, |m| (m / rustgis_data::memory::SHARE / MEMORY_PER_BYTE).clamp(BYTES_FLOOR, ceiling())) +} + +/// The most [`max_bytes`] grows to in this build: [`BYTES_CEILING`] on a 64-bit build. A 32-bit +/// build stays at [`BYTES_FLOOR`] whatever the memory: an open project takes 2 to 5 times its file +/// in address space, which 4 GiB doesn't hold for a larger one. +fn ceiling() -> u64 { + if usize::BITS >= 64 { BYTES_CEILING } else { BYTES_FLOOR } +} + +/// The memory a 64-bit build needs for [`max_bytes`] to reach `bytes`: the inverse of +/// [`max_bytes_for`] above the floor. +fn memory_for(bytes: u64) -> u64 { + bytes.saturating_mul(rustgis_data::memory::SHARE).saturating_mul(MEMORY_PER_BYTE) +} + +/// `value` as JSON text, or `None` as soon as it passes `max` bytes: the text of a project too +/// large to keep is never built whole, only up to the limit. +pub(crate) fn json_within(value: &T, max: u64) -> Result, String> { + struct Capped { + buf: Vec, + max: usize, + over: bool, + } + impl std::io::Write for Capped { + fn write(&mut self, b: &[u8]) -> std::io::Result { + if self.buf.len().saturating_add(b.len()) > self.max { + self.over = true; + return Err(std::io::Error::other("past the limit")); + } + // A machine short of memory gets an error, not an abort. + self.buf.try_reserve(b.len()).map_err(|_| std::io::Error::other("out of memory"))?; + self.buf.extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut w = Capped { buf: Vec::new(), max: usize::try_from(max).unwrap_or(usize::MAX), over: false }; + match serde_json::to_writer(&mut w, value) { + Ok(()) => String::from_utf8(w.buf).map(Some).map_err(|e| e.to_string()), + Err(_) if w.over => Ok(None), + Err(e) => Err(e.to_string()), + } +} + +/// The limit as a refusal says it: in MiB and, where it follows the computer's memory, that it +/// does, so a project refused here but opened elsewhere is understood. +pub fn limit_said(max: u64) -> String { + // A 32-bit build keeps the floor whatever the memory: its limit doesn't follow it. + if rustgis_data::memory::physical().is_some() && ceiling() > BYTES_FLOOR { + format!("{} MiB on this computer, a limit that follows its memory", max >> 20) + } else { + format!("{} MiB", max >> 20) + } +} + +#[cfg(test)] +thread_local! { + /// A smaller [`max_bytes`] for the tests on this thread ([`with_max_bytes`]), so a test of + /// the limit doesn't build a project of gigabytes. + static TEST_MAX: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// Runs `f` with [`max_bytes`] at `max` on the calling thread only (work `f` hands to another +/// thread sees the real limit); the limit is restored even if `f` panics. +#[cfg(test)] +pub(crate) fn with_max_bytes(max: u64, f: impl FnOnce() -> T) -> T { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + TEST_MAX.with(|m| m.set(None)); + } + } + TEST_MAX.with(|m| m.set(Some(max))); + let _restore = Restore; + f() +} /// Runtime capabilities never survive a project boundary, including an in-process save restore. /// `None` for an address that holds none, and for a wrapper that doesn't parse (kept as it is, @@ -544,8 +658,9 @@ struct FileFormat { /// metadata is legacy content; arrays include legacy style files. Writers remain unchanged /// until a change needs a newer format, at which point it must declare that format. pub(crate) fn check_format(text: &str, kind: &str) -> Result<(), String> { - if text.len() as u64 > MAX_BYTES { - return Err(format!("the {kind} file is too large")); + let max = max_bytes(); + if text.len() as u64 > max { + return Err(format!("the {kind} file is too large (maximum {})", limit_said(max))); } if text.trim_start().starts_with('[') { return Ok(()); @@ -1018,11 +1133,12 @@ impl Project { pub fn recovery_text_with_report(mut self) -> Result<(String, crate::durable::Report), String> { self.omit_pending_imports(); let unrecorded = self.bound_address_losses(false); - let (text, mut report) = crate::durable::output_owned(self, project_text)?; + // Refused as a recovery copy, not as a save, and never built past the limit. + let (text, mut report) = crate::durable::output_owned(self, |p| { + let max = max_bytes(); + json_within(p, max)?.ok_or_else(|| format!("the project is too large to keep a recovery copy (maximum {})", limit_said(max))) + })?; report.addresses = report.addresses.saturating_add(u32::try_from(unrecorded).unwrap_or(u32::MAX)); - if text.len() as u64 > MAX_BYTES { - return Err("the project is too large to keep a recovery copy".into()); - } Ok((text, report)) } @@ -1084,10 +1200,37 @@ impl Project { Ok(p) } - /// Opens a project file (a regular file within [`MAX_BYTES`]; see `read_file`), resolving + /// Opens a project file (a regular file within [`max_bytes`]; see `read_file`), resolving /// its relative paths against its folder. pub fn load(path: &Path) -> Result { - let bytes = read_file(path, MAX_BYTES)?; + let max = max_bytes(); + // Past the limit: said with its size and the limit here, before anything is read. + if let Ok(meta) = std::fs::metadata(path) + && meta.is_file() + && meta.len() > max + { + // A computer with more memory opens it, up to the ceiling; past that none does. + // The memory a 64-bit build needs to open it (whole GiB, rounded up). + let needed = memory_for(meta.len()).div_ceil(1 << 30); + let more = if meta.len() > BYTES_CEILING { + format!(": no computer opens a project past {} MiB", BYTES_CEILING >> 20) + } else if meta.len() > ceiling() { + // Only a 32-bit build stops below the ceiling. + format!(": a 64-bit RustGIS on a computer with at least {needed} GiB of memory opens it") + } else if rustgis_data::memory::physical().is_some() { + format!(": a computer with at least {needed} GiB of memory opens it") + } else { + String::new() + }; + return Err(format!( + "{} is {} MiB, more than a project can be (maximum {}){more}", + path.display(), + // Rounded up: a file a byte past 512 MiB isn't "512 MiB". + meta.len().div_ceil(1 << 20), + limit_said(max) + )); + } + let bytes = read_file(path, max)?; let text = String::from_utf8(bytes).map_err(|_| format!("{} isn't a RustGIS project", path.display()))?; let dir = std::path::absolute(path).ok().and_then(|p| p.parent().map(Path::to_path_buf)); Self::read_json(&text, true, dir.as_deref(), true) @@ -1353,11 +1496,73 @@ impl Durable for ModelGraph { /// Enforces the opening bound inside the formatter, before any destination is replaced. fn project_text(p: &Project) -> Result { - let text = serde_json::to_string(p).map_err(|e| e.to_string())?; - if text.len() as u64 > MAX_BYTES { - return Err(format!("the project is too large to save (maximum {} MiB)", MAX_BYTES >> 20)); + let max = max_bytes(); + json_within(p, max)?.ok_or_else(|| format!("the project is too large to save (maximum {})", limit_said(max))) +} + +#[cfg(test)] +mod limit_tests { + use super::*; + + const GIB: u64 = 1 << 30; + + /// A quarter of the memory, at 5 bytes a byte of project: a twentieth of it, from 512 MiB to + /// 4 GiB; 512 MiB where the memory isn't known (the browser). + #[test] + fn the_project_limit_follows_the_memory() { + assert_eq!(max_bytes_for(None), BYTES_FLOOR); + assert_eq!(max_bytes_for(Some(8 * GIB)), BYTES_FLOOR, "never below the floor"); + // Within this build's ceiling: a 32-bit build stays at the floor. + let within = |v: u64| v.clamp(BYTES_FLOOR, ceiling()); + assert_eq!(max_bytes_for(Some(16 * GIB)), within(16 * GIB / 20)); + assert_eq!(max_bytes_for(Some(64 * GIB)), within(64 * GIB / 20)); + assert_eq!(max_bytes_for(Some(80 * GIB)), ceiling()); + assert_eq!(max_bytes_for(Some(1024 * GIB)), ceiling(), "never above the ceiling"); + assert_eq!(max_bytes_for(Some(u64::MAX)), ceiling(), "no overflow"); + // The 659 MiB project of #1615 opens on a computer sold as 16 GB, which reports less (in a + // 64-bit build, as the app is). + if usize::BITS >= 64 { + assert!(max_bytes_for(Some(15 * GIB)) > 659 << 20); + assert_eq!(ceiling(), BYTES_CEILING); + } else { + assert_eq!(ceiling(), BYTES_FLOOR); + } + let max = max_bytes(); + assert!((BYTES_FLOOR..=BYTES_CEILING).contains(&max), "{max}"); + // The memory named for a refused file is enough to open it, and not much more. + let size: u64 = 659 << 20; + let needed = memory_for(size).div_ceil(GIB) * GIB; + if usize::BITS >= 64 { + assert!(max_bytes_for(Some(needed)) >= size && max_bytes_for(Some(needed - GIB)) < size); + } + assert_eq!(needed / GIB, 13); + } + + /// Writing stops at the limit: what comes after it is never serialized. + #[test] + fn serializing_stops_at_the_limit() { + struct Probe<'a>(&'a std::cell::Cell); + impl Serialize for Probe<'_> { + fn serialize(&self, s: S) -> Result { + self.0.set(true); + s.serialize_unit() + } + } + let visited = std::cell::Cell::new(false); + let v = ("x".repeat(100), Probe(&visited)); + assert_eq!(json_within(&v, 50).ok(), Some(None)); + assert!(!visited.get(), "the value after the limit was serialized"); + assert!(json_within(&v, 1000).ok().flatten().is_some() && visited.get()); + } + + /// The text stops at the limit: whole within it, none past it, never built past it. + #[test] + fn json_is_never_built_past_the_limit() { + let v = serde_json::json!({"a": "x".repeat(100)}); + let whole = serde_json::to_string(&v).unwrap_or_default(); + assert_eq!(json_within(&v, whole.len() as u64).ok().flatten(), Some(whole.clone())); + assert_eq!(json_within(&v, whole.len() as u64 - 1).ok(), Some(None)); } - Ok(text) } #[cfg(test)] diff --git a/crates/engine/src/roles.rs b/crates/engine/src/roles.rs index cc3897628..62173f88a 100644 --- a/crates/engine/src/roles.rs +++ b/crates/engine/src/roles.rs @@ -376,7 +376,7 @@ impl Durable for VecSteps { /// Reads documents at the existing reader bound, without normalization, trust or restore-cache /// access. Unchanged documents keep their exact text; only changed typed copies are serialized. fn document(kind: Document, v: &Value, depth: usize, report: &mut Report) -> Option { - let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::MAX_BYTES) else { return invalid(v, report) }; + let Some(text) = v.as_str().filter(|s| s.len() as u64 <= crate::project::max_bytes()) else { return invalid(v, report) }; if serde_json::from_str::(text).is_ok_and(|f| f.format > u64::from(crate::project::FORMAT)) { report.unchecked = report.unchecked.saturating_add(1); return None; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 82bbe3928..f7f1ac08a 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -2113,12 +2113,15 @@ fn oversized_project_save_preserves_file_and_live_state() { s.run("project.rename", &json!({"name": "Unsaved work"})).unwrap(); let source = dir.join("roads.geojson").display().to_string(); s.project.maps[0].layers[0].dataset.source = Some(source.clone()); - // A single large value reaches the real limit without a large data fixture on disk. - s.project.thumbnail = Some("x".repeat(crate::project::MAX_BYTES as usize)); - let error = s.run("project.saveAs", &json!({"path": path})).unwrap_err(); - assert!(error.contains("too large"), "{error}"); - let error = s.run("project.package", &json!({"path": package})).unwrap_err(); - assert!(error.contains("too large"), "{error}"); + // A single large value reaches the limit without a large data fixture on disk; the limit is + // held at 1 MiB here, as it follows this computer's memory (#1615). + let max = 1 << 20; + s.project.thumbnail = Some("x".repeat(max as usize)); + let (error, package_error) = crate::project::with_max_bytes(max, || { + (s.run("project.saveAs", &json!({"path": path})).unwrap_err(), s.run("project.package", &json!({"path": package})).unwrap_err()) + }); + assert!(error.contains("too large to save (maximum 1 MiB"), "{error}"); + assert!(package_error.contains("too large"), "{package_error}"); assert_eq!(std::fs::read(&path).unwrap(), original); assert_eq!(std::fs::read(&package).unwrap(), original_package); assert_eq!(s.project.name, "Unsaved work"); @@ -2132,6 +2135,43 @@ fn oversized_project_save_preserves_file_and_live_state() { std::fs::remove_dir_all(&dir).unwrap(); } +/// The project limit follows the computer's memory (#1615): a project as large as the limit saves +/// and opens again on the same computer, and one past it is refused before it is read, saying its +/// size and the limit. +#[test] +fn a_project_within_the_limit_saves_and_opens_and_one_past_it_says_both_sizes() { + let dir = std::env::temp_dir().join(format!("rustgis-project-limit-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("near.rgis"); + let max: u64 = 2 << 20; + let mut s = world(); + let small = { + s.run("project.saveAs", &json!({"path": path})).unwrap(); + std::fs::metadata(&path).unwrap().len() + }; + // Just under the limit: written and read back whole, the same limit both ways. + s.project.thumbnail = Some("x".repeat((max - small - 1024) as usize)); + crate::project::with_max_bytes(max, || { + s.run("project.saveAs", &json!({"path": path})).unwrap(); + assert!(std::fs::metadata(&path).unwrap().len() <= max); + assert_eq!(Project::load(&path).unwrap().thumbnail.map(|t| t.len()), s.project.thumbnail.as_ref().map(|t| t.len())); + }); + // Past it (as a project saved on a computer with more memory): refused, saying both sizes. + let big = dir.join("big.rgis"); + std::fs::write(&big, vec![b' '; (3 << 20) as usize]).unwrap(); + let e = crate::project::with_max_bytes(max, || Project::load(&big).unwrap_err()); + assert!(e.contains("is 3 MiB, more than a project can be (maximum 2 MiB"), "{e}"); + let e = crate::project::with_max_bytes(max, || crate::project::check_format(&" ".repeat(3 << 20), "project").unwrap_err()); + assert!(e.contains("too large (maximum 2 MiB"), "{e}"); + // Import Maps reads a project file under the same limit: the one saved above, not the one past it. + crate::project::with_max_bytes(max, || { + assert!(crate::cmd::items::read_text(&json!({"path": path})).is_ok()); + let e = crate::cmd::items::read_text(&json!({"path": big})).unwrap_err(); + assert!(e.contains("too large"), "{e}"); + }); + std::fs::remove_dir_all(&dir).ok(); +} + #[test] fn project_extension_is_rgis_and_legacy_rgisx_opens() { let mut s = world(); @@ -8266,6 +8306,7 @@ fn analysis_workflows_models_scripts_and_raster_functions() { assert!(limit("pagedFeatures") >= rustgis_data::ogc::PAGED_FEATURES_FLOOR, "{env}"); assert!(limit("densityCells") >= rustgis_analysis::DENSITY_CELLS_FLOOR, "{env}"); assert!(limit("bufferBytes") >= 512 << 20 && limit("threadScratchBytes") >= 256 << 20, "{env}"); + assert!(limit("projectBytes") as u64 == crate::project::max_bytes() && limit("projectBytes") >= 512 << 20, "{env}"); // Network route along the rivers: two points on the same river. let rivers = s.map().layer_by_name("Rivers").unwrap(); let (a, b2) = match rivers.dataset.features.iter().filter_map(|f| f.geometry.as_ref()).find_map(|g| match g { diff --git a/crates/ui-egui/src/autosave.rs b/crates/ui-egui/src/autosave.rs index b56ab561d..31553ddc2 100644 --- a/crates/ui-egui/src/autosave.rs +++ b/crates/ui-egui/src/autosave.rs @@ -728,11 +728,13 @@ mod folder { fn read_pending(&self) -> Result { let path = self.list().first().map(|(p, _)| p.clone()).ok_or("there is no unsaved work to recover")?; let mut bytes = Vec::new(); + let max = rustgis_engine::project::max_bytes() + MAX_HEADER; std::fs::File::open(&path) - .and_then(|f| f.take(rustgis_engine::project::MAX_BYTES + MAX_HEADER + 1).read_to_end(&mut bytes)) + .and_then(|f| f.take(max + 1).read_to_end(&mut bytes)) .map_err(|e| format!("{}: {e}", path.display()))?; - if bytes.len() as u64 > rustgis_engine::project::MAX_BYTES + MAX_HEADER { - return Err("the recovery copy is too large".into()); + if bytes.len() as u64 > max { + let mib = ((max - MAX_HEADER) >> 20).to_string(); + return Err(crate::i18n::fmt("the recovery copy is too large (maximum {mib} MiB on this computer)", &[("mib", &mib)])); } // The description line goes in place: a copy near the size limit isn't held twice. let mut text = String::from_utf8(bytes).map_err(|_| "the recovery copy isn't text".to_string())?; diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index ec6623878..b55a7d121 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -1207,6 +1207,7 @@ below the web app's fixed limits and never above a ceiling. | `densityCells`: cells of a Kernel Density raster | 16 million | 16 bytes a cell | 512 million | | `bufferBytes`: memory for Buffer's output and Select Layer By Location's buffered selectors | 512 MiB (128 MiB in the web app) | the bytes themselves | 8 GiB | | `threadScratchBytes`: working memory a tool's threads take together, beyond one thread's | 256 MiB | a sixteenth of memory, on top of the quarter | 4 GiB | +| `projectBytes`: the largest project file saved or opened (see [Projects](workspace.md#projects)) | 512 MiB | 5 bytes for each byte of the file (about 800 MiB for every 16 GB) | 4 GiB | So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster diff --git a/docs/user-guide/workspace.md b/docs/user-guide/workspace.md index c3aedf98d..831ea6e88 100644 --- a/docs/user-guide/workspace.md +++ b/docs/user-guide/workspace.md @@ -11,6 +11,20 @@ streamed layers reopen their sources, and file-backed rasters reopen their files sources accessible when moving a project. See [Project items](data.md#project-items) for templates, packages and files for individual items. +A project file holds the features of every layer loaded whole (GeoJSON, KML/KMZ, GPX, and +Shapefiles of up to 100,000 features), so a project of many such layers grows large. How large a +project can be saved and opened follows the computer's memory: on the desktop, as much as a +quarter of it holds open (about 800 MiB of project for every 16 GB), from 512 MiB up to 4 GiB; in +the web app and a 32-bit build, 512 MiB. Save refuses a project past it, rather than writing one that can't be opened +again, and Open refuses a larger file, saying its size, the limit, and how much memory a computer +needs to open it. A project saved on a +computer with more memory may need one like it to open. A project package (`.rgispkg`) holds +at most 1 GiB once unzipped (256 MiB in the web app), its project and its files together, as zip +archives do here; the package file itself is smaller. `gp.env` reports the limit as +`projectBytes`. A GeoPackage, FlatGeobuf, GeoParquet or Shapefile of more than 100,000 features +(or any, added with `"stream": true`) is read in place and stays out of the project file; a +smaller one is loaded whole. + **Open** also accepts a project package (`.rgispkg`), as do drag-and-drop and the desktop command line. A package opens as a new, unsaved project; Save writes a plain `.rgis` file. Opening or creating a project clears undo history; saving does not.