From 2499cc2c0115a4b88745177fc0dd7c86a4737432 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 09:15:13 -0300 Subject: [PATCH 1/7] fix(data, engine): share a document's address, and work out a WMS root's systems once Two findings from #1526's last review round, reproduced on its head: - Each layer's CapabilitySource held its own copy of the capabilities address, which nothing bounds: 100,000 collections with a 100 KB address peaked at 9.7 GB. It is now one Arc per document. - A WMS layer naming systems of its own re-parsed its parent's whole list: a root of 1000 systems and 100,000 such layers took 97 s. A layer's systems now keep only what wms_native uses (Web Mercator among them, and the first 64 usable), and a sublayer adds its own to its parent's first ones. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/ogc.rs | 91 +++++++++++++++++++++++++------- crates/engine/src/cmd/service.rs | 13 +++-- 2 files changed, 81 insertions(+), 23 deletions(-) diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index 4a8d098e2..d3053bdb7 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -1006,11 +1006,7 @@ fn wms(url: &str, root: &Node) -> Service { // (A layer naming none of its own shares its parent's list, and what is worked out of it.) let own: Vec = layer.children().filter(|c| c.is_element() && matches!(local(c), "CRS" | "SRS")).map(|c| text_of(&c)).collect(); - let crs: Rc = if own.is_empty() { - inherited_crs - } else { - Rc::new(Systems { names: inherited_crs.names.iter().cloned().chain(own).collect(), ..Systems::default() }) - }; + let crs: Rc = if own.is_empty() { inherited_crs } else { Rc::new(Systems::with_own(&inherited_crs, &own)) }; let bbox = child(&layer, "EX_GeographicBoundingBox") .and_then(|b| { let f = |k: &str| child_text(&b, k).and_then(|t| t.parse::().ok()); @@ -1040,9 +1036,8 @@ fn wms(url: &str, root: &Node) -> Service { let Some(name) = child_text(&layer, "Name") else { continue }; // In Web Mercator, as the service draws it, else in a system of its own, warped (#769); // with the other systems it is offered in, for a map drawn in one of them. - let mercator = crs.names.iter().any(|c| c.split_whitespace().any(is_web_mercator)); - let listed = crs.candidates.get_or_init(|| wms_candidates(&crs.names)); - let mut native = wms_native(&layer, listed, v13, bbox, mercator, &mut layer_boxes); + let mercator = crs.mercator; + let mut native = wms_native(&layer, &crs.first, v13, bbox, mercator, &mut layer_boxes); if !mercator && native.is_none() { skipped += 1; continue; @@ -1146,12 +1141,50 @@ fn wms(url: &str, root: &Node) -> Service { service } -/// The coordinate systems a WMS layer is offered in, its own and those it inherits, and those of -/// them RustGIS knows (worked out once, for the layers sharing the list). +/// The coordinate systems a WMS layer is offered in, its own and those it inherits: whether Web +/// Mercator is among them, and the first [`warp::MAX_SYSTEMS`] RustGIS can draw the layer in, in +/// order of preference (see [`wms_candidates`]), the only ones [`wms_native`] takes. A layer +/// naming systems of its own adds them to its parent's first ones, so the many a service's root +/// may list (a GeoServer's thousands) are worked out once, not again for each layer (#1519). #[derive(Default)] struct Systems { - names: Vec, - candidates: std::cell::OnceCell>, + /// How many names the list has, its parents' included: where a sublayer's own come in it. + names: usize, + mercator: bool, + first: Vec, +} + +impl Systems { + fn with_own(parent: &Systems, own: &[String]) -> Systems { + // (The parent's are usable already, and whether one is depends only on its code.) + let mut all = parent.first.clone(); + for mut c in wms_candidates(own).into_iter().filter(|c| warp::usable(c.1).is_some()) { + c.3 += parent.names; + // One name for each system, as in `wms_candidates`: CRS:84 over EPSG:4326, else the + // first listed. (A system past the parent's first ones comes after them all here too.) + match all.iter_mut().find(|p| p.1 == c.1) { + Some(p) if warp::is_crs84(&c.0) && !warp::is_crs84(&p.0) => *p = c, + Some(_) => {} + None => all.push(c), + } + } + all.sort_by_key(|c| (candidate_rank(c), c.3)); + Systems { + names: parent.names + own.iter().map(|c| c.split_whitespace().count()).sum::(), + mercator: parent.mercator || own.iter().any(|c| c.split_whitespace().any(is_web_mercator)), + first: all.into_iter().take(warp::MAX_SYSTEMS).collect(), + } + } +} + +/// Where a system comes in a WMS layer's order of preference: projected, CRS:84, other +/// geographic. +fn candidate_rank((name, _, c, _): &Candidate) -> u8 { + match (c.is_geographic(), warp::is_crs84(name)) { + (false, _) => 0, + (true, true) => 1, + (true, false) => 2, + } } /// A system of a WMS layer's list that RustGIS knows: its name there, its EPSG code, it, and its @@ -1179,13 +1212,8 @@ fn wms_candidates(crs: &[String]) -> Vec { // One name for each system (EPSG:4326 and CRS:84 differ only in their axes: CRS:84 kept). listed.sort_by_cached_key(|(name, code, _, _)| (*code, !warp::is_crs84(name))); listed.dedup_by_key(|(_, code, _, _)| *code); - let rank = |(name, _, c, _): &Candidate| match (c.is_geographic(), warp::is_crs84(name)) { - (false, _) => 0, - (true, true) => 1, - (true, false) => 2, - }; // In the order the service lists them, the preferred first. - listed.sort_by_key(|l| (rank(l), l.3)); + listed.sort_by_key(|l| (candidate_rank(l), l.3)); listed } @@ -5803,6 +5831,33 @@ mod tests { } } + /// A sublayer's systems, its own added to its parent's first ones, are those its whole list + /// gives, so a root's many are worked out once (#1519). + #[test] + fn a_sublayers_systems_build_on_its_parents_first_ones() { + let first = |names: &[String]| -> Vec<(String, u32)> { + wms_candidates(names).into_iter().filter(|c| warp::usable(c.1).is_some()).take(warp::MAX_SYSTEMS).map(|c| (c.0, c.1)).collect() + }; + let codes = |r: std::ops::Range| r.map(|c| format!("EPSG:{c}")).collect::>(); + let many = [codes(32601..32671), vec!["EPSG:4326".into(), "EPSG:3857".into()], codes(2000..2100)].concat(); + let few = vec!["EPSG:4326".to_string(), "EPSG:3857".into()]; + let owns = [ + vec!["CRS:84".to_string()], + vec!["EPSG:32633".into(), "EPSG:2056".into()], + vec!["EPSG:4326 EPSG:31983".into()], + codes(32701..32761), + ]; + for root in [&many, &few] { + let parent = Systems::with_own(&Systems::default(), root); + for own in &owns { + let child = Systems::with_own(&parent, own); + let whole: Vec = root.iter().chain(own).cloned().collect(); + assert_eq!(child.first.iter().map(|c| (c.0.clone(), c.1)).collect::>(), first(&whole), "{own:?}"); + assert!(child.mercator && !Systems::with_own(&Systems::default(), own).mercator); + } + } + } + #[test] fn wms_layers_in_web_mercator() { let xml = r#" diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index 460a8d11d..d4725d2a9 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -89,9 +89,11 @@ pub(crate) fn without_token(p: &Value) -> Result, String> { Ok(Some(owned)) } -/// One layer's exact capabilities document and the origin history used to reach it. +/// One layer's exact capabilities document and the origin history used to reach it. The address +/// is shared by the document's layers: nothing bounds its length, and a document has up to +/// `MAX_READ_LAYERS` of them (#1519). struct CapabilitySource { - address: String, + address: std::sync::Arc, crossed_origin: bool, } @@ -216,9 +218,10 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result } // Parsers bound each document's collections/layers; visited documents bound the merge. // Distinct offers may share a name (including generated tile names and collection ids). + let shared: std::sync::Arc = address.as_str().into(); for layer in service.layers { result.service.layers.push(layer); - result.sources.push(CapabilitySource { address: address.clone(), crossed_origin }); + result.sources.push(CapabilitySource { address: shared.clone(), crossed_origin }); } result.service.withheld_addresses = result.service.withheld_addresses.saturating_add(service.withheld_addresses); result.service.past_read_limit = result.service.past_read_limit.take().or(service.past_read_limit); @@ -684,7 +687,7 @@ fn layer_source(layer: &mut Value, source: CapabilitySource) { offer.get("type").and_then(Value::as_str) == Some("features") && ps(offer, "url").is_some_and(|url| crosses_origin(&source.address, url)) }); - layer["source_url"] = json!(source.address); + layer["source_url"] = json!(&*source.address); if source.crossed_origin || foreign_features { layer["anonymous"] = json!(true); if let Some(url) = layer.get_mut("offer").and_then(|o| o.get_mut("url")) @@ -746,7 +749,7 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis } })?; let capability_crossed = source.crossed_origin; - let source = source.address.as_str(); + let source = &*source.address; // A time-enabled layer follows the time slider, at each of its steps, unless a time is asked // for: one of its times (the layer named with it), or "default" (`false`), the service's. // (`"time": true`, the slider, as for a Zarr variable, `layer.addZarr`'s.) From 81547c9f0fbd44046344fa6c53b2f3b355e4a509 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 14:56:45 -0300 Subject: [PATCH 2/7] fix(engine, data): bound the addresses a listing repeats, and check systems only up to the first 64 Review round 1 of #1621: - service.layers gave each listed layer its document's address as source_url, and nothing bounds that address: 2000 collections under a 1 MB address peaked at 14.4 GB. A listing now repeats at most 64 MiB of addresses, says so in a note, and service.add's refusal counts the layers it lists. - Systems::with_own asked warp::usable (which makes a transformer) of every own system, a root's thousands included; it now asks only until the first 64 usable ones, as wms_native did, and the parent's are known to be usable. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/ogc.rs | 28 +++++++++++++++++++--------- crates/engine/src/cmd/service.rs | 28 ++++++++++++++++++++++++++-- crates/engine/src/tests.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index d3053bdb7..b33ad4c3d 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -1148,7 +1148,8 @@ fn wms(url: &str, root: &Node) -> Service { /// may list (a GeoServer's thousands) are worked out once, not again for each layer (#1519). #[derive(Default)] struct Systems { - /// How many names the list has, its parents' included: where a sublayer's own come in it. + /// Where a sublayer's own names are numbered from: all the names its parents list, those + /// RustGIS leaves out included (only the order matters). names: usize, mercator: bool, first: Vec, @@ -1156,23 +1157,32 @@ struct Systems { impl Systems { fn with_own(parent: &Systems, own: &[String]) -> Systems { - // (The parent's are usable already, and whether one is depends only on its code.) - let mut all = parent.first.clone(); - for mut c in wms_candidates(own).into_iter().filter(|c| warp::usable(c.1).is_some()) { + // Each with whether it is known to be usable: the parent's are. Whether one is depends only + // on its code, and finding out makes its transformer, so it is asked only of those that + // come before the first `MAX_SYSTEMS` usable ones, as `wms_native` did of the whole list. + let mut all: Vec<(Candidate, bool)> = parent.first.iter().map(|c| (c.clone(), true)).collect(); + let inherited = all.len(); + // (`wms_candidates` gives one name for each system: only the parent's can repeat one.) + for mut c in wms_candidates(own) { c.3 += parent.names; // One name for each system, as in `wms_candidates`: CRS:84 over EPSG:4326, else the // first listed. (A system past the parent's first ones comes after them all here too.) - match all.iter_mut().find(|p| p.1 == c.1) { - Some(p) if warp::is_crs84(&c.0) && !warp::is_crs84(&p.0) => *p = c, + match all[..inherited].iter_mut().find(|(p, _)| p.1 == c.1) { + Some((p, _)) if warp::is_crs84(&c.0) && !warp::is_crs84(&p.0) => *p = c, Some(_) => {} - None => all.push(c), + None => all.push((c, false)), } } - all.sort_by_key(|c| (candidate_rank(c), c.3)); + all.sort_by_key(|(c, _)| (candidate_rank(c), c.3)); Systems { names: parent.names + own.iter().map(|c| c.split_whitespace().count()).sum::(), mercator: parent.mercator || own.iter().any(|c| c.split_whitespace().any(is_web_mercator)), - first: all.into_iter().take(warp::MAX_SYSTEMS).collect(), + first: all + .into_iter() + .filter(|(c, known)| *known || warp::usable(c.1).is_some()) + .take(warp::MAX_SYSTEMS) + .map(|(c, _)| c) + .collect(), } } } diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index d4725d2a9..c0617bd00 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -91,7 +91,7 @@ pub(crate) fn without_token(p: &Value) -> Result, String> { /// One layer's exact capabilities document and the origin history used to reach it. The address /// is shared by the document's layers: nothing bounds its length, and a document has up to -/// `MAX_READ_LAYERS` of them (#1519). +/// 100,000 of them, the most `rustgis_data::ogc` reads (#1519). struct CapabilitySource { address: std::sync::Arc, crossed_origin: bool, @@ -126,6 +126,10 @@ impl Discovery { const MAX_SERVICE_DOCUMENTS: usize = 4; +/// Most bytes of document addresses a listing repeats: each listed layer's `source_url` is its +/// document's address, which nothing bounds (a landing page links to any collections address). +const MAX_LISTED_ADDRESS_BYTES: usize = if cfg!(test) { 8 << 20 } else { 64 << 20 }; + /// Whether what this `service.add` would wait for is left off the frame: it is a plugin command's, /// run by the desktop app, which runs the command again when it has come (#1319). fn leaves_waits(s: &Session) -> bool { @@ -144,8 +148,28 @@ fn capabilities(s: &mut Session, p: &Value) -> Result { } /// Reads a service's documents, fetching what `p` doesn't give: waiting for them, or with `left`, -/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. +/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. Long +/// document addresses list fewer layers ([`MAX_LISTED_ADDRESS_BYTES`]), said in a note. fn discover(s: &mut Session, p: &Value, left: bool) -> Result { + let mut found = read_documents(s, p, left)?; + let listed = found.service.listed.unwrap_or(usize::MAX).min(found.sources.len()); + let mut bytes = 0usize; + let cut = found.sources[..listed].iter().position(|source| { + bytes = bytes.saturating_add(source.address.len()); + bytes > MAX_LISTED_ADDRESS_BYTES + }); + // (At least one, whatever its address.) + if let Some((cut, long)) = cut.map(|cut| (cut.max(1), found.sources[cut].address.len())).filter(|&(cut, _)| cut < listed) { + found.service.listed = Some(cut); + found.service.notes.push(format!( + "each listed layer repeats its document's address, {long} bytes long here, and RustGIS lists at most {} MiB of them", + MAX_LISTED_ADDRESS_BYTES >> 20 + )); + } + Ok(found) +} + +fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result { let original = url_of(p)?; let mut address = crate::render::rustgis_tiles::logins::inherit(original, &rustgis_data::ogc::capabilities_url(original, ps(p, "type"))); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ce2971cd4..0723dc174 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -14889,6 +14889,35 @@ fn a_layer_past_the_listing_is_added_by_name() { assert!(e.contains("more than 2000"), "{e}"); } +/// Each listed layer repeats its document's address (`source_url`), which nothing bounds: a long +/// one lists fewer layers, said, and `service.add`'s refusal of a name counts those listed. +#[test] +fn a_long_document_address_lists_fewer_layers() { + let collections: Vec = (1..=600) + .map(|i| json!({"id": format!("c{i}"), "links": [{"rel": "items", "href": format!("https://api.example.org/c{i}/items")}]})) + .collect(); + let text = json!({"collections": collections}).to_string(); + let mut s = Session::new(Project::default()); + let short = s.run("service.layers", &json!({"url": "https://api.example.org/collections", "text": text})).unwrap(); + // (Other notes, such as how many features a collection is read up to here, aren't these.) + let says = |out: &Value, text: &str| out["notes"].as_array().unwrap().iter().any(|n| n == text); + assert_eq!(short["count"], 600); + assert!(short.get("listedNote").is_none() && !short["notes"].to_string().contains("repeats"), "{}", short["notes"]); + let url = format!("https://api.example.org/{}/collections", "a".repeat(16 << 10)); + let long = s.run("service.layers", &json!({"url": url, "text": text})).unwrap(); + let bytes = long["layers"][0]["source_url"].as_str().unwrap().len(); + // The test build lists 8 MiB of addresses (its length doesn't divide them). + let listed = (8 << 20) / bytes; + assert!(listed < 600 && (8 << 20) % bytes != 0, "{bytes}"); + assert_eq!((&long["count"], &long["total"]), (&json!(listed), &json!(600))); + let cut = format!("only the first {listed} of 600 layers are listed; service.add takes any of them by name"); + assert_eq!(long["listedNote"], json!(cut)); + let why = format!("each listed layer repeats its document's address, {bytes} bytes long here, and RustGIS lists at most 8 MiB of them"); + assert!(says(&long, &why) && says(&long, &cut), "{}", long["notes"]); + let e = s.run("service.add", &json!({"url": url, "layer": "nao", "text": text})).unwrap_err(); + assert!(e.ends_with(&format!("no layer 'nao' (service.layers lists the first {listed} of 600 layers)")), "{e}"); +} + /// A layer past what RustGIS reads of a document is refused for that reason, not as absent. #[test] fn a_layer_past_the_read_limit_is_refused_saying_so() { From 1095409fa2f9208dbae4bcbc48e2b9b55d13e750 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 15:36:32 -0300 Subject: [PATCH 3/7] fix(data, engine): stop reading at the address budget whatever the listing Review round 2 of #1621: - read_limit and the OGC API collections loop lowered how many layers are read only down to the listing (2000), so 2000 collections without an items link under a 1 MB address copied it 2000 times while parsing (2 GB). Reading now stops at MAX_COPIED_ADDRESS_BYTES of copies after at least one layer, and an OGC API document read short is incomplete, so a service group refuses it. - The listing's note says it keeps at least one layer, whatever its address. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/ogc.rs | 34 ++++++++++++++++++++++++-------- crates/engine/src/cmd/service.rs | 2 +- crates/engine/src/tests.rs | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index b33ad4c3d..3c83be902 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -475,9 +475,11 @@ const MAX_READ_LAYERS: usize = if cfg!(test) { 3_000 } else { 100_000 }; const MAX_COPIED_ADDRESS_BYTES: usize = if cfg!(test) { 8 << 20 } else { 64 << 20 }; /// How many layers are read from a document whose layers each copy `per_layer` bytes of its -/// request addresses: [`MAX_READ_LAYERS`], fewer for long addresses, never fewer than are listed. +/// request addresses: [`MAX_READ_LAYERS`], fewer for long addresses, at least one. (Fewer than +/// are listed only for addresses over 32 KiB, which no service needs: 2000 copies of a 1 MB one +/// would take 2 GB.) fn read_limit(per_layer: usize) -> usize { - (MAX_COPIED_ADDRESS_BYTES / per_layer.max(1)).clamp(MAX_LISTED_LAYERS, MAX_READ_LAYERS) + (MAX_COPIED_ADDRESS_BYTES / per_layer.max(1)).clamp(1, MAX_READ_LAYERS) } /// Says why a document past a [`read_limit`] lower than [`MAX_READ_LAYERS`] was read no further. @@ -2647,8 +2649,8 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { // are listed. let mut three_d: Vec<((&str, bool), Vec<&str>)> = Vec::new(); // A collection's layers copy the document's own address when their items or tiles links are - // missing or relative, and nothing bounds that address: past the listing, reading stops once - // the layers' addresses reach `MAX_COPIED_ADDRESS_BYTES`. + // missing or relative, and nothing bounds that address: reading stops once the layers' + // addresses reach `MAX_COPIED_ADDRESS_BYTES` (after the first collection, as `read_limit`). let (mut copied, mut counted) = (0usize, 0); let mut read = collections.len().min(MAX_READ_LAYERS); for (i, c) in collections.iter().take(MAX_READ_LAYERS).enumerate() { @@ -2658,7 +2660,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { if i == MAX_LISTED_LAYERS { service.listed = Some(service.layers.len()); } - if i >= MAX_LISTED_LAYERS && copied > MAX_COPIED_ADDRESS_BYTES { + if i > 0 && copied > MAX_COPIED_ADDRESS_BYTES { read = i; break; } @@ -2786,7 +2788,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { note_past_read_limit(&mut service, collections.len() - read, "collections", read); note_copied_addresses(&mut service, copied.div_ceil(read.max(1)), read); } - service.incomplete = unreadable > 0 || collections.len() > MAX_LISTED_LAYERS; + service.incomplete = unreadable > 0 || collections.len() > MAX_LISTED_LAYERS.min(read); if service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })) { service.notes.push(format!("a collection is read up to {} features on this computer, page by page", max_paged_features())); } @@ -7037,8 +7039,8 @@ mod tests { assert_eq!(s.listed, cut.then_some(MAX_LISTED_LAYERS), "{url}"); assert_eq!(parse(url, doc(MAX_LISTED_LAYERS).as_bytes()).unwrap().listed, None, "{url}"); } - // A long request address, copied into each layer, lowers how many are read, though never - // below the listing. + // A long request address, copied into each layer, lowers how many are read (a 4 KB one not + // below the listing). let href = format!("https://x.org/{}?", "a".repeat(4000)); let types = many(MAX_READ_LAYERS, |i| format!("t{i}")); let long = format!( @@ -7068,6 +7070,22 @@ mod tests { assert!(s.notes.iter().any(|n| n.starts_with("each of its layers copies 40")), "{:?}", s.notes); let s = parse("https://x.org/collections", bare.as_bytes()).unwrap(); assert_eq!((s.layers.len(), s.past_read_limit), (MAX_READ_LAYERS, None)); + // An address so long that the listing's copies would pass the budget reads fewer than are + // listed (at least one), and the collections are said to be incomplete for a group. + let huge = "a".repeat(1 << 20); + let few = serde_json::json!({"collections": list(100, |i| serde_json::json!({"id": format!("c{i}")}))}).to_string(); + let s = parse(&format!("https://x.org/{huge}/collections"), few.as_bytes()).unwrap(); + let read = s.layers.len(); + assert!((1..100).contains(&read) && s.incomplete && s.listed.is_none(), "{read}"); + let note = format!("{PAST_READ_LIMIT} {read} collections from one document: {} left out", 100 - read); + assert_eq!(s.past_read_limit.as_ref(), Some(¬e)); + let types = many(100, |i| format!("t{i}")); + let wfs = format!( + r#"{types}"# + ); + let s = parse("https://x.org/wfs", wfs.as_bytes()).unwrap(); + assert!((1..100).contains(&s.layers.len()), "{}", s.layers.len()); + assert!(s.notes.iter().any(|n| n.starts_with("each of its layers copies 10")), "{:?}", s.notes); // A WMS is read in document order: the layers past the limit are its last, and a parent // comes before its sublayers. let s = parse("https://x.org/wms", wms(MAX_READ_LAYERS + 1).as_bytes()).unwrap(); diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index c0617bd00..d5f6b5c6e 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -162,7 +162,7 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result if let Some((cut, long)) = cut.map(|cut| (cut.max(1), found.sources[cut].address.len())).filter(|&(cut, _)| cut < listed) { found.service.listed = Some(cut); found.service.notes.push(format!( - "each listed layer repeats its document's address, {long} bytes long here, and RustGIS lists at most {} MiB of them", + "each listed layer repeats its document's address, {long} bytes long here, and RustGIS lists at most {} MiB of them (at least one layer)", MAX_LISTED_ADDRESS_BYTES >> 20 )); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 0723dc174..293bcaf25 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -14912,7 +14912,9 @@ fn a_long_document_address_lists_fewer_layers() { assert_eq!((&long["count"], &long["total"]), (&json!(listed), &json!(600))); let cut = format!("only the first {listed} of 600 layers are listed; service.add takes any of them by name"); assert_eq!(long["listedNote"], json!(cut)); - let why = format!("each listed layer repeats its document's address, {bytes} bytes long here, and RustGIS lists at most 8 MiB of them"); + let why = format!( + "each listed layer repeats its document's address, {bytes} bytes long here, and RustGIS lists at most 8 MiB of them (at least one layer)" + ); assert!(says(&long, &why) && says(&long, &cut), "{}", long["notes"]); let e = s.run("service.add", &json!({"url": url, "layer": "nao", "text": text})).unwrap_err(); assert!(e.ends_with(&format!("no layer 'nao' (service.layers lists the first {listed} of 600 layers)")), "{e}"); From 9f7eccf7ff848da319e02b4788575dcdb3e7b438 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 15:45:39 -0300 Subject: [PATCH 4/7] fix(engine): a service group's refusal names a too-long address; user guide on the new read limits Review round 3 of #1621: an OGC API document read short by a long address is incomplete, and the group's refusal said only "more than 2000, or unreadable addresses". The user guide said a long address still reads at least 2,000 layers. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/service_group.rs | 2 +- docs/user-guide/data-formats.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/service_group.rs b/crates/engine/src/cmd/service_group.rs index bcb1e403a..919278200 100644 --- a/crates/engine/src/cmd/service_group.rs +++ b/crates/engine/src/cmd/service_group.rs @@ -120,7 +120,7 @@ fn listing(s: &mut Session, p: &Value) -> CmdResult { } if found.incomplete { return Err( - "the service lists collections RustGIS cannot read (more than 2000, or unreadable addresses); the group was not changed".into(), + "the service lists collections RustGIS cannot read (more than 2000, an address too long to read them all, or unreadable addresses); the group was not changed".into(), ); } let previous = target(s, p)?; diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index b330ac8c3..2d9ea2ff1 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -314,8 +314,10 @@ A long list stops at its first 2,000 layers, and a note says how many the servic API service stops at its first 2,000 collections instead, and each can make up to three layers (features, map tiles and vector tiles), so its list can be longer. Which ones come first is up to the service, and some change their order between answers. `service.add` still takes any of them by name, up to 100,000 in one document, fewer -(but at least 2,000) when the document's request address (an OGC API's, its own), which each -layer's repeats, is very long. An ArcGIS service or directory is an exception: it is read up to +when the document's request address (an OGC API's, its own), which each layer's repeats, is +very long: at least 2,000 up to about 32 KB, and fewer than are listed past that, at least one, +with a note. A list also stops early, with a note, once its layers' document addresses (each +layer's `source_url`) reach 64 MiB, which takes an address over 32 KB too. An ArcGIS service or directory is an exception: it is read up to 2,000 layers, folders and services, and lists all it reads. A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit. From eac68452257f8c11a25f5051e8d8d8134f762e81 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 15:55:41 -0300 Subject: [PATCH 5/7] fix(engine): a service group refuses a list the address budget cut Review round 4 of #1621: a group takes every layer read, not those listed, so a listing cut by MAX_LISTED_ADDRESS_BYTES still let a group copy every source_url. Such a document is now incomplete, and the group says why. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/service.rs | 2 ++ crates/engine/src/tests.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index d5f6b5c6e..fc8a6bb10 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -161,6 +161,8 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result // (At least one, whatever its address.) if let Some((cut, long)) = cut.map(|cut| (cut.max(1), found.sources[cut].address.len())).filter(|&(cut, _)| cut < listed) { found.service.listed = Some(cut); + // A service group takes every layer read, not those listed: it can't take part of this. + found.incomplete = true; found.service.notes.push(format!( "each listed layer repeats its document's address, {long} bytes long here, and RustGIS lists at most {} MiB of them (at least one layer)", MAX_LISTED_ADDRESS_BYTES >> 20 diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 293bcaf25..ded3553d8 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -14918,6 +14918,10 @@ fn a_long_document_address_lists_fewer_layers() { assert!(says(&long, &why) && says(&long, &cut), "{}", long["notes"]); let e = s.run("service.add", &json!({"url": url, "layer": "nao", "text": text})).unwrap_err(); assert!(e.ends_with(&format!("no layer 'nao' (service.layers lists the first {listed} of 600 layers)")), "{e}"); + // A service group would take them all, each with its address: it refuses the list instead. + let e = s.run("service.groupPlan", &json!({"url": url, "text": text})).unwrap_err(); + assert!(e.contains("an address too long to read them all"), "{e}"); + assert!(s.run("service.groupPlan", &json!({"url": "https://api.example.org/collections", "text": text})).is_ok()); } /// A layer past what RustGIS reads of a document is refused for that reason, not as absent. From 874a9d0f59c3925eb29aa49fa2d91954b54e9e7d Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 15:56:21 -0300 Subject: [PATCH 6/7] docs: a WMS's request addresses count together, and an OGC API's longer list stops sooner Review round 4 of #1621. Co-Authored-By: Claude Opus 5.5 --- docs/user-guide/data-formats.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 2d9ea2ff1..3f84ce6bb 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -314,10 +314,12 @@ A long list stops at its first 2,000 layers, and a note says how many the servic API service stops at its first 2,000 collections instead, and each can make up to three layers (features, map tiles and vector tiles), so its list can be longer. Which ones come first is up to the service, and some change their order between answers. `service.add` still takes any of them by name, up to 100,000 in one document, fewer -when the document's request address (an OGC API's, its own), which each layer's repeats, is -very long: at least 2,000 up to about 32 KB, and fewer than are listed past that, at least one, -with a note. A list also stops early, with a note, once its layers' document addresses (each -layer's `source_url`) reach 64 MiB, which takes an address over 32 KB too. An ArcGIS service or directory is an exception: it is read up to +when the document's request addresses (an OGC API's, its own; a WMS's GetMap, GetFeatureInfo +and legend addresses together), which each layer's repeat, are very long: at least 2,000 up to +about 32 KB, and fewer than are listed past that, at least one, with a note. A list also stops +early, with a note, once its layers' document addresses (each layer's `source_url`) reach +64 MiB: 2,000 layers each repeating an address over 32 KB, or an OGC API's longer list one a +little shorter. An ArcGIS service or directory is an exception: it is read up to 2,000 layers, folders and services, and lists all it reads. A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit. From 20cc13296f77ebc5b79a6391da5702d9e56bcbd7 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Sun, 8 Nov 2026 15:56:41 -0300 Subject: [PATCH 7/7] docs: reword the listing's address stop Co-Authored-By: Claude Opus 5.5 --- docs/user-guide/data-formats.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 3f84ce6bb..bff8d5c68 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -318,8 +318,8 @@ when the document's request addresses (an OGC API's, its own; a WMS's GetMap, Ge and legend addresses together), which each layer's repeat, are very long: at least 2,000 up to about 32 KB, and fewer than are listed past that, at least one, with a note. A list also stops early, with a note, once its layers' document addresses (each layer's `source_url`) reach -64 MiB: 2,000 layers each repeating an address over 32 KB, or an OGC API's longer list one a -little shorter. An ArcGIS service or directory is an exception: it is read up to +64 MiB: 2,000 layers each repeating an address over 32 KB, or the more layers of an OGC API's +list each repeating a somewhat shorter one. An ArcGIS service or directory is an exception: it is read up to 2,000 layers, folders and services, and lists all it reads. A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit.