diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index 4a8d098e2..3c83be902 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -475,9 +475,11 @@ const MAX_READ_LAYERS: usize = if cfg!(test) { 3_000 } else { 100_000 }; const MAX_COPIED_ADDRESS_BYTES: usize = if cfg!(test) { 8 << 20 } else { 64 << 20 }; /// How many layers are read from a document whose layers each copy `per_layer` bytes of its -/// request addresses: [`MAX_READ_LAYERS`], fewer for long addresses, never fewer than are listed. +/// request addresses: [`MAX_READ_LAYERS`], fewer for long addresses, at least one. (Fewer than +/// are listed only for addresses over 32 KiB, which no service needs: 2000 copies of a 1 MB one +/// would take 2 GB.) fn read_limit(per_layer: usize) -> usize { - (MAX_COPIED_ADDRESS_BYTES / per_layer.max(1)).clamp(MAX_LISTED_LAYERS, MAX_READ_LAYERS) + (MAX_COPIED_ADDRESS_BYTES / per_layer.max(1)).clamp(1, MAX_READ_LAYERS) } /// Says why a document past a [`read_limit`] lower than [`MAX_READ_LAYERS`] was read no further. @@ -1006,11 +1008,7 @@ fn wms(url: &str, root: &Node) -> Service { // (A layer naming none of its own shares its parent's list, and what is worked out of it.) let own: Vec = layer.children().filter(|c| c.is_element() && matches!(local(c), "CRS" | "SRS")).map(|c| text_of(&c)).collect(); - let crs: Rc = if own.is_empty() { - inherited_crs - } else { - Rc::new(Systems { names: inherited_crs.names.iter().cloned().chain(own).collect(), ..Systems::default() }) - }; + let crs: Rc = if own.is_empty() { inherited_crs } else { Rc::new(Systems::with_own(&inherited_crs, &own)) }; let bbox = child(&layer, "EX_GeographicBoundingBox") .and_then(|b| { let f = |k: &str| child_text(&b, k).and_then(|t| t.parse::().ok()); @@ -1040,9 +1038,8 @@ fn wms(url: &str, root: &Node) -> Service { let Some(name) = child_text(&layer, "Name") else { continue }; // In Web Mercator, as the service draws it, else in a system of its own, warped (#769); // with the other systems it is offered in, for a map drawn in one of them. - let mercator = crs.names.iter().any(|c| c.split_whitespace().any(is_web_mercator)); - let listed = crs.candidates.get_or_init(|| wms_candidates(&crs.names)); - let mut native = wms_native(&layer, listed, v13, bbox, mercator, &mut layer_boxes); + let mercator = crs.mercator; + let mut native = wms_native(&layer, &crs.first, v13, bbox, mercator, &mut layer_boxes); if !mercator && native.is_none() { skipped += 1; continue; @@ -1146,12 +1143,60 @@ fn wms(url: &str, root: &Node) -> Service { service } -/// The coordinate systems a WMS layer is offered in, its own and those it inherits, and those of -/// them RustGIS knows (worked out once, for the layers sharing the list). +/// The coordinate systems a WMS layer is offered in, its own and those it inherits: whether Web +/// Mercator is among them, and the first [`warp::MAX_SYSTEMS`] RustGIS can draw the layer in, in +/// order of preference (see [`wms_candidates`]), the only ones [`wms_native`] takes. A layer +/// naming systems of its own adds them to its parent's first ones, so the many a service's root +/// may list (a GeoServer's thousands) are worked out once, not again for each layer (#1519). #[derive(Default)] struct Systems { - names: Vec, - candidates: std::cell::OnceCell>, + /// Where a sublayer's own names are numbered from: all the names its parents list, those + /// RustGIS leaves out included (only the order matters). + names: usize, + mercator: bool, + first: Vec, +} + +impl Systems { + fn with_own(parent: &Systems, own: &[String]) -> Systems { + // Each with whether it is known to be usable: the parent's are. Whether one is depends only + // on its code, and finding out makes its transformer, so it is asked only of those that + // come before the first `MAX_SYSTEMS` usable ones, as `wms_native` did of the whole list. + let mut all: Vec<(Candidate, bool)> = parent.first.iter().map(|c| (c.clone(), true)).collect(); + let inherited = all.len(); + // (`wms_candidates` gives one name for each system: only the parent's can repeat one.) + for mut c in wms_candidates(own) { + c.3 += parent.names; + // One name for each system, as in `wms_candidates`: CRS:84 over EPSG:4326, else the + // first listed. (A system past the parent's first ones comes after them all here too.) + match all[..inherited].iter_mut().find(|(p, _)| p.1 == c.1) { + Some((p, _)) if warp::is_crs84(&c.0) && !warp::is_crs84(&p.0) => *p = c, + Some(_) => {} + None => all.push((c, false)), + } + } + all.sort_by_key(|(c, _)| (candidate_rank(c), c.3)); + Systems { + names: parent.names + own.iter().map(|c| c.split_whitespace().count()).sum::(), + mercator: parent.mercator || own.iter().any(|c| c.split_whitespace().any(is_web_mercator)), + first: all + .into_iter() + .filter(|(c, known)| *known || warp::usable(c.1).is_some()) + .take(warp::MAX_SYSTEMS) + .map(|(c, _)| c) + .collect(), + } + } +} + +/// Where a system comes in a WMS layer's order of preference: projected, CRS:84, other +/// geographic. +fn candidate_rank((name, _, c, _): &Candidate) -> u8 { + match (c.is_geographic(), warp::is_crs84(name)) { + (false, _) => 0, + (true, true) => 1, + (true, false) => 2, + } } /// A system of a WMS layer's list that RustGIS knows: its name there, its EPSG code, it, and its @@ -1179,13 +1224,8 @@ fn wms_candidates(crs: &[String]) -> Vec { // One name for each system (EPSG:4326 and CRS:84 differ only in their axes: CRS:84 kept). listed.sort_by_cached_key(|(name, code, _, _)| (*code, !warp::is_crs84(name))); listed.dedup_by_key(|(_, code, _, _)| *code); - let rank = |(name, _, c, _): &Candidate| match (c.is_geographic(), warp::is_crs84(name)) { - (false, _) => 0, - (true, true) => 1, - (true, false) => 2, - }; // In the order the service lists them, the preferred first. - listed.sort_by_key(|l| (rank(l), l.3)); + listed.sort_by_key(|l| (candidate_rank(l), l.3)); listed } @@ -2609,8 +2649,8 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { // are listed. let mut three_d: Vec<((&str, bool), Vec<&str>)> = Vec::new(); // A collection's layers copy the document's own address when their items or tiles links are - // missing or relative, and nothing bounds that address: past the listing, reading stops once - // the layers' addresses reach `MAX_COPIED_ADDRESS_BYTES`. + // missing or relative, and nothing bounds that address: reading stops once the layers' + // addresses reach `MAX_COPIED_ADDRESS_BYTES` (after the first collection, as `read_limit`). let (mut copied, mut counted) = (0usize, 0); let mut read = collections.len().min(MAX_READ_LAYERS); for (i, c) in collections.iter().take(MAX_READ_LAYERS).enumerate() { @@ -2620,7 +2660,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { if i == MAX_LISTED_LAYERS { service.listed = Some(service.layers.len()); } - if i >= MAX_LISTED_LAYERS && copied > MAX_COPIED_ADDRESS_BYTES { + if i > 0 && copied > MAX_COPIED_ADDRESS_BYTES { read = i; break; } @@ -2748,7 +2788,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { note_past_read_limit(&mut service, collections.len() - read, "collections", read); note_copied_addresses(&mut service, copied.div_ceil(read.max(1)), read); } - service.incomplete = unreadable > 0 || collections.len() > MAX_LISTED_LAYERS; + service.incomplete = unreadable > 0 || collections.len() > MAX_LISTED_LAYERS.min(read); if service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })) { service.notes.push(format!("a collection is read up to {} features on this computer, page by page", max_paged_features())); } @@ -5803,6 +5843,33 @@ mod tests { } } + /// A sublayer's systems, its own added to its parent's first ones, are those its whole list + /// gives, so a root's many are worked out once (#1519). + #[test] + fn a_sublayers_systems_build_on_its_parents_first_ones() { + let first = |names: &[String]| -> Vec<(String, u32)> { + wms_candidates(names).into_iter().filter(|c| warp::usable(c.1).is_some()).take(warp::MAX_SYSTEMS).map(|c| (c.0, c.1)).collect() + }; + let codes = |r: std::ops::Range| r.map(|c| format!("EPSG:{c}")).collect::>(); + let many = [codes(32601..32671), vec!["EPSG:4326".into(), "EPSG:3857".into()], codes(2000..2100)].concat(); + let few = vec!["EPSG:4326".to_string(), "EPSG:3857".into()]; + let owns = [ + vec!["CRS:84".to_string()], + vec!["EPSG:32633".into(), "EPSG:2056".into()], + vec!["EPSG:4326 EPSG:31983".into()], + codes(32701..32761), + ]; + for root in [&many, &few] { + let parent = Systems::with_own(&Systems::default(), root); + for own in &owns { + let child = Systems::with_own(&parent, own); + let whole: Vec = root.iter().chain(own).cloned().collect(); + assert_eq!(child.first.iter().map(|c| (c.0.clone(), c.1)).collect::>(), first(&whole), "{own:?}"); + assert!(child.mercator && !Systems::with_own(&Systems::default(), own).mercator); + } + } + } + #[test] fn wms_layers_in_web_mercator() { let xml = r#" @@ -6972,8 +7039,8 @@ mod tests { assert_eq!(s.listed, cut.then_some(MAX_LISTED_LAYERS), "{url}"); assert_eq!(parse(url, doc(MAX_LISTED_LAYERS).as_bytes()).unwrap().listed, None, "{url}"); } - // A long request address, copied into each layer, lowers how many are read, though never - // below the listing. + // A long request address, copied into each layer, lowers how many are read (a 4 KB one not + // below the listing). let href = format!("https://x.org/{}?", "a".repeat(4000)); let types = many(MAX_READ_LAYERS, |i| format!("t{i}")); let long = format!( @@ -7003,6 +7070,22 @@ mod tests { assert!(s.notes.iter().any(|n| n.starts_with("each of its layers copies 40")), "{:?}", s.notes); let s = parse("https://x.org/collections", bare.as_bytes()).unwrap(); assert_eq!((s.layers.len(), s.past_read_limit), (MAX_READ_LAYERS, None)); + // An address so long that the listing's copies would pass the budget reads fewer than are + // listed (at least one), and the collections are said to be incomplete for a group. + let huge = "a".repeat(1 << 20); + let few = serde_json::json!({"collections": list(100, |i| serde_json::json!({"id": format!("c{i}")}))}).to_string(); + let s = parse(&format!("https://x.org/{huge}/collections"), few.as_bytes()).unwrap(); + let read = s.layers.len(); + assert!((1..100).contains(&read) && s.incomplete && s.listed.is_none(), "{read}"); + let note = format!("{PAST_READ_LIMIT} {read} collections from one document: {} left out", 100 - read); + assert_eq!(s.past_read_limit.as_ref(), Some(¬e)); + let types = many(100, |i| format!("t{i}")); + let wfs = format!( + r#"{types}"# + ); + let s = parse("https://x.org/wfs", wfs.as_bytes()).unwrap(); + assert!((1..100).contains(&s.layers.len()), "{}", s.layers.len()); + assert!(s.notes.iter().any(|n| n.starts_with("each of its layers copies 10")), "{:?}", s.notes); // A WMS is read in document order: the layers past the limit are its last, and a parent // comes before its sublayers. let s = parse("https://x.org/wms", wms(MAX_READ_LAYERS + 1).as_bytes()).unwrap(); diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index 460a8d11d..fc8a6bb10 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -89,9 +89,11 @@ pub(crate) fn without_token(p: &Value) -> Result, String> { Ok(Some(owned)) } -/// One layer's exact capabilities document and the origin history used to reach it. +/// One layer's exact capabilities document and the origin history used to reach it. The address +/// is shared by the document's layers: nothing bounds its length, and a document has up to +/// 100,000 of them, the most `rustgis_data::ogc` reads (#1519). struct CapabilitySource { - address: String, + address: std::sync::Arc, crossed_origin: bool, } @@ -124,6 +126,10 @@ impl Discovery { const MAX_SERVICE_DOCUMENTS: usize = 4; +/// Most bytes of document addresses a listing repeats: each listed layer's `source_url` is its +/// document's address, which nothing bounds (a landing page links to any collections address). +const MAX_LISTED_ADDRESS_BYTES: usize = if cfg!(test) { 8 << 20 } else { 64 << 20 }; + /// Whether what this `service.add` would wait for is left off the frame: it is a plugin command's, /// run by the desktop app, which runs the command again when it has come (#1319). fn leaves_waits(s: &Session) -> bool { @@ -142,8 +148,30 @@ fn capabilities(s: &mut Session, p: &Value) -> Result { } /// Reads a service's documents, fetching what `p` doesn't give: waiting for them, or with `left`, -/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. +/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. Long +/// document addresses list fewer layers ([`MAX_LISTED_ADDRESS_BYTES`]), said in a note. fn discover(s: &mut Session, p: &Value, left: bool) -> Result { + let mut found = read_documents(s, p, left)?; + let listed = found.service.listed.unwrap_or(usize::MAX).min(found.sources.len()); + let mut bytes = 0usize; + let cut = found.sources[..listed].iter().position(|source| { + bytes = bytes.saturating_add(source.address.len()); + bytes > MAX_LISTED_ADDRESS_BYTES + }); + // (At least one, whatever its address.) + if let Some((cut, long)) = cut.map(|cut| (cut.max(1), found.sources[cut].address.len())).filter(|&(cut, _)| cut < listed) { + found.service.listed = Some(cut); + // A service group takes every layer read, not those listed: it can't take part of this. + found.incomplete = true; + found.service.notes.push(format!( + "each listed layer repeats its document's address, {long} bytes long here, and RustGIS lists at most {} MiB of them (at least one layer)", + MAX_LISTED_ADDRESS_BYTES >> 20 + )); + } + Ok(found) +} + +fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result { let original = url_of(p)?; let mut address = crate::render::rustgis_tiles::logins::inherit(original, &rustgis_data::ogc::capabilities_url(original, ps(p, "type"))); @@ -216,9 +244,10 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result } // Parsers bound each document's collections/layers; visited documents bound the merge. // Distinct offers may share a name (including generated tile names and collection ids). + let shared: std::sync::Arc = address.as_str().into(); for layer in service.layers { result.service.layers.push(layer); - result.sources.push(CapabilitySource { address: address.clone(), crossed_origin }); + result.sources.push(CapabilitySource { address: shared.clone(), crossed_origin }); } result.service.withheld_addresses = result.service.withheld_addresses.saturating_add(service.withheld_addresses); result.service.past_read_limit = result.service.past_read_limit.take().or(service.past_read_limit); @@ -684,7 +713,7 @@ fn layer_source(layer: &mut Value, source: CapabilitySource) { offer.get("type").and_then(Value::as_str) == Some("features") && ps(offer, "url").is_some_and(|url| crosses_origin(&source.address, url)) }); - layer["source_url"] = json!(source.address); + layer["source_url"] = json!(&*source.address); if source.crossed_origin || foreign_features { layer["anonymous"] = json!(true); if let Some(url) = layer.get_mut("offer").and_then(|o| o.get_mut("url")) @@ -746,7 +775,7 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis } })?; let capability_crossed = source.crossed_origin; - let source = source.address.as_str(); + let source = &*source.address; // A time-enabled layer follows the time slider, at each of its steps, unless a time is asked // for: one of its times (the layer named with it), or "default" (`false`), the service's. // (`"time": true`, the slider, as for a Zarr variable, `layer.addZarr`'s.) diff --git a/crates/engine/src/cmd/service_group.rs b/crates/engine/src/cmd/service_group.rs index bcb1e403a..919278200 100644 --- a/crates/engine/src/cmd/service_group.rs +++ b/crates/engine/src/cmd/service_group.rs @@ -120,7 +120,7 @@ fn listing(s: &mut Session, p: &Value) -> CmdResult { } if found.incomplete { return Err( - "the service lists collections RustGIS cannot read (more than 2000, or unreadable addresses); the group was not changed".into(), + "the service lists collections RustGIS cannot read (more than 2000, an address too long to read them all, or unreadable addresses); the group was not changed".into(), ); } let previous = target(s, p)?; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ce2971cd4..ded3553d8 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -14889,6 +14889,41 @@ fn a_layer_past_the_listing_is_added_by_name() { assert!(e.contains("more than 2000"), "{e}"); } +/// Each listed layer repeats its document's address (`source_url`), which nothing bounds: a long +/// one lists fewer layers, said, and `service.add`'s refusal of a name counts those listed. +#[test] +fn a_long_document_address_lists_fewer_layers() { + let collections: Vec = (1..=600) + .map(|i| json!({"id": format!("c{i}"), "links": [{"rel": "items", "href": format!("https://api.example.org/c{i}/items")}]})) + .collect(); + let text = json!({"collections": collections}).to_string(); + let mut s = Session::new(Project::default()); + let short = s.run("service.layers", &json!({"url": "https://api.example.org/collections", "text": text})).unwrap(); + // (Other notes, such as how many features a collection is read up to here, aren't these.) + let says = |out: &Value, text: &str| out["notes"].as_array().unwrap().iter().any(|n| n == text); + assert_eq!(short["count"], 600); + assert!(short.get("listedNote").is_none() && !short["notes"].to_string().contains("repeats"), "{}", short["notes"]); + let url = format!("https://api.example.org/{}/collections", "a".repeat(16 << 10)); + let long = s.run("service.layers", &json!({"url": url, "text": text})).unwrap(); + let bytes = long["layers"][0]["source_url"].as_str().unwrap().len(); + // The test build lists 8 MiB of addresses (its length doesn't divide them). + let listed = (8 << 20) / bytes; + assert!(listed < 600 && (8 << 20) % bytes != 0, "{bytes}"); + assert_eq!((&long["count"], &long["total"]), (&json!(listed), &json!(600))); + let cut = format!("only the first {listed} of 600 layers are listed; service.add takes any of them by name"); + assert_eq!(long["listedNote"], json!(cut)); + let why = format!( + "each listed layer repeats its document's address, {bytes} bytes long here, and RustGIS lists at most 8 MiB of them (at least one layer)" + ); + assert!(says(&long, &why) && says(&long, &cut), "{}", long["notes"]); + let e = s.run("service.add", &json!({"url": url, "layer": "nao", "text": text})).unwrap_err(); + assert!(e.ends_with(&format!("no layer 'nao' (service.layers lists the first {listed} of 600 layers)")), "{e}"); + // A service group would take them all, each with its address: it refuses the list instead. + let e = s.run("service.groupPlan", &json!({"url": url, "text": text})).unwrap_err(); + assert!(e.contains("an address too long to read them all"), "{e}"); + assert!(s.run("service.groupPlan", &json!({"url": "https://api.example.org/collections", "text": text})).is_ok()); +} + /// A layer past what RustGIS reads of a document is refused for that reason, not as absent. #[test] fn a_layer_past_the_read_limit_is_refused_saying_so() { diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index b330ac8c3..bff8d5c68 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -314,8 +314,12 @@ A long list stops at its first 2,000 layers, and a note says how many the servic API service stops at its first 2,000 collections instead, and each can make up to three layers (features, map tiles and vector tiles), so its list can be longer. Which ones come first is up to the service, and some change their order between answers. `service.add` still takes any of them by name, up to 100,000 in one document, fewer -(but at least 2,000) when the document's request address (an OGC API's, its own), which each -layer's repeats, is very long. An ArcGIS service or directory is an exception: it is read up to +when the document's request addresses (an OGC API's, its own; a WMS's GetMap, GetFeatureInfo +and legend addresses together), which each layer's repeat, are very long: at least 2,000 up to +about 32 KB, and fewer than are listed past that, at least one, with a note. A list also stops +early, with a note, once its layers' document addresses (each layer's `source_url`) reach +64 MiB: 2,000 layers each repeating an address over 32 KB, or the more layers of an OGC API's +list each repeating a somewhat shorter one. An ArcGIS service or directory is an exception: it is read up to 2,000 layers, folders and services, and lists all it reads. A dynamic MapServer offers each of those layers twice, as features and as a map, so its list can be twice as long. A note says how many are left out past either limit.