fix(data, engine): share a capabilities document's address, and work out a WMS root's systems once (#1526 follow-ups)

crates/data/src/ogc.rs

@@ -475,9 +475,11 @@ const MAX_READ_LAYERS: usize = if cfg!(test) { 3_000 } else { 100_000 };
 const MAX_COPIED_ADDRESS_BYTES: usize = if cfg!(test) { 8 << 20 } else { 64 << 20 };
 
 /// How many layers are read from a document whose layers each copy `per_layer` bytes of its
-/// request addresses: [`MAX_READ_LAYERS`], fewer for long addresses, never fewer than are listed.
+/// request addresses: [`MAX_READ_LAYERS`], fewer for long addresses, at least one. (Fewer than
+/// are listed only for addresses over 32 KiB, which no service needs: 2000 copies of a 1 MB one
+/// would take 2 GB.)
 fn read_limit(per_layer: usize) -> usize {
-    (MAX_COPIED_ADDRESS_BYTES / per_layer.max(1)).clamp(MAX_LISTED_LAYERS, MAX_READ_LAYERS)
+    (MAX_COPIED_ADDRESS_BYTES / per_layer.max(1)).clamp(1, MAX_READ_LAYERS)
 }
 
 /// Says why a document past a [`read_limit`] lower than [`MAX_READ_LAYERS`] was read no further.
@@ -1006,11 +1008,7 @@ fn wms(url: &str, root: &Node) -> Service {
         // (A layer naming none of its own shares its parent's list, and what is worked out of it.)
         let own: Vec<String> =
             layer.children().filter(|c| c.is_element() && matches!(local(c), "CRS" | "SRS")).map(|c| text_of(&c)).collect();
-        let crs: Rc<Systems> = if own.is_empty() {
-            inherited_crs
-        } else {
-            Rc::new(Systems { names: inherited_crs.names.iter().cloned().chain(own).collect(), ..Systems::default() })
-        };
+        let crs: Rc<Systems> = if own.is_empty() { inherited_crs } else { Rc::new(Systems::with_own(&inherited_crs, &own)) };
         let bbox = child(&layer, "EX_GeographicBoundingBox")
             .and_then(|b| {
                 let f = |k: &str| child_text(&b, k).and_then(|t| t.parse::<f64>().ok());
@@ -1040,9 +1038,8 @@ fn wms(url: &str, root: &Node) -> Service {
         let Some(name) = child_text(&layer, "Name") else { continue };
         // In Web Mercator, as the service draws it, else in a system of its own, warped (#769);
         // with the other systems it is offered in, for a map drawn in one of them.
-        let mercator = crs.names.iter().any(|c| c.split_whitespace().any(is_web_mercator));
-        let listed = crs.candidates.get_or_init(|| wms_candidates(&crs.names));
-        let mut native = wms_native(&layer, listed, v13, bbox, mercator, &mut layer_boxes);
+        let mercator = crs.mercator;
+        let mut native = wms_native(&layer, &crs.first, v13, bbox, mercator, &mut layer_boxes);
         if !mercator && native.is_none() {
             skipped += 1;
             continue;
@@ -1146,12 +1143,60 @@ fn wms(url: &str, root: &Node) -> Service {
     service
 }
 
-/// The coordinate systems a WMS layer is offered in, its own and those it inherits, and those of
-/// them RustGIS knows (worked out once, for the layers sharing the list).
+/// The coordinate systems a WMS layer is offered in, its own and those it inherits: whether Web
+/// Mercator is among them, and the first [`warp::MAX_SYSTEMS`] RustGIS can draw the layer in, in
+/// order of preference (see [`wms_candidates`]), the only ones [`wms_native`] takes. A layer
+/// naming systems of its own adds them to its parent's first ones, so the many a service's root
+/// may list (a GeoServer's thousands) are worked out once, not again for each layer (#1519).
 #[derive(Default)]
 struct Systems {
-    names: Vec<String>,
-    candidates: std::cell::OnceCell<Vec<Candidate>>,
+    /// Where a sublayer's own names are numbered from: all the names its parents list, those
+    /// RustGIS leaves out included (only the order matters).
+    names: usize,
+    mercator: bool,
+    first: Vec<Candidate>,
+}
+
+impl Systems {
+    fn with_own(parent: &Systems, own: &[String]) -> Systems {
+        // Each with whether it is known to be usable: the parent's are. Whether one is depends only
+        // on its code, and finding out makes its transformer, so it is asked only of those that
+        // come before the first `MAX_SYSTEMS` usable ones, as `wms_native` did of the whole list.
+        let mut all: Vec<(Candidate, bool)> = parent.first.iter().map(|c| (c.clone(), true)).collect();
+        let inherited = all.len();
+        // (`wms_candidates` gives one name for each system: only the parent's can repeat one.)
+        for mut c in wms_candidates(own) {
+            c.3 += parent.names;
+            // One name for each system, as in `wms_candidates`: CRS:84 over EPSG:4326, else the
+            // first listed. (A system past the parent's first ones comes after them all here too.)
+            match all[..inherited].iter_mut().find(|(p, _)| p.1 == c.1) {
+                Some((p, _)) if warp::is_crs84(&c.0) && !warp::is_crs84(&p.0) => *p = c,
+                Some(_) => {}
+                None => all.push((c, false)),
+            }
+        }
+        all.sort_by_key(|(c, _)| (candidate_rank(c), c.3));
+        Systems {
+            names: parent.names + own.iter().map(|c| c.split_whitespace().count()).sum::<usize>(),
+            mercator: parent.mercator || own.iter().any(|c| c.split_whitespace().any(is_web_mercator)),
+            first: all
+                .into_iter()
+                .filter(|(c, known)| *known || warp::usable(c.1).is_some())
+                .take(warp::MAX_SYSTEMS)
+                .map(|(c, _)| c)
+                .collect(),
+        }
+    }
+}
+
+/// Where a system comes in a WMS layer's order of preference: projected, CRS:84, other
+/// geographic.
+fn candidate_rank((name, _, c, _): &Candidate) -> u8 {
+    match (c.is_geographic(), warp::is_crs84(name)) {
+        (false, _) => 0,
+        (true, true) => 1,
+        (true, false) => 2,
+    }
 }
 
 /// A system of a WMS layer's list that RustGIS knows: its name there, its EPSG code, it, and its
@@ -1179,13 +1224,8 @@ fn wms_candidates(crs: &[String]) -> Vec<Candidate> {
     // One name for each system (EPSG:4326 and CRS:84 differ only in their axes: CRS:84 kept).
     listed.sort_by_cached_key(|(name, code, _, _)| (*code, !warp::is_crs84(name)));
     listed.dedup_by_key(|(_, code, _, _)| *code);
-    let rank = |(name, _, c, _): &Candidate| match (c.is_geographic(), warp::is_crs84(name)) {
-        (false, _) => 0,
-        (true, true) => 1,
-        (true, false) => 2,
-    };
     // In the order the service lists them, the preferred first.
-    listed.sort_by_key(|l| (rank(l), l.3));
+    listed.sort_by_key(|l| (candidate_rank(l), l.3));
     listed
 }
 
@@ -2609,8 +2649,8 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result<Service, String> {
     // are listed.
     let mut three_d: Vec<((&str, bool), Vec<&str>)> = Vec::new();
     // A collection's layers copy the document's own address when their items or tiles links are
-    // missing or relative, and nothing bounds that address: past the listing, reading stops once
-    // the layers' addresses reach `MAX_COPIED_ADDRESS_BYTES`.
+    // missing or relative, and nothing bounds that address: reading stops once the layers'
+    // addresses reach `MAX_COPIED_ADDRESS_BYTES` (after the first collection, as `read_limit`).
     let (mut copied, mut counted) = (0usize, 0);
     let mut read = collections.len().min(MAX_READ_LAYERS);
     for (i, c) in collections.iter().take(MAX_READ_LAYERS).enumerate() {
@@ -2620,7 +2660,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result<Service, String> {
         if i == MAX_LISTED_LAYERS {
             service.listed = Some(service.layers.len());
         }
-        if i >= MAX_LISTED_LAYERS && copied > MAX_COPIED_ADDRESS_BYTES {
+        if i > 0 && copied > MAX_COPIED_ADDRESS_BYTES {
             read = i;
             break;
         }
@@ -2748,7 +2788,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result<Service, String> {
         note_past_read_limit(&mut service, collections.len() - read, "collections", read);
         note_copied_addresses(&mut service, copied.div_ceil(read.max(1)), read);
     }
-    service.incomplete = unreadable > 0 || collections.len() > MAX_LISTED_LAYERS;
+    service.incomplete = unreadable > 0 || collections.len() > MAX_LISTED_LAYERS.min(read);
     if service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })) {
         service.notes.push(format!("a collection is read up to {} features on this computer, page by page", max_paged_features()));
     }
@@ -5803,6 +5843,33 @@ mod tests {
         }
     }
 
+    /// A sublayer's systems, its own added to its parent's first ones, are those its whole list
+    /// gives, so a root's many are worked out once (#1519).
+    #[test]
+    fn a_sublayers_systems_build_on_its_parents_first_ones() {
+        let first = |names: &[String]| -> Vec<(String, u32)> {
+            wms_candidates(names).into_iter().filter(|c| warp::usable(c.1).is_some()).take(warp::MAX_SYSTEMS).map(|c| (c.0, c.1)).collect()
+        };
+        let codes = |r: std::ops::Range<u32>| r.map(|c| format!("EPSG:{c}")).collect::<Vec<_>>();
+        let many = [codes(32601..32671), vec!["EPSG:4326".into(), "EPSG:3857".into()], codes(2000..2100)].concat();
+        let few = vec!["EPSG:4326".to_string(), "EPSG:3857".into()];
+        let owns = [
+            vec!["CRS:84".to_string()],
+            vec!["EPSG:32633".into(), "EPSG:2056".into()],
+            vec!["EPSG:4326 EPSG:31983".into()],
+            codes(32701..32761),
+        ];
+        for root in [&many, &few] {
+            let parent = Systems::with_own(&Systems::default(), root);
+            for own in &owns {
+                let child = Systems::with_own(&parent, own);
+                let whole: Vec<String> = root.iter().chain(own).cloned().collect();
+                assert_eq!(child.first.iter().map(|c| (c.0.clone(), c.1)).collect::<Vec<_>>(), first(&whole), "{own:?}");
+                assert!(child.mercator && !Systems::with_own(&Systems::default(), own).mercator);
+            }
+        }
+    }
+
     #[test]
     fn wms_layers_in_web_mercator() {
         let xml = r#"<?xml version="1.0"?><WMS_Capabilities version="1.3.0" xmlns="http://www.opengis.net/wms" xmlns:xlink="http://www.w3.org/1999/xlink">
@@ -6972,8 +7039,8 @@ mod tests {
             assert_eq!(s.listed, cut.then_some(MAX_LISTED_LAYERS), "{url}");
             assert_eq!(parse(url, doc(MAX_LISTED_LAYERS).as_bytes()).unwrap().listed, None, "{url}");
         }
-        // A long request address, copied into each layer, lowers how many are read, though never
-        // below the listing.
+        // A long request address, copied into each layer, lowers how many are read (a 4 KB one not
+        // below the listing).
         let href = format!("https://x.org/{}?", "a".repeat(4000));
         let types = many(MAX_READ_LAYERS, |i| format!("<FeatureType><Name>t{i}</Name></FeatureType>"));
         let long = format!(
@@ -7003,6 +7070,22 @@ mod tests {
         assert!(s.notes.iter().any(|n| n.starts_with("each of its layers copies 40")), "{:?}", s.notes);
         let s = parse("https://x.org/collections", bare.as_bytes()).unwrap();
         assert_eq!((s.layers.len(), s.past_read_limit), (MAX_READ_LAYERS, None));
+        // An address so long that the listing's copies would pass the budget reads fewer than are
+        // listed (at least one), and the collections are said to be incomplete for a group.
+        let huge = "a".repeat(1 << 20);
+        let few = serde_json::json!({"collections": list(100, |i| serde_json::json!({"id": format!("c{i}")}))}).to_string();
+        let s = parse(&format!("https://x.org/{huge}/collections"), few.as_bytes()).unwrap();
+        let read = s.layers.len();
+        assert!((1..100).contains(&read) && s.incomplete && s.listed.is_none(), "{read}");
+        let note = format!("{PAST_READ_LIMIT} {read} collections from one document: {} left out", 100 - read);
+        assert_eq!(s.past_read_limit.as_ref(), Some(&note));
+        let types = many(100, |i| format!("<FeatureType><Name>t{i}</Name></FeatureType>"));
+        let wfs = format!(
+            r#"<WFS_Capabilities version="2.0.0"><OperationsMetadata><Operation name="GetFeature"><DCP><HTTP><Get href="https://x.org/{huge}?"/></HTTP></DCP></Operation></OperationsMetadata><FeatureTypeList>{types}</FeatureTypeList></WFS_Capabilities>"#
+        );
+        let s = parse("https://x.org/wfs", wfs.as_bytes()).unwrap();
+        assert!((1..100).contains(&s.layers.len()), "{}", s.layers.len());
+        assert!(s.notes.iter().any(|n| n.starts_with("each of its layers copies 10")), "{:?}", s.notes);
         // A WMS is read in document order: the layers past the limit are its last, and a parent
         // comes before its sublayers.
         let s = parse("https://x.org/wms", wms(MAX_READ_LAYERS + 1).as_bytes()).unwrap();

crates/engine/src/cmd/service.rs

@@ -89,9 +89,11 @@ pub(crate) fn without_token(p: &Value) -> Result<Option<Value>, String> {
     Ok(Some(owned))
 }
 
-/// One layer's exact capabilities document and the origin history used to reach it.
+/// One layer's exact capabilities document and the origin history used to reach it. The address
+/// is shared by the document's layers: nothing bounds its length, and a document has up to
+/// 100,000 of them, the most `rustgis_data::ogc` reads (#1519).
 struct CapabilitySource {
-    address: String,
+    address: std::sync::Arc<str>,
     crossed_origin: bool,
 }
 
@@ -124,6 +126,10 @@ impl Discovery {
 
 const MAX_SERVICE_DOCUMENTS: usize = 4;
 
+/// Most bytes of document addresses a listing repeats: each listed layer's `source_url` is its
+/// document's address, which nothing bounds (a landing page links to any collections address).
+const MAX_LISTED_ADDRESS_BYTES: usize = if cfg!(test) { 8 << 20 } else { 64 << 20 };
+
 /// Whether what this `service.add` would wait for is left off the frame: it is a plugin command's,
 /// run by the desktop app, which runs the command again when it has come (#1319).
 fn leaves_waits(s: &Session) -> bool {
@@ -142,8 +148,30 @@ fn capabilities(s: &mut Session, p: &Value) -> Result<Discovery, String> {
 }
 
 /// Reads a service's documents, fetching what `p` doesn't give: waiting for them, or with `left`,
-/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`.
+/// off the frame ([`Session::left_document`]), when a document not in yet is its `next`. Long
+/// document addresses list fewer layers ([`MAX_LISTED_ADDRESS_BYTES`]), said in a note.
 fn discover(s: &mut Session, p: &Value, left: bool) -> Result<Discovery, String> {
+    let mut found = read_documents(s, p, left)?;
+    let listed = found.service.listed.unwrap_or(usize::MAX).min(found.sources.len());
+    let mut bytes = 0usize;
+    let cut = found.sources[..listed].iter().position(|source| {
+        bytes = bytes.saturating_add(source.address.len());
+        bytes > MAX_LISTED_ADDRESS_BYTES
+    });
+    // (At least one, whatever its address.)
+    if let Some((cut, long)) = cut.map(|cut| (cut.max(1), found.sources[cut].address.len())).filter(|&(cut, _)| cut < listed) {
+        found.service.listed = Some(cut);
+        // A service group takes every layer read, not those listed: it can't take part of this.
+        found.incomplete = true;
+        found.service.notes.push(format!(
+            "each listed layer repeats its document's address, {long} bytes long here, and RustGIS lists at most {} MiB of them (at least one layer)",
+            MAX_LISTED_ADDRESS_BYTES >> 20
+        ));
+    }
+    Ok(found)
+}
+
+fn read_documents(s: &mut Session, p: &Value, left: bool) -> Result<Discovery, String> {
     let original = url_of(p)?;
     let mut address =
         crate::render::rustgis_tiles::logins::inherit(original, &rustgis_data::ogc::capabilities_url(original, ps(p, "type")));
@@ -216,9 +244,10 @@ fn discover(s: &mut Session, p: &Value, left: bool) -> Result<Discovery, String>
         }
         // Parsers bound each document's collections/layers; visited documents bound the merge.
         // Distinct offers may share a name (including generated tile names and collection ids).
+        let shared: std::sync::Arc<str> = address.as_str().into();
         for layer in service.layers {
             result.service.layers.push(layer);
-            result.sources.push(CapabilitySource { address: address.clone(), crossed_origin });
+            result.sources.push(CapabilitySource { address: shared.clone(), crossed_origin });
         }
         result.service.withheld_addresses = result.service.withheld_addresses.saturating_add(service.withheld_addresses);
         result.service.past_read_limit = result.service.past_read_limit.take().or(service.past_read_limit);
@@ -684,7 +713,7 @@ fn layer_source(layer: &mut Value, source: CapabilitySource) {
         offer.get("type").and_then(Value::as_str) == Some("features")
             && ps(offer, "url").is_some_and(|url| crosses_origin(&source.address, url))
     });
-    layer["source_url"] = json!(source.address);
+    layer["source_url"] = json!(&*source.address);
     if source.crossed_origin || foreign_features {
         layer["anonymous"] = json!(true);
         if let Some(url) = layer.get_mut("offer").and_then(|o| o.get_mut("url"))
@@ -746,7 +775,7 @@ fn add_discovered(s: &mut Session, p: &Value, url: &str, want: &str, found: &Dis
             }
         })?;
     let capability_crossed = source.crossed_origin;
-    let source = source.address.as_str();
+    let source = &*source.address;
     // A time-enabled layer follows the time slider, at each of its steps, unless a time is asked
     // for: one of its times (the layer named with it), or "default" (`false`), the service's.
     // (`"time": true`, the slider, as for a Zarr variable, `layer.addZarr`'s.)

crates/engine/src/cmd/service_group.rs

@@ -120,7 +120,7 @@ fn listing(s: &mut Session, p: &Value) -> CmdResult {
     }
     if found.incomplete {
         return Err(
-            "the service lists collections RustGIS cannot read (more than 2000, or unreadable addresses); the group was not changed".into(),
+            "the service lists collections RustGIS cannot read (more than 2000, an address too long to read them all, or unreadable addresses); the group was not changed".into(),
         );
     }
     let previous = target(s, p)?;