From fd0a0fd54bc0e745db9f880ccf38b0b64b094302 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 16:53:58 -0300 Subject: [PATCH 1/7] feat(embeddings): AlphaEarth's years on the time slider (embeddings.addCube) (#1720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tiles: search_aef_series / aef_series find each AlphaEarth tile's file of every year in one read of the GeoParquet index, matched by place (each year's export names its files afresh); year_start_ms, aef_place. - engine: embeddings.addCube adds a tile layer per tile with a COG per year in time_urls, the slider on over the years (YYYY), as stac.addCube does; missing tile-years, a partial index read, tiles past a search's list and a second file for a year are said. The slider part of stac.addCube is shared as cube_slider. - map.setTiles gives a new look (bands, range) to every file of a data cube, not only the one shown. - ui: Satellite Embeddings › All years, on the time slider, Add years, and Cancel while a search or read runs. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 267 +++++++++++++++++++++++++++- crates/engine/src/cmd/stac.rs | 75 ++++---- crates/engine/src/cmd/tiles.rs | 10 +- crates/tiles/src/embeddings.rs | 194 +++++++++++++++++++- crates/ui-egui/src/chrome.rs | 2 +- crates/ui-egui/src/embeddingsui.rs | 145 ++++++++++++--- docs/user-guide/web-data.md | 7 +- 7 files changed, 628 insertions(+), 72 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index ee6ce21fd0..ae1bf3ee56 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -6,6 +6,8 @@ //! On the desktop and the command line these commands read what they need, waiting for it; the //! web app reads it off the frame and hands it over (`ran`). +use std::collections::BTreeMap; + use rustgis_data::raster; use rustgis_map::raster::RasterLayer; use serde_json::{Value, json}; @@ -27,6 +29,9 @@ pub fn commands() -> Vec { C::new("embeddings.add", "Add Satellite Embeddings", "Home › Add › Web Data", add).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params( r#"{"dataset": "alphaearth" | "earth-index" | "tessera", "url"?: "AlphaEarth's tile or Earth Index's file, as embeddings.search lists it (Earth Index: default the MGRS tile at the view's centre)", "year"?: number (Tessera, default 2024), "bands"?: [red, green, blue] (bands from 1; AlphaEarth default [2, 17, 10], Tessera [1, 2, 3]), "stretch"?: number (AlphaEarth: dequantised values shown over ±stretch, default 0.3), "dimension"?: number (Earth Index: a vector value kept as each point's `value`, from 1), "bbox"?: [west, south, east, north] | "view" (Earth Index and Tessera: the area read, default the map's extent), "name"?: text, "ran"?: (internal: what the app read)}"#, ), + C::new("embeddings.addCube", "Add Satellite Embeddings over Time", "Home › Add › Web Data", add_cube).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params( + r#"{"dataset": "alphaearth", "bbox"?: [west, south, east, north] | "view" (the area searched, default the map's extent), "start"?: year (default 2017), "end"?: year (default 2025), "bands"?: [red, green, blue] (bands from 1, default [2, 17, 10]), "stretch"?: number (dequantised values shown over ±stretch, default 0.3), "name"?: text, "found"?: (internal: the tiles the app found, each {years: {year: address}})} → a tile layer per AlphaEarth tile, a file per year, following the time slider (turned on over the years when it is off)"#, + ), ] } @@ -134,11 +139,7 @@ fn add(s: &mut Session, p: &Value) -> CmdResult { .filter(|u| u.starts_with("https://")) .ok_or("url is required: an AlphaEarth tile (embeddings.search lists them)")?; let b = bands(p, 64, emb::AEF_DEFAULT_BANDS)?; - let stretch = f(p, "stretch").map(|v| v as f32).unwrap_or(emb::AEF_DEFAULT_STRETCH); - if !(stretch.is_finite() && stretch > 0.0 && stretch <= 1.0) { - return Err("stretch is a number above 0, at most 1".into()); - } - let look = emb::aef_look(b, stretch); + let look = emb::aef_look(b, stretch(p)?); let file = crate::render::rustgis_tiles::cog::split(url).0; // The year from the tile's address (…/annual/2024/10N/…), when it has one. let year = file.split('/').find(|s| s.len() == 4 && s.parse::().is_ok()); @@ -163,6 +164,194 @@ fn add(s: &mut Session, p: &Value) -> CmdResult { } } +/// AlphaEarth's stretch a command names: dequantised values shown over ±stretch. +fn stretch(p: &Value) -> Result { + let stretch = f(p, "stretch").map(|v| v as f32).unwrap_or(emb::AEF_DEFAULT_STRETCH); + if !(stretch.is_finite() && stretch > 0.0 && stretch <= 1.0) { + return Err("stretch is a number above 0, at most 1".into()); + } + Ok(stretch) +} + +/// `embeddings.addCube` (#1720): AlphaEarth's tiles where the map is as a data cube, a year a step +/// of the time slider. Each tile is a tile layer reading its file of the year the slider is at +/// (`time_urls`, as `stac.addCube` makes them); a year a tile has no file for shows nothing, said. +/// The tiles come from one read of the index, which holds every year (the command line waits for +/// it), or from the app's search off the frame (`found`). Natively only the first tile's latest +/// year's header is read before the layers are added, as `embeddings.add` reads one tile's; the +/// rest are read in place as the map draws them. +fn add_cube(s: &mut Session, p: &Value) -> CmdResult { + use crate::render::rustgis_tiles::cog::{self, Look}; + let d = dataset(p)?; + if d.id != "alphaearth" { + return Err(format!("{}: only AlphaEarth's years make a time cube here", d.name)); + } + let (first, last) = d.years.unwrap_or((2017, 2025)); + let year_param = |k: &str, default: i32| match f(p, k) { + None => Ok(default), + Some(y) if y.fract() == 0.0 && y >= f64::from(first) && y <= f64::from(last) => Ok(y as i32), + Some(_) => Err(format!("{k} is a year from {first} to {last}")), + }; + let (start, end) = (year_param("start", first)?, year_param("end", last)?); + if start > end { + return Err("start is a year before end, or the same one".into()); + } + let b = bands(p, 64, emb::AEF_DEFAULT_BANDS)?; + let look = Look { cube: true, ..emb::aef_look(b, stretch(p)?) }; + let given = p.get("found").filter(|v| !v.is_null()); + let series: Vec> = match given { + Some(v) => found(v, start, end)?, + None => search_series(s, p, (start, end))?, + }; + let series: Vec> = series.into_iter().filter(|y| !y.is_empty()).collect(); + if series.is_empty() { + return Err(format!("AlphaEarth has no tiles there for {start}–{end}")); + } + let room = rustgis_map::tiles::MAX_TILE_LAYERS.saturating_sub(s.map().tiles.len()); + if series.len() > room { + return Err(format!( + "{} AlphaEarth tiles, and the map has room for {room} more tile layers (at most {}): take a smaller area", + series.len(), + rustgis_map::tiles::MAX_TILE_LAYERS + )); + } + let base = ps(p, "name") + .map(str::trim) + .filter(|n| !n.is_empty()) + .map_or_else(|| format!("AlphaEarth {start}–{end} ({})", band_label(b)), str::to_string); + // A cube that can't be added whole takes out what it added (as stac.addCube does). + let (tiles_before, layers_before): (Vec, Vec) = + (s.map().tiles.clone(), s.map().layers.iter().map(|l| l.id).collect()); + let undo = |s: &mut Session| { + let m = s.map_mut(); + m.tiles = tiles_before.clone(); + m.layers.retain(|l| layers_before.contains(&l.id)); + }; + // In the browser nothing may wait, nor may the app (it hands its search over) or a plugin's + // command in it (on the frame). + let mut probe = cfg!(not(target_arch = "wasm32")) && !s.leave_downloads && given.is_none(); + let span = usize::try_from(end - start + 1).unwrap_or(1); + let (mut dates, mut missing, many) = (Vec::new(), 0usize, series.len() > 1); + for years in &series { + missing += span.saturating_sub(years.len()); + let urls: Vec<(f64, String)> = years.iter().map(|(y, u)| (emb::year_start_ms(*y), cog::with_look(u, &look))).collect(); + let Some((_, latest)) = urls.last() else { continue }; + let file = cog::split(latest).0.to_string(); + #[cfg(not(target_arch = "wasm32"))] + if probe { + match cog::probe_blocking(&file) { + Ok(true) => {} + Ok(false) => { + undo(s); + return Err(format!("{file}: not a cloud-optimised GeoTIFF")); + } + Err(e) => { + undo(s); + return Err(e); + } + } + } + let name = match (many, emb::aef_place(&file)) { + (true, Some(place)) => format!("{base} · {place}"), + _ => base.clone(), + }; + let r = match super::layer::add_cog(s, &json!({"name": name}), latest, probe) { + Ok(r) => r, + Err(e) => { + undo(s); + return Err(e); + } + }; + probe = false; + let at = r.get("index").and_then(Value::as_u64).and_then(|i| usize::try_from(i).ok()); + match at.and_then(|i| s.map_mut().tiles.get_mut(i)).filter(|t| cog::split(&t.url).0 == file) { + Some(t) => t.time_urls = urls.clone(), + None => { + undo(s); + return Err(format!("{file}: not added as a tile layer that follows the time slider")); + } + } + dates.extend(urls.iter().map(|(d, _)| *d)); + } + if let Err(e) = super::stac::cube_slider(s, &mut dates, &json!({"unit": "year", "format": "YYYY"})) { + undo(s); + return Err(e); + } + if missing > 0 { + s.notify( + format!( + "{base}: {missing} of the {} tile-years have no file in AlphaEarth's index; in those years the tile shows nothing", + series.len() * span + ), + false, + ); + } + Ok(json!({"tiles": series.len(), "years": dates.len(), "missing": missing})) +} + +/// The tiles the app found and hands over (`found`): each a year's file by year, kept to the +/// years asked for. +fn found(v: &Value, start: i32, end: i32) -> Result>, String> { + let bad = || "found is a list of tiles, each {\"years\": {\"2017\": an https address, …}}".to_string(); + let list = v.as_array().ok_or_else(bad)?; + if list.len() > emb::MAX_TILES { + return Err(format!("found holds {} tiles, more than the {} a search lists", list.len(), emb::MAX_TILES)); + } + list.iter() + .map(|t| { + let mut out = BTreeMap::new(); + for (y, u) in t.get("years").and_then(Value::as_object).ok_or_else(bad)? { + let y: i32 = y.trim().parse().map_err(|_| bad())?; + let u = u.as_str().map(str::trim).filter(|u| u.starts_with("https://") && u.len() <= 4000).ok_or_else(bad)?; + if (start..=end).contains(&y) { + out.insert(y, u.to_string()); + } + } + Ok(out) + }) + .collect() +} + +/// AlphaEarth's tiles over the years where the command says, from one read of the index, waiting +/// for it (the command line), with what the read left out said. +fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result>, String> { + let area = bbox(s, p)?; + #[cfg(target_arch = "wasm32")] + { + let _ = (area, years); + Err("in the web app, add AlphaEarth's years with Home › Add › Web Data › Satellite Embeddings".into()) + } + #[cfg(not(target_arch = "wasm32"))] + { + // A plugin's command in the app runs on the frame, where nothing may wait for the index. + if s.leave_downloads { + return Err( + "AlphaEarth's index is read off the frame here: add its years from Home › Add › Web Data › Satellite Embeddings".into() + ); + } + let f = emb::search_aef_series_blocking(area, years)?; + series_notices(s, &f); + Ok(f.series.into_iter().map(|x| x.years).collect()) + } +} + +/// What a search over the years left out: rows past one read, tiles past a search's list, and a +/// tile's second file for a year. +fn series_notices(s: &mut Session, f: &emb::SeriesFound) { + if !f.complete { + s.notify("AlphaEarth: the area holds more tiles than one search reads; zoom in to take them all", false); + } + if f.dropped > 0 { + s.notify( + format!("AlphaEarth: {} tiles past the {} a search lists were left out; zoom in for them", f.dropped, emb::MAX_TILES), + false, + ); + } + if f.duplicates > 0 { + s.notify(format!("AlphaEarth: {} tile-years had a second file in the index; the first was taken", f.duplicates), false); + } +} + /// Bytes the app read and handed over, by their id (`ran`). fn handed(s: &mut Session, p: &Value) -> Result)>, String> { let Some(id) = f(p, "ran").filter(|v| *v >= 0.0 && v.fract() == 0.0).map(|v| v as u64) else { return Ok(None) }; @@ -313,6 +502,74 @@ mod tests { } assert_eq!(bands(&json!({"bands": [2, 17, 10]}), 64, [0, 0, 0]), Ok([1, 16, 9])); assert_eq!(band_label([1, 16, 9]), "2, 17, 10"); + let tile = json!([{"years": {"2017": "https://x.org/2017/23S/a-1-2.tiff"}}]); + for bad in [ + json!({"dataset": "tessera", "found": tile}), + json!({"dataset": "clay", "found": tile}), + json!({"dataset": "alphaearth", "start": 2016, "found": tile}), + json!({"dataset": "alphaearth", "end": 2024.5, "found": tile}), + json!({"dataset": "alphaearth", "start": 2024, "end": 2020, "found": tile}), + json!({"dataset": "alphaearth", "stretch": 2, "found": tile}), + json!({"dataset": "alphaearth", "bands": [1, 2], "found": tile}), + json!({"dataset": "alphaearth", "found": "tiles"}), + json!({"dataset": "alphaearth", "found": [{"years": {"2017": "file:///etc/passwd"}}]}), + json!({"dataset": "alphaearth", "found": [{"years": {"later": "https://x.org/a.tiff"}}]}), + json!({"dataset": "alphaearth", "found": [{"files": []}]}), + // Nothing in the years asked for. + json!({"dataset": "alphaearth", "start": 2020, "found": tile}), + json!({"dataset": "alphaearth", "found": vec![json!({"years": {}}); emb::MAX_TILES + 1]}), + ] { + assert!(s.run("embeddings.addCube", &bad).is_err(), "{bad}"); + } + assert!(s.map().tiles.is_empty() && s.map().time.is_none(), "a refused cube leaves the map as it was"); + } + + /// The tiles the app found make a cube: a tile layer each, a file a year in the slider's order, + /// the slider on over the years; the years a tile has no file for are said. + #[test] + fn alphaearth_years_on_the_time_slider() { + let mut s = Session::new(Project::default()); + let file = |y: i32, place: &str| format!("https://127.0.0.1:9/annual/{y}/23S/x{y}-{place}.tiff"); + let (a, b) = ("0000008192-0000000000", "0000000000-0000000000"); + let found = json!([ + {"years": {"2025": file(2025, a), "2017": file(2017, a), "2016": file(2016, a)}}, + {"years": {"2020": file(2020, b)}}, + ]); + let r = s.run("embeddings.addCube", &json!({"dataset": "alphaearth", "found": found, "bands": [1, 2, 3]})).unwrap(); + assert_eq!((r["tiles"].as_u64(), r["years"].as_u64(), r["missing"].as_u64()), (Some(2), Some(3), Some(15))); + assert_eq!(s.map().tiles.len(), 2); + let named = |s: &Session, place: &str| s.map().tiles.iter().find(|t| t.name.ends_with(place)).cloned().unwrap(); + let first = named(&s, a); + assert_eq!(first.name, format!("AlphaEarth 2017–2025 (1, 2, 3) · 23S {a}")); + let dates: Vec = first.time_urls.iter().map(|(d, _)| *d).collect(); + assert_eq!(dates, vec![emb::year_start_ms(2017), emb::year_start_ms(2025)], "2016 is outside the years asked for"); + let (f, look) = crate::render::rustgis_tiles::cog::split(&first.time_urls[0].1); + assert_eq!(f, file(2017, a)); + assert!(look.aef && look.cube && look.bands == vec![0, 1, 2], "{look:?}"); + let t = s.map().time.clone().unwrap(); + assert_eq!(t.dates, vec![emb::year_start_ms(2017), emb::year_start_ms(2020), emb::year_start_ms(2025)]); + assert_eq!(t.format, "YYYY"); + assert!(s.notices.iter().any(|n| n.text.contains("15 of the 18 tile-years have no file"))); + // The slider shows each tile's file of its year, or nothing in a year it has none for. + s.run("time.goTo", &json!({"step": 1})).unwrap(); + assert!(named(&s, a).time_hidden, "no file in 2020"); + let other = named(&s, b); + assert!(!other.time_hidden && other.url.starts_with(&file(2020, b)), "{}", other.url); + // Every year there: nothing said. + let mut t = Session::new(Project::default()); + let one = json!([{"years": {"2024": file(2024, "1-2"), "2025": file(2025, "1-2")}}]); + t.run("embeddings.addCube", &json!({"dataset": "alphaearth", "start": 2024, "found": one, "name": "AEF"})).unwrap(); + assert_eq!(t.map().tiles.first().map(|l| l.name.as_str()), Some("AEF")); + assert!(!t.notices.iter().any(|n| n.text.contains("tile-years"))); + // Other bands (Apply to layers) hold at every year, not only the one shown. + t.run("map.setTiles", &json!({"index": 0, "bands": [3, 4, 5]})).unwrap(); + t.run("time.goTo", &json!({"step": 0})).unwrap(); + let tile = t.map().tiles.first().cloned().unwrap(); + assert!(tile.url.starts_with(&file(2024, "1-2")), "{}", tile.url); + for (_, u) in tile.time_urls.iter().chain([&(0.0, tile.url.clone())]) { + let look = crate::render::rustgis_tiles::cog::split(u).1; + assert!(look.bands == vec![2, 3, 4] && look.aef && look.cube, "{u}"); + } } /// A plugin's command in the app (`leave_downloads`) doesn't read an AlphaEarth tile's header diff --git a/crates/engine/src/cmd/stac.rs b/crates/engine/src/cmd/stac.rs index fe90d01dc4..6c70ac6140 100644 --- a/crates/engine/src/cmd/stac.rs +++ b/crates/engine/src/cmd/stac.rs @@ -1098,42 +1098,7 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { } } } - dates.sort_by(f64::total_cmp); - dates.dedup(); - // At most MAX_CUBE_ITEMS dates, well under the slider's steps; were it ever more, the cube - // is refused rather than left with files the slider can't reach. - if dates.len() > rustgis_map::time::MAX_STEPS { - undo(s); - return Err(format!("{} dates; the time slider takes {} at most", dates.len(), rustgis_map::time::MAX_STEPS)); - } - // The slider over the cube's dates: on over them when it is off; when it is on, taking - // them in (its own dates and the cube's, or its span widened to theirs), so every file - // can be reached. - let slider = match s.map().time.as_ref() { - None => { - let d: Vec = dates.iter().map(|d| json!(d)).collect(); - json!({"dates": d}) - } - Some(t) if !t.dates.is_empty() => { - let mut all: Vec = t.dates.iter().chain(&dates).copied().collect(); - all.sort_by(f64::total_cmp); - all.dedup(); - if all.len() > rustgis_map::time::MAX_STEPS { - undo(s); - return Err(format!( - "the time slider's dates and the cube's make {}, more than its {} steps: take a shorter period", - all.len(), - rustgis_map::time::MAX_STEPS - )); - } - json!({"dates": all, "date": t.current}) - } - Some(t) => { - let (first, last) = (dates.first().copied().unwrap_or(t.start), dates.last().copied().unwrap_or(t.end)); - json!({"start": t.start.min(first), "end": t.end.max(last), "date": t.current}) - } - }; - if let Err(e) = super::time::set(s, &slider) { + if let Err(e) = cube_slider(s, &mut dates, &json!({})) { undo(s); return Err(e); } @@ -1148,6 +1113,44 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { } } +/// Turns the time slider on over a data cube's dates (sorted and deduplicated here), or, when it +/// is on, takes them in (its own dates and the cube's, or its span widened to theirs), so every +/// file can be reached. `fresh`: what else a slider turned on here is set to (AlphaEarth's years +/// shown as years). Refused, the slider unchanged, when the dates are more than its steps: a cube +/// left with files the slider can't reach would say nothing of them. +pub(crate) fn cube_slider(s: &mut Session, dates: &mut Vec, fresh: &Value) -> Result<(), String> { + dates.sort_by(f64::total_cmp); + dates.dedup(); + if dates.len() > rustgis_map::time::MAX_STEPS { + return Err(format!("{} dates; the time slider takes {} at most", dates.len(), rustgis_map::time::MAX_STEPS)); + } + let slider = match s.map().time.as_ref() { + None => { + let mut q = if fresh.is_object() { fresh.clone() } else { json!({}) }; + q["dates"] = json!(dates); + q + } + Some(t) if !t.dates.is_empty() => { + let mut all: Vec = t.dates.iter().chain(dates.iter()).copied().collect(); + all.sort_by(f64::total_cmp); + all.dedup(); + if all.len() > rustgis_map::time::MAX_STEPS { + return Err(format!( + "the time slider's dates and the cube's make {}, more than its {} steps: take a shorter period", + all.len(), + rustgis_map::time::MAX_STEPS + )); + } + json!({"dates": all, "date": t.current}) + } + Some(t) => { + let (first, last) = (dates.first().copied().unwrap_or(t.start), dates.last().copied().unwrap_or(t.end)); + json!({"start": t.start.min(first), "end": t.end.max(last), "date": t.current}) + } + }; + super::time::set(s, &slider).map(|_| ()) +} + #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use rustgis_data::raster::GeoTransform; diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 3eb8549944..c6ac8f7922 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -48,6 +48,7 @@ pub const NETWORK: &[&str] = &[ "portal.files", "embeddings.search", "embeddings.add", + "embeddings.addCube", "earthdata.collections", "earthdata.granules", "nldi.flowline", @@ -982,9 +983,14 @@ fn set(s: &mut Session, p: &Value) -> CmdResult { t.name = r.chars().take(120).collect(); } if let Some(url) = new_url { - // A dated layer keeps the look at every date: its template takes it too. + // A dated layer keeps the look at every date: its template takes it too, and a data + // cube's file of each date (the slider would bring the old look back at the next step). + let look = cog::split(&url).1; if let Some(template) = t.time_url.as_mut() { - *template = cog::with_look(template, &cog::split(&url).1); + *template = cog::with_look(template, &look); + } + for (_, u) in &mut t.time_urls { + *u = cog::with_look(u, &look); } t.url = url; } diff --git a/crates/tiles/src/embeddings.rs b/crates/tiles/src/embeddings.rs index 98611f9690..3ae8e2cd52 100644 --- a/crates/tiles/src/embeddings.rs +++ b/crates/tiles/src/embeddings.rs @@ -17,6 +17,8 @@ //! Nothing here waits on the network: searches and reads take a callback (the app's, off the //! frame), with blocking forms for the command line. +use std::collections::{BTreeMap, HashMap}; + use rustgis_data::source::{Batch, Request, parquet::Vectors}; use rustgis_data::{Dataset, Feature, Field, FieldType, Geometry, GeometryType, Value}; use rustgis_geom::{Coord, Extent}; @@ -273,6 +275,14 @@ fn year_of(t: i64) -> i32 { /// The tiles of a year in what the index read, nearest the box's centre first (at most /// [`MAX_TILES`]). pub fn aef_tiles(fields: &[Field], batch: &Batch, year: i32, bbox: [f64; 4]) -> Vec { + let mut out = aef_rows(fields, batch, bbox, |y| y == year); + out.dedup_by(|a, b| a.1.url == b.1.url); + out.into_iter().take(MAX_TILES).map(|(_, t)| t).collect() +} + +/// Every tile of the years `keep` takes in what the index read, each titled with its own year, +/// with its squared distance from the box's centre, nearest first. +fn aef_rows(fields: &[Field], batch: &Batch, bbox: [f64; 4], keep: impl Fn(i32) -> bool) -> Vec<(f64, Tile)> { let col = |n: &str| fields.iter().position(|f| f.name == n); let (href, epsg, date, id) = (col(AEF_HREF), col("proj:epsg"), col("datetime"), col("id")); let text = |f: &Feature, c: Option| c.and_then(|c| f.attrs.get(c)).map(|v| v.to_string()).unwrap_or_default(); @@ -281,12 +291,12 @@ pub fn aef_tiles(fields: &[Field], batch: &Batch, year: i32, bbox: [f64; 4]) -> .features .iter() .filter_map(|f| { - let y = date.and_then(|c| f.attrs.get(c)).and_then(|v| match v { + let year = date.and_then(|c| f.attrs.get(c)).and_then(|v| match v { Value::Int(t) => Some(year_of(*t)), Value::Text(t) => t.get(..4).and_then(|y| y.parse().ok()), _ => None, })?; - if y != year { + if !keep(year) { return None; } let url = https(&text(f, href)); @@ -327,8 +337,100 @@ pub fn aef_tiles(fields: &[Field], batch: &Batch, year: i32, bbox: [f64; 4]) -> }) .collect(); out.sort_by(|a, b| a.0.total_cmp(&b.0)); - out.dedup_by(|a, b| a.1.url == b.1.url); - out.into_iter().take(MAX_TILES).map(|(_, t)| t).collect() + out +} + +/// One AlphaEarth tile over the years. Each year's export names its files afresh (a prefix of +/// its own: `xdwdu82ch9yffuj03-0000008192-0000000000.tiff` in 2017, +/// `x1i7lpngfblt9wshv-0000008192-0000000000.tiff` in 2024), so a tile's years are matched by +/// place: its coordinate system and footprint. +#[derive(Clone, Debug, PartialEq)] +pub struct Series { + /// The tile as a card shows it: its latest year's footprint, titled with its years. + pub tile: Tile, + /// Each year's file. + pub years: BTreeMap, +} + +/// What a search over the years found. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SeriesFound { + /// The tiles, nearest the box's centre first (at most [`MAX_TILES`]). + pub series: Vec, + /// Tiles past [`MAX_TILES`], left out. + pub dropped: usize, + /// Rows left out because their tile already had another file that year. + pub duplicates: usize, + /// Whether the index was read whole for the box. + pub complete: bool, +} + +/// A tile's place, the same every year: its coordinate system and its footprint's corners to +/// about a metre. +fn place_key(t: &Tile) -> String { + let crs = t.detail.split(" · ").next().unwrap_or(""); + let b = t.bbox.map(|v| (v * 1e5).round() as i64); + format!("{crs} {} {} {} {}", b[0], b[1], b[2], b[3]) +} + +/// The tiles of the years `first` to `last` in what the index read, each with its file a year, +/// nearest the box's centre first. +pub fn aef_series(fields: &[Field], batch: &Batch, (first, last): (i32, i32), bbox: [f64; 4]) -> SeriesFound { + let mut found = SeriesFound { complete: batch.complete, ..Default::default() }; + let mut at: HashMap = HashMap::new(); + for (_, t) in aef_rows(fields, batch, bbox, |y| (first..=last).contains(&y)) { + let Some(year) = t.year else { continue }; + let i = *at.entry(place_key(&t)).or_insert_with(|| { + found.series.push(Series { tile: t.clone(), years: BTreeMap::new() }); + found.series.len() - 1 + }); + let Some(s) = found.series.get_mut(i) else { continue }; + match s.years.get(&year) { + Some(u) if *u != t.url => found.duplicates += 1, + Some(_) => {} + None => { + if s.tile.year.is_none_or(|y| year > y) { + s.tile = t.clone(); + } + s.years.insert(year, t.url); + } + } + } + for s in &mut found.series { + let (a, z) = (s.years.keys().next().copied().unwrap_or(first), s.years.keys().next_back().copied().unwrap_or(last)); + let zone = s.tile.title.split(" · ").nth(1).map(|z| format!(" · {z}")).unwrap_or_default(); + let crs = s.tile.detail.split(" · ").next().unwrap_or("").to_string(); + s.tile.title = if a == z { format!("AlphaEarth {a}{zone}") } else { format!("AlphaEarth {a}–{z}{zone}") }; + s.tile.detail = format!("{crs} · 64 bands · 10 m · {} of {} years", s.years.len(), last - first + 1); + s.tile.year = None; + } + found.dropped = found.series.len().saturating_sub(MAX_TILES); + found.series.truncate(MAX_TILES); + found +} + +/// The first moment of a year, in milliseconds since 1970 (UTC): a time slider's step for it. +pub fn year_start_ms(year: i32) -> f64 { + // Howard Hinnant's days_from_civil for 1 January, which it counts as month 11 of the year + // before (March is its month 1). + let y = i64::from(year) - 1; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * 10 + 2) / 5; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146_097 + doe - 719_468; + days as f64 * 86_400_000.0 +} + +/// An AlphaEarth file's place, as a layer's name gives it: its UTM zone's folder and its tile's +/// row and column (`23S 0000008192-0000000000` from `…/2024/23S/x1i7…-0000008192-0000000000.tiff`). +pub fn aef_place(url: &str) -> Option { + let path = url.split(['?', '#']).next().unwrap_or(url); + let mut parts = path.rsplit('/'); + let file = parts.next()?.trim_end_matches(".tiff").trim_end_matches(".tif"); + let zone = parts.next().filter(|z| !z.is_empty())?; + let (_, place) = file.split_once('-')?; + Some(format!("{zone} {place}")) } /// What a search found: the tiles, and whether the index was read whole for the box (a box @@ -364,6 +466,30 @@ pub fn search_aef_blocking(bbox: [f64; 4], year: i32) -> Result { Ok((aef_tiles(&fields, &batch, year, bbox), batch.complete)) } +/// Finds AlphaEarth's tiles in a box over the years `first` to `last`, each with its file a year: +/// the same one read of the index as [`search_aef`] (its rows hold every year). +pub fn search_aef_series(bbox: [f64; 4], years: (i32, i32), done: Box) + Send>) { + let bbox = match check_bbox(bbox) { + Ok(b) => b, + Err(e) => return done(Err(e)), + }; + rustgis_data::source::query_url( + AEF_INDEX, + vec![AEF_HREF.into()], + aef_request(bbox), + Box::new(move |r| done(r.map(|(fields, batch)| aef_series(&fields, &batch, years, bbox)).map_err(|e| e.to_string()))), + ); +} + +/// [`search_aef_series`], waiting for it (the command line). +#[cfg(not(target_arch = "wasm32"))] +pub fn search_aef_series_blocking(bbox: [f64; 4], years: (i32, i32)) -> Result { + let bbox = check_bbox(bbox)?; + let (fields, batch) = + rustgis_data::source::query_url_blocking(AEF_INDEX, vec![AEF_HREF.into()], aef_request(bbox)).map_err(|e| e.to_string())?; + Ok(aef_series(&fields, &batch, years, bbox)) +} + // Earth Index. /// Where Earth Index's files are. @@ -801,6 +927,66 @@ mod tests { assert!(aef_tiles(&[], &batch, 2021, [0.0, 0.0, 1.0, 1.0]).is_empty()); } + /// A tile's years are matched by place, not by name (each year's export names its files + /// afresh); another place is another tile, a second file for a tile's year is counted. + #[test] + fn index_rows_to_series() { + let fields = vec![ + Field::new("id", FieldType::Text), + Field::new("datetime", FieldType::Integer), + Field::new(AEF_HREF, FieldType::Text), + Field::new("proj:epsg", FieldType::Text), + ]; + let row = |x: f64, year: i32, href: &str| Feature { + id: 1, + geometry: Some(Geometry::Polygon(vec![vec![vec![ + Coord::new(x, -21.0), + Coord::new(x + 0.8, -21.0), + Coord::new(x + 0.8, -20.7), + Coord::new(x, -20.7), + Coord::new(x, -21.0), + ]]])), + attrs: vec![ + Value::Text("1".into()), + Value::Int(year_start_ms(year) as i64), + Value::Text(href.into()), + Value::Text("EPSG:32723".into()), + ], + }; + let base = "s3://us-west-2.opendata.source.coop/tge-labs/aef/v1/annual"; + let batch = Batch { + features: vec![ + row(-43.4, 2024, &format!("{base}/2024/23S/xb-0000008192-0000000000.tiff")), + row(-43.4, 2017, &format!("{base}/2017/23S/xa-0000008192-0000000000.tiff")), + row(-43.4, 2025, &format!("{base}/2025/23S/xc-0000008192-0000000000.tiff")), + // The same tile and year twice: the first kept, the other counted. + row(-43.4, 2025, &format!("{base}/2025/23S/xd-0000008192-0000000000.tiff")), + // Another place; and a year outside those asked for. + row(-44.2, 2020, &format!("{base}/2020/23S/xe-0000000000-0000000000.tiff")), + row(-43.4, 2016, &format!("{base}/2016/23S/xf-0000008192-0000000000.tiff")), + ], + matched: 6, + complete: false, + skipped: 0, + }; + let f = aef_series(&fields, &batch, (2017, 2025), [-43.0, -20.9, -42.9, -20.8]); + assert_eq!((f.series.len(), f.duplicates, f.dropped, f.complete), (2, 1, 0, false)); + let s = &f.series[0]; + assert_eq!(s.years.keys().copied().collect::>(), vec![2017, 2024, 2025]); + assert_eq!(s.years[&2025], "https://data.source.coop/tge-labs/aef/v1/annual/2025/23S/xc-0000008192-0000000000.tiff"); + assert_eq!((s.tile.title.as_str(), s.tile.year), ("AlphaEarth 2017–2025 · UTM 23S", None)); + assert_eq!(s.tile.detail, "EPSG:32723 · 64 bands · 10 m · 3 of 9 years"); + assert_eq!(s.tile.id, "xc-0000008192-0000000000", "the latest year's file stands for the tile"); + assert_eq!(f.series[1].tile.title, "AlphaEarth 2020 · UTM 23S"); + assert_eq!(aef_place(&s.years[&2017]).as_deref(), Some("23S 0000008192-0000000000")); + assert_eq!(aef_place("https://h/x.tiff#bands=1"), None); + assert_eq!(year_start_ms(1970), 0.0); + assert_eq!(year_start_ms(2017), 1_483_228_800_000.0); + assert_eq!(year_start_ms(2024), 1_704_067_200_000.0); + assert_eq!(year_of(year_start_ms(2000) as i64), 2000); + assert_eq!(aef_series(&[], &batch, (2017, 2025), [0.0, 0.0, 1.0, 1.0]).series, Vec::new()); + } + #[test] fn principal_component_colours() { // Two clusters along one direction: their first component tells them apart. diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index be97db7f94..e1d567bb1c 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -877,7 +877,7 @@ fn search_action(id: &str) -> Option { "osm.download" => Dialog("osm"), "overture.add" => Dialog("overture"), "boundaries.add" => Dialog("boundaries"), - "embeddings.add" | "embeddings.search" => Dialog("embeddings"), + "embeddings.add" | "embeddings.addCube" | "embeddings.search" => Dialog("embeddings"), "cube.catalog" | "cube.open" => Dialog("dataCubes"), "nationalMap.add" => Dialog("nationalMap"), "fieldsOfTheWorld.add" => Dialog("fieldsOfTheWorld"), diff --git a/crates/ui-egui/src/embeddingsui.rs b/crates/ui-egui/src/embeddingsui.rs index 63e14f98c9..7e6399552a 100644 --- a/crates/ui-egui/src/embeddingsui.rs +++ b/crates/ui-egui/src/embeddingsui.rs @@ -2,11 +2,15 @@ //! resolution, years and links), its tiles found where the map is (footprints drawn), and one //! shown on the map: AlphaEarth's three chosen bands as red, green and blue with a stretch, //! Earth Index's points coloured by their vectors' principal components, Tessera's three bands of -//! the view as a raster. Searches and reads go off the frame; what they read is handed to the -//! `embeddings.*` commands, so the change is undoable like any other. +//! the view as a raster. AlphaEarth's tiles can also be found over all its years and shown on +//! the time slider, a year a step (`embeddings.addCube`). Searches and reads go off the frame, +//! with Cancel; what they read is handed to the `embeddings.*` commands, so the change is +//! undoable like any other. + +use std::collections::BTreeMap; use egui::vec2; -use rustgis_engine::render::rustgis_tiles::embeddings::{self as emb, EmbeddingDataset, Found, Tile, tessera}; +use rustgis_engine::render::rustgis_tiles::embeddings::{self as emb, EmbeddingDataset, Found, SeriesFound, Tile, tessera}; use rustgis_engine::render::rustgis_tiles::{cache, cog}; use serde_json::json; @@ -15,6 +19,8 @@ use crate::{RustGisApp, Slot}; /// A read under way: a search, or a tile's data for the map. enum Read { Search(Slot>), + /// AlphaEarth's tiles over its years. + Series(Slot>), /// Bytes to hand to `embeddings.add` (Earth Index's points as GeoJSON, Tessera's GeoTIFF), /// and what they were read for, as the read began. Data(Slot, String>>, Request), @@ -43,11 +49,15 @@ pub struct State { pub tessera_bands: [usize; 3], /// Earth Index: a vector value kept as each point's `value` (0-based), if any. pub dimension: Option, + /// AlphaEarth: search its tiles over all its years, each added as a cube on the time slider. + pub all_years: bool, reading: Option, /// The search's box and dataset. /// The search's box and dataset, and the project it was for (`Session::opened`). searched: Option<([f64; 4], &'static str, u64)>, pub found: Vec, + /// Each tile found's file a year, by `found`'s order, when the search was over the years. + years: Vec>, /// The footprints layer of the last search, and its project (`Session::opened`): layer ids /// are a map's own, so another project's may be the same. footprints: Option<(u32, u64)>, @@ -64,9 +74,11 @@ impl Default for State { stretch: emb::AEF_DEFAULT_STRETCH, tessera_bands: tessera::DEFAULT_BANDS, dimension: None, + all_years: false, reading: None, searched: None, found: Vec::new(), + years: Vec::new(), footprints: None, error: None, info: None, @@ -93,6 +105,7 @@ impl State { } self.dataset = d.id; self.found.clear(); + self.years.clear(); self.info = None; self.error = None; if let Some((a, b)) = d.years { @@ -110,11 +123,34 @@ fn view_bbox(app: &RustGisApp) -> Result<[f64; 4], String> { emb::check_bbox([e.min_x, e.min_y, e.max_x, e.max_y]) } +/// What a search listed: its tiles, and for a search over the years each one's files and what +/// the search left out. +#[derive(Default)] +struct Listed { + tiles: Vec, + complete: bool, + years: Vec>, + dropped: usize, + duplicates: usize, +} + +impl Listed { + fn of_series(f: SeriesFound) -> Listed { + let (tiles, years) = f.series.into_iter().map(|s| (s.tile, s.years)).unzip(); + Listed { tiles, complete: f.complete, years, dropped: f.dropped, duplicates: f.duplicates } + } +} + /// Lands a search or a read. pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { let landed = match &app.ui.embeddings.reading { None => return, - Some(Read::Search(slot)) => slot.lock().unwrap_or_else(|e| e.into_inner()).take().map(Ok), + Some(Read::Search(slot)) => slot + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take() + .map(|r| Ok(r.map(|(tiles, complete)| Listed { tiles, complete, ..Default::default() }))), + Some(Read::Series(slot)) => slot.lock().unwrap_or_else(|e| e.into_inner()).take().map(|r| Ok(r.map(Listed::of_series))), Some(Read::Data(slot, _)) => slot.lock().unwrap_or_else(|e| e.into_inner()).take().map(Err), }; let Some(landed) = landed else { @@ -124,7 +160,7 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { let read = app.ui.embeddings.reading.take(); match landed { Ok(found) => match found { - Ok((tiles, complete)) => { + Ok(Listed { tiles, complete, years, dropped, duplicates }) => { let st = &mut app.ui.embeddings; // A search for a dataset no longer picked: its tiles aren't this one's. // Or for a project since closed. @@ -132,7 +168,7 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { if st.searched.is_some_and(|(_, d, o)| d != st.dataset || o != opened) { return; } - st.info = Some(if tiles.is_empty() { + let mut info = if tiles.is_empty() { tl!("Nothing found where the map is").to_string() } else if complete { crate::i18n::fmt("{n} found where the map is", &[("n", &tiles.len().to_string())]) @@ -141,8 +177,19 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { "{n} found; the map holds more than one search reads: zoom in for the rest", &[("n", &tiles.len().to_string())], ) - }); + }; + if dropped > 0 { + info.push_str(&crate::i18n::fmt(" · {n} more left out: zoom in for them", &[("n", &dropped.to_string())])); + } + if duplicates > 0 { + info.push_str(&crate::i18n::fmt( + " · {n} tile-years had a second file; the first was taken", + &[("n", &duplicates.to_string())], + )); + } + st.info = Some(info); st.found = tiles; + st.years = years; show_footprints(app); } Err(e) => app.ui.embeddings.error = Some(e), @@ -219,7 +266,22 @@ fn search(app: &mut RustGisApp) { st.error = None; st.info = None; st.found.clear(); + st.years.clear(); st.searched = Some((bbox, st.dataset, app.session.opened)); + if st.dataset == "alphaearth" && st.all_years { + let years = picked(st).years.unwrap_or((st.year, st.year)); + let slot: Slot> = Slot::default(); + let land = slot.clone(); + emb::search_aef_series( + bbox, + years, + Box::new(move |r| { + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(r); + }), + ); + st.reading = Some(Read::Series(slot)); + return; + } let slot: Slot> = Slot::default(); let land = slot.clone(); let done: Box) + Send> = Box::new(move |r| { @@ -241,8 +303,21 @@ fn add(app: &mut RustGisApp, i: usize) { match tile.dataset { "alphaearth" => { let st = &app.ui.embeddings; - let p = json!({"dataset": "alphaearth", "url": tile.url, "bands": st.aef_bands.map(|b| b + 1), "stretch": st.stretch}); - match app.run("embeddings.add", p) { + let (bands, stretch) = (st.aef_bands.map(|b| b + 1), st.stretch); + // A tile found over the years: its files, handed over, on the time slider. + let (id, p) = match st.years.get(i).filter(|y| !y.is_empty()) { + Some(years) => { + let years: serde_json::Map = years.iter().map(|(y, u)| (y.to_string(), json!(u))).collect(); + let (first, last) = picked(st).years.unwrap_or((st.year, st.year)); + let found = json!([{"years": years}]); + ( + "embeddings.addCube", + json!({"dataset": "alphaearth", "found": found, "start": first, "end": last, "bands": bands, "stretch": stretch}), + ) + } + None => ("embeddings.add", json!({"dataset": "alphaearth", "url": tile.url, "bands": bands, "stretch": stretch})), + }; + match app.run(id, p) { Ok(_) => app.map.invalidate(), Err(e) => app.ui.embeddings.error = Some(e), } @@ -402,17 +477,27 @@ pub fn pane(app: &mut RustGisApp, ui: &mut egui::Ui) { && last > first { let input_label = ui.label(tl!("Year")); - egui::ComboBox::from_id_salt("emb_year") - .selected_text(st.year.to_string()) - .width(w) - .show_ui(ui, |ui| { - for y in (first..=last).rev() { - ui.selectable_value(&mut st.year, y, y.to_string()); - } - }) - .response - .labelled_by(input_label.id); + let all = d.id == "alphaearth" && st.all_years; + ui.add_enabled_ui(!all, |ui| { + egui::ComboBox::from_id_salt("emb_year") + .selected_text(if all { format!("{first}–{last}") } else { st.year.to_string() }) + .width(w) + .show_ui(ui, |ui| { + for y in (first..=last).rev() { + ui.selectable_value(&mut st.year, y, y.to_string()); + } + }) + .response + .labelled_by(input_label.id); + }); ui.end_row(); + if d.id == "alphaearth" { + ui.label(""); + ui.checkbox(&mut st.all_years, tl!("All years, on the time slider")).on_hover_text(tl!( + "Find each tile's file of every year; Add years shows them as one layer that follows the time slider" + )); + ui.end_row(); + } } match d.id { "alphaearth" => { @@ -474,15 +559,26 @@ pub fn pane(app: &mut RustGisApp, ui: &mut egui::Ui) { search(app); } let st = &app.ui.embeddings; + let mut cancel = false; if busy { ui.horizontal(|ui| { ui.spinner(); ui.label(match st.reading { - Some(Read::Search(_)) => tl!("Searching…"), + Some(Read::Search(_) | Read::Series(_)) => tl!("Searching…"), _ => tl!("Reading the embeddings…"), }); + cancel = ui.button(tl!("Cancel")).clicked(); }); } + if cancel { + // The read runs on, but what it brings is dropped with its slot. + if let Some(Read::Data(_, req)) = app.ui.embeddings.reading.take() { + app.session.download_failed(req.id); + } + app.ui.embeddings.reading = None; + app.ui.embeddings.info = Some(tl!("Cancelled").to_string()); + } + let st = &app.ui.embeddings; if let Some(e) = &st.error { ui.label(egui::RichText::new(e).color(t.red)); } @@ -491,8 +587,9 @@ pub fn pane(app: &mut RustGisApp, ui: &mut egui::Ui) { } let mut act = None; for (i, tile) in st.found.iter().enumerate() { + let years = st.years.get(i).is_some_and(|y| !y.is_empty()); ui.push_id(i, |ui| { - if let Some(a) = card(ui, &t, tile, i, busy) { + if let Some(a) = card(ui, &t, tile, i, busy, years) { act = Some(a); } }); @@ -548,7 +645,7 @@ enum Act { } /// A result's card: its title, what it is, and its actions (Zoom, Add, the file's address). -fn card(ui: &mut egui::Ui, t: &crate::theme::Tokens, tile: &Tile, i: usize, busy: bool) -> Option { +fn card(ui: &mut egui::Ui, t: &crate::theme::Tokens, tile: &Tile, i: usize, busy: bool, years: bool) -> Option { let mut act = None; egui::Frame::NONE.fill(t.input).stroke(egui::Stroke::new(1.0, t.border)).corner_radius(6).inner_margin(egui::Margin::same(8)).show( ui, @@ -564,6 +661,10 @@ fn card(ui: &mut egui::Ui, t: &crate::theme::Tokens, tile: &Tile, i: usize, busy act = Some(Act::Zoom(tile.bbox)); } let (label, hint) = match tile.dataset { + "alphaearth" if years => ( + tl!("Add years"), + tl!("Show the bands picked of every year on the time slider, read in place as the map needs them"), + ), "alphaearth" => (tl!("Add"), tl!("Show the bands picked, read in place as the map needs them")), "earth-index" => (tl!("Add points"), tl!("Read the points in the view, coloured by their principal components")), _ => (tl!("Add view"), tl!("Read the bands picked for the view (at most about 6 km a side)")), diff --git a/docs/user-guide/web-data.md b/docs/user-guide/web-data.md index eaf2ad832d..66fbeb09d8 100644 --- a/docs/user-guide/web-data.md +++ b/docs/user-guide/web-data.md @@ -629,7 +629,7 @@ their footprints drawn as a layer. | Dataset | What RustGIS does with it | |---|---| -| **AlphaEarth Foundations** (Google's Satellite Embedding; 64 values a 10 m pixel, 2017–2025) | Finds the year's UTM tiles where the map is in the dataset's GeoParquet index on [Source Cooperative](https://source.coop/tge-labs/aef), read in place (its bounding boxes, then the rows that meet the map). **Add** shows a tile as a cloud-optimised GeoTIFF read in place: three bands (**Bands**, from 1; A01, A16 and A09 by default) as red, green and blue, dequantised (`sign(v)·(v/127.5)²`) and stretched over ±**Stretch** (0.3). Only the three bands shown are read, a band at a time. **Apply to layers** shows other bands or another stretch on the tiles already added. | +| **AlphaEarth Foundations** (Google's Satellite Embedding; 64 values a 10 m pixel, 2017–2025) | Finds the year's UTM tiles where the map is in the dataset's GeoParquet index on [Source Cooperative](https://source.coop/tge-labs/aef), read in place (its bounding boxes, then the rows that meet the map). **Add** shows a tile as a cloud-optimised GeoTIFF read in place: three bands (**Bands**, from 1; A01, A16 and A09 by default) as red, green and blue, dequantised (`sign(v)·(v/127.5)²`) and stretched over ±**Stretch** (0.3). Only the three bands shown are read, a band at a time. **Apply to layers** shows other bands or another stretch on the tiles already added. With **All years, on the time slider**, the search finds each tile's file of every year (2017–2025, matched by place, since each year's files have names of their own) and **Add years** shows them as one tile layer that follows the time slider, a year a step (turned on over the years when it is off); a year a tile has no file for shows nothing, and is said. | | **Earth Index** (Earth Genome; 384 values every ~320 m, 2024) | Names the Sentinel-2 (MGRS) tiles where the map is and keeps those with a file. **Add points** reads the file's points in the view in place (every point's position, then the vectors of those in view: zoom to a few kilometres) and colours them by their vectors' first three principal components (64 colours); **Keep value** keeps one of each vector's values as the points' `value` field. | | **Tessera** (Cambridge; 128 values a 10 m pixel, 2017–2025) | **Add view** reads three bands (1, 2 and 3 by default) of the view from Tessera's [Zarr store](https://data.source.coop/tessera/tessera/zarr/v1.1-dclimate) in place (the shard's index, then the chunks the view needs: at most about 6 km a side, its centre when the view is larger), as a 32-bit raster in UTM. | | **Clay**, **Major TOM**, **Copernicus-Embed** | Listed with links to their data and papers; RustGIS doesn't read them. | @@ -640,10 +640,13 @@ AlphaEarth's tiles are tile layers that keep their look in their address in the Symbology pane. GeoTIFFs with their bands stored apart (planar), signed bytes and bottom-up images are read in place too. The commands: `embeddings.datasets`, `embeddings.search` (`"dataset"`, `"year"`, `"bbox"` or the view, `"footprints"`) and `embeddings.add` (`"dataset"`, `"url"`, -`"bands"`, `"stretch"`, `"dimension"`, `"bbox"`, `"year"`): +`"bands"`, `"stretch"`, `"dimension"`, `"bbox"`, `"year"`) and `embeddings.addCube` (AlphaEarth's tiles +where the map is over the years `"start"` to `"end"`, with `"bbox"`, `"bands"`, `"stretch"`, `"name"`; at +most 32 tile layers on a map): ```sh rustgis-cli run --cmd 'map.zoomToExtent={"extent":[-122.55,37.7,-122.3,37.85]}' --cmd 'embeddings.search={"dataset":"alphaearth","year":2024}' --cmd 'embeddings.add={"dataset":"alphaearth","url":"https://data.source.coop/tge-labs/aef/v1/annual/2024/10N/xcyo46pot2fg6a61t-0000008192-0000000000.tiff"}' --cmd 'share.exportMap={"path":"aef.png"}' +rustgis-cli run --cmd 'map.zoomToExtent={"extent":[-42.91,-20.78,-42.85,-20.735]}' --cmd 'embeddings.addCube={"dataset":"alphaearth"}' --cmd 'time.goTo={"date":"2017"}' --cmd 'share.exportMap={"path":"aef_2017.png"}' --cmd 'time.goTo={"date":"2025"}' --cmd 'share.exportMap={"path":"aef_2025.png"}' rustgis-cli run --cmd 'map.zoomToExtent={"extent":[-83.96,35.94,-83.9,35.98]}' --cmd 'embeddings.add={"dataset":"tessera","year":2024}' --cmd 'share.exportMap={"path":"tessera.png"}' ``` From a3bf4048edd0048b3ae58b8fe895729c0e2c0db3 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 20:11:05 -0300 Subject: [PATCH 2/7] fix(embeddings): first review of addCube (#1720) - series_notices is native only, as its one caller: the wasm32 build passes with -D warnings. - add_cube's doc says why it runs on its caller's thread without found. - The missing-years notice tells an incomplete read from the index's own gap; the pane hands its search's completeness over (complete). - A tile handed over with no file in the years asked for is left out and said. - A tile's years are matched by footprints agreeing to 1e-4 degrees, not by a rounded key a rounding boundary could split. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 49 +++++++++++++++++++++++------ crates/tiles/src/embeddings.rs | 39 +++++++++++++++-------- crates/ui-egui/src/embeddingsui.rs | 9 +++++- 3 files changed, 73 insertions(+), 24 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index ae1bf3ee56..89399b5c33 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -30,7 +30,7 @@ pub fn commands() -> Vec { r#"{"dataset": "alphaearth" | "earth-index" | "tessera", "url"?: "AlphaEarth's tile or Earth Index's file, as embeddings.search lists it (Earth Index: default the MGRS tile at the view's centre)", "year"?: number (Tessera, default 2024), "bands"?: [red, green, blue] (bands from 1; AlphaEarth default [2, 17, 10], Tessera [1, 2, 3]), "stretch"?: number (AlphaEarth: dequantised values shown over ±stretch, default 0.3), "dimension"?: number (Earth Index: a vector value kept as each point's `value`, from 1), "bbox"?: [west, south, east, north] | "view" (Earth Index and Tessera: the area read, default the map's extent), "name"?: text, "ran"?: (internal: what the app read)}"#, ), C::new("embeddings.addCube", "Add Satellite Embeddings over Time", "Home › Add › Web Data", add_cube).roles(&[]).effects(crate::policy::Effects::NONE.with(crate::policy::Effect::Network)).params( - r#"{"dataset": "alphaearth", "bbox"?: [west, south, east, north] | "view" (the area searched, default the map's extent), "start"?: year (default 2017), "end"?: year (default 2025), "bands"?: [red, green, blue] (bands from 1, default [2, 17, 10]), "stretch"?: number (dequantised values shown over ±stretch, default 0.3), "name"?: text, "found"?: (internal: the tiles the app found, each {years: {year: address}})} → a tile layer per AlphaEarth tile, a file per year, following the time slider (turned on over the years when it is off)"#, + r#"{"dataset": "alphaearth", "bbox"?: [west, south, east, north] | "view" (the area searched, default the map's extent), "start"?: year (default 2017), "end"?: year (default 2025), "bands"?: [red, green, blue] (bands from 1, default [2, 17, 10]), "stretch"?: number (dequantised values shown over ±stretch, default 0.3), "name"?: text, "found"?: (internal: the tiles the app found, each {years: {year: address}}), "complete"?: (internal: false when the app's search read only part of the index)} → a tile layer per AlphaEarth tile, a file per year, following the time slider (turned on over the years when it is off)"#, ), ] } @@ -180,6 +180,13 @@ fn stretch(p: &Value) -> Result { /// it), or from the app's search off the frame (`found`). Natively only the first tile's latest /// year's header is read before the layers are added, as `embeddings.add` reads one tile's; the /// rest are read in place as the map draws them. +/// +/// Without `found` the command runs on its caller's thread and waits there for the index and that +/// header, as `embeddings.search` and `embeddings.add` do: the command line, Python and the +/// assistant have no frame to hold and want the layers there when the command returns. The app's +/// own routes (ribbon, command search) open the pane, which searches off the frame and hands its +/// tiles over (`found`); a plugin's command in the app (`leave_downloads`) is refused rather than +/// wait on the frame. fn add_cube(s: &mut Session, p: &Value) -> CmdResult { use crate::render::rustgis_tiles::cog::{self, Look}; let d = dataset(p)?; @@ -199,10 +206,13 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { let b = bands(p, 64, emb::AEF_DEFAULT_BANDS)?; let look = Look { cube: true, ..emb::aef_look(b, stretch(p)?) }; let given = p.get("found").filter(|v| !v.is_null()); - let series: Vec> = match given { - Some(v) => found(v, start, end)?, + // Whether the index was read whole: a tile-year with no file is then the index's gap, not + // the read's. + let (series, complete) = match given { + Some(v) => (found(v, start, end)?, super::b(p, "complete") != Some(false)), None => search_series(s, p, (start, end))?, }; + let listed = series.len(); let series: Vec> = series.into_iter().filter(|y| !y.is_empty()).collect(); if series.is_empty() { return Err(format!("AlphaEarth has no tiles there for {start}–{end}")); @@ -277,12 +287,19 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { undo(s); return Err(e); } + if listed > series.len() { + s.notify(format!("{base}: {} of the {listed} tiles have no file in {start}–{end} and were left out", listed - series.len()), false); + } if missing > 0 { + let total = series.len() * span; s.notify( - format!( - "{base}: {missing} of the {} tile-years have no file in AlphaEarth's index; in those years the tile shows nothing", - series.len() * span - ), + if complete { + format!("{base}: {missing} of the {total} tile-years have no file in AlphaEarth's index; in those years the tile shows nothing") + } else { + format!( + "{base}: {missing} of the {total} tile-years have no file in the part of AlphaEarth's index that was read (zoom in to read it whole); in those years the tile shows nothing" + ) + }, false, ); } @@ -313,8 +330,9 @@ fn found(v: &Value, start: i32, end: i32) -> Result>, } /// AlphaEarth's tiles over the years where the command says, from one read of the index, waiting -/// for it (the command line), with what the read left out said. -fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result>, String> { +/// for it (the command line), with what the read left out said; and whether it read the index +/// whole for the area. +fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result<(Vec>, bool), String> { let area = bbox(s, p)?; #[cfg(target_arch = "wasm32")] { @@ -331,12 +349,13 @@ fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result String { - let crs = t.detail.split(" · ").next().unwrap_or(""); - let b = t.bbox.map(|v| (v * 1e5).round() as i64); - format!("{crs} {} {} {} {}", b[0], b[1], b[2], b[3]) +/// How far apart (degrees, about 10 m) two years' footprint corners may be and still be one +/// tile's: far above the noise of the index's coordinates, far below a tile's 82 km, so no +/// rounding boundary splits a tile in two. +const SAME_PLACE: f64 = 1e-4; + +/// Whether two rows are one tile's: the same coordinate system, and footprints whose corners +/// agree to [`SAME_PLACE`]. +fn same_place(a: &Tile, b: &Tile) -> bool { + let crs = |t: &Tile| t.detail.split(" · ").next().unwrap_or("").to_string(); + crs(a) == crs(b) && a.bbox.iter().zip(b.bbox.iter()).all(|(x, y)| (x - y).abs() <= SAME_PLACE) } /// The tiles of the years `first` to `last` in what the index read, each with its file a year, /// nearest the box's centre first. pub fn aef_series(fields: &[Field], batch: &Batch, (first, last): (i32, i32), bbox: [f64; 4]) -> SeriesFound { let mut found = SeriesFound { complete: batch.complete, ..Default::default() }; - let mut at: HashMap = HashMap::new(); + // Each tile's place as first seen: matched against, so a tile's latest footprint taking over + // its card doesn't move what later rows are matched to. A read holds at most MAX_INDEX_ROWS + // rows, so comparing each with the tiles so far is cheap. + let mut places: Vec = Vec::new(); for (_, t) in aef_rows(fields, batch, bbox, |y| (first..=last).contains(&y)) { let Some(year) = t.year else { continue }; - let i = *at.entry(place_key(&t)).or_insert_with(|| { - found.series.push(Series { tile: t.clone(), years: BTreeMap::new() }); - found.series.len() - 1 - }); + let i = match places.iter().position(|p| same_place(p, &t)) { + Some(i) => i, + None => { + places.push(t.clone()); + found.series.push(Series { tile: t.clone(), years: BTreeMap::new() }); + found.series.len() - 1 + } + }; let Some(s) = found.series.get_mut(i) else { continue }; match s.years.get(&year) { Some(u) if *u != t.url => found.duplicates += 1, @@ -956,7 +967,9 @@ mod tests { let base = "s3://us-west-2.opendata.source.coop/tge-labs/aef/v1/annual"; let batch = Batch { features: vec![ - row(-43.4, 2024, &format!("{base}/2024/23S/xb-0000008192-0000000000.tiff")), + // A footprint a hair off another year's, across a 1e-5° rounding boundary: the + // same tile still. + row(-43.400_005_1, 2024, &format!("{base}/2024/23S/xb-0000008192-0000000000.tiff")), row(-43.4, 2017, &format!("{base}/2017/23S/xa-0000008192-0000000000.tiff")), row(-43.4, 2025, &format!("{base}/2025/23S/xc-0000008192-0000000000.tiff")), // The same tile and year twice: the first kept, the other counted. diff --git a/crates/ui-egui/src/embeddingsui.rs b/crates/ui-egui/src/embeddingsui.rs index 7e6399552a..3d3d21a251 100644 --- a/crates/ui-egui/src/embeddingsui.rs +++ b/crates/ui-egui/src/embeddingsui.rs @@ -58,6 +58,8 @@ pub struct State { pub found: Vec, /// Each tile found's file a year, by `found`'s order, when the search was over the years. years: Vec>, + /// Whether the search over the years read the index whole (handed over as `complete`). + years_whole: bool, /// The footprints layer of the last search, and its project (`Session::opened`): layer ids /// are a map's own, so another project's may be the same. footprints: Option<(u32, u64)>, @@ -79,6 +81,7 @@ impl Default for State { searched: None, found: Vec::new(), years: Vec::new(), + years_whole: true, footprints: None, error: None, info: None, @@ -190,6 +193,7 @@ pub fn poll(app: &mut RustGisApp, ctx: &egui::Context) { st.info = Some(info); st.found = tiles; st.years = years; + st.years_whole = complete; show_footprints(app); } Err(e) => app.ui.embeddings.error = Some(e), @@ -312,7 +316,10 @@ fn add(app: &mut RustGisApp, i: usize) { let found = json!([{"years": years}]); ( "embeddings.addCube", - json!({"dataset": "alphaearth", "found": found, "start": first, "end": last, "bands": bands, "stretch": stretch}), + json!({ + "dataset": "alphaearth", "found": found, "complete": st.years_whole, + "start": first, "end": last, "bands": bands, "stretch": stretch, + }), ) } None => ("embeddings.add", json!({"dataset": "alphaearth", "url": tile.url, "bands": bands, "stretch": stretch})), From 7b4131fbaacc50072b1c926e0db79a06bffee7bb Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 20:28:01 -0300 Subject: [PATCH 3/7] fix(embeddings): say the handed-over files outside the years asked for (#1720) embeddings.addCube's found dropped a file outside start..=end without a word: a caller passing a 2016 file with start 2017 lost it silently. The files left out are now counted and said. The pane's Cancel comment says why a search's bounded index read is left to run on. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 42 +++++++++++++++++++++++------ crates/ui-egui/src/embeddingsui.rs | 5 +++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index 89399b5c33..66e95628cd 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -207,10 +207,16 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { let look = Look { cube: true, ..emb::aef_look(b, stretch(p)?) }; let given = p.get("found").filter(|v| !v.is_null()); // Whether the index was read whole: a tile-year with no file is then the index's gap, not - // the read's. - let (series, complete) = match given { - Some(v) => (found(v, start, end)?, super::b(p, "complete") != Some(false)), - None => search_series(s, p, (start, end))?, + // the read's. And the files handed over outside the years asked for, left out. + let (series, complete, outside) = match given { + Some(v) => { + let (series, outside) = found(v, start, end)?; + (series, super::b(p, "complete") != Some(false), outside) + } + None => { + let (series, complete) = search_series(s, p, (start, end))?; + (series, complete, 0) + } }; let listed = series.len(); let series: Vec> = series.into_iter().filter(|y| !y.is_empty()).collect(); @@ -287,6 +293,16 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { undo(s); return Err(e); } + if outside > 0 { + s.notify( + if outside == 1 { + format!("{base}: 1 file handed over is outside {start}–{end} and was left out") + } else { + format!("{base}: {outside} files handed over are outside {start}–{end} and were left out") + }, + false, + ); + } if listed > series.len() { s.notify(format!("{base}: {} of the {listed} tiles have no file in {start}–{end} and were left out", listed - series.len()), false); } @@ -307,14 +323,16 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { } /// The tiles the app found and hands over (`found`): each a year's file by year, kept to the -/// years asked for. -fn found(v: &Value, start: i32, end: i32) -> Result>, String> { +/// years asked for; and how many files were outside them (said by the caller). +fn found(v: &Value, start: i32, end: i32) -> Result<(Vec>, usize), String> { let bad = || "found is a list of tiles, each {\"years\": {\"2017\": an https address, …}}".to_string(); let list = v.as_array().ok_or_else(bad)?; if list.len() > emb::MAX_TILES { return Err(format!("found holds {} tiles, more than the {} a search lists", list.len(), emb::MAX_TILES)); } - list.iter() + let mut outside = 0usize; + let series = list + .iter() .map(|t| { let mut out = BTreeMap::new(); for (y, u) in t.get("years").and_then(Value::as_object).ok_or_else(bad)? { @@ -322,11 +340,14 @@ fn found(v: &Value, start: i32, end: i32) -> Result>, let u = u.as_str().map(str::trim).filter(|u| u.starts_with("https://") && u.len() <= 4000).ok_or_else(bad)?; if (start..=end).contains(&y) { out.insert(y, u.to_string()); + } else { + outside += 1; } } Ok(out) }) - .collect() + .collect::, String>>()?; + Ok((series, outside)) } /// AlphaEarth's tiles over the years where the command says, from one read of the index, waiting @@ -569,6 +590,11 @@ mod tests { assert_eq!(t.dates, vec![emb::year_start_ms(2017), emb::year_start_ms(2020), emb::year_start_ms(2025)]); assert_eq!(t.format, "YYYY"); assert!(s.notices.iter().any(|n| n.text.contains("15 of the 18 tile-years have no file"))); + assert!( + s.notices.iter().any(|n| n.text.contains("1 file handed over is outside 2017–2025 and was left out")), + "a file outside the years asked for is said: {:?}", + s.notices.iter().map(|n| &n.text).collect::>() + ); // The slider shows each tile's file of its year, or nothing in a year it has none for. s.run("time.goTo", &json!({"step": 1})).unwrap(); assert!(named(&s, a).time_hidden, "no file in 2020"); diff --git a/crates/ui-egui/src/embeddingsui.rs b/crates/ui-egui/src/embeddingsui.rs index 3d3d21a251..20a159fc41 100644 --- a/crates/ui-egui/src/embeddingsui.rs +++ b/crates/ui-egui/src/embeddingsui.rs @@ -578,7 +578,10 @@ pub fn pane(app: &mut RustGisApp, ui: &mut egui::Ui) { }); } if cancel { - // The read runs on, but what it brings is dropped with its slot. + // What the read brings is dropped with its slot, and a data read's download released. + // A search's read of the index runs on to its end: it is bounded (one read, at most + // MAX_INDEX_ROWS rows) and the index query has no way to be told to stop yet, as for + // every other search here. if let Some(Read::Data(_, req)) = app.ui.embeddings.reading.take() { app.session.download_failed(req.id); } From 2ad313621beea96b77659f13d28e09061b81f7a8 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 21:28:41 -0300 Subject: [PATCH 4/7] fix(embeddings): addCube's names, doc and place match after review (#1720) - A cube whose name a tile layer already has takes its tile's place in it, so two tiles added from the pane one at a time can be told apart. - add_cube's doc says no header is read with found. - same_place compares borrowed CRS text, after the corners, with no allocation per comparison. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 22 +++++++++++++++++----- crates/tiles/src/embeddings.rs | 6 ++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index 66e95628cd..00f403c4d0 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -177,9 +177,10 @@ fn stretch(p: &Value) -> Result { /// of the time slider. Each tile is a tile layer reading its file of the year the slider is at /// (`time_urls`, as `stac.addCube` makes them); a year a tile has no file for shows nothing, said. /// The tiles come from one read of the index, which holds every year (the command line waits for -/// it), or from the app's search off the frame (`found`). Natively only the first tile's latest -/// year's header is read before the layers are added, as `embeddings.add` reads one tile's; the -/// rest are read in place as the map draws them. +/// it), or from the app's search off the frame (`found`). Without `found`, natively, only the +/// first tile's latest year's header is read before the layers are added, as `embeddings.add` +/// reads one tile's; with `found` (the pane, on the frame) no header is read. The rest are read in +/// place as the map draws them. /// /// Without `found` the command runs on its caller's thread and waits there for the index and that /// header, as `embeddings.search` and `embeddings.add` do: the command line, Python and the @@ -247,7 +248,10 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { // command in it (on the frame). let mut probe = cfg!(not(target_arch = "wasm32")) && !s.leave_downloads && given.is_none(); let span = usize::try_from(end - start + 1).unwrap_or(1); - let (mut dates, mut missing, many) = (Vec::new(), 0usize, series.len() > 1); + // A tile's place goes in its layer's name when the cube has several, or when a tile layer + // already has the name (the pane adds one tile at a time). + let tagged = series.len() > 1 || s.map().tiles.iter().any(|t| t.name == base); + let (mut dates, mut missing) = (Vec::new(), 0usize); for years in &series { missing += span.saturating_sub(years.len()); let urls: Vec<(f64, String)> = years.iter().map(|(y, u)| (emb::year_start_ms(*y), cog::with_look(u, &look))).collect(); @@ -267,7 +271,7 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { } } } - let name = match (many, emb::aef_place(&file)) { + let name = match (tagged, emb::aef_place(&file)) { (true, Some(place)) => format!("{base} · {place}"), _ => base.clone(), }; @@ -606,6 +610,14 @@ mod tests { t.run("embeddings.addCube", &json!({"dataset": "alphaearth", "start": 2024, "found": one, "name": "AEF"})).unwrap(); assert_eq!(t.map().tiles.first().map(|l| l.name.as_str()), Some("AEF")); assert!(!t.notices.iter().any(|n| n.text.contains("tile-years"))); + // The pane adds a tile at a time: a second cube of the same name takes its place in it. + let mut w = Session::new(Project::default()); + for place in [a, b] { + let one = json!([{"years": {"2024": file(2024, place)}}]); + w.run("embeddings.addCube", &json!({"dataset": "alphaearth", "found": one})).unwrap(); + } + let names: Vec<&str> = w.map().tiles.iter().map(|l| l.name.as_str()).collect(); + assert!(names.len() == 2 && names[0] != names[1] && names.iter().any(|n| n.ends_with(b)), "{names:?}"); // A search that read part of the index: the missing years are the read's, said so; a tile // with no file in the years asked for is left out, said. let mut u = Session::new(Project::default()); diff --git a/crates/tiles/src/embeddings.rs b/crates/tiles/src/embeddings.rs index 4a35ebabeb..bd3b799575 100644 --- a/crates/tiles/src/embeddings.rs +++ b/crates/tiles/src/embeddings.rs @@ -373,8 +373,10 @@ const SAME_PLACE: f64 = 1e-4; /// Whether two rows are one tile's: the same coordinate system, and footprints whose corners /// agree to [`SAME_PLACE`]. fn same_place(a: &Tile, b: &Tile) -> bool { - let crs = |t: &Tile| t.detail.split(" · ").next().unwrap_or("").to_string(); - crs(a) == crs(b) && a.bbox.iter().zip(b.bbox.iter()).all(|(x, y)| (x - y).abs() <= SAME_PLACE) + fn crs(t: &Tile) -> &str { + t.detail.split(" · ").next().unwrap_or("") + } + a.bbox.iter().zip(b.bbox.iter()).all(|(x, y)| (x - y).abs() <= SAME_PLACE) && crs(a) == crs(b) } /// The tiles of the years `first` to `last` in what the index read, each with its file a year, From 8fe204e6e1d920ed709768cae23079a696994387 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 22:06:45 -0300 Subject: [PATCH 5/7] fix(embeddings): refuse addCube's bad years and a year named twice (#1720) - start or end given as something other than a year ("2020-01-01", true) took the default 2017-2025 without a word; only a missing or null one does now, anything else is refused. - A found tile naming one year twice (" 2017" and "2017") kept one of its files and lost the other silently; it is refused. Both reproduced in the parameters test first. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index 00f403c4d0..21489d9afe 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -195,10 +195,13 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { return Err(format!("{}: only AlphaEarth's years make a time cube here", d.name)); } let (first, last) = d.years.unwrap_or((2017, 2025)); - let year_param = |k: &str, default: i32| match f(p, k) { + // Only a year left out (or null) takes the default: one given that isn't a year is refused. + let year_param = |k: &str, default: i32| match p.get(k).filter(|v| !v.is_null()) { None => Ok(default), - Some(y) if y.fract() == 0.0 && y >= f64::from(first) && y <= f64::from(last) => Ok(y as i32), - Some(_) => Err(format!("{k} is a year from {first} to {last}")), + Some(_) => match f(p, k) { + Some(y) if y.fract() == 0.0 && y >= f64::from(first) && y <= f64::from(last) => Ok(y as i32), + _ => Err(format!("{k} is a year from {first} to {last}")), + }, }; let (start, end) = (year_param("start", first)?, year_param("end", last)?); if start > end { @@ -338,9 +341,14 @@ fn found(v: &Value, start: i32, end: i32) -> Result<(Vec>, let series = list .iter() .map(|t| { - let mut out = BTreeMap::new(); + let (mut out, mut seen) = (BTreeMap::new(), Vec::new()); for (y, u) in t.get("years").and_then(Value::as_object).ok_or_else(bad)? { let y: i32 = y.trim().parse().map_err(|_| bad())?; + // Two keys of one year (" 2017" and "2017"): one file would be lost, so refused. + if seen.contains(&y) { + return Err(format!("found: a tile names the year {y} twice")); + } + seen.push(y); let u = u.as_str().map(str::trim).filter(|u| u.starts_with("https://") && u.len() <= 4000).ok_or_else(bad)?; if (start..=end).contains(&y) { out.insert(y, u.to_string()); @@ -559,6 +567,11 @@ mod tests { json!({"dataset": "alphaearth", "found": [{"years": {"2017": "file:///etc/passwd"}}]}), json!({"dataset": "alphaearth", "found": [{"years": {"later": "https://x.org/a.tiff"}}]}), json!({"dataset": "alphaearth", "found": [{"files": []}]}), + // A year that isn't one, not taken as the default. + json!({"dataset": "alphaearth", "start": "2020-01-01", "found": tile}), + json!({"dataset": "alphaearth", "end": true, "found": tile}), + // A year named twice, one file of it lost. + json!({"dataset": "alphaearth", "found": [{"years": {"2017": "https://x.org/a.tiff", " 2017": "https://x.org/b.tiff"}}]}), // Nothing in the years asked for. json!({"dataset": "alphaearth", "start": 2020, "found": tile}), json!({"dataset": "alphaearth", "found": vec![json!({"years": {}}); emb::MAX_TILES + 1]}), From 792df078d851a197e1fe0cdf10e4b77cb70fa035 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 22:40:29 -0300 Subject: [PATCH 6/7] fix(embeddings): refuse a stretch that isn't a number; say a search's gaps only for a cube added (#1720) - stretch given as something other than a number ("big", true) took the default 0.3 without a word, in embeddings.add and addCube; only a missing or null one does now. Reproduced in the parameters test first. - What a search over the years left out is said once the cube is added, not before a refusal that adds nothing. - A tile's years are checked for repeats with a set. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 53 +++++++++++++++++------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index 21489d9afe..a79d54fe02 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -164,9 +164,13 @@ fn add(s: &mut Session, p: &Value) -> CmdResult { } } -/// AlphaEarth's stretch a command names: dequantised values shown over ±stretch. +/// AlphaEarth's stretch a command names: dequantised values shown over ±stretch. Only a stretch +/// left out (or null) takes the default: one given that isn't a number is refused. fn stretch(p: &Value) -> Result { - let stretch = f(p, "stretch").map(|v| v as f32).unwrap_or(emb::AEF_DEFAULT_STRETCH); + let stretch = match p.get("stretch").filter(|v| !v.is_null()) { + None => emb::AEF_DEFAULT_STRETCH, + Some(_) => f(p, "stretch").map_or(f32::NAN, |v| v as f32), + }; if !(stretch.is_finite() && stretch > 0.0 && stretch <= 1.0) { return Err("stretch is a number above 0, at most 1".into()); } @@ -211,15 +215,16 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { let look = Look { cube: true, ..emb::aef_look(b, stretch(p)?) }; let given = p.get("found").filter(|v| !v.is_null()); // Whether the index was read whole: a tile-year with no file is then the index's gap, not - // the read's. And the files handed over outside the years asked for, left out. - let (series, complete, outside) = match given { + // the read's. The files handed over outside the years asked for, left out. And what the + // search left out, said once the cube is added (a refused one adds nothing to speak of). + let (series, complete, outside, notes) = match given { Some(v) => { let (series, outside) = found(v, start, end)?; - (series, super::b(p, "complete") != Some(false), outside) + (series, super::b(p, "complete") != Some(false), outside, Vec::new()) } None => { - let (series, complete) = search_series(s, p, (start, end))?; - (series, complete, 0) + let (series, complete, notes) = search_series(s, p, (start, end))?; + (series, complete, 0, notes) } }; let listed = series.len(); @@ -300,6 +305,9 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { undo(s); return Err(e); } + for note in notes { + s.notify(note, false); + } if outside > 0 { s.notify( if outside == 1 { @@ -341,14 +349,13 @@ fn found(v: &Value, start: i32, end: i32) -> Result<(Vec>, let series = list .iter() .map(|t| { - let (mut out, mut seen) = (BTreeMap::new(), Vec::new()); + let (mut out, mut seen) = (BTreeMap::new(), std::collections::BTreeSet::new()); for (y, u) in t.get("years").and_then(Value::as_object).ok_or_else(bad)? { let y: i32 = y.trim().parse().map_err(|_| bad())?; // Two keys of one year (" 2017" and "2017"): one file would be lost, so refused. - if seen.contains(&y) { + if !seen.insert(y) { return Err(format!("found: a tile names the year {y} twice")); } - seen.push(y); let u = u.as_str().map(str::trim).filter(|u| u.starts_with("https://") && u.len() <= 4000).ok_or_else(bad)?; if (start..=end).contains(&y) { out.insert(y, u.to_string()); @@ -363,9 +370,9 @@ fn found(v: &Value, start: i32, end: i32) -> Result<(Vec>, } /// AlphaEarth's tiles over the years where the command says, from one read of the index, waiting -/// for it (the command line), with what the read left out said; and whether it read the index -/// whole for the area. -fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result<(Vec>, bool), String> { +/// for it (the command line); whether it read the index whole for the area; and what the read +/// left out, to be said. +fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result<(Vec>, bool, Vec), String> { let area = bbox(s, p)?; #[cfg(target_arch = "wasm32")] { @@ -381,27 +388,26 @@ fn search_series(s: &mut Session, p: &Value, years: (i32, i32)) -> Result<(Vec Vec { + let mut notes = Vec::new(); if !f.complete { - s.notify("AlphaEarth: the area holds more tiles than one search reads; zoom in to take them all", false); + notes.push("AlphaEarth: the area holds more tiles than one search reads; zoom in to take them all".to_string()); } if f.dropped > 0 { - s.notify( - format!("AlphaEarth: {} tiles past the {} a search lists were left out; zoom in for them", f.dropped, emb::MAX_TILES), - false, - ); + notes.push(format!("AlphaEarth: {} tiles past the {} a search lists were left out; zoom in for them", f.dropped, emb::MAX_TILES)); } if f.duplicates > 0 { - s.notify(format!("AlphaEarth: {} tile-years had a second file in the index; the first was taken", f.duplicates), false); + notes.push(format!("AlphaEarth: {} tile-years had a second file in the index; the first was taken", f.duplicates)); } + notes } /// Bytes the app read and handed over, by their id (`ran`). @@ -562,6 +568,9 @@ mod tests { json!({"dataset": "alphaearth", "end": 2024.5, "found": tile}), json!({"dataset": "alphaearth", "start": 2024, "end": 2020, "found": tile}), json!({"dataset": "alphaearth", "stretch": 2, "found": tile}), + // A stretch that isn't a number, not taken as the default. + json!({"dataset": "alphaearth", "stretch": "big", "found": tile}), + json!({"dataset": "alphaearth", "stretch": true, "found": tile}), json!({"dataset": "alphaearth", "bands": [1, 2], "found": tile}), json!({"dataset": "alphaearth", "found": "tiles"}), json!({"dataset": "alphaearth", "found": [{"years": {"2017": "file:///etc/passwd"}}]}), From 71825eedad5efdeb3ec94f26dc363c1e07c2bcb4 Mon Sep 17 00:00:00 2001 From: elpidioassinaturas Date: Tue, 10 Nov 2026 23:25:35 -0300 Subject: [PATCH 7/7] docs(embeddings): addCube's doc says the app's assistant waits on the frame (#1720) The doc said the assistant has no frame to hold. The app's assistant runs its tools on the frame without leave_downloads, as it does for embeddings.search and layer.addUrl, so addCube without found waits there too. The doc now says so. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/embeddings.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/embeddings.rs b/crates/engine/src/cmd/embeddings.rs index a79d54fe02..29232abff8 100644 --- a/crates/engine/src/cmd/embeddings.rs +++ b/crates/engine/src/cmd/embeddings.rs @@ -187,11 +187,14 @@ fn stretch(p: &Value) -> Result { /// place as the map draws them. /// /// Without `found` the command runs on its caller's thread and waits there for the index and that -/// header, as `embeddings.search` and `embeddings.add` do: the command line, Python and the -/// assistant have no frame to hold and want the layers there when the command returns. The app's +/// header, as `embeddings.search` and `embeddings.add` do, so that the caller finds the layers +/// there when the command returns. The command line and Python have no window to hold. The app's /// own routes (ribbon, command search) open the pane, which searches off the frame and hands its /// tiles over (`found`); a plugin's command in the app (`leave_downloads`) is refused rather than -/// wait on the frame. +/// wait on the frame. The app's assistant runs its tools on the frame without `leave_downloads` +/// (see [`Session::run_leaving_downloads`]), so there it waits on the frame, as it does for every +/// command that waits for the network (`embeddings.search`, `layer.addUrl`); moving the +/// assistant's tools off the frame is a change for all of them, not for this one. fn add_cube(s: &mut Session, p: &Value) -> CmdResult { use crate::render::rustgis_tiles::cog::{self, Look}; let d = dataset(p)?;