diff --git a/crates/buzz-test-client/src/loadgen/desktop.rs b/crates/buzz-test-client/src/loadgen/desktop.rs new file mode 100644 index 00000000000..2575d16061e --- /dev/null +++ b/crates/buzz-test-client/src/loadgen/desktop.rs @@ -0,0 +1,2719 @@ +//! A desktop app client. It makes the relay requests the desktop app makes, +//! modeled on the app's live subscription manager (the desktop app's +//! `live.ts`) and its session store. It follows the request pattern from +//! before (`legacy`) or after (`current`) the app's client volume fixes. +//! +//! The planning functions here say what a connection sends: the REQs it +//! opens, the HTTP reads it makes, and what a recovery tick or a renewal +//! adds. [`desktop`] executes those plans over one socket, plus one HTTP read +//! task per client. See `perf/LOADGEN.md` for the model and what it omits. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use buzz_ws_client::{NostrWsConnection, RelayMessage, WsClientError}; +use futures_util::stream::{self, StreamExt}; +use nostr::Keys; +use serde_json::{json, Value}; + +use super::identity::{self, Role}; +use super::net::http_reason; +use super::run::{ + connect, observe, roots, stopped, Activity, Backoff, Directory, Rng, Seen, Shared, Step, + MAX_IDLE, +}; +use super::scenario::{DesktopMode, WideSplit}; +use super::stats::{Op, Pending}; + +/// Kinds the app subscribes to for every joined channel. +const CHANNEL_KINDS: [u32; 20] = [ + 9, 40002, 40008, 45001, 45003, 40099, 40100, 40003, 5, 9005, 7, 39000, 39002, 39005, 20002, + 45010, 48100, 48101, 48102, 48103, +]; +/// Kinds of a channel window page: channel rows without notices. +const WINDOW_KINDS: [u32; 5] = [9, 40002, 40008, 48100, 48103]; +/// Per-filter limit of a replaying channel filter. +const REPLAY_LIMIT: usize = 500; +/// Channels per replay REQ (one filter each), and per `legacy` batch REQ. +const REPLAY_BATCH: usize = 10; +/// Channel ids in one `current` live-only batch filter. +const LIVE_BATCH: usize = 2048; +/// REQs awaiting EOSE at once while a connection establishes. +const SETUP_CONCURRENCY: usize = 4; +/// How long a REQ may wait for its EOSE. +const REQ_TIMEOUT: Duration = Duration::from_secs(10); +/// A resumed subscription starts this long before its cursor. +const RESUME_LAG_SECS: u64 = 960; +/// A first subscription starts this long before now. +const STARTUP_FLOOR_SECS: u64 = 300; +/// HTTP reads in flight at once per client. +const HTTP_CONCURRENCY: usize = 6; +/// Deadline of one HTTP read. +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); +/// Page size of roster, metadata and profile reads. +const PAGE: usize = 500; +/// Most roster pages one read follows. +const MAX_ROSTER_PAGES: usize = 20; +/// Profiles the client holds. +const PROFILE_CAP: usize = 1024; +/// Status authors read. +const STATUS_AUTHORS: usize = 100; +/// Channels per unread repair read. +const UNREAD_CHANNELS: usize = 128; +/// Events per channel window page. +const WINDOW_PAGE: usize = 20; +/// Older window pages a catch-up reads after the head, at most. +const CATCH_UP_PAGES: usize = 5; +/// The app's custom emoji set. +const EMOJI_SET: &str = "buzz:custom-emoji"; + +/// What a subscription carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Route { + /// The global profiles route: profiles, statuses and custom emoji. + Profiles, + /// The global membership route: grants, read state and reminders. + Membership, + /// A channel batch, which stays open. + Channels, + /// A replay of channels behind a live-only batch, closed at its EOSE. + Replay, +} + +/// One REQ frame. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Req { + pub(super) id: String, + pub(super) filters: Vec, + pub(super) route: Route, +} + +impl Req { + fn frame(&self) -> Value { + let mut frame = vec![json!("REQ"), json!(self.id)]; + frame.extend(self.filters.iter().cloned()); + Value::Array(frame) + } + + /// A route whose replay counts as cut whatever it sends: a live-only + /// channel batch over more than ten channels replays nothing itself. + fn cut_by_design(&self) -> bool { + let live_only = + self.route == Route::Channels && self.filters.iter().all(|f| f["limit"] == json!(0)); + let scope: usize = self + .filters + .iter() + .map(|f| f["#h"].as_array().map_or(0, Vec::len)) + .sum(); + live_only && scope > REPLAY_BATCH + } +} + +/// Whether a route's replay was cut at its EOSE: it replayed the app's limit +/// of events across its filters, or it was cut by design. +pub(super) fn limited(cut_by_design: bool, events: usize) -> bool { + cut_by_design || events >= REPLAY_LIMIT +} + +fn channel_filter(ids: &[String], since: u64, limit: usize) -> Value { + json!({"kinds": CHANNEL_KINDS, "#h": ids, "since": since, "limit": limit}) +} + +/// The two global subscriptions, profiles and membership. +fn global_reqs(viewer: &str, since: u64) -> [Req; 2] { + let limit = REPLAY_LIMIT; + [ + Req { + id: "profiles".into(), + filters: vec![ + json!({"kinds": [0, 10100, 30177], "since": since, "limit": limit}), + json!({"kinds": [30315], "#d": ["general"], "since": since, "limit": limit}), + json!({"kinds": [30030], "#d": [EMOJI_SET], "since": since, "limit": limit}), + ], + route: Route::Profiles, + }, + Req { + id: "membership".into(), + filters: vec![ + json!({"kinds": [44100, 44101], "#p": [viewer], "since": since, "limit": limit}), + json!({"kinds": [44100, 44101], "#p": [viewer], "#t": ["system"], "since": since, "limit": limit}), + json!({"kinds": [30078], "authors": [viewer], "#t": ["read-state"], "since": since, "limit": limit}), + json!({"kinds": [30300], "authors": [viewer], "since": since, "limit": limit}), + ], + route: Route::Membership, + }, + ] +} + +/// Channel batch subscriptions, which stay open. Up to 10 channels share +/// one REQ with a filter per channel. Beyond that, `legacy` sends a REQ per +/// 10 channels, and `current` sends one live-only filter (limit 0) per 2048 +/// channels and replays separately ([`replay_reqs`]). `limit` applies to +/// per-channel filters: [`REPLAY_LIMIT`] on connect, 0 for a tick's +/// replacement. +pub(super) fn batch_reqs( + mode: DesktopMode, + channels: &[String], + since: u64, + limit: usize, + generation: u64, +) -> Vec { + let id = |i: usize| format!("batch-{generation}-{i}"); + if mode == DesktopMode::Legacy || channels.len() <= REPLAY_BATCH { + channels + .chunks(REPLAY_BATCH) + .enumerate() + .map(|(i, chunk)| Req { + id: id(i), + filters: chunk + .iter() + .map(|c| channel_filter(std::slice::from_ref(c), since, limit)) + .collect(), + route: Route::Channels, + }) + .collect() + } else { + channels + .chunks(LIVE_BATCH) + .enumerate() + .map(|(i, chunk)| Req { + id: id(i), + filters: vec![channel_filter(chunk, since, 0)], + route: Route::Channels, + }) + .collect() + } +} + +/// `current` mode's replay of more than 10 channels behind live-only +/// batches: a REQ per 10 channels, one filter each, closed at its EOSE. +pub(super) fn replay_reqs( + mode: DesktopMode, + channels: &[String], + since: u64, + generation: u64, +) -> Vec { + if mode == DesktopMode::Legacy || channels.len() <= REPLAY_BATCH { + return Vec::new(); + } + channels + .chunks(REPLAY_BATCH) + .enumerate() + .map(|(i, chunk)| Req { + id: format!("replay-{generation}-{i}"), + filters: chunk + .iter() + .map(|c| channel_filter(std::slice::from_ref(c), since, REPLAY_LIMIT)) + .collect(), + route: Route::Replay, + }) + .collect() +} + +/// Every REQ a connection sends while it establishes, in order. +pub(super) fn connect_reqs( + mode: DesktopMode, + viewer: &str, + channels: &[String], + since: u64, + generation: u64, +) -> Vec { + let mut reqs = Vec::from(global_reqs(viewer, since)); + reqs.extend(batch_reqs(mode, channels, since, REPLAY_LIMIT, generation)); + reqs.extend(replay_reqs(mode, channels, since, generation)); + reqs +} + +/// Why a connection is being established. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Situation { + /// The client's first connection. + Cold, + /// Any other reconnect: routes resume from the last connection. + Reconnect, + /// A `current` identity renewal: routes are carried over, and none + /// establishes unless one of them replays a cut window. + Renewal, +} + +/// What an identity renewal is in `mode`. Before the fixes it was an +/// ordinary reconnect. +pub(super) fn renewal(mode: DesktopMode) -> Situation { + match mode { + DesktopMode::Legacy => Situation::Reconnect, + DesktopMode::Current => Situation::Renewal, + } +} + +/// One HTTP read job. A job may make several requests: pages, or chunks of +/// ids. `Window(i)` and `Thread(i)` are the client's i-th open window and +/// thread. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Fetch { + Nip11, + Roster, + Metadata, + Sidebar, + Profiles, + Statuses, + Emoji, + /// Unread repair. With `snapshot`, a full repair: the read-state + /// snapshot first, then the repair. Without, the membership route's + /// replay covered read state and only the repair runs. + Unread { + snapshot: bool, + }, + Reminders, + Window(usize), + Thread(usize), +} + +const FULL_REPAIR: Fetch = Fetch::Unread { snapshot: true }; +const COVERED_REPAIR: Fetch = Fetch::Unread { snapshot: false }; + +impl Fetch { + /// Whether this job, queued or running, already does `other`'s work. + fn covers(self, other: Fetch) -> bool { + self == other || (self == FULL_REPAIR && other == COVERED_REPAIR) + } +} + +fn channel_recovery(windows: usize, threads: usize) -> impl Iterator { + (0..windows) + .map(Fetch::Window) + .chain((0..threads).map(Fetch::Thread)) +} + +/// The reads a connection makes as soon as it opens. `legacy` reads +/// everything here: all of it cold, all but channel names and sidebar +/// preferences on a reconnect, which it treats a renewal as. `current` +/// reads only what no route covers; the rest follows each route's +/// establishment ([`establish_fetches`]). +pub(super) fn connect_fetches( + mode: DesktopMode, + situation: Situation, + windows: usize, + threads: usize, +) -> Vec { + let cold = situation == Situation::Cold; + match (mode, situation) { + (DesktopMode::Legacy, _) => { + let mut reads = vec![Fetch::Nip11, Fetch::Roster]; + if cold { + reads.extend([Fetch::Metadata, Fetch::Sidebar]); + } + reads.extend([ + Fetch::Profiles, + Fetch::Statuses, + Fetch::Emoji, + FULL_REPAIR, + Fetch::Reminders, + ]); + reads.extend(channel_recovery(windows, threads)); + reads + } + (DesktopMode::Current, Situation::Cold) => { + vec![Fetch::Nip11, Fetch::Metadata, Fetch::Sidebar] + } + (DesktopMode::Current, Situation::Reconnect) => vec![Fetch::Nip11], + (DesktopMode::Current, Situation::Renewal) => Vec::new(), + } +} + +/// Reads still owed after a failed or unfinished one, as the app's stores +/// track them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Owed { + pub(super) roster: bool, + pub(super) profiles: bool, + /// The read-state snapshot, owed until a snapshot read succeeds. Only a + /// full repair reads it, so a covered repair never settles it. + pub(super) snapshot: bool, + /// The unread repair of the channels. + pub(super) unread: bool, +} + +/// The reads `current` makes when `route` establishes. A `continuous` +/// global route replayed every change since the last connection, so it +/// skips its full read unless one is still owed: profiles resume their +/// statuses and emoji, and membership resumes read state and reminders and +/// only repairs unreads, unless the read-state snapshot is owed, which takes +/// a full repair. A channel route recovers the open windows and threads. +pub(super) fn establish_fetches( + route: Route, + continuous: bool, + owed: Owed, + windows: usize, + threads: usize, +) -> Vec { + match route { + Route::Profiles if !continuous => vec![Fetch::Profiles, Fetch::Statuses, Fetch::Emoji], + Route::Profiles if owed.profiles => vec![Fetch::Profiles], + Route::Profiles => Vec::new(), + Route::Membership => { + let mut reads = Vec::new(); + if !continuous || owed.roster { + reads.push(Fetch::Roster); + } + if continuous && !owed.snapshot { + reads.push(COVERED_REPAIR); + } else if continuous { + reads.push(FULL_REPAIR); + } else { + reads.extend([FULL_REPAIR, Fetch::Reminders]); + } + reads + } + Route::Channels => channel_recovery(windows, threads).collect(), + Route::Replay => Vec::new(), + } +} + +/// Which of a connection's routes establish, and when. Each route kind +/// establishes once per connection. A route resumes only if the previous +/// connection had it caught up; one that had not counts as cut. On a +/// renewal, routes are carried: one whose replay kept continuity establishes +/// nothing, until any route's replay is cut, which establishes every route +/// again. +#[derive(Debug)] +pub(super) struct Establishment { + /// Routes the previous connection had caught up. + resumed: Vec, + carried: bool, + /// Routes that reached EOSE while carried, with their continuity. + held: Vec<(Route, bool)>, + done: Vec, + /// Routes established only because their EOSE never came: recovered, + /// but not live, so not caught up. + timed_out: Vec, +} + +impl Establishment { + /// A connection's establishment, after one that had `caught_up` routes + /// (see [`Establishment::caught_up`]). A cold start resumes nothing. + pub(super) fn new(situation: Situation, caught_up: &[Route]) -> Self { + let resumed = match situation { + Situation::Cold => Vec::new(), + _ => caught_up.to_vec(), + }; + Self { + resumed, + carried: situation == Situation::Renewal, + held: Vec::new(), + done: Vec::new(), + timed_out: Vec::new(), + } + } + + /// `route` reached its EOSE, its replay `cut` or not. Returns the routes + /// that establish now, each with its continuity. + pub(super) fn eose(&mut self, route: Route, cut: bool) -> Vec<(Route, bool)> { + if route == Route::Replay { + return Vec::new(); + } + let cut = cut || !self.resumed.contains(&route); + let mut now = Vec::new(); + if self.carried { + if !cut { + self.held.push((route, true)); + return Vec::new(); + } + self.carried = false; + now = std::mem::take(&mut self.held); + } + now.push((route, !cut)); + now.retain(|(route, _)| { + let first = !self.done.contains(route); + if first { + self.done.push(*route); + } + first + }); + now + } + + /// `route` missed its EOSE deadline. It is recovered as if its replay + /// were cut, but only an EOSE proves a route live: it is not caught up, + /// so the next connection cannot resume or carry it. + pub(super) fn timeout(&mut self, route: Route) -> Vec<(Route, bool)> { + if route != Route::Replay { + self.timed_out.push(route); + } + self.eose(route, true) + } + + /// The routes this connection has caught up: established at their EOSE, + /// or carried with their continuity kept. + pub(super) fn caught_up(&self) -> Vec { + let held = self.held.iter().map(|(route, _)| *route); + let done = self.done.iter().copied(); + done.filter(|route| !self.timed_out.contains(route)) + .chain(held) + .collect() + } +} + +/// What a recovery tick re-sends or rereads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Due { + /// Replace every channel batch with a live-only copy. + pub(super) batches: bool, + pub(super) roster: bool, + /// A full unread repair. + pub(super) unread: bool, +} + +/// How long since each tick item was last done. `None`: never, or a +/// replacement that never established. +#[derive(Clone, Copy, Debug)] +pub(super) struct Clocks { + /// Since the batches were last sent. + pub(super) batches: Option, + /// Since the roster was last read in full. + pub(super) roster: Option, + /// Since a full unread repair was last asked for. + pub(super) unread: Option, +} + +/// A tick's work. `legacy` does all of it every tick. `current` waits for +/// `backstop` since each was last done, and retries owed reads at once. +pub(super) fn tick(mode: DesktopMode, backstop: Duration, clocks: Clocks, owed: Owed) -> Due { + let stale = |since: Option| since.is_none_or(|d| d >= backstop); + match mode { + DesktopMode::Legacy => Due { + batches: true, + roster: true, + unread: true, + }, + DesktopMode::Current => Due { + batches: stale(clocks.batches), + roster: owed.roster || stale(clocks.roster), + unread: owed.unread || owed.snapshot || stale(clocks.unread), + }, + } +} + +/// The HTTP reads a tick makes. +pub(super) fn tick_fetches(due: Due) -> Vec { + let mut reads = Vec::new(); + if due.roster { + reads.push(Fetch::Roster); + } + if due.unread { + reads.push(FULL_REPAIR); + } + reads +} + +/// A window page's position: `(created_at, id)` of the row the next page +/// starts after. +pub(super) type Cursor = (u64, String); + +/// The relay's verdict on a window page: the cursor of the next page, or +/// `None` once history is exhausted. The relay answers every window with +/// one kind 39006 bounds event for this channel and cursor, signed by +/// `relay` (when known); rows, aux events and summaries say nothing about +/// exhaustion. Anything else is an error, as in the app. +pub(super) fn window_bounds( + events: &[Value], + channel: &str, + cursor: Option<&Cursor>, + relay: Option<&str>, +) -> Result, String> { + let suffix = match cursor { + Some((until, id)) => format!("{channel}:{until}:{id}"), + None => format!("{channel}:head"), + }; + let bounds: Vec<&Value> = events + .iter() + .filter(|e| { + e["kind"] == json!(39006) + && relay.is_none_or(|r| e["pubkey"] == json!(r)) + && has_tag(e, "h", channel) + && has_tag(e, "d", &suffix) + }) + .collect(); + let [bound] = bounds[..] else { + return Err(format!("{} window bounds for {suffix}", bounds.len())); + }; + let body: Value = serde_json::from_str(bound["content"].as_str().unwrap_or_default()) + .map_err(|e| format!("window bounds: {e}"))?; + let has_more = body["has_more"] + .as_bool() + .ok_or("window bounds omit has_more")?; + let next = match &body["next_cursor"] { + Value::Null => None, + next => { + let until = next["created_at"].as_u64(); + let id = next["id"].as_str().filter(|id| id.len() == 64); + let (Some(until), Some(id)) = (until, id) else { + return Err("window cursor is malformed".into()); + }; + Some((until, id.to_string())) + } + }; + if has_more != next.is_some() { + return Err("window bounds contradict their cursor".into()); + } + // History runs newest first, then by ascending id at equal times. + if let (Some((until, id)), Some((next_until, next_id))) = (cursor, &next) { + if next_until > until || (next_until == until && next_id <= id) { + return Err("window cursor does not advance".into()); + } + } + Ok(next) +} + +fn has_tag(event: &Value, name: &str, value: &str) -> bool { + event["tags"].as_array().is_some_and(|tags| { + tags.iter() + .any(|t| t[0] == json!(name) && t[1] == json!(value)) + }) +} + +/// The last row of a page in the relay's order (newest first, then by +/// ascending id): the cursor of the page after it. +pub(super) fn last_row(page: &[Value]) -> Option<(u64, &str)> { + page.iter() + .filter_map(|e| Some((e["created_at"].as_u64()?, e["id"].as_str()?))) + .min_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(a.1))) +} + +/// Points `filter` at the page after `cursor`. The relay takes `until` and +/// `before_id` together. +fn page_before(filter: &mut Value, (until, id): (u64, &str)) { + filter["until"] = json!(until); + filter["before_id"] = json!(id); +} + +/// The events of a `/query` response: an array, or the object a read-state +/// snapshot answers with. +fn events(text: &str) -> serde_json::Result> { + match serde_json::from_str(text)? { + Value::Object(mut body) => { + serde_json::from_value(body.remove("events").unwrap_or_default()) + } + other => serde_json::from_value(other), + } +} + +/// Where a resumed subscription starts: [`RESUME_LAG_SECS`] before the +/// last connection dropped, but not before the client's own start floor. +fn resume_since(started: u64, cursor: Option, now: u64) -> u64 { + match cursor { + Some(cursor) => cursor + .saturating_sub(RESUME_LAG_SECS) + .max(started.saturating_sub(STARTUP_FLOOR_SECS)), + None => now.saturating_sub(STARTUP_FLOOR_SECS), + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +/// Up to [`PROFILE_CAP`] other humans in `channels` of `community`, in +/// channel order: the profiles the client holds. +fn co_members( + directory: &Directory, + community: usize, + me: usize, + channels: &[usize], +) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + for &channel in channels { + for member in &directory.members[community][channel] { + if member.role == Role::Human && member.index != me && seen.insert(member.index) { + out.push(directory.humans[member.index].clone()); + if out.len() == PROFILE_CAP { + return out; + } + } + } + } + out +} + +/// What the client's reads have settled: what is still owed and when the +/// roster was last read in full. Its read tasks write it; ticks and route +/// establishments read it. +#[derive(Debug)] +struct Ledger { + owed: Owed, + roster_read: Option, +} + +/// What a client's reads use. Its sessions and read tasks share it. +struct Reader { + shared: Arc, + community: usize, + keys: Keys, + viewer: String, + channels: Vec, + profiles: Vec, + windows: usize, + /// The relay's own pubkey from NIP-11 `self`, the author of notices and + /// window bounds. + relay: std::sync::Mutex>, + /// When the last unread repair started (Unix seconds). + unread_mark: std::sync::Mutex>, + ledger: std::sync::Mutex, +} + +impl Reader { + /// One `POST /query`, timed as `op`. Returns the events, or `None` once + /// it failed (recorded) or the run is stopping (not started). + async fn query(&self, op: Op, filters: Value) -> Option> { + self.query_with(op, filters, |events| Ok(events.to_vec())) + .await + } + + /// [`Reader::query`], where the response only succeeds once `check` + /// accepts its events. + async fn query_with( + &self, + op: Op, + filters: Value, + check: impl FnOnce(&[Value]) -> Result, + ) -> Option { + if stopped(&self.shared) { + return None; + } + let shared = &self.shared; + let base = &shared.communities[self.community].http_url; + let body = filters.to_string().into_bytes(); + let pending = shared.stats.begin(op); + let response = tokio::time::timeout( + HTTP_TIMEOUT, + shared.net.post(&self.keys, base, "/query", body, None), + ) + .await; + match response { + Err(_) => pending.err("read timeout"), + Ok(Err(e)) => pending.err(&e.to_string()), + Ok(Ok((status, text))) if !status.is_success() => { + pending.err(&http_reason(status, &text)) + } + Ok(Ok((_, text))) => match events(&text) { + Ok(events) => match check(&events) { + Ok(value) => { + pending.ok(); + return Some(value); + } + Err(e) => pending.err(&e), + }, + Err(e) => pending.err(&format!("decode: {e}")), + }, + } + None + } + + fn owed(&self) -> Owed { + lock(&self.ledger).owed + } + + async fn fetch(&self, fetch: Fetch) { + let viewer = &self.viewer; + match fetch { + Fetch::Nip11 => self.nip11().await, + Fetch::Roster => { + let read = self.roster().await; + let mut ledger = lock(&self.ledger); + ledger.owed.roster = !read; + if read { + ledger.roster_read = Some(Instant::now()); + } + } + Fetch::Metadata => { + for ids in self.channels.chunks(PAGE) { + let filter = json!([{"kinds": [39000], "#d": ids, "limit": PAGE}]); + self.query(Op::DesktopMetadata, filter).await; + } + } + Fetch::Sidebar => { + let filters: Vec = [ + "channel-sections", + "channel-stars", + "channel-mutes", + "channel-sort", + "channel-filter", + ] + .iter() + .map(|d| json!({"kinds": [30078], "authors": [viewer], "#d": [d], "limit": 1})) + .collect(); + self.query(Op::DesktopSidebar, Value::Array(filters)).await; + } + Fetch::Profiles => { + lock(&self.ledger).owed.profiles = true; + let mut read = true; + for authors in self.profiles.chunks(PAGE) { + let filter = json!([{"kinds": [0], "authors": authors, "limit": PAGE}]); + read &= self.query(Op::DesktopProfiles, filter).await.is_some(); + } + lock(&self.ledger).owed.profiles = !read; + } + Fetch::Statuses => { + let authors = &self.profiles[..self.profiles.len().min(STATUS_AUTHORS)]; + if !authors.is_empty() { + let filter = json!([{ + "kinds": [30315], "#d": ["general"], "authors": authors, "limit": authors.len(), + }]); + self.query(Op::DesktopStatuses, filter).await; + } + } + Fetch::Emoji => { + let filter = json!([{"kinds": [30030], "#d": [EMOJI_SET], "limit": PAGE}]); + self.query(Op::DesktopEmoji, filter).await; + } + Fetch::Unread { snapshot } => { + // The snapshot and the channel repair settle separately: + // a repair that succeeds without the snapshot leaves it owed. + if snapshot { + let filter = json!([{ + "kinds": [30078], "authors": [viewer], "read_state_snapshot": 1, + }]); + let read = self.query(Op::DesktopReadState, filter).await.is_some(); + let mut ledger = lock(&self.ledger); + ledger.owed.snapshot = !read; + if !read { + // The repair waits for the snapshot it builds on. + ledger.owed.unread = true; + return; + } + } + let repaired = self.repair().await; + lock(&self.ledger).owed.unread = !repaired; + } + Fetch::Reminders => { + let filter = json!([{"kinds": [30300], "authors": [viewer], "limit": 201}]); + self.query(Op::DesktopReminders, filter).await; + } + Fetch::Window(i) => self.window(&self.channels[i]).await, + Fetch::Thread(i) => self.thread(i).await, + } + } + + /// The roster in pages until a short page. Whether every page was read. + async fn roster(&self) -> bool { + let mut filter = json!({"kinds": [39002], "#p": [self.viewer], "limit": PAGE}); + for _ in 0..MAX_ROSTER_PAGES { + let Some(page) = self.query(Op::DesktopRoster, json!([filter])).await else { + return false; + }; + match last_row(&page) { + Some(cursor) if page.len() >= PAGE => page_before(&mut filter, cursor), + _ => break, + } + } + true + } + + /// An unread repair of the channels, one chunk after another, as the + /// app does. Whether it all succeeded. + async fn repair(&self) -> bool { + let now = unix_now(); + let mark = lock(&self.unread_mark).replace(now); + for ids in self.channels.chunks(UNREAD_CHANNELS) { + let mut filter = json!({ + "kinds": [9, 40002, 40008], "#h": ids, "include_aux": true, "limit": REPLAY_LIMIT, + }); + if let Some(mark) = mark { + filter["since"] = json!(mark.saturating_sub(RESUME_LAG_SECS)); + } + if self + .query(Op::DesktopUnread, json!([filter])) + .await + .is_none() + { + return false; + } + } + true + } + + /// `GET /` for the relay's NIP-11 document, remembering its `self`. + async fn nip11(&self) { + if stopped(&self.shared) { + return; + } + let shared = &self.shared; + let base = &shared.communities[self.community].http_url; + let pending = shared.stats.begin(Op::DesktopNip11); + match tokio::time::timeout(HTTP_TIMEOUT, shared.net.relay_info(base)).await { + Err(_) => pending.err("read timeout"), + Ok(Err(e)) => pending.err(&e.to_string()), + Ok(Ok((status, text))) if !status.is_success() => { + pending.err(&http_reason(status, &text)) + } + Ok(Ok((_, text))) => { + pending.ok(); + let info: Value = serde_json::from_str(&text).unwrap_or_default(); + if let Some(relay) = info["self"].as_str() { + *lock(&self.relay) = Some(relay.to_string()); + } + } + } + } + + /// A window catch-up: the head page, then older pages while the relay's + /// bounds say there are more (at most [`CATCH_UP_PAGES`]), then the + /// channel's notices. + async fn window(&self, channel: &str) { + let relay = lock(&self.relay).clone(); + let mut cursor: Option = None; + for _ in 0..=CATCH_UP_PAGES { + let mut filter = json!({ + "kinds": WINDOW_KINDS, "#h": [channel], "limit": WINDOW_PAGE, + "top_level": true, "include_aux": true, "include_summaries": true, + }); + if let Some((until, id)) = &cursor { + page_before(&mut filter, (*until, id)); + } + let at = cursor.as_ref(); + let next = self + .query_with(Op::DesktopWindow, json!([filter]), |events| { + window_bounds(events, channel, at, relay.as_deref()) + }) + .await; + match next { + Some(Some(next)) => cursor = Some(next), + _ => break, + } + } + let mut notices = json!({"kinds": [40099], "#h": [channel], "limit": REPLAY_LIMIT}); + if let Some(relay) = relay { + notices["authors"] = json!([relay]); + } + self.query(Op::DesktopNotices, json!([notices])).await; + } + + /// Refreshes open thread `i`: a seeded root in one of the open windows' + /// channels, read by id and then its first thread window page. + async fn thread(&self, i: usize) { + let windows = self.windows.clamp(1, self.channels.len()); + let channel = &self.channels[i % windows]; + let shared = &self.shared; + let community = &shared.communities[self.community]; + let found = roots(shared, community, &self.keys, channel).await; + let Some(root) = found.get((i / windows) % found.len().max(1)) else { + return; + }; + let lookup = json!([{"ids": [root], "#h": [channel], "limit": 1}]); + if self.query(Op::DesktopThreadRoot, lookup).await.is_none() { + return; + } + let page = json!([{ + "kinds": [9, 40002], "#h": [channel], "#e": [root], "depth_limit": 100, + "include_aux": true, "thread_window": true, "limit": 10, + }]); + self.query(Op::DesktopThreadWindow, page).await; + } +} + +/// The value behind `mutex`; a panicked holder never left it half-written. +fn lock(mutex: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// A client's HTTP reads, run by one task at most [`HTTP_CONCURRENCY`] at a +/// time, so reads never pile up behind a slow relay. A read that comes due +/// while the same work is running or queued is not added again; a read +/// leaves the running set as soon as it settles, so a failed one can be +/// retried while others are still under way. Dropping it (the client was +/// cancelled) aborts the task, so its unfinished reads are recorded. +#[derive(Default)] +struct Fetcher { + task: Option>, + work: Arc>, + /// Wakes the task for newly queued reads. + wake: Arc, +} + +/// A [`Fetcher`]'s reads, shared with its task. +#[derive(Default)] +struct Work { + /// Whether the task is under way; it clears this when it runs dry. + active: bool, + running: Vec, + queued: Vec, +} + +impl Work { + /// The next queued read that may start beside the `running` ones, taken + /// off the queue. NIP-11 runs alone and before anything queued with it, + /// since notices and window bounds need the relay's pubkey. + fn next(&mut self) -> Option { + if self.running.len() >= HTTP_CONCURRENCY || self.running.contains(&Fetch::Nip11) { + return None; + } + let i = match self.queued.iter().position(|f| *f == Fetch::Nip11) { + Some(_) if !self.running.is_empty() => return None, + Some(i) => i, + None if self.queued.is_empty() => return None, + None => 0, + }; + let fetch = self.queued.remove(i); + self.running.push(fetch); + Some(fetch) + } +} + +impl Fetcher { + fn start(&mut self, reader: &Arc, fetches: Vec) { + let mut work = lock(&self.work); + for fetch in fetches { + let covered = |have: &Fetch| have.covers(fetch); + if work.running.iter().any(covered) || work.queued.iter().any(covered) { + continue; + } + // A full repair queued now does a queued covered one's work too. + work.queued.retain(|have| !fetch.covers(*have)); + work.queued.push(fetch); + } + if work.queued.is_empty() { + return; + } + if work.active { + self.wake.notify_one(); + return; + } + work.active = true; + drop(work); + let task = drain(reader.clone(), self.work.clone(), self.wake.clone()); + self.task = Some(tokio::spawn(task)); + } + + /// Waits for the reads under way and queued. + async fn finish(&mut self) { + if let Some(task) = self.task.take() { + let _ = task.await; + } + } +} + +/// A [`Fetcher`]'s task: starts queued reads as they may start, until none +/// are running or queued. +async fn drain( + reader: Arc, + work: Arc>, + wake: Arc, +) { + let mut jobs = stream::FuturesUnordered::new(); + loop { + { + let mut work = lock(&work); + while let Some(fetch) = work.next() { + let reader = reader.clone(); + jobs.push(async move { + reader.fetch(fetch).await; + fetch + }); + } + if work.running.is_empty() { + work.active = false; + return; + } + } + tokio::select! { + Some(fetch) = jobs.next() => { + let mut work = lock(&work); + if let Some(i) = work.running.iter().position(|f| *f == fetch) { + work.running.remove(i); + } + } + () = wake.notified() => {} + } + } +} + +impl Drop for Fetcher { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} + +/// How a session ended. +enum End { + Stopped, + /// A planned identity renewal. + Renewal, + /// The human's scheduled session end, an unplanned reconnect. + Dropped, + Failed, +} + +/// A REQ awaiting its EOSE. +struct Waiting<'a> { + route: Route, + /// [`Req::cut_by_design`]. + cut: bool, + /// For a tick's replacement, the batch it takes over at its EOSE. + slot: Option, + /// Events replayed so far. + events: usize, + pending: Pending<'a>, + sent: Instant, +} + +/// A desktop client. Its sessions differ by [`Situation`]: the first is +/// cold, a renewal follows [`renewal`], and any other reconnect is a +/// [`Situation::Reconnect`]. +pub(super) async fn desktop(shared: Arc, global: usize) { + run_client(shared, global, REQ_TIMEOUT).await; +} + +/// [`desktop`], where a REQ may wait `req_timeout` for its EOSE. +async fn run_client(shared: Arc, global: usize, req_timeout: Duration) { + let scenario = &shared.scenario; + let profile = &scenario.desktop; + let community = global % shared.communities.len(); + let keys = identity::keys(&scenario.run_id, Role::Human, global); + let indices = scenario.member_channel_indices(Role::Human, global); + let directory = shared.directory().await; + let reader = Arc::new(Reader { + shared: shared.clone(), + community, + viewer: keys.public_key().to_hex(), + keys: keys.clone(), + channels: scenario.member_channels(community, Role::Human, global), + profiles: co_members(&directory, community, global, &indices), + windows: profile.windows.min(indices.len()), + relay: Default::default(), + unread_mark: Default::default(), + ledger: std::sync::Mutex::new(Ledger { + owed: Owed { + profiles: true, + ..Owed::default() + }, + roster_read: None, + }), + }); + let mut client = Client { + shared: shared.clone(), + reader, + rng: Rng::new(global as u64 ^ 0xD35C_70B0), + fetcher: Fetcher::default(), + situation: Situation::Cold, + req_timeout, + next_tick: Instant::now() + Duration::from_secs_f64(profile.tick_secs), + unread_at: None, + caught_up: Vec::new(), + generation: 0, + first_renewal: true, + last_message: None, + split: scenario.wide_split(Role::Human, global), + }; + let started = unix_now(); + let mut cursor = None; + let mut backoff = Backoff::new(); + let target = &shared.communities[community]; + while let Some(mut conn) = + connect(&shared, target, &keys, None, &mut client.rng, &mut backoff).await + { + let since = resume_since(started, cursor, unix_now()); + let end = client.session(&mut conn, since).await; + cursor = Some(unix_now()); + let _ = conn.disconnect().await; + client.situation = match end { + End::Stopped => break, + End::Renewal => renewal(profile.mode), + End::Dropped => { + backoff = Backoff::new(); + Situation::Reconnect + } + End::Failed => { + // A failed session backs off like a failed connect. + backoff.wait(&shared, &mut client.rng).await; + Situation::Reconnect + } + }; + } + client.fetcher.finish().await; +} + +struct Client { + shared: Arc, + reader: Arc, + rng: Rng, + fetcher: Fetcher, + situation: Situation, + req_timeout: Duration, + /// The recovery tick runs on the client's clock, across connections. + next_tick: Instant, + /// When a full unread repair was last asked for. + unread_at: Option, + /// The routes the last connection had caught up. + caught_up: Vec, + generation: u64, + first_renewal: bool, + last_message: Option, + split: Option, +} + +impl Client { + /// Starts `fetches`. Asking for a full unread repair restarts its + /// backstop; a covered repair reads no snapshot, so it does not. + fn fetch(&mut self, fetches: Vec) { + if fetches.contains(&FULL_REPAIR) { + self.unread_at = Some(Instant::now()); + } + self.fetcher.start(&self.reader, fetches); + } + + /// One connection: establishes per the situation, then runs ticks, + /// presence and the human's own traffic until it ends. + async fn session(&mut self, conn: &mut NostrWsConnection, since: u64) -> End { + let shared = self.shared.clone(); + let stats = &shared.stats; + let scenario = &shared.scenario; + let profile = &scenario.desktop; + let mode = profile.mode; + let reader = self.reader.clone(); + let channels = &reader.channels; + let community = &shared.communities[reader.community]; + let (windows, threads) = (reader.windows, profile.threads); + + self.generation += 1; + let reqs = connect_reqs(mode, &reader.viewer, channels, since, self.generation); + let mut batches: Vec = reqs + .iter() + .filter(|r| r.route == Route::Channels) + .map(|r| r.id.clone()) + .collect(); + // `None` once a replacement missed its deadline: the next tick retries. + let mut batches_at = Some(Instant::now()); + let mut setup: VecDeque<(Req, Option)> = + reqs.into_iter().map(|r| (r, None)).collect(); + self.fetch(connect_fetches(mode, self.situation, windows, threads)); + let mut establishment = Establishment::new(self.situation, &self.caught_up); + + let mut waiting: HashMap> = HashMap::new(); + let mut open: HashSet = HashSet::new(); + let mut activity = Activity::new(&scenario.human, &mut self.rng, self.split); + let tick_every = Duration::from_secs_f64(profile.tick_secs); + let backstop = Duration::from_secs_f64(profile.backstop_secs); + let next_renewal = (profile.renewal_secs > 0.0).then(|| { + // Clients connecting together do not renew together. + let mut secs = profile.renewal_secs; + if std::mem::take(&mut self.first_renewal) { + secs *= self.rng.unit(); + } + Instant::now() + Duration::from_secs_f64(secs) + }); + let disconnected = |e: WsClientError| { + stats.err(Op::WsConnect, &format!("disconnected: {e}")); + End::Failed + }; + + let end = 'session: loop { + if stopped(&shared) { + break End::Stopped; + } + while waiting.len() < SETUP_CONCURRENCY { + let Some((req, slot)) = setup.pop_front() else { + break; + }; + if let Err(e) = conn.send_raw(&req.frame()).await { + break 'session disconnected(e); + } + let op = match req.route { + Route::Replay => Op::DesktopReplay, + _ => Op::DesktopLive, + }; + let waited = Waiting { + route: req.route, + cut: req.cut_by_design(), + slot, + events: 0, + pending: stats.begin(op), + sent: Instant::now(), + }; + waiting.insert(req.id, waited); + } + + // A REQ past its deadline: a replay or a replacement is closed (a + // replacement keeps the batch it would have replaced, and the next + // tick tries again); a connection's live route stays open, and + // establishes as cut, since it may have missed history. + let expired: Vec = waiting + .iter() + .filter(|(_, w)| w.sent.elapsed() >= self.req_timeout) + .map(|(id, _)| id.clone()) + .collect(); + let mut established = Vec::new(); + for id in expired { + let Some(w) = waiting.remove(&id) else { + continue; + }; + w.pending.err("EOSE timeout"); + if w.route == Route::Replay || w.slot.is_some() { + if w.slot.is_some() { + batches_at = None; + } + if let Err(e) = close(conn, &id).await { + break 'session disconnected(e); + } + } else { + open.insert(id); + established.extend(establishment.timeout(w.route)); + } + } + + let due = [activity.due(), Some(self.next_tick), next_renewal] + .into_iter() + .flatten() + .min(); + let wait = due + .map(|d| d.saturating_duration_since(Instant::now())) + .unwrap_or(MAX_IDLE) + .min(MAX_IDLE); + match conn.next_event(wait).await { + Ok(RelayMessage::Eose { subscription_id }) => { + if let Some(w) = waiting.remove(&subscription_id) { + w.pending.ok(); + let retired = match (w.route, w.slot) { + (Route::Replay, _) => Some(subscription_id), + (_, Some(slot)) => { + open.insert(subscription_id.clone()); + let old = std::mem::replace(&mut batches[slot], subscription_id); + open.remove(&old); + Some(old) + } + (route, None) => { + open.insert(subscription_id); + let cut = limited(w.cut, w.events); + established.extend(establishment.eose(route, cut)); + None + } + }; + if let Some(id) = retired { + if let Err(e) = close(conn, &id).await { + break 'session disconnected(e); + } + } + } + } + Ok(RelayMessage::Closed { + subscription_id, + message, + }) => { + // Live subscriptions the relay refuses end the session, + // as for the basic human; a refused replay is an error. + if let Some(w) = waiting.remove(&subscription_id) { + w.pending.err(&message); + if w.route != Route::Replay { + break End::Failed; + } + } else if open.remove(&subscription_id) { + stats.err( + Op::DesktopLive, + &format!("CLOSED {subscription_id}: {message}"), + ); + break End::Failed; + } + } + Ok(message) => { + // An event before EOSE is replay, which the app counts + // against its limit; only events after EOSE are delivery. + if let RelayMessage::Event { + subscription_id, .. + } = &message + { + if let Some(w) = waiting.get_mut(subscription_id) { + w.events += 1; + } else if open.contains(subscription_id) { + if let Some(target) = observe(stats, &message) { + self.last_message = Some(target); + } + } + } + } + Err(WsClientError::Timeout) => {} + Err(e) => break disconnected(e), + } + + // `legacy` read everything when it connected. + if mode == DesktopMode::Current { + let mut reads = Vec::new(); + for (route, continuous) in established { + reads.extend(establish_fetches( + route, + continuous, + reader.owed(), + windows, + threads, + )); + } + if !reads.is_empty() { + self.fetch(reads); + } + } + + let step = activity + .step( + &shared, + community, + &reader.keys, + channels, + conn, + &mut self.rng, + &mut self.last_message, + ) + .await; + match step { + Ok(Step::Continue) => {} + Ok(Step::SessionEnd) => break End::Dropped, + Err(e) => break disconnected(e), + } + + let now = Instant::now(); + if now >= self.next_tick { + self.next_tick = now + tick_every; + let (owed, roster_read) = { + let ledger = lock(&reader.ledger); + (ledger.owed, ledger.roster_read) + }; + let clocks = Clocks { + batches: batches_at.map(|t| now.saturating_duration_since(t)), + roster: roster_read.map(|t| now.saturating_duration_since(t)), + unread: self.unread_at.map(|t| now.saturating_duration_since(t)), + }; + let due = tick(mode, backstop, clocks, owed); + // A replacement still waiting for its EOSE is not replaced again. + let replacing = setup.iter().any(|(_, slot)| slot.is_some()) + || waiting.values().any(|w| w.slot.is_some()); + if due.batches && !replacing { + self.generation += 1; + let fresh = batch_reqs(mode, channels, unix_now(), 0, self.generation); + setup.extend( + fresh + .into_iter() + .zip(0..batches.len()) + .map(|(r, i)| (r, Some(i))), + ); + batches_at = Some(now); + } + self.fetch(tick_fetches(due)); + } + if next_renewal.is_some_and(|t| now >= t) { + break End::Renewal; + } + }; + // Subscriptions still setting up are abandoned with the socket, not failed. + for (_, w) in waiting.drain() { + w.pending.forget(); + } + self.caught_up = establishment.caught_up(); + end + } +} + +async fn close(conn: &mut NostrWsConnection, id: &str) -> Result<(), WsClientError> { + conn.send_raw(&json!(["CLOSE", id])).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ids(n: usize) -> Vec { + (0..n).map(|i| format!("c{i}")).collect() + } + + /// (filters, limit of the first filter, `#h` ids of the first filter, + /// replay) per REQ. + fn shape(reqs: &[Req]) -> Vec<(usize, u64, usize, bool)> { + reqs.iter() + .map(|r| { + let first = &r.filters[0]; + ( + r.filters.len(), + first["limit"].as_u64().unwrap(), + first["#h"].as_array().map_or(0, Vec::len), + r.route == Route::Replay, + ) + }) + .collect() + } + + #[test] + fn up_to_ten_channels_share_one_replaying_batch_in_both_modes() { + for mode in [DesktopMode::Legacy, DesktopMode::Current] { + let reqs = connect_reqs(mode, "v", &ids(5), 100, 1); + let names: Vec<&str> = reqs.iter().map(|r| r.id.as_str()).collect(); + assert_eq!(names, ["profiles", "membership", "batch-1-0"], "{mode:?}"); + assert_eq!(shape(&reqs[2..]), [(5, 500, 1, false)], "{mode:?}"); + assert_eq!(reqs[2].filters[0]["since"], 100); + assert_eq!(reqs[2].filters[0]["kinds"], json!(CHANNEL_KINDS)); + assert!(reqs.iter().all(|r| !r.cut_by_design()), "{mode:?}"); + } + } + + #[test] + fn more_than_ten_channels_batch_by_ten_before_and_replay_separately_after() { + let legacy = connect_reqs(DesktopMode::Legacy, "v", &ids(25), 100, 1); + assert_eq!( + shape(&legacy[2..]), + [(10, 500, 1, false), (10, 500, 1, false), (5, 500, 1, false)] + ); + assert!(legacy.iter().all(|r| !r.cut_by_design())); + let current = connect_reqs(DesktopMode::Current, "v", &ids(25), 100, 1); + assert_eq!( + shape(¤t[2..]), + [ + (1, 0, 25, false), + (10, 500, 1, true), + (10, 500, 1, true), + (5, 500, 1, true) + ] + ); + // The live-only batch replays nothing, so its window counts as cut. + let cut: Vec<_> = current.iter().map(Req::cut_by_design).collect(); + assert_eq!(cut, [false, false, true, false, false, false]); + // One live-only filter holds up to 2048 channels. + let wide = connect_reqs(DesktopMode::Current, "v", &ids(2049), 100, 1); + let live: Vec<_> = wide[2..] + .iter() + .filter(|r| r.route == Route::Channels) + .collect(); + assert_eq!(live.len(), 2); + assert_eq!(wide.len(), 2 + 2 + 205); + } + + #[test] + fn tick_replacements_are_live_only() { + let legacy = batch_reqs(DesktopMode::Legacy, &ids(25), 100, 0, 2); + assert_eq!( + shape(&legacy), + [(10, 0, 1, false), (10, 0, 1, false), (5, 0, 1, false)] + ); + let current = batch_reqs(DesktopMode::Current, &ids(25), 100, 0, 2); + assert_eq!(shape(¤t), [(1, 0, 25, false)]); + let small = batch_reqs(DesktopMode::Current, &ids(5), 100, 0, 2); + assert_eq!(shape(&small), [(5, 0, 1, false)]); + assert_eq!(small[0].id, "batch-2-0"); + } + + #[test] + fn a_replay_is_limited_at_the_apps_cap() { + assert!(!limited(false, REPLAY_LIMIT - 1)); + assert!(limited(false, REPLAY_LIMIT)); + assert!(limited(true, 0)); + } + + const NO_OWED: Owed = Owed { + roster: false, + profiles: false, + snapshot: false, + unread: false, + }; + + #[test] + fn legacy_ticks_do_everything_and_current_ticks_wait_for_the_backstop() { + let backstop = Duration::from_secs(900); + let minute = Some(Duration::from_secs(60)); + let clocks = |batches, roster, unread| Clocks { + batches, + roster, + unread, + }; + let all = Due { + batches: true, + roster: true, + unread: true, + }; + let none = Due { + batches: false, + roster: false, + unread: false, + }; + let fresh = clocks(minute, minute, minute); + assert_eq!(tick(DesktopMode::Legacy, backstop, fresh, NO_OWED), all); + assert_eq!(tick(DesktopMode::Current, backstop, fresh, NO_OWED), none); + let stale = clocks(Some(backstop), Some(backstop), Some(backstop)); + assert_eq!(tick(DesktopMode::Current, backstop, stale, NO_OWED), all); + // Never done counts as stale. + let never = clocks(None, None, None); + assert_eq!(tick(DesktopMode::Current, backstop, never, NO_OWED), all); + // Each backstop runs on its own clock. + let roster_only = tick( + DesktopMode::Current, + backstop, + clocks(minute, Some(backstop), minute), + NO_OWED, + ); + assert_eq!(tick_fetches(roster_only), [Fetch::Roster]); + assert_eq!(tick_fetches(all), [Fetch::Roster, FULL_REPAIR]); + assert!(tick_fetches(none).is_empty()); + } + + #[test] + fn owed_reads_retry_on_the_next_tick_before_their_backstop() { + let backstop = Duration::from_secs(900); + let minute = Some(Duration::from_secs(60)); + let fresh = Clocks { + batches: minute, + roster: minute, + unread: minute, + }; + let owed = Owed { + roster: true, + unread: true, + ..NO_OWED + }; + let due = tick(DesktopMode::Current, backstop, fresh, owed); + assert!(!due.batches); + assert_eq!(tick_fetches(due), [Fetch::Roster, FULL_REPAIR]); + let snapshot = Owed { + snapshot: true, + ..NO_OWED + }; + let due = tick(DesktopMode::Current, backstop, fresh, snapshot); + assert_eq!(tick_fetches(due), [FULL_REPAIR]); + } + + #[test] + fn renewals_reconnect_before_and_carry_routes_after() { + assert_eq!(renewal(DesktopMode::Legacy), Situation::Reconnect); + assert_eq!(renewal(DesktopMode::Current), Situation::Renewal); + + let cold = connect_fetches(DesktopMode::Legacy, Situation::Cold, 2, 1); + assert_eq!(cold.len(), 9 + 3); + let reconnect = connect_fetches(DesktopMode::Legacy, Situation::Reconnect, 2, 1); + // A reconnect rereads everything but names and sidebar preferences. + let skipped: Vec<_> = cold.iter().filter(|r| !reconnect.contains(r)).collect(); + assert_eq!(skipped, [&Fetch::Metadata, &Fetch::Sidebar]); + + // `current` reads up front only what no route covers. + let current = |s| connect_fetches(DesktopMode::Current, s, 2, 1); + assert_eq!( + current(Situation::Cold), + [Fetch::Nip11, Fetch::Metadata, Fetch::Sidebar] + ); + assert_eq!(current(Situation::Reconnect), [Fetch::Nip11]); + assert!(current(Situation::Renewal).is_empty()); + } + + #[test] + fn a_route_establishes_its_reads_by_continuity() { + let channels = [Fetch::Window(0), Fetch::Window(1), Fetch::Thread(0)]; + let est = |route, continuous, owed| establish_fetches(route, continuous, owed, 2, 1); + assert_eq!( + est(Route::Profiles, false, NO_OWED), + [Fetch::Profiles, Fetch::Statuses, Fetch::Emoji] + ); + assert!(est(Route::Profiles, true, NO_OWED).is_empty()); + let profiles_owed = Owed { + profiles: true, + ..NO_OWED + }; + assert_eq!(est(Route::Profiles, true, profiles_owed), [Fetch::Profiles]); + assert_eq!( + est(Route::Membership, false, NO_OWED), + [Fetch::Roster, FULL_REPAIR, Fetch::Reminders] + ); + assert_eq!(est(Route::Membership, true, NO_OWED), [COVERED_REPAIR]); + let roster_owed = Owed { + roster: true, + ..NO_OWED + }; + assert_eq!( + est(Route::Membership, true, roster_owed), + [Fetch::Roster, COVERED_REPAIR] + ); + // A continuous route cannot settle an owed snapshot without reading it. + let snapshot_owed = Owed { + snapshot: true, + ..NO_OWED + }; + assert_eq!(est(Route::Membership, true, snapshot_owed), [FULL_REPAIR]); + assert_eq!(est(Route::Channels, true, NO_OWED), channels); + assert_eq!(est(Route::Channels, false, NO_OWED), channels); + assert!(est(Route::Replay, false, NO_OWED).is_empty()); + assert!(FULL_REPAIR.covers(COVERED_REPAIR)); + assert!(!COVERED_REPAIR.covers(FULL_REPAIR)); + } + + #[test] + fn routes_establish_once_and_a_renewal_holds_them_until_a_cut() { + let all = [Route::Profiles, Route::Membership, Route::Channels]; + // Cold: every route establishes at its EOSE, not continuous. + let mut cold = Establishment::new(Situation::Cold, &all); + assert_eq!( + cold.eose(Route::Profiles, false), + [(Route::Profiles, false)] + ); + assert!(cold.eose(Route::Replay, false).is_empty()); + assert_eq!( + cold.eose(Route::Channels, false), + [(Route::Channels, false)] + ); + assert!(cold.eose(Route::Channels, false).is_empty()); + assert_eq!(cold.caught_up(), [Route::Profiles, Route::Channels]); + + // Reconnect: each route by its own replay, and only if the last + // connection had it caught up. + let mut reconnect = Establishment::new(Situation::Reconnect, &all[..2]); + assert_eq!( + reconnect.eose(Route::Membership, false), + [(Route::Membership, true)] + ); + assert_eq!( + reconnect.eose(Route::Profiles, true), + [(Route::Profiles, false)] + ); + assert_eq!( + reconnect.eose(Route::Channels, false), + [(Route::Channels, false)] + ); + + // Renewal without a cut: nothing establishes, all stay caught up. + let mut quiet = Establishment::new(Situation::Renewal, &all); + for route in all { + assert!(quiet.eose(route, false).is_empty()); + } + assert_eq!(quiet.caught_up(), all); + + // Renewal with a cut: the held routes establish with it. + let mut cut = Establishment::new(Situation::Renewal, &all); + assert!(cut.eose(Route::Profiles, false).is_empty()); + assert_eq!( + cut.eose(Route::Channels, true), + [(Route::Profiles, true), (Route::Channels, false)] + ); + assert_eq!( + cut.eose(Route::Membership, false), + [(Route::Membership, true)] + ); + + // A timed-out route recovers as cut, but is not caught up. + let mut late = Establishment::new(Situation::Reconnect, &all); + assert_eq!( + late.timeout(Route::Membership), + [(Route::Membership, false)] + ); + assert_eq!(late.eose(Route::Profiles, false), [(Route::Profiles, true)]); + assert_eq!(late.caught_up(), [Route::Profiles]); + + // A renewal before a route ever caught up reads it in full. + let mut early = Establishment::new(Situation::Renewal, &all[..1]); + assert_eq!( + early.eose(Route::Membership, false), + [(Route::Membership, false)] + ); + } + + #[test] + fn resumed_subscriptions_start_before_the_cursor_but_not_before_the_floor() { + assert_eq!(resume_since(10_000, None, 20_000), 19_700); + assert_eq!(resume_since(10_000, Some(15_000), 20_000), 14_040); + assert_eq!(resume_since(10_000, Some(10_100), 20_000), 9_700); + } + + fn hex(c: char) -> String { + std::iter::repeat_n(c, 64).collect() + } + + fn bound(d: &str, content: Value) -> Value { + json!({ + "kind": 39006, "pubkey": "relay", "content": content.to_string(), + "tags": [["d", d], ["h", "c0"]], + }) + } + + #[test] + fn window_pages_follow_the_relays_bounds() { + let next = json!({"has_more": true, "next_cursor": {"created_at": 1000, "id": hex('b')}}); + let row = json!({"kind": 9, "created_at": 1000, "id": hex('a')}); + let head = [row.clone(), bound("c0:head", next.clone())]; + let cursor = (1000, hex('b')); + assert_eq!( + window_bounds(&head, "c0", None, Some("relay")), + Ok(Some(cursor.clone())) + ); + // Exhausted. + let done = json!({"has_more": false, "next_cursor": null}); + let d = format!("c0:1000:{}", hex('b')); + let last = [bound(&d, done.clone())]; + assert_eq!(window_bounds(&last, "c0", Some(&cursor), None), Ok(None)); + // No bounds, two bounds, another signer, or the wrong cursor. + assert!(window_bounds(&[row], "c0", None, None).is_err()); + let twice = [ + bound("c0:head", next.clone()), + bound("c0:head", next.clone()), + ]; + assert!(window_bounds(&twice, "c0", None, None).is_err()); + assert!(window_bounds(&head, "c0", None, Some("other")).is_err()); + assert!(window_bounds(&last, "c0", None, None).is_err()); + // Contradictions and a cursor that does not move on. + let contradict = + json!({"has_more": false, "next_cursor": {"created_at": 1000, "id": hex('b')}}); + assert!(window_bounds(&[bound("c0:head", contradict)], "c0", None, None).is_err()); + let back = [bound( + &d, + json!({"has_more": true, "next_cursor": {"created_at": 1000, "id": hex('a')}}), + )]; + assert!(window_bounds(&back, "c0", Some(&cursor), None).is_err()); + let short = json!({"has_more": true, "next_cursor": {"created_at": 1000, "id": "ab"}}); + assert!(window_bounds(&[bound("c0:head", short)], "c0", None, None).is_err()); + } + + #[test] + fn the_last_row_is_the_oldest_then_the_highest_id() { + let page = [ + json!({"created_at": 1000, "id": "b"}), + json!({"created_at": 999, "id": "a"}), + json!({"created_at": 999, "id": "c"}), + json!({"created_at": 1001, "id": "z"}), + ]; + assert_eq!(last_row(&page), Some((999, "c"))); + assert_eq!(last_row(&[]), None); + } + + // The client against loopback fakes: every connect, REQ, CLOSE and HTTP + // read below goes through the production paths. + + use super::super::net::Net; + use super::super::run::kind; + use super::super::scenario::Scenario; + use super::super::seed::tests::fake_http; + use super::super::stats::Stats; + use nostr::{EventBuilder, Tag}; + use tokio::sync::watch; + + type Shared2 = Arc>; + + /// `(connection, verb, subscription id)` of every REQ and CLOSE the fake + /// relay got, connections numbered as accepted. + type Frames = Shared2>; + + /// How the fake relay answers a REQ on a connection: events replayed + /// before its EOSE, and how long the EOSE waits. + type Plan = Arc (usize, Duration) + Send + Sync>; + + fn instant() -> Plan { + Arc::new(|_, _| (0, Duration::ZERO)) + } + + /// A loopback relay that challenges, accepts AUTH and every EVENT, answers + /// each REQ per `plan` and records REQs and CLOSEs. With `drop_first`, it + /// closes its first connection that long after accepting it. + async fn fake_ws_relay(plan: Plan, drop_first: Option) -> (String, Frames) { + use futures_util::SinkExt; + use tokio_tungstenite::tungstenite::Message; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let frames = Frames::default(); + let seen = frames.clone(); + let replayed = EventBuilder::new(kind(9), "replayed") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let replayed = serde_json::to_value(replayed).unwrap(); + tokio::spawn(async move { + let mut conn = 0; + while let Ok((stream, _)) = listener.accept().await { + let (seen, plan, replayed) = (seen.clone(), plan.clone(), replayed.clone()); + let index = conn; + conn += 1; + tokio::spawn(async move { + let Ok(ws) = tokio_tungstenite::accept_async(stream).await else { + return; + }; + let (mut sink, mut stream) = ws.split(); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + let close = matches!(message, Message::Close(_)); + if sink.send(message).await.is_err() || close { + return; + } + } + }); + let text = |v: Value| Message::Text(v.to_string().into()); + let _ = tx.send(text(json!(["AUTH", "challenge"]))); + if let Some(after) = drop_first.filter(|_| index == 0) { + let tx = tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(after).await; + let _ = tx.send(Message::Close(None)); + }); + } + while let Some(Ok(Message::Text(raw))) = stream.next().await { + let frame: Value = serde_json::from_str(&raw).unwrap(); + let verb = frame[0].as_str().unwrap_or_default().to_string(); + let id = frame[1].as_str().unwrap_or_default().to_string(); + match verb.as_str() { + "AUTH" | "EVENT" => { + let _ = tx.send(text(json!(["OK", frame[1]["id"], true, ""]))); + } + "REQ" => { + let (events, delay) = plan(index, &id); + let (tx, replayed, id) = (tx.clone(), replayed.clone(), id.clone()); + tokio::spawn(async move { + for _ in 0..events { + let _ = tx.send(text(json!(["EVENT", id, replayed]))); + } + tokio::time::sleep(delay).await; + let _ = tx.send(text(json!(["EOSE", id]))); + }); + } + _ => {} + } + if verb == "REQ" || verb == "CLOSE" { + seen.lock().unwrap().push((index, verb, id)); + } + } + }); + } + }); + (addr, frames) + } + + /// Reads the fake API fails once each, by name: `roster`, `read_state`, + /// `unread`, `reminders`. + type FailOnce = &'static [&'static str]; + + /// A read the fake API answers late, by name, and how late. + type Slow = Option<(&'static str, Duration)>; + + fn signed(keys: &Keys, k: u32, content: &str, at: u64, tags: Vec) -> Value { + let event = EventBuilder::new(kind(k), content) + .custom_created_at(nostr::Timestamp::from(at)) + .tags(tags) + .sign_with_keys(keys) + .unwrap(); + serde_json::to_value(event).unwrap() + } + + fn tag(name: &str, value: &str) -> Tag { + Tag::parse([name, value]).unwrap() + } + + /// A loopback HTTP API, recording every query's first filter. NIP-11 + /// names the relay's key; the owner's root search finds one seeded root. + /// A window head page holds a full page of rows at one timestamp, an + /// older aux event and a summary; its relay-signed bounds point after the + /// last row. The page there is full too, and its bounds say history is + /// exhausted. Any other window cursor is refused. The read-state snapshot + /// answers in its own shape; `fail` reads fail the first time, and a + /// `slow` read blocks its connection's handler (run such a test on the + /// multi-thread runtime). + async fn fake_api(run_id: &str, fail: FailOnce, slow: Slow) -> (String, Shared2>) { + let owner = identity::keys(run_id, Role::Owner, 0); + let relay = Keys::generate(); + let root = Value::Array(vec![signed(&owner, 9, "lg-root 0", 2_000, vec![])]); + let rows = |at: u64| -> Vec { + (0..WINDOW_PAGE) + .map(|i| signed(&owner, 9, &format!("w{i}"), at, vec![])) + .collect() + }; + let head_rows = rows(1_000); + let last = head_rows + .iter() + .filter_map(|e| e["id"].as_str()) + .max() + .unwrap() + .to_string(); + let older = rows(990); + let info = json!({"self": relay.public_key().to_hex()}).to_string(); + let owner_hex = owner.public_key().to_hex(); + let queries = Shared2::>::default(); + let recorded = queries.clone(); + let failed = Shared2::>::default(); + let snapshot = + json!({"read_state_snapshot": 1, "complete": true, "events": []}).to_string(); + let http = fake_http(move |method, target, body| { + if method == "GET" && target == "/" { + return (200, info.clone()); + } + let filters: Value = serde_json::from_slice(body).unwrap_or_default(); + let filter = filters[0].clone(); + recorded.lock().unwrap().push(filter.clone()); + let name = if filter["kinds"] == json!([39002]) { + "roster" + } else if filter["kinds"] == json!([9, 40002, 40008]) { + "unread" + } else if filter.get("read_state_snapshot").is_some() { + "read_state" + } else if filter["kinds"] == json!([30300]) { + "reminders" + } else { + "" + }; + if let Some((_, delay)) = slow.filter(|(slow, _)| *slow == name) { + std::thread::sleep(delay); + } + if fail.contains(&name) && !failed.lock().unwrap().contains(&name) { + failed.lock().unwrap().push(name); + return (500, r#"{"error":"once"}"#.into()); + } + if filter.get("read_state_snapshot").is_some() { + return (200, snapshot.clone()); + } + if filter.get("top_level").is_some() { + let channel = filter["#h"][0].as_str().unwrap_or_default(); + let (d, mut page, next) = match (filter.get("until"), filter.get("before_id")) { + (None, None) => { + let mut page = head_rows.clone(); + page.push(signed(&owner, 7, "+", 900, vec![tag("h", channel)])); + page.push(signed(&relay, 39005, "{}", 1_000, vec![tag("h", channel)])); + let next = json!({"created_at": 1_000, "id": last}); + (format!("{channel}:head"), page, next) + } + (Some(until), Some(id)) if *until == json!(1_000) && *id == json!(last) => { + (format!("{channel}:1000:{last}"), older.clone(), Value::Null) + } + _ => return (400, r#"{"error":"unknown window cursor"}"#.into()), + }; + let content = json!({"has_more": !next.is_null(), "next_cursor": next}); + let tags = vec![tag("d", &d), tag("h", channel)]; + page.push(signed(&relay, 39006, &content.to_string(), 1_000, tags)); + return (200, Value::Array(page).to_string()); + } + if filter["kinds"] == json!([9]) && filter["authors"] == json!([owner_hex]) { + (200, root.to_string()) + } else { + (200, "[]".into()) + } + }) + .await; + (http, queries) + } + + /// `humans` desktop humans in one community of 25 channels, each in + /// `joined`, with two windows and one thread open and no traffic of + /// their own. + fn desktops(ws: &str, http: &str, humans: usize, joined: usize, desktop: &str) -> Scenario { + Scenario::parse(&format!( + r#" +run_id = "d" +[target] +ws_url = "ws://{ws}" +http_url = "http://{http}" +[shape] +communities = 1 +channels_per_community = 25 +[population] +humans = {humans} +agents = 0 +channels_per_member = {joined} +[human] +presence_interval_secs = 3600.0 +typing_per_hour = 0.0 +messages_per_hour = 0.0 +reactions_per_hour = 0.0 +history_limit = 1 +[desktop] +fraction = 1.0 +windows = 2 +threads = 1 +{desktop} +[[ramp]] +users = 1.0 +hold_secs = 1 +"# + )) + .unwrap() + } + + /// Per-op `(count, errors)` by op name, plus the relay's frames and the + /// API's queries. + struct Outcome { + ops: HashMap<&'static str, (u64, u64)>, + frames: Vec<(usize, String, String)>, + queries: Vec, + } + + impl Outcome { + fn count(&self, op: &str) -> u64 { + self.ops.get(op).map_or(0, |c| c.0) + } + + fn errors(&self, op: &str) -> u64 { + self.ops.get(op).map_or(0, |c| c.1) + } + + fn saw(&self, verb: &str, id: &str) -> bool { + self.at(verb, id).is_some() + } + + /// Where a frame came in the relay's record. + fn at(&self, verb: &str, id: &str) -> Option { + self.frames + .iter() + .position(|(_, v, i)| v == verb && i == id) + } + + /// Counts of the desktop ops, the ones the model owns. + fn desktop(&self) -> HashMap<&'static str, u64> { + self.ops + .iter() + .filter(|(op, _)| op.starts_with("desktop_")) + .map(|(op, c)| (*op, c.0)) + .collect() + } + + /// Desktop op counts beyond `before`, zeroes dropped. + fn since(&self, before: &HashMap<&'static str, u64>) -> HashMap<&'static str, u64> { + self.desktop() + .into_iter() + .map(|(op, n)| (op, n - before.get(op).copied().unwrap_or(0))) + .filter(|(_, n)| *n > 0) + .collect() + } + } + + fn outcome(stats: &Stats, frames: &Frames, queries: &Shared2>) -> Outcome { + let ops = stats + .snapshot() + .since(&Default::default()) + .into_iter() + .filter(|(_, s)| s.count + s.errors > 0) + .map(|(op, s)| (op.name(), (s.count, s.errors))) + .collect(); + Outcome { + ops, + frames: frames.lock().unwrap().clone(), + queries: queries.lock().unwrap().clone(), + } + } + + /// One loopback run. + struct Rig { + /// Humans in the scenario, whose profiles a client reads. + humans: usize, + /// Clients that run: the first `clients` humans. + clients: usize, + joined: usize, + desktop: String, + plan: Plan, + drop_first: Option, + fail: FailOnce, + slow: Slow, + req_timeout: Duration, + } + + /// One client, with a co-member whose profile it reads. + fn solo(joined: usize, desktop: &str) -> Rig { + Rig { + clients: 1, + ..rig(2, joined, desktop) + } + } + + fn rig(humans: usize, joined: usize, desktop: &str) -> Rig { + Rig { + humans, + clients: humans, + joined, + desktop: desktop.into(), + plan: instant(), + drop_first: None, + fail: &[], + slow: None, + req_timeout: REQ_TIMEOUT, + } + } + + /// Runs the clients for at least `min`, until `done` holds, then stops + /// them and returns what they did. Fails after 15 s without `done`. + async fn drive(rig: Rig, min: Duration, done: impl Fn(&Outcome) -> bool) -> Outcome { + let (ws, frames) = fake_ws_relay(rig.plan, rig.drop_first).await; + let (http, queries) = fake_api("d", rig.fail, rig.slow).await; + let scenario = desktops(&ws, &http, rig.humans, rig.joined, &rig.desktop); + let net = Net::new(&scenario).await.unwrap(); + let stats = Arc::new(Stats::default()); + let (stop_tx, stop) = watch::channel(false); + let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); + let clients: Vec<_> = (0..rig.clients) + .map(|g| tokio::spawn(run_client(shared.clone(), g, rig.req_timeout))) + .collect(); + let started = Instant::now(); + loop { + let now = outcome(&stats, &frames, &queries); + if started.elapsed() >= min && done(&now) { + break; + } + assert!( + started.elapsed() < Duration::from_secs(15), + "never done: {:?} {:?}", + now.ops, + now.frames + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + stop_tx.send(true).unwrap(); + for client in clients { + tokio::time::timeout(Duration::from_secs(5), client) + .await + .expect("a stopped client returns") + .unwrap(); + } + outcome(&stats, &frames, &queries) + } + + fn no_errors(out: &Outcome) { + let failed: Vec<_> = out.ops.iter().filter(|(_, c)| c.1 > 0).collect(); + assert!(failed.is_empty(), "errors: {failed:?}"); + } + + /// What one cold start reads over HTTP with two windows and a thread. + fn cold_reads(clients: u64) -> HashMap<&'static str, u64> { + [ + ("desktop_nip11", 1), + ("desktop_roster", 1), + ("desktop_metadata", 1), + ("desktop_sidebar", 1), + ("desktop_profiles", 1), + ("desktop_statuses", 1), + ("desktop_emoji", 1), + ("desktop_read_state", 1), + ("desktop_unread", 1), + ("desktop_reminders", 1), + // Per window, the head page and the page its bounds point to. + ("desktop_window", 4), + ("desktop_notices", 2), + ("desktop_thread_root", 1), + ("desktop_thread_window", 1), + ] + .into_iter() + .map(|(op, n)| (op, n * clients)) + .collect() + } + + fn with( + mut ops: HashMap<&'static str, u64>, + extra: &[(&'static str, u64)], + ) -> HashMap<&'static str, u64> { + ops.extend(extra.iter().copied()); + ops + } + + fn ops(list: &[(&'static str, u64)]) -> HashMap<&'static str, u64> { + list.iter().copied().collect() + } + + /// What a channel route's establishment reads: both windows (two pages + /// and notices each) and the thread. + const CHANNEL_RECOVERY: [(&str, u64); 4] = [ + ("desktop_window", 4), + ("desktop_notices", 2), + ("desktop_thread_root", 1), + ("desktop_thread_window", 1), + ]; + + const QUIET: &str = "tick_secs = 3600.0"; + + #[tokio::test] + async fn a_cold_start_sends_the_modeled_requests_once() { + // Legacy: per-channel batches of ten for 25 channels; current: one + // live-only batch and three replays. Both read the same over HTTP. + let cases = [ + ("legacy", 25, vec![("desktop_live_eose", 2 * 5)]), + ( + "current", + 25, + vec![("desktop_live_eose", 2 * 3), ("desktop_replay_eose", 2 * 3)], + ), + ("legacy", 5, vec![("desktop_live_eose", 2 * 3)]), + ("current", 5, vec![("desktop_live_eose", 2 * 3)]), + ]; + for (mode, joined, subs) in cases { + let want = with(cold_reads(2), &subs); + let toml = format!("mode = \"{mode}\"\n{QUIET}"); + let out = drive(rig(2, joined, &toml), Duration::from_millis(300), |o| { + o.desktop() == want + }) + .await; + no_errors(&out); + assert_eq!(out.desktop(), want, "{mode} J={joined}"); + assert_eq!(out.count("ws_connect_auth"), 2, "{mode} J={joined}"); + } + } + + #[tokio::test] + async fn a_window_catch_up_follows_the_relays_cursor_until_it_is_exhausted() { + let toml = format!("mode = \"current\"\n{QUIET}"); + let out = drive(solo(5, &toml), Duration::ZERO, |o| { + o.count("desktop_notices") == 2 + }) + .await; + no_errors(&out); + let top: Vec<&Value> = out + .queries + .iter() + .filter(|f| f.get("top_level").is_some()) + .collect(); + let channel = &top[0]["#h"]; + let windows: Vec<&Value> = top.into_iter().filter(|f| f["#h"] == *channel).collect(); + // The head, then the page after the head's last row (not after its + // older aux event), and no third read of an exhausted history, + // though that page came back full. + assert_eq!(windows.len(), 2, "{windows:?}"); + assert!(windows[0].get("until").is_none()); + assert_eq!(windows[1]["until"], 1_000); + let ids: Vec<&str> = out + .queries + .iter() + .filter(|f| f.get("before_id").is_some()) + .filter_map(|f| f["before_id"].as_str()) + .collect(); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|id| id.len() == 64 && *id == ids[0])); + assert_eq!(out.count("desktop_window"), 4); + } + + /// Holds once `ready` has held for `settle`: what a connection reads has + /// had time to finish. + fn settled(ready: impl Fn(&Outcome) -> bool, settle: Duration) -> impl Fn(&Outcome) -> bool { + let since = std::cell::Cell::new(None::); + move |o| { + if !ready(o) { + return false; + } + let at = since.get().unwrap_or_else(Instant::now); + since.set(Some(at)); + at.elapsed() >= settle + } + } + + /// The reads beyond its cold start of a `current` client in 5 channels, + /// once its first renewal's connection established with each replay + /// sending 499 events, except 500 on subscriptions whose id starts + /// with `cut`. + async fn renewed(cut: &'static str) -> HashMap<&'static str, u64> { + let mut rig = solo( + 5, + "mode = \"current\"\nrenewal_secs = 4.0\ntick_secs = 3600.0", + ); + rig.plan = Arc::new(move |conn, id| { + let events = match conn { + 0 => 0, + _ if id.starts_with(cut) => REPLAY_LIMIT, + _ => REPLAY_LIMIT - 1, + }; + (events, Duration::ZERO) + }); + let cold = with(cold_reads(1), &[("desktop_live_eose", 3)]); + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("desktop_live_eose") >= 6, + Duration::from_millis(800), + ), + ) + .await; + no_errors(&out); + assert_eq!(out.count("ws_connect_auth"), 2, "{cut}"); + out.since(&cold) + .into_iter() + .filter(|(op, _)| *op != "desktop_live_eose") + .collect() + } + + #[tokio::test] + async fn a_renewal_recovers_only_what_a_cut_replay_lost() { + // 499 events everywhere: every route kept continuity. + assert_eq!(renewed("none").await, ops(&[])); + // A cut channel replay: the windows and the thread recover, and the + // carried membership route only repairs unreads. + assert_eq!( + renewed("batch").await, + ops(&[CHANNEL_RECOVERY.as_slice(), &[("desktop_unread", 1)]].concat()) + ); + // A cut membership replay: its full refresh, and the channels. + assert_eq!( + renewed("membership").await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[ + ("desktop_roster", 1), + ("desktop_read_state", 1), + ("desktop_unread", 1), + ("desktop_reminders", 1), + ] + ] + .concat()) + ); + // A cut profiles replay: its refresh, the channels, and the covered + // membership route's repair. + assert_eq!( + renewed("profiles").await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[ + ("desktop_profiles", 1), + ("desktop_statuses", 1), + ("desktop_emoji", 1), + ("desktop_unread", 1), + ] + ] + .concat()) + ); + } + + /// The reads beyond its cold start of a `current` client in 5 channels + /// whose first connection drops after it established, once the next + /// connection established; its global replays send `globals` events. + async fn reconnected(globals: usize) -> HashMap<&'static str, u64> { + let mut rig = solo(5, &format!("mode = \"current\"\n{QUIET}")); + rig.drop_first = Some(Duration::from_millis(700)); + rig.plan = Arc::new(move |conn, id| { + let global = id == "profiles" || id == "membership"; + let events = if conn > 0 && global { globals } else { 0 }; + (events, Duration::ZERO) + }); + let cold = with(cold_reads(1), &[("desktop_live_eose", 3)]); + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("ws_connect_auth") >= 2 && o.count("desktop_live_eose") >= 6, + Duration::from_millis(800), + ), + ) + .await; + // The drop is the only error. + let failed: Vec<_> = out + .ops + .iter() + .filter(|(op, c)| c.1 > 0 && **op != "ws_connect_auth") + .collect(); + assert!(failed.is_empty(), "errors: {failed:?}"); + assert_eq!(out.count("ws_connect_auth"), 2); + out.since(&cold) + .into_iter() + .filter(|(op, _)| *op != "desktop_live_eose") + .collect() + } + + #[tokio::test] + async fn a_reconnect_rereads_a_global_only_when_its_replay_was_cut() { + // Complete replays: the globals resume, unreads are only repaired, + // and the channels recover. + assert_eq!( + reconnected(REPLAY_LIMIT - 1).await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[("desktop_nip11", 1), ("desktop_unread", 1)] + ] + .concat()) + ); + // Truncated global replays: both globals refresh in full. + assert_eq!( + reconnected(REPLAY_LIMIT).await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[ + ("desktop_nip11", 1), + ("desktop_profiles", 1), + ("desktop_statuses", 1), + ("desktop_emoji", 1), + ("desktop_roster", 1), + ("desktop_read_state", 1), + ("desktop_unread", 1), + ("desktop_reminders", 1), + ] + ] + .concat()) + ); + } + + #[tokio::test] + async fn read_state_snapshots_recur_under_renewals_shorter_than_the_backstop() { + // Every renewal of 25 channels establishes (the live-only batch is + // cut by design) and repairs unreads without a snapshot; that must + // not hold off the snapshot's backstop. + let toml = "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 2.0\nrenewal_secs = 1.0"; + let out = drive(solo(25, toml), Duration::ZERO, |o| { + o.count("desktop_read_state") >= 2 && o.count("ws_connect_auth") >= 3 + }) + .await; + no_errors(&out); + assert!(out.count("desktop_unread") > out.count("desktop_read_state")); + } + + #[tokio::test] + async fn a_replacement_past_its_deadline_is_closed_and_the_next_tick_retries() { + let mut rig = solo( + 25, + "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 1.0", + ); + rig.req_timeout = Duration::from_millis(300); + rig.plan = Arc::new(|_, id| { + let late = if id == "batch-2-0" { + Duration::from_secs(1) + } else { + Duration::ZERO + }; + (0, late) + }); + let out = drive(rig, Duration::ZERO, |o| o.saw("CLOSE", "batch-1-0")).await; + assert_eq!(out.errors("desktop_live_eose"), 1); + // The late replacement is closed; the next tick replaces the batch. + let closed = out + .at("CLOSE", "batch-2-0") + .expect("late replacement closed"); + let next = out.at("REQ", "batch-3-0").expect("next tick replaced"); + let retired = out.at("CLOSE", "batch-1-0").expect("old batch closed"); + assert!(closed < next && next < retired, "{:?}", out.frames); + // One batch wire stays open. + let opened = |verb: &str| { + out.frames + .iter() + .filter(|(_, v, id)| v == verb && id.starts_with("batch-")) + .count() + }; + assert_eq!(opened("REQ") - opened("CLOSE"), 1, "{:?}", out.frames); + } + + #[tokio::test] + async fn a_failed_read_retries_on_the_next_tick_and_a_good_one_does_not() { + let mut rig = solo(25, "mode = \"current\"\ntick_secs = 1.0"); + rig.fail = &["roster", "unread"]; + let out = drive(rig, Duration::from_millis(3_500), |o| { + o.count("desktop_roster") == 1 && o.count("desktop_unread") == 1 + }) + .await; + let ops = &out.ops; + assert_eq!( + (out.count("desktop_roster"), out.errors("desktop_roster")), + (1, 1), + "{ops:?}" + ); + assert_eq!( + (out.count("desktop_unread"), out.errors("desktop_unread")), + (1, 1), + "{ops:?}" + ); + // The retry reread the snapshot with its repair; nothing else reran. + assert_eq!(out.count("desktop_read_state"), 2); + assert_eq!(out.count("desktop_reminders"), 1); + assert_eq!(out.count("desktop_profiles"), 1); + } + + #[tokio::test] + async fn an_owed_snapshot_survives_a_covered_repair() { + // The cold snapshot read fails. Before any tick, the connection + // drops and the membership route resumes with a complete replay: + // its repair must read the snapshot still owed. + let mut rig = solo(5, &format!("mode = \"current\"\n{QUIET}")); + rig.drop_first = Some(Duration::from_millis(700)); + rig.fail = &["read_state"]; + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("ws_connect_auth") >= 2 && o.count("desktop_live_eose") >= 6, + Duration::from_millis(800), + ), + ) + .await; + let ops = &out.ops; + assert_eq!( + ( + out.count("desktop_read_state"), + out.errors("desktop_read_state") + ), + (1, 1), + "{ops:?}" + ); + // The failed snapshot held the cold repair back; the resumed route + // repaired once and reread neither roster nor reminders. + assert_eq!(out.count("desktop_unread"), 1, "{ops:?}"); + assert_eq!(out.count("desktop_roster"), 1, "{ops:?}"); + assert_eq!(out.count("desktop_reminders"), 1, "{ops:?}"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_failed_read_retries_while_an_unrelated_slow_one_runs() { + // The cold roster read fails while reminders take 3 s; the tick at + // 1 s retries the roster without waiting for reminders. + let mut rig = solo(5, "mode = \"current\"\ntick_secs = 1.0"); + rig.fail = &["roster"]; + rig.slow = Some(("reminders", Duration::from_secs(3))); + let reminders = std::cell::Cell::new(None); + let out = drive(rig, Duration::ZERO, |o| { + let retried = o.count("desktop_roster") == 1; + if retried && reminders.get().is_none() { + reminders.set(Some(o.count("desktop_reminders"))); + } + retried + }) + .await; + assert_eq!( + reminders.get(), + Some(0), + "the roster retry waited for reminders" + ); + assert_eq!(out.errors("desktop_roster"), 1); + assert_eq!(out.count("desktop_reminders"), 1); + } + + #[tokio::test] + async fn a_timed_out_route_is_recovered_again_after_a_renewal() { + // The first connection's membership EOSE never comes: the route is + // recovered as cut, but it was never live. The renewal's complete + // replay therefore cannot carry it, and it recovers in full again. + // The first renewal is jittered; with this period it comes after the + // short deadline and before the test ends. + let mut rig = solo( + 5, + "mode = \"current\"\nrenewal_secs = 20.0\ntick_secs = 3600.0", + ); + rig.req_timeout = Duration::from_millis(100); + rig.plan = Arc::new(|conn, id| { + let withheld = conn == 0 && id == "membership"; + let delay = if withheld { + Duration::from_secs(60) + } else { + Duration::ZERO + }; + (0, delay) + }); + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("ws_connect_auth") >= 2 && o.count("desktop_live_eose") >= 5, + Duration::from_millis(800), + ), + ) + .await; + let ops = &out.ops; + assert_eq!(out.errors("desktop_live_eose"), 1, "{ops:?}"); + assert_eq!(out.count("ws_connect_auth"), 2, "{ops:?}"); + for op in ["desktop_roster", "desktop_read_state", "desktop_reminders"] { + assert_eq!(out.count(op), 2, "{op}: {ops:?}"); + } + // The routes that were live carried: no second profile read. + assert_eq!(out.count("desktop_profiles"), 1, "{ops:?}"); + } + + #[tokio::test] + async fn legacy_ticks_reread_and_replace_batches_every_tick() { + let out = drive( + rig(2, 25, "mode = \"legacy\"\ntick_secs = 1.0"), + Duration::ZERO, + |o| o.count("desktop_roster") >= 4 && o.saw("CLOSE", "batch-1-2"), + ) + .await; + no_errors(&out); + // Each replacement is sent before the batch it replaces is closed. + assert!(out.saw("REQ", "batch-2-0") && out.saw("CLOSE", "batch-1-0")); + assert!(out.count("desktop_read_state") >= 4); + assert!(out.count("desktop_unread") >= 4); + // Ticks never reread names or open windows. + assert_eq!(out.count("desktop_metadata"), 2); + assert_eq!(out.count("desktop_window"), 8); + } + + #[tokio::test] + async fn current_ticks_wait_for_the_backstop() { + let before = drive( + rig(2, 25, "mode = \"current\"\ntick_secs = 1.0"), + Duration::from_millis(2500), + |o| o.count("desktop_thread_window") == 2, + ) + .await; + no_errors(&before); + assert_eq!( + before.desktop(), + with( + cold_reads(2), + &[("desktop_live_eose", 6), ("desktop_replay_eose", 6)] + ) + ); + assert!(!before + .frames + .iter() + .any(|(_, _, id)| id.starts_with("batch-2"))); + + let after = drive( + rig( + 2, + 25, + "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 1.0", + ), + Duration::ZERO, + |o| o.count("desktop_roster") >= 4 && o.saw("CLOSE", "batch-1-0"), + ) + .await; + no_errors(&after); + assert!(after.saw("REQ", "batch-2-0")); + assert!(after.count("desktop_unread") >= 4); + // A replacement is live-only: no replays after the connect's three. + assert_eq!(after.count("desktop_replay_eose"), 6); + } + + #[tokio::test] + async fn legacy_renewals_are_full_reconnects() { + let out = drive( + rig(2, 25, "mode = \"legacy\"\nrenewal_secs = 1.0\n"), + Duration::ZERO, + |o| o.count("desktop_nip11") >= 4 && o.count("desktop_roster") >= 4, + ) + .await; + no_errors(&out); + assert!(out.count("ws_connect_auth") >= 4); + // A reconnect keeps channel names and sidebar preferences. + assert_eq!(out.count("desktop_metadata"), 2); + assert_eq!(out.count("desktop_sidebar"), 2); + } + + #[tokio::test] + async fn current_renewals_carry_small_sessions_and_recover_channels_in_large_ones() { + // More than ten channels: the live-only batch is cut, so replays, + // channel catch-up and an unread repair again, but no global reads. + let large = drive( + rig(2, 25, "mode = \"current\"\nrenewal_secs = 1.0\n"), + Duration::ZERO, + |o| o.count("desktop_replay_eose") >= 12 && o.count("desktop_unread") >= 4, + ) + .await; + no_errors(&large); + assert!(large.count("ws_connect_auth") >= 4); + for op in [ + "desktop_nip11", + "desktop_roster", + "desktop_metadata", + "desktop_profiles", + "desktop_read_state", + ] { + assert_eq!(large.count(op), 2, "{op}"); + } + assert!(large.count("desktop_window") >= 16); + + // Up to ten channels: subscriptions are re-sent and nothing is read, + // but for one unread repair when a client's first renewal (jittered) + // came before its cold connection caught up. + let small = drive( + rig(2, 5, "mode = \"current\"\nrenewal_secs = 1.0\n"), + Duration::ZERO, + |o| o.count("desktop_live_eose") >= 12 && o.count("desktop_thread_window") == 2, + ) + .await; + no_errors(&small); + assert!(small.count("ws_connect_auth") >= 4); + let reads: HashMap<_, _> = small + .desktop() + .into_iter() + .filter(|(op, _)| *op != "desktop_live_eose" && *op != "desktop_unread") + .collect(); + let mut cold = cold_reads(2); + cold.remove("desktop_unread"); + assert_eq!(reads, cold); + assert!(small.count("desktop_unread") <= 4); + } +} diff --git a/crates/buzz-test-client/src/loadgen/identity.rs b/crates/buzz-test-client/src/loadgen/identity.rs index 44f8cc8a9c3..4ed86986bb9 100644 --- a/crates/buzz-test-client/src/loadgen/identity.rs +++ b/crates/buzz-test-client/src/loadgen/identity.rs @@ -7,7 +7,7 @@ use nostr::{EventBuilder, JsonUtil, Keys, Kind, SecretKey, Tag}; use sha2::{Digest, Sha256}; /// Role of a derived identity; part of the derivation so roles never collide. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Role { Owner, Human, diff --git a/crates/buzz-test-client/src/loadgen/mod.rs b/crates/buzz-test-client/src/loadgen/mod.rs index 8bf6bda6075..e56809d0b4a 100644 --- a/crates/buzz-test-client/src/loadgen/mod.rs +++ b/crates/buzz-test-client/src/loadgen/mod.rs @@ -11,6 +11,7 @@ //! scenario's `run_id`, so independent shards (`--shard i/n`) agree on the //! fixture without sharing state. +pub mod desktop; pub mod identity; pub mod net; pub mod run; diff --git a/crates/buzz-test-client/src/loadgen/net.rs b/crates/buzz-test-client/src/loadgen/net.rs index 2b44a550fff..f8971ffc8c7 100644 --- a/crates/buzz-test-client/src/loadgen/net.rs +++ b/crates/buzz-test-client/src/loadgen/net.rs @@ -131,6 +131,18 @@ impl Net { Ok((status, response.text().await?)) } + /// `GET {base}/` for the relay's NIP-11 information document. + pub async fn relay_info(&self, base: &str) -> anyhow::Result<(reqwest::StatusCode, String)> { + let response = self + .http + .get(self.transport_url(&format!("{base}/"))) + .header("Accept", "application/nostr+json") + .send() + .await?; + let status = response.status(); + Ok((status, response.text().await?)) + } + /// Submits `event` through HTTP `/events`; errors unless the relay accepted it. pub async fn submit( &self, diff --git a/crates/buzz-test-client/src/loadgen/run.rs b/crates/buzz-test-client/src/loadgen/run.rs index 8b78b467d3e..6a35fb0dbfd 100644 --- a/crates/buzz-test-client/src/loadgen/run.rs +++ b/crates/buzz-test-client/src/loadgen/run.rs @@ -16,16 +16,17 @@ use nostr::{Event, EventBuilder, Filter, Keys, Kind, PublicKey, Tag, Timestamp}; use serde_json::{json, Value}; use tokio::sync::{watch, Mutex}; +use super::desktop; use super::identity::{self, Role}; use super::net::{http_reason, Net}; -use super::scenario::{Community, Scenario}; +use super::scenario::{Community, HumanProfile, Members, Scenario, WideSplit}; use super::seed::{find_roots, hot_root_keys, reply_tags, HOT_ROOT_PREFIX, ROOT_PREFIX}; use super::stats::{abort_reason, ops_json, Op, Stats}; /// Content prefix of timed messages: `lg:`. const TIMED_PREFIX: &str = "lg:"; /// Longest an idle client blocks on its socket before rechecking the stop flag. -const MAX_IDLE: Duration = Duration::from_secs(1); +pub(super) const MAX_IDLE: Duration = Duration::from_secs(1); /// How long stopped clients get to settle what they have in flight and /// disconnect before they are cancelled. const SHUTDOWN_GRACE: Duration = Duration::from_secs(15); @@ -57,14 +58,69 @@ impl Shard { } } -struct Shared { - scenario: Scenario, - communities: Vec, - net: Net, - stats: Arc, - stop: watch::Receiver, +pub(super) struct Shared { + pub(super) scenario: Scenario, + pub(super) communities: Vec, + pub(super) net: Net, + pub(super) stats: Arc, + pub(super) stop: watch::Receiver, /// Seeded thread roots per channel id, resolved lazily by whichever client needs them first. roots: Mutex>>>, + /// Channel members and human pubkeys, built once by the first desktop + /// client that needs them. + pub(super) directory: tokio::sync::OnceCell>, +} + +/// Who is in each channel, for clients that read their co-members' profiles. +pub(super) struct Directory { + pub(super) members: Members, + /// Hex pubkey of every human up to the peak population, by global index. + pub(super) humans: Vec, +} + +impl Shared { + pub(super) fn new( + scenario: Scenario, + net: Net, + stats: Arc, + stop: watch::Receiver, + ) -> Self { + Self { + communities: scenario.communities(), + scenario, + net, + stats, + stop, + roots: Mutex::new(HashMap::new()), + directory: tokio::sync::OnceCell::new(), + } + } + + /// The [`Directory`], derived off the async workers on first use: it + /// derives a key for every human up to the peak population. + pub(super) async fn directory(&self) -> Arc { + self.directory + .get_or_init(|| async { + let scenario = self.scenario.clone(); + tokio::task::spawn_blocking(move || { + let humans = (0..scenario.peak().humans) + .map(|i| { + identity::keys(&scenario.run_id, Role::Human, i) + .public_key() + .to_hex() + }) + .collect(); + Arc::new(Directory { + members: scenario.channel_members(), + humans, + }) + }) + .await + .unwrap_or_else(|e| std::panic::resume_unwind(e.into_panic())) + }) + .await + .clone() + } } /// Runs every ramp step and returns the JSON report; `Err` when a step aborted. @@ -82,14 +138,7 @@ async fn run_with( ) -> Result { let stats = Arc::new(Stats::default()); let (stop_tx, stop) = watch::channel(false); - let shared = Arc::new(Shared { - communities: scenario.communities(), - scenario, - net, - stats: stats.clone(), - stop, - roots: Mutex::new(HashMap::new()), - }); + let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); let scenario = &shared.scenario; let pop = &scenario.population; let connect_gap = Duration::from_secs_f64(1.0 / pop.connect_rate_per_sec); @@ -121,6 +170,8 @@ async fn run_with( tasks.push(tokio::spawn(async move { if spawn_agent { agent(shared, global).await + } else if shared.scenario.is_desktop(global) { + desktop::desktop(shared, global).await } else { human(shared, global).await } @@ -258,10 +309,10 @@ async fn shutdown(tasks: Vec>, grace: Duration) -> ( } /// Small deterministic PRNG so every client draws an independent schedule. -struct Rng(u64); +pub(super) struct Rng(u64); impl Rng { - fn new(seed: u64) -> Self { + pub(super) fn new(seed: u64) -> Self { Self(seed ^ 0x9E37_79B9_7F4A_7C15) } @@ -273,16 +324,16 @@ impl Rng { z ^ (z >> 31) } - fn unit(&mut self) -> f64 { + pub(super) fn unit(&mut self) -> f64 { (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 } - fn below(&mut self, n: usize) -> usize { + pub(super) fn below(&mut self, n: usize) -> usize { (self.next_u64() % n.max(1) as u64) as usize } /// Next arrival of a Poisson process with `per_hour` events per hour. - fn after(&mut self, per_hour: f64) -> Option { + pub(super) fn after(&mut self, per_hour: f64) -> Option { if per_hour <= 0.0 { return None; } @@ -291,43 +342,43 @@ impl Rng { } } -fn now_micros() -> u128 { +pub(super) fn now_micros() -> u128 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_micros() } -fn tag(parts: [&str; N]) -> Tag { +pub(super) fn tag(parts: [&str; N]) -> Tag { Tag::parse(parts).expect("static tag shape") } -fn kind(k: u32) -> Kind { +pub(super) fn kind(k: u32) -> Kind { Kind::Custom(k as u16) } -fn stopped(shared: &Shared) -> bool { +pub(super) fn stopped(shared: &Shared) -> bool { *shared.stop.borrow() } /// Resolves once the run is stopping. -async fn stopping(shared: &Shared) { +pub(super) async fn stopping(shared: &Shared) { let mut stop = shared.stop.clone(); let _ = stop.wait_for(|s| *s).await; } /// Jittered exponential backoff between failed connects or failed sessions. -struct Backoff(f64); +pub(super) struct Backoff(f64); impl Backoff { const MAX_SECS: f64 = 30.0; - fn new() -> Self { + pub(super) fn new() -> Self { Self(1.0) } /// Sleeps for the next interval, returning early when the run stops. - async fn wait(&mut self, shared: &Shared, rng: &mut Rng) { + pub(super) async fn wait(&mut self, shared: &Shared, rng: &mut Rng) { let wait = Duration::from_secs_f64(self.0 * (0.5 + rng.unit())); self.0 = (self.0 * 2.0).min(Self::MAX_SECS); tokio::select! { @@ -339,7 +390,7 @@ impl Backoff { /// A stream message a client can react to: its id and channel. #[derive(Clone)] -struct Seen { +pub(super) struct Seen { id: String, channel: String, } @@ -398,7 +449,7 @@ fn live( /// Records end-to-end delivery latency for a live timed message and returns /// it as a reaction target. Only live subscriptions call this: history /// replays old messages, whose age is not delivery latency. -fn observe(stats: &Stats, message: &RelayMessage) -> Option { +pub(super) fn observe(stats: &Stats, message: &RelayMessage) -> Option { if let RelayMessage::Event { event, .. } = message { if let Some(sent) = event .content @@ -437,7 +488,7 @@ async fn publish( /// Connects with retry until connected or stopped. Each attempt is bounded /// by the connect deadline, which should be shorter than [`SHUTDOWN_GRACE`] /// so an attempt under way at the stop settles before clients are cancelled. -async fn connect( +pub(super) async fn connect( shared: &Shared, community: &Community, keys: &Keys, @@ -470,7 +521,7 @@ async fn connect( /// Thread roots of `channel`, cached once found. Concurrent first callers may /// each query; a failed query is not cached, so a later reply retries it. -async fn roots( +pub(super) async fn roots( shared: &Shared, community: &Community, reader: &Keys, @@ -508,8 +559,8 @@ async fn human(shared: Arc, global: usize) { let profile = &scenario.human; let community = &shared.communities[global % shared.communities.len()]; let keys = identity::keys(&scenario.run_id, Role::Human, global); - let me = keys.public_key().to_hex(); - let channels = scenario.member_channels(community.index, global); + let channels = scenario.member_channels(community.index, Role::Human, global); + let split = scenario.wide_split(Role::Human, global); let peers: Vec = (1..=10) .map(|k| { let peer = (global + k * shared.communities.len()) % scenario.population.humans.max(1); @@ -603,28 +654,10 @@ async fn human(shared: Arc, global: usize) { } } - let interval = profile.presence_interval_secs; - let mut next_presence = - Some(Instant::now() + Duration::from_secs_f64(interval * rng.unit())); - let mut next_typing = rng.after(profile.typing_per_hour); - let mut next_message = rng.after(profile.messages_per_hour); - let mut next_reaction = rng.after(profile.reactions_per_hour); - let session_end = (profile.mean_session_secs > 0.0) - .then(|| rng.after(3600.0 / profile.mean_session_secs)) - .flatten(); - + let mut activity = Activity::new(profile, &mut rng, split); while ok && !stopped(&shared) { - let due = [ - next_presence, - next_typing, - next_message, - next_reaction, - session_end, - ] - .into_iter() - .flatten() - .min(); - let wait = due + let wait = activity + .due() .map(|d| d.saturating_duration_since(Instant::now())) .unwrap_or(MAX_IDLE) .min(MAX_IDLE); @@ -646,59 +679,27 @@ async fn human(shared: Arc, global: usize) { break; } } - let now = Instant::now(); - let result = if next_presence.is_some_and(|t| t <= now) { - next_presence = Some(now + Duration::from_secs_f64(interval)); - let event = - EventBuilder::new(kind(KIND_PRESENCE_UPDATE), "online").sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishEphemeral, event).await - } else if next_typing.is_some_and(|t| t <= now) { - next_typing = rng.after(profile.typing_per_hour); - let channel = &channels[rng.below(channels.len())]; - let event = EventBuilder::new(kind(KIND_TYPING_INDICATOR), "") - .tags([tag(["h", channel])]) - .sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishEphemeral, event).await - } else if next_message.is_some_and(|t| t <= now) { - next_message = rng.after(profile.messages_per_hour); - let channel = &channels[rng.below(channels.len())]; - let mut tags = vec![tag(["h", channel])]; - if rng.unit() < profile.thread_reply_fraction { - let roots = roots(&shared, community, &keys, channel).await; - if !roots.is_empty() { - tags.push(tag(["e", &roots[rng.below(roots.len())], "", "reply"])); - } - } - let event = EventBuilder::new( - kind(KIND_STREAM_MESSAGE), - format!("{TIMED_PREFIX}{} {me}", now_micros()), + let outcome = activity + .step( + &shared, + community, + &keys, + &channels, + &mut conn, + &mut rng, + &mut last_message, ) - .tags(tags) - .sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishMessage, event).await - } else if next_reaction.is_some_and(|t| t <= now) { - next_reaction = rng.after(profile.reactions_per_hour); - // React to each seen message at most once; the relay rejects duplicates. - match last_message.take() { - Some(target) => { - let event = EventBuilder::new(kind(KIND_REACTION), "+") - .tags([tag(["h", &target.channel]), tag(["e", &target.id])]) - .sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishMessage, event).await - } - None => Ok(()), + .await; + match outcome { + Ok(Step::Continue) => {} + Ok(Step::SessionEnd) => break, + Err(e) => { + shared + .stats + .err(Op::WsConnect, &format!("disconnected: {e}")); + ok = false; + break; } - } else if session_end.is_some_and(|t| t <= now) { - break; - } else { - Ok(()) - }; - if let Err(e) = result { - shared - .stats - .err(Op::WsConnect, &format!("disconnected: {e}")); - ok = false; - break; } } let _ = conn.disconnect().await; @@ -715,7 +716,154 @@ async fn human(shared: Arc, global: usize) { } } -async fn send( +/// What [`Activity::step`] leaves the session to do. +pub(super) enum Step { + Continue, + /// The session reached its scheduled end; reconnect. + SessionEnd, +} + +/// A human's own traffic on its socket: presence heartbeats, typing, +/// messages (some as replies into seeded threads), reactions and the end of +/// its session, each on its own schedule. +pub(super) struct Activity { + interval: f64, + next_presence: Option, + next_typing: Option, + next_message: Option, + next_reaction: Option, + session_end: Option, + /// Set when `population.wide_activity_share` applies to this member. + split: Option, + /// With a split, the latest message seen in a wide (0) and a regular (1) + /// channel, so reactions follow the split too. + recent: [Option; 2], +} + +/// Index into `channels` of the channel a member's next typing indicator, +/// message or reaction goes to. Without a split every channel is equally +/// likely; with one, a wide channel gets `share` of the picks unless the +/// member has no regular channels. +fn pick_channel(split: Option, channels: &[String], rng: &mut Rng) -> usize { + match split { + Some(WideSplit { wide, share }) if wide > 0 && wide < channels.len() => { + if rng.unit() < share { + rng.below(wide) + } else { + wide + rng.below(channels.len() - wide) + } + } + _ => rng.below(channels.len()), + } +} + +impl Activity { + /// Schedules from now, the first heartbeat at a random point within one + /// interval so clients connecting together do not beat together. + pub(super) fn new(profile: &HumanProfile, rng: &mut Rng, split: Option) -> Self { + let interval = profile.presence_interval_secs; + Self { + interval, + next_presence: Some(Instant::now() + Duration::from_secs_f64(interval * rng.unit())), + next_typing: rng.after(profile.typing_per_hour), + next_message: rng.after(profile.messages_per_hour), + next_reaction: rng.after(profile.reactions_per_hour), + session_end: (profile.mean_session_secs > 0.0) + .then(|| rng.after(3600.0 / profile.mean_session_secs)) + .flatten(), + split, + recent: [None, None], + } + } + + /// When the next item is due. + pub(super) fn due(&self) -> Option { + [ + self.next_presence, + self.next_typing, + self.next_message, + self.next_reaction, + self.session_end, + ] + .into_iter() + .flatten() + .min() + } + + /// Runs the first due item, if any. An error means the socket broke. + #[allow(clippy::too_many_arguments)] + pub(super) async fn step( + &mut self, + shared: &Shared, + community: &Community, + keys: &Keys, + channels: &[String], + conn: &mut NostrWsConnection, + rng: &mut Rng, + last_message: &mut Option, + ) -> Result { + let profile = &shared.scenario.human; + if let Some(split) = self.split { + if let Some(seen) = last_message.take() { + let wide = channels[..split.wide.min(channels.len())].contains(&seen.channel); + self.recent[usize::from(!wide)] = Some(seen); + } + } + let now = Instant::now(); + if self.next_presence.is_some_and(|t| t <= now) { + self.next_presence = Some(now + Duration::from_secs_f64(self.interval)); + let event = + EventBuilder::new(kind(KIND_PRESENCE_UPDATE), "online").sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishEphemeral, event).await?; + } else if self.next_typing.is_some_and(|t| t <= now) { + self.next_typing = rng.after(profile.typing_per_hour); + let channel = &channels[pick_channel(self.split, channels, rng)]; + let event = EventBuilder::new(kind(KIND_TYPING_INDICATOR), "") + .tags([tag(["h", channel])]) + .sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishEphemeral, event).await?; + } else if self.next_message.is_some_and(|t| t <= now) { + self.next_message = rng.after(profile.messages_per_hour); + let channel = &channels[pick_channel(self.split, channels, rng)]; + let mut tags = vec![tag(["h", channel])]; + if rng.unit() < profile.thread_reply_fraction { + let roots = roots(shared, community, keys, channel).await; + if !roots.is_empty() { + tags.push(tag(["e", &roots[rng.below(roots.len())], "", "reply"])); + } + } + let me = keys.public_key().to_hex(); + let event = EventBuilder::new( + kind(KIND_STREAM_MESSAGE), + format!("{TIMED_PREFIX}{} {me}", now_micros()), + ) + .tags(tags) + .sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishMessage, event).await?; + } else if self.next_reaction.is_some_and(|t| t <= now) { + self.next_reaction = rng.after(profile.reactions_per_hour); + // React to each seen message at most once; the relay rejects duplicates. + let target = match self.split { + None => last_message.take(), + Some(split) => { + let i = pick_channel(self.split, channels, rng); + self.recent[usize::from(i >= split.wide)].take() + } + }; + if let Some(target) = target { + let event = EventBuilder::new(kind(KIND_REACTION), "+") + .tags([tag(["h", &target.channel]), tag(["e", &target.id])]) + .sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishMessage, event).await?; + } + } else if self.session_end.is_some_and(|t| t <= now) { + return Ok(Step::SessionEnd); + } + Ok(Step::Continue) + } +} + +pub(super) async fn send( stats: &Stats, conn: &mut NostrWsConnection, op: Op, @@ -850,7 +998,7 @@ async fn agent(shared: Arc, global: usize) { global % scenario.population.humans.max(1), ); let auth = identity::nip_oa_tag(&owner, &keys); - let channels = scenario.member_channels(community.index, global); + let channels = scenario.member_channels(community.index, Role::Agent, global); let me = keys.public_key().to_hex(); let mut rng = Rng::new(u64::MAX - global as u64); let mut backoff = Backoff::new(); @@ -1473,6 +1621,11 @@ hold_secs = 1 json!(["EOSE", frame[1]]) } Some("EVENT") => { + let tags = frame[1]["tags"].as_array().cloned().unwrap_or_default(); + if let Some(h) = tags.iter().find(|t| t[0] == "h") { + let h = h[1].as_str().unwrap_or_default().to_string(); + seen.channels.lock().unwrap().push(h); + } let Some(mut delay) = relay.ok_delay else { continue; }; @@ -1505,6 +1658,8 @@ hold_secs = 1 struct Seen { reqs: AtomicUsize, closed: AtomicUsize, + /// The `h` tag of every EVENT that had one. + channels: std::sync::Mutex>, } impl Seen { @@ -1530,6 +1685,112 @@ hold_secs = 1 } } + #[test] + fn wide_activity_share_splits_picks_and_unset_keeps_the_uniform_pick() { + let channels: Vec = (0..6).map(|i| format!("c{i}")).collect(); + let draws = 100_000; + let wide_share = |split: Option| { + let mut rng = Rng::new(7); + let wide = (0..draws) + .filter(|_| pick_channel(split, &channels, &mut rng) < 3) + .count(); + wide as f64 / draws as f64 + }; + let split = |share| Some(WideSplit { wide: 3, share }); + // Unset: the same draws as before the knob, so every channel is as likely. + let (mut a, mut b) = (Rng::new(7), Rng::new(7)); + for _ in 0..1_000 { + assert_eq!( + pick_channel(None, &channels, &mut a), + b.below(channels.len()) + ); + } + assert!((wide_share(None) - 0.5).abs() < 0.01); + assert!((wide_share(split(0.1)) - 0.1).abs() < 0.01); + assert!((wide_share(split(0.8)) - 0.8).abs() < 0.01); + assert_eq!(wide_share(split(1.0)), 1.0); + assert_eq!(wide_share(split(0.0)), 0.0); + // Regular picks stay uniform over the regular channels. + let mut rng = Rng::new(7); + let mut per = [0usize; 6]; + for _ in 0..draws { + per[pick_channel(split(0.1), &channels, &mut rng)] += 1; + } + for n in &per[3..] { + assert!((*n as f64 / draws as f64 - 0.3).abs() < 0.01, "{per:?}"); + } + // A member in wide channels only sends everything to them. + let mut rng = Rng::new(7); + let only_wide = Some(WideSplit { + wide: 6, + share: 0.1, + }); + let mut seen = [false; 6]; + for _ in 0..1_000 { + seen[pick_channel(only_wide, &channels, &mut rng)] = true; + } + assert_eq!(seen, [true; 6]); + } + + /// One human in three wide and three regular channels that only types, + /// ten times a second. + fn typing_wide_member(addr: &str, share: &str) -> Scenario { + Scenario::parse(&format!( + r#" +run_id = "t" +[target] +ws_url = "ws://{addr}" +http_url = "http://{addr}" +[shape] +communities = 1 +channels_per_community = 6 +wide_channels = 3 +[population] +humans = 1 +agents = 0 +channels_per_member = 3 +wide_member_fraction = 1.0 +{share} +[human] +presence_interval_secs = 3600.0 +typing_per_hour = 36000.0 +messages_per_hour = 0.0 +reactions_per_hour = 0.0 +history_limit = 1 +[[ramp]] +users = 1.0 +hold_secs = 2 +"# + )) + .unwrap() + } + + #[tokio::test] + async fn typing_follows_the_wide_activity_share() { + let mut wide_counts = Vec::new(); + for share in ["", "wide_activity_share = 0.0", "wide_activity_share = 1.0"] { + let (addr, seen) = fake_ws_relay(FakeRelay::PROMPT).await; + let scenario = typing_wide_member(&addr, share); + let wide: Vec = scenario.channel_ids(0)[..3].to_vec(); + run_lone_human(scenario, Duration::from_millis(300)) + .await + .expect("the run passes"); + let channels = seen.channels.lock().unwrap().clone(); + assert!(channels.len() >= 5, "{share}: {channels:?}"); + let in_wide = channels.iter().filter(|c| wide.contains(c)).count(); + wide_counts.push((in_wide, channels.len())); + } + let [(unset, n), (none, _), (all, m)] = wide_counts[..] else { + unreachable!() + }; + assert!( + unset > 0 && unset < n, + "unset picks both kinds: {wide_counts:?}" + ); + assert_eq!(none, 0, "{wide_counts:?}"); + assert_eq!(all, m, "{wide_counts:?}"); + } + /// One human, no agents, one `hold_secs` step, and a presence heartbeat /// every second; other traffic is off so the heartbeat is the only publish. fn lone_human(addr: &str, hold_secs: u64, max_error_rate: f64) -> Scenario { @@ -1962,14 +2223,7 @@ hold_secs = 1 let net = Net::new(&scenario).await.unwrap(); let stats = Arc::new(Stats::default()); let (stop_tx, stop) = watch::channel(false); - let shared = Arc::new(Shared { - communities: scenario.communities(), - scenario, - net, - stats: stats.clone(), - stop, - roots: Mutex::new(HashMap::new()), - }); + let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); let authors: Vec<_> = (0..8) .map(|i| tokio::spawn(hot_author(shared.clone(), i))) .collect(); diff --git a/crates/buzz-test-client/src/loadgen/scenario.rs b/crates/buzz-test-client/src/loadgen/scenario.rs index f874d4b9c0f..aabfb899e5e 100644 --- a/crates/buzz-test-client/src/loadgen/scenario.rs +++ b/crates/buzz-test-client/src/loadgen/scenario.rs @@ -2,7 +2,7 @@ use serde::Deserialize; -use super::identity; +use super::identity::{self, Role}; /// A load scenario, parsed from TOML. #[derive(Clone, Debug, Deserialize)] @@ -19,6 +19,8 @@ pub struct Scenario { pub agent: AgentProfile, #[serde(default)] pub hot_thread: HotThread, + #[serde(default)] + pub desktop: DesktopProfile, pub ramp: Vec, #[serde(default)] pub abort: Abort, @@ -82,6 +84,16 @@ pub struct Shape { /// Concurrent seeding requests. #[serde(default = "default_seed_concurrency")] pub seed_concurrency: usize, + /// Create channels private and add every member with kind:9000, so the + /// relay checks real membership on every read and subscription. Off + /// creates open channels nobody joins. + #[serde(default)] + pub private_channels: bool, + /// The first `wide_channels` channels of each community are its wide + /// channels, which `population.wide_member_fraction` of humans join on + /// top of their own. Other memberships use the remaining channels. + #[serde(default)] + pub wide_channels: usize, } fn default_seed_concurrency() -> usize { @@ -99,6 +111,38 @@ pub struct Population { /// New connections per second per shard while ramping. #[serde(default = "default_connect_rate")] pub connect_rate_per_sec: f64, + /// Fraction of humans that also join every wide channel. + #[serde(default)] + pub wide_member_fraction: f64, + /// Share of its typing, messages and reactions a member of the wide + /// channels sends into them; the rest goes to its regular channels. + /// Unset, typing and messages pick any of a member's channels equally and + /// reactions go to the latest message it saw. + #[serde(default)] + pub wide_activity_share: Option, + /// Humans whose channel count differs from `channels_per_member`, such + /// as a few heavy users in many channels. Agents keep `channels_per_member`. + #[serde(default)] + pub tiers: Vec, +} + +/// A member's split of its own traffic between the wide channels that lead +/// its channel list and its regular channels. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct WideSplit { + /// Wide channels at the start of the member's channels. + pub wide: usize, + /// Share of picks that land in a wide channel. + pub share: f64, +} + +/// `fraction` of humans join `channels` channels instead of +/// `population.channels_per_member`, not counting wide channels. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Tier { + pub fraction: f64, + pub channels: usize, } fn default_connect_rate() -> f64 { @@ -181,6 +225,57 @@ impl HotThread { } } +/// Which desktop app request pattern a desktop client follows. +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum DesktopMode { + /// Before the client volume fixes: every recovery tick replaces each live + /// channel batch and rereads the roster, read state and unreads, and an + /// identity renewal is a full reconnect. + Legacy, + /// After them: tick reads and batch re-sends wait for a backstop, and a + /// renewal re-sends subscriptions from their cursors. + #[default] + Current, +} + +/// Humans that run the desktop app's relay request pattern instead of the +/// basic human client: HTTP reads of the roster, profiles, read state, +/// unreads and open windows on connect, batched channel subscriptions, a +/// recovery tick, and optional periodic identity renewal. Disabled while +/// `fraction` is 0. +#[derive(Clone, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DesktopProfile { + /// Fraction of humans that are desktop clients. + pub fraction: f64, + pub mode: DesktopMode, + /// Identity renewal interval, a reconnect the client plans (0: never). + pub renewal_secs: f64, + /// Channel windows kept open, each caught up on every establishment. + pub windows: usize, + /// Thread views kept open, each refreshed on every establishment. + pub threads: usize, + /// Recovery tick interval. + pub tick_secs: f64, + /// `current` mode's backstop for tick reads and batch re-sends. + pub backstop_secs: f64, +} + +impl Default for DesktopProfile { + fn default() -> Self { + Self { + fraction: 0.0, + mode: DesktopMode::Current, + renewal_secs: 0.0, + windows: 3, + threads: 1, + tick_secs: 60.0, + backstop_secs: 900.0, + } + } +} + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct RampStep { @@ -206,6 +301,41 @@ impl Default for Abort { } } +/// A member of channels: a human or agent by global index. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Member { + pub role: Role, + pub index: usize, +} + +/// Members of every channel, indexed `[community][channel]`. +pub type Members = Vec>>; + +/// Clients each kind reaches at the largest ramp step. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Peak { + pub humans: usize, + pub agents: usize, + pub hot_authors: usize, +} + +/// Salts for [`draw`], one per independent per-member choice. +const TIER_SALT: u64 = 1; +const WIDE_SALT: u64 = 2; +const DESKTOP_SALT: u64 = 3; + +/// A deterministic draw in `[0, 1)` for member `index`, independent for each +/// `salt` and of the client schedules, so every shard and `seed` agree. +fn draw(index: usize, salt: u64) -> f64 { + let mut z = (index as u64) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(salt.wrapping_mul(0xD1B5_4A32_D192_ED03)); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^= z >> 31; + (z >> 11) as f64 / (1u64 << 53) as f64 +} + /// One community's endpoints. #[derive(Clone, Debug)] pub struct Community { @@ -253,10 +383,35 @@ impl Scenario { .into(), ); } + let n = self.shape.channels_per_community; + let wide = self.shape.wide_channels; + // Every member needs at least one channel outside the wide ones. + if wide >= n && wide > 0 { + return Err("shape.wide_channels must be below channels_per_community".into()); + } if self.population.channels_per_member == 0 - || self.population.channels_per_member > self.shape.channels_per_community + || self.population.channels_per_member > n - wide { - return Err("population.channels_per_member must be 1..=channels_per_community".into()); + return Err( + "population.channels_per_member must be 1..=channels_per_community - wide_channels" + .into(), + ); + } + let mut tiered = 0.0; + for tier in &self.population.tiers { + if !(tier.fraction > 0.0 && tier.fraction <= 1.0) { + return Err("population.tiers fraction must be above 0 and at most 1".into()); + } + if tier.channels == 0 || tier.channels > n - wide { + return Err( + "population.tiers channels must be 1..=channels_per_community - wide_channels" + .into(), + ); + } + tiered += tier.fraction; + } + if tiered > 1.0 + 1e-9 { + return Err("population.tiers fractions must sum to at most 1".into()); } if self.ramp.is_empty() { return Err("ramp needs at least one step".into()); @@ -281,6 +436,7 @@ impl Scenario { ), ("agent.query_interval_secs", a.query_interval_secs), ("abort.max_p99_ms", self.abort.max_p99_ms), + ("desktop.backstop_secs", self.desktop.backstop_secs), ]; for (name, value) in positive { if !value.is_finite() || value <= 0.0 { @@ -291,6 +447,14 @@ impl Scenario { if !h.presence_interval_secs.is_finite() || h.presence_interval_secs < 1.0 { return Err("human.presence_interval_secs must be a finite number >= 1".into()); } + // Neither the recovery tick nor a renewal runs more than once a second. + let d = &self.desktop; + if !d.tick_secs.is_finite() || d.tick_secs < 1.0 { + return Err("desktop.tick_secs must be a finite number >= 1".into()); + } + if !d.renewal_secs.is_finite() || !(d.renewal_secs == 0.0 || d.renewal_secs >= 1.0) { + return Err("desktop.renewal_secs must be 0 or a finite number >= 1".into()); + } let non_negative = [ ("human.typing_per_hour", h.typing_per_hour), ("human.messages_per_hour", h.messages_per_hour), @@ -315,6 +479,15 @@ impl Scenario { let fractions = [ ("human.thread_reply_fraction", h.thread_reply_fraction), ("abort.max_error_rate", self.abort.max_error_rate), + ( + "population.wide_member_fraction", + self.population.wide_member_fraction, + ), + ("desktop.fraction", self.desktop.fraction), + ( + "population.wide_activity_share", + self.population.wide_activity_share.unwrap_or(0.0), + ), ]; for (name, value) in fractions { if !(0.0..=1.0).contains(&value) { @@ -354,18 +527,103 @@ impl Scenario { .collect() } - /// Channels member `member` (global human or agent index) belongs to. - pub fn member_channels(&self, community: usize, member: usize) -> Vec { - let n = self.shape.channels_per_community; - (0..self.population.channels_per_member) - .map(|k| identity::channel_id(&self.run_id, community, (member + k) % n)) + /// Indices, within its community, of the channels member `index` of + /// `role` (a global human or agent index) belongs to: the wide channels + /// if it joins them, then a run of consecutive regular channels starting + /// at its own index. Without wide channels or tiers this is + /// `channels_per_member` channels from `index % channels_per_community`. + pub fn member_channel_indices(&self, role: Role, index: usize) -> Vec { + let wide = self.shape.wide_channels; + let regular = self.shape.channels_per_community - wide; + let mut channels = Vec::new(); + let mut count = self.population.channels_per_member; + if self.joins_wide(role, index) { + channels.extend(0..wide); + } + if role == Role::Human { + count = self.tier_channels(index); + } + channels.extend((0..count).map(|k| wide + (index + k) % regular)); + channels + } + + /// Whether member `index` of `role` joins the wide channels. + fn joins_wide(&self, role: Role, index: usize) -> bool { + role == Role::Human + && self.shape.wide_channels > 0 + && draw(index, WIDE_SALT) < self.population.wide_member_fraction + } + + /// How member `index` of `role` splits its own traffic, when + /// `population.wide_activity_share` applies to it: the number of wide + /// channels leading its channel list and the share they get. + pub fn wide_split(&self, role: Role, index: usize) -> Option { + let share = self.population.wide_activity_share?; + self.joins_wide(role, index).then_some(WideSplit { + wide: self.shape.wide_channels, + share, + }) + } + + /// Regular (not wide) channels human `index` joins: its tier's count, or + /// `channels_per_member` for humans outside every tier. + fn tier_channels(&self, index: usize) -> usize { + let u = draw(index, TIER_SALT); + let mut bound = 0.0; + for tier in &self.population.tiers { + bound += tier.fraction; + if u < bound { + return tier.channels; + } + } + self.population.channels_per_member + } + + /// Channel ids member `index` of `role` belongs to in `community`. + pub fn member_channels(&self, community: usize, role: Role, index: usize) -> Vec { + self.member_channel_indices(role, index) + .into_iter() + .map(|channel| identity::channel_id(&self.run_id, community, channel)) .collect() } + /// Whether human `index` runs the desktop client. + pub fn is_desktop(&self, index: usize) -> bool { + draw(index, DESKTOP_SALT) < self.desktop.fraction + } + /// Largest multiplier any ramp step reaches. pub fn peak_users(&self) -> f64 { self.ramp.iter().map(|s| s.users).fold(0.0, f64::max) } + + /// Clients of each kind at the largest ramp step, rounded as `run` does. + pub fn peak(&self) -> Peak { + let at = |base: usize| (base as f64 * self.peak_users()).ceil() as usize; + Peak { + humans: at(self.population.humans), + agents: at(self.population.agents), + hot_authors: at(self.hot_thread.authors), + } + } + + /// Humans and agents of every channel up to the peak population, indexed + /// `[community][channel]`, humans first and each kind by index. A client + /// with global index `g` lives in community `g % communities`. + pub fn channel_members(&self) -> Members { + let shape = &self.shape; + let mut members = vec![vec![Vec::new(); shape.channels_per_community]; shape.communities]; + let peak = self.peak(); + for (role, count) in [(Role::Human, peak.humans), (Role::Agent, peak.agents)] { + for index in 0..count { + let community = &mut members[index % shape.communities]; + for channel in self.member_channel_indices(role, index) { + community[channel].push(Member { role, index }); + } + } + } + members + } } /// `url` with its host (and port) replaced by `host`. @@ -403,7 +661,7 @@ hold_secs = 10 assert_eq!(s.communities().len(), 1); assert_eq!(s.communities()[0].ws_url, "ws://localhost:3000"); assert_eq!(s.human.presence_interval_secs, 60.0); - assert_eq!(s.member_channels(0, 3).len(), 2); + assert_eq!(s.member_channels(0, Role::Human, 3).len(), 2); } #[test] @@ -519,6 +777,201 @@ hold_secs = 10 assert!(err.contains("port"), "{err}"); } + /// MINIMAL with `n` channels, `shape` and `population` lines added, and + /// `extra` TOML after it. + fn shaped(n: usize, shape: &str, population: &str, extra: &str) -> Result { + let text = MINIMAL + .replace( + "channels_per_community = 4", + &format!("channels_per_community = {n}\n{shape}"), + ) + .replace( + "channels_per_member = 2", + &format!("channels_per_member = 2\n{population}"), + ); + Scenario::parse(&format!("{text}{extra}")) + } + + #[test] + fn without_wide_channels_or_tiers_every_member_keeps_its_run_of_channels() { + let s = Scenario::parse(MINIMAL).unwrap(); + for role in [Role::Human, Role::Agent] { + for m in 0..50 { + let want: Vec = (0..2).map(|k| (m + k) % 4).collect(); + assert_eq!(s.member_channel_indices(role, m), want, "{role:?} {m}"); + } + } + let ids = s.member_channels(0, Role::Human, 3); + assert_eq!(ids, [3, 0].map(|c| identity::channel_id("t", 0, c))); + assert_eq!(s.desktop.fraction, 0.0); + assert!(!(0..1000).any(|m| s.is_desktop(m))); + assert!(!s.shape.private_channels); + } + + #[test] + fn wide_channels_and_tiers_are_deterministic_and_honor_their_fractions() { + let s = shaped( + 12, + "wide_channels = 2", + "wide_member_fraction = 0.5\n[[population.tiers]]\nfraction = 0.1\nchannels = 8\n[[population.tiers]]\nfraction = 0.2\nchannels = 1", + "[desktop]\nfraction = 0.25\n", + ) + .unwrap(); + let again = shaped( + 12, + "wide_channels = 2", + "wide_member_fraction = 0.5\n[[population.tiers]]\nfraction = 0.1\nchannels = 8\n[[population.tiers]]\nfraction = 0.2\nchannels = 1", + "[desktop]\nfraction = 0.25\n", + ) + .unwrap(); + let n = 20_000; + let (mut wide, mut heavy, mut light, mut desktop) = (0, 0, 0, 0); + for m in 0..n { + let channels = s.member_channel_indices(Role::Human, m); + assert_eq!(channels, again.member_channel_indices(Role::Human, m)); + let joins_wide = channels.starts_with(&[0, 1]); + let regular: Vec = channels.iter().copied().filter(|c| *c >= 2).collect(); + assert_eq!(regular.len() + 2 * usize::from(joins_wide), channels.len()); + // Regular channels are a run of distinct channels from the member's index. + let want: Vec = (0..regular.len()).map(|k| 2 + (m + k) % 10).collect(); + assert_eq!(regular, want, "member {m}"); + wide += usize::from(joins_wide); + heavy += usize::from(regular.len() == 8); + light += usize::from(regular.len() == 1); + desktop += usize::from(s.is_desktop(m)); + // Agents never join wide channels and keep channels_per_member. + let agent = s.member_channel_indices(Role::Agent, m); + assert_eq!(agent, [2 + m % 10, 2 + (m + 1) % 10]); + } + let share = |count: usize| count as f64 / n as f64; + assert!((share(wide) - 0.5).abs() < 0.02, "wide {wide}"); + assert!((share(heavy) - 0.1).abs() < 0.02, "heavy {heavy}"); + assert!((share(light) - 0.2).abs() < 0.02, "light {light}"); + assert!((share(desktop) - 0.25).abs() < 0.02, "desktop {desktop}"); + } + + #[test] + fn channel_members_inverts_the_assignment_up_to_the_peak() { + let s = shaped( + 6, + "wide_channels = 1", + "wide_member_fraction = 0.3\n[[population.tiers]]\nfraction = 0.5\nchannels = 4", + "[[ramp]]\nusers = 2.5\nhold_secs = 10\n", + ) + .unwrap(); + let peak = s.peak(); + assert_eq!((peak.humans, peak.agents), (25, 5)); + let members = s.channel_members(); + assert_eq!(members.len(), 1); + let mut listed = 0; + for (role, count) in [(Role::Human, 25), (Role::Agent, 5)] { + for index in 0..count { + for c in s.member_channel_indices(role, index) { + assert!(members[0][c].contains(&Member { role, index })); + listed += 1; + } + } + } + assert_eq!(listed, members[0].iter().map(Vec::len).sum::()); + // Humans come first in every channel. + for channel in &members[0] { + let first_agent = channel.iter().position(|m| m.role == Role::Agent); + let last_human = channel.iter().rposition(|m| m.role == Role::Human); + if let (Some(a), Some(h)) = (first_agent, last_human) { + assert!(h < a); + } + } + } + + #[test] + fn wide_activity_share_applies_only_to_members_of_wide_channels() { + let wide = "wide_channels = 2"; + let unset = shaped(6, wide, "wide_member_fraction = 1.0", "").unwrap(); + assert_eq!(unset.population.wide_activity_share, None); + assert_eq!(unset.wide_split(Role::Human, 0), None); + let set = shaped( + 6, + wide, + "wide_member_fraction = 1.0\nwide_activity_share = 0.25", + "", + ) + .unwrap(); + let split = WideSplit { + wide: 2, + share: 0.25, + }; + for m in 0..20 { + assert_eq!(set.wide_split(Role::Human, m), Some(split)); + // The wide channels lead the member's channels, as the split assumes. + assert_eq!(set.member_channel_indices(Role::Human, m)[..2], [0, 1]); + // Agents never join wide channels. + assert_eq!(set.wide_split(Role::Agent, m), None); + } + let outside = shaped(6, wide, "wide_activity_share = 0.25", "").unwrap(); + assert!((0..20).all(|m| outside.wide_split(Role::Human, m).is_none())); + } + + #[test] + fn shape_and_desktop_knobs_are_validated() { + let rejected = [ + ("wide_channels = 4", "", ""), + // Two members' channels no longer fit outside three wide channels. + ("wide_channels = 3", "", ""), + ("", "wide_member_fraction = 1.5", ""), + ("", "wide_member_fraction = nan", ""), + ("", "wide_activity_share = 1.5", ""), + ("", "wide_activity_share = -0.1", ""), + ("", "wide_activity_share = nan", ""), + ("", "[[population.tiers]]\nfraction = 0.0\nchannels = 1", ""), + ("", "[[population.tiers]]\nfraction = 1.5\nchannels = 1", ""), + ("", "[[population.tiers]]\nfraction = 0.6\nchannels = 1\n[[population.tiers]]\nfraction = 0.6\nchannels = 2", ""), + ("", "[[population.tiers]]\nfraction = 0.5\nchannels = 0", ""), + ("", "[[population.tiers]]\nfraction = 0.5\nchannels = 5", ""), + ("wide_channels = 1", "[[population.tiers]]\nfraction = 0.5\nchannels = 4", ""), + ("", "[[population.tiers]]\nfraction = 0.5\nchannels = 1\nextra = 1", ""), + ("", "", "[desktop]\nfraction = 1.5\n"), + ("", "", "[desktop]\nfraction = -0.1\n"), + ("", "", "[desktop]\nmode = \"bogus\"\n"), + ("", "", "[desktop]\ntick_secs = 0.5\n"), + ("", "", "[desktop]\ntick_secs = inf\n"), + ("", "", "[desktop]\nrenewal_secs = 0.5\n"), + ("", "", "[desktop]\nrenewal_secs = -1.0\n"), + ("", "", "[desktop]\nbackstop_secs = 0.0\n"), + ("", "", "[desktop]\nwindow = 1\n"), + ]; + for (shape, population, extra) in rejected { + assert!( + shaped(4, shape, population, extra).is_err(), + "accepted: {shape} {population} {extra}" + ); + } + let accepted = [ + ("wide_channels = 2", "wide_member_fraction = 1.0", ""), + ("private_channels = true", "", ""), + ( + "", + "[[population.tiers]]\nfraction = 0.5\nchannels = 4\n[[population.tiers]]\nfraction = 0.5\nchannels = 1", + "", + ), + ( + "", + "", + "[desktop]\nfraction = 1.0\nmode = \"legacy\"\nrenewal_secs = 1.0\ntick_secs = 1.0\nwindows = 0\nthreads = 0\n", + ), + ]; + for (shape, population, extra) in accepted { + let s = shaped(4, shape, population, extra) + .unwrap_or_else(|e| panic!("rejected {shape} {population} {extra}: {e}")); + assert!(!s.member_channel_indices(Role::Human, 0).is_empty()); + } + let legacy = shaped(4, "", "", "[desktop]\nmode = \"legacy\"\n").unwrap(); + assert_eq!(legacy.desktop.mode, DesktopMode::Legacy); + assert_eq!( + Scenario::parse(MINIMAL).unwrap().desktop.mode, + DesktopMode::Current + ); + } + #[test] fn thread_depth_is_limited_to_what_the_relay_accepts() { let with_depth = |depth: usize| { diff --git a/crates/buzz-test-client/src/loadgen/seed.rs b/crates/buzz-test-client/src/loadgen/seed.rs index 61379bc27d1..9742d001fde 100644 --- a/crates/buzz-test-client/src/loadgen/seed.rs +++ b/crates/buzz-test-client/src/loadgen/seed.rs @@ -5,13 +5,13 @@ use std::sync::Arc; use std::time::Instant; use futures_util::stream::{self, StreamExt}; -use nostr::{Event, EventBuilder, Keys, Kind, Tag}; +use nostr::{Event, EventBuilder, JsonUtil, Keys, Kind, Tag}; use reqwest::StatusCode; use serde_json::json; use super::identity::{self, Role}; -use super::net::{http_reason, Net}; -use super::scenario::{Community, Scenario}; +use super::net::{http_reason, truncate, Net}; +use super::scenario::{Community, Member, Members, Scenario}; use super::stats::{Op, Snapshot, Stats}; /// Content prefix of seeded thread roots; `run` finds roots by it. @@ -24,17 +24,31 @@ pub fn hot_root_keys(run_id: &str) -> Keys { identity::keys(run_id, Role::HotRoot, 0) } +/// Most people one kind:9000 may name, the relay's limit. +const MAX_ADDS_PER_EVENT: usize = 100; + /// Seeds every community. Idempotent for communities this run's fixture -/// owner owns, and for channels; history and threads are appended on every -/// call. Fails unless every planned post -/// was accepted. +/// owner owns, for channels and for memberships; history and threads are +/// appended on every call. Fails unless every planned post was accepted and, +/// with private channels, every member was added. pub async fn seed(scenario: &Scenario, net: &Net, stats: &Arc) -> anyhow::Result<()> { + let members = scenario + .shape + .private_channels + .then(|| scenario.channel_members()); for community in scenario.communities() { let owner = identity::keys(&scenario.run_id, Role::Owner, community.index); if let Some(host) = &community.host { provision(scenario, net, host, &owner).await?; } create_channels(scenario, net, &community, &owner).await?; + if let Some(members) = &members { + let added = add_members(scenario, net, &community, &owner, members).await?; + eprintln!( + "seed: community {} memberships added ({added} people)", + community.index + ); + } eprintln!( "seed: community {} ready ({} channels)", community.index, scenario.shape.channels_per_community @@ -46,8 +60,18 @@ pub async fn seed(scenario: &Scenario, net: &Net, stats: &Arc) -> anyhow: let before = stats.snapshot(); let (roots, rest) = plan_posts(scenario); let planned: usize = roots.iter().chain(&rest).map(Vec::len).sum(); - let root_ids = post_all(scenario, net, stats, &communities, roots, &RootIds::new()).await; - post_all(scenario, net, stats, &communities, rest, &root_ids).await; + let members = members.as_ref(); + let root_ids = post_all( + scenario, + net, + stats, + &communities, + members, + roots, + &RootIds::new(), + ) + .await; + post_all(scenario, net, stats, &communities, members, rest, &root_ids).await; let accepted = seeded(&stats.snapshot(), &before); eprintln!( "seed: {accepted} of {planned} posts accepted in {:.1}s", @@ -82,6 +106,7 @@ async fn post_all( net: &Net, stats: &Stats, communities: &[Community], + members: Option<&Members>, chains: Vec, roots: &RootIds, ) -> RootIds { @@ -91,7 +116,8 @@ async fn post_all( let mut previous: Option = None; for post in chain { let community = &communities[post.community]; - let Some((keys, event)) = post.build(scenario, roots, previous.as_deref()) else { + let Some((keys, event)) = post.build(scenario, members, roots, previous.as_deref()) + else { stats.err(Op::Seed, "thread parent missing"); previous = None; continue; @@ -193,6 +219,11 @@ async fn create_channels( community: &Community, owner: &Keys, ) -> anyhow::Result<()> { + let visibility = if scenario.shape.private_channels { + "private" + } else { + "open" + }; let results: Vec> = stream::iter(scenario.channel_ids(community.index)) .enumerate() .map(|(i, channel)| async move { @@ -201,7 +232,7 @@ async fn create_channels( tag(["h", &channel]), tag(["name", &format!("lg-{}-{i}", scenario.run_id)]), tag(["channel_type", "stream"]), - tag(["visibility", "open"]), + tag(["visibility", visibility]), ]) .sign_with_keys(owner)?; match net.submit(owner, community, &event, None).await { @@ -215,6 +246,100 @@ async fn create_channels( results.into_iter().collect() } +/// Adds every member of `community`'s channels with kind:9000 events from +/// the channel owner, at most [`MAX_ADDS_PER_EVENT`] people each and +/// `seed_concurrency` at a time. Channel 0 of community 0 also gets the +/// hot-thread root author and authors. Adding a current member at its role +/// changes nothing, so re-running is safe. Returns how many people were +/// named; fails if any event, or any person in one, was refused. +async fn add_members( + scenario: &Scenario, + net: &Net, + community: &Community, + owner: &Keys, + members: &Members, +) -> anyhow::Result { + let run_id = &scenario.run_id; + let mut adds = Vec::new(); + for (channel, list) in members[community.index].iter().enumerate() { + let mut people: Vec = list + .iter() + .map(|m| identity::keys(run_id, m.role, m.index)) + .collect(); + if community.index == 0 && channel == 0 && scenario.hot_thread.enabled() { + people.push(hot_root_keys(run_id)); + people.extend( + (0..scenario.peak().hot_authors) + .map(|i| identity::keys(run_id, Role::HotAuthor, i)), + ); + } + let id = identity::channel_id(run_id, community.index, channel); + for chunk in people.chunks(MAX_ADDS_PER_EVENT) { + adds.push(( + id.clone(), + chunk.iter().map(|k| k.public_key().to_hex()).collect(), + )); + } + } + let named: usize = adds + .iter() + .map(|(_, p): &(String, Vec)| p.len()) + .sum(); + let failures: Vec = stream::iter(adds) + .map(|(channel, people)| async move { + put_users(net, community, owner, &channel, &people) + .await + .err() + .map(|e| format!("{channel}: {e}")) + }) + .buffer_unordered(scenario.shape.seed_concurrency) + .filter_map(|failure| async move { failure }) + .collect() + .await; + if let Some(first) = failures.first() { + anyhow::bail!( + "{} membership events failed in community {}; first: {first}", + failures.len(), + community.index + ); + } + Ok(named) +} + +/// Submits one kind:9000 naming `people` in `channel`. The relay accepts an +/// event that adds some people and refuses others, listing the refused ones +/// under `rejected` in its message, so that counts as a failure too. +async fn put_users( + net: &Net, + community: &Community, + owner: &Keys, + channel: &str, + people: &[String], +) -> anyhow::Result<()> { + let tags = std::iter::once(tag(["h", channel])).chain(people.iter().map(|p| tag(["p", p]))); + let event = EventBuilder::new(Kind::Custom(9000), "") + .tags(tags) + .sign_with_keys(owner)?; + let (status, body) = net + .post( + owner, + &community.http_url, + "/events", + event.as_json().into_bytes(), + None, + ) + .await?; + if !status.is_success() { + anyhow::bail!("{}", http_reason(status, &body)); + } + let reply: serde_json::Value = serde_json::from_str(&body)?; + let message = reply["message"].as_str().unwrap_or_default(); + if reply["accepted"].as_bool() != Some(true) || message.contains("\"rejected\"") { + anyhow::bail!("rejected: {}", truncate(message)); + } + Ok(()) +} + /// One seeded post: history message, thread root, or thread reply. struct Post { community: usize, @@ -299,20 +424,16 @@ fn plan_posts(scenario: &Scenario) -> (Vec, Vec) { impl Post { /// Builds the event. A reply attaches to `previous`, the accepted reply /// before it in its thread; the first reply attaches to the thread root. + /// With `members` (private channels), authors are the channel's humans. fn build( &self, scenario: &Scenario, + members: Option<&Members>, roots: &RootIds, previous: Option<&str>, ) -> Option<(Keys, Event)> { let channel = identity::channel_id(&scenario.run_id, self.community, self.channel); - // Rotate authors across humans to stay under per-pubkey rate limits. - let humans = scenario.population.humans.max(1); - let author = identity::keys( - &scenario.run_id, - Role::Human, - (self.seq * scenario.shape.communities + self.community) % humans, - ); + let author = self.author(scenario, members); let mut tags = vec![tag(["h", &channel])]; let (keys, content) = match self.kind { PostKind::History => (author, format!("lg-history {}", self.seq)), @@ -396,6 +517,28 @@ pub async fn find_roots( Ok(roots.into_iter().map(|(_, id)| id).collect()) } +impl Post { + /// Rotates authors across humans to stay under per-pubkey rate limits: + /// across all humans for open channels, across the channel's own human + /// members for private ones, falling back to the owner when it has none. + fn author(&self, scenario: &Scenario, members: Option<&Members>) -> Keys { + let run_id = &scenario.run_id; + let Some(members) = members else { + let humans = scenario.population.humans.max(1); + let index = (self.seq * scenario.shape.communities + self.community) % humans; + return identity::keys(run_id, Role::Human, index); + }; + let humans: Vec<&Member> = members[self.community][self.channel] + .iter() + .filter(|m| m.role == Role::Human) + .collect(); + match humans.get(self.seq % humans.len().max(1)) { + Some(m) => identity::keys(run_id, Role::Human, m.index), + None => identity::keys(run_id, Role::Owner, self.community), + } + } +} + fn tag(parts: [&str; N]) -> Tag { Tag::parse(parts).expect("static tag shape") } @@ -604,6 +747,161 @@ hold_secs = 1 assert!(err.to_string().contains("seeded 10 of 15"), "{err}"); } + /// Seeds `scenario` against a relay that accepts everything, answering + /// kind:9000 with `put_user_message`. Returns the outcome and every + /// event submitted, in order. + async fn seed_logged( + scenario: &str, + put_user_message: &'static str, + ) -> (anyhow::Result<()>, Scenario, Vec) { + let log: Arc>> = Default::default(); + let logged = log.clone(); + let addr = fake_http(move |_, _, body| { + let event: Event = serde_json::from_slice(body).unwrap(); + let message = if event.kind == Kind::Custom(9000) { + put_user_message + } else { + "" + }; + logged.lock().unwrap().push(event); + ( + 200, + json!({"accepted": true, "message": message}).to_string(), + ) + }) + .await; + let scenario = Scenario::parse(&scenario.replace("{addr}", &addr)).unwrap(); + let net = Net::new(&scenario).await.unwrap(); + let stats = Arc::new(Stats::default()); + let outcome = seed(&scenario, &net, &stats).await; + let events = log.lock().unwrap().clone(); + (outcome, scenario, events) + } + + fn values(event: &Event, name: &str) -> Vec { + event + .tags + .iter() + .filter(|t| t.kind().as_str() == name) + .filter_map(|t| t.content().map(str::to_string)) + .collect() + } + + const PRIVATE: &str = r#" +run_id = "t" +[target] +ws_url = "ws://{addr}" +http_url = "http://{addr}" +[shape] +communities = 1 +channels_per_community = 3 +history_messages_per_channel = 3 +private_channels = true +wide_channels = 1 +[population] +humans = 150 +agents = 2 +channels_per_member = 1 +wide_member_fraction = 1.0 +[hot_thread] +seed_replies = 2 +authors = 2 +[[ramp]] +users = 1.0 +hold_secs = 1 +[[ramp]] +users = 1.5 +hold_secs = 1 +"#; + + #[tokio::test] + async fn private_channels_add_every_member_before_anything_is_posted() { + let (outcome, scenario, events) = seed_logged(PRIVATE, "").await; + outcome.expect("seed passes"); + let kind = |e: &Event| e.kind.as_u16(); + let creates: Vec<&Event> = events.iter().filter(|e| kind(e) == 9007).collect(); + assert_eq!(creates.len(), 3); + assert!(creates + .iter() + .all(|e| values(e, "visibility") == ["private"])); + let first_put = events.iter().position(|e| kind(e) == 9000).unwrap(); + let last_put = events.iter().rposition(|e| kind(e) == 9000).unwrap(); + let first_post = events.iter().position(|e| kind(e) == 9).unwrap(); + let last_create = events.iter().rposition(|e| kind(e) == 9007).unwrap(); + assert!(last_create < first_put && last_put < first_post); + + let members = scenario.channel_members(); + let owner = identity::keys("t", Role::Owner, 0); + let mut added: HashMap> = HashMap::new(); + for put in events.iter().filter(|e| kind(e) == 9000) { + assert_eq!(put.pubkey, owner.public_key()); + let people = values(put, "p"); + assert!(!people.is_empty() && people.len() <= MAX_ADDS_PER_EVENT); + let [channel] = values(put, "h").try_into().unwrap(); + added.entry(channel).or_default().extend(people); + } + // Every human up to the peak (225) joins the wide channel 0, and so do + // the hot thread's root author and its 3 peak authors. + for (c, list) in members[0].iter().enumerate() { + let mut want: Vec = list + .iter() + .map(|m| identity::keys("t", m.role, m.index).public_key().to_hex()) + .collect(); + if c == 0 { + want.push(hot_root_keys("t").public_key().to_hex()); + want.extend((0..3).map(|i| { + identity::keys("t", Role::HotAuthor, i) + .public_key() + .to_hex() + })); + assert_eq!(want.len(), 225 + 4); + } + let mut got = added.remove(&identity::channel_id("t", 0, c)).unwrap(); + got.sort(); + want.sort(); + assert_eq!(got, want, "channel {c}"); + } + assert!(added.is_empty()); + + // Every post is authored by a human member of its channel. + for post in events.iter().filter(|e| kind(e) == 9) { + let [channel] = values(post, "h").try_into().unwrap(); + let c = (0..3) + .find(|c| identity::channel_id("t", 0, *c) == channel) + .unwrap(); + let author = post.pubkey.to_hex(); + let member = members[0][c].iter().any(|m| { + m.role == Role::Human + && identity::keys("t", Role::Human, m.index) + .public_key() + .to_hex() + == author + }); + let hot_root = author == hot_root_keys("t").public_key().to_hex(); + assert!(member || hot_root, "{author} posted in channel {c}"); + } + } + + #[tokio::test] + async fn refused_members_fail_the_seed_and_open_channels_add_nobody() { + // The relay accepts an event that refuses some of the people it names. + let refused = + r#"members: {"added":["aa"],"rejected":[{"reason":"blocked","pubkeys":["bb"]}]}"#; + let (outcome, _, _) = seed_logged(PRIVATE, refused).await; + let err = outcome.expect_err("a refused member fails the seed"); + assert!(err.to_string().contains("membership"), "{err}"); + // Re-adding current members changes nothing and passes. + let (outcome, _, _) = seed_logged(PRIVATE, r#"members: {"already_member":["aa"]}"#).await; + outcome.expect("re-running the seed passes"); + + let open = PRIVATE.replace("private_channels = true", ""); + let (outcome, _, events) = seed_logged(&open, "").await; + outcome.expect("seed passes"); + assert!(!events.iter().any(|e| e.kind.as_u16() == 9000)); + let creates: Vec<&Event> = events.iter().filter(|e| e.kind.as_u16() == 9007).collect(); + assert!(creates.iter().all(|e| values(e, "visibility") == ["open"])); + } + #[test] fn reply_k_points_reply_at_reply_k_minus_one_and_root_at_the_root() { let scenario = Scenario::parse( @@ -636,7 +934,7 @@ hold_secs = 1 let mut previous: Option = None; for (k, post) in rest[0].iter().enumerate() { let (_, event) = post - .build(&scenario, &known, previous.as_deref()) + .build(&scenario, None, &known, previous.as_deref()) .expect("reply builds once its parent is known"); let markers = buzz_core::nip10::parse_thread_markers(&event.tags); assert_eq!(markers.root.as_deref(), Some(root_id.as_str()), "reply {k}"); @@ -645,7 +943,7 @@ hold_secs = 1 previous = Some(event.id.to_hex()); } // A reply cannot be placed without its predecessor. - assert!(rest[0][2].build(&scenario, &known, None).is_none()); + assert!(rest[0][2].build(&scenario, None, &known, None).is_none()); } /// An operator endpoint that lists hosts by owner, as `GET diff --git a/crates/buzz-test-client/src/loadgen/stats.rs b/crates/buzz-test-client/src/loadgen/stats.rs index 62e3f43234a..aacf4928b96 100644 --- a/crates/buzz-test-client/src/loadgen/stats.rs +++ b/crates/buzz-test-client/src/loadgen/stats.rs @@ -21,10 +21,30 @@ pub enum Op { Subscribe, HotReply, ThreadFetch, + /// Desktop client: NIP-11 `GET /` on connect. + DesktopNip11, + /// Desktop client: EOSE of a live subscription (global or channel batch). + DesktopLive, + /// Desktop client: EOSE of a closing channel replay subscription. + DesktopReplay, + /// Desktop client HTTP `/query` reads, one per request type. + DesktopRoster, + DesktopMetadata, + DesktopSidebar, + DesktopProfiles, + DesktopStatuses, + DesktopEmoji, + DesktopReadState, + DesktopUnread, + DesktopReminders, + DesktopWindow, + DesktopNotices, + DesktopThreadRoot, + DesktopThreadWindow, } impl Op { - pub const ALL: [Op; 11] = [ + pub const ALL: [Op; 27] = [ Op::WsConnect, Op::History, Op::PublishEphemeral, @@ -36,6 +56,22 @@ impl Op { Op::Subscribe, Op::HotReply, Op::ThreadFetch, + Op::DesktopNip11, + Op::DesktopLive, + Op::DesktopReplay, + Op::DesktopRoster, + Op::DesktopMetadata, + Op::DesktopSidebar, + Op::DesktopProfiles, + Op::DesktopStatuses, + Op::DesktopEmoji, + Op::DesktopReadState, + Op::DesktopUnread, + Op::DesktopReminders, + Op::DesktopWindow, + Op::DesktopNotices, + Op::DesktopThreadRoot, + Op::DesktopThreadWindow, ]; pub fn name(self) -> &'static str { @@ -51,6 +87,22 @@ impl Op { Op::Subscribe => "subscribe_eose", Op::HotReply => "hot_reply", Op::ThreadFetch => "thread_fetch", + Op::DesktopNip11 => "desktop_nip11", + Op::DesktopLive => "desktop_live_eose", + Op::DesktopReplay => "desktop_replay_eose", + Op::DesktopRoster => "desktop_roster", + Op::DesktopMetadata => "desktop_metadata", + Op::DesktopSidebar => "desktop_sidebar", + Op::DesktopProfiles => "desktop_profiles", + Op::DesktopStatuses => "desktop_statuses", + Op::DesktopEmoji => "desktop_emoji", + Op::DesktopReadState => "desktop_read_state", + Op::DesktopUnread => "desktop_unread", + Op::DesktopReminders => "desktop_reminders", + Op::DesktopWindow => "desktop_window", + Op::DesktopNotices => "desktop_notices", + Op::DesktopThreadRoot => "desktop_thread_root", + Op::DesktopThreadWindow => "desktop_thread_window", } } } diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index b6413326aa7..d0a9df95cb7 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -23,6 +23,10 @@ cargo run --release -p buzz-test-client --bin buzz-loadgen -- teardown perf/scen | Humans / agents | `population.humans`, `population.agents`, multiplied by each `[[ramp]]` step's `users` | | Throughput per client | `[human]` and `[agent]` rates | | Reconnect churn | `human.mean_session_secs`, `population.connect_rate_per_sec` | +| Private membership | `shape.private_channels` | +| Large channels | `shape.wide_channels`, `population.wide_member_fraction`, `population.wide_activity_share` | +| Uneven membership | `population.tiers` | +| Desktop clients | `[desktop]`: `fraction`, `mode`, `windows`, `threads`, `tick_secs`, `backstop_secs`, `renewal_secs` | `shape.thread_depth` seeds each thread as a chain: reply `k` replies to reply `k-1`, so the thread's deepest reply sits at depth `thread_depth`. The relay @@ -42,6 +46,27 @@ agents) reply to that root and read its newest 100 replies over HTTP. Their replies are timed, so humans subscribed to that channel record delivery. Leave `seed_replies` and `authors` at 0 to disable it. +Members are assigned deterministically by global index, so every shard and +`seed` agree on them without coordination: + +- `shape.wide_channels` reserves the first K channels of each community as + wide channels. Each human joins them with probability + `population.wide_member_fraction`, on top of its own channels. Agents never + do. +- `population.wide_activity_share`, when set, is the share of its typing, + messages and reactions a member of the wide channels sends into them; the + rest goes to its regular channels. Unset, typing and messages pick any of + a member's channels equally, so a member of three wide and three regular + channels sends half of them into the wide ones, and reactions go to the + latest message the member saw. It has no effect on members outside the wide + channels. +- `population.tiers` is a list of `{ fraction, channels }`. That fraction of + humans joins `channels` channels instead of `channels_per_member`. Fractions + sum to at most 1; the rest keep `channels_per_member`. +- A member's own channels are a run of consecutive channels after the wide + ones, starting at its index. With no wide channels and no tiers this is the + assignment of earlier versions. + ## What each client does - **Human**: authenticates over WebSocket and subscribes to its channels' @@ -52,6 +77,10 @@ Leave `seed_replies` and `authors` at 0 to disable it. - **Agent**: authenticates with a NIP-OA tag from its owner, polls HTTP `/query` (channel window plus mentions), posts messages through HTTP `/events`, and sends encrypted kind:24200 observer frames to its owner. +- **Desktop human**: the `desktop.fraction` of humans that model the desktop + app instead of the basic human above. They keep the basic human's presence, + typing, messages and reactions; the subscriptions and HTTP reads are the + app's, described under [Desktop clients](#desktop-clients). - **Hot-thread author**: finds the seeded hot root, then posts replies to it through HTTP `/events` (`hot_reply`) and reads the thread through HTTP `/query` (`thread_fetch`). Until the root is found every attempt counts as @@ -80,8 +109,100 @@ relay checks against its own relay URL. Without a host template, the generator connects to `ws_url` and `http_url` as written and `connect_addr` has no effect. -Seeded identities do not join communities or channels. A relay that requires -membership rejects both `seed` and `run`. +By default channels are open and seeded identities join nothing, so a relay +that requires membership rejects both `seed` and `run`. With +`shape.private_channels = true`, `seed` creates the channels private and adds +every member: the humans and agents assigned to each channel up to the +largest ramp step, and the hot-thread root and authors in its channel. It +sends kind:9000 events of up to 100 members, `seed_concurrency` at a time, +signed by the community owner, before posting any history. A rerun adds +nobody new, and any member the relay refuses fails the seed. Seeded history +in a private channel is posted by its own members. + +## Desktop clients + +A desktop client models the subscriptions and reads the desktop app makes, +in two modes: `legacy`, the app before its client volume fixes, and +`current` (default), after them. Each request type is its own operation in +the report. + +On a **cold start** it: + +1. reads the relay's NIP-11 document (`desktop_nip11`); +2. opens its profile and membership subscriptions, and its channel + subscriptions (`desktop_live_eose`). With up to ten channels both modes + open one per-channel batch that replays the newest 500 events. With more, + `legacy` opens one such batch per ten channels; `current` opens one + live-only subscription per 2048 channels and replays the channels ten at a + time in separate subscriptions it closes at EOSE (`desktop_replay_eose`). + At most four are set up at once, each with a 10 s EOSE timeout; +3. reads over HTTP, at most six at a time with a 30 s timeout: its roster + (`desktop_roster`, paged), channel names (`desktop_metadata`), sidebar + preferences (`desktop_sidebar`), co-member profiles (`desktop_profiles`) + and statuses (`desktop_statuses`), the custom emoji set + (`desktop_emoji`), read state (`desktop_read_state`), unread repair in + chunks of 128 channels (`desktop_unread`), reminders + (`desktop_reminders`), and for each of `windows` open channels a window + catch-up (`desktop_window`) and its notices (`desktop_notices`), and for + each of `threads` open threads a seeded root lookup and its first thread + page (`desktop_thread_root`, `desktop_thread_window`). A window catch-up + reads the head page, then follows the cursor in the relay's window + bounds (kind 39006) for up to five older pages, until the bounds say + history is exhausted. + +`legacy` makes all of these reads as it connects. `current` reads only +NIP-11, names and sidebar preferences then, and the rest as each +subscription reaches its EOSE ("establishes"): profiles, statuses and emoji +with the profile subscription; roster, read state, unread repair and +reminders with the membership subscription; windows and threads with the +channel subscriptions. + +A subscription's replay is cut when it replays 500 events, or, for the +live-only subscription, always: it replays nothing itself. A subscription +that misses its EOSE deadline counts as cut, and so does one the previous +connection never caught up. Only an EOSE catches a subscription up: one +that missed its deadline is recovered, but the next connection cannot +resume or carry it. When a `current` client reconnects, a profile or +membership subscription whose replay was not cut resumed every change, so +it skips its full read: profiles read nothing unless an earlier read is +still owed, and membership only repairs unreads, without the read-state +snapshot, unless a snapshot read failed since, which a full repair retries. +A cut one reads as on a cold start. The channel subscriptions always +recover the windows and threads. + +Every `tick_secs` the client runs a recovery tick, on its own clock across +connections. In `legacy` each tick replaces its channel subscriptions and +rereads roster, read state and unread. In `current` each of the three waits +`backstop_secs` since it last ran: since the batches were last replaced, +since the roster was last read in full, and since a full unread repair (with +its read-state snapshot) was last asked for. A failed roster read, unread +repair or read-state snapshot is owed and runs again on the next tick; a +repair without the snapshot does not settle an owed snapshot. An owed +profile read runs again when the profile subscription next establishes. A +replacement is live-only and is opened before the subscription it replaces +is closed. A replacement that misses its EOSE deadline is closed, the +subscription it would have replaced stays, and the next tick tries again. + +Every `renewal_secs` (0 disables it; the first is jittered) the client renews +its identity and reconnects. In `legacy` that is an ordinary reconnect, which +rereads everything but channel names and sidebar preferences. In `current` +the subscriptions are carried over: none establishes while their replays +keep continuity, so nothing is read. Once any replay is cut, every +subscription establishes as on a reconnect. With more than ten channels the +live-only subscription is always cut, so the replays, windows, threads and +an unread repair run again. A human's session end +(`human.mean_session_secs`) is an ordinary reconnect in both modes. Resumed +subscriptions start 960 s before the previous session ended, but not before +300 s before the client started. + +Each client runs its reads in one task, at most six at a time. A read that +comes due while the same read is running or queued is not added again, so a +slow relay does not pile reads up behind it. A read leaves the running set +as soon as it settles, so a failed one can retry while a slow one is still +under way. + +The app also establishes the channel subscriptions when a tick's replacement +reaches its EOSE; the generator does not model that. ## Sharding diff --git a/perf/scenarios/local-desktop.toml b/perf/scenarios/local-desktop.toml new file mode 100644 index 00000000000..518fa65ee78 --- /dev/null +++ b/perf/scenarios/local-desktop.toml @@ -0,0 +1,53 @@ +# Short run of desktop clients in private channels against a local relay +# (`just relay`). Ticks, backstops and renewals are seconds apart instead of +# minutes, so every desktop request type shows up within the ramp. Set +# `desktop.mode = "legacy"` to compare the client before its volume fixes. +# cargo run -p buzz-test-client --bin buzz-loadgen -- seed perf/scenarios/local-desktop.toml +# cargo run -p buzz-test-client --bin buzz-loadgen -- run perf/scenarios/local-desktop.toml +run_id = "local-desktop" + +[target] +ws_url = "ws://localhost:3000" +http_url = "http://localhost:3000" + +[shape] +communities = 1 +channels_per_community = 30 +history_messages_per_channel = 20 +thread_roots_per_channel = 1 +thread_depth = 5 +# Members are added to each channel with kind:9000; the first two channels +# are joined by half of the humans on top of their own. +private_channels = true +wide_channels = 2 + +[population] +humans = 40 +agents = 4 +channels_per_member = 5 +connect_rate_per_sec = 20.0 +wide_member_fraction = 0.5 +# Members of the wide channels send a fifth of their own traffic there. +wide_activity_share = 0.2 +# A quarter of the humans are in 15 channels instead of 5, so their +# subscriptions batch past ten channels. +tiers = [{ fraction = 0.25, channels = 15 }] + +[human] +presence_interval_secs = 10.0 +typing_per_hour = 120.0 +messages_per_hour = 60.0 +reactions_per_hour = 30.0 + +[desktop] +fraction = 0.5 +mode = "current" +windows = 3 +threads = 1 +tick_secs = 5.0 +backstop_secs = 15.0 +renewal_secs = 20.0 + +[[ramp]] +users = 1.0 +hold_secs = 60