From afe3edf5bbff7d223e53666dcdf6a6db9c482642 Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Tue, 3 Nov 2026 12:13:07 -0500 Subject: [PATCH 1/4] feat(loadgen): model desktop clients and large-community shape Add opt-in scenario knobs for communities shaped like a large deployment and for desktop app clients: - shape.private_channels: seed creates channels private and adds every member with kind:9000, seed_concurrency at a time, idempotently. - shape.wide_channels + population.wide_member_fraction: the first K channels of each community are joined by a fraction of humans. - population.tiers: fractions of humans in more channels than channels_per_member. - [desktop]: a fraction of humans run the desktop app's subscriptions and HTTP reads, in legacy (before the client volume fixes) or current mode, with cold start, recovery ticks, backstops and identity renewals; each request type reports as its own op. Every knob defaults to the previous behavior and assignment. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Luke Tornquist --- .../buzz-test-client/src/loadgen/desktop.rs | 1465 +++++++++++++++++ .../buzz-test-client/src/loadgen/identity.rs | 2 +- crates/buzz-test-client/src/loadgen/mod.rs | 1 + crates/buzz-test-client/src/loadgen/net.rs | 12 + crates/buzz-test-client/src/loadgen/run.rs | 337 ++-- .../buzz-test-client/src/loadgen/scenario.rs | 402 ++++- crates/buzz-test-client/src/loadgen/seed.rs | 336 +++- crates/buzz-test-client/src/loadgen/stats.rs | 54 +- perf/LOADGEN.md | 80 +- perf/scenarios/local-desktop.toml | 51 + 10 files changed, 2591 insertions(+), 149 deletions(-) create mode 100644 crates/buzz-test-client/src/loadgen/desktop.rs create mode 100644 perf/scenarios/local-desktop.toml diff --git a/crates/buzz-test-client/src/loadgen/desktop.rs b/crates/buzz-test-client/src/loadgen/desktop.rs new file mode 100644 index 00000000000..af4b3bacbcf --- /dev/null +++ b/crates/buzz-test-client/src/loadgen/desktop.rs @@ -0,0 +1,1465 @@ +//! A desktop app client. It makes the relay requests the desktop app makes, +//! modeled on the app's live subscription manager (the desktop app's +//! `live.ts`) and its session store. It follows the request pattern from +//! before (`legacy`) or after (`current`) the app's client volume fixes. +//! +//! The planning functions here say what a connection sends: the REQs it +//! opens, the HTTP reads it makes, and what a recovery tick or a renewal +//! adds. [`desktop`] executes those plans over one socket, plus one HTTP read +//! task per client. See `perf/LOADGEN.md` for the model and what it omits. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use buzz_ws_client::{NostrWsConnection, RelayMessage, WsClientError}; +use futures_util::stream::{self, StreamExt}; +use nostr::Keys; +use serde_json::{json, Value}; + +use super::identity::{self, Role}; +use super::net::http_reason; +use super::run::{ + connect, observe, roots, stopped, Activity, Backoff, Directory, Rng, Seen, Shared, Step, + MAX_IDLE, +}; +use super::scenario::DesktopMode; +use super::stats::{Op, Pending}; + +/// Kinds the app subscribes to for every joined channel. +const CHANNEL_KINDS: [u32; 20] = [ + 9, 40002, 40008, 45001, 45003, 40099, 40100, 40003, 5, 9005, 7, 39000, 39002, 39005, 20002, + 45010, 48100, 48101, 48102, 48103, +]; +/// Kinds of a channel window page: channel rows without notices. +const WINDOW_KINDS: [u32; 5] = [9, 40002, 40008, 48100, 48103]; +/// Per-filter limit of a replaying channel filter. +const REPLAY_LIMIT: usize = 500; +/// Channels per replay REQ (one filter each), and per `legacy` batch REQ. +const REPLAY_BATCH: usize = 10; +/// Channel ids in one `current` live-only batch filter. +const LIVE_BATCH: usize = 2048; +/// REQs awaiting EOSE at once while a connection establishes. +const SETUP_CONCURRENCY: usize = 4; +/// How long a REQ may wait for its EOSE. +const REQ_TIMEOUT: Duration = Duration::from_secs(10); +/// A resumed subscription starts this long before its cursor. +const RESUME_LAG_SECS: u64 = 960; +/// A first subscription starts this long before now. +const STARTUP_FLOOR_SECS: u64 = 300; +/// HTTP reads in flight at once per client. +const HTTP_CONCURRENCY: usize = 6; +/// Deadline of one HTTP read. +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); +/// Page size of roster, metadata and profile reads. +const PAGE: usize = 500; +/// Most roster pages one read follows. +const MAX_ROSTER_PAGES: usize = 20; +/// Profiles the client holds. +const PROFILE_CAP: usize = 1024; +/// Status authors read. +const STATUS_AUTHORS: usize = 100; +/// Channels per unread repair read. +const UNREAD_CHANNELS: usize = 128; +/// Events per channel window page. +const WINDOW_PAGE: usize = 20; +/// Older window pages a catch-up reads after the head, at most. +const CATCH_UP_PAGES: usize = 5; +/// The app's custom emoji set. +const EMOJI_SET: &str = "buzz:custom-emoji"; + +/// One REQ frame. +#[derive(Clone, Debug, PartialEq)] +pub(super) struct Req { + pub(super) id: String, + pub(super) filters: Vec, + /// A replay subscription, closed at its EOSE. Others stay open. + pub(super) replay: bool, +} + +impl Req { + fn frame(&self) -> Value { + let mut frame = vec![json!("REQ"), json!(self.id)]; + frame.extend(self.filters.iter().cloned()); + Value::Array(frame) + } +} + +fn channel_filter(ids: &[String], since: u64, limit: usize) -> Value { + json!({"kinds": CHANNEL_KINDS, "#h": ids, "since": since, "limit": limit}) +} + +/// The two global subscriptions, profiles and membership. +fn global_reqs(viewer: &str, since: u64) -> [Req; 2] { + let limit = REPLAY_LIMIT; + [ + Req { + id: "profiles".into(), + filters: vec![ + json!({"kinds": [0, 10100, 30177], "since": since, "limit": limit}), + json!({"kinds": [30315], "#d": ["general"], "since": since, "limit": limit}), + json!({"kinds": [30030], "#d": [EMOJI_SET], "since": since, "limit": limit}), + ], + replay: false, + }, + Req { + id: "membership".into(), + filters: vec![ + json!({"kinds": [44100, 44101], "#p": [viewer], "since": since, "limit": limit}), + json!({"kinds": [44100, 44101], "#p": [viewer], "#t": ["system"], "since": since, "limit": limit}), + json!({"kinds": [30078], "authors": [viewer], "#t": ["read-state"], "since": since, "limit": limit}), + json!({"kinds": [30300], "authors": [viewer], "since": since, "limit": limit}), + ], + replay: false, + }, + ] +} + +/// Channel batch subscriptions, which stay open. Up to 10 channels share +/// one REQ with a filter per channel. Beyond that, `legacy` sends a REQ per +/// 10 channels, and `current` sends one live-only filter (limit 0) per 2048 +/// channels and replays separately ([`replay_reqs`]). `limit` applies to +/// per-channel filters: [`REPLAY_LIMIT`] on connect, 0 for a tick's +/// replacement. +pub(super) fn batch_reqs( + mode: DesktopMode, + channels: &[String], + since: u64, + limit: usize, + generation: u64, +) -> Vec { + let id = |i: usize| format!("batch-{generation}-{i}"); + if mode == DesktopMode::Legacy || channels.len() <= REPLAY_BATCH { + channels + .chunks(REPLAY_BATCH) + .enumerate() + .map(|(i, chunk)| Req { + id: id(i), + filters: chunk + .iter() + .map(|c| channel_filter(std::slice::from_ref(c), since, limit)) + .collect(), + replay: false, + }) + .collect() + } else { + channels + .chunks(LIVE_BATCH) + .enumerate() + .map(|(i, chunk)| Req { + id: id(i), + filters: vec![channel_filter(chunk, since, 0)], + replay: false, + }) + .collect() + } +} + +/// `current` mode's replay of more than 10 channels behind live-only +/// batches: a REQ per 10 channels, one filter each, closed at its EOSE. +pub(super) fn replay_reqs( + mode: DesktopMode, + channels: &[String], + since: u64, + generation: u64, +) -> Vec { + if mode == DesktopMode::Legacy || channels.len() <= REPLAY_BATCH { + return Vec::new(); + } + channels + .chunks(REPLAY_BATCH) + .enumerate() + .map(|(i, chunk)| Req { + id: format!("replay-{generation}-{i}"), + filters: chunk + .iter() + .map(|c| channel_filter(std::slice::from_ref(c), since, REPLAY_LIMIT)) + .collect(), + replay: true, + }) + .collect() +} + +/// Every REQ a connection sends while it establishes, in order. +pub(super) fn connect_reqs( + mode: DesktopMode, + viewer: &str, + channels: &[String], + since: u64, + generation: u64, +) -> Vec { + let mut reqs = Vec::from(global_reqs(viewer, since)); + reqs.extend(batch_reqs(mode, channels, since, REPLAY_LIMIT, generation)); + reqs.extend(replay_reqs(mode, channels, since, generation)); + reqs +} + +/// Why a connection is being established. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Situation { + /// The client's first connection. + Cold, + /// A reconnect whose replay is not continuous. Everything is read again + /// except channel names and sidebar preferences, which the client + /// already holds. + Reconnect, + /// A `current` renewal with every channel in one replaying batch: the + /// subscriptions are re-sent and nothing is read. + QuietRenewal, + /// A `current` renewal whose batches are live-only (more than 10 + /// channels). Channel routes establish again, so windows, threads and + /// unreads are refreshed; global reads are skipped. + LimitedRenewal, +} + +/// What an identity renewal is for a client in `mode` with `joined` channels. +/// Before the fixes, it was an ordinary reconnect. +pub(super) fn renewal(mode: DesktopMode, joined: usize) -> Situation { + match mode { + DesktopMode::Legacy => Situation::Reconnect, + DesktopMode::Current if joined <= REPLAY_BATCH => Situation::QuietRenewal, + DesktopMode::Current => Situation::LimitedRenewal, + } +} + +/// One HTTP read job. A job may make several requests: pages, or chunks of +/// ids. `Window(i)` and `Thread(i)` are the client's i-th open window and +/// thread. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Fetch { + Nip11, + Roster, + Metadata, + Sidebar, + Profiles, + Statuses, + Emoji, + ReadState, + Unread, + Reminders, + Window(usize), + Thread(usize), +} + +/// The reads a connection makes once established in `situation`, with +/// `windows` channel windows and `threads` thread views open. +pub(super) fn connect_fetches(situation: Situation, windows: usize, threads: usize) -> Vec { + let mut reads = match situation { + Situation::Cold => vec![ + Fetch::Nip11, + Fetch::Roster, + Fetch::Metadata, + Fetch::Sidebar, + Fetch::Profiles, + Fetch::Statuses, + Fetch::Emoji, + Fetch::ReadState, + Fetch::Unread, + Fetch::Reminders, + ], + Situation::Reconnect => vec![ + Fetch::Nip11, + Fetch::Roster, + Fetch::Profiles, + Fetch::Statuses, + Fetch::Emoji, + Fetch::ReadState, + Fetch::Unread, + Fetch::Reminders, + ], + Situation::QuietRenewal => return Vec::new(), + Situation::LimitedRenewal => vec![Fetch::Unread], + }; + reads.extend((0..windows).map(Fetch::Window)); + reads.extend((0..threads).map(Fetch::Thread)); + reads +} + +/// What a recovery tick re-sends or rereads. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Due { + /// Replace every channel batch with a live-only copy. + pub(super) batches: bool, + pub(super) roster: bool, + /// Read state and unread repair. + pub(super) unread: bool, +} + +/// A tick's work. `legacy` does all of it every tick. `current` waits for +/// `backstop` since each was last done. +pub(super) fn tick( + mode: DesktopMode, + backstop: Duration, + since_batches: Duration, + since_roster: Duration, + since_unread: Duration, +) -> Due { + match mode { + DesktopMode::Legacy => Due { + batches: true, + roster: true, + unread: true, + }, + DesktopMode::Current => Due { + batches: since_batches >= backstop, + roster: since_roster >= backstop, + unread: since_unread >= backstop, + }, + } +} + +/// The HTTP reads a tick makes. +pub(super) fn tick_fetches(due: Due) -> Vec { + let mut reads = Vec::new(); + if due.roster { + reads.push(Fetch::Roster); + } + if due.unread { + reads.extend([Fetch::ReadState, Fetch::Unread]); + } + reads +} + +/// Where a resumed subscription starts: [`RESUME_LAG_SECS`] before the +/// last connection dropped, but not before the client's own start floor. +fn resume_since(started: u64, cursor: Option, now: u64) -> u64 { + match cursor { + Some(cursor) => cursor + .saturating_sub(RESUME_LAG_SECS) + .max(started.saturating_sub(STARTUP_FLOOR_SECS)), + None => now.saturating_sub(STARTUP_FLOOR_SECS), + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +/// Up to [`PROFILE_CAP`] other humans in `channels` of `community`, in +/// channel order: the profiles the client holds. +fn co_members( + directory: &Directory, + community: usize, + me: usize, + channels: &[usize], +) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + for &channel in channels { + for member in &directory.members[community][channel] { + if member.role == Role::Human && member.index != me && seen.insert(member.index) { + out.push(directory.humans[member.index].clone()); + if out.len() == PROFILE_CAP { + return out; + } + } + } + } + out +} + +/// What a client's reads use. Its sessions and read tasks share it. +struct Reader { + shared: Arc, + community: usize, + keys: Keys, + viewer: String, + channels: Vec, + profiles: Vec, + windows: usize, + /// The relay's own pubkey from NIP-11 `self`, the author of notices. + relay: std::sync::Mutex>, + /// When the last unread repair started (Unix seconds). + unread_mark: std::sync::Mutex>, +} + +impl Reader { + /// One `POST /query`, timed as `op`. Returns the events, or `None` once + /// it failed (recorded) or the run is stopping (not started). + async fn query(&self, op: Op, filters: Value) -> Option> { + if stopped(&self.shared) { + return None; + } + let shared = &self.shared; + let base = &shared.communities[self.community].http_url; + let body = filters.to_string().into_bytes(); + let pending = shared.stats.begin(op); + let response = tokio::time::timeout( + HTTP_TIMEOUT, + shared.net.post(&self.keys, base, "/query", body, None), + ) + .await; + match response { + Err(_) => pending.err("read timeout"), + Ok(Err(e)) => pending.err(&e.to_string()), + Ok(Ok((status, text))) if !status.is_success() => { + pending.err(&http_reason(status, &text)) + } + Ok(Ok((_, text))) => match serde_json::from_str::>(&text) { + Ok(events) => { + pending.ok(); + return Some(events); + } + Err(e) => pending.err(&format!("decode: {e}")), + }, + } + None + } + + async fn fetch(&self, fetch: Fetch) { + let viewer = &self.viewer; + match fetch { + Fetch::Nip11 => self.nip11().await, + Fetch::Roster => { + let mut filter = json!({"kinds": [39002], "#p": [viewer], "limit": PAGE}); + for _ in 0..MAX_ROSTER_PAGES { + let Some(page) = self.query(Op::DesktopRoster, json!([filter])).await else { + break; + }; + match oldest(&page) { + Some(until) if page.len() >= PAGE => filter["until"] = json!(until), + _ => break, + } + } + } + Fetch::Metadata => { + for ids in self.channels.chunks(PAGE) { + let filter = json!([{"kinds": [39000], "#d": ids, "limit": PAGE}]); + self.query(Op::DesktopMetadata, filter).await; + } + } + Fetch::Sidebar => { + let filters: Vec = [ + "channel-sections", + "channel-stars", + "channel-mutes", + "channel-sort", + "channel-filter", + ] + .iter() + .map(|d| json!({"kinds": [30078], "authors": [viewer], "#d": [d], "limit": 1})) + .collect(); + self.query(Op::DesktopSidebar, Value::Array(filters)).await; + } + Fetch::Profiles => { + for authors in self.profiles.chunks(PAGE) { + let filter = json!([{"kinds": [0], "authors": authors, "limit": PAGE}]); + self.query(Op::DesktopProfiles, filter).await; + } + } + Fetch::Statuses => { + let authors = &self.profiles[..self.profiles.len().min(STATUS_AUTHORS)]; + if !authors.is_empty() { + let filter = json!([{ + "kinds": [30315], "#d": ["general"], "authors": authors, "limit": authors.len(), + }]); + self.query(Op::DesktopStatuses, filter).await; + } + } + Fetch::Emoji => { + let filter = json!([{"kinds": [30030], "#d": [EMOJI_SET], "limit": PAGE}]); + self.query(Op::DesktopEmoji, filter).await; + } + Fetch::ReadState => { + let filter = + json!([{"kinds": [30078], "authors": [viewer], "read_state_snapshot": 1}]); + self.query(Op::DesktopReadState, filter).await; + } + Fetch::Unread => { + let now = unix_now(); + let mark = lock(&self.unread_mark).replace(now); + // Sequential, as the app repairs one chunk after another. + for ids in self.channels.chunks(UNREAD_CHANNELS) { + let mut filter = json!({ + "kinds": [9, 40002, 40008], "#h": ids, "include_aux": true, "limit": REPLAY_LIMIT, + }); + if let Some(mark) = mark { + filter["since"] = json!(mark.saturating_sub(RESUME_LAG_SECS)); + } + if self + .query(Op::DesktopUnread, json!([filter])) + .await + .is_none() + { + break; + } + } + } + Fetch::Reminders => { + let filter = json!([{"kinds": [30300], "authors": [viewer], "limit": 201}]); + self.query(Op::DesktopReminders, filter).await; + } + Fetch::Window(i) => self.window(&self.channels[i]).await, + Fetch::Thread(i) => self.thread(i).await, + } + } + + /// `GET /` for the relay's NIP-11 document, remembering its `self`. + async fn nip11(&self) { + if stopped(&self.shared) { + return; + } + let shared = &self.shared; + let base = &shared.communities[self.community].http_url; + let pending = shared.stats.begin(Op::DesktopNip11); + match tokio::time::timeout(HTTP_TIMEOUT, shared.net.relay_info(base)).await { + Err(_) => pending.err("read timeout"), + Ok(Err(e)) => pending.err(&e.to_string()), + Ok(Ok((status, text))) if !status.is_success() => { + pending.err(&http_reason(status, &text)) + } + Ok(Ok((_, text))) => { + pending.ok(); + let info: Value = serde_json::from_str(&text).unwrap_or_default(); + if let Some(relay) = info["self"].as_str() { + *lock(&self.relay) = Some(relay.to_string()); + } + } + } + } + + /// A window catch-up: the head page, then older pages while each comes + /// back full (at most [`CATCH_UP_PAGES`]), then the channel's notices. + async fn window(&self, channel: &str) { + let mut filter = json!({ + "kinds": WINDOW_KINDS, "#h": [channel], "limit": WINDOW_PAGE, + "top_level": true, "include_aux": true, "include_summaries": true, + }); + for _ in 0..=CATCH_UP_PAGES { + let Some(page) = self.query(Op::DesktopWindow, json!([filter])).await else { + break; + }; + match oldest(&page) { + Some(until) if page.len() >= WINDOW_PAGE => filter["until"] = json!(until), + _ => break, + } + } + let mut notices = json!({"kinds": [40099], "#h": [channel], "limit": REPLAY_LIMIT}); + if let Some(relay) = lock(&self.relay).clone() { + notices["authors"] = json!([relay]); + } + self.query(Op::DesktopNotices, json!([notices])).await; + } + + /// Refreshes open thread `i`: a seeded root in one of the open windows' + /// channels, read by id and then its first thread window page. + async fn thread(&self, i: usize) { + let windows = self.windows.clamp(1, self.channels.len()); + let channel = &self.channels[i % windows]; + let shared = &self.shared; + let community = &shared.communities[self.community]; + let found = roots(shared, community, &self.keys, channel).await; + let Some(root) = found.get((i / windows) % found.len().max(1)) else { + return; + }; + let lookup = json!([{"ids": [root], "#h": [channel], "limit": 1}]); + if self.query(Op::DesktopThreadRoot, lookup).await.is_none() { + return; + } + let page = json!([{ + "kinds": [9, 40002], "#h": [channel], "#e": [root], "depth_limit": 100, + "include_aux": true, "thread_window": true, "limit": 10, + }]); + self.query(Op::DesktopThreadWindow, page).await; + } +} + +/// The value behind `mutex`; a panicked holder never left it half-written. +fn lock(mutex: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// `created_at` of the oldest event in a page. +fn oldest(page: &[Value]) -> Option { + page.iter().filter_map(|e| e["created_at"].as_u64()).min() +} + +/// Runs `fetches`: NIP-11 first, since notices need the relay's pubkey, then +/// the rest at most [`HTTP_CONCURRENCY`] at a time. +async fn run_fetches(reader: Arc, fetches: Vec) { + if fetches.contains(&Fetch::Nip11) { + reader.nip11().await; + } + stream::iter(fetches.into_iter().filter(|f| *f != Fetch::Nip11)) + .map(|fetch| { + let reader = reader.clone(); + async move { reader.fetch(fetch).await } + }) + .buffer_unordered(HTTP_CONCURRENCY) + .collect::>() + .await; +} + +/// A client's HTTP reads: one task at a time, so reads never pile up behind +/// a slow relay. Reads that come due while it runs are queued, once each, +/// for the next task. Dropping it (the client was cancelled) aborts the task, +/// so its unfinished reads are recorded. +#[derive(Default)] +struct Fetcher { + task: Option>, + queued: Vec, +} + +impl Fetcher { + fn start(&mut self, reader: &Arc, fetches: Vec) { + for fetch in fetches { + if !self.queued.contains(&fetch) { + self.queued.push(fetch); + } + } + self.poll(reader); + } + + /// Starts the queued reads once the previous task finished. + fn poll(&mut self, reader: &Arc) { + if self.queued.is_empty() || self.task.as_ref().is_some_and(|t| !t.is_finished()) { + return; + } + let fetches = std::mem::take(&mut self.queued); + self.task = Some(tokio::spawn(run_fetches(reader.clone(), fetches))); + } + + /// Waits for the task under way. Reads still queued are dropped. + async fn finish(&mut self) { + if let Some(task) = self.task.take() { + let _ = task.await; + } + } +} + +impl Drop for Fetcher { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} + +/// How a session ended. +enum End { + Stopped, + /// A planned identity renewal. + Renewal, + /// The human's scheduled session end, an unplanned reconnect. + Dropped, + Failed, +} + +/// A REQ awaiting its EOSE. +struct Waiting<'a> { + replay: bool, + pending: Pending<'a>, + sent: Instant, +} + +/// A desktop client. Its sessions differ by [`Situation`]: the first is +/// cold, a renewal follows [`renewal`], and any other reconnect is a +/// [`Situation::Reconnect`]. +pub(super) async fn desktop(shared: Arc, global: usize) { + let scenario = &shared.scenario; + let profile = &scenario.desktop; + let community = global % shared.communities.len(); + let keys = identity::keys(&scenario.run_id, Role::Human, global); + let indices = scenario.member_channel_indices(Role::Human, global); + let directory = shared.directory().await; + let reader = Arc::new(Reader { + shared: shared.clone(), + community, + viewer: keys.public_key().to_hex(), + keys: keys.clone(), + channels: scenario.member_channels(community, Role::Human, global), + profiles: co_members(&directory, community, global, &indices), + windows: profile.windows.min(indices.len()), + relay: Default::default(), + unread_mark: Default::default(), + }); + let mut client = Client { + shared: shared.clone(), + reader, + rng: Rng::new(global as u64 ^ 0xD35C_70B0), + fetcher: Fetcher::default(), + situation: Situation::Cold, + roster_at: Instant::now(), + unread_at: Instant::now(), + generation: 0, + first_renewal: true, + last_message: None, + }; + let started = unix_now(); + let mut cursor = None; + let mut backoff = Backoff::new(); + let target = &shared.communities[community]; + while let Some(mut conn) = + connect(&shared, target, &keys, None, &mut client.rng, &mut backoff).await + { + let since = resume_since(started, cursor, unix_now()); + let end = client.session(&mut conn, since).await; + cursor = Some(unix_now()); + let _ = conn.disconnect().await; + client.situation = match end { + End::Stopped => break, + End::Renewal => renewal(profile.mode, client.reader.channels.len()), + End::Dropped => { + backoff = Backoff::new(); + Situation::Reconnect + } + End::Failed => { + // A failed session backs off like a failed connect. + backoff.wait(&shared, &mut client.rng).await; + Situation::Reconnect + } + }; + } + client.fetcher.finish().await; +} + +struct Client { + shared: Arc, + reader: Arc, + rng: Rng, + fetcher: Fetcher, + situation: Situation, + roster_at: Instant, + unread_at: Instant, + generation: u64, + first_renewal: bool, + last_message: Option, +} + +impl Client { + /// One connection: establishes per the situation, then runs ticks, + /// presence and the human's own traffic until it ends. + async fn session(&mut self, conn: &mut NostrWsConnection, since: u64) -> End { + let shared = self.shared.clone(); + let stats = &shared.stats; + let scenario = &shared.scenario; + let profile = &scenario.desktop; + let mode = profile.mode; + let reader = self.reader.clone(); + let channels = &reader.channels; + let community = &shared.communities[reader.community]; + + self.generation += 1; + let reqs = connect_reqs(mode, &reader.viewer, channels, since, self.generation); + let mut batches: Vec = reqs + .iter() + .filter(|r| r.id.starts_with("batch-")) + .map(|r| r.id.clone()) + .collect(); + let mut batches_at = Instant::now(); + let mut setup: VecDeque = reqs.into(); + let reads = connect_fetches(self.situation, reader.windows, profile.threads); + if reads.contains(&Fetch::Roster) { + self.roster_at = Instant::now(); + } + if reads.contains(&Fetch::Unread) { + self.unread_at = Instant::now(); + } + self.fetcher.start(&reader, reads); + + let mut waiting: HashMap> = HashMap::new(); + let mut open: HashSet = HashSet::new(); + // New batch id to the batch it replaces, closed at the new one's EOSE. + let mut replacing: HashMap = HashMap::new(); + let mut activity = Activity::new(&scenario.human, &mut self.rng); + let tick_every = Duration::from_secs_f64(profile.tick_secs); + let backstop = Duration::from_secs_f64(profile.backstop_secs); + let mut next_tick = Instant::now() + tick_every; + let next_renewal = (profile.renewal_secs > 0.0).then(|| { + // Clients connecting together do not renew together. + let mut secs = profile.renewal_secs; + if std::mem::take(&mut self.first_renewal) { + secs *= self.rng.unit(); + } + Instant::now() + Duration::from_secs_f64(secs) + }); + let disconnected = |e: WsClientError| { + stats.err(Op::WsConnect, &format!("disconnected: {e}")); + End::Failed + }; + + let end = 'session: loop { + if stopped(&shared) { + break End::Stopped; + } + while waiting.len() < SETUP_CONCURRENCY { + let Some(req) = setup.pop_front() else { + break; + }; + if let Err(e) = conn.send_raw(&req.frame()).await { + break 'session disconnected(e); + } + let op = if req.replay { + Op::DesktopReplay + } else { + Op::DesktopLive + }; + let waited = Waiting { + replay: req.replay, + pending: stats.begin(op), + sent: Instant::now(), + }; + waiting.insert(req.id, waited); + } + let expired: Vec = waiting + .iter() + .filter(|(_, w)| w.sent.elapsed() >= REQ_TIMEOUT) + .map(|(id, _)| id.clone()) + .collect(); + for id in expired { + let Some(w) = waiting.remove(&id) else { + continue; + }; + w.pending.err("EOSE timeout"); + // A late live subscription may still deliver; a replay is abandoned. + if w.replay { + if let Err(e) = close(conn, &id).await { + break 'session disconnected(e); + } + } else { + open.insert(id); + } + } + self.fetcher.poll(&reader); + + let due = [activity.due(), Some(next_tick), next_renewal] + .into_iter() + .flatten() + .min(); + let wait = due + .map(|d| d.saturating_duration_since(Instant::now())) + .unwrap_or(MAX_IDLE) + .min(MAX_IDLE); + match conn.next_event(wait).await { + Ok(RelayMessage::Eose { subscription_id }) => { + if let Some(w) = waiting.remove(&subscription_id) { + w.pending.ok(); + let retired = if w.replay { + Some(subscription_id) + } else { + let old = replacing.remove(&subscription_id); + open.insert(subscription_id); + old.inspect(|old| { + open.remove(old); + }) + }; + if let Some(id) = retired { + if let Err(e) = close(conn, &id).await { + break 'session disconnected(e); + } + } + } + } + Ok(RelayMessage::Closed { + subscription_id, + message, + }) => { + // Live subscriptions the relay refuses end the session, + // as for the basic human; a refused replay is an error. + if let Some(w) = waiting.remove(&subscription_id) { + w.pending.err(&message); + if !w.replay { + break End::Failed; + } + } else if open.remove(&subscription_id) { + stats.err( + Op::DesktopLive, + &format!("CLOSED {subscription_id}: {message}"), + ); + break End::Failed; + } + } + Ok(message) => { + // Only live events after EOSE are delivery; replays are history. + let live = matches!( + &message, + RelayMessage::Event { subscription_id, .. } if open.contains(subscription_id) + ); + if live { + if let Some(target) = observe(stats, &message) { + self.last_message = Some(target); + } + } + } + Err(WsClientError::Timeout) => {} + Err(e) => break disconnected(e), + } + + let step = activity + .step( + &shared, + community, + &reader.keys, + channels, + conn, + &mut self.rng, + &mut self.last_message, + ) + .await; + match step { + Ok(Step::Continue) => {} + Ok(Step::SessionEnd) => break End::Dropped, + Err(e) => break disconnected(e), + } + + let now = Instant::now(); + if now >= next_tick { + next_tick = now + tick_every; + let due = tick( + mode, + backstop, + batches_at.elapsed(), + self.roster_at.elapsed(), + self.unread_at.elapsed(), + ); + // A replacement still waiting for its EOSE is not replaced again. + if due.batches && replacing.is_empty() { + self.generation += 1; + let fresh = batch_reqs(mode, channels, unix_now(), 0, self.generation); + for (new, old) in fresh.iter().zip(&batches) { + replacing.insert(new.id.clone(), old.clone()); + } + batches = fresh.iter().map(|r| r.id.clone()).collect(); + setup.extend(fresh); + batches_at = now; + } + if due.roster { + self.roster_at = now; + } + if due.unread { + self.unread_at = now; + } + self.fetcher.start(&reader, tick_fetches(due)); + } + if next_renewal.is_some_and(|t| now >= t) { + break End::Renewal; + } + }; + // Subscriptions still setting up are abandoned with the socket, not failed. + for (_, w) in waiting.drain() { + w.pending.forget(); + } + end + } +} + +async fn close(conn: &mut NostrWsConnection, id: &str) -> Result<(), WsClientError> { + conn.send_raw(&json!(["CLOSE", id])).await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ids(n: usize) -> Vec { + (0..n).map(|i| format!("c{i}")).collect() + } + + /// (filters, limit of the first filter, `#h` ids of the first filter) per REQ. + fn shape(reqs: &[Req]) -> Vec<(usize, u64, usize, bool)> { + reqs.iter() + .map(|r| { + let first = &r.filters[0]; + ( + r.filters.len(), + first["limit"].as_u64().unwrap(), + first["#h"].as_array().map_or(0, Vec::len), + r.replay, + ) + }) + .collect() + } + + #[test] + fn up_to_ten_channels_share_one_replaying_batch_in_both_modes() { + for mode in [DesktopMode::Legacy, DesktopMode::Current] { + let reqs = connect_reqs(mode, "v", &ids(5), 100, 1); + let names: Vec<&str> = reqs.iter().map(|r| r.id.as_str()).collect(); + assert_eq!(names, ["profiles", "membership", "batch-1-0"], "{mode:?}"); + assert_eq!(shape(&reqs[2..]), [(5, 500, 1, false)], "{mode:?}"); + assert_eq!(reqs[2].filters[0]["since"], 100); + assert_eq!(reqs[2].filters[0]["kinds"], json!(CHANNEL_KINDS)); + } + } + + #[test] + fn more_than_ten_channels_batch_by_ten_before_and_replay_separately_after() { + let legacy = connect_reqs(DesktopMode::Legacy, "v", &ids(25), 100, 1); + assert_eq!( + shape(&legacy[2..]), + [(10, 500, 1, false), (10, 500, 1, false), (5, 500, 1, false)] + ); + let current = connect_reqs(DesktopMode::Current, "v", &ids(25), 100, 1); + assert_eq!( + shape(¤t[2..]), + [ + (1, 0, 25, false), + (10, 500, 1, true), + (10, 500, 1, true), + (5, 500, 1, true) + ] + ); + // One live-only filter holds up to 2048 channels. + let wide = connect_reqs(DesktopMode::Current, "v", &ids(2049), 100, 1); + let live: Vec<_> = wide[2..].iter().filter(|r| !r.replay).collect(); + assert_eq!(live.len(), 2); + assert_eq!(wide.len(), 2 + 2 + 205); + } + + #[test] + fn tick_replacements_are_live_only() { + let legacy = batch_reqs(DesktopMode::Legacy, &ids(25), 100, 0, 2); + assert_eq!( + shape(&legacy), + [(10, 0, 1, false), (10, 0, 1, false), (5, 0, 1, false)] + ); + let current = batch_reqs(DesktopMode::Current, &ids(25), 100, 0, 2); + assert_eq!(shape(¤t), [(1, 0, 25, false)]); + let small = batch_reqs(DesktopMode::Current, &ids(5), 100, 0, 2); + assert_eq!(shape(&small), [(5, 0, 1, false)]); + assert_eq!(small[0].id, "batch-2-0"); + } + + #[test] + fn legacy_ticks_do_everything_and_current_ticks_wait_for_the_backstop() { + let backstop = Duration::from_secs(900); + let minute = Duration::from_secs(60); + let all = Due { + batches: true, + roster: true, + unread: true, + }; + let none = Due { + batches: false, + roster: false, + unread: false, + }; + assert_eq!( + tick(DesktopMode::Legacy, backstop, minute, minute, minute), + all + ); + assert_eq!( + tick(DesktopMode::Current, backstop, minute, minute, minute), + none + ); + assert_eq!( + tick(DesktopMode::Current, backstop, backstop, backstop, backstop), + all + ); + // Each backstop runs on its own clock. + let roster_only = tick(DesktopMode::Current, backstop, minute, backstop, minute); + assert_eq!(tick_fetches(roster_only), [Fetch::Roster]); + assert_eq!( + tick_fetches(all), + [Fetch::Roster, Fetch::ReadState, Fetch::Unread] + ); + assert!(tick_fetches(none).is_empty()); + } + + #[test] + fn renewals_reconnect_before_and_carry_or_limit_after() { + assert_eq!(renewal(DesktopMode::Legacy, 5), Situation::Reconnect); + assert_eq!(renewal(DesktopMode::Legacy, 25), Situation::Reconnect); + assert_eq!(renewal(DesktopMode::Current, 10), Situation::QuietRenewal); + assert_eq!(renewal(DesktopMode::Current, 11), Situation::LimitedRenewal); + + assert!(connect_fetches(Situation::QuietRenewal, 2, 1).is_empty()); + assert_eq!( + connect_fetches(Situation::LimitedRenewal, 2, 1), + [ + Fetch::Unread, + Fetch::Window(0), + Fetch::Window(1), + Fetch::Thread(0) + ] + ); + let cold = connect_fetches(Situation::Cold, 2, 1); + assert_eq!(cold.len(), 10 + 3); + let reconnect = connect_fetches(Situation::Reconnect, 2, 1); + // A reconnect rereads everything but names and sidebar preferences. + let skipped: Vec<_> = cold.iter().filter(|r| !reconnect.contains(r)).collect(); + assert_eq!(skipped, [&Fetch::Metadata, &Fetch::Sidebar]); + } + + #[test] + fn resumed_subscriptions_start_before_the_cursor_but_not_before_the_floor() { + assert_eq!(resume_since(10_000, None, 20_000), 19_700); + assert_eq!(resume_since(10_000, Some(15_000), 20_000), 14_040); + assert_eq!(resume_since(10_000, Some(10_100), 20_000), 9_700); + } + + // The client against loopback fakes: every connect, REQ, CLOSE and HTTP + // read below goes through the production paths. + + use super::super::net::Net; + use super::super::run::kind; + use super::super::scenario::Scenario; + use super::super::seed::tests::fake_http; + use super::super::stats::Stats; + use nostr::EventBuilder; + use tokio::sync::watch; + + /// `(verb, subscription id)` of every REQ and CLOSE the fake relay got. + type Frames = Arc>>; + + /// A loopback relay that challenges, accepts AUTH and every EVENT, answers + /// each REQ with EOSE and records REQs and CLOSEs. + async fn fake_ws_relay() -> (String, Frames) { + use futures_util::SinkExt; + use tokio_tungstenite::tungstenite::Message; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let frames = Frames::default(); + let seen = frames.clone(); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let seen = seen.clone(); + tokio::spawn(async move { + let Ok(mut ws) = tokio_tungstenite::accept_async(stream).await else { + return; + }; + let challenge = json!(["AUTH", "challenge"]).to_string(); + if ws.send(Message::Text(challenge.into())).await.is_err() { + return; + } + while let Some(Ok(Message::Text(text))) = ws.next().await { + let frame: Value = serde_json::from_str(&text).unwrap(); + let verb = frame[0].as_str().unwrap_or_default().to_string(); + let reply = match verb.as_str() { + "AUTH" | "EVENT" => json!(["OK", frame[1]["id"], true, ""]), + "REQ" => json!(["EOSE", frame[1]]), + _ => Value::Null, + }; + if verb == "REQ" || verb == "CLOSE" { + let id = frame[1].as_str().unwrap_or_default().to_string(); + seen.lock().unwrap().push((verb, id)); + } + if !reply.is_null() + && ws + .send(Message::Text(reply.to_string().into())) + .await + .is_err() + { + return; + } + } + }); + } + }); + (addr, frames) + } + + /// A loopback HTTP API: NIP-11 names a relay key, the owner's root search + /// finds one seeded root, and every other query is empty. + async fn fake_api(run_id: &str) -> String { + let owner = identity::keys(run_id, Role::Owner, 0); + let root = EventBuilder::new(kind(9), "lg-root 0") + .sign_with_keys(&owner) + .unwrap(); + let root = serde_json::to_string(&[root]).unwrap(); + let info = json!({"self": Keys::generate().public_key().to_hex()}).to_string(); + let owner = owner.public_key().to_hex(); + fake_http(move |method, target, body| { + if method == "GET" && target == "/" { + return (200, info.clone()); + } + let filters: Value = serde_json::from_slice(body).unwrap_or_default(); + let filter = &filters[0]; + if filter["kinds"] == json!([9]) && filter["authors"] == json!([owner]) { + (200, root.clone()) + } else { + (200, "[]".into()) + } + }) + .await + } + + /// Two desktop humans in one community of 25 channels, each in `joined`, + /// with two windows and one thread open and no traffic of their own. + fn two_desktops(ws: &str, http: &str, joined: usize, desktop: &str) -> Scenario { + Scenario::parse(&format!( + r#" +run_id = "d" +[target] +ws_url = "ws://{ws}" +http_url = "http://{http}" +[shape] +communities = 1 +channels_per_community = 25 +[population] +humans = 2 +agents = 0 +channels_per_member = {joined} +[human] +presence_interval_secs = 3600.0 +typing_per_hour = 0.0 +messages_per_hour = 0.0 +reactions_per_hour = 0.0 +history_limit = 1 +[desktop] +fraction = 1.0 +windows = 2 +threads = 1 +{desktop} +[[ramp]] +users = 1.0 +hold_secs = 1 +"# + )) + .unwrap() + } + + /// Per-op `(count, errors)` by op name, plus the relay's frames. + struct Outcome { + ops: HashMap<&'static str, (u64, u64)>, + frames: Vec<(String, String)>, + } + + impl Outcome { + fn count(&self, op: &str) -> u64 { + self.ops.get(op).map_or(0, |c| c.0) + } + + fn saw(&self, verb: &str, id: &str) -> bool { + self.frames.iter().any(|(v, i)| v == verb && i == id) + } + + /// Counts of the desktop ops, the ones the model owns. + fn desktop(&self) -> HashMap<&'static str, u64> { + self.ops + .iter() + .filter(|(op, _)| op.starts_with("desktop_")) + .map(|(op, c)| (*op, c.0)) + .collect() + } + } + + fn outcome(stats: &Stats, frames: &Frames) -> Outcome { + let ops = stats + .snapshot() + .since(&Default::default()) + .into_iter() + .filter(|(_, s)| s.count + s.errors > 0) + .map(|(op, s)| (op.name(), (s.count, s.errors))) + .collect(); + Outcome { + ops, + frames: frames.lock().unwrap().clone(), + } + } + + /// Runs both clients for at least `min`, until `done` holds, then stops + /// them and returns what they did. Fails after 15 s without `done`. + async fn drive( + joined: usize, + desktop_toml: &str, + min: Duration, + done: impl Fn(&Outcome) -> bool, + ) -> Outcome { + let (ws, frames) = fake_ws_relay().await; + let http = fake_api("d").await; + let scenario = two_desktops(&ws, &http, joined, desktop_toml); + let net = Net::new(&scenario).await.unwrap(); + let stats = Arc::new(Stats::default()); + let (stop_tx, stop) = watch::channel(false); + let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); + let clients: Vec<_> = (0..2) + .map(|g| tokio::spawn(desktop(shared.clone(), g))) + .collect(); + let started = Instant::now(); + loop { + let now = outcome(&stats, &frames); + if started.elapsed() >= min && done(&now) { + break; + } + assert!( + started.elapsed() < Duration::from_secs(15), + "never done: {:?} {:?}", + now.ops, + now.frames + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + stop_tx.send(true).unwrap(); + for client in clients { + tokio::time::timeout(Duration::from_secs(5), client) + .await + .expect("a stopped client returns") + .unwrap(); + } + let out = outcome(&stats, &frames); + let failed: Vec<_> = out.ops.iter().filter(|(_, c)| c.1 > 0).collect(); + assert!(failed.is_empty(), "errors: {failed:?}"); + out + } + + /// What one cold start reads over HTTP with two windows and a thread. + fn cold_reads(clients: u64) -> HashMap<&'static str, u64> { + [ + ("desktop_nip11", 1), + ("desktop_roster", 1), + ("desktop_metadata", 1), + ("desktop_sidebar", 1), + ("desktop_profiles", 1), + ("desktop_statuses", 1), + ("desktop_emoji", 1), + ("desktop_read_state", 1), + ("desktop_unread", 1), + ("desktop_reminders", 1), + ("desktop_window", 2), + ("desktop_notices", 2), + ("desktop_thread_root", 1), + ("desktop_thread_window", 1), + ] + .into_iter() + .map(|(op, n)| (op, n * clients)) + .collect() + } + + fn with( + mut ops: HashMap<&'static str, u64>, + extra: &[(&'static str, u64)], + ) -> HashMap<&'static str, u64> { + ops.extend(extra.iter().copied()); + ops + } + + const QUIET: &str = "tick_secs = 3600.0"; + + #[tokio::test] + async fn a_cold_start_sends_the_modeled_requests_once() { + // Legacy: per-channel batches of ten for 25 channels; current: one + // live-only batch and three replays. Both read the same over HTTP. + let cases = [ + ("legacy", 25, vec![("desktop_live_eose", 2 * 5)]), + ( + "current", + 25, + vec![("desktop_live_eose", 2 * 3), ("desktop_replay_eose", 2 * 3)], + ), + ("legacy", 5, vec![("desktop_live_eose", 2 * 3)]), + ("current", 5, vec![("desktop_live_eose", 2 * 3)]), + ]; + for (mode, joined, subs) in cases { + let want = with(cold_reads(2), &subs); + let toml = format!("mode = \"{mode}\"\n{QUIET}"); + let out = drive(joined, &toml, Duration::from_millis(300), |o| { + o.desktop() == want + }) + .await; + assert_eq!(out.desktop(), want, "{mode} J={joined}"); + assert_eq!(out.count("ws_connect_auth"), 2, "{mode} J={joined}"); + } + } + + #[tokio::test] + async fn legacy_ticks_reread_and_replace_batches_every_tick() { + let out = drive( + 25, + "mode = \"legacy\"\ntick_secs = 1.0", + Duration::ZERO, + |o| o.count("desktop_roster") >= 4 && o.saw("CLOSE", "batch-1-2"), + ) + .await; + // Each replacement is sent before the batch it replaces is closed. + assert!(out.saw("REQ", "batch-2-0") && out.saw("CLOSE", "batch-1-0")); + assert!(out.count("desktop_read_state") >= 4); + assert!(out.count("desktop_unread") >= 4); + // Ticks never reread names or open windows. + assert_eq!(out.count("desktop_metadata"), 2); + assert_eq!(out.count("desktop_window"), 4); + } + + #[tokio::test] + async fn current_ticks_wait_for_the_backstop() { + let before = drive( + 25, + "mode = \"current\"\ntick_secs = 1.0", + Duration::from_millis(2500), + |o| o.count("desktop_thread_window") == 2, + ) + .await; + assert_eq!( + before.desktop(), + with( + cold_reads(2), + &[("desktop_live_eose", 6), ("desktop_replay_eose", 6)] + ) + ); + assert!(!before + .frames + .iter() + .any(|(_, id)| id.starts_with("batch-2"))); + + let after = drive( + 25, + "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 1.0", + Duration::ZERO, + |o| o.count("desktop_roster") >= 4 && o.saw("CLOSE", "batch-1-0"), + ) + .await; + assert!(after.saw("REQ", "batch-2-0")); + assert!(after.count("desktop_unread") >= 4); + // A replacement is live-only: no replays after the connect's three. + assert_eq!(after.count("desktop_replay_eose"), 6); + } + + #[tokio::test] + async fn legacy_renewals_are_full_reconnects() { + let out = drive( + 25, + "mode = \"legacy\"\nrenewal_secs = 1.0\n", + Duration::ZERO, + |o| o.count("desktop_nip11") >= 4 && o.count("desktop_roster") >= 4, + ) + .await; + assert!(out.count("ws_connect_auth") >= 4); + // A reconnect keeps channel names and sidebar preferences. + assert_eq!(out.count("desktop_metadata"), 2); + assert_eq!(out.count("desktop_sidebar"), 2); + } + + #[tokio::test] + async fn current_renewals_carry_small_sessions_and_limit_large_ones() { + // More than ten channels: replays and channel catch-up again, no globals. + let large = drive( + 25, + "mode = \"current\"\nrenewal_secs = 1.0\n", + Duration::ZERO, + |o| o.count("desktop_replay_eose") >= 12 && o.count("desktop_unread") >= 4, + ) + .await; + assert!(large.count("ws_connect_auth") >= 4); + for op in [ + "desktop_nip11", + "desktop_roster", + "desktop_metadata", + "desktop_profiles", + ] { + assert_eq!(large.count(op), 2, "{op}"); + } + assert!(large.count("desktop_window") >= 8); + + // Up to ten channels: subscriptions are re-sent and nothing is read. + let small = drive( + 5, + "mode = \"current\"\nrenewal_secs = 1.0\n", + Duration::ZERO, + |o| o.count("desktop_live_eose") >= 12 && o.count("desktop_thread_window") == 2, + ) + .await; + assert!(small.count("ws_connect_auth") >= 4); + let reads: HashMap<_, _> = small + .desktop() + .into_iter() + .filter(|(op, _)| *op != "desktop_live_eose") + .collect(); + assert_eq!(reads, cold_reads(2)); + } +} diff --git a/crates/buzz-test-client/src/loadgen/identity.rs b/crates/buzz-test-client/src/loadgen/identity.rs index 44f8cc8a9c3..4ed86986bb9 100644 --- a/crates/buzz-test-client/src/loadgen/identity.rs +++ b/crates/buzz-test-client/src/loadgen/identity.rs @@ -7,7 +7,7 @@ use nostr::{EventBuilder, JsonUtil, Keys, Kind, SecretKey, Tag}; use sha2::{Digest, Sha256}; /// Role of a derived identity; part of the derivation so roles never collide. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Role { Owner, Human, diff --git a/crates/buzz-test-client/src/loadgen/mod.rs b/crates/buzz-test-client/src/loadgen/mod.rs index 8bf6bda6075..e56809d0b4a 100644 --- a/crates/buzz-test-client/src/loadgen/mod.rs +++ b/crates/buzz-test-client/src/loadgen/mod.rs @@ -11,6 +11,7 @@ //! scenario's `run_id`, so independent shards (`--shard i/n`) agree on the //! fixture without sharing state. +pub mod desktop; pub mod identity; pub mod net; pub mod run; diff --git a/crates/buzz-test-client/src/loadgen/net.rs b/crates/buzz-test-client/src/loadgen/net.rs index 2b44a550fff..f8971ffc8c7 100644 --- a/crates/buzz-test-client/src/loadgen/net.rs +++ b/crates/buzz-test-client/src/loadgen/net.rs @@ -131,6 +131,18 @@ impl Net { Ok((status, response.text().await?)) } + /// `GET {base}/` for the relay's NIP-11 information document. + pub async fn relay_info(&self, base: &str) -> anyhow::Result<(reqwest::StatusCode, String)> { + let response = self + .http + .get(self.transport_url(&format!("{base}/"))) + .header("Accept", "application/nostr+json") + .send() + .await?; + let status = response.status(); + Ok((status, response.text().await?)) + } + /// Submits `event` through HTTP `/events`; errors unless the relay accepted it. pub async fn submit( &self, diff --git a/crates/buzz-test-client/src/loadgen/run.rs b/crates/buzz-test-client/src/loadgen/run.rs index 8b78b467d3e..e95f6e766e1 100644 --- a/crates/buzz-test-client/src/loadgen/run.rs +++ b/crates/buzz-test-client/src/loadgen/run.rs @@ -16,16 +16,17 @@ use nostr::{Event, EventBuilder, Filter, Keys, Kind, PublicKey, Tag, Timestamp}; use serde_json::{json, Value}; use tokio::sync::{watch, Mutex}; +use super::desktop; use super::identity::{self, Role}; use super::net::{http_reason, Net}; -use super::scenario::{Community, Scenario}; +use super::scenario::{Community, HumanProfile, Members, Scenario}; use super::seed::{find_roots, hot_root_keys, reply_tags, HOT_ROOT_PREFIX, ROOT_PREFIX}; use super::stats::{abort_reason, ops_json, Op, Stats}; /// Content prefix of timed messages: `lg:`. const TIMED_PREFIX: &str = "lg:"; /// Longest an idle client blocks on its socket before rechecking the stop flag. -const MAX_IDLE: Duration = Duration::from_secs(1); +pub(super) const MAX_IDLE: Duration = Duration::from_secs(1); /// How long stopped clients get to settle what they have in flight and /// disconnect before they are cancelled. const SHUTDOWN_GRACE: Duration = Duration::from_secs(15); @@ -57,14 +58,69 @@ impl Shard { } } -struct Shared { - scenario: Scenario, - communities: Vec, - net: Net, - stats: Arc, - stop: watch::Receiver, +pub(super) struct Shared { + pub(super) scenario: Scenario, + pub(super) communities: Vec, + pub(super) net: Net, + pub(super) stats: Arc, + pub(super) stop: watch::Receiver, /// Seeded thread roots per channel id, resolved lazily by whichever client needs them first. roots: Mutex>>>, + /// Channel members and human pubkeys, built once by the first desktop + /// client that needs them. + pub(super) directory: tokio::sync::OnceCell>, +} + +/// Who is in each channel, for clients that read their co-members' profiles. +pub(super) struct Directory { + pub(super) members: Members, + /// Hex pubkey of every human up to the peak population, by global index. + pub(super) humans: Vec, +} + +impl Shared { + pub(super) fn new( + scenario: Scenario, + net: Net, + stats: Arc, + stop: watch::Receiver, + ) -> Self { + Self { + communities: scenario.communities(), + scenario, + net, + stats, + stop, + roots: Mutex::new(HashMap::new()), + directory: tokio::sync::OnceCell::new(), + } + } + + /// The [`Directory`], derived off the async workers on first use: it + /// derives a key for every human up to the peak population. + pub(super) async fn directory(&self) -> Arc { + self.directory + .get_or_init(|| async { + let scenario = self.scenario.clone(); + tokio::task::spawn_blocking(move || { + let humans = (0..scenario.peak().humans) + .map(|i| { + identity::keys(&scenario.run_id, Role::Human, i) + .public_key() + .to_hex() + }) + .collect(); + Arc::new(Directory { + members: scenario.channel_members(), + humans, + }) + }) + .await + .unwrap_or_else(|e| std::panic::resume_unwind(e.into_panic())) + }) + .await + .clone() + } } /// Runs every ramp step and returns the JSON report; `Err` when a step aborted. @@ -82,14 +138,7 @@ async fn run_with( ) -> Result { let stats = Arc::new(Stats::default()); let (stop_tx, stop) = watch::channel(false); - let shared = Arc::new(Shared { - communities: scenario.communities(), - scenario, - net, - stats: stats.clone(), - stop, - roots: Mutex::new(HashMap::new()), - }); + let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); let scenario = &shared.scenario; let pop = &scenario.population; let connect_gap = Duration::from_secs_f64(1.0 / pop.connect_rate_per_sec); @@ -121,6 +170,8 @@ async fn run_with( tasks.push(tokio::spawn(async move { if spawn_agent { agent(shared, global).await + } else if shared.scenario.is_desktop(global) { + desktop::desktop(shared, global).await } else { human(shared, global).await } @@ -258,10 +309,10 @@ async fn shutdown(tasks: Vec>, grace: Duration) -> ( } /// Small deterministic PRNG so every client draws an independent schedule. -struct Rng(u64); +pub(super) struct Rng(u64); impl Rng { - fn new(seed: u64) -> Self { + pub(super) fn new(seed: u64) -> Self { Self(seed ^ 0x9E37_79B9_7F4A_7C15) } @@ -273,16 +324,16 @@ impl Rng { z ^ (z >> 31) } - fn unit(&mut self) -> f64 { + pub(super) fn unit(&mut self) -> f64 { (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 } - fn below(&mut self, n: usize) -> usize { + pub(super) fn below(&mut self, n: usize) -> usize { (self.next_u64() % n.max(1) as u64) as usize } /// Next arrival of a Poisson process with `per_hour` events per hour. - fn after(&mut self, per_hour: f64) -> Option { + pub(super) fn after(&mut self, per_hour: f64) -> Option { if per_hour <= 0.0 { return None; } @@ -291,43 +342,43 @@ impl Rng { } } -fn now_micros() -> u128 { +pub(super) fn now_micros() -> u128 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_micros() } -fn tag(parts: [&str; N]) -> Tag { +pub(super) fn tag(parts: [&str; N]) -> Tag { Tag::parse(parts).expect("static tag shape") } -fn kind(k: u32) -> Kind { +pub(super) fn kind(k: u32) -> Kind { Kind::Custom(k as u16) } -fn stopped(shared: &Shared) -> bool { +pub(super) fn stopped(shared: &Shared) -> bool { *shared.stop.borrow() } /// Resolves once the run is stopping. -async fn stopping(shared: &Shared) { +pub(super) async fn stopping(shared: &Shared) { let mut stop = shared.stop.clone(); let _ = stop.wait_for(|s| *s).await; } /// Jittered exponential backoff between failed connects or failed sessions. -struct Backoff(f64); +pub(super) struct Backoff(f64); impl Backoff { const MAX_SECS: f64 = 30.0; - fn new() -> Self { + pub(super) fn new() -> Self { Self(1.0) } /// Sleeps for the next interval, returning early when the run stops. - async fn wait(&mut self, shared: &Shared, rng: &mut Rng) { + pub(super) async fn wait(&mut self, shared: &Shared, rng: &mut Rng) { let wait = Duration::from_secs_f64(self.0 * (0.5 + rng.unit())); self.0 = (self.0 * 2.0).min(Self::MAX_SECS); tokio::select! { @@ -339,7 +390,7 @@ impl Backoff { /// A stream message a client can react to: its id and channel. #[derive(Clone)] -struct Seen { +pub(super) struct Seen { id: String, channel: String, } @@ -398,7 +449,7 @@ fn live( /// Records end-to-end delivery latency for a live timed message and returns /// it as a reaction target. Only live subscriptions call this: history /// replays old messages, whose age is not delivery latency. -fn observe(stats: &Stats, message: &RelayMessage) -> Option { +pub(super) fn observe(stats: &Stats, message: &RelayMessage) -> Option { if let RelayMessage::Event { event, .. } = message { if let Some(sent) = event .content @@ -437,7 +488,7 @@ async fn publish( /// Connects with retry until connected or stopped. Each attempt is bounded /// by the connect deadline, which should be shorter than [`SHUTDOWN_GRACE`] /// so an attempt under way at the stop settles before clients are cancelled. -async fn connect( +pub(super) async fn connect( shared: &Shared, community: &Community, keys: &Keys, @@ -470,7 +521,7 @@ async fn connect( /// Thread roots of `channel`, cached once found. Concurrent first callers may /// each query; a failed query is not cached, so a later reply retries it. -async fn roots( +pub(super) async fn roots( shared: &Shared, community: &Community, reader: &Keys, @@ -508,8 +559,7 @@ async fn human(shared: Arc, global: usize) { let profile = &scenario.human; let community = &shared.communities[global % shared.communities.len()]; let keys = identity::keys(&scenario.run_id, Role::Human, global); - let me = keys.public_key().to_hex(); - let channels = scenario.member_channels(community.index, global); + let channels = scenario.member_channels(community.index, Role::Human, global); let peers: Vec = (1..=10) .map(|k| { let peer = (global + k * shared.communities.len()) % scenario.population.humans.max(1); @@ -603,28 +653,10 @@ async fn human(shared: Arc, global: usize) { } } - let interval = profile.presence_interval_secs; - let mut next_presence = - Some(Instant::now() + Duration::from_secs_f64(interval * rng.unit())); - let mut next_typing = rng.after(profile.typing_per_hour); - let mut next_message = rng.after(profile.messages_per_hour); - let mut next_reaction = rng.after(profile.reactions_per_hour); - let session_end = (profile.mean_session_secs > 0.0) - .then(|| rng.after(3600.0 / profile.mean_session_secs)) - .flatten(); - + let mut activity = Activity::new(profile, &mut rng); while ok && !stopped(&shared) { - let due = [ - next_presence, - next_typing, - next_message, - next_reaction, - session_end, - ] - .into_iter() - .flatten() - .min(); - let wait = due + let wait = activity + .due() .map(|d| d.saturating_duration_since(Instant::now())) .unwrap_or(MAX_IDLE) .min(MAX_IDLE); @@ -646,59 +678,27 @@ async fn human(shared: Arc, global: usize) { break; } } - let now = Instant::now(); - let result = if next_presence.is_some_and(|t| t <= now) { - next_presence = Some(now + Duration::from_secs_f64(interval)); - let event = - EventBuilder::new(kind(KIND_PRESENCE_UPDATE), "online").sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishEphemeral, event).await - } else if next_typing.is_some_and(|t| t <= now) { - next_typing = rng.after(profile.typing_per_hour); - let channel = &channels[rng.below(channels.len())]; - let event = EventBuilder::new(kind(KIND_TYPING_INDICATOR), "") - .tags([tag(["h", channel])]) - .sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishEphemeral, event).await - } else if next_message.is_some_and(|t| t <= now) { - next_message = rng.after(profile.messages_per_hour); - let channel = &channels[rng.below(channels.len())]; - let mut tags = vec![tag(["h", channel])]; - if rng.unit() < profile.thread_reply_fraction { - let roots = roots(&shared, community, &keys, channel).await; - if !roots.is_empty() { - tags.push(tag(["e", &roots[rng.below(roots.len())], "", "reply"])); - } - } - let event = EventBuilder::new( - kind(KIND_STREAM_MESSAGE), - format!("{TIMED_PREFIX}{} {me}", now_micros()), + let outcome = activity + .step( + &shared, + community, + &keys, + &channels, + &mut conn, + &mut rng, + &mut last_message, ) - .tags(tags) - .sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishMessage, event).await - } else if next_reaction.is_some_and(|t| t <= now) { - next_reaction = rng.after(profile.reactions_per_hour); - // React to each seen message at most once; the relay rejects duplicates. - match last_message.take() { - Some(target) => { - let event = EventBuilder::new(kind(KIND_REACTION), "+") - .tags([tag(["h", &target.channel]), tag(["e", &target.id])]) - .sign_with_keys(&keys); - send(&shared.stats, &mut conn, Op::PublishMessage, event).await - } - None => Ok(()), + .await; + match outcome { + Ok(Step::Continue) => {} + Ok(Step::SessionEnd) => break, + Err(e) => { + shared + .stats + .err(Op::WsConnect, &format!("disconnected: {e}")); + ok = false; + break; } - } else if session_end.is_some_and(|t| t <= now) { - break; - } else { - Ok(()) - }; - if let Err(e) = result { - shared - .stats - .err(Op::WsConnect, &format!("disconnected: {e}")); - ok = false; - break; } } let _ = conn.disconnect().await; @@ -715,7 +715,117 @@ async fn human(shared: Arc, global: usize) { } } -async fn send( +/// What [`Activity::step`] leaves the session to do. +pub(super) enum Step { + Continue, + /// The session reached its scheduled end; reconnect. + SessionEnd, +} + +/// A human's own traffic on its socket: presence heartbeats, typing, +/// messages (some as replies into seeded threads), reactions and the end of +/// its session, each on its own schedule. +pub(super) struct Activity { + interval: f64, + next_presence: Option, + next_typing: Option, + next_message: Option, + next_reaction: Option, + session_end: Option, +} + +impl Activity { + /// Schedules from now, the first heartbeat at a random point within one + /// interval so clients connecting together do not beat together. + pub(super) fn new(profile: &HumanProfile, rng: &mut Rng) -> Self { + let interval = profile.presence_interval_secs; + Self { + interval, + next_presence: Some(Instant::now() + Duration::from_secs_f64(interval * rng.unit())), + next_typing: rng.after(profile.typing_per_hour), + next_message: rng.after(profile.messages_per_hour), + next_reaction: rng.after(profile.reactions_per_hour), + session_end: (profile.mean_session_secs > 0.0) + .then(|| rng.after(3600.0 / profile.mean_session_secs)) + .flatten(), + } + } + + /// When the next item is due. + pub(super) fn due(&self) -> Option { + [ + self.next_presence, + self.next_typing, + self.next_message, + self.next_reaction, + self.session_end, + ] + .into_iter() + .flatten() + .min() + } + + /// Runs the first due item, if any. An error means the socket broke. + #[allow(clippy::too_many_arguments)] + pub(super) async fn step( + &mut self, + shared: &Shared, + community: &Community, + keys: &Keys, + channels: &[String], + conn: &mut NostrWsConnection, + rng: &mut Rng, + last_message: &mut Option, + ) -> Result { + let profile = &shared.scenario.human; + let now = Instant::now(); + if self.next_presence.is_some_and(|t| t <= now) { + self.next_presence = Some(now + Duration::from_secs_f64(self.interval)); + let event = + EventBuilder::new(kind(KIND_PRESENCE_UPDATE), "online").sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishEphemeral, event).await?; + } else if self.next_typing.is_some_and(|t| t <= now) { + self.next_typing = rng.after(profile.typing_per_hour); + let channel = &channels[rng.below(channels.len())]; + let event = EventBuilder::new(kind(KIND_TYPING_INDICATOR), "") + .tags([tag(["h", channel])]) + .sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishEphemeral, event).await?; + } else if self.next_message.is_some_and(|t| t <= now) { + self.next_message = rng.after(profile.messages_per_hour); + let channel = &channels[rng.below(channels.len())]; + let mut tags = vec![tag(["h", channel])]; + if rng.unit() < profile.thread_reply_fraction { + let roots = roots(shared, community, keys, channel).await; + if !roots.is_empty() { + tags.push(tag(["e", &roots[rng.below(roots.len())], "", "reply"])); + } + } + let me = keys.public_key().to_hex(); + let event = EventBuilder::new( + kind(KIND_STREAM_MESSAGE), + format!("{TIMED_PREFIX}{} {me}", now_micros()), + ) + .tags(tags) + .sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishMessage, event).await?; + } else if self.next_reaction.is_some_and(|t| t <= now) { + self.next_reaction = rng.after(profile.reactions_per_hour); + // React to each seen message at most once; the relay rejects duplicates. + if let Some(target) = last_message.take() { + let event = EventBuilder::new(kind(KIND_REACTION), "+") + .tags([tag(["h", &target.channel]), tag(["e", &target.id])]) + .sign_with_keys(keys); + send(&shared.stats, conn, Op::PublishMessage, event).await?; + } + } else if self.session_end.is_some_and(|t| t <= now) { + return Ok(Step::SessionEnd); + } + Ok(Step::Continue) + } +} + +pub(super) async fn send( stats: &Stats, conn: &mut NostrWsConnection, op: Op, @@ -850,7 +960,7 @@ async fn agent(shared: Arc, global: usize) { global % scenario.population.humans.max(1), ); let auth = identity::nip_oa_tag(&owner, &keys); - let channels = scenario.member_channels(community.index, global); + let channels = scenario.member_channels(community.index, Role::Agent, global); let me = keys.public_key().to_hex(); let mut rng = Rng::new(u64::MAX - global as u64); let mut backoff = Backoff::new(); @@ -1962,14 +2072,7 @@ hold_secs = 1 let net = Net::new(&scenario).await.unwrap(); let stats = Arc::new(Stats::default()); let (stop_tx, stop) = watch::channel(false); - let shared = Arc::new(Shared { - communities: scenario.communities(), - scenario, - net, - stats: stats.clone(), - stop, - roots: Mutex::new(HashMap::new()), - }); + let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); let authors: Vec<_> = (0..8) .map(|i| tokio::spawn(hot_author(shared.clone(), i))) .collect(); diff --git a/crates/buzz-test-client/src/loadgen/scenario.rs b/crates/buzz-test-client/src/loadgen/scenario.rs index f874d4b9c0f..e44cb495859 100644 --- a/crates/buzz-test-client/src/loadgen/scenario.rs +++ b/crates/buzz-test-client/src/loadgen/scenario.rs @@ -2,7 +2,7 @@ use serde::Deserialize; -use super::identity; +use super::identity::{self, Role}; /// A load scenario, parsed from TOML. #[derive(Clone, Debug, Deserialize)] @@ -19,6 +19,8 @@ pub struct Scenario { pub agent: AgentProfile, #[serde(default)] pub hot_thread: HotThread, + #[serde(default)] + pub desktop: DesktopProfile, pub ramp: Vec, #[serde(default)] pub abort: Abort, @@ -82,6 +84,16 @@ pub struct Shape { /// Concurrent seeding requests. #[serde(default = "default_seed_concurrency")] pub seed_concurrency: usize, + /// Create channels private and add every member with kind:9000, so the + /// relay checks real membership on every read and subscription. Off + /// creates open channels nobody joins. + #[serde(default)] + pub private_channels: bool, + /// The first `wide_channels` channels of each community are its wide + /// channels, which `population.wide_member_fraction` of humans join on + /// top of their own. Other memberships use the remaining channels. + #[serde(default)] + pub wide_channels: usize, } fn default_seed_concurrency() -> usize { @@ -99,6 +111,22 @@ pub struct Population { /// New connections per second per shard while ramping. #[serde(default = "default_connect_rate")] pub connect_rate_per_sec: f64, + /// Fraction of humans that also join every wide channel. + #[serde(default)] + pub wide_member_fraction: f64, + /// Humans whose channel count differs from `channels_per_member`, such + /// as a few heavy users in many channels. Agents keep `channels_per_member`. + #[serde(default)] + pub tiers: Vec, +} + +/// `fraction` of humans join `channels` channels instead of +/// `population.channels_per_member`, not counting wide channels. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Tier { + pub fraction: f64, + pub channels: usize, } fn default_connect_rate() -> f64 { @@ -181,6 +209,57 @@ impl HotThread { } } +/// Which desktop app request pattern a desktop client follows. +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum DesktopMode { + /// Before the client volume fixes: every recovery tick replaces each live + /// channel batch and rereads the roster, read state and unreads, and an + /// identity renewal is a full reconnect. + Legacy, + /// After them: tick reads and batch re-sends wait for a backstop, and a + /// renewal re-sends subscriptions from their cursors. + #[default] + Current, +} + +/// Humans that run the desktop app's relay request pattern instead of the +/// basic human client: HTTP reads of the roster, profiles, read state, +/// unreads and open windows on connect, batched channel subscriptions, a +/// recovery tick, and optional periodic identity renewal. Disabled while +/// `fraction` is 0. +#[derive(Clone, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DesktopProfile { + /// Fraction of humans that are desktop clients. + pub fraction: f64, + pub mode: DesktopMode, + /// Identity renewal interval, a reconnect the client plans (0: never). + pub renewal_secs: f64, + /// Channel windows kept open, each caught up on every establishment. + pub windows: usize, + /// Thread views kept open, each refreshed on every establishment. + pub threads: usize, + /// Recovery tick interval. + pub tick_secs: f64, + /// `current` mode's backstop for tick reads and batch re-sends. + pub backstop_secs: f64, +} + +impl Default for DesktopProfile { + fn default() -> Self { + Self { + fraction: 0.0, + mode: DesktopMode::Current, + renewal_secs: 0.0, + windows: 3, + threads: 1, + tick_secs: 60.0, + backstop_secs: 900.0, + } + } +} + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct RampStep { @@ -206,6 +285,41 @@ impl Default for Abort { } } +/// A member of channels: a human or agent by global index. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Member { + pub role: Role, + pub index: usize, +} + +/// Members of every channel, indexed `[community][channel]`. +pub type Members = Vec>>; + +/// Clients each kind reaches at the largest ramp step. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Peak { + pub humans: usize, + pub agents: usize, + pub hot_authors: usize, +} + +/// Salts for [`draw`], one per independent per-member choice. +const TIER_SALT: u64 = 1; +const WIDE_SALT: u64 = 2; +const DESKTOP_SALT: u64 = 3; + +/// A deterministic draw in `[0, 1)` for member `index`, independent for each +/// `salt` and of the client schedules, so every shard and `seed` agree. +fn draw(index: usize, salt: u64) -> f64 { + let mut z = (index as u64) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(salt.wrapping_mul(0xD1B5_4A32_D192_ED03)); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^= z >> 31; + (z >> 11) as f64 / (1u64 << 53) as f64 +} + /// One community's endpoints. #[derive(Clone, Debug)] pub struct Community { @@ -253,10 +367,35 @@ impl Scenario { .into(), ); } + let n = self.shape.channels_per_community; + let wide = self.shape.wide_channels; + // Every member needs at least one channel outside the wide ones. + if wide >= n && wide > 0 { + return Err("shape.wide_channels must be below channels_per_community".into()); + } if self.population.channels_per_member == 0 - || self.population.channels_per_member > self.shape.channels_per_community + || self.population.channels_per_member > n - wide { - return Err("population.channels_per_member must be 1..=channels_per_community".into()); + return Err( + "population.channels_per_member must be 1..=channels_per_community - wide_channels" + .into(), + ); + } + let mut tiered = 0.0; + for tier in &self.population.tiers { + if !(tier.fraction > 0.0 && tier.fraction <= 1.0) { + return Err("population.tiers fraction must be above 0 and at most 1".into()); + } + if tier.channels == 0 || tier.channels > n - wide { + return Err( + "population.tiers channels must be 1..=channels_per_community - wide_channels" + .into(), + ); + } + tiered += tier.fraction; + } + if tiered > 1.0 + 1e-9 { + return Err("population.tiers fractions must sum to at most 1".into()); } if self.ramp.is_empty() { return Err("ramp needs at least one step".into()); @@ -281,6 +420,7 @@ impl Scenario { ), ("agent.query_interval_secs", a.query_interval_secs), ("abort.max_p99_ms", self.abort.max_p99_ms), + ("desktop.backstop_secs", self.desktop.backstop_secs), ]; for (name, value) in positive { if !value.is_finite() || value <= 0.0 { @@ -291,6 +431,14 @@ impl Scenario { if !h.presence_interval_secs.is_finite() || h.presence_interval_secs < 1.0 { return Err("human.presence_interval_secs must be a finite number >= 1".into()); } + // Neither the recovery tick nor a renewal runs more than once a second. + let d = &self.desktop; + if !d.tick_secs.is_finite() || d.tick_secs < 1.0 { + return Err("desktop.tick_secs must be a finite number >= 1".into()); + } + if !d.renewal_secs.is_finite() || !(d.renewal_secs == 0.0 || d.renewal_secs >= 1.0) { + return Err("desktop.renewal_secs must be 0 or a finite number >= 1".into()); + } let non_negative = [ ("human.typing_per_hour", h.typing_per_hour), ("human.messages_per_hour", h.messages_per_hour), @@ -315,6 +463,11 @@ impl Scenario { let fractions = [ ("human.thread_reply_fraction", h.thread_reply_fraction), ("abort.max_error_rate", self.abort.max_error_rate), + ( + "population.wide_member_fraction", + self.population.wide_member_fraction, + ), + ("desktop.fraction", self.desktop.fraction), ]; for (name, value) in fractions { if !(0.0..=1.0).contains(&value) { @@ -354,18 +507,85 @@ impl Scenario { .collect() } - /// Channels member `member` (global human or agent index) belongs to. - pub fn member_channels(&self, community: usize, member: usize) -> Vec { - let n = self.shape.channels_per_community; - (0..self.population.channels_per_member) - .map(|k| identity::channel_id(&self.run_id, community, (member + k) % n)) + /// Indices, within its community, of the channels member `index` of + /// `role` (a global human or agent index) belongs to: the wide channels + /// if it joins them, then a run of consecutive regular channels starting + /// at its own index. Without wide channels or tiers this is + /// `channels_per_member` channels from `index % channels_per_community`. + pub fn member_channel_indices(&self, role: Role, index: usize) -> Vec { + let wide = self.shape.wide_channels; + let regular = self.shape.channels_per_community - wide; + let mut channels = Vec::new(); + let mut count = self.population.channels_per_member; + if role == Role::Human { + if wide > 0 && draw(index, WIDE_SALT) < self.population.wide_member_fraction { + channels.extend(0..wide); + } + count = self.tier_channels(index); + } + channels.extend((0..count).map(|k| wide + (index + k) % regular)); + channels + } + + /// Regular (not wide) channels human `index` joins: its tier's count, or + /// `channels_per_member` for humans outside every tier. + fn tier_channels(&self, index: usize) -> usize { + let u = draw(index, TIER_SALT); + let mut bound = 0.0; + for tier in &self.population.tiers { + bound += tier.fraction; + if u < bound { + return tier.channels; + } + } + self.population.channels_per_member + } + + /// Channel ids member `index` of `role` belongs to in `community`. + pub fn member_channels(&self, community: usize, role: Role, index: usize) -> Vec { + self.member_channel_indices(role, index) + .into_iter() + .map(|channel| identity::channel_id(&self.run_id, community, channel)) .collect() } + /// Whether human `index` runs the desktop client. + pub fn is_desktop(&self, index: usize) -> bool { + draw(index, DESKTOP_SALT) < self.desktop.fraction + } + /// Largest multiplier any ramp step reaches. pub fn peak_users(&self) -> f64 { self.ramp.iter().map(|s| s.users).fold(0.0, f64::max) } + + /// Clients of each kind at the largest ramp step, rounded as `run` does. + pub fn peak(&self) -> Peak { + let at = |base: usize| (base as f64 * self.peak_users()).ceil() as usize; + Peak { + humans: at(self.population.humans), + agents: at(self.population.agents), + hot_authors: at(self.hot_thread.authors), + } + } + + /// Humans and agents of every channel up to the peak population, indexed + /// `[community][channel]`, humans first and each kind by index. A client + /// with global index `g` lives in community `g % communities`. + pub fn channel_members(&self) -> Members { + let shape = &self.shape; + let mut members = vec![vec![Vec::new(); shape.channels_per_community]; shape.communities]; + let peak = self.peak(); + for (role, count) in [(Role::Human, peak.humans), (Role::Agent, peak.agents)] { + for index in 0..count { + let community = &mut members[index % shape.communities]; + for channel in self.member_channel_indices(role, index) { + community[channel].push(Member { role, index }); + } + } + } + members + } } /// `url` with its host (and port) replaced by `host`. @@ -403,7 +623,7 @@ hold_secs = 10 assert_eq!(s.communities().len(), 1); assert_eq!(s.communities()[0].ws_url, "ws://localhost:3000"); assert_eq!(s.human.presence_interval_secs, 60.0); - assert_eq!(s.member_channels(0, 3).len(), 2); + assert_eq!(s.member_channels(0, Role::Human, 3).len(), 2); } #[test] @@ -519,6 +739,170 @@ hold_secs = 10 assert!(err.contains("port"), "{err}"); } + /// MINIMAL with `n` channels, `shape` and `population` lines added, and + /// `extra` TOML after it. + fn shaped(n: usize, shape: &str, population: &str, extra: &str) -> Result { + let text = MINIMAL + .replace( + "channels_per_community = 4", + &format!("channels_per_community = {n}\n{shape}"), + ) + .replace( + "channels_per_member = 2", + &format!("channels_per_member = 2\n{population}"), + ); + Scenario::parse(&format!("{text}{extra}")) + } + + #[test] + fn without_wide_channels_or_tiers_every_member_keeps_its_run_of_channels() { + let s = Scenario::parse(MINIMAL).unwrap(); + for role in [Role::Human, Role::Agent] { + for m in 0..50 { + let want: Vec = (0..2).map(|k| (m + k) % 4).collect(); + assert_eq!(s.member_channel_indices(role, m), want, "{role:?} {m}"); + } + } + let ids = s.member_channels(0, Role::Human, 3); + assert_eq!(ids, [3, 0].map(|c| identity::channel_id("t", 0, c))); + assert_eq!(s.desktop.fraction, 0.0); + assert!(!(0..1000).any(|m| s.is_desktop(m))); + assert!(!s.shape.private_channels); + } + + #[test] + fn wide_channels_and_tiers_are_deterministic_and_honor_their_fractions() { + let s = shaped( + 12, + "wide_channels = 2", + "wide_member_fraction = 0.5\n[[population.tiers]]\nfraction = 0.1\nchannels = 8\n[[population.tiers]]\nfraction = 0.2\nchannels = 1", + "[desktop]\nfraction = 0.25\n", + ) + .unwrap(); + let again = shaped( + 12, + "wide_channels = 2", + "wide_member_fraction = 0.5\n[[population.tiers]]\nfraction = 0.1\nchannels = 8\n[[population.tiers]]\nfraction = 0.2\nchannels = 1", + "[desktop]\nfraction = 0.25\n", + ) + .unwrap(); + let n = 20_000; + let (mut wide, mut heavy, mut light, mut desktop) = (0, 0, 0, 0); + for m in 0..n { + let channels = s.member_channel_indices(Role::Human, m); + assert_eq!(channels, again.member_channel_indices(Role::Human, m)); + let joins_wide = channels.starts_with(&[0, 1]); + let regular: Vec = channels.iter().copied().filter(|c| *c >= 2).collect(); + assert_eq!(regular.len() + 2 * usize::from(joins_wide), channels.len()); + // Regular channels are a run of distinct channels from the member's index. + let want: Vec = (0..regular.len()).map(|k| 2 + (m + k) % 10).collect(); + assert_eq!(regular, want, "member {m}"); + wide += usize::from(joins_wide); + heavy += usize::from(regular.len() == 8); + light += usize::from(regular.len() == 1); + desktop += usize::from(s.is_desktop(m)); + // Agents never join wide channels and keep channels_per_member. + let agent = s.member_channel_indices(Role::Agent, m); + assert_eq!(agent, [2 + m % 10, 2 + (m + 1) % 10]); + } + let share = |count: usize| count as f64 / n as f64; + assert!((share(wide) - 0.5).abs() < 0.02, "wide {wide}"); + assert!((share(heavy) - 0.1).abs() < 0.02, "heavy {heavy}"); + assert!((share(light) - 0.2).abs() < 0.02, "light {light}"); + assert!((share(desktop) - 0.25).abs() < 0.02, "desktop {desktop}"); + } + + #[test] + fn channel_members_inverts_the_assignment_up_to_the_peak() { + let s = shaped( + 6, + "wide_channels = 1", + "wide_member_fraction = 0.3\n[[population.tiers]]\nfraction = 0.5\nchannels = 4", + "[[ramp]]\nusers = 2.5\nhold_secs = 10\n", + ) + .unwrap(); + let peak = s.peak(); + assert_eq!((peak.humans, peak.agents), (25, 5)); + let members = s.channel_members(); + assert_eq!(members.len(), 1); + let mut listed = 0; + for (role, count) in [(Role::Human, 25), (Role::Agent, 5)] { + for index in 0..count { + for c in s.member_channel_indices(role, index) { + assert!(members[0][c].contains(&Member { role, index })); + listed += 1; + } + } + } + assert_eq!(listed, members[0].iter().map(Vec::len).sum::()); + // Humans come first in every channel. + for channel in &members[0] { + let first_agent = channel.iter().position(|m| m.role == Role::Agent); + let last_human = channel.iter().rposition(|m| m.role == Role::Human); + if let (Some(a), Some(h)) = (first_agent, last_human) { + assert!(h < a); + } + } + } + + #[test] + fn shape_and_desktop_knobs_are_validated() { + let rejected = [ + ("wide_channels = 4", "", ""), + // Two members' channels no longer fit outside three wide channels. + ("wide_channels = 3", "", ""), + ("", "wide_member_fraction = 1.5", ""), + ("", "wide_member_fraction = nan", ""), + ("", "[[population.tiers]]\nfraction = 0.0\nchannels = 1", ""), + ("", "[[population.tiers]]\nfraction = 1.5\nchannels = 1", ""), + ("", "[[population.tiers]]\nfraction = 0.6\nchannels = 1\n[[population.tiers]]\nfraction = 0.6\nchannels = 2", ""), + ("", "[[population.tiers]]\nfraction = 0.5\nchannels = 0", ""), + ("", "[[population.tiers]]\nfraction = 0.5\nchannels = 5", ""), + ("wide_channels = 1", "[[population.tiers]]\nfraction = 0.5\nchannels = 4", ""), + ("", "[[population.tiers]]\nfraction = 0.5\nchannels = 1\nextra = 1", ""), + ("", "", "[desktop]\nfraction = 1.5\n"), + ("", "", "[desktop]\nfraction = -0.1\n"), + ("", "", "[desktop]\nmode = \"bogus\"\n"), + ("", "", "[desktop]\ntick_secs = 0.5\n"), + ("", "", "[desktop]\ntick_secs = inf\n"), + ("", "", "[desktop]\nrenewal_secs = 0.5\n"), + ("", "", "[desktop]\nrenewal_secs = -1.0\n"), + ("", "", "[desktop]\nbackstop_secs = 0.0\n"), + ("", "", "[desktop]\nwindow = 1\n"), + ]; + for (shape, population, extra) in rejected { + assert!( + shaped(4, shape, population, extra).is_err(), + "accepted: {shape} {population} {extra}" + ); + } + let accepted = [ + ("wide_channels = 2", "wide_member_fraction = 1.0", ""), + ("private_channels = true", "", ""), + ( + "", + "[[population.tiers]]\nfraction = 0.5\nchannels = 4\n[[population.tiers]]\nfraction = 0.5\nchannels = 1", + "", + ), + ( + "", + "", + "[desktop]\nfraction = 1.0\nmode = \"legacy\"\nrenewal_secs = 1.0\ntick_secs = 1.0\nwindows = 0\nthreads = 0\n", + ), + ]; + for (shape, population, extra) in accepted { + let s = shaped(4, shape, population, extra) + .unwrap_or_else(|e| panic!("rejected {shape} {population} {extra}: {e}")); + assert!(!s.member_channel_indices(Role::Human, 0).is_empty()); + } + let legacy = shaped(4, "", "", "[desktop]\nmode = \"legacy\"\n").unwrap(); + assert_eq!(legacy.desktop.mode, DesktopMode::Legacy); + assert_eq!( + Scenario::parse(MINIMAL).unwrap().desktop.mode, + DesktopMode::Current + ); + } + #[test] fn thread_depth_is_limited_to_what_the_relay_accepts() { let with_depth = |depth: usize| { diff --git a/crates/buzz-test-client/src/loadgen/seed.rs b/crates/buzz-test-client/src/loadgen/seed.rs index 61379bc27d1..9742d001fde 100644 --- a/crates/buzz-test-client/src/loadgen/seed.rs +++ b/crates/buzz-test-client/src/loadgen/seed.rs @@ -5,13 +5,13 @@ use std::sync::Arc; use std::time::Instant; use futures_util::stream::{self, StreamExt}; -use nostr::{Event, EventBuilder, Keys, Kind, Tag}; +use nostr::{Event, EventBuilder, JsonUtil, Keys, Kind, Tag}; use reqwest::StatusCode; use serde_json::json; use super::identity::{self, Role}; -use super::net::{http_reason, Net}; -use super::scenario::{Community, Scenario}; +use super::net::{http_reason, truncate, Net}; +use super::scenario::{Community, Member, Members, Scenario}; use super::stats::{Op, Snapshot, Stats}; /// Content prefix of seeded thread roots; `run` finds roots by it. @@ -24,17 +24,31 @@ pub fn hot_root_keys(run_id: &str) -> Keys { identity::keys(run_id, Role::HotRoot, 0) } +/// Most people one kind:9000 may name, the relay's limit. +const MAX_ADDS_PER_EVENT: usize = 100; + /// Seeds every community. Idempotent for communities this run's fixture -/// owner owns, and for channels; history and threads are appended on every -/// call. Fails unless every planned post -/// was accepted. +/// owner owns, for channels and for memberships; history and threads are +/// appended on every call. Fails unless every planned post was accepted and, +/// with private channels, every member was added. pub async fn seed(scenario: &Scenario, net: &Net, stats: &Arc) -> anyhow::Result<()> { + let members = scenario + .shape + .private_channels + .then(|| scenario.channel_members()); for community in scenario.communities() { let owner = identity::keys(&scenario.run_id, Role::Owner, community.index); if let Some(host) = &community.host { provision(scenario, net, host, &owner).await?; } create_channels(scenario, net, &community, &owner).await?; + if let Some(members) = &members { + let added = add_members(scenario, net, &community, &owner, members).await?; + eprintln!( + "seed: community {} memberships added ({added} people)", + community.index + ); + } eprintln!( "seed: community {} ready ({} channels)", community.index, scenario.shape.channels_per_community @@ -46,8 +60,18 @@ pub async fn seed(scenario: &Scenario, net: &Net, stats: &Arc) -> anyhow: let before = stats.snapshot(); let (roots, rest) = plan_posts(scenario); let planned: usize = roots.iter().chain(&rest).map(Vec::len).sum(); - let root_ids = post_all(scenario, net, stats, &communities, roots, &RootIds::new()).await; - post_all(scenario, net, stats, &communities, rest, &root_ids).await; + let members = members.as_ref(); + let root_ids = post_all( + scenario, + net, + stats, + &communities, + members, + roots, + &RootIds::new(), + ) + .await; + post_all(scenario, net, stats, &communities, members, rest, &root_ids).await; let accepted = seeded(&stats.snapshot(), &before); eprintln!( "seed: {accepted} of {planned} posts accepted in {:.1}s", @@ -82,6 +106,7 @@ async fn post_all( net: &Net, stats: &Stats, communities: &[Community], + members: Option<&Members>, chains: Vec, roots: &RootIds, ) -> RootIds { @@ -91,7 +116,8 @@ async fn post_all( let mut previous: Option = None; for post in chain { let community = &communities[post.community]; - let Some((keys, event)) = post.build(scenario, roots, previous.as_deref()) else { + let Some((keys, event)) = post.build(scenario, members, roots, previous.as_deref()) + else { stats.err(Op::Seed, "thread parent missing"); previous = None; continue; @@ -193,6 +219,11 @@ async fn create_channels( community: &Community, owner: &Keys, ) -> anyhow::Result<()> { + let visibility = if scenario.shape.private_channels { + "private" + } else { + "open" + }; let results: Vec> = stream::iter(scenario.channel_ids(community.index)) .enumerate() .map(|(i, channel)| async move { @@ -201,7 +232,7 @@ async fn create_channels( tag(["h", &channel]), tag(["name", &format!("lg-{}-{i}", scenario.run_id)]), tag(["channel_type", "stream"]), - tag(["visibility", "open"]), + tag(["visibility", visibility]), ]) .sign_with_keys(owner)?; match net.submit(owner, community, &event, None).await { @@ -215,6 +246,100 @@ async fn create_channels( results.into_iter().collect() } +/// Adds every member of `community`'s channels with kind:9000 events from +/// the channel owner, at most [`MAX_ADDS_PER_EVENT`] people each and +/// `seed_concurrency` at a time. Channel 0 of community 0 also gets the +/// hot-thread root author and authors. Adding a current member at its role +/// changes nothing, so re-running is safe. Returns how many people were +/// named; fails if any event, or any person in one, was refused. +async fn add_members( + scenario: &Scenario, + net: &Net, + community: &Community, + owner: &Keys, + members: &Members, +) -> anyhow::Result { + let run_id = &scenario.run_id; + let mut adds = Vec::new(); + for (channel, list) in members[community.index].iter().enumerate() { + let mut people: Vec = list + .iter() + .map(|m| identity::keys(run_id, m.role, m.index)) + .collect(); + if community.index == 0 && channel == 0 && scenario.hot_thread.enabled() { + people.push(hot_root_keys(run_id)); + people.extend( + (0..scenario.peak().hot_authors) + .map(|i| identity::keys(run_id, Role::HotAuthor, i)), + ); + } + let id = identity::channel_id(run_id, community.index, channel); + for chunk in people.chunks(MAX_ADDS_PER_EVENT) { + adds.push(( + id.clone(), + chunk.iter().map(|k| k.public_key().to_hex()).collect(), + )); + } + } + let named: usize = adds + .iter() + .map(|(_, p): &(String, Vec)| p.len()) + .sum(); + let failures: Vec = stream::iter(adds) + .map(|(channel, people)| async move { + put_users(net, community, owner, &channel, &people) + .await + .err() + .map(|e| format!("{channel}: {e}")) + }) + .buffer_unordered(scenario.shape.seed_concurrency) + .filter_map(|failure| async move { failure }) + .collect() + .await; + if let Some(first) = failures.first() { + anyhow::bail!( + "{} membership events failed in community {}; first: {first}", + failures.len(), + community.index + ); + } + Ok(named) +} + +/// Submits one kind:9000 naming `people` in `channel`. The relay accepts an +/// event that adds some people and refuses others, listing the refused ones +/// under `rejected` in its message, so that counts as a failure too. +async fn put_users( + net: &Net, + community: &Community, + owner: &Keys, + channel: &str, + people: &[String], +) -> anyhow::Result<()> { + let tags = std::iter::once(tag(["h", channel])).chain(people.iter().map(|p| tag(["p", p]))); + let event = EventBuilder::new(Kind::Custom(9000), "") + .tags(tags) + .sign_with_keys(owner)?; + let (status, body) = net + .post( + owner, + &community.http_url, + "/events", + event.as_json().into_bytes(), + None, + ) + .await?; + if !status.is_success() { + anyhow::bail!("{}", http_reason(status, &body)); + } + let reply: serde_json::Value = serde_json::from_str(&body)?; + let message = reply["message"].as_str().unwrap_or_default(); + if reply["accepted"].as_bool() != Some(true) || message.contains("\"rejected\"") { + anyhow::bail!("rejected: {}", truncate(message)); + } + Ok(()) +} + /// One seeded post: history message, thread root, or thread reply. struct Post { community: usize, @@ -299,20 +424,16 @@ fn plan_posts(scenario: &Scenario) -> (Vec, Vec) { impl Post { /// Builds the event. A reply attaches to `previous`, the accepted reply /// before it in its thread; the first reply attaches to the thread root. + /// With `members` (private channels), authors are the channel's humans. fn build( &self, scenario: &Scenario, + members: Option<&Members>, roots: &RootIds, previous: Option<&str>, ) -> Option<(Keys, Event)> { let channel = identity::channel_id(&scenario.run_id, self.community, self.channel); - // Rotate authors across humans to stay under per-pubkey rate limits. - let humans = scenario.population.humans.max(1); - let author = identity::keys( - &scenario.run_id, - Role::Human, - (self.seq * scenario.shape.communities + self.community) % humans, - ); + let author = self.author(scenario, members); let mut tags = vec![tag(["h", &channel])]; let (keys, content) = match self.kind { PostKind::History => (author, format!("lg-history {}", self.seq)), @@ -396,6 +517,28 @@ pub async fn find_roots( Ok(roots.into_iter().map(|(_, id)| id).collect()) } +impl Post { + /// Rotates authors across humans to stay under per-pubkey rate limits: + /// across all humans for open channels, across the channel's own human + /// members for private ones, falling back to the owner when it has none. + fn author(&self, scenario: &Scenario, members: Option<&Members>) -> Keys { + let run_id = &scenario.run_id; + let Some(members) = members else { + let humans = scenario.population.humans.max(1); + let index = (self.seq * scenario.shape.communities + self.community) % humans; + return identity::keys(run_id, Role::Human, index); + }; + let humans: Vec<&Member> = members[self.community][self.channel] + .iter() + .filter(|m| m.role == Role::Human) + .collect(); + match humans.get(self.seq % humans.len().max(1)) { + Some(m) => identity::keys(run_id, Role::Human, m.index), + None => identity::keys(run_id, Role::Owner, self.community), + } + } +} + fn tag(parts: [&str; N]) -> Tag { Tag::parse(parts).expect("static tag shape") } @@ -604,6 +747,161 @@ hold_secs = 1 assert!(err.to_string().contains("seeded 10 of 15"), "{err}"); } + /// Seeds `scenario` against a relay that accepts everything, answering + /// kind:9000 with `put_user_message`. Returns the outcome and every + /// event submitted, in order. + async fn seed_logged( + scenario: &str, + put_user_message: &'static str, + ) -> (anyhow::Result<()>, Scenario, Vec) { + let log: Arc>> = Default::default(); + let logged = log.clone(); + let addr = fake_http(move |_, _, body| { + let event: Event = serde_json::from_slice(body).unwrap(); + let message = if event.kind == Kind::Custom(9000) { + put_user_message + } else { + "" + }; + logged.lock().unwrap().push(event); + ( + 200, + json!({"accepted": true, "message": message}).to_string(), + ) + }) + .await; + let scenario = Scenario::parse(&scenario.replace("{addr}", &addr)).unwrap(); + let net = Net::new(&scenario).await.unwrap(); + let stats = Arc::new(Stats::default()); + let outcome = seed(&scenario, &net, &stats).await; + let events = log.lock().unwrap().clone(); + (outcome, scenario, events) + } + + fn values(event: &Event, name: &str) -> Vec { + event + .tags + .iter() + .filter(|t| t.kind().as_str() == name) + .filter_map(|t| t.content().map(str::to_string)) + .collect() + } + + const PRIVATE: &str = r#" +run_id = "t" +[target] +ws_url = "ws://{addr}" +http_url = "http://{addr}" +[shape] +communities = 1 +channels_per_community = 3 +history_messages_per_channel = 3 +private_channels = true +wide_channels = 1 +[population] +humans = 150 +agents = 2 +channels_per_member = 1 +wide_member_fraction = 1.0 +[hot_thread] +seed_replies = 2 +authors = 2 +[[ramp]] +users = 1.0 +hold_secs = 1 +[[ramp]] +users = 1.5 +hold_secs = 1 +"#; + + #[tokio::test] + async fn private_channels_add_every_member_before_anything_is_posted() { + let (outcome, scenario, events) = seed_logged(PRIVATE, "").await; + outcome.expect("seed passes"); + let kind = |e: &Event| e.kind.as_u16(); + let creates: Vec<&Event> = events.iter().filter(|e| kind(e) == 9007).collect(); + assert_eq!(creates.len(), 3); + assert!(creates + .iter() + .all(|e| values(e, "visibility") == ["private"])); + let first_put = events.iter().position(|e| kind(e) == 9000).unwrap(); + let last_put = events.iter().rposition(|e| kind(e) == 9000).unwrap(); + let first_post = events.iter().position(|e| kind(e) == 9).unwrap(); + let last_create = events.iter().rposition(|e| kind(e) == 9007).unwrap(); + assert!(last_create < first_put && last_put < first_post); + + let members = scenario.channel_members(); + let owner = identity::keys("t", Role::Owner, 0); + let mut added: HashMap> = HashMap::new(); + for put in events.iter().filter(|e| kind(e) == 9000) { + assert_eq!(put.pubkey, owner.public_key()); + let people = values(put, "p"); + assert!(!people.is_empty() && people.len() <= MAX_ADDS_PER_EVENT); + let [channel] = values(put, "h").try_into().unwrap(); + added.entry(channel).or_default().extend(people); + } + // Every human up to the peak (225) joins the wide channel 0, and so do + // the hot thread's root author and its 3 peak authors. + for (c, list) in members[0].iter().enumerate() { + let mut want: Vec = list + .iter() + .map(|m| identity::keys("t", m.role, m.index).public_key().to_hex()) + .collect(); + if c == 0 { + want.push(hot_root_keys("t").public_key().to_hex()); + want.extend((0..3).map(|i| { + identity::keys("t", Role::HotAuthor, i) + .public_key() + .to_hex() + })); + assert_eq!(want.len(), 225 + 4); + } + let mut got = added.remove(&identity::channel_id("t", 0, c)).unwrap(); + got.sort(); + want.sort(); + assert_eq!(got, want, "channel {c}"); + } + assert!(added.is_empty()); + + // Every post is authored by a human member of its channel. + for post in events.iter().filter(|e| kind(e) == 9) { + let [channel] = values(post, "h").try_into().unwrap(); + let c = (0..3) + .find(|c| identity::channel_id("t", 0, *c) == channel) + .unwrap(); + let author = post.pubkey.to_hex(); + let member = members[0][c].iter().any(|m| { + m.role == Role::Human + && identity::keys("t", Role::Human, m.index) + .public_key() + .to_hex() + == author + }); + let hot_root = author == hot_root_keys("t").public_key().to_hex(); + assert!(member || hot_root, "{author} posted in channel {c}"); + } + } + + #[tokio::test] + async fn refused_members_fail_the_seed_and_open_channels_add_nobody() { + // The relay accepts an event that refuses some of the people it names. + let refused = + r#"members: {"added":["aa"],"rejected":[{"reason":"blocked","pubkeys":["bb"]}]}"#; + let (outcome, _, _) = seed_logged(PRIVATE, refused).await; + let err = outcome.expect_err("a refused member fails the seed"); + assert!(err.to_string().contains("membership"), "{err}"); + // Re-adding current members changes nothing and passes. + let (outcome, _, _) = seed_logged(PRIVATE, r#"members: {"already_member":["aa"]}"#).await; + outcome.expect("re-running the seed passes"); + + let open = PRIVATE.replace("private_channels = true", ""); + let (outcome, _, events) = seed_logged(&open, "").await; + outcome.expect("seed passes"); + assert!(!events.iter().any(|e| e.kind.as_u16() == 9000)); + let creates: Vec<&Event> = events.iter().filter(|e| e.kind.as_u16() == 9007).collect(); + assert!(creates.iter().all(|e| values(e, "visibility") == ["open"])); + } + #[test] fn reply_k_points_reply_at_reply_k_minus_one_and_root_at_the_root() { let scenario = Scenario::parse( @@ -636,7 +934,7 @@ hold_secs = 1 let mut previous: Option = None; for (k, post) in rest[0].iter().enumerate() { let (_, event) = post - .build(&scenario, &known, previous.as_deref()) + .build(&scenario, None, &known, previous.as_deref()) .expect("reply builds once its parent is known"); let markers = buzz_core::nip10::parse_thread_markers(&event.tags); assert_eq!(markers.root.as_deref(), Some(root_id.as_str()), "reply {k}"); @@ -645,7 +943,7 @@ hold_secs = 1 previous = Some(event.id.to_hex()); } // A reply cannot be placed without its predecessor. - assert!(rest[0][2].build(&scenario, &known, None).is_none()); + assert!(rest[0][2].build(&scenario, None, &known, None).is_none()); } /// An operator endpoint that lists hosts by owner, as `GET diff --git a/crates/buzz-test-client/src/loadgen/stats.rs b/crates/buzz-test-client/src/loadgen/stats.rs index 62e3f43234a..aacf4928b96 100644 --- a/crates/buzz-test-client/src/loadgen/stats.rs +++ b/crates/buzz-test-client/src/loadgen/stats.rs @@ -21,10 +21,30 @@ pub enum Op { Subscribe, HotReply, ThreadFetch, + /// Desktop client: NIP-11 `GET /` on connect. + DesktopNip11, + /// Desktop client: EOSE of a live subscription (global or channel batch). + DesktopLive, + /// Desktop client: EOSE of a closing channel replay subscription. + DesktopReplay, + /// Desktop client HTTP `/query` reads, one per request type. + DesktopRoster, + DesktopMetadata, + DesktopSidebar, + DesktopProfiles, + DesktopStatuses, + DesktopEmoji, + DesktopReadState, + DesktopUnread, + DesktopReminders, + DesktopWindow, + DesktopNotices, + DesktopThreadRoot, + DesktopThreadWindow, } impl Op { - pub const ALL: [Op; 11] = [ + pub const ALL: [Op; 27] = [ Op::WsConnect, Op::History, Op::PublishEphemeral, @@ -36,6 +56,22 @@ impl Op { Op::Subscribe, Op::HotReply, Op::ThreadFetch, + Op::DesktopNip11, + Op::DesktopLive, + Op::DesktopReplay, + Op::DesktopRoster, + Op::DesktopMetadata, + Op::DesktopSidebar, + Op::DesktopProfiles, + Op::DesktopStatuses, + Op::DesktopEmoji, + Op::DesktopReadState, + Op::DesktopUnread, + Op::DesktopReminders, + Op::DesktopWindow, + Op::DesktopNotices, + Op::DesktopThreadRoot, + Op::DesktopThreadWindow, ]; pub fn name(self) -> &'static str { @@ -51,6 +87,22 @@ impl Op { Op::Subscribe => "subscribe_eose", Op::HotReply => "hot_reply", Op::ThreadFetch => "thread_fetch", + Op::DesktopNip11 => "desktop_nip11", + Op::DesktopLive => "desktop_live_eose", + Op::DesktopReplay => "desktop_replay_eose", + Op::DesktopRoster => "desktop_roster", + Op::DesktopMetadata => "desktop_metadata", + Op::DesktopSidebar => "desktop_sidebar", + Op::DesktopProfiles => "desktop_profiles", + Op::DesktopStatuses => "desktop_statuses", + Op::DesktopEmoji => "desktop_emoji", + Op::DesktopReadState => "desktop_read_state", + Op::DesktopUnread => "desktop_unread", + Op::DesktopReminders => "desktop_reminders", + Op::DesktopWindow => "desktop_window", + Op::DesktopNotices => "desktop_notices", + Op::DesktopThreadRoot => "desktop_thread_root", + Op::DesktopThreadWindow => "desktop_thread_window", } } } diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index b6413326aa7..f46482b25ea 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -23,6 +23,10 @@ cargo run --release -p buzz-test-client --bin buzz-loadgen -- teardown perf/scen | Humans / agents | `population.humans`, `population.agents`, multiplied by each `[[ramp]]` step's `users` | | Throughput per client | `[human]` and `[agent]` rates | | Reconnect churn | `human.mean_session_secs`, `population.connect_rate_per_sec` | +| Private membership | `shape.private_channels` | +| Large channels | `shape.wide_channels`, `population.wide_member_fraction` | +| Uneven membership | `population.tiers` | +| Desktop clients | `[desktop]`: `fraction`, `mode`, `windows`, `threads`, `tick_secs`, `backstop_secs`, `renewal_secs` | `shape.thread_depth` seeds each thread as a chain: reply `k` replies to reply `k-1`, so the thread's deepest reply sits at depth `thread_depth`. The relay @@ -42,6 +46,20 @@ agents) reply to that root and read its newest 100 replies over HTTP. Their replies are timed, so humans subscribed to that channel record delivery. Leave `seed_replies` and `authors` at 0 to disable it. +Members are assigned deterministically by global index, so every shard and +`seed` agree on them without coordination: + +- `shape.wide_channels` reserves the first K channels of each community as + wide channels. Each human joins them with probability + `population.wide_member_fraction`, on top of its own channels. Agents never + do. +- `population.tiers` is a list of `{ fraction, channels }`. That fraction of + humans joins `channels` channels instead of `channels_per_member`. Fractions + sum to at most 1; the rest keep `channels_per_member`. +- A member's own channels are a run of consecutive channels after the wide + ones, starting at its index. With no wide channels and no tiers this is the + assignment of earlier versions. + ## What each client does - **Human**: authenticates over WebSocket and subscribes to its channels' @@ -52,6 +70,10 @@ Leave `seed_replies` and `authors` at 0 to disable it. - **Agent**: authenticates with a NIP-OA tag from its owner, polls HTTP `/query` (channel window plus mentions), posts messages through HTTP `/events`, and sends encrypted kind:24200 observer frames to its owner. +- **Desktop human**: the `desktop.fraction` of humans that model the desktop + app instead of the basic human above. They keep the basic human's presence, + typing, messages and reactions; the subscriptions and HTTP reads are the + app's, described under [Desktop clients](#desktop-clients). - **Hot-thread author**: finds the seeded hot root, then posts replies to it through HTTP `/events` (`hot_reply`) and reads the thread through HTTP `/query` (`thread_fetch`). Until the root is found every attempt counts as @@ -80,8 +102,62 @@ relay checks against its own relay URL. Without a host template, the generator connects to `ws_url` and `http_url` as written and `connect_addr` has no effect. -Seeded identities do not join communities or channels. A relay that requires -membership rejects both `seed` and `run`. +By default channels are open and seeded identities join nothing, so a relay +that requires membership rejects both `seed` and `run`. With +`shape.private_channels = true`, `seed` creates the channels private and adds +every member: the humans and agents assigned to each channel up to the +largest ramp step, and the hot-thread root and authors in its channel. It +sends kind:9000 events of up to 100 members, `seed_concurrency` at a time, +signed by the community owner, before posting any history. A rerun adds +nobody new, and any member the relay refuses fails the seed. Seeded history +in a private channel is posted by its own members. + +## Desktop clients + +A desktop client models the subscriptions and reads the desktop app makes, +in two modes: `legacy`, the app before its client volume fixes, and +`current` (default), after them. Each request type is its own operation in +the report. + +On a **cold start** it: + +1. reads the relay's NIP-11 document (`desktop_nip11`); +2. opens its profile and membership subscriptions, and its channel + subscriptions (`desktop_live_eose`). With up to ten channels both modes + open one per-channel batch that replays the newest 500 events. With more, + `legacy` opens one such batch per ten channels; `current` opens one + live-only subscription per 2048 channels and replays the channels ten at a + time in separate subscriptions it closes at EOSE (`desktop_replay_eose`). + At most four are set up at once, each with a 10 s EOSE timeout; +3. reads over HTTP, at most six at a time with a 30 s timeout: its roster + (`desktop_roster`, paged), channel names (`desktop_metadata`), sidebar + preferences (`desktop_sidebar`), co-member profiles (`desktop_profiles`) + and statuses (`desktop_statuses`), the custom emoji set + (`desktop_emoji`), read state (`desktop_read_state`), unread repair in + chunks of 128 channels (`desktop_unread`), reminders + (`desktop_reminders`), and for each of `windows` open channels a window + catch-up (`desktop_window`, up to five older pages while pages come back + full) and its notices (`desktop_notices`), and for each of `threads` open + threads a seeded root lookup and its first thread page + (`desktop_thread_root`, `desktop_thread_window`). + +Every `tick_secs` the client runs a recovery tick. In `legacy` each tick +replaces its channel subscriptions and rereads roster, read state and unread. +In `current` each of the three waits `backstop_secs` since it last ran, and +a replacement is live-only. A replacement is opened before the subscription +it replaces is closed. + +Every `renewal_secs` (0 disables it; the first is jittered) the client renews +its identity and reconnects. In `legacy` that is an ordinary reconnect, which +rereads everything but channel names and sidebar preferences. In `current` +with up to ten channels the subscriptions are re-sent and nothing is read; +with more, the replays, unread repair, windows and threads run again. A +human's session end (`human.mean_session_secs`) is an ordinary reconnect in +both modes. Resumed subscriptions start 960 s before the previous session +ended, but not before 300 s before the client started. + +Reads that come due while earlier ones are still running wait for them, once +each, so a slow relay does not pile reads up behind it. ## Sharding diff --git a/perf/scenarios/local-desktop.toml b/perf/scenarios/local-desktop.toml new file mode 100644 index 00000000000..accea580b04 --- /dev/null +++ b/perf/scenarios/local-desktop.toml @@ -0,0 +1,51 @@ +# Short run of desktop clients in private channels against a local relay +# (`just relay`). Ticks, backstops and renewals are seconds apart instead of +# minutes, so every desktop request type shows up within the ramp. Set +# `desktop.mode = "legacy"` to compare the client before its volume fixes. +# cargo run -p buzz-test-client --bin buzz-loadgen -- seed perf/scenarios/local-desktop.toml +# cargo run -p buzz-test-client --bin buzz-loadgen -- run perf/scenarios/local-desktop.toml +run_id = "local-desktop" + +[target] +ws_url = "ws://localhost:3000" +http_url = "http://localhost:3000" + +[shape] +communities = 1 +channels_per_community = 30 +history_messages_per_channel = 20 +thread_roots_per_channel = 1 +thread_depth = 5 +# Members are added to each channel with kind:9000; the first two channels +# are joined by half of the humans on top of their own. +private_channels = true +wide_channels = 2 + +[population] +humans = 40 +agents = 4 +channels_per_member = 5 +connect_rate_per_sec = 20.0 +wide_member_fraction = 0.5 +# A quarter of the humans are in 15 channels instead of 5, so their +# subscriptions batch past ten channels. +tiers = [{ fraction = 0.25, channels = 15 }] + +[human] +presence_interval_secs = 10.0 +typing_per_hour = 120.0 +messages_per_hour = 60.0 +reactions_per_hour = 30.0 + +[desktop] +fraction = 0.5 +mode = "current" +windows = 3 +threads = 1 +tick_secs = 5.0 +backstop_secs = 15.0 +renewal_secs = 20.0 + +[[ramp]] +users = 1.0 +hold_secs = 60 From 5b92db99f38cf20ec1dba48a1e9cba0cf7f680ec Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Tue, 3 Nov 2026 12:20:16 -0500 Subject: [PATCH 2/4] feat(loadgen): split wide-channel activity; page and read state like the relay - population.wide_activity_share: optional share of a wide-channel member's typing, messages and reactions that goes to the wide channels; unset keeps the uniform pick and its exact draws. - Desktop older pages send until with before_id, which top-level window pages require, and the read-state snapshot's object response decodes. Both failed against a local relay; the test fakes now enforce the same contract. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Luke Tornquist --- .../buzz-test-client/src/loadgen/desktop.rs | 71 ++++++-- crates/buzz-test-client/src/loadgen/run.rs | 163 +++++++++++++++++- .../buzz-test-client/src/loadgen/scenario.rs | 75 +++++++- perf/LOADGEN.md | 9 +- perf/scenarios/local-desktop.toml | 2 + 5 files changed, 298 insertions(+), 22 deletions(-) diff --git a/crates/buzz-test-client/src/loadgen/desktop.rs b/crates/buzz-test-client/src/loadgen/desktop.rs index af4b3bacbcf..5c335c23466 100644 --- a/crates/buzz-test-client/src/loadgen/desktop.rs +++ b/crates/buzz-test-client/src/loadgen/desktop.rs @@ -23,7 +23,7 @@ use super::run::{ connect, observe, roots, stopped, Activity, Backoff, Directory, Rng, Seen, Shared, Step, MAX_IDLE, }; -use super::scenario::DesktopMode; +use super::scenario::{DesktopMode, WideSplit}; use super::stats::{Op, Pending}; /// Kinds the app subscribes to for every joined channel. @@ -398,7 +398,7 @@ impl Reader { Ok(Ok((status, text))) if !status.is_success() => { pending.err(&http_reason(status, &text)) } - Ok(Ok((_, text))) => match serde_json::from_str::>(&text) { + Ok(Ok((_, text))) => match events(&text) { Ok(events) => { pending.ok(); return Some(events); @@ -420,7 +420,7 @@ impl Reader { break; }; match oldest(&page) { - Some(until) if page.len() >= PAGE => filter["until"] = json!(until), + Some(cursor) if page.len() >= PAGE => page_before(&mut filter, cursor), _ => break, } } @@ -533,7 +533,7 @@ impl Reader { break; }; match oldest(&page) { - Some(until) if page.len() >= WINDOW_PAGE => filter["until"] = json!(until), + Some(cursor) if page.len() >= WINDOW_PAGE => page_before(&mut filter, cursor), _ => break, } } @@ -574,9 +574,29 @@ fn lock(mutex: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// `created_at` of the oldest event in a page. -fn oldest(page: &[Value]) -> Option { - page.iter().filter_map(|e| e["created_at"].as_u64()).min() +/// `(created_at, id)` of the oldest event in a page. +fn oldest(page: &[Value]) -> Option<(u64, &str)> { + page.iter() + .filter_map(|e| Some((e["created_at"].as_u64()?, e["id"].as_str()?))) + .min() +} + +/// Points `filter` at the page before `cursor`, the oldest event so far. The +/// relay takes `until` and `before_id` together. +fn page_before(filter: &mut Value, (until, id): (u64, &str)) { + filter["until"] = json!(until); + filter["before_id"] = json!(id); +} + +/// The events of a `/query` response: an array, or the object a read-state +/// snapshot answers with. +fn events(text: &str) -> serde_json::Result> { + match serde_json::from_str(text)? { + Value::Object(mut body) => { + serde_json::from_value(body.remove("events").unwrap_or_default()) + } + other => serde_json::from_value(other), + } } /// Runs `fetches`: NIP-11 first, since notices need the relay's pubkey, then @@ -689,6 +709,7 @@ pub(super) async fn desktop(shared: Arc, global: usize) { generation: 0, first_renewal: true, last_message: None, + split: scenario.wide_split(Role::Human, global), }; let started = unix_now(); let mut cursor = None; @@ -729,6 +750,7 @@ struct Client { generation: u64, first_renewal: bool, last_message: Option, + split: Option, } impl Client { @@ -766,7 +788,7 @@ impl Client { let mut open: HashSet = HashSet::new(); // New batch id to the batch it replaces, closed at the new one's EOSE. let mut replacing: HashMap = HashMap::new(); - let mut activity = Activity::new(&scenario.human, &mut self.rng); + let mut activity = Activity::new(&scenario.human, &mut self.rng, self.split); let tick_every = Duration::from_secs_f64(profile.tick_secs); let backstop = Duration::from_secs_f64(profile.backstop_secs); let mut next_tick = Instant::now() + tick_every; @@ -1156,13 +1178,27 @@ mod tests { } /// A loopback HTTP API: NIP-11 names a relay key, the owner's root search - /// finds one seeded root, and every other query is empty. + /// finds one seeded root, a window head page is full and the page before + /// it empty, the read-state snapshot answers in its own shape, and every + /// other query is empty. Like the relay, it refuses a top-level page + /// cursor without both `until` and `before_id`. async fn fake_api(run_id: &str) -> String { let owner = identity::keys(run_id, Role::Owner, 0); let root = EventBuilder::new(kind(9), "lg-root 0") .sign_with_keys(&owner) .unwrap(); let root = serde_json::to_string(&[root]).unwrap(); + let head: Vec<_> = (0..WINDOW_PAGE as u64) + .map(|i| { + EventBuilder::new(kind(9), "w") + .custom_created_at(nostr::Timestamp::from(1_000 + i)) + .sign_with_keys(&owner) + .unwrap() + }) + .collect(); + let head = serde_json::to_string(&head).unwrap(); + let snapshot = + json!({"read_state_snapshot": 1, "complete": true, "events": []}).to_string(); let info = json!({"self": Keys::generate().public_key().to_hex()}).to_string(); let owner = owner.public_key().to_hex(); fake_http(move |method, target, body| { @@ -1171,6 +1207,16 @@ mod tests { } let filters: Value = serde_json::from_slice(body).unwrap_or_default(); let filter = &filters[0]; + if filter.get("read_state_snapshot").is_some() { + return (200, snapshot.clone()); + } + if filter.get("top_level").is_some() { + return match (filter.get("until"), filter.get("before_id")) { + (None, None) => (200, head.clone()), + (Some(_), Some(_)) => (200, "[]".into()), + _ => (400, r#"{"error":"top_level cursor requires both"}"#.into()), + }; + } if filter["kinds"] == json!([9]) && filter["authors"] == json!([owner]) { (200, root.clone()) } else { @@ -1312,7 +1358,8 @@ hold_secs = 1 ("desktop_read_state", 1), ("desktop_unread", 1), ("desktop_reminders", 1), - ("desktop_window", 2), + // Per window, a full head page and the empty page before it. + ("desktop_window", 4), ("desktop_notices", 2), ("desktop_thread_root", 1), ("desktop_thread_window", 1), @@ -1373,7 +1420,7 @@ hold_secs = 1 assert!(out.count("desktop_unread") >= 4); // Ticks never reread names or open windows. assert_eq!(out.count("desktop_metadata"), 2); - assert_eq!(out.count("desktop_window"), 4); + assert_eq!(out.count("desktop_window"), 8); } #[tokio::test] @@ -1444,7 +1491,7 @@ hold_secs = 1 ] { assert_eq!(large.count(op), 2, "{op}"); } - assert!(large.count("desktop_window") >= 8); + assert!(large.count("desktop_window") >= 16); // Up to ten channels: subscriptions are re-sent and nothing is read. let small = drive( diff --git a/crates/buzz-test-client/src/loadgen/run.rs b/crates/buzz-test-client/src/loadgen/run.rs index e95f6e766e1..6a35fb0dbfd 100644 --- a/crates/buzz-test-client/src/loadgen/run.rs +++ b/crates/buzz-test-client/src/loadgen/run.rs @@ -19,7 +19,7 @@ use tokio::sync::{watch, Mutex}; use super::desktop; use super::identity::{self, Role}; use super::net::{http_reason, Net}; -use super::scenario::{Community, HumanProfile, Members, Scenario}; +use super::scenario::{Community, HumanProfile, Members, Scenario, WideSplit}; use super::seed::{find_roots, hot_root_keys, reply_tags, HOT_ROOT_PREFIX, ROOT_PREFIX}; use super::stats::{abort_reason, ops_json, Op, Stats}; @@ -560,6 +560,7 @@ async fn human(shared: Arc, global: usize) { let community = &shared.communities[global % shared.communities.len()]; let keys = identity::keys(&scenario.run_id, Role::Human, global); let channels = scenario.member_channels(community.index, Role::Human, global); + let split = scenario.wide_split(Role::Human, global); let peers: Vec = (1..=10) .map(|k| { let peer = (global + k * shared.communities.len()) % scenario.population.humans.max(1); @@ -653,7 +654,7 @@ async fn human(shared: Arc, global: usize) { } } - let mut activity = Activity::new(profile, &mut rng); + let mut activity = Activity::new(profile, &mut rng, split); while ok && !stopped(&shared) { let wait = activity .due() @@ -732,12 +733,34 @@ pub(super) struct Activity { next_message: Option, next_reaction: Option, session_end: Option, + /// Set when `population.wide_activity_share` applies to this member. + split: Option, + /// With a split, the latest message seen in a wide (0) and a regular (1) + /// channel, so reactions follow the split too. + recent: [Option; 2], +} + +/// Index into `channels` of the channel a member's next typing indicator, +/// message or reaction goes to. Without a split every channel is equally +/// likely; with one, a wide channel gets `share` of the picks unless the +/// member has no regular channels. +fn pick_channel(split: Option, channels: &[String], rng: &mut Rng) -> usize { + match split { + Some(WideSplit { wide, share }) if wide > 0 && wide < channels.len() => { + if rng.unit() < share { + rng.below(wide) + } else { + wide + rng.below(channels.len() - wide) + } + } + _ => rng.below(channels.len()), + } } impl Activity { /// Schedules from now, the first heartbeat at a random point within one /// interval so clients connecting together do not beat together. - pub(super) fn new(profile: &HumanProfile, rng: &mut Rng) -> Self { + pub(super) fn new(profile: &HumanProfile, rng: &mut Rng, split: Option) -> Self { let interval = profile.presence_interval_secs; Self { interval, @@ -748,6 +771,8 @@ impl Activity { session_end: (profile.mean_session_secs > 0.0) .then(|| rng.after(3600.0 / profile.mean_session_secs)) .flatten(), + split, + recent: [None, None], } } @@ -778,6 +803,12 @@ impl Activity { last_message: &mut Option, ) -> Result { let profile = &shared.scenario.human; + if let Some(split) = self.split { + if let Some(seen) = last_message.take() { + let wide = channels[..split.wide.min(channels.len())].contains(&seen.channel); + self.recent[usize::from(!wide)] = Some(seen); + } + } let now = Instant::now(); if self.next_presence.is_some_and(|t| t <= now) { self.next_presence = Some(now + Duration::from_secs_f64(self.interval)); @@ -786,14 +817,14 @@ impl Activity { send(&shared.stats, conn, Op::PublishEphemeral, event).await?; } else if self.next_typing.is_some_and(|t| t <= now) { self.next_typing = rng.after(profile.typing_per_hour); - let channel = &channels[rng.below(channels.len())]; + let channel = &channels[pick_channel(self.split, channels, rng)]; let event = EventBuilder::new(kind(KIND_TYPING_INDICATOR), "") .tags([tag(["h", channel])]) .sign_with_keys(keys); send(&shared.stats, conn, Op::PublishEphemeral, event).await?; } else if self.next_message.is_some_and(|t| t <= now) { self.next_message = rng.after(profile.messages_per_hour); - let channel = &channels[rng.below(channels.len())]; + let channel = &channels[pick_channel(self.split, channels, rng)]; let mut tags = vec![tag(["h", channel])]; if rng.unit() < profile.thread_reply_fraction { let roots = roots(shared, community, keys, channel).await; @@ -812,7 +843,14 @@ impl Activity { } else if self.next_reaction.is_some_and(|t| t <= now) { self.next_reaction = rng.after(profile.reactions_per_hour); // React to each seen message at most once; the relay rejects duplicates. - if let Some(target) = last_message.take() { + let target = match self.split { + None => last_message.take(), + Some(split) => { + let i = pick_channel(self.split, channels, rng); + self.recent[usize::from(i >= split.wide)].take() + } + }; + if let Some(target) = target { let event = EventBuilder::new(kind(KIND_REACTION), "+") .tags([tag(["h", &target.channel]), tag(["e", &target.id])]) .sign_with_keys(keys); @@ -1583,6 +1621,11 @@ hold_secs = 1 json!(["EOSE", frame[1]]) } Some("EVENT") => { + let tags = frame[1]["tags"].as_array().cloned().unwrap_or_default(); + if let Some(h) = tags.iter().find(|t| t[0] == "h") { + let h = h[1].as_str().unwrap_or_default().to_string(); + seen.channels.lock().unwrap().push(h); + } let Some(mut delay) = relay.ok_delay else { continue; }; @@ -1615,6 +1658,8 @@ hold_secs = 1 struct Seen { reqs: AtomicUsize, closed: AtomicUsize, + /// The `h` tag of every EVENT that had one. + channels: std::sync::Mutex>, } impl Seen { @@ -1640,6 +1685,112 @@ hold_secs = 1 } } + #[test] + fn wide_activity_share_splits_picks_and_unset_keeps_the_uniform_pick() { + let channels: Vec = (0..6).map(|i| format!("c{i}")).collect(); + let draws = 100_000; + let wide_share = |split: Option| { + let mut rng = Rng::new(7); + let wide = (0..draws) + .filter(|_| pick_channel(split, &channels, &mut rng) < 3) + .count(); + wide as f64 / draws as f64 + }; + let split = |share| Some(WideSplit { wide: 3, share }); + // Unset: the same draws as before the knob, so every channel is as likely. + let (mut a, mut b) = (Rng::new(7), Rng::new(7)); + for _ in 0..1_000 { + assert_eq!( + pick_channel(None, &channels, &mut a), + b.below(channels.len()) + ); + } + assert!((wide_share(None) - 0.5).abs() < 0.01); + assert!((wide_share(split(0.1)) - 0.1).abs() < 0.01); + assert!((wide_share(split(0.8)) - 0.8).abs() < 0.01); + assert_eq!(wide_share(split(1.0)), 1.0); + assert_eq!(wide_share(split(0.0)), 0.0); + // Regular picks stay uniform over the regular channels. + let mut rng = Rng::new(7); + let mut per = [0usize; 6]; + for _ in 0..draws { + per[pick_channel(split(0.1), &channels, &mut rng)] += 1; + } + for n in &per[3..] { + assert!((*n as f64 / draws as f64 - 0.3).abs() < 0.01, "{per:?}"); + } + // A member in wide channels only sends everything to them. + let mut rng = Rng::new(7); + let only_wide = Some(WideSplit { + wide: 6, + share: 0.1, + }); + let mut seen = [false; 6]; + for _ in 0..1_000 { + seen[pick_channel(only_wide, &channels, &mut rng)] = true; + } + assert_eq!(seen, [true; 6]); + } + + /// One human in three wide and three regular channels that only types, + /// ten times a second. + fn typing_wide_member(addr: &str, share: &str) -> Scenario { + Scenario::parse(&format!( + r#" +run_id = "t" +[target] +ws_url = "ws://{addr}" +http_url = "http://{addr}" +[shape] +communities = 1 +channels_per_community = 6 +wide_channels = 3 +[population] +humans = 1 +agents = 0 +channels_per_member = 3 +wide_member_fraction = 1.0 +{share} +[human] +presence_interval_secs = 3600.0 +typing_per_hour = 36000.0 +messages_per_hour = 0.0 +reactions_per_hour = 0.0 +history_limit = 1 +[[ramp]] +users = 1.0 +hold_secs = 2 +"# + )) + .unwrap() + } + + #[tokio::test] + async fn typing_follows_the_wide_activity_share() { + let mut wide_counts = Vec::new(); + for share in ["", "wide_activity_share = 0.0", "wide_activity_share = 1.0"] { + let (addr, seen) = fake_ws_relay(FakeRelay::PROMPT).await; + let scenario = typing_wide_member(&addr, share); + let wide: Vec = scenario.channel_ids(0)[..3].to_vec(); + run_lone_human(scenario, Duration::from_millis(300)) + .await + .expect("the run passes"); + let channels = seen.channels.lock().unwrap().clone(); + assert!(channels.len() >= 5, "{share}: {channels:?}"); + let in_wide = channels.iter().filter(|c| wide.contains(c)).count(); + wide_counts.push((in_wide, channels.len())); + } + let [(unset, n), (none, _), (all, m)] = wide_counts[..] else { + unreachable!() + }; + assert!( + unset > 0 && unset < n, + "unset picks both kinds: {wide_counts:?}" + ); + assert_eq!(none, 0, "{wide_counts:?}"); + assert_eq!(all, m, "{wide_counts:?}"); + } + /// One human, no agents, one `hold_secs` step, and a presence heartbeat /// every second; other traffic is off so the heartbeat is the only publish. fn lone_human(addr: &str, hold_secs: u64, max_error_rate: f64) -> Scenario { diff --git a/crates/buzz-test-client/src/loadgen/scenario.rs b/crates/buzz-test-client/src/loadgen/scenario.rs index e44cb495859..aabfb899e5e 100644 --- a/crates/buzz-test-client/src/loadgen/scenario.rs +++ b/crates/buzz-test-client/src/loadgen/scenario.rs @@ -114,12 +114,28 @@ pub struct Population { /// Fraction of humans that also join every wide channel. #[serde(default)] pub wide_member_fraction: f64, + /// Share of its typing, messages and reactions a member of the wide + /// channels sends into them; the rest goes to its regular channels. + /// Unset, typing and messages pick any of a member's channels equally and + /// reactions go to the latest message it saw. + #[serde(default)] + pub wide_activity_share: Option, /// Humans whose channel count differs from `channels_per_member`, such /// as a few heavy users in many channels. Agents keep `channels_per_member`. #[serde(default)] pub tiers: Vec, } +/// A member's split of its own traffic between the wide channels that lead +/// its channel list and its regular channels. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct WideSplit { + /// Wide channels at the start of the member's channels. + pub wide: usize, + /// Share of picks that land in a wide channel. + pub share: f64, +} + /// `fraction` of humans join `channels` channels instead of /// `population.channels_per_member`, not counting wide channels. #[derive(Clone, Debug, Deserialize)] @@ -468,6 +484,10 @@ impl Scenario { self.population.wide_member_fraction, ), ("desktop.fraction", self.desktop.fraction), + ( + "population.wide_activity_share", + self.population.wide_activity_share.unwrap_or(0.0), + ), ]; for (name, value) in fractions { if !(0.0..=1.0).contains(&value) { @@ -517,16 +537,34 @@ impl Scenario { let regular = self.shape.channels_per_community - wide; let mut channels = Vec::new(); let mut count = self.population.channels_per_member; + if self.joins_wide(role, index) { + channels.extend(0..wide); + } if role == Role::Human { - if wide > 0 && draw(index, WIDE_SALT) < self.population.wide_member_fraction { - channels.extend(0..wide); - } count = self.tier_channels(index); } channels.extend((0..count).map(|k| wide + (index + k) % regular)); channels } + /// Whether member `index` of `role` joins the wide channels. + fn joins_wide(&self, role: Role, index: usize) -> bool { + role == Role::Human + && self.shape.wide_channels > 0 + && draw(index, WIDE_SALT) < self.population.wide_member_fraction + } + + /// How member `index` of `role` splits its own traffic, when + /// `population.wide_activity_share` applies to it: the number of wide + /// channels leading its channel list and the share they get. + pub fn wide_split(&self, role: Role, index: usize) -> Option { + let share = self.population.wide_activity_share?; + self.joins_wide(role, index).then_some(WideSplit { + wide: self.shape.wide_channels, + share, + }) + } + /// Regular (not wide) channels human `index` joins: its tier's count, or /// `channels_per_member` for humans outside every tier. fn tier_channels(&self, index: usize) -> usize { @@ -845,6 +883,34 @@ hold_secs = 10 } } + #[test] + fn wide_activity_share_applies_only_to_members_of_wide_channels() { + let wide = "wide_channels = 2"; + let unset = shaped(6, wide, "wide_member_fraction = 1.0", "").unwrap(); + assert_eq!(unset.population.wide_activity_share, None); + assert_eq!(unset.wide_split(Role::Human, 0), None); + let set = shaped( + 6, + wide, + "wide_member_fraction = 1.0\nwide_activity_share = 0.25", + "", + ) + .unwrap(); + let split = WideSplit { + wide: 2, + share: 0.25, + }; + for m in 0..20 { + assert_eq!(set.wide_split(Role::Human, m), Some(split)); + // The wide channels lead the member's channels, as the split assumes. + assert_eq!(set.member_channel_indices(Role::Human, m)[..2], [0, 1]); + // Agents never join wide channels. + assert_eq!(set.wide_split(Role::Agent, m), None); + } + let outside = shaped(6, wide, "wide_activity_share = 0.25", "").unwrap(); + assert!((0..20).all(|m| outside.wide_split(Role::Human, m).is_none())); + } + #[test] fn shape_and_desktop_knobs_are_validated() { let rejected = [ @@ -853,6 +919,9 @@ hold_secs = 10 ("wide_channels = 3", "", ""), ("", "wide_member_fraction = 1.5", ""), ("", "wide_member_fraction = nan", ""), + ("", "wide_activity_share = 1.5", ""), + ("", "wide_activity_share = -0.1", ""), + ("", "wide_activity_share = nan", ""), ("", "[[population.tiers]]\nfraction = 0.0\nchannels = 1", ""), ("", "[[population.tiers]]\nfraction = 1.5\nchannels = 1", ""), ("", "[[population.tiers]]\nfraction = 0.6\nchannels = 1\n[[population.tiers]]\nfraction = 0.6\nchannels = 2", ""), diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index f46482b25ea..8f31177b9e0 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -24,7 +24,7 @@ cargo run --release -p buzz-test-client --bin buzz-loadgen -- teardown perf/scen | Throughput per client | `[human]` and `[agent]` rates | | Reconnect churn | `human.mean_session_secs`, `population.connect_rate_per_sec` | | Private membership | `shape.private_channels` | -| Large channels | `shape.wide_channels`, `population.wide_member_fraction` | +| Large channels | `shape.wide_channels`, `population.wide_member_fraction`, `population.wide_activity_share` | | Uneven membership | `population.tiers` | | Desktop clients | `[desktop]`: `fraction`, `mode`, `windows`, `threads`, `tick_secs`, `backstop_secs`, `renewal_secs` | @@ -53,6 +53,13 @@ Members are assigned deterministically by global index, so every shard and wide channels. Each human joins them with probability `population.wide_member_fraction`, on top of its own channels. Agents never do. +- `population.wide_activity_share`, when set, is the share of its typing, + messages and reactions a member of the wide channels sends into them; the + rest goes to its regular channels. Unset, typing and messages pick any of + a member's channels equally, so a member of three wide and three regular + channels sends half of them into the wide ones, and reactions go to the + latest message the member saw. It has no effect on members outside the wide + channels. - `population.tiers` is a list of `{ fraction, channels }`. That fraction of humans joins `channels` channels instead of `channels_per_member`. Fractions sum to at most 1; the rest keep `channels_per_member`. diff --git a/perf/scenarios/local-desktop.toml b/perf/scenarios/local-desktop.toml index accea580b04..518fa65ee78 100644 --- a/perf/scenarios/local-desktop.toml +++ b/perf/scenarios/local-desktop.toml @@ -27,6 +27,8 @@ agents = 4 channels_per_member = 5 connect_rate_per_sec = 20.0 wide_member_fraction = 0.5 +# Members of the wide channels send a fifth of their own traffic there. +wide_activity_share = 0.2 # A quarter of the humans are in 15 channels instead of 5, so their # subscriptions batch past ten channels. tiers = [{ fraction = 0.25, channels = 15 }] From 7a4dcefe6d1d09e15576bb48f0083c96d4bbf178 Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Tue, 3 Nov 2026 12:54:11 -0500 Subject: [PATCH 3/4] fix(loadgen): establish desktop routes by replay outcome; follow window bounds Review fixes for the desktop client model: - Window catch-ups follow the cursor in the relay's kind 39006 window bounds, validated as the app does, instead of guessing from page length and the oldest event (which an older aux event skews). - Each route's replay is counted to its EOSE; 500 events, a missed deadline, or a live-only batch over ten channels is a cut. `current` reads follow each route's establishment: a continuous global skips its full read, membership then only repairs unreads, and channel routes recover windows and threads. A renewal carries routes until one is cut. - Only a full unread repair (snapshot plus repair) restarts its backstop. - A tick replacement that misses its deadline is closed, keeps the batch it would have replaced, and the next tick retries. - Read outcomes are kept: a failed roster or unread read is owed and retries on the next tick; the roster clock advances only on success. The tick clock runs across connections, and the read task drains its own queue so reads that come due mid-round start without waiting for the socket. Signed-off-by: Luke Tornquist Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 --- .../buzz-test-client/src/loadgen/desktop.rs | 1744 +++++++++++++---- perf/LOADGEN.md | 68 +- 2 files changed, 1427 insertions(+), 385 deletions(-) diff --git a/crates/buzz-test-client/src/loadgen/desktop.rs b/crates/buzz-test-client/src/loadgen/desktop.rs index 5c335c23466..cf2ca966fd8 100644 --- a/crates/buzz-test-client/src/loadgen/desktop.rs +++ b/crates/buzz-test-client/src/loadgen/desktop.rs @@ -68,13 +68,25 @@ const CATCH_UP_PAGES: usize = 5; /// The app's custom emoji set. const EMOJI_SET: &str = "buzz:custom-emoji"; +/// What a subscription carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Route { + /// The global profiles route: profiles, statuses and custom emoji. + Profiles, + /// The global membership route: grants, read state and reminders. + Membership, + /// A channel batch, which stays open. + Channels, + /// A replay of channels behind a live-only batch, closed at its EOSE. + Replay, +} + /// One REQ frame. #[derive(Clone, Debug, PartialEq)] pub(super) struct Req { pub(super) id: String, pub(super) filters: Vec, - /// A replay subscription, closed at its EOSE. Others stay open. - pub(super) replay: bool, + pub(super) route: Route, } impl Req { @@ -83,6 +95,25 @@ impl Req { frame.extend(self.filters.iter().cloned()); Value::Array(frame) } + + /// A route whose replay counts as cut whatever it sends: a live-only + /// channel batch over more than ten channels replays nothing itself. + fn cut_by_design(&self) -> bool { + let live_only = + self.route == Route::Channels && self.filters.iter().all(|f| f["limit"] == json!(0)); + let scope: usize = self + .filters + .iter() + .map(|f| f["#h"].as_array().map_or(0, Vec::len)) + .sum(); + live_only && scope > REPLAY_BATCH + } +} + +/// Whether a route's replay was cut at its EOSE: it replayed the app's limit +/// of events across its filters, or it was cut by design. +pub(super) fn limited(cut_by_design: bool, events: usize) -> bool { + cut_by_design || events >= REPLAY_LIMIT } fn channel_filter(ids: &[String], since: u64, limit: usize) -> Value { @@ -100,7 +131,7 @@ fn global_reqs(viewer: &str, since: u64) -> [Req; 2] { json!({"kinds": [30315], "#d": ["general"], "since": since, "limit": limit}), json!({"kinds": [30030], "#d": [EMOJI_SET], "since": since, "limit": limit}), ], - replay: false, + route: Route::Profiles, }, Req { id: "membership".into(), @@ -110,7 +141,7 @@ fn global_reqs(viewer: &str, since: u64) -> [Req; 2] { json!({"kinds": [30078], "authors": [viewer], "#t": ["read-state"], "since": since, "limit": limit}), json!({"kinds": [30300], "authors": [viewer], "since": since, "limit": limit}), ], - replay: false, + route: Route::Membership, }, ] } @@ -139,7 +170,7 @@ pub(super) fn batch_reqs( .iter() .map(|c| channel_filter(std::slice::from_ref(c), since, limit)) .collect(), - replay: false, + route: Route::Channels, }) .collect() } else { @@ -149,7 +180,7 @@ pub(super) fn batch_reqs( .map(|(i, chunk)| Req { id: id(i), filters: vec![channel_filter(chunk, since, 0)], - replay: false, + route: Route::Channels, }) .collect() } @@ -175,7 +206,7 @@ pub(super) fn replay_reqs( .iter() .map(|c| channel_filter(std::slice::from_ref(c), since, REPLAY_LIMIT)) .collect(), - replay: true, + route: Route::Replay, }) .collect() } @@ -199,26 +230,19 @@ pub(super) fn connect_reqs( pub(super) enum Situation { /// The client's first connection. Cold, - /// A reconnect whose replay is not continuous. Everything is read again - /// except channel names and sidebar preferences, which the client - /// already holds. + /// Any other reconnect: routes resume from the last connection. Reconnect, - /// A `current` renewal with every channel in one replaying batch: the - /// subscriptions are re-sent and nothing is read. - QuietRenewal, - /// A `current` renewal whose batches are live-only (more than 10 - /// channels). Channel routes establish again, so windows, threads and - /// unreads are refreshed; global reads are skipped. - LimitedRenewal, + /// A `current` identity renewal: routes are carried over, and none + /// establishes unless one of them replays a cut window. + Renewal, } -/// What an identity renewal is for a client in `mode` with `joined` channels. -/// Before the fixes, it was an ordinary reconnect. -pub(super) fn renewal(mode: DesktopMode, joined: usize) -> Situation { +/// What an identity renewal is in `mode`. Before the fixes it was an +/// ordinary reconnect. +pub(super) fn renewal(mode: DesktopMode) -> Situation { match mode { DesktopMode::Legacy => Situation::Reconnect, - DesktopMode::Current if joined <= REPLAY_BATCH => Situation::QuietRenewal, - DesktopMode::Current => Situation::LimitedRenewal, + DesktopMode::Current => Situation::Renewal, } } @@ -234,45 +258,178 @@ pub(super) enum Fetch { Profiles, Statuses, Emoji, - ReadState, - Unread, + /// Unread repair. With `snapshot`, a full repair: the read-state + /// snapshot first, then the repair. Without, the membership route's + /// replay covered read state and only the repair runs. + Unread { + snapshot: bool, + }, Reminders, Window(usize), Thread(usize), } -/// The reads a connection makes once established in `situation`, with -/// `windows` channel windows and `threads` thread views open. -pub(super) fn connect_fetches(situation: Situation, windows: usize, threads: usize) -> Vec { - let mut reads = match situation { - Situation::Cold => vec![ - Fetch::Nip11, - Fetch::Roster, - Fetch::Metadata, - Fetch::Sidebar, - Fetch::Profiles, - Fetch::Statuses, - Fetch::Emoji, - Fetch::ReadState, - Fetch::Unread, - Fetch::Reminders, - ], - Situation::Reconnect => vec![ - Fetch::Nip11, - Fetch::Roster, - Fetch::Profiles, - Fetch::Statuses, - Fetch::Emoji, - Fetch::ReadState, - Fetch::Unread, - Fetch::Reminders, - ], - Situation::QuietRenewal => return Vec::new(), - Situation::LimitedRenewal => vec![Fetch::Unread], - }; - reads.extend((0..windows).map(Fetch::Window)); - reads.extend((0..threads).map(Fetch::Thread)); - reads +const FULL_REPAIR: Fetch = Fetch::Unread { snapshot: true }; +const COVERED_REPAIR: Fetch = Fetch::Unread { snapshot: false }; + +impl Fetch { + /// Whether this job, queued or running, already does `other`'s work. + fn covers(self, other: Fetch) -> bool { + self == other || (self == FULL_REPAIR && other == COVERED_REPAIR) + } +} + +fn channel_recovery(windows: usize, threads: usize) -> impl Iterator { + (0..windows) + .map(Fetch::Window) + .chain((0..threads).map(Fetch::Thread)) +} + +/// The reads a connection makes as soon as it opens. `legacy` reads +/// everything here: all of it cold, all but channel names and sidebar +/// preferences on a reconnect, which it treats a renewal as. `current` +/// reads only what no route covers; the rest follows each route's +/// establishment ([`establish_fetches`]). +pub(super) fn connect_fetches( + mode: DesktopMode, + situation: Situation, + windows: usize, + threads: usize, +) -> Vec { + let cold = situation == Situation::Cold; + match (mode, situation) { + (DesktopMode::Legacy, _) => { + let mut reads = vec![Fetch::Nip11, Fetch::Roster]; + if cold { + reads.extend([Fetch::Metadata, Fetch::Sidebar]); + } + reads.extend([ + Fetch::Profiles, + Fetch::Statuses, + Fetch::Emoji, + FULL_REPAIR, + Fetch::Reminders, + ]); + reads.extend(channel_recovery(windows, threads)); + reads + } + (DesktopMode::Current, Situation::Cold) => { + vec![Fetch::Nip11, Fetch::Metadata, Fetch::Sidebar] + } + (DesktopMode::Current, Situation::Reconnect) => vec![Fetch::Nip11], + (DesktopMode::Current, Situation::Renewal) => Vec::new(), + } +} + +/// Reads still owed after a failed or unfinished one, as the app's stores +/// track them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Owed { + pub(super) roster: bool, + pub(super) profiles: bool, + pub(super) unread: bool, +} + +/// The reads `current` makes when `route` establishes. A `continuous` +/// global route replayed every change since the last connection, so it +/// skips its full read unless one is still owed: profiles resume their +/// statuses and emoji, and membership resumes read state and reminders and +/// only repairs unreads. A channel route recovers the open windows and +/// threads. +pub(super) fn establish_fetches( + route: Route, + continuous: bool, + owed: Owed, + windows: usize, + threads: usize, +) -> Vec { + match route { + Route::Profiles if !continuous => vec![Fetch::Profiles, Fetch::Statuses, Fetch::Emoji], + Route::Profiles if owed.profiles => vec![Fetch::Profiles], + Route::Profiles => Vec::new(), + Route::Membership => { + let mut reads = Vec::new(); + if !continuous || owed.roster { + reads.push(Fetch::Roster); + } + if continuous { + reads.push(COVERED_REPAIR); + } else { + reads.extend([FULL_REPAIR, Fetch::Reminders]); + } + reads + } + Route::Channels => channel_recovery(windows, threads).collect(), + Route::Replay => Vec::new(), + } +} + +/// Which of a connection's routes establish, and when. Each route kind +/// establishes once per connection. A route resumes only if the previous +/// connection had it caught up; one that had not counts as cut. On a +/// renewal, routes are carried: one whose replay kept continuity establishes +/// nothing, until any route's replay is cut, which establishes every route +/// again. +#[derive(Debug)] +pub(super) struct Establishment { + /// Routes the previous connection had caught up. + resumed: Vec, + carried: bool, + /// Routes that reached EOSE while carried, with their continuity. + held: Vec<(Route, bool)>, + done: Vec, +} + +impl Establishment { + /// A connection's establishment, after one that had `caught_up` routes + /// (see [`Establishment::caught_up`]). A cold start resumes nothing. + pub(super) fn new(situation: Situation, caught_up: &[Route]) -> Self { + let resumed = match situation { + Situation::Cold => Vec::new(), + _ => caught_up.to_vec(), + }; + Self { + resumed, + carried: situation == Situation::Renewal, + held: Vec::new(), + done: Vec::new(), + } + } + + /// `route` reached its EOSE, its replay `cut` or not. A route that missed + /// its deadline counts as cut. Returns the routes that establish now, + /// each with its continuity. + pub(super) fn eose(&mut self, route: Route, cut: bool) -> Vec<(Route, bool)> { + if route == Route::Replay { + return Vec::new(); + } + let cut = cut || !self.resumed.contains(&route); + let mut now = Vec::new(); + if self.carried { + if !cut { + self.held.push((route, true)); + return Vec::new(); + } + self.carried = false; + now = std::mem::take(&mut self.held); + } + now.push((route, !cut)); + now.retain(|(route, _)| { + let first = !self.done.contains(route); + if first { + self.done.push(*route); + } + first + }); + now + } + + /// The routes this connection has caught up: established, or carried + /// with their continuity kept. + pub(super) fn caught_up(&self) -> Vec { + let held = self.held.iter().map(|(route, _)| *route); + self.done.iter().copied().chain(held).collect() + } } /// What a recovery tick re-sends or rereads. @@ -281,19 +438,26 @@ pub(super) struct Due { /// Replace every channel batch with a live-only copy. pub(super) batches: bool, pub(super) roster: bool, - /// Read state and unread repair. + /// A full unread repair. pub(super) unread: bool, } +/// How long since each tick item was last done. `None`: never, or a +/// replacement that never established. +#[derive(Clone, Copy, Debug)] +pub(super) struct Clocks { + /// Since the batches were last sent. + pub(super) batches: Option, + /// Since the roster was last read in full. + pub(super) roster: Option, + /// Since a full unread repair was last asked for. + pub(super) unread: Option, +} + /// A tick's work. `legacy` does all of it every tick. `current` waits for -/// `backstop` since each was last done. -pub(super) fn tick( - mode: DesktopMode, - backstop: Duration, - since_batches: Duration, - since_roster: Duration, - since_unread: Duration, -) -> Due { +/// `backstop` since each was last done, and retries owed reads at once. +pub(super) fn tick(mode: DesktopMode, backstop: Duration, clocks: Clocks, owed: Owed) -> Due { + let stale = |since: Option| since.is_none_or(|d| d >= backstop); match mode { DesktopMode::Legacy => Due { batches: true, @@ -301,9 +465,9 @@ pub(super) fn tick( unread: true, }, DesktopMode::Current => Due { - batches: since_batches >= backstop, - roster: since_roster >= backstop, - unread: since_unread >= backstop, + batches: stale(clocks.batches), + roster: owed.roster || stale(clocks.roster), + unread: owed.unread || stale(clocks.unread), }, } } @@ -315,11 +479,103 @@ pub(super) fn tick_fetches(due: Due) -> Vec { reads.push(Fetch::Roster); } if due.unread { - reads.extend([Fetch::ReadState, Fetch::Unread]); + reads.push(FULL_REPAIR); } reads } +/// A window page's position: `(created_at, id)` of the row the next page +/// starts after. +pub(super) type Cursor = (u64, String); + +/// The relay's verdict on a window page: the cursor of the next page, or +/// `None` once history is exhausted. The relay answers every window with +/// one kind 39006 bounds event for this channel and cursor, signed by +/// `relay` (when known); rows, aux events and summaries say nothing about +/// exhaustion. Anything else is an error, as in the app. +pub(super) fn window_bounds( + events: &[Value], + channel: &str, + cursor: Option<&Cursor>, + relay: Option<&str>, +) -> Result, String> { + let suffix = match cursor { + Some((until, id)) => format!("{channel}:{until}:{id}"), + None => format!("{channel}:head"), + }; + let bounds: Vec<&Value> = events + .iter() + .filter(|e| { + e["kind"] == json!(39006) + && relay.is_none_or(|r| e["pubkey"] == json!(r)) + && has_tag(e, "h", channel) + && has_tag(e, "d", &suffix) + }) + .collect(); + let [bound] = bounds[..] else { + return Err(format!("{} window bounds for {suffix}", bounds.len())); + }; + let body: Value = serde_json::from_str(bound["content"].as_str().unwrap_or_default()) + .map_err(|e| format!("window bounds: {e}"))?; + let has_more = body["has_more"] + .as_bool() + .ok_or("window bounds omit has_more")?; + let next = match &body["next_cursor"] { + Value::Null => None, + next => { + let until = next["created_at"].as_u64(); + let id = next["id"].as_str().filter(|id| id.len() == 64); + let (Some(until), Some(id)) = (until, id) else { + return Err("window cursor is malformed".into()); + }; + Some((until, id.to_string())) + } + }; + if has_more != next.is_some() { + return Err("window bounds contradict their cursor".into()); + } + // History runs newest first, then by ascending id at equal times. + if let (Some((until, id)), Some((next_until, next_id))) = (cursor, &next) { + if next_until > until || (next_until == until && next_id <= id) { + return Err("window cursor does not advance".into()); + } + } + Ok(next) +} + +fn has_tag(event: &Value, name: &str, value: &str) -> bool { + event["tags"].as_array().is_some_and(|tags| { + tags.iter() + .any(|t| t[0] == json!(name) && t[1] == json!(value)) + }) +} + +/// The last row of a page in the relay's order (newest first, then by +/// ascending id): the cursor of the page after it. +pub(super) fn last_row(page: &[Value]) -> Option<(u64, &str)> { + page.iter() + .filter_map(|e| Some((e["created_at"].as_u64()?, e["id"].as_str()?))) + .min_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(a.1))) +} + +/// Points `filter` at the page after `cursor`. The relay takes `until` and +/// `before_id` together. +fn page_before(filter: &mut Value, (until, id): (u64, &str)) { + filter["until"] = json!(until); + filter["before_id"] = json!(id); +} + +/// The events of a `/query` response: an array, or the object a read-state +/// snapshot answers with. +fn events(text: &str) -> serde_json::Result> { + match serde_json::from_str(text)? { + Value::Object(mut body) => { + serde_json::from_value(body.remove("events").unwrap_or_default()) + } + other => serde_json::from_value(other), + } +} + /// Where a resumed subscription starts: [`RESUME_LAG_SECS`] before the /// last connection dropped, but not before the client's own start floor. fn resume_since(started: u64, cursor: Option, now: u64) -> u64 { @@ -361,6 +617,15 @@ fn co_members( out } +/// What the client's reads have settled: what is still owed and when the +/// roster was last read in full. Its read tasks write it; ticks and route +/// establishments read it. +#[derive(Debug)] +struct Ledger { + owed: Owed, + roster_read: Option, +} + /// What a client's reads use. Its sessions and read tasks share it. struct Reader { shared: Arc, @@ -370,16 +635,30 @@ struct Reader { channels: Vec, profiles: Vec, windows: usize, - /// The relay's own pubkey from NIP-11 `self`, the author of notices. + /// The relay's own pubkey from NIP-11 `self`, the author of notices and + /// window bounds. relay: std::sync::Mutex>, /// When the last unread repair started (Unix seconds). unread_mark: std::sync::Mutex>, + ledger: std::sync::Mutex, } impl Reader { /// One `POST /query`, timed as `op`. Returns the events, or `None` once /// it failed (recorded) or the run is stopping (not started). async fn query(&self, op: Op, filters: Value) -> Option> { + self.query_with(op, filters, |events| Ok(events.to_vec())) + .await + } + + /// [`Reader::query`], where the response only succeeds once `check` + /// accepts its events. + async fn query_with( + &self, + op: Op, + filters: Value, + check: impl FnOnce(&[Value]) -> Result, + ) -> Option { if stopped(&self.shared) { return None; } @@ -399,30 +678,33 @@ impl Reader { pending.err(&http_reason(status, &text)) } Ok(Ok((_, text))) => match events(&text) { - Ok(events) => { - pending.ok(); - return Some(events); - } + Ok(events) => match check(&events) { + Ok(value) => { + pending.ok(); + return Some(value); + } + Err(e) => pending.err(&e), + }, Err(e) => pending.err(&format!("decode: {e}")), }, } None } + fn owed(&self) -> Owed { + lock(&self.ledger).owed + } + async fn fetch(&self, fetch: Fetch) { let viewer = &self.viewer; match fetch { Fetch::Nip11 => self.nip11().await, Fetch::Roster => { - let mut filter = json!({"kinds": [39002], "#p": [viewer], "limit": PAGE}); - for _ in 0..MAX_ROSTER_PAGES { - let Some(page) = self.query(Op::DesktopRoster, json!([filter])).await else { - break; - }; - match oldest(&page) { - Some(cursor) if page.len() >= PAGE => page_before(&mut filter, cursor), - _ => break, - } + let read = self.roster().await; + let mut ledger = lock(&self.ledger); + ledger.owed.roster = !read; + if read { + ledger.roster_read = Some(Instant::now()); } } Fetch::Metadata => { @@ -445,10 +727,13 @@ impl Reader { self.query(Op::DesktopSidebar, Value::Array(filters)).await; } Fetch::Profiles => { + lock(&self.ledger).owed.profiles = true; + let mut read = true; for authors in self.profiles.chunks(PAGE) { let filter = json!([{"kinds": [0], "authors": authors, "limit": PAGE}]); - self.query(Op::DesktopProfiles, filter).await; + read &= self.query(Op::DesktopProfiles, filter).await.is_some(); } + lock(&self.ledger).owed.profiles = !read; } Fetch::Statuses => { let authors = &self.profiles[..self.profiles.len().min(STATUS_AUTHORS)]; @@ -463,30 +748,9 @@ impl Reader { let filter = json!([{"kinds": [30030], "#d": [EMOJI_SET], "limit": PAGE}]); self.query(Op::DesktopEmoji, filter).await; } - Fetch::ReadState => { - let filter = - json!([{"kinds": [30078], "authors": [viewer], "read_state_snapshot": 1}]); - self.query(Op::DesktopReadState, filter).await; - } - Fetch::Unread => { - let now = unix_now(); - let mark = lock(&self.unread_mark).replace(now); - // Sequential, as the app repairs one chunk after another. - for ids in self.channels.chunks(UNREAD_CHANNELS) { - let mut filter = json!({ - "kinds": [9, 40002, 40008], "#h": ids, "include_aux": true, "limit": REPLAY_LIMIT, - }); - if let Some(mark) = mark { - filter["since"] = json!(mark.saturating_sub(RESUME_LAG_SECS)); - } - if self - .query(Op::DesktopUnread, json!([filter])) - .await - .is_none() - { - break; - } - } + Fetch::Unread { snapshot } => { + let repaired = self.repair(snapshot).await; + lock(&self.ledger).owed.unread = !repaired; } Fetch::Reminders => { let filter = json!([{"kinds": [30300], "authors": [viewer], "limit": 201}]); @@ -497,6 +761,52 @@ impl Reader { } } + /// The roster in pages until a short page. Whether every page was read. + async fn roster(&self) -> bool { + let mut filter = json!({"kinds": [39002], "#p": [self.viewer], "limit": PAGE}); + for _ in 0..MAX_ROSTER_PAGES { + let Some(page) = self.query(Op::DesktopRoster, json!([filter])).await else { + return false; + }; + match last_row(&page) { + Some(cursor) if page.len() >= PAGE => page_before(&mut filter, cursor), + _ => break, + } + } + true + } + + /// An unread repair, after the read-state snapshot when `snapshot`. The + /// repair reads its channels one chunk after another, as the app does. + /// Whether it all succeeded. + async fn repair(&self, snapshot: bool) -> bool { + if snapshot { + let filter = + json!([{"kinds": [30078], "authors": [self.viewer], "read_state_snapshot": 1}]); + if self.query(Op::DesktopReadState, filter).await.is_none() { + return false; + } + } + let now = unix_now(); + let mark = lock(&self.unread_mark).replace(now); + for ids in self.channels.chunks(UNREAD_CHANNELS) { + let mut filter = json!({ + "kinds": [9, 40002, 40008], "#h": ids, "include_aux": true, "limit": REPLAY_LIMIT, + }); + if let Some(mark) = mark { + filter["since"] = json!(mark.saturating_sub(RESUME_LAG_SECS)); + } + if self + .query(Op::DesktopUnread, json!([filter])) + .await + .is_none() + { + return false; + } + } + true + } + /// `GET /` for the relay's NIP-11 document, remembering its `self`. async fn nip11(&self) { if stopped(&self.shared) { @@ -521,24 +831,33 @@ impl Reader { } } - /// A window catch-up: the head page, then older pages while each comes - /// back full (at most [`CATCH_UP_PAGES`]), then the channel's notices. + /// A window catch-up: the head page, then older pages while the relay's + /// bounds say there are more (at most [`CATCH_UP_PAGES`]), then the + /// channel's notices. async fn window(&self, channel: &str) { - let mut filter = json!({ - "kinds": WINDOW_KINDS, "#h": [channel], "limit": WINDOW_PAGE, - "top_level": true, "include_aux": true, "include_summaries": true, - }); + let relay = lock(&self.relay).clone(); + let mut cursor: Option = None; for _ in 0..=CATCH_UP_PAGES { - let Some(page) = self.query(Op::DesktopWindow, json!([filter])).await else { - break; - }; - match oldest(&page) { - Some(cursor) if page.len() >= WINDOW_PAGE => page_before(&mut filter, cursor), + let mut filter = json!({ + "kinds": WINDOW_KINDS, "#h": [channel], "limit": WINDOW_PAGE, + "top_level": true, "include_aux": true, "include_summaries": true, + }); + if let Some((until, id)) = &cursor { + page_before(&mut filter, (*until, id)); + } + let at = cursor.as_ref(); + let next = self + .query_with(Op::DesktopWindow, json!([filter]), |events| { + window_bounds(events, channel, at, relay.as_deref()) + }) + .await; + match next { + Some(Some(next)) => cursor = Some(next), _ => break, } } let mut notices = json!({"kinds": [40099], "#h": [channel], "limit": REPLAY_LIMIT}); - if let Some(relay) = lock(&self.relay).clone() { + if let Some(relay) = relay { notices["authors"] = json!([relay]); } self.query(Op::DesktopNotices, json!([notices])).await; @@ -574,33 +893,8 @@ fn lock(mutex: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// `(created_at, id)` of the oldest event in a page. -fn oldest(page: &[Value]) -> Option<(u64, &str)> { - page.iter() - .filter_map(|e| Some((e["created_at"].as_u64()?, e["id"].as_str()?))) - .min() -} - -/// Points `filter` at the page before `cursor`, the oldest event so far. The -/// relay takes `until` and `before_id` together. -fn page_before(filter: &mut Value, (until, id): (u64, &str)) { - filter["until"] = json!(until); - filter["before_id"] = json!(id); -} - -/// The events of a `/query` response: an array, or the object a read-state -/// snapshot answers with. -fn events(text: &str) -> serde_json::Result> { - match serde_json::from_str(text)? { - Value::Object(mut body) => { - serde_json::from_value(body.remove("events").unwrap_or_default()) - } - other => serde_json::from_value(other), - } -} - -/// Runs `fetches`: NIP-11 first, since notices need the relay's pubkey, then -/// the rest at most [`HTTP_CONCURRENCY`] at a time. +/// Runs `fetches`: NIP-11 first, since notices and window bounds need the +/// relay's pubkey, then the rest at most [`HTTP_CONCURRENCY`] at a time. async fn run_fetches(reader: Arc, fetches: Vec) { if fetches.contains(&Fetch::Nip11) { reader.nip11().await; @@ -616,35 +910,60 @@ async fn run_fetches(reader: Arc, fetches: Vec) { } /// A client's HTTP reads: one task at a time, so reads never pile up behind -/// a slow relay. Reads that come due while it runs are queued, once each, -/// for the next task. Dropping it (the client was cancelled) aborts the task, -/// so its unfinished reads are recorded. +/// a slow relay. Reads that come due while it runs are queued for its next +/// round; one whose work is already running or queued is not added again. +/// Dropping it (the client was cancelled) aborts the task, so its unfinished +/// reads are recorded. #[derive(Default)] struct Fetcher { task: Option>, + work: Arc>, +} + +/// A [`Fetcher`]'s reads, shared with its task. +#[derive(Default)] +struct Work { + /// Whether the task is under way; it clears this when it runs dry. + active: bool, + running: Vec, queued: Vec, } impl Fetcher { fn start(&mut self, reader: &Arc, fetches: Vec) { + let mut work = lock(&self.work); for fetch in fetches { - if !self.queued.contains(&fetch) { - self.queued.push(fetch); + let covered = |have: &Fetch| have.covers(fetch); + if work.running.iter().any(covered) || work.queued.iter().any(covered) { + continue; } + // A full repair queued now does a queued covered one's work too. + work.queued.retain(|have| !fetch.covers(*have)); + work.queued.push(fetch); } - self.poll(reader); - } - - /// Starts the queued reads once the previous task finished. - fn poll(&mut self, reader: &Arc) { - if self.queued.is_empty() || self.task.as_ref().is_some_and(|t| !t.is_finished()) { + if work.active || work.queued.is_empty() { return; } - let fetches = std::mem::take(&mut self.queued); - self.task = Some(tokio::spawn(run_fetches(reader.clone(), fetches))); + work.active = true; + drop(work); + let (reader, work) = (reader.clone(), self.work.clone()); + self.task = Some(tokio::spawn(async move { + loop { + let fetches = { + let mut work = lock(&work); + work.running = std::mem::take(&mut work.queued); + if work.running.is_empty() { + work.active = false; + return; + } + work.running.clone() + }; + run_fetches(reader.clone(), fetches).await; + } + })); } - /// Waits for the task under way. Reads still queued are dropped. + /// Waits for the task under way, and the reads queued behind it. async fn finish(&mut self) { if let Some(task) = self.task.take() { let _ = task.await; @@ -672,7 +991,13 @@ enum End { /// A REQ awaiting its EOSE. struct Waiting<'a> { - replay: bool, + route: Route, + /// [`Req::cut_by_design`]. + cut: bool, + /// For a tick's replacement, the batch it takes over at its EOSE. + slot: Option, + /// Events replayed so far. + events: usize, pending: Pending<'a>, sent: Instant, } @@ -681,6 +1006,11 @@ struct Waiting<'a> { /// cold, a renewal follows [`renewal`], and any other reconnect is a /// [`Situation::Reconnect`]. pub(super) async fn desktop(shared: Arc, global: usize) { + run_client(shared, global, REQ_TIMEOUT).await; +} + +/// [`desktop`], where a REQ may wait `req_timeout` for its EOSE. +async fn run_client(shared: Arc, global: usize, req_timeout: Duration) { let scenario = &shared.scenario; let profile = &scenario.desktop; let community = global % shared.communities.len(); @@ -697,6 +1027,13 @@ pub(super) async fn desktop(shared: Arc, global: usize) { windows: profile.windows.min(indices.len()), relay: Default::default(), unread_mark: Default::default(), + ledger: std::sync::Mutex::new(Ledger { + owed: Owed { + profiles: true, + ..Owed::default() + }, + roster_read: None, + }), }); let mut client = Client { shared: shared.clone(), @@ -704,8 +1041,10 @@ pub(super) async fn desktop(shared: Arc, global: usize) { rng: Rng::new(global as u64 ^ 0xD35C_70B0), fetcher: Fetcher::default(), situation: Situation::Cold, - roster_at: Instant::now(), - unread_at: Instant::now(), + req_timeout, + next_tick: Instant::now() + Duration::from_secs_f64(profile.tick_secs), + unread_at: None, + caught_up: Vec::new(), generation: 0, first_renewal: true, last_message: None, @@ -724,7 +1063,7 @@ pub(super) async fn desktop(shared: Arc, global: usize) { let _ = conn.disconnect().await; client.situation = match end { End::Stopped => break, - End::Renewal => renewal(profile.mode, client.reader.channels.len()), + End::Renewal => renewal(profile.mode), End::Dropped => { backoff = Backoff::new(); Situation::Reconnect @@ -745,8 +1084,13 @@ struct Client { rng: Rng, fetcher: Fetcher, situation: Situation, - roster_at: Instant, - unread_at: Instant, + req_timeout: Duration, + /// The recovery tick runs on the client's clock, across connections. + next_tick: Instant, + /// When a full unread repair was last asked for. + unread_at: Option, + /// The routes the last connection had caught up. + caught_up: Vec, generation: u64, first_renewal: bool, last_message: Option, @@ -754,6 +1098,15 @@ struct Client { } impl Client { + /// Starts `fetches`. Asking for a full unread repair restarts its + /// backstop; a covered repair reads no snapshot, so it does not. + fn fetch(&mut self, fetches: Vec) { + if fetches.contains(&FULL_REPAIR) { + self.unread_at = Some(Instant::now()); + } + self.fetcher.start(&self.reader, fetches); + } + /// One connection: establishes per the situation, then runs ticks, /// presence and the human's own traffic until it ends. async fn session(&mut self, conn: &mut NostrWsConnection, since: u64) -> End { @@ -765,33 +1118,27 @@ impl Client { let reader = self.reader.clone(); let channels = &reader.channels; let community = &shared.communities[reader.community]; + let (windows, threads) = (reader.windows, profile.threads); self.generation += 1; let reqs = connect_reqs(mode, &reader.viewer, channels, since, self.generation); let mut batches: Vec = reqs .iter() - .filter(|r| r.id.starts_with("batch-")) + .filter(|r| r.route == Route::Channels) .map(|r| r.id.clone()) .collect(); - let mut batches_at = Instant::now(); - let mut setup: VecDeque = reqs.into(); - let reads = connect_fetches(self.situation, reader.windows, profile.threads); - if reads.contains(&Fetch::Roster) { - self.roster_at = Instant::now(); - } - if reads.contains(&Fetch::Unread) { - self.unread_at = Instant::now(); - } - self.fetcher.start(&reader, reads); + // `None` once a replacement missed its deadline: the next tick retries. + let mut batches_at = Some(Instant::now()); + let mut setup: VecDeque<(Req, Option)> = + reqs.into_iter().map(|r| (r, None)).collect(); + self.fetch(connect_fetches(mode, self.situation, windows, threads)); + let mut establishment = Establishment::new(self.situation, &self.caught_up); let mut waiting: HashMap> = HashMap::new(); let mut open: HashSet = HashSet::new(); - // New batch id to the batch it replaces, closed at the new one's EOSE. - let mut replacing: HashMap = HashMap::new(); let mut activity = Activity::new(&scenario.human, &mut self.rng, self.split); let tick_every = Duration::from_secs_f64(profile.tick_secs); let backstop = Duration::from_secs_f64(profile.backstop_secs); - let mut next_tick = Instant::now() + tick_every; let next_renewal = (profile.renewal_secs > 0.0).then(|| { // Clients connecting together do not renew together. let mut secs = profile.renewal_secs; @@ -810,46 +1157,56 @@ impl Client { break End::Stopped; } while waiting.len() < SETUP_CONCURRENCY { - let Some(req) = setup.pop_front() else { + let Some((req, slot)) = setup.pop_front() else { break; }; if let Err(e) = conn.send_raw(&req.frame()).await { break 'session disconnected(e); } - let op = if req.replay { - Op::DesktopReplay - } else { - Op::DesktopLive + let op = match req.route { + Route::Replay => Op::DesktopReplay, + _ => Op::DesktopLive, }; let waited = Waiting { - replay: req.replay, + route: req.route, + cut: req.cut_by_design(), + slot, + events: 0, pending: stats.begin(op), sent: Instant::now(), }; waiting.insert(req.id, waited); } + + // A REQ past its deadline: a replay or a replacement is closed (a + // replacement keeps the batch it would have replaced, and the next + // tick tries again); a connection's live route stays open, and + // establishes as cut, since it may have missed history. let expired: Vec = waiting .iter() - .filter(|(_, w)| w.sent.elapsed() >= REQ_TIMEOUT) + .filter(|(_, w)| w.sent.elapsed() >= self.req_timeout) .map(|(id, _)| id.clone()) .collect(); + let mut established = Vec::new(); for id in expired { let Some(w) = waiting.remove(&id) else { continue; }; w.pending.err("EOSE timeout"); - // A late live subscription may still deliver; a replay is abandoned. - if w.replay { + if w.route == Route::Replay || w.slot.is_some() { + if w.slot.is_some() { + batches_at = None; + } if let Err(e) = close(conn, &id).await { break 'session disconnected(e); } } else { open.insert(id); + established.extend(establishment.eose(w.route, true)); } } - self.fetcher.poll(&reader); - let due = [activity.due(), Some(next_tick), next_renewal] + let due = [activity.due(), Some(self.next_tick), next_renewal] .into_iter() .flatten() .min(); @@ -861,14 +1218,20 @@ impl Client { Ok(RelayMessage::Eose { subscription_id }) => { if let Some(w) = waiting.remove(&subscription_id) { w.pending.ok(); - let retired = if w.replay { - Some(subscription_id) - } else { - let old = replacing.remove(&subscription_id); - open.insert(subscription_id); - old.inspect(|old| { - open.remove(old); - }) + let retired = match (w.route, w.slot) { + (Route::Replay, _) => Some(subscription_id), + (_, Some(slot)) => { + open.insert(subscription_id.clone()); + let old = std::mem::replace(&mut batches[slot], subscription_id); + open.remove(&old); + Some(old) + } + (route, None) => { + open.insert(subscription_id); + let cut = limited(w.cut, w.events); + established.extend(establishment.eose(route, cut)); + None + } }; if let Some(id) = retired { if let Err(e) = close(conn, &id).await { @@ -885,7 +1248,7 @@ impl Client { // as for the basic human; a refused replay is an error. if let Some(w) = waiting.remove(&subscription_id) { w.pending.err(&message); - if !w.replay { + if w.route != Route::Replay { break End::Failed; } } else if open.remove(&subscription_id) { @@ -897,14 +1260,18 @@ impl Client { } } Ok(message) => { - // Only live events after EOSE are delivery; replays are history. - let live = matches!( - &message, - RelayMessage::Event { subscription_id, .. } if open.contains(subscription_id) - ); - if live { - if let Some(target) = observe(stats, &message) { - self.last_message = Some(target); + // An event before EOSE is replay, which the app counts + // against its limit; only events after EOSE are delivery. + if let RelayMessage::Event { + subscription_id, .. + } = &message + { + if let Some(w) = waiting.get_mut(subscription_id) { + w.events += 1; + } else if open.contains(subscription_id) { + if let Some(target) = observe(stats, &message) { + self.last_message = Some(target); + } } } } @@ -912,6 +1279,23 @@ impl Client { Err(e) => break disconnected(e), } + // `legacy` read everything when it connected. + if mode == DesktopMode::Current { + let mut reads = Vec::new(); + for (route, continuous) in established { + reads.extend(establish_fetches( + route, + continuous, + reader.owed(), + windows, + threads, + )); + } + if !reads.is_empty() { + self.fetch(reads); + } + } + let step = activity .step( &shared, @@ -930,33 +1314,33 @@ impl Client { } let now = Instant::now(); - if now >= next_tick { - next_tick = now + tick_every; - let due = tick( - mode, - backstop, - batches_at.elapsed(), - self.roster_at.elapsed(), - self.unread_at.elapsed(), - ); + if now >= self.next_tick { + self.next_tick = now + tick_every; + let (owed, roster_read) = { + let ledger = lock(&reader.ledger); + (ledger.owed, ledger.roster_read) + }; + let clocks = Clocks { + batches: batches_at.map(|t| now.saturating_duration_since(t)), + roster: roster_read.map(|t| now.saturating_duration_since(t)), + unread: self.unread_at.map(|t| now.saturating_duration_since(t)), + }; + let due = tick(mode, backstop, clocks, owed); // A replacement still waiting for its EOSE is not replaced again. - if due.batches && replacing.is_empty() { + let replacing = setup.iter().any(|(_, slot)| slot.is_some()) + || waiting.values().any(|w| w.slot.is_some()); + if due.batches && !replacing { self.generation += 1; let fresh = batch_reqs(mode, channels, unix_now(), 0, self.generation); - for (new, old) in fresh.iter().zip(&batches) { - replacing.insert(new.id.clone(), old.clone()); - } - batches = fresh.iter().map(|r| r.id.clone()).collect(); - setup.extend(fresh); - batches_at = now; - } - if due.roster { - self.roster_at = now; - } - if due.unread { - self.unread_at = now; + setup.extend( + fresh + .into_iter() + .zip(0..batches.len()) + .map(|(r, i)| (r, Some(i))), + ); + batches_at = Some(now); } - self.fetcher.start(&reader, tick_fetches(due)); + self.fetch(tick_fetches(due)); } if next_renewal.is_some_and(|t| now >= t) { break End::Renewal; @@ -966,6 +1350,7 @@ impl Client { for (_, w) in waiting.drain() { w.pending.forget(); } + self.caught_up = establishment.caught_up(); end } } @@ -982,7 +1367,8 @@ mod tests { (0..n).map(|i| format!("c{i}")).collect() } - /// (filters, limit of the first filter, `#h` ids of the first filter) per REQ. + /// (filters, limit of the first filter, `#h` ids of the first filter, + /// replay) per REQ. fn shape(reqs: &[Req]) -> Vec<(usize, u64, usize, bool)> { reqs.iter() .map(|r| { @@ -991,7 +1377,7 @@ mod tests { r.filters.len(), first["limit"].as_u64().unwrap(), first["#h"].as_array().map_or(0, Vec::len), - r.replay, + r.route == Route::Replay, ) }) .collect() @@ -1006,6 +1392,7 @@ mod tests { assert_eq!(shape(&reqs[2..]), [(5, 500, 1, false)], "{mode:?}"); assert_eq!(reqs[2].filters[0]["since"], 100); assert_eq!(reqs[2].filters[0]["kinds"], json!(CHANNEL_KINDS)); + assert!(reqs.iter().all(|r| !r.cut_by_design()), "{mode:?}"); } } @@ -1016,6 +1403,7 @@ mod tests { shape(&legacy[2..]), [(10, 500, 1, false), (10, 500, 1, false), (5, 500, 1, false)] ); + assert!(legacy.iter().all(|r| !r.cut_by_design())); let current = connect_reqs(DesktopMode::Current, "v", &ids(25), 100, 1); assert_eq!( shape(¤t[2..]), @@ -1026,9 +1414,15 @@ mod tests { (5, 500, 1, true) ] ); + // The live-only batch replays nothing, so its window counts as cut. + let cut: Vec<_> = current.iter().map(Req::cut_by_design).collect(); + assert_eq!(cut, [false, false, true, false, false, false]); // One live-only filter holds up to 2048 channels. let wide = connect_reqs(DesktopMode::Current, "v", &ids(2049), 100, 1); - let live: Vec<_> = wide[2..].iter().filter(|r| !r.replay).collect(); + let live: Vec<_> = wide[2..] + .iter() + .filter(|r| r.route == Route::Channels) + .collect(); assert_eq!(live.len(), 2); assert_eq!(wide.len(), 2 + 2 + 205); } @@ -1047,10 +1441,28 @@ mod tests { assert_eq!(small[0].id, "batch-2-0"); } + #[test] + fn a_replay_is_limited_at_the_apps_cap() { + assert!(!limited(false, REPLAY_LIMIT - 1)); + assert!(limited(false, REPLAY_LIMIT)); + assert!(limited(true, 0)); + } + + const NO_OWED: Owed = Owed { + roster: false, + profiles: false, + unread: false, + }; + #[test] fn legacy_ticks_do_everything_and_current_ticks_wait_for_the_backstop() { let backstop = Duration::from_secs(900); - let minute = Duration::from_secs(60); + let minute = Some(Duration::from_secs(60)); + let clocks = |batches, roster, unread| Clocks { + batches, + roster, + unread, + }; let all = Due { batches: true, roster: true, @@ -1061,51 +1473,159 @@ mod tests { roster: false, unread: false, }; + let fresh = clocks(minute, minute, minute); + assert_eq!(tick(DesktopMode::Legacy, backstop, fresh, NO_OWED), all); + assert_eq!(tick(DesktopMode::Current, backstop, fresh, NO_OWED), none); + let stale = clocks(Some(backstop), Some(backstop), Some(backstop)); + assert_eq!(tick(DesktopMode::Current, backstop, stale, NO_OWED), all); + // Never done counts as stale. + let never = clocks(None, None, None); + assert_eq!(tick(DesktopMode::Current, backstop, never, NO_OWED), all); + // Each backstop runs on its own clock. + let roster_only = tick( + DesktopMode::Current, + backstop, + clocks(minute, Some(backstop), minute), + NO_OWED, + ); + assert_eq!(tick_fetches(roster_only), [Fetch::Roster]); + assert_eq!(tick_fetches(all), [Fetch::Roster, FULL_REPAIR]); + assert!(tick_fetches(none).is_empty()); + } + + #[test] + fn owed_reads_retry_on_the_next_tick_before_their_backstop() { + let backstop = Duration::from_secs(900); + let minute = Some(Duration::from_secs(60)); + let fresh = Clocks { + batches: minute, + roster: minute, + unread: minute, + }; + let owed = Owed { + roster: true, + unread: true, + ..NO_OWED + }; + let due = tick(DesktopMode::Current, backstop, fresh, owed); + assert!(!due.batches); + assert_eq!(tick_fetches(due), [Fetch::Roster, FULL_REPAIR]); + } + + #[test] + fn renewals_reconnect_before_and_carry_routes_after() { + assert_eq!(renewal(DesktopMode::Legacy), Situation::Reconnect); + assert_eq!(renewal(DesktopMode::Current), Situation::Renewal); + + let cold = connect_fetches(DesktopMode::Legacy, Situation::Cold, 2, 1); + assert_eq!(cold.len(), 9 + 3); + let reconnect = connect_fetches(DesktopMode::Legacy, Situation::Reconnect, 2, 1); + // A reconnect rereads everything but names and sidebar preferences. + let skipped: Vec<_> = cold.iter().filter(|r| !reconnect.contains(r)).collect(); + assert_eq!(skipped, [&Fetch::Metadata, &Fetch::Sidebar]); + + // `current` reads up front only what no route covers. + let current = |s| connect_fetches(DesktopMode::Current, s, 2, 1); assert_eq!( - tick(DesktopMode::Legacy, backstop, minute, minute, minute), - all + current(Situation::Cold), + [Fetch::Nip11, Fetch::Metadata, Fetch::Sidebar] ); + assert_eq!(current(Situation::Reconnect), [Fetch::Nip11]); + assert!(current(Situation::Renewal).is_empty()); + } + + #[test] + fn a_route_establishes_its_reads_by_continuity() { + let channels = [Fetch::Window(0), Fetch::Window(1), Fetch::Thread(0)]; + let est = |route, continuous, owed| establish_fetches(route, continuous, owed, 2, 1); assert_eq!( - tick(DesktopMode::Current, backstop, minute, minute, minute), - none + est(Route::Profiles, false, NO_OWED), + [Fetch::Profiles, Fetch::Statuses, Fetch::Emoji] ); + assert!(est(Route::Profiles, true, NO_OWED).is_empty()); + let profiles_owed = Owed { + profiles: true, + ..NO_OWED + }; + assert_eq!(est(Route::Profiles, true, profiles_owed), [Fetch::Profiles]); assert_eq!( - tick(DesktopMode::Current, backstop, backstop, backstop, backstop), - all + est(Route::Membership, false, NO_OWED), + [Fetch::Roster, FULL_REPAIR, Fetch::Reminders] ); - // Each backstop runs on its own clock. - let roster_only = tick(DesktopMode::Current, backstop, minute, backstop, minute); - assert_eq!(tick_fetches(roster_only), [Fetch::Roster]); + assert_eq!(est(Route::Membership, true, NO_OWED), [COVERED_REPAIR]); + let roster_owed = Owed { + roster: true, + ..NO_OWED + }; assert_eq!( - tick_fetches(all), - [Fetch::Roster, Fetch::ReadState, Fetch::Unread] + est(Route::Membership, true, roster_owed), + [Fetch::Roster, COVERED_REPAIR] ); - assert!(tick_fetches(none).is_empty()); + assert_eq!(est(Route::Channels, true, NO_OWED), channels); + assert_eq!(est(Route::Channels, false, NO_OWED), channels); + assert!(est(Route::Replay, false, NO_OWED).is_empty()); + assert!(FULL_REPAIR.covers(COVERED_REPAIR)); + assert!(!COVERED_REPAIR.covers(FULL_REPAIR)); } #[test] - fn renewals_reconnect_before_and_carry_or_limit_after() { - assert_eq!(renewal(DesktopMode::Legacy, 5), Situation::Reconnect); - assert_eq!(renewal(DesktopMode::Legacy, 25), Situation::Reconnect); - assert_eq!(renewal(DesktopMode::Current, 10), Situation::QuietRenewal); - assert_eq!(renewal(DesktopMode::Current, 11), Situation::LimitedRenewal); + fn routes_establish_once_and_a_renewal_holds_them_until_a_cut() { + let all = [Route::Profiles, Route::Membership, Route::Channels]; + // Cold: every route establishes at its EOSE, not continuous. + let mut cold = Establishment::new(Situation::Cold, &all); + assert_eq!( + cold.eose(Route::Profiles, false), + [(Route::Profiles, false)] + ); + assert!(cold.eose(Route::Replay, false).is_empty()); + assert_eq!( + cold.eose(Route::Channels, false), + [(Route::Channels, false)] + ); + assert!(cold.eose(Route::Channels, false).is_empty()); + assert_eq!(cold.caught_up(), [Route::Profiles, Route::Channels]); - assert!(connect_fetches(Situation::QuietRenewal, 2, 1).is_empty()); + // Reconnect: each route by its own replay, and only if the last + // connection had it caught up. + let mut reconnect = Establishment::new(Situation::Reconnect, &all[..2]); assert_eq!( - connect_fetches(Situation::LimitedRenewal, 2, 1), - [ - Fetch::Unread, - Fetch::Window(0), - Fetch::Window(1), - Fetch::Thread(0) - ] + reconnect.eose(Route::Membership, false), + [(Route::Membership, true)] + ); + assert_eq!( + reconnect.eose(Route::Profiles, true), + [(Route::Profiles, false)] + ); + assert_eq!( + reconnect.eose(Route::Channels, false), + [(Route::Channels, false)] + ); + + // Renewal without a cut: nothing establishes, all stay caught up. + let mut quiet = Establishment::new(Situation::Renewal, &all); + for route in all { + assert!(quiet.eose(route, false).is_empty()); + } + assert_eq!(quiet.caught_up(), all); + + // Renewal with a cut: the held routes establish with it. + let mut cut = Establishment::new(Situation::Renewal, &all); + assert!(cut.eose(Route::Profiles, false).is_empty()); + assert_eq!( + cut.eose(Route::Channels, true), + [(Route::Profiles, true), (Route::Channels, false)] + ); + assert_eq!( + cut.eose(Route::Membership, false), + [(Route::Membership, true)] + ); + + // A renewal before a route ever caught up reads it in full. + let mut early = Establishment::new(Situation::Renewal, &all[..1]); + assert_eq!( + early.eose(Route::Membership, false), + [(Route::Membership, false)] ); - let cold = connect_fetches(Situation::Cold, 2, 1); - assert_eq!(cold.len(), 10 + 3); - let reconnect = connect_fetches(Situation::Reconnect, 2, 1); - // A reconnect rereads everything but names and sidebar preferences. - let skipped: Vec<_> = cold.iter().filter(|r| !reconnect.contains(r)).collect(); - assert_eq!(skipped, [&Fetch::Metadata, &Fetch::Sidebar]); } #[test] @@ -1115,6 +1635,66 @@ mod tests { assert_eq!(resume_since(10_000, Some(10_100), 20_000), 9_700); } + fn hex(c: char) -> String { + std::iter::repeat_n(c, 64).collect() + } + + fn bound(d: &str, content: Value) -> Value { + json!({ + "kind": 39006, "pubkey": "relay", "content": content.to_string(), + "tags": [["d", d], ["h", "c0"]], + }) + } + + #[test] + fn window_pages_follow_the_relays_bounds() { + let next = json!({"has_more": true, "next_cursor": {"created_at": 1000, "id": hex('b')}}); + let row = json!({"kind": 9, "created_at": 1000, "id": hex('a')}); + let head = [row.clone(), bound("c0:head", next.clone())]; + let cursor = (1000, hex('b')); + assert_eq!( + window_bounds(&head, "c0", None, Some("relay")), + Ok(Some(cursor.clone())) + ); + // Exhausted. + let done = json!({"has_more": false, "next_cursor": null}); + let d = format!("c0:1000:{}", hex('b')); + let last = [bound(&d, done.clone())]; + assert_eq!(window_bounds(&last, "c0", Some(&cursor), None), Ok(None)); + // No bounds, two bounds, another signer, or the wrong cursor. + assert!(window_bounds(&[row], "c0", None, None).is_err()); + let twice = [ + bound("c0:head", next.clone()), + bound("c0:head", next.clone()), + ]; + assert!(window_bounds(&twice, "c0", None, None).is_err()); + assert!(window_bounds(&head, "c0", None, Some("other")).is_err()); + assert!(window_bounds(&last, "c0", None, None).is_err()); + // Contradictions and a cursor that does not move on. + let contradict = + json!({"has_more": false, "next_cursor": {"created_at": 1000, "id": hex('b')}}); + assert!(window_bounds(&[bound("c0:head", contradict)], "c0", None, None).is_err()); + let back = [bound( + &d, + json!({"has_more": true, "next_cursor": {"created_at": 1000, "id": hex('a')}}), + )]; + assert!(window_bounds(&back, "c0", Some(&cursor), None).is_err()); + let short = json!({"has_more": true, "next_cursor": {"created_at": 1000, "id": "ab"}}); + assert!(window_bounds(&[bound("c0:head", short)], "c0", None, None).is_err()); + } + + #[test] + fn the_last_row_is_the_oldest_then_the_highest_id() { + let page = [ + json!({"created_at": 1000, "id": "b"}), + json!({"created_at": 999, "id": "a"}), + json!({"created_at": 999, "id": "c"}), + json!({"created_at": 1001, "id": "z"}), + ]; + assert_eq!(last_row(&page), Some((999, "c"))); + assert_eq!(last_row(&[]), None); + } + // The client against loopback fakes: every connect, REQ, CLOSE and HTTP // read below goes through the production paths. @@ -1123,15 +1703,27 @@ mod tests { use super::super::scenario::Scenario; use super::super::seed::tests::fake_http; use super::super::stats::Stats; - use nostr::EventBuilder; + use nostr::{EventBuilder, Tag}; use tokio::sync::watch; - /// `(verb, subscription id)` of every REQ and CLOSE the fake relay got. - type Frames = Arc>>; + type Shared2 = Arc>; + + /// `(connection, verb, subscription id)` of every REQ and CLOSE the fake + /// relay got, connections numbered as accepted. + type Frames = Shared2>; + + /// How the fake relay answers a REQ on a connection: events replayed + /// before its EOSE, and how long the EOSE waits. + type Plan = Arc (usize, Duration) + Send + Sync>; + + fn instant() -> Plan { + Arc::new(|_, _| (0, Duration::ZERO)) + } /// A loopback relay that challenges, accepts AUTH and every EVENT, answers - /// each REQ with EOSE and records REQs and CLOSEs. - async fn fake_ws_relay() -> (String, Frames) { + /// each REQ per `plan` and records REQs and CLOSEs. With `drop_first`, it + /// closes its first connection that long after accepting it. + async fn fake_ws_relay(plan: Plan, drop_first: Option) -> (String, Frames) { use futures_util::SinkExt; use tokio_tungstenite::tungstenite::Message; @@ -1139,36 +1731,62 @@ mod tests { let addr = listener.local_addr().unwrap().to_string(); let frames = Frames::default(); let seen = frames.clone(); + let replayed = EventBuilder::new(kind(9), "replayed") + .sign_with_keys(&Keys::generate()) + .unwrap(); + let replayed = serde_json::to_value(replayed).unwrap(); tokio::spawn(async move { + let mut conn = 0; while let Ok((stream, _)) = listener.accept().await { - let seen = seen.clone(); + let (seen, plan, replayed) = (seen.clone(), plan.clone(), replayed.clone()); + let index = conn; + conn += 1; tokio::spawn(async move { - let Ok(mut ws) = tokio_tungstenite::accept_async(stream).await else { + let Ok(ws) = tokio_tungstenite::accept_async(stream).await else { return; }; - let challenge = json!(["AUTH", "challenge"]).to_string(); - if ws.send(Message::Text(challenge.into())).await.is_err() { - return; + let (mut sink, mut stream) = ws.split(); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + tokio::spawn(async move { + while let Some(message) = rx.recv().await { + let close = matches!(message, Message::Close(_)); + if sink.send(message).await.is_err() || close { + return; + } + } + }); + let text = |v: Value| Message::Text(v.to_string().into()); + let _ = tx.send(text(json!(["AUTH", "challenge"]))); + if let Some(after) = drop_first.filter(|_| index == 0) { + let tx = tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(after).await; + let _ = tx.send(Message::Close(None)); + }); } - while let Some(Ok(Message::Text(text))) = ws.next().await { - let frame: Value = serde_json::from_str(&text).unwrap(); + while let Some(Ok(Message::Text(raw))) = stream.next().await { + let frame: Value = serde_json::from_str(&raw).unwrap(); let verb = frame[0].as_str().unwrap_or_default().to_string(); - let reply = match verb.as_str() { - "AUTH" | "EVENT" => json!(["OK", frame[1]["id"], true, ""]), - "REQ" => json!(["EOSE", frame[1]]), - _ => Value::Null, - }; - if verb == "REQ" || verb == "CLOSE" { - let id = frame[1].as_str().unwrap_or_default().to_string(); - seen.lock().unwrap().push((verb, id)); + let id = frame[1].as_str().unwrap_or_default().to_string(); + match verb.as_str() { + "AUTH" | "EVENT" => { + let _ = tx.send(text(json!(["OK", frame[1]["id"], true, ""]))); + } + "REQ" => { + let (events, delay) = plan(index, &id); + let (tx, replayed, id) = (tx.clone(), replayed.clone(), id.clone()); + tokio::spawn(async move { + for _ in 0..events { + let _ = tx.send(text(json!(["EVENT", id, replayed]))); + } + tokio::time::sleep(delay).await; + let _ = tx.send(text(json!(["EOSE", id]))); + }); + } + _ => {} } - if !reply.is_null() - && ws - .send(Message::Text(reply.to_string().into())) - .await - .is_err() - { - return; + if verb == "REQ" || verb == "CLOSE" { + seen.lock().unwrap().push((index, verb, id)); } } }); @@ -1177,58 +1795,108 @@ mod tests { (addr, frames) } - /// A loopback HTTP API: NIP-11 names a relay key, the owner's root search - /// finds one seeded root, a window head page is full and the page before - /// it empty, the read-state snapshot answers in its own shape, and every - /// other query is empty. Like the relay, it refuses a top-level page - /// cursor without both `until` and `before_id`. - async fn fake_api(run_id: &str) -> String { - let owner = identity::keys(run_id, Role::Owner, 0); - let root = EventBuilder::new(kind(9), "lg-root 0") - .sign_with_keys(&owner) + /// Reads the fake API fails once each, by name: `roster`, `unread`. + type FailOnce = &'static [&'static str]; + + fn signed(keys: &Keys, k: u32, content: &str, at: u64, tags: Vec) -> Value { + let event = EventBuilder::new(kind(k), content) + .custom_created_at(nostr::Timestamp::from(at)) + .tags(tags) + .sign_with_keys(keys) .unwrap(); - let root = serde_json::to_string(&[root]).unwrap(); - let head: Vec<_> = (0..WINDOW_PAGE as u64) - .map(|i| { - EventBuilder::new(kind(9), "w") - .custom_created_at(nostr::Timestamp::from(1_000 + i)) - .sign_with_keys(&owner) - .unwrap() - }) - .collect(); - let head = serde_json::to_string(&head).unwrap(); + serde_json::to_value(event).unwrap() + } + + fn tag(name: &str, value: &str) -> Tag { + Tag::parse([name, value]).unwrap() + } + + /// A loopback HTTP API, recording every query's first filter. NIP-11 + /// names the relay's key; the owner's root search finds one seeded root. + /// A window head page holds a full page of rows at one timestamp, an + /// older aux event and a summary; its relay-signed bounds point after the + /// last row. The page there is full too, and its bounds say history is + /// exhausted. Any other window cursor is refused. The read-state snapshot + /// answers in its own shape; `fail` reads fail the first time. + async fn fake_api(run_id: &str, fail: FailOnce) -> (String, Shared2>) { + let owner = identity::keys(run_id, Role::Owner, 0); + let relay = Keys::generate(); + let root = Value::Array(vec![signed(&owner, 9, "lg-root 0", 2_000, vec![])]); + let rows = |at: u64| -> Vec { + (0..WINDOW_PAGE) + .map(|i| signed(&owner, 9, &format!("w{i}"), at, vec![])) + .collect() + }; + let head_rows = rows(1_000); + let last = head_rows + .iter() + .filter_map(|e| e["id"].as_str()) + .max() + .unwrap() + .to_string(); + let older = rows(990); + let info = json!({"self": relay.public_key().to_hex()}).to_string(); + let owner_hex = owner.public_key().to_hex(); + let queries = Shared2::>::default(); + let recorded = queries.clone(); + let failed = Shared2::>::default(); let snapshot = json!({"read_state_snapshot": 1, "complete": true, "events": []}).to_string(); - let info = json!({"self": Keys::generate().public_key().to_hex()}).to_string(); - let owner = owner.public_key().to_hex(); - fake_http(move |method, target, body| { + let http = fake_http(move |method, target, body| { if method == "GET" && target == "/" { return (200, info.clone()); } let filters: Value = serde_json::from_slice(body).unwrap_or_default(); - let filter = &filters[0]; + let filter = filters[0].clone(); + recorded.lock().unwrap().push(filter.clone()); + let name = if filter["kinds"] == json!([39002]) { + "roster" + } else if filter["kinds"] == json!([9, 40002, 40008]) { + "unread" + } else { + "" + }; + if fail.contains(&name) && !failed.lock().unwrap().contains(&name) { + failed.lock().unwrap().push(name); + return (500, r#"{"error":"once"}"#.into()); + } if filter.get("read_state_snapshot").is_some() { return (200, snapshot.clone()); } if filter.get("top_level").is_some() { - return match (filter.get("until"), filter.get("before_id")) { - (None, None) => (200, head.clone()), - (Some(_), Some(_)) => (200, "[]".into()), - _ => (400, r#"{"error":"top_level cursor requires both"}"#.into()), + let channel = filter["#h"][0].as_str().unwrap_or_default(); + let (d, mut page, next) = match (filter.get("until"), filter.get("before_id")) { + (None, None) => { + let mut page = head_rows.clone(); + page.push(signed(&owner, 7, "+", 900, vec![tag("h", channel)])); + page.push(signed(&relay, 39005, "{}", 1_000, vec![tag("h", channel)])); + let next = json!({"created_at": 1_000, "id": last}); + (format!("{channel}:head"), page, next) + } + (Some(until), Some(id)) if *until == json!(1_000) && *id == json!(last) => { + (format!("{channel}:1000:{last}"), older.clone(), Value::Null) + } + _ => return (400, r#"{"error":"unknown window cursor"}"#.into()), }; + let content = json!({"has_more": !next.is_null(), "next_cursor": next}); + let tags = vec![tag("d", &d), tag("h", channel)]; + page.push(signed(&relay, 39006, &content.to_string(), 1_000, tags)); + return (200, Value::Array(page).to_string()); } - if filter["kinds"] == json!([9]) && filter["authors"] == json!([owner]) { - (200, root.clone()) + if filter["kinds"] == json!([9]) && filter["authors"] == json!([owner_hex]) { + (200, root.to_string()) } else { (200, "[]".into()) } }) - .await + .await; + (http, queries) } - /// Two desktop humans in one community of 25 channels, each in `joined`, - /// with two windows and one thread open and no traffic of their own. - fn two_desktops(ws: &str, http: &str, joined: usize, desktop: &str) -> Scenario { + /// `humans` desktop humans in one community of 25 channels, each in + /// `joined`, with two windows and one thread open and no traffic of + /// their own. + fn desktops(ws: &str, http: &str, humans: usize, joined: usize, desktop: &str) -> Scenario { Scenario::parse(&format!( r#" run_id = "d" @@ -1239,7 +1907,7 @@ http_url = "http://{http}" communities = 1 channels_per_community = 25 [population] -humans = 2 +humans = {humans} agents = 0 channels_per_member = {joined} [human] @@ -1261,10 +1929,12 @@ hold_secs = 1 .unwrap() } - /// Per-op `(count, errors)` by op name, plus the relay's frames. + /// Per-op `(count, errors)` by op name, plus the relay's frames and the + /// API's queries. struct Outcome { ops: HashMap<&'static str, (u64, u64)>, - frames: Vec<(String, String)>, + frames: Vec<(usize, String, String)>, + queries: Vec, } impl Outcome { @@ -1272,8 +1942,19 @@ hold_secs = 1 self.ops.get(op).map_or(0, |c| c.0) } + fn errors(&self, op: &str) -> u64 { + self.ops.get(op).map_or(0, |c| c.1) + } + fn saw(&self, verb: &str, id: &str) -> bool { - self.frames.iter().any(|(v, i)| v == verb && i == id) + self.at(verb, id).is_some() + } + + /// Where a frame came in the relay's record. + fn at(&self, verb: &str, id: &str) -> Option { + self.frames + .iter() + .position(|(_, v, i)| v == verb && i == id) } /// Counts of the desktop ops, the ones the model owns. @@ -1284,9 +1965,18 @@ hold_secs = 1 .map(|(op, c)| (*op, c.0)) .collect() } + + /// Desktop op counts beyond `before`, zeroes dropped. + fn since(&self, before: &HashMap<&'static str, u64>) -> HashMap<&'static str, u64> { + self.desktop() + .into_iter() + .map(|(op, n)| (op, n - before.get(op).copied().unwrap_or(0))) + .filter(|(_, n)| *n > 0) + .collect() + } } - fn outcome(stats: &Stats, frames: &Frames) -> Outcome { + fn outcome(stats: &Stats, frames: &Frames, queries: &Shared2>) -> Outcome { let ops = stats .snapshot() .since(&Default::default()) @@ -1297,30 +1987,61 @@ hold_secs = 1 Outcome { ops, frames: frames.lock().unwrap().clone(), + queries: queries.lock().unwrap().clone(), } } - /// Runs both clients for at least `min`, until `done` holds, then stops - /// them and returns what they did. Fails after 15 s without `done`. - async fn drive( + /// One loopback run. + struct Rig { + /// Humans in the scenario, whose profiles a client reads. + humans: usize, + /// Clients that run: the first `clients` humans. + clients: usize, joined: usize, - desktop_toml: &str, - min: Duration, - done: impl Fn(&Outcome) -> bool, - ) -> Outcome { - let (ws, frames) = fake_ws_relay().await; - let http = fake_api("d").await; - let scenario = two_desktops(&ws, &http, joined, desktop_toml); + desktop: String, + plan: Plan, + drop_first: Option, + fail: FailOnce, + req_timeout: Duration, + } + + /// One client, with a co-member whose profile it reads. + fn solo(joined: usize, desktop: &str) -> Rig { + Rig { + clients: 1, + ..rig(2, joined, desktop) + } + } + + fn rig(humans: usize, joined: usize, desktop: &str) -> Rig { + Rig { + humans, + clients: humans, + joined, + desktop: desktop.into(), + plan: instant(), + drop_first: None, + fail: &[], + req_timeout: REQ_TIMEOUT, + } + } + + /// Runs the clients for at least `min`, until `done` holds, then stops + /// them and returns what they did. Fails after 15 s without `done`. + async fn drive(rig: Rig, min: Duration, done: impl Fn(&Outcome) -> bool) -> Outcome { + let (ws, frames) = fake_ws_relay(rig.plan, rig.drop_first).await; + let (http, queries) = fake_api("d", rig.fail).await; + let scenario = desktops(&ws, &http, rig.humans, rig.joined, &rig.desktop); let net = Net::new(&scenario).await.unwrap(); let stats = Arc::new(Stats::default()); let (stop_tx, stop) = watch::channel(false); let shared = Arc::new(Shared::new(scenario, net, stats.clone(), stop)); - let clients: Vec<_> = (0..2) - .map(|g| tokio::spawn(desktop(shared.clone(), g))) + let clients: Vec<_> = (0..rig.clients) + .map(|g| tokio::spawn(run_client(shared.clone(), g, rig.req_timeout))) .collect(); let started = Instant::now(); loop { - let now = outcome(&stats, &frames); + let now = outcome(&stats, &frames, &queries); if started.elapsed() >= min && done(&now) { break; } @@ -1339,10 +2060,12 @@ hold_secs = 1 .expect("a stopped client returns") .unwrap(); } - let out = outcome(&stats, &frames); + outcome(&stats, &frames, &queries) + } + + fn no_errors(out: &Outcome) { let failed: Vec<_> = out.ops.iter().filter(|(_, c)| c.1 > 0).collect(); assert!(failed.is_empty(), "errors: {failed:?}"); - out } /// What one cold start reads over HTTP with two windows and a thread. @@ -1358,7 +2081,7 @@ hold_secs = 1 ("desktop_read_state", 1), ("desktop_unread", 1), ("desktop_reminders", 1), - // Per window, a full head page and the empty page before it. + // Per window, the head page and the page its bounds point to. ("desktop_window", 4), ("desktop_notices", 2), ("desktop_thread_root", 1), @@ -1377,6 +2100,19 @@ hold_secs = 1 ops } + fn ops(list: &[(&'static str, u64)]) -> HashMap<&'static str, u64> { + list.iter().copied().collect() + } + + /// What a channel route's establishment reads: both windows (two pages + /// and notices each) and the thread. + const CHANNEL_RECOVERY: [(&str, u64); 4] = [ + ("desktop_window", 4), + ("desktop_notices", 2), + ("desktop_thread_root", 1), + ("desktop_thread_window", 1), + ]; + const QUIET: &str = "tick_secs = 3600.0"; #[tokio::test] @@ -1396,24 +2132,287 @@ hold_secs = 1 for (mode, joined, subs) in cases { let want = with(cold_reads(2), &subs); let toml = format!("mode = \"{mode}\"\n{QUIET}"); - let out = drive(joined, &toml, Duration::from_millis(300), |o| { + let out = drive(rig(2, joined, &toml), Duration::from_millis(300), |o| { o.desktop() == want }) .await; + no_errors(&out); assert_eq!(out.desktop(), want, "{mode} J={joined}"); assert_eq!(out.count("ws_connect_auth"), 2, "{mode} J={joined}"); } } #[tokio::test] - async fn legacy_ticks_reread_and_replace_batches_every_tick() { + async fn a_window_catch_up_follows_the_relays_cursor_until_it_is_exhausted() { + let toml = format!("mode = \"current\"\n{QUIET}"); + let out = drive(solo(5, &toml), Duration::ZERO, |o| { + o.count("desktop_notices") == 2 + }) + .await; + no_errors(&out); + let top: Vec<&Value> = out + .queries + .iter() + .filter(|f| f.get("top_level").is_some()) + .collect(); + let channel = &top[0]["#h"]; + let windows: Vec<&Value> = top.into_iter().filter(|f| f["#h"] == *channel).collect(); + // The head, then the page after the head's last row (not after its + // older aux event), and no third read of an exhausted history, + // though that page came back full. + assert_eq!(windows.len(), 2, "{windows:?}"); + assert!(windows[0].get("until").is_none()); + assert_eq!(windows[1]["until"], 1_000); + let ids: Vec<&str> = out + .queries + .iter() + .filter(|f| f.get("before_id").is_some()) + .filter_map(|f| f["before_id"].as_str()) + .collect(); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|id| id.len() == 64 && *id == ids[0])); + assert_eq!(out.count("desktop_window"), 4); + } + + /// Holds once `ready` has held for `settle`: what a connection reads has + /// had time to finish. + fn settled(ready: impl Fn(&Outcome) -> bool, settle: Duration) -> impl Fn(&Outcome) -> bool { + let since = std::cell::Cell::new(None::); + move |o| { + if !ready(o) { + return false; + } + let at = since.get().unwrap_or_else(Instant::now); + since.set(Some(at)); + at.elapsed() >= settle + } + } + + /// The reads beyond its cold start of a `current` client in 5 channels, + /// once its first renewal's connection established with each replay + /// sending 499 events, except 500 on subscriptions whose id starts + /// with `cut`. + async fn renewed(cut: &'static str) -> HashMap<&'static str, u64> { + let mut rig = solo( + 5, + "mode = \"current\"\nrenewal_secs = 4.0\ntick_secs = 3600.0", + ); + rig.plan = Arc::new(move |conn, id| { + let events = match conn { + 0 => 0, + _ if id.starts_with(cut) => REPLAY_LIMIT, + _ => REPLAY_LIMIT - 1, + }; + (events, Duration::ZERO) + }); + let cold = with(cold_reads(1), &[("desktop_live_eose", 3)]); let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("desktop_live_eose") >= 6, + Duration::from_millis(800), + ), + ) + .await; + no_errors(&out); + assert_eq!(out.count("ws_connect_auth"), 2, "{cut}"); + out.since(&cold) + .into_iter() + .filter(|(op, _)| *op != "desktop_live_eose") + .collect() + } + + #[tokio::test] + async fn a_renewal_recovers_only_what_a_cut_replay_lost() { + // 499 events everywhere: every route kept continuity. + assert_eq!(renewed("none").await, ops(&[])); + // A cut channel replay: the windows and the thread recover, and the + // carried membership route only repairs unreads. + assert_eq!( + renewed("batch").await, + ops(&[CHANNEL_RECOVERY.as_slice(), &[("desktop_unread", 1)]].concat()) + ); + // A cut membership replay: its full refresh, and the channels. + assert_eq!( + renewed("membership").await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[ + ("desktop_roster", 1), + ("desktop_read_state", 1), + ("desktop_unread", 1), + ("desktop_reminders", 1), + ] + ] + .concat()) + ); + // A cut profiles replay: its refresh, the channels, and the covered + // membership route's repair. + assert_eq!( + renewed("profiles").await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[ + ("desktop_profiles", 1), + ("desktop_statuses", 1), + ("desktop_emoji", 1), + ("desktop_unread", 1), + ] + ] + .concat()) + ); + } + + /// The reads beyond its cold start of a `current` client in 5 channels + /// whose first connection drops after it established, once the next + /// connection established; its global replays send `globals` events. + async fn reconnected(globals: usize) -> HashMap<&'static str, u64> { + let mut rig = solo(5, &format!("mode = \"current\"\n{QUIET}")); + rig.drop_first = Some(Duration::from_millis(700)); + rig.plan = Arc::new(move |conn, id| { + let global = id == "profiles" || id == "membership"; + let events = if conn > 0 && global { globals } else { 0 }; + (events, Duration::ZERO) + }); + let cold = with(cold_reads(1), &[("desktop_live_eose", 3)]); + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("ws_connect_auth") >= 2 && o.count("desktop_live_eose") >= 6, + Duration::from_millis(800), + ), + ) + .await; + // The drop is the only error. + let failed: Vec<_> = out + .ops + .iter() + .filter(|(op, c)| c.1 > 0 && **op != "ws_connect_auth") + .collect(); + assert!(failed.is_empty(), "errors: {failed:?}"); + assert_eq!(out.count("ws_connect_auth"), 2); + out.since(&cold) + .into_iter() + .filter(|(op, _)| *op != "desktop_live_eose") + .collect() + } + + #[tokio::test] + async fn a_reconnect_rereads_a_global_only_when_its_replay_was_cut() { + // Complete replays: the globals resume, unreads are only repaired, + // and the channels recover. + assert_eq!( + reconnected(REPLAY_LIMIT - 1).await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[("desktop_nip11", 1), ("desktop_unread", 1)] + ] + .concat()) + ); + // Truncated global replays: both globals refresh in full. + assert_eq!( + reconnected(REPLAY_LIMIT).await, + ops(&[ + CHANNEL_RECOVERY.as_slice(), + &[ + ("desktop_nip11", 1), + ("desktop_profiles", 1), + ("desktop_statuses", 1), + ("desktop_emoji", 1), + ("desktop_roster", 1), + ("desktop_read_state", 1), + ("desktop_unread", 1), + ("desktop_reminders", 1), + ] + ] + .concat()) + ); + } + + #[tokio::test] + async fn read_state_snapshots_recur_under_renewals_shorter_than_the_backstop() { + // Every renewal of 25 channels establishes (the live-only batch is + // cut by design) and repairs unreads without a snapshot; that must + // not hold off the snapshot's backstop. + let toml = "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 2.0\nrenewal_secs = 1.0"; + let out = drive(solo(25, toml), Duration::ZERO, |o| { + o.count("desktop_read_state") >= 2 && o.count("ws_connect_auth") >= 3 + }) + .await; + no_errors(&out); + assert!(out.count("desktop_unread") > out.count("desktop_read_state")); + } + + #[tokio::test] + async fn a_replacement_past_its_deadline_is_closed_and_the_next_tick_retries() { + let mut rig = solo( 25, - "mode = \"legacy\"\ntick_secs = 1.0", + "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 1.0", + ); + rig.req_timeout = Duration::from_millis(300); + rig.plan = Arc::new(|_, id| { + let late = if id == "batch-2-0" { + Duration::from_secs(1) + } else { + Duration::ZERO + }; + (0, late) + }); + let out = drive(rig, Duration::ZERO, |o| o.saw("CLOSE", "batch-1-0")).await; + assert_eq!(out.errors("desktop_live_eose"), 1); + // The late replacement is closed; the next tick replaces the batch. + let closed = out + .at("CLOSE", "batch-2-0") + .expect("late replacement closed"); + let next = out.at("REQ", "batch-3-0").expect("next tick replaced"); + let retired = out.at("CLOSE", "batch-1-0").expect("old batch closed"); + assert!(closed < next && next < retired, "{:?}", out.frames); + // One batch wire stays open. + let opened = |verb: &str| { + out.frames + .iter() + .filter(|(_, v, id)| v == verb && id.starts_with("batch-")) + .count() + }; + assert_eq!(opened("REQ") - opened("CLOSE"), 1, "{:?}", out.frames); + } + + #[tokio::test] + async fn a_failed_read_retries_on_the_next_tick_and_a_good_one_does_not() { + let mut rig = solo(25, "mode = \"current\"\ntick_secs = 1.0"); + rig.fail = &["roster", "unread"]; + let out = drive(rig, Duration::from_millis(3_500), |o| { + o.count("desktop_roster") == 1 && o.count("desktop_unread") == 1 + }) + .await; + let ops = &out.ops; + assert_eq!( + (out.count("desktop_roster"), out.errors("desktop_roster")), + (1, 1), + "{ops:?}" + ); + assert_eq!( + (out.count("desktop_unread"), out.errors("desktop_unread")), + (1, 1), + "{ops:?}" + ); + // The retry reread the snapshot with its repair; nothing else reran. + assert_eq!(out.count("desktop_read_state"), 2); + assert_eq!(out.count("desktop_reminders"), 1); + assert_eq!(out.count("desktop_profiles"), 1); + } + + #[tokio::test] + async fn legacy_ticks_reread_and_replace_batches_every_tick() { + let out = drive( + rig(2, 25, "mode = \"legacy\"\ntick_secs = 1.0"), Duration::ZERO, |o| o.count("desktop_roster") >= 4 && o.saw("CLOSE", "batch-1-2"), ) .await; + no_errors(&out); // Each replacement is sent before the batch it replaces is closed. assert!(out.saw("REQ", "batch-2-0") && out.saw("CLOSE", "batch-1-0")); assert!(out.count("desktop_read_state") >= 4); @@ -1426,12 +2425,12 @@ hold_secs = 1 #[tokio::test] async fn current_ticks_wait_for_the_backstop() { let before = drive( - 25, - "mode = \"current\"\ntick_secs = 1.0", + rig(2, 25, "mode = \"current\"\ntick_secs = 1.0"), Duration::from_millis(2500), |o| o.count("desktop_thread_window") == 2, ) .await; + no_errors(&before); assert_eq!( before.desktop(), with( @@ -1442,15 +2441,19 @@ hold_secs = 1 assert!(!before .frames .iter() - .any(|(_, id)| id.starts_with("batch-2"))); + .any(|(_, _, id)| id.starts_with("batch-2"))); let after = drive( - 25, - "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 1.0", + rig( + 2, + 25, + "mode = \"current\"\ntick_secs = 1.0\nbackstop_secs = 1.0", + ), Duration::ZERO, |o| o.count("desktop_roster") >= 4 && o.saw("CLOSE", "batch-1-0"), ) .await; + no_errors(&after); assert!(after.saw("REQ", "batch-2-0")); assert!(after.count("desktop_unread") >= 4); // A replacement is live-only: no replays after the connect's three. @@ -1460,12 +2463,12 @@ hold_secs = 1 #[tokio::test] async fn legacy_renewals_are_full_reconnects() { let out = drive( - 25, - "mode = \"legacy\"\nrenewal_secs = 1.0\n", + rig(2, 25, "mode = \"legacy\"\nrenewal_secs = 1.0\n"), Duration::ZERO, |o| o.count("desktop_nip11") >= 4 && o.count("desktop_roster") >= 4, ) .await; + no_errors(&out); assert!(out.count("ws_connect_auth") >= 4); // A reconnect keeps channel names and sidebar preferences. assert_eq!(out.count("desktop_metadata"), 2); @@ -1473,40 +2476,47 @@ hold_secs = 1 } #[tokio::test] - async fn current_renewals_carry_small_sessions_and_limit_large_ones() { - // More than ten channels: replays and channel catch-up again, no globals. + async fn current_renewals_carry_small_sessions_and_recover_channels_in_large_ones() { + // More than ten channels: the live-only batch is cut, so replays, + // channel catch-up and an unread repair again, but no global reads. let large = drive( - 25, - "mode = \"current\"\nrenewal_secs = 1.0\n", + rig(2, 25, "mode = \"current\"\nrenewal_secs = 1.0\n"), Duration::ZERO, |o| o.count("desktop_replay_eose") >= 12 && o.count("desktop_unread") >= 4, ) .await; + no_errors(&large); assert!(large.count("ws_connect_auth") >= 4); for op in [ "desktop_nip11", "desktop_roster", "desktop_metadata", "desktop_profiles", + "desktop_read_state", ] { assert_eq!(large.count(op), 2, "{op}"); } assert!(large.count("desktop_window") >= 16); - // Up to ten channels: subscriptions are re-sent and nothing is read. + // Up to ten channels: subscriptions are re-sent and nothing is read, + // but for one unread repair when a client's first renewal (jittered) + // came before its cold connection caught up. let small = drive( - 5, - "mode = \"current\"\nrenewal_secs = 1.0\n", + rig(2, 5, "mode = \"current\"\nrenewal_secs = 1.0\n"), Duration::ZERO, |o| o.count("desktop_live_eose") >= 12 && o.count("desktop_thread_window") == 2, ) .await; + no_errors(&small); assert!(small.count("ws_connect_auth") >= 4); let reads: HashMap<_, _> = small .desktop() .into_iter() - .filter(|(op, _)| *op != "desktop_live_eose") + .filter(|(op, _)| *op != "desktop_live_eose" && *op != "desktop_unread") .collect(); - assert_eq!(reads, cold_reads(2)); + let mut cold = cold_reads(2); + cold.remove("desktop_unread"); + assert_eq!(reads, cold); + assert!(small.count("desktop_unread") <= 4); } } diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index 8f31177b9e0..1a06d079568 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -143,28 +143,60 @@ On a **cold start** it: (`desktop_emoji`), read state (`desktop_read_state`), unread repair in chunks of 128 channels (`desktop_unread`), reminders (`desktop_reminders`), and for each of `windows` open channels a window - catch-up (`desktop_window`, up to five older pages while pages come back - full) and its notices (`desktop_notices`), and for each of `threads` open - threads a seeded root lookup and its first thread page - (`desktop_thread_root`, `desktop_thread_window`). - -Every `tick_secs` the client runs a recovery tick. In `legacy` each tick -replaces its channel subscriptions and rereads roster, read state and unread. -In `current` each of the three waits `backstop_secs` since it last ran, and -a replacement is live-only. A replacement is opened before the subscription -it replaces is closed. + catch-up (`desktop_window`) and its notices (`desktop_notices`), and for + each of `threads` open threads a seeded root lookup and its first thread + page (`desktop_thread_root`, `desktop_thread_window`). A window catch-up + reads the head page, then follows the cursor in the relay's window + bounds (kind 39006) for up to five older pages, until the bounds say + history is exhausted. + +`legacy` makes all of these reads as it connects. `current` reads only +NIP-11, names and sidebar preferences then, and the rest as each +subscription reaches its EOSE ("establishes"): profiles, statuses and emoji +with the profile subscription; roster, read state, unread repair and +reminders with the membership subscription; windows and threads with the +channel subscriptions. + +A subscription's replay is cut when it replays 500 events, or, for the +live-only subscription, always: it replays nothing itself. A subscription +that misses its EOSE deadline counts as cut, and so does one the previous +connection never caught up. When a `current` client reconnects, a profile +or membership subscription whose replay was not cut resumed every change, +so it skips its full read: profiles read nothing unless an earlier read is +still owed, and membership only repairs unreads, without the read-state +snapshot. A cut one reads as on a cold start. The +channel subscriptions always recover the windows and threads. + +Every `tick_secs` the client runs a recovery tick, on its own clock across +connections. In `legacy` each tick replaces its channel subscriptions and +rereads roster, read state and unread. In `current` each of the three waits +`backstop_secs` since it last ran: since the batches were last replaced, +since the roster was last read in full, and since a full unread repair (with +its read-state snapshot) was last asked for. A failed roster or unread read +is owed and runs again on the next tick; an owed profile read runs again +when the profile subscription next establishes. A replacement is live-only +and is opened before the subscription it replaces is closed. A replacement +that misses its EOSE deadline is closed, the subscription it would have +replaced stays, and the next tick tries again. Every `renewal_secs` (0 disables it; the first is jittered) the client renews its identity and reconnects. In `legacy` that is an ordinary reconnect, which rereads everything but channel names and sidebar preferences. In `current` -with up to ten channels the subscriptions are re-sent and nothing is read; -with more, the replays, unread repair, windows and threads run again. A -human's session end (`human.mean_session_secs`) is an ordinary reconnect in -both modes. Resumed subscriptions start 960 s before the previous session -ended, but not before 300 s before the client started. - -Reads that come due while earlier ones are still running wait for them, once -each, so a slow relay does not pile reads up behind it. +the subscriptions are carried over: none establishes while their replays +keep continuity, so nothing is read. Once any replay is cut, every +subscription establishes as on a reconnect. With more than ten channels the +live-only subscription is always cut, so the replays, windows, threads and +an unread repair run again. A human's session end +(`human.mean_session_secs`) is an ordinary reconnect in both modes. Resumed +subscriptions start 960 s before the previous session ended, but not before +300 s before the client started. + +Reads run in one task at a time per client. A read that comes due while the +same read is running or queued is not added again, so a slow relay does not +pile reads up behind it. + +The app also establishes the channel subscriptions when a tick's replacement +reaches its EOSE; the generator does not model that. ## Sharding From 15135c9db9008ff395a40cc2854f7cae914ffe78 Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Tue, 3 Nov 2026 13:09:42 -0500 Subject: [PATCH 4/4] fix(loadgen): keep snapshot debt, settle reads singly, don't resume timed-out routes Re-review fixes for the desktop client model: - The read-state snapshot is owed apart from the unread repair. A covered repair no longer settles it: a continuous membership route with a snapshot owed runs the full repair, and the tick retries it. - Each HTTP read leaves the running set as soon as it settles, and the read task starts queued reads as slots free (NIP-11 still first and alone). A failed read's retry no longer waits for an unrelated slow one. - A live route that misses its EOSE deadline is still recovered as cut, but is not caught up: the next connection cannot resume or carry it. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Luke Tornquist --- .../buzz-test-client/src/loadgen/desktop.rs | 327 ++++++++++++++---- perf/LOADGEN.md | 34 +- 2 files changed, 282 insertions(+), 79 deletions(-) diff --git a/crates/buzz-test-client/src/loadgen/desktop.rs b/crates/buzz-test-client/src/loadgen/desktop.rs index cf2ca966fd8..2575d16061e 100644 --- a/crates/buzz-test-client/src/loadgen/desktop.rs +++ b/crates/buzz-test-client/src/loadgen/desktop.rs @@ -327,6 +327,10 @@ pub(super) fn connect_fetches( pub(super) struct Owed { pub(super) roster: bool, pub(super) profiles: bool, + /// The read-state snapshot, owed until a snapshot read succeeds. Only a + /// full repair reads it, so a covered repair never settles it. + pub(super) snapshot: bool, + /// The unread repair of the channels. pub(super) unread: bool, } @@ -334,8 +338,8 @@ pub(super) struct Owed { /// global route replayed every change since the last connection, so it /// skips its full read unless one is still owed: profiles resume their /// statuses and emoji, and membership resumes read state and reminders and -/// only repairs unreads. A channel route recovers the open windows and -/// threads. +/// only repairs unreads, unless the read-state snapshot is owed, which takes +/// a full repair. A channel route recovers the open windows and threads. pub(super) fn establish_fetches( route: Route, continuous: bool, @@ -352,8 +356,10 @@ pub(super) fn establish_fetches( if !continuous || owed.roster { reads.push(Fetch::Roster); } - if continuous { + if continuous && !owed.snapshot { reads.push(COVERED_REPAIR); + } else if continuous { + reads.push(FULL_REPAIR); } else { reads.extend([FULL_REPAIR, Fetch::Reminders]); } @@ -378,6 +384,9 @@ pub(super) struct Establishment { /// Routes that reached EOSE while carried, with their continuity. held: Vec<(Route, bool)>, done: Vec, + /// Routes established only because their EOSE never came: recovered, + /// but not live, so not caught up. + timed_out: Vec, } impl Establishment { @@ -393,12 +402,12 @@ impl Establishment { carried: situation == Situation::Renewal, held: Vec::new(), done: Vec::new(), + timed_out: Vec::new(), } } - /// `route` reached its EOSE, its replay `cut` or not. A route that missed - /// its deadline counts as cut. Returns the routes that establish now, - /// each with its continuity. + /// `route` reached its EOSE, its replay `cut` or not. Returns the routes + /// that establish now, each with its continuity. pub(super) fn eose(&mut self, route: Route, cut: bool) -> Vec<(Route, bool)> { if route == Route::Replay { return Vec::new(); @@ -424,11 +433,24 @@ impl Establishment { now } - /// The routes this connection has caught up: established, or carried - /// with their continuity kept. + /// `route` missed its EOSE deadline. It is recovered as if its replay + /// were cut, but only an EOSE proves a route live: it is not caught up, + /// so the next connection cannot resume or carry it. + pub(super) fn timeout(&mut self, route: Route) -> Vec<(Route, bool)> { + if route != Route::Replay { + self.timed_out.push(route); + } + self.eose(route, true) + } + + /// The routes this connection has caught up: established at their EOSE, + /// or carried with their continuity kept. pub(super) fn caught_up(&self) -> Vec { let held = self.held.iter().map(|(route, _)| *route); - self.done.iter().copied().chain(held).collect() + let done = self.done.iter().copied(); + done.filter(|route| !self.timed_out.contains(route)) + .chain(held) + .collect() } } @@ -467,7 +489,7 @@ pub(super) fn tick(mode: DesktopMode, backstop: Duration, clocks: Clocks, owed: DesktopMode::Current => Due { batches: stale(clocks.batches), roster: owed.roster || stale(clocks.roster), - unread: owed.unread || stale(clocks.unread), + unread: owed.unread || owed.snapshot || stale(clocks.unread), }, } } @@ -749,7 +771,22 @@ impl Reader { self.query(Op::DesktopEmoji, filter).await; } Fetch::Unread { snapshot } => { - let repaired = self.repair(snapshot).await; + // The snapshot and the channel repair settle separately: + // a repair that succeeds without the snapshot leaves it owed. + if snapshot { + let filter = json!([{ + "kinds": [30078], "authors": [viewer], "read_state_snapshot": 1, + }]); + let read = self.query(Op::DesktopReadState, filter).await.is_some(); + let mut ledger = lock(&self.ledger); + ledger.owed.snapshot = !read; + if !read { + // The repair waits for the snapshot it builds on. + ledger.owed.unread = true; + return; + } + } + let repaired = self.repair().await; lock(&self.ledger).owed.unread = !repaired; } Fetch::Reminders => { @@ -776,17 +813,9 @@ impl Reader { true } - /// An unread repair, after the read-state snapshot when `snapshot`. The - /// repair reads its channels one chunk after another, as the app does. - /// Whether it all succeeded. - async fn repair(&self, snapshot: bool) -> bool { - if snapshot { - let filter = - json!([{"kinds": [30078], "authors": [self.viewer], "read_state_snapshot": 1}]); - if self.query(Op::DesktopReadState, filter).await.is_none() { - return false; - } - } + /// An unread repair of the channels, one chunk after another, as the + /// app does. Whether it all succeeded. + async fn repair(&self) -> bool { let now = unix_now(); let mark = lock(&self.unread_mark).replace(now); for ids in self.channels.chunks(UNREAD_CHANNELS) { @@ -893,31 +922,18 @@ fn lock(mutex: &std::sync::Mutex) -> std::sync::MutexGuard<'_, T> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// Runs `fetches`: NIP-11 first, since notices and window bounds need the -/// relay's pubkey, then the rest at most [`HTTP_CONCURRENCY`] at a time. -async fn run_fetches(reader: Arc, fetches: Vec) { - if fetches.contains(&Fetch::Nip11) { - reader.nip11().await; - } - stream::iter(fetches.into_iter().filter(|f| *f != Fetch::Nip11)) - .map(|fetch| { - let reader = reader.clone(); - async move { reader.fetch(fetch).await } - }) - .buffer_unordered(HTTP_CONCURRENCY) - .collect::>() - .await; -} - -/// A client's HTTP reads: one task at a time, so reads never pile up behind -/// a slow relay. Reads that come due while it runs are queued for its next -/// round; one whose work is already running or queued is not added again. -/// Dropping it (the client was cancelled) aborts the task, so its unfinished -/// reads are recorded. +/// A client's HTTP reads, run by one task at most [`HTTP_CONCURRENCY`] at a +/// time, so reads never pile up behind a slow relay. A read that comes due +/// while the same work is running or queued is not added again; a read +/// leaves the running set as soon as it settles, so a failed one can be +/// retried while others are still under way. Dropping it (the client was +/// cancelled) aborts the task, so its unfinished reads are recorded. #[derive(Default)] struct Fetcher { task: Option>, work: Arc>, + /// Wakes the task for newly queued reads. + wake: Arc, } /// A [`Fetcher`]'s reads, shared with its task. @@ -929,6 +945,26 @@ struct Work { queued: Vec, } +impl Work { + /// The next queued read that may start beside the `running` ones, taken + /// off the queue. NIP-11 runs alone and before anything queued with it, + /// since notices and window bounds need the relay's pubkey. + fn next(&mut self) -> Option { + if self.running.len() >= HTTP_CONCURRENCY || self.running.contains(&Fetch::Nip11) { + return None; + } + let i = match self.queued.iter().position(|f| *f == Fetch::Nip11) { + Some(_) if !self.running.is_empty() => return None, + Some(i) => i, + None if self.queued.is_empty() => return None, + None => 0, + }; + let fetch = self.queued.remove(i); + self.running.push(fetch); + Some(fetch) + } +} + impl Fetcher { fn start(&mut self, reader: &Arc, fetches: Vec) { let mut work = lock(&self.work); @@ -941,29 +977,20 @@ impl Fetcher { work.queued.retain(|have| !fetch.covers(*have)); work.queued.push(fetch); } - if work.active || work.queued.is_empty() { + if work.queued.is_empty() { + return; + } + if work.active { + self.wake.notify_one(); return; } work.active = true; drop(work); - let (reader, work) = (reader.clone(), self.work.clone()); - self.task = Some(tokio::spawn(async move { - loop { - let fetches = { - let mut work = lock(&work); - work.running = std::mem::take(&mut work.queued); - if work.running.is_empty() { - work.active = false; - return; - } - work.running.clone() - }; - run_fetches(reader.clone(), fetches).await; - } - })); + let task = drain(reader.clone(), self.work.clone(), self.wake.clone()); + self.task = Some(tokio::spawn(task)); } - /// Waits for the task under way, and the reads queued behind it. + /// Waits for the reads under way and queued. async fn finish(&mut self) { if let Some(task) = self.task.take() { let _ = task.await; @@ -971,6 +998,41 @@ impl Fetcher { } } +/// A [`Fetcher`]'s task: starts queued reads as they may start, until none +/// are running or queued. +async fn drain( + reader: Arc, + work: Arc>, + wake: Arc, +) { + let mut jobs = stream::FuturesUnordered::new(); + loop { + { + let mut work = lock(&work); + while let Some(fetch) = work.next() { + let reader = reader.clone(); + jobs.push(async move { + reader.fetch(fetch).await; + fetch + }); + } + if work.running.is_empty() { + work.active = false; + return; + } + } + tokio::select! { + Some(fetch) = jobs.next() => { + let mut work = lock(&work); + if let Some(i) = work.running.iter().position(|f| *f == fetch) { + work.running.remove(i); + } + } + () = wake.notified() => {} + } + } +} + impl Drop for Fetcher { fn drop(&mut self) { if let Some(task) = &self.task { @@ -1202,7 +1264,7 @@ impl Client { } } else { open.insert(id); - established.extend(establishment.eose(w.route, true)); + established.extend(establishment.timeout(w.route)); } } @@ -1451,6 +1513,7 @@ mod tests { const NO_OWED: Owed = Owed { roster: false, profiles: false, + snapshot: false, unread: false, }; @@ -1510,6 +1573,12 @@ mod tests { let due = tick(DesktopMode::Current, backstop, fresh, owed); assert!(!due.batches); assert_eq!(tick_fetches(due), [Fetch::Roster, FULL_REPAIR]); + let snapshot = Owed { + snapshot: true, + ..NO_OWED + }; + let due = tick(DesktopMode::Current, backstop, fresh, snapshot); + assert_eq!(tick_fetches(due), [FULL_REPAIR]); } #[test] @@ -1561,6 +1630,12 @@ mod tests { est(Route::Membership, true, roster_owed), [Fetch::Roster, COVERED_REPAIR] ); + // A continuous route cannot settle an owed snapshot without reading it. + let snapshot_owed = Owed { + snapshot: true, + ..NO_OWED + }; + assert_eq!(est(Route::Membership, true, snapshot_owed), [FULL_REPAIR]); assert_eq!(est(Route::Channels, true, NO_OWED), channels); assert_eq!(est(Route::Channels, false, NO_OWED), channels); assert!(est(Route::Replay, false, NO_OWED).is_empty()); @@ -1620,6 +1695,15 @@ mod tests { [(Route::Membership, true)] ); + // A timed-out route recovers as cut, but is not caught up. + let mut late = Establishment::new(Situation::Reconnect, &all); + assert_eq!( + late.timeout(Route::Membership), + [(Route::Membership, false)] + ); + assert_eq!(late.eose(Route::Profiles, false), [(Route::Profiles, true)]); + assert_eq!(late.caught_up(), [Route::Profiles]); + // A renewal before a route ever caught up reads it in full. let mut early = Establishment::new(Situation::Renewal, &all[..1]); assert_eq!( @@ -1795,9 +1879,13 @@ mod tests { (addr, frames) } - /// Reads the fake API fails once each, by name: `roster`, `unread`. + /// Reads the fake API fails once each, by name: `roster`, `read_state`, + /// `unread`, `reminders`. type FailOnce = &'static [&'static str]; + /// A read the fake API answers late, by name, and how late. + type Slow = Option<(&'static str, Duration)>; + fn signed(keys: &Keys, k: u32, content: &str, at: u64, tags: Vec) -> Value { let event = EventBuilder::new(kind(k), content) .custom_created_at(nostr::Timestamp::from(at)) @@ -1817,8 +1905,10 @@ mod tests { /// older aux event and a summary; its relay-signed bounds point after the /// last row. The page there is full too, and its bounds say history is /// exhausted. Any other window cursor is refused. The read-state snapshot - /// answers in its own shape; `fail` reads fail the first time. - async fn fake_api(run_id: &str, fail: FailOnce) -> (String, Shared2>) { + /// answers in its own shape; `fail` reads fail the first time, and a + /// `slow` read blocks its connection's handler (run such a test on the + /// multi-thread runtime). + async fn fake_api(run_id: &str, fail: FailOnce, slow: Slow) -> (String, Shared2>) { let owner = identity::keys(run_id, Role::Owner, 0); let relay = Keys::generate(); let root = Value::Array(vec![signed(&owner, 9, "lg-root 0", 2_000, vec![])]); @@ -1853,9 +1943,16 @@ mod tests { "roster" } else if filter["kinds"] == json!([9, 40002, 40008]) { "unread" + } else if filter.get("read_state_snapshot").is_some() { + "read_state" + } else if filter["kinds"] == json!([30300]) { + "reminders" } else { "" }; + if let Some((_, delay)) = slow.filter(|(slow, _)| *slow == name) { + std::thread::sleep(delay); + } if fail.contains(&name) && !failed.lock().unwrap().contains(&name) { failed.lock().unwrap().push(name); return (500, r#"{"error":"once"}"#.into()); @@ -2002,6 +2099,7 @@ hold_secs = 1 plan: Plan, drop_first: Option, fail: FailOnce, + slow: Slow, req_timeout: Duration, } @@ -2022,6 +2120,7 @@ hold_secs = 1 plan: instant(), drop_first: None, fail: &[], + slow: None, req_timeout: REQ_TIMEOUT, } } @@ -2030,7 +2129,7 @@ hold_secs = 1 /// them and returns what they did. Fails after 15 s without `done`. async fn drive(rig: Rig, min: Duration, done: impl Fn(&Outcome) -> bool) -> Outcome { let (ws, frames) = fake_ws_relay(rig.plan, rig.drop_first).await; - let (http, queries) = fake_api("d", rig.fail).await; + let (http, queries) = fake_api("d", rig.fail, rig.slow).await; let scenario = desktops(&ws, &http, rig.humans, rig.joined, &rig.desktop); let net = Net::new(&scenario).await.unwrap(); let stats = Arc::new(Stats::default()); @@ -2404,6 +2503,104 @@ hold_secs = 1 assert_eq!(out.count("desktop_profiles"), 1); } + #[tokio::test] + async fn an_owed_snapshot_survives_a_covered_repair() { + // The cold snapshot read fails. Before any tick, the connection + // drops and the membership route resumes with a complete replay: + // its repair must read the snapshot still owed. + let mut rig = solo(5, &format!("mode = \"current\"\n{QUIET}")); + rig.drop_first = Some(Duration::from_millis(700)); + rig.fail = &["read_state"]; + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("ws_connect_auth") >= 2 && o.count("desktop_live_eose") >= 6, + Duration::from_millis(800), + ), + ) + .await; + let ops = &out.ops; + assert_eq!( + ( + out.count("desktop_read_state"), + out.errors("desktop_read_state") + ), + (1, 1), + "{ops:?}" + ); + // The failed snapshot held the cold repair back; the resumed route + // repaired once and reread neither roster nor reminders. + assert_eq!(out.count("desktop_unread"), 1, "{ops:?}"); + assert_eq!(out.count("desktop_roster"), 1, "{ops:?}"); + assert_eq!(out.count("desktop_reminders"), 1, "{ops:?}"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_failed_read_retries_while_an_unrelated_slow_one_runs() { + // The cold roster read fails while reminders take 3 s; the tick at + // 1 s retries the roster without waiting for reminders. + let mut rig = solo(5, "mode = \"current\"\ntick_secs = 1.0"); + rig.fail = &["roster"]; + rig.slow = Some(("reminders", Duration::from_secs(3))); + let reminders = std::cell::Cell::new(None); + let out = drive(rig, Duration::ZERO, |o| { + let retried = o.count("desktop_roster") == 1; + if retried && reminders.get().is_none() { + reminders.set(Some(o.count("desktop_reminders"))); + } + retried + }) + .await; + assert_eq!( + reminders.get(), + Some(0), + "the roster retry waited for reminders" + ); + assert_eq!(out.errors("desktop_roster"), 1); + assert_eq!(out.count("desktop_reminders"), 1); + } + + #[tokio::test] + async fn a_timed_out_route_is_recovered_again_after_a_renewal() { + // The first connection's membership EOSE never comes: the route is + // recovered as cut, but it was never live. The renewal's complete + // replay therefore cannot carry it, and it recovers in full again. + // The first renewal is jittered; with this period it comes after the + // short deadline and before the test ends. + let mut rig = solo( + 5, + "mode = \"current\"\nrenewal_secs = 20.0\ntick_secs = 3600.0", + ); + rig.req_timeout = Duration::from_millis(100); + rig.plan = Arc::new(|conn, id| { + let withheld = conn == 0 && id == "membership"; + let delay = if withheld { + Duration::from_secs(60) + } else { + Duration::ZERO + }; + (0, delay) + }); + let out = drive( + rig, + Duration::ZERO, + settled( + |o| o.count("ws_connect_auth") >= 2 && o.count("desktop_live_eose") >= 5, + Duration::from_millis(800), + ), + ) + .await; + let ops = &out.ops; + assert_eq!(out.errors("desktop_live_eose"), 1, "{ops:?}"); + assert_eq!(out.count("ws_connect_auth"), 2, "{ops:?}"); + for op in ["desktop_roster", "desktop_read_state", "desktop_reminders"] { + assert_eq!(out.count(op), 2, "{op}: {ops:?}"); + } + // The routes that were live carried: no second profile read. + assert_eq!(out.count("desktop_profiles"), 1, "{ops:?}"); + } + #[tokio::test] async fn legacy_ticks_reread_and_replace_batches_every_tick() { let out = drive( diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index 1a06d079568..d0a9df95cb7 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -160,24 +160,28 @@ channel subscriptions. A subscription's replay is cut when it replays 500 events, or, for the live-only subscription, always: it replays nothing itself. A subscription that misses its EOSE deadline counts as cut, and so does one the previous -connection never caught up. When a `current` client reconnects, a profile -or membership subscription whose replay was not cut resumed every change, -so it skips its full read: profiles read nothing unless an earlier read is +connection never caught up. Only an EOSE catches a subscription up: one +that missed its deadline is recovered, but the next connection cannot +resume or carry it. When a `current` client reconnects, a profile or +membership subscription whose replay was not cut resumed every change, so +it skips its full read: profiles read nothing unless an earlier read is still owed, and membership only repairs unreads, without the read-state -snapshot. A cut one reads as on a cold start. The -channel subscriptions always recover the windows and threads. +snapshot, unless a snapshot read failed since, which a full repair retries. +A cut one reads as on a cold start. The channel subscriptions always +recover the windows and threads. Every `tick_secs` the client runs a recovery tick, on its own clock across connections. In `legacy` each tick replaces its channel subscriptions and rereads roster, read state and unread. In `current` each of the three waits `backstop_secs` since it last ran: since the batches were last replaced, since the roster was last read in full, and since a full unread repair (with -its read-state snapshot) was last asked for. A failed roster or unread read -is owed and runs again on the next tick; an owed profile read runs again -when the profile subscription next establishes. A replacement is live-only -and is opened before the subscription it replaces is closed. A replacement -that misses its EOSE deadline is closed, the subscription it would have -replaced stays, and the next tick tries again. +its read-state snapshot) was last asked for. A failed roster read, unread +repair or read-state snapshot is owed and runs again on the next tick; a +repair without the snapshot does not settle an owed snapshot. An owed +profile read runs again when the profile subscription next establishes. A +replacement is live-only and is opened before the subscription it replaces +is closed. A replacement that misses its EOSE deadline is closed, the +subscription it would have replaced stays, and the next tick tries again. Every `renewal_secs` (0 disables it; the first is jittered) the client renews its identity and reconnects. In `legacy` that is an ordinary reconnect, which @@ -191,9 +195,11 @@ an unread repair run again. A human's session end subscriptions start 960 s before the previous session ended, but not before 300 s before the client started. -Reads run in one task at a time per client. A read that comes due while the -same read is running or queued is not added again, so a slow relay does not -pile reads up behind it. +Each client runs its reads in one task, at most six at a time. A read that +comes due while the same read is running or queued is not added again, so a +slow relay does not pile reads up behind it. A read leaves the running set +as soon as it settles, so a failed one can retry while a slow one is still +under way. The app also establishes the channel subscriptions when a tick's replacement reaches its EOSE; the generator does not model that.