diff --git a/.env.example b/.env.example index 73d4d9f6eab..7c5a1996f57 100644 --- a/.env.example +++ b/.env.example @@ -41,6 +41,12 @@ PGDATABASE=buzz # Redis 7 # ----------------------------------------------------------------------------- REDIS_URL=redis://localhost:6379 + +# File-share authorization rollout (independent of NIP-FI). +# off: skip share checks; log-only (default): count would-deny/errors but allow; +# enforce: deny unauthorized reads. Authentication/community checks always apply. +# Read at startup: change deployment config and restart every relay pod. +BUZZ_MEDIA_READ_MODE=log-only # Max connections in the relay's shared Redis pool (default 16). # BUZZ_REDIS_POOL_SIZE=16 diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index c02086dfda2..9dafd9624cd 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -598,6 +598,9 @@ jobs: relay-e2e: name: Relay E2E + env: + BUZZ_MEDIA_READ_MODE: enforce + BUZZ_USAGE_METRICS_INTERVAL_SECS: "5" runs-on: ubuntu-latest timeout-minutes: 20 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.rust) @@ -643,7 +646,7 @@ jobs: # --no-fail-fast: without it cargo stops after the first failing binary, # so one broken case hides every later binary's result. run: | - cargo test -p buzz-test-client --no-fail-fast --test e2e_media --test e2e_media_extended --test e2e_media_video -- --ignored --nocapture + cargo test -p buzz-test-client --no-fail-fast --test e2e_media --test e2e_media_extended --test e2e_media_review --test e2e_media_video -- --ignored --nocapture env: RELAY_URL: ws://localhost:3000 RELAY_HTTP_URL: http://localhost:3000 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7ff26b4ef74..34f2afa2d5a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -504,6 +504,8 @@ shape is an explicit transition, not a flag day: never spans communities. Serving leases use the same application admission. Startup and deletion catalog validation tolerate the universal fence objects being absent; inventories still record live fences for older executors. + Desired-state bootstraps install only explicitly declared legacy fences. + New tenant-owned tables rely on application admission, without a fence trigger. - **Replica floor (transition foundation):** runtime now provides a shared replica-floor lock helper for compliant channel-event writers, and the writer probe handshake takes the exclusive counterpart before sampling `S`, scanning diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 4f91369d6df..9d7f396dbf5 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -100,6 +100,22 @@ enum Command { #[arg(long, default_value_t = 10_000_000)] max_objects: u64, }, + /// Record media shares for events stored before share recording existed. + /// + /// Run after every relay pod records shares and + /// before enforcing channel-scoped media reads. Historical references are + /// trusted only before each community's schema-recorded cutoff. + /// Restores uploader proof from surviving server media_uploaded audit rows; + /// files without that evidence still need an upload of the exact bytes. + /// Later references use current permissions, so any run can grant references + /// live recording previously refused if their authors have since gained access. + /// Accept the resulting lasting grants before the first backfill; otherwise + /// defer it. Each invocation restarts the scan and deduplicates existing rows. + BackfillMediaShares { + /// Events read and recorded per transaction. + #[arg(long, default_value_t = 1000, value_parser = clap::value_parser!(i64).range(1..))] + batch_size: i64, + }, /// Run the partition catalog audit using a read-only database session. PartitionAudit { /// Future months to include in the coverage check. @@ -193,6 +209,14 @@ async fn run(cli: Cli) -> Result { Ok(0) } Command::StorageSnapshot { max_objects } => cmd_storage_snapshot(max_objects).await, + Command::BackfillMediaShares { batch_size } => { + let added = connect_db() + .await? + .backfill_media_shares(batch_size) + .await?; + println!("Recorded {added} media shares."); + Ok(0) + } Command::PartitionAudit { months_ahead } => cmd_partition_audit(months_ahead).await, Command::AddMember { pubkey, role } => cmd_add_member(pubkey, role).await, Command::RemoveMember { pubkey, role } => cmd_remove_member(pubkey, role).await, diff --git a/crates/buzz-audit/src/action.rs b/crates/buzz-audit/src/action.rs index be7ccc35454..78acb15f2fc 100644 --- a/crates/buzz-audit/src/action.rs +++ b/crates/buzz-audit/src/action.rs @@ -28,6 +28,8 @@ pub enum AuditAction { RateLimitExceeded, /// A media file was uploaded via the Blossom endpoint. MediaUploaded, + /// A moderator received temporary access to a reported message's file. + ModerationMediaRead, } impl AuditAction { @@ -45,6 +47,7 @@ impl AuditAction { Self::AuthFailure => "auth_failure", Self::RateLimitExceeded => "rate_limit_exceeded", Self::MediaUploaded => "media_uploaded", + Self::ModerationMediaRead => "moderation_media_read", } } @@ -60,6 +63,7 @@ impl AuditAction { Self::AuthFailure, Self::RateLimitExceeded, Self::MediaUploaded, + Self::ModerationMediaRead, ]; } diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index e7669c874b0..0c89cd62c0f 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -24,6 +24,8 @@ pub mod git_perms; pub mod invite; /// Buzz kind number registry — custom event type constants. pub mod kind; +/// Which media blobs an event references, for channel-scoped media access. +pub mod media; /// Network utilities — SSRF-safe IP classification. pub mod network; /// NIP-10 thread-marker parsing — shared `root`/`reply` marker resolver. diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs new file mode 100644 index 00000000000..1685ae5f566 --- /dev/null +++ b/crates/buzz-core/src/media.rs @@ -0,0 +1,151 @@ +//! Which media blobs an event shares, for channel-scoped media access. + +use std::collections::BTreeSet; + +use crate::kind::{ + event_is_shared, is_author_only_event_kind, is_shared_gated_kind, KIND_AGENT_ENGRAM, + P_GATED_KINDS, +}; + +/// Hashes of every blob `event` shares into its channel, or with the whole +/// community when it is channel-less: each local media path `/media/` +/// anywhere in its content or tags (attachments, posters, thumbnails, link +/// previews, avatars, custom emoji, pasted links). +/// +/// Empty for an event only its author or tagged recipients can read, since +/// referencing a blob there shows it to no one new. +pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { + let kind = u32::from(event.kind.as_u16()); + if is_author_only_event_kind(event) + || P_GATED_KINDS.contains(&kind) + || kind == KIND_AGENT_ENGRAM + || (is_shared_gated_kind(kind) && !event_is_shared(event)) + { + return Vec::new(); + } + // JSON content may escape `/` as `\/`; scan the path as clients will see it. + let content = event.content.replace("\\/", "/"); + let mut hashes: BTreeSet = media_path_hashes(&content).collect(); + for tag in event.tags.iter() { + hashes.extend( + tag.as_slice() + .iter() + .flat_map(|part| media_path_hashes(part)), + ); + } + hashes.into_iter().collect() +} + +/// The media blob used as a kind:0 profile's picture. A registered agent may +/// publish this one field using its owner's upload proof; arbitrary content +/// and tags must not inherit that authority. +pub fn profile_picture_blob_hash(event: &nostr::Event) -> Option { + if event.kind != nostr::Kind::Metadata { + return None; + } + let profile: serde_json::Value = serde_json::from_str(&event.content).ok()?; + let hash = media_path_hashes(profile.get("picture")?.as_str()?).next(); + hash +} + +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) +} + +/// The hash in each `/media/` path in `text`, where the hash is not +/// followed by another hex digit. +fn media_path_hashes(text: &str) -> impl Iterator + '_ { + text.match_indices("/media/").filter_map(|(at, path)| { + let rest = &text[at + path.len()..]; + let hash = rest.get(..64)?; + let bounded = !rest[64..].starts_with(|c: char| matches!(c, '0'..='9' | 'a'..='f')); + (is_sha256_hex(hash) && bounded).then(|| hash.to_owned()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kind::{KIND_EVENT_REMINDER, KIND_PERSONA}; + use nostr::{EventBuilder, Keys, Kind, Tag}; + + fn media_url(sha256: &str, ext: &str) -> String { + format!("https://relay.example/media/{sha256}.{ext}") + } + + fn signed(builder: EventBuilder) -> nostr::Event { + builder.sign_with_keys(&Keys::generate()).unwrap() + } + + #[test] + fn shares_every_local_media_path() { + let (video, poster, embed) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); + let event = signed( + EventBuilder::new( + Kind::TextNote, + format!("look ![image]({})", media_url(&embed, "png")), + ) + .tags([ + Tag::parse([ + "imeta", + &format!("url {}", media_url(&video, "mp4")), + &format!("x {video}"), + &format!("image /media/{poster}.jpg"), + ]) + .unwrap(), + Tag::parse([ + "r", + &format!("https://elsewhere.example/{}", "e".repeat(64)), + ]) + .unwrap(), + Tag::parse(["r", &format!("/media/{}0", "f".repeat(64))]).unwrap(), + ]), + ); + assert_eq!(shared_blob_hashes(&event), vec![video, poster, embed]); + } + + #[test] + fn shares_avatars_emoji_and_shared_personas_only() { + let (avatar, emoji) = ("c".repeat(64), "d".repeat(64)); + let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar, "jpg")); + let profile = signed(EventBuilder::new(Kind::Metadata, &content)); + assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); + let escaped = signed(EventBuilder::new( + Kind::Metadata, + content.replace('/', "\\/"), + )); + assert_eq!(shared_blob_hashes(&escaped), vec![avatar.clone()]); + + let emoji_set = signed( + EventBuilder::new(Kind::from(30030), "").tags([Tag::parse([ + "emoji", + "party", + &media_url(&emoji, "png"), + ]) + .unwrap()]), + ); + assert_eq!(shared_blob_hashes(&emoji_set), vec![emoji]); + + let persona = |shared: bool| { + let mut tags = vec![Tag::parse(["d", "helper"]).unwrap()]; + if shared { + tags.push(Tag::parse(["shared", "true"]).unwrap()); + } + signed(EventBuilder::new(Kind::from(KIND_PERSONA as u16), &content).tags(tags)) + }; + assert_eq!(shared_blob_hashes(&persona(true)), vec![avatar]); + assert!(shared_blob_hashes(&persona(false)).is_empty()); + } + + #[test] + fn author_only_events_share_nothing() { + let reminder = signed( + EventBuilder::new(Kind::from(KIND_EVENT_REMINDER as u16), "").tags([Tag::parse([ + "imeta", + &format!("url /media/{}.jpg", "a".repeat(64)), + ]) + .unwrap()]), + ); + assert!(shared_blob_hashes(&reminder).is_empty()); + } +} diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f495983a28d..7fa87929f45 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -66,10 +66,10 @@ pub(crate) use runtime::{ }; pub use store::{ admin_moderation, allowlist, api_token, archived_identities, artifact, channel, - channel_members, community, deletion, dm, event, feed, git_repo, moderation, nip_fi_binding, - operator_listener, partition, personal_read, product_feedback, push, reaction, read_state, - relay_admin_actions, relay_invite, relay_members, relay_operators, reminder, replaceable, - storage_accounting, thread, thread_window, usage, user, workflow, + channel_members, community, deletion, dm, event, feed, git_repo, media_cleanup, moderation, + nip_fi_binding, operator_listener, partition, personal_read, product_feedback, push, reaction, + read_state, relay_admin_actions, relay_invite, relay_members, relay_operators, reminder, + replaceable, storage_accounting, thread, thread_window, usage, user, workflow, }; pub use allowlist::AllowlistEntry; diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index af877332529..12f89c0504c 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -14,6 +14,10 @@ use crate::Result; static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations"); +#[cfg(test)] +#[path = "tests/schema_fence_bootstrap_postgres_tests.rs"] +mod schema_fence_bootstrap_postgres_tests; + /// Run all pending Buzz database migrations. /// /// The entire run holds the exclusive [`SCHEMA_DESTRUCTION_LOCK_KEY`] session @@ -769,7 +773,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 67); + assert_eq!(migrations.len(), 68); assert_eq!(migrations[64].version, 65); let retire_ttl = strip_sql_comments(migrations[64].sql.as_str()); for statement in [ @@ -806,6 +810,16 @@ mod postgres_tests { .sql .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); + assert_eq!(migrations[67].version, 68); + assert!(strip_sql_comments(migrations[67].sql.as_str()) + .trim_start() + .starts_with("SET LOCAL lock_timeout = '5s';")); + for table in ["media_uploaders", "media_shares"] { + assert!(migrations[67] + .sql + .as_str() + .contains(&format!("CREATE TABLE {table}"))); + } assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql diff --git a/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs new file mode 100644 index 00000000000..67eb9a9a990 --- /dev/null +++ b/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs @@ -0,0 +1,39 @@ +use sqlx::PgPool; + +/// A fresh bootstrap keeps the declared backstops but never discovers new +/// community tables and silently attaches a trigger to them. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_bootstrap_keeps_only_explicit_legacy_fences() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + sqlx::raw_sql("CREATE TABLE new_tenant_probe (community_id UUID NOT NULL)") + .execute(&pool) + .await + .unwrap(); + // Only checked-in schema files contribute SQL to this test bootstrap. + sqlx::raw_sql(sqlx::AssertSqlSafe( + crate::test_support::desired_state_schema_sql(), + )) + .execute(&pool) + .await + .unwrap(); + + for (table, expected) in [ + ("new_tenant_probe", false), + ("events", true), + ("users", true), + ] { + let fenced: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_trigger \ + WHERE tgrelid = $1::regclass \ + AND tgfoid = 'enforce_community_write_fence()'::regprocedure)", + ) + .bind(table) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(fenced, expected, "unexpected fence on {table}"); + } +} diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 28e64ff61c8..111016ede1a 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -556,13 +556,8 @@ impl LockedMemberSnapshot { "member snapshot event id already exists".into(), )); } - crate::store::event_follow_up::after_admitted_insert( - &mut self.tx, - event.id.as_bytes().as_slice(), - kind, - Some(channel_id), - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut self.tx, event, Some(channel_id)) + .await?; crate::insert_mentions_in_transaction(&mut self.tx, event, Some(channel_id)).await?; Ok(( buzz_core::StoredEvent::with_received_at( diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index c5d62451278..c236d4158c4 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -90,6 +90,8 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "media_shares", + "media_uploaders", "moderation_actions", "moderation_reports", "parameterized_event_watermarks", @@ -156,6 +158,8 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "audit_log", "community_bans", "community_nip_fi_issuers", + "media_shares", + "media_uploaders", "pubkey_allowlist", "relay_members", "users", @@ -4430,7 +4434,7 @@ mod postgres_tests { }) .await .expect("connect isolated migration DB"); - crate::migration::run_migrations_through(&db.pool, 66) + crate::migration::run_migrations_through(&db.pool, 68) .await .expect("migrate with fence objects present"); let store = db.deletion_store(); @@ -4458,6 +4462,8 @@ DROP TRIGGER IF EXISTS community_write_fence_event_mentions ON event_mentions; DROP TRIGGER IF EXISTS community_write_fence_events ON events; DROP TRIGGER IF EXISTS community_write_fence_git_repo_names ON git_repo_names; DROP TRIGGER IF EXISTS community_write_fence_join_policy_acceptances ON join_policy_acceptances; +DROP TRIGGER IF EXISTS community_write_fence_media_shares ON media_shares; +DROP TRIGGER IF EXISTS community_write_fence_media_uploaders ON media_uploaders; DROP TRIGGER IF EXISTS community_write_fence_moderation_actions ON moderation_actions; DROP TRIGGER IF EXISTS community_write_fence_moderation_reports ON moderation_reports; DROP TRIGGER IF EXISTS community_write_fence_parameterized_event_watermarks ON parameterized_event_watermarks; @@ -4501,14 +4507,13 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); store.validate_catalog().await.expect("deletion catalog"); let inventory = store.inventory_schema(community).await.expect("inventory"); assert_eq!(inventory.scoped_tables.len(), EXPECTED_SCOPED_TABLES.len()); - assert_eq!( - inventory.fenced_tables.len(), - if objects_present { - EXPECTED_SCOPED_TABLES.len() - } else { - 0 - } - ); + // Tables added after fence retirement are admitted in the app and + // carry no fence, so only legacy tables are fenced. + assert_eq!(inventory.fenced_tables.is_empty(), !objects_present); + assert!(inventory + .fenced_tables + .iter() + .all(|table| inventory.scoped_tables.contains(table))); let lease = store .acquire_serving_write_lease( community, @@ -8489,6 +8494,31 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("count relay admin rows") } + async fn seed_unfenced_media(db: &Db, community: Uuid) -> nostr::Event { + let community = CommunityId::from_uuid(community); + let keys = nostr::Keys::generate(); + let sha = "ab".repeat(32); + let store = db.deletion_store(); + let lease = store + .acquire_serving_write_lease( + community, + "media_upload", + "purge-test", + Duration::from_secs(60), + ) + .await + .unwrap(); + db.record_media_uploader(&lease, &sha, keys.public_key().as_bytes()) + .await + .unwrap(); + store.release_serving_write_lease(&lease).await.unwrap(); + let event = nostr::EventBuilder::new(nostr::Kind::TextNote, format!("/media/{sha}")) + .sign_with_keys(&keys) + .unwrap(); + db.insert_event(community, &event, None).await.unwrap(); + event + } + /// Report-backed staff actions FK `moderation_reports`, and outbox rows FK /// their action. Purge must delete the community's report-backed and direct /// rows in FK order and leave other communities' rows alone. @@ -8498,6 +8528,21 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); let (db, store) = store().await; let (request, inventory) = inventoried_request(&db, &store).await; let community = *request.community_id.as_uuid(); + let media = seed_unfenced_media(&db, community).await; + let backstops: i64 = sqlx::query_scalar( + "SELECT (SELECT count(*) FROM pg_constraint WHERE contype = 'f' \ + AND conrelid IN ('media_shares'::regclass, 'media_uploaders'::regclass)) \ + + (SELECT count(*) FROM pg_trigger \ + WHERE tgrelid IN ('media_shares'::regclass, 'media_uploaders'::regclass) \ + AND tgfoid = 'enforce_community_write_fence()'::regprocedure)", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!( + backstops, 0, + "new media tables rely on application admission" + ); seed_relay_admin_rows(&db, community).await; let bystander = db .ensure_configured_community(&format!("bystander-{}.example", Uuid::new_v4().simple())) @@ -8507,6 +8552,7 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .as_uuid() .to_owned(); seed_relay_admin_rows(&db, bystander).await; + seed_unfenced_media(&db, bystander).await; let live = store .inventory_schema(request.community_id) .await @@ -8524,6 +8570,10 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("claim") .expect("won claim"); store.begin_quiescing(&claim.lease).await.expect("quiesce"); + assert!(db + .insert_event(request.community_id, &media, None) + .await + .is_err()); let generation = store.fence(&claim.lease).await.expect("fence"); let token = LeaseToken { fence_generation: Some(generation), @@ -8540,6 +8590,18 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("bindings"); let deleted = store.purge_postgres(&token).await.expect("purge postgres"); + for table in ["media_shares", "media_uploaders"] { + assert_eq!(deleted.get(table), Some(&1)); + } + let media_rows: (i64, i64) = sqlx::query_as( + "SELECT (SELECT count(*) FROM media_shares WHERE community_id = $1), \ + (SELECT count(*) FROM media_uploaders WHERE community_id = $1)", + ) + .bind(bystander) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(media_rows, (1, 1), "another community's media must survive"); assert_eq!(deleted.get("relay_admin_outbox"), Some(&2)); assert_eq!(deleted.get("relay_admin_actions"), Some(&2)); assert_eq!(relay_admin_row_counts(&db, community).await, (0, 0)); diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index c279e4831d0..3aba787a523 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -515,13 +515,7 @@ async fn insert_event_on( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(( @@ -1905,13 +1899,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; if let Some(ref meta) = thread_meta { let broadcast_val: bool = meta.broadcast; diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index d2853b4d89d..9079e496539 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -1,7 +1,10 @@ //! App-owned follow-up work for a newly inserted `events` row. //! -//! Two pieces of work follow every durable event insert: +//! Three pieces of work follow every durable event insert: //! +//! - **Media grants.** References grant access only after their author passes +//! the sharing check. Replacements record these before retiring the old head, +//! inside their savepoint; delivery is staged only after it succeeds. //! - **Push match enqueue.** Eligible event IDs are staged in memory. Commit //! submits them to a bounded best-effort worker using a dedicated connection. //! Push errors, delays, overload, and shutdown can drop wakes, never messages. @@ -37,18 +40,53 @@ const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; /// Follow-up for an event row inserted in an admitted transaction. /// /// Call only when the INSERT affected a row, after releasing any savepoint -/// that may roll it back. Stages push work for after commit and records TTL work. +/// that may roll it back. Stages push work for after commit, records TTL work, +/// and shares the blobs the event references into its channel in the same +/// transaction, so a stored reference always carries its share. pub(crate) async fn after_admitted_insert( tx: &mut AdmittedTx, - event_id: &[u8], - kind: i32, + event: &nostr::Event, channel_id: Option, ) -> Result<()> { - tx.record_push_event(event_id, kind); - tx.record_channel_event(channel_id, kind); + record_admitted_media(tx, event, channel_id).await?; + stage_admitted_insert(tx, event, channel_id); + Ok(()) +} + +/// Record media inside a replacement's savepoint, before retiring its old head. +/// This preserves authority carried by a reference the new head keeps. +pub(crate) async fn record_admitted_media( + tx: &mut AdmittedTx, + event: &nostr::Event, + channel_id: Option, +) -> Result<()> { + let hashes = buzz_core::media::shared_blob_hashes(event); + if !hashes.is_empty() { + let community = tx.community(); + crate::store::media_reference::record_media_shares( + tx.conn(), + community, + &hashes, + event, + channel_id, + ) + .await?; + } Ok(()) } +/// Stage delivery only after the insertion's savepoint has succeeded. +/// Call after [`record_admitted_media`] when splitting the two phases. +pub(crate) fn stage_admitted_insert( + tx: &mut AdmittedTx, + event: &nostr::Event, + channel_id: Option, +) { + let kind = i32::from(event.kind.as_u16()); + tx.record_push_event(event.id.as_bytes(), kind); + tx.record_channel_event(channel_id, kind); +} + /// The channel whose TTL an event of `kind` in `channel_id` refreshes, if any. pub(crate) fn ttl_refresh_channel(channel_id: Option, kind: i32) -> Option { channel_id.filter(|_| kind != KIND_CHANNEL_CREATE) diff --git a/crates/buzz-db/src/store/media_cleanup.rs b/crates/buzz-db/src/store/media_cleanup.rs new file mode 100644 index 00000000000..a07c51b52b4 --- /dev/null +++ b/crates/buzz-db/src/store/media_cleanup.rs @@ -0,0 +1,120 @@ +//! Bounded, tenant-admitted reclamation of grants whose sources are gone. + +use std::collections::BTreeMap; + +use buzz_datastore_tracing::datastore_span; +use sqlx::FromRow; +use uuid::Uuid; + +use crate::{Db, Result}; + +/// Volatile primary-key position for the periodic media-share reaper. +/// Restarting a scan is safe: cleanup never creates an authorization grant. +#[derive(Clone, Debug, FromRow)] +pub struct MediaShareCleanupCursor { + community_id: Uuid, + sha256: String, + event_id: Vec, +} + +/// Committed cleanup and the next scan position; `None` starts another cycle. +pub struct MediaShareCleanupBatch { + /// Number of inactive grant rows removed. + pub removed: u64, + /// Continue after this key, or wrap to the beginning on the next tick. + pub next_cursor: Option, +} + +impl Db { + /// Inspect at most `batch_size` grants in primary-key order and remove + /// inactive ones under each community's write admission. + /// + /// Keep the first channel grant per blob, even if inactive: otherwise + /// removing its last historical channel row would restore uploader preview + /// access. This bounds dead channel history to one row per blob; public + /// profile/emoji grants need no such marker. Live sources are never removed. + /// Tenant-local errors retry in a later scan cycle, as do skipped locks. + #[datastore_span(name = "reap_inactive_media_shares", system = "postgresql")] + pub async fn reap_inactive_media_shares( + &self, + after: Option<&MediaShareCleanupCursor>, + batch_size: u16, + ) -> Result { + if !(1..=1000).contains(&batch_size) { + return Err(crate::DbError::InvalidData( + "media cleanup batch size must be between 1 and 1000".into(), + )); + } + let mut conn = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; + // Intentional fleet-wide candidate discovery, bounded by the leading + // primary key. It reads only keys; every mutation is admitted below. + let rows = sqlx::query_as::<_, MediaShareCleanupCursor>( + "SELECT community_id, sha256, event_id FROM media_shares \ + WHERE (community_id, sha256, event_id) > ($1, $2, $3) \ + ORDER BY community_id, sha256, event_id LIMIT $4", + ) + .bind(after.map_or(Uuid::nil(), |key| key.community_id)) + .bind(after.map_or("", |key| key.sha256.as_str())) + .bind(after.map_or(&[][..], |key| key.event_id.as_slice())) + .bind(i64::from(batch_size)) + .fetch_all(&mut *conn) + .await?; + drop(conn); + let next_cursor = (rows.len() == usize::from(batch_size)) + .then(|| rows.last().cloned()) + .flatten(); + let mut groups = BTreeMap::, Vec>)>::new(); + for row in rows { + let (hashes, ids) = groups.entry(row.community_id).or_default(); + hashes.push(row.sha256); + ids.push(row.event_id); + } + let groups = &groups; + let removed = crate::sweep_admitted_communities( + &self.pool, + groups.keys().copied(), + crate::observability::WriterOperation::Maintenance, + "media_share_reap", + |community, mut tx| async move { + sqlx::raw_sql("SET LOCAL lock_timeout = '1s'; SET LOCAL statement_timeout = '5s'") + .execute(tx.conn()) + .await?; + let (hashes, ids) = &groups[&community]; + let removed = sqlx::query(concat!( + "DELETE FROM media_shares s \ + USING unnest($2::text[], $3::bytea[]) AS batch(sha256, event_id) \ + WHERE s.community_id = $1 AND s.sha256 = batch.sha256 \ + AND s.event_id = batch.event_id \ + AND NOT EXISTS (SELECT 1 FROM events e \ + WHERE e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + ") AND (s.channel_id IS NULL OR EXISTS ( \ + SELECT 1 FROM media_shares previous \ + WHERE previous.community_id = s.community_id \ + AND previous.sha256 = s.sha256 AND previous.channel_id IS NOT NULL \ + AND previous.event_id < s.event_id))", + )) + .bind(community) + .bind(hashes) + .bind(ids) + .execute(tx.conn()) + .await? + .rows_affected(); + tx.commit().await?; + Ok(removed) + }, + ) + .await + .into_iter() + .sum(); + Ok(MediaShareCleanupBatch { + removed, + next_cursor, + }) + } +} diff --git a/crates/buzz-db/src/store/media_grant.rs b/crates/buzz-db/src/store/media_grant.rs new file mode 100644 index 00000000000..b1aa93977a2 --- /dev/null +++ b/crates/buzz-db/src/store/media_grant.rs @@ -0,0 +1,143 @@ +//! Provenance shared by live media recording, legacy replacement, and backfill. + +use buzz_core::{CommunityId, StoredEvent}; +use chrono::{DateTime, Utc}; +use nostr::Event; +use sqlx::PgConnection; +use uuid::Uuid; + +use crate::{AdmittedTx, Result}; + +/// Resolve the first valid edit target, as the relay's edit validator does. +/// Nested edits and missing/deleted/cross-channel targets grant no file access. +pub(crate) async fn edit_target( + conn: &mut PgConnection, + community: CommunityId, + event: &Event, + channel: Option, +) -> Result, DateTime)>> { + if event.kind.as_u16() != buzz_core::kind::KIND_STREAM_MESSAGE_EDIT as u16 { + return Ok(None); + } + let Some(channel) = channel else { + return Ok(None); + }; + let Some(target) = event.tags.iter().find_map(|tag| { + let parts = tag.as_slice(); + (parts.first().is_some_and(|name| name == "e")) + .then(|| { + parts + .get(1) + .and_then(|id| nostr::EventId::from_hex(id).ok()) + }) + .flatten() + }) else { + return Ok(None); + }; + Ok(sqlx::query_as( + "SELECT id, created_at FROM events \ + WHERE community_id = $1 AND id = $2 AND channel_id = $3 \ + AND deleted_at IS NULL AND kind <> 40003 \ + ORDER BY created_at DESC LIMIT 1", + ) + .bind(community.as_uuid()) + .bind(target.as_bytes().as_slice()) + .bind(channel) + .fetch_optional(conn) + .await?) +} + +/// Record only server-received pre-cutoff references, with their live source +/// and (for edits) target. The receipt check is repeated in the INSERT. +pub(crate) async fn record_history( + conn: &mut PgConnection, + community: CommunityId, + events: &[StoredEvent], +) -> Result { + let mut hashes = Vec::new(); + let mut channels = Vec::new(); + let mut ids = Vec::new(); + let mut times = Vec::new(); + let mut targets = Vec::new(); + let mut target_times = Vec::new(); + for stored in events { + let event = &stored.event; + let blobs = buzz_core::media::shared_blob_hashes(event); + if blobs.is_empty() { + continue; + } + let target = edit_target(conn, community, event, stored.channel_id).await?; + if event.kind.as_u16() == 40003 && target.is_none() { + continue; + } + let seconds = event.created_at.as_secs() as i64; + let created_at = DateTime::from_timestamp(seconds, 0) + .ok_or(crate::DbError::InvalidTimestamp(seconds))?; + for hash in blobs { + hashes.push(hash); + channels.push(stored.channel_id); + ids.push(event.id.to_bytes().to_vec()); + times.push(created_at); + targets.push(target.as_ref().map(|(id, _)| id.clone())); + target_times.push(target.as_ref().map(|(_, at)| *at)); + } + } + if hashes.is_empty() { + return Ok(0); + } + Ok(sqlx::query( + "INSERT INTO media_shares \ + (community_id, sha256, channel_id, event_id, event_created_at, \ + target_event_id, target_event_created_at) \ + SELECT $1, t.h, t.c, e.id, e.created_at, t.target_id, t.target_at \ + FROM unnest($2::text[], $3::uuid[], $4::bytea[], $5::timestamptz[], \ + $6::bytea[], $7::timestamptz[]) \ + AS t(h, c, event_id, event_created_at, target_id, target_at) \ + JOIN events e ON e.community_id = $1 AND e.id = t.event_id \ + AND e.created_at = t.event_created_at AND e.deleted_at IS NULL \ + JOIN communities c ON c.id = e.community_id \ + AND e.received_at < c.media_shares_recording_started_at \ + ORDER BY t.h, e.id ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(hashes) + .bind(channels) + .bind(ids) + .bind(times) + .bind(targets) + .bind(target_times) + .execute(conn) + .await? + .rows_affected()) +} + +/// Recover an eligible legacy head inside its replacement transaction, while +/// it still grants access. Only references the replacement keeps survive it. +pub(crate) async fn bootstrap_replacement( + tx: &mut AdmittedTx, + incoming: &Event, + previous_id: &[u8], +) -> Result<()> { + if buzz_core::media::shared_blob_hashes(incoming).is_empty() { + return Ok(()); + } + let community = tx.community(); + let row = sqlx::query( + "SELECT e.id, e.pubkey, e.created_at, e.kind, e.tags, e.content, e.sig, \ + e.received_at, e.channel_id \ + FROM events e JOIN communities c ON c.id = e.community_id \ + WHERE e.community_id = $1 AND e.id = $2 AND e.deleted_at IS NULL \ + AND e.received_at < c.media_shares_recording_started_at \ + ORDER BY e.created_at DESC LIMIT 1", + ) + .bind(community.as_uuid()) + .bind(previous_id) + .fetch_optional(tx.conn()) + .await?; + if let Some(row) = row { + if let Some(stored) = crate::event::row_to_stored_event(row)? { + record_history(tx.conn(), community, &[stored]).await?; + } + } + Ok(()) +} diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs new file mode 100644 index 00000000000..d4afe0f4610 --- /dev/null +++ b/crates/buzz-db/src/store/media_reference.rs @@ -0,0 +1,1002 @@ +//! Where each media blob is shared, and media read authorization from it. + +use buzz_datastore_tracing::datastore_span; +use chrono::{DateTime, Utc}; +use sqlx::{PgConnection, PgPool, Row}; +use uuid::Uuid; + +use crate::{error::Result, Db}; +use buzz_core::CommunityId; + +/// Both the reference and its edited message must survive. The target lookup +/// uses its recorded timestamp to reach the original event's partition. +macro_rules! live_source_sql { + () => { + "e.deleted_at IS NULL AND (e.kind <> 40003 OR EXISTS ( \ + SELECT 1 FROM events target \ + WHERE target.community_id = s.community_id AND target.id = s.target_event_id \ + AND target.created_at = s.target_event_created_at AND target.deleted_at IS NULL \ + AND target.kind <> 40003 AND target.channel_id IS NOT DISTINCT FROM e.channel_id))" + }; +} +pub(crate) use live_source_sql; + +/// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. +/// +/// A share admits everyone when it is community-wide, or readers of its +/// channel (open, or one they actively belong to). An unknown or unshared hash +/// grants nothing. Deleted or purged source events grant nothing. +macro_rules! readable_sql { + () => { + concat!( + "( \ + EXISTS (SELECT 1 FROM media_shares s \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + " \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + " \ + JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $3 AND cm.removed_at IS NULL))))" + ) + }; +} + +/// Share into `channel_id` (or with the whole community when `None`) each blob +/// in `sha256s` that the event's author can read or has uploaded (uploading +/// the exact bytes proves possession). An agent's profile picture may also +/// use its registered owner's upload proof. Any other blob is left unshared, +/// so a reference built from a leaked hash shows it to no one new. Called with the +/// transaction that stores the referencing event; returns the shares added. +pub(crate) async fn record_media_shares( + conn: &mut PgConnection, + community_id: CommunityId, + sha256s: &[String], + event: &nostr::Event, + channel_id: Option, +) -> Result { + let created_at_secs = event.created_at.as_secs() as i64; + let created_at = DateTime::from_timestamp(created_at_secs, 0) + .ok_or(crate::DbError::InvalidTimestamp(created_at_secs))?; + let target = super::media_grant::edit_target(conn, community_id, event, channel_id).await?; + if event.kind.as_u16() == 40003 && target.is_none() { + return Ok(0); + } + let added = sqlx::query(concat!( + "INSERT INTO media_shares (community_id, sha256, channel_id, event_id, event_created_at, \ + target_event_id, target_event_created_at) \ + SELECT $1, b.sha256, $4, $6, $7, $8, $9 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + WHERE (EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR (b.sha256 = $5 AND EXISTS ( \ + SELECT 1 FROM users agent JOIN media_uploaders u \ + ON u.community_id = agent.community_id AND u.pubkey = agent.agent_owner_pubkey \ + WHERE agent.community_id = $1 AND agent.pubkey = $3 AND u.sha256 = b.sha256)) \ + OR ", + readable_sql!(), + ") AND EXISTS (SELECT 1 FROM events e \ + WHERE e.community_id = $1 AND e.id = $6 AND e.created_at = $7 \ + AND e.deleted_at IS NULL) \ + ORDER BY b.sha256 ON CONFLICT DO NOTHING" + )) + .bind(community_id.as_uuid()) + .bind(sha256s) + .bind(event.pubkey.as_bytes().as_slice()) + .bind(channel_id) + .bind(buzz_core::media::profile_picture_blob_hash(event)) + .bind(event.id.as_bytes().as_slice()) + .bind(created_at) + .bind(target.as_ref().map(|(id, _)| id)) + .bind(target.as_ref().map(|(_, at)| *at)) + .execute(conn) + .await? + .rows_affected(); + Ok(added) +} + +/// Whether `reader` has a readable share, or uploaded a blob not yet shared +/// into any channel. Upload proof alone does not bypass existing channel shares. +pub async fn can_read_blob( + pool: &PgPool, + community_id: CommunityId, + sha256: &str, + reader: &[u8], +) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let readable = sqlx::query_scalar(concat!( + "SELECT ", + readable_sql!(), + " OR (NOT EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND s.channel_id IS NOT NULL) \ + AND EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3)) \ + FROM (SELECT $2::text AS sha256) b" + )) + .bind(community_id.as_uuid()) + .bind(sha256) + .bind(reader) + .fetch_one(&mut *connection) + .await?; + Ok(readable) +} + +impl Db { + /// Whether `reader` may read the blob; see [`can_read_blob`]. + #[datastore_span(name = "can_read_blob", system = "postgresql")] + pub async fn can_read_blob( + &self, + community_id: CommunityId, + sha256: &str, + reader: &[u8], + ) -> Result { + can_read_blob(&self.pool, community_id, sha256, reader).await + } + + /// Record the shares of undeleted events stored in an active community, + /// reading `batch_size` events at a time; returns + /// the number of shares added. + /// + /// Retained server upload audits restore uploader proof before references + /// are evaluated. Missing, malformed, or unattributed audit rows establish + /// no ownership; those uploads still need the exact bytes uploaded again. + /// + /// Shares come from the same rule as live writes. Events received before + /// share recording was installed predate the readability check, so their + /// references are recorded as they stand; later events get the live, + /// checked recording using the author's current permissions. Earlier refusals + /// are not remembered: any run can grant old references after access changes. + /// Rows are deduplicated, but authorization decisions are not preserved. + #[datastore_span(name = "backfill_media_shares", system = "postgresql")] + pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { + if batch_size <= 0 { + return Err(crate::DbError::InvalidData( + "batch size must be positive".into(), + )); + } + let communities: Vec<(Uuid, DateTime)> = sqlx::query_as( + "SELECT id, media_shares_recording_started_at FROM communities \ + WHERE deleted_at IS NULL AND deletion_state = 'active' ORDER BY id", + ) + .fetch_all(&self.pool) + .await?; + let mut added = 0; + for (community, installed) in communities { + let community = CommunityId::from_uuid(community); + self.backfill_media_uploaders(community, batch_size).await?; + let mut cursor: Option<(DateTime, Vec)> = None; + loop { + let rows = sqlx::query( + "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, \ + channel_id \ + FROM events \ + WHERE community_id = $1 AND deleted_at IS NULL \ + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) \ + ORDER BY created_at, id LIMIT $4", + ) + .bind(community.as_uuid()) + .bind(cursor.as_ref().map(|(at, _)| *at)) + .bind( + cursor + .as_ref() + .map(|(_, id)| id.clone()) + .unwrap_or_default(), + ) + .bind(batch_size) + .fetch_all(&self.pool) + .await?; + let Some(last) = rows.last() else { break }; + cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); + let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; + + let mut history = Vec::new(); + let mut recorded_live = Vec::new(); + for row in rows { + let received_at: DateTime = row.try_get("received_at")?; + let Some(stored) = crate::event::row_to_stored_event(row)? else { + continue; + }; + let hashes = buzz_core::media::shared_blob_hashes(&stored.event); + if hashes.is_empty() { + continue; + } + if received_at < installed { + history.push(stored); + } else { + recorded_live.push((stored.event, hashes, stored.channel_id)); + } + } + if !history.is_empty() { + let mut tx = self.begin_media_share_transaction(community).await?; + added += + super::media_grant::record_history(tx.conn(), community, &history).await?; + tx.commit().await?; + } + for (event, hashes, channel) in recorded_live { + let mut tx = self.begin_media_share_transaction(community).await?; + added += + record_media_shares(tx.conn(), community, &hashes, &event, channel).await?; + tx.commit().await?; + } + if !full_batch { + break; + } + } + } + Ok(added) + } + + async fn begin_media_share_transaction( + &self, + community: CommunityId, + ) -> Result { + crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await + } +} + +#[cfg(test)] +mod postgres_tests { + use std::time::Duration; + + use super::*; + use crate::channel::{ChannelType, ChannelVisibility}; + use nostr::{EventBuilder, Keys, Kind, Tag}; + use sqlx::PgPool; + + async fn fixture() -> (Db, CommunityId) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let host = format!("media-shares-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("seed community") + .id; + (db, community) + } + + async fn channel( + db: &Db, + community: CommunityId, + owner: &Keys, + visibility: ChannelVisibility, + ) -> Uuid { + db.create_channel( + community, + &format!("media-{}", Uuid::new_v4().simple()), + ChannelType::Stream, + visibility, + None, + &owner.public_key().to_bytes(), + None, + ) + .await + .expect("create channel") + .id + } + + fn attachment(keys: &Keys, sha256: &str, channel: Option) -> nostr::Event { + let mut tags = vec![Tag::parse(["imeta", &format!("url /media/{sha256}.jpg")]).unwrap()]; + if let Some(channel) = channel { + tags.push(Tag::parse(["h", &channel.to_string()]).unwrap()); + } + EventBuilder::new(Kind::from(9), format!("attachment {}", Uuid::new_v4())) + .tags(tags) + .sign_with_keys(keys) + .unwrap() + } + + async fn shares(db: &Db, community: CommunityId, sha256: &str) -> Vec> { + let mut shares: Vec> = sqlx::query_scalar( + "SELECT DISTINCT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + ) + .bind(community.as_uuid()) + .bind(sha256) + .fetch_all(&db.pool) + .await + .expect("read shares"); + shares.sort(); + shares + } + + fn unique_sha() -> String { + format!("{:064x}", Uuid::new_v4().as_u128()) + } + + async fn upload(db: &Db, community: CommunityId, uploader: &Keys, sha256: &str) { + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community, + "media_upload", + "media-shares-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + db.record_media_uploader(&lease, sha256, &uploader.public_key().to_bytes()) + .await + .expect("record uploader"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stored_references_share_only_blobs_their_author_can_read() { + let (db, community) = fixture().await; + let (alice, bob) = (Keys::generate(), Keys::generate()); + let sha = unique_sha(); + let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; + + for uploader_present in [false, true] { + if uploader_present { + upload(&db, community, &alice, &sha).await; + } + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references an unshared hash"); + assert!( + shares(&db, community, &sha).await.is_empty(), + "a hash alone grants nothing, even before its uploader posts it" + ); + } + db.insert_event( + community, + &attachment(&alice, &sha, Some(private)), + Some(private), + ) + .await + .expect("alice posts privately"); + assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references the hash"); + assert_eq!( + shares(&db, community, &sha).await, + vec![Some(private)], + "a hash alone must not share a blob its author cannot read" + ); + + upload(&db, community, &bob, &sha).await; + db.insert_event(community, &attachment(&bob, &sha, Some(bobs)), Some(bobs)) + .await + .expect("bob posts his copy"); + let mut expected = vec![Some(private), Some(bobs)]; + expected.sort(); + assert_eq!( + shares(&db, community, &sha).await, + expected, + "an uploader may share their own bytes" + ); + + let excluded = unique_sha(); + upload(&db, community, &alice, &excluded).await; + use buzz_core::kind::{ + AUTHOR_ONLY_KINDS, KIND_AGENT_ENGRAM, KIND_DELETION, KIND_EVENT_REMINDER, + P_GATED_KINDS, SHARED_GATED_KINDS, + }; + let kinds = AUTHOR_ONLY_KINDS + .iter() + .chain(P_GATED_KINDS) + .chain(SHARED_GATED_KINDS) + .copied() + .chain([KIND_AGENT_ENGRAM, KIND_DELETION]); + for kind in kinds { + if (20_000..30_000).contains(&kind) { + continue; // Ephemeral frames have no stored reference to backfill. + } + let mut tags = vec![Tag::parse(["d", "private-media"]).unwrap()]; + if kind == KIND_DELETION { + tags.push(Tag::parse(["k", &KIND_EVENT_REMINDER.to_string()]).unwrap()); + } + let event = EventBuilder::new(Kind::from(kind as u16), format!("/media/{excluded}")) + .tags(tags) + .sign_with_keys(&alice) + .unwrap(); + db.insert_event(community, &event, None).await.unwrap(); + assert!( + db.get_event_by_id(community, event.id.as_bytes()) + .await + .unwrap() + .is_some(), + "kind {kind} must exercise a stored event" + ); + assert!( + shares(&db, community, &excluded).await.is_empty(), + "private kind {kind} must create no grant even for its uploader" + ); + } + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn backfill_trusts_history_and_rechecks_recorded_events() { + check_backfill().await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn deleting_a_reference_revokes_only_its_own_media_grant() { + let (db, community) = fixture().await; + let (owner, reader) = (Keys::generate(), Keys::generate()); + let open = channel(&db, community, &owner, ChannelVisibility::Open).await; + for channel_id in [Some(open), None] { + let sha = unique_sha(); + upload(&db, community, &owner, &sha).await; + let first = attachment(&owner, &sha, channel_id); + let last = attachment(&owner, &sha, channel_id); + for event in [&first, &last] { + db.insert_event(community, event, channel_id).await.unwrap(); + } + let sources: Vec> = sqlx::query_scalar( + "SELECT event_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + ) + .bind(community.as_uuid()) + .bind(&sha) + .fetch_all(&db.pool) + .await + .unwrap(); + assert_eq!(sources.len(), 2, "each message must retain its own grant"); + for event in [&first, &last] { + assert!(sources.contains(&event.id.to_bytes().to_vec())); + } + let edit_only = if let Some(channel) = channel_id { + let edit_sha = unique_sha(); + upload(&db, community, &owner, &edit_sha).await; + for target in [&first, &last] { + let edit = EventBuilder::new( + Kind::from(40003), + format!("retained /media/{sha}, added /media/{edit_sha}"), + ) + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &target.id.to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &edit, channel_id).await.unwrap(); + // The backfill must also reject surviving legacy edits + // once their logical message has disappeared. + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query( + "UPDATE events SET received_at = (SELECT media_shares_recording_started_at \ + FROM communities WHERE id = $1) - interval '1 second' \ + WHERE community_id = $1 AND id = $2", + ) + .bind(community.as_uuid()) + .bind(edit.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + } + assert!(readable_share(&db, community, &edit_sha, &reader).await); + Some(edit_sha) + } else { + None + }; + assert!(readable_share(&db, community, &sha, &reader).await); + for (event, still_shared) in [(&first, true), (&last, false)] { + assert!( + db.soft_delete_event_and_update_thread( + community, + event.id.as_bytes(), + None, + None, + ) + .await + .unwrap() + ); + assert_eq!( + readable_share(&db, community, &sha, &reader).await, + still_shared, + "only an undeleted source message can grant access" + ); + if let Some(edit_sha) = &edit_only { + assert_eq!( + readable_share(&db, community, edit_sha, &reader).await, + still_shared, + "edit-only attachments must share the target message's lifetime" + ); + } + } + db.insert_event(community, &attachment(&reader, &sha, None), None) + .await + .unwrap(); + assert!(!readable_share(&db, community, &sha, &reader).await); + db.backfill_media_shares(1).await.unwrap(); + assert!(!readable_share(&db, community, &sha, &reader).await); + if let Some(edit_sha) = &edit_only { + assert!(!readable_share(&db, community, edit_sha, &reader).await); + } + + let purged = attachment(&owner, &sha, channel_id); + db.insert_event(community, &purged, channel_id) + .await + .unwrap(); + assert!(readable_share(&db, community, &sha, &reader).await); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query("DELETE FROM events WHERE community_id = $1 AND id = $2") + .bind(community.as_uuid()) + .bind(purged.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert!( + !readable_share(&db, community, &sha, &reader).await, + "a purged source cannot leave a dangling grant" + ); + assert!(reap_all(&db).await > 0); + assert_eq!( + shares(&db, community, &sha).await.len(), + usize::from(channel_id.is_some()), + "cleanup retains at most one channel boundary, never public history" + ); + if let Some(edit_sha) = &edit_only { + assert_eq!( + shares(&db, community, edit_sha).await.len(), + 1, + "inactive edit grants compact to one channel marker" + ); + } + assert!(!readable_share(&db, community, &sha, &reader).await); + assert_eq!(reap_all(&db).await, 0, "cleanup is idempotent"); + } + + // Fleet cleanup skips a quiescing community while reclaiming another + // community's dead copy of the same hash and keeping its live copy. + let sha = unique_sha(); + upload(&db, community, &owner, &sha).await; + let retired = attachment(&owner, &sha, None); + db.insert_event(community, &retired, None).await.unwrap(); + db.soft_delete_event_and_update_thread(community, retired.id.as_bytes(), None, None) + .await + .unwrap(); + quiesce(&db, community).await; + let (other, other_community) = fixture().await; + upload(&other, other_community, &owner, &sha).await; + let old = attachment(&owner, &sha, None); + let current = attachment(&owner, &sha, None); + for event in [&old, ¤t] { + other + .insert_event(other_community, event, None) + .await + .unwrap(); + } + other + .soft_delete_event_and_update_thread(other_community, old.id.as_bytes(), None, None) + .await + .unwrap(); + assert!(reap_all(&db).await > 0); + assert_eq!( + shares(&db, community, &sha).await.len(), + 1, + "cleanup may not write into a quiescing community" + ); + assert_eq!(shares(&other, other_community, &sha).await.len(), 1); + assert!(readable_share(&other, other_community, &sha, &reader).await); + } + + async fn readable_share(db: &Db, community: CommunityId, sha: &str, reader: &Keys) -> bool { + sqlx::query_scalar(concat!( + "SELECT ", + readable_sql!(), + " FROM (SELECT $2::text AS sha256) b" + )) + .bind(community.as_uuid()) + .bind(sha) + .bind(reader.public_key().to_bytes().to_vec()) + .fetch_one(&db.pool) + .await + .unwrap() + } + + async fn quiesce(db: &Db, community: CommunityId) { + let mut lifecycle = db.pool.begin().await.unwrap(); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.as_uuid().to_string()) + .execute(&mut *lifecycle) + .await + .unwrap(); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(community.as_uuid()) + .execute(&mut *lifecycle) + .await + .unwrap(); + lifecycle.commit().await.unwrap(); + } + + async fn reap_all(db: &Db) -> u64 { + let mut cursor = None; + let mut removed = 0; + // One row per page exercises progress across live rows and retained + // channel markers as well as rows deleted by a previous page. + for _ in 0..10_000 { + let batch = db + .reap_inactive_media_shares(cursor.as_ref(), 1) + .await + .unwrap(); + removed += batch.removed; + cursor = batch.next_cursor; + if cursor.is_none() { + return removed; + } + } + panic!("cleanup must finish its bounded primary-key scan"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn replacement_preserves_only_the_media_it_continues_to_reference() { + let (db, community) = fixture().await; + let viewer = Keys::generate(); + for kind in [Kind::Metadata, Kind::from(30030)] { + let (uploader, publisher) = (Keys::generate(), Keys::generate()); + let sha = unique_sha(); + upload(&db, community, &uploader, &sha).await; + let base = nostr::Timestamp::now().as_secs(); + let make_for = |sha: &str, keys: &Keys, kind: Kind, step: u64, picture: bool| { + let url = if picture { + format!("/media/{sha}.jpg") + } else { + String::new() + }; + let mut tags = Vec::new(); + let content = if kind == Kind::Metadata { + serde_json::json!({"picture": url, "name": format!("profile-{step}")}) + .to_string() + } else { + tags.push(Tag::parse(["d", "media-continuity"]).unwrap()); + if picture { + tags.push(Tag::parse(["emoji", "sample", &url]).unwrap()); + } + String::new() + }; + EventBuilder::new(kind, content) + .tags(tags) + .custom_created_at(nostr::Timestamp::from_secs(base + step)) + .sign_with_keys(keys) + .unwrap() + }; + let make = |keys: &Keys, kind: Kind, step: u64, picture: bool| { + make_for(&sha, keys, kind, step, picture) + }; + let publish = async |event: &nostr::Event| { + if event.kind == Kind::Metadata { + db.replace_addressable_event(community, event, None) + .await + .unwrap() + .1 + } else { + db.replace_parameterized_event(community, event, "media-continuity", None) + .await + .unwrap() + .1 + } + }; + assert!(publish(&make(&uploader, Kind::Metadata, 0, true)).await); + assert!(publish(&make(&publisher, kind, 0, true)).await); + assert!(publish(&make(&uploader, Kind::Metadata, 1, false)).await); + assert!(readable_share(&db, community, &sha, &viewer).await); + assert!(publish(&make(&publisher, kind, 1, true)).await); + assert!(reap_all(&db).await > 0); + assert_eq!( + shares(&db, community, &sha).await.len(), + 1, + "cleanup keeps the surviving profile/emoji head" + ); + assert!( + readable_share(&db, community, &sha, &viewer).await, + "a replacement must carry forward an authorized reference" + ); + assert!(publish(&make(&publisher, kind, 2, false)).await); + assert!(!readable_share(&db, community, &sha, &viewer).await); + assert!(publish(&make(&publisher, kind, 3, true)).await); + assert!( + !readable_share(&db, community, &sha, &viewer).await, + "a later replacement cannot revive a previously removed grant" + ); + + // An unbackfilled legacy head has neither proof nor shares. A + // normal name-only update must retain the eligible old reference. + let legacy_keys = Keys::generate(); + let legacy_sha = unique_sha(); + let legacy_make = + |step, picture| make_for(&legacy_sha, &legacy_keys, kind, step, picture); + let legacy = legacy_make(0, true); + assert!(publish(&legacy).await); + assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query( + "UPDATE events SET received_at = (SELECT media_shares_recording_started_at \ + FROM communities WHERE id = $1) - interval '1 second' \ + WHERE community_id = $1 AND id = $2", + ) + .bind(community.as_uuid()) + .bind(legacy.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert!(publish(&legacy_make(1, true)).await); + assert!(reap_all(&db).await > 0); + assert_eq!(shares(&db, community, &legacy_sha).await.len(), 1); + db.backfill_media_shares(1).await.unwrap(); + assert!( + readable_share(&db, community, &legacy_sha, &viewer).await, + "a retained legacy avatar/emoji must survive replacement before backfill" + ); + assert!(publish(&legacy_make(2, false)).await); + assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); + assert!(publish(&legacy_make(3, true)).await); + assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); + assert!(reap_all(&db).await > 0); + assert!( + shares(&db, community, &legacy_sha).await.is_empty(), + "replaced avatar/emoji heads must not accumulate inactive rows" + ); + assert!(shares(&db, community, &sha).await.is_empty()); + } + } + + async fn check_backfill() { + let (db, community) = fixture().await; + let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); + let (sha, historical, deleted_history) = (unique_sha(), unique_sha(), unique_sha()); + let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + upload(&db, community, &alice, &sha).await; + db.insert_event( + community, + &attachment(&alice, &sha, Some(private)), + Some(private), + ) + .await + .expect("alice posts privately"); + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references the hash"); + let old = attachment(&carol, &historical, None); + db.insert_event(community, &old, None) + .await + .expect("carol posts before share recording"); + let deleted = attachment(&carol, &deleted_history, None); + db.insert_event(community, &deleted, None).await.unwrap(); + db.soft_delete_event_and_update_thread(community, deleted.id.as_bytes(), None, None) + .await + .unwrap(); + + let mut tx = crate::begin_community_write_transaction( + &db.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await + .expect("admit"); + sqlx::query( + "UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = ANY($2)", + ) + .bind(community.as_uuid()) + .bind(vec![ + old.id.to_bytes().to_vec(), + deleted.id.to_bytes().to_vec(), + ]) + .execute(tx.conn()) + .await + .expect("age carol's event"); + sqlx::query("DELETE FROM media_shares WHERE community_id = $1") + .bind(community.as_uuid()) + .execute(tx.conn()) + .await + .expect("forget shares"); + tx.commit().await.expect("commit"); + + // Surviving server upload audits recover legacy ownership; malformed, + // missing, or unrelated audit evidence never creates uploader proof. + let (recovered, unproven) = (unique_sha(), unique_sha()); + let (uploader, stranger) = (Keys::generate(), Keys::generate()); + db.insert_event(community, &attachment(&uploader, &recovered, None), None) + .await + .unwrap(); + db.insert_event(community, &attachment(&stranger, &unproven, None), None) + .await + .unwrap(); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + for (seq, action, actor, object) in [ + ( + 1_i64, + "media_uploaded", + Some(uploader.public_key().to_bytes().to_vec()), + recovered.clone(), + ), + (2, "media_uploaded", None, unproven.clone()), + (3, "media_uploaded", Some(vec![1; 31]), unproven.clone()), + ( + 4, + "event_created", + Some(stranger.public_key().to_bytes().to_vec()), + unproven.clone(), + ), + ( + 5, + "media_uploaded", + Some(stranger.public_key().to_bytes().to_vec()), + "bad-hash".into(), + ), + ] { + sqlx::query( + "INSERT INTO audit_log (community_id, seq, hash, action, actor_pubkey, object_id) \ + VALUES ($1,$2,$3,$4,$5,$6)", + ) + .bind(community.as_uuid()) + .bind(seq) + .bind(vec![seq as u8; 32]) + .bind(action) + .bind(actor) + .bind(object) + .execute(tx.conn()) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + let (fenced, fenced_community) = fixture().await; + let mut tx = fenced + .begin_media_share_transaction(fenced_community) + .await + .unwrap(); + sqlx::query( + "INSERT INTO audit_log (community_id, seq, hash, action, actor_pubkey, object_id) \ + VALUES ($1,1,$2,'media_uploaded',$3,$4)", + ) + .bind(fenced_community.as_uuid()) + .bind(vec![1_u8; 32]) + .bind(stranger.public_key().to_bytes().as_slice()) + .bind(&unproven) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + quiesce(&fenced, fenced_community).await; + + db.backfill_media_shares(1).await.expect("backfill"); + assert_eq!(db.backfill_media_shares(1).await.expect("rerun"), 0); + assert_eq!( + shares(&db, community, &sha).await, + vec![Some(private)], + "a reference live recording refused must stay unshared" + ); + assert_eq!(shares(&db, community, &historical).await, vec![None]); + let fenced_proofs: i64 = + sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1") + .bind(fenced_community.as_uuid()) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!( + fenced_proofs, 0, + "audit recovery must not write across a fenced tenant boundary" + ); + assert_eq!( + shares(&db, community, &recovered).await, + vec![None], + "retained upload evidence must recover an uploader's intended share" + ); + assert!(shares(&db, community, &unproven).await.is_empty()); + let proofs: Vec<(String, Vec)> = sqlx::query_as( + "SELECT sha256, pubkey FROM media_uploaders WHERE community_id = $1 AND sha256 = ANY($2)", + ).bind(community.as_uuid()).bind(vec![recovered.clone(), unproven]) + .fetch_all(&db.pool).await.unwrap(); + assert_eq!( + proofs, + vec![(recovered, uploader.public_key().to_bytes().to_vec())] + ); + assert!( + shares(&db, community, &deleted_history).await.is_empty(), + "legacy deleted messages must never gain a share" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn migration_schema_backfill_trusts_history_and_rechecks_recorded_events() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to migration DB"); + crate::migration::run_migrations_through(&pool, 68) + .await + .expect("migrate"); + pool.close().await; + check_backfill().await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn agent_profile_picture_uses_only_its_registered_owners_upload_proof() { + let (db, community) = fixture().await; + let (owner, agent) = (Keys::generate(), Keys::generate()); + let (avatar, secret, unknown) = (unique_sha(), unique_sha(), unique_sha()); + for sha in [&avatar, &secret] { + upload(&db, community, &owner, sha).await; + } + let profile = |picture: &str| { + EventBuilder::new( + Kind::Metadata, + serde_json::json!({ + "name": Uuid::new_v4().to_string(), + "picture": format!("/media/{picture}.jpg"), + "about": format!("/media/{secret}.jpg"), + }) + .to_string(), + ) + .tags([Tag::parse(["p", &owner.public_key().to_hex()]).unwrap()]) + .sign_with_keys(&agent) + .unwrap() + }; + db.insert_event(community, &profile(&avatar), None) + .await + .unwrap(); + assert!( + shares(&db, community, &avatar).await.is_empty(), + "an unverified owner tag grants nothing" + ); + for keys in [&owner, &agent] { + db.ensure_user(community, &keys.public_key().to_bytes()) + .await + .unwrap(); + } + db.set_agent_owner( + community, + &agent.public_key().to_bytes(), + &owner.public_key().to_bytes(), + ) + .await + .unwrap(); + db.insert_event(community, &profile(&avatar), None) + .await + .unwrap(); + assert_eq!(shares(&db, community, &avatar).await, vec![None]); + assert!( + shares(&db, community, &secret).await.is_empty(), + "owner proof applies only to picture" + ); + db.insert_event(community, &profile(&unknown), None) + .await + .unwrap(); + assert!( + shares(&db, community, &unknown).await.is_empty(), + "the owner must have uploaded these bytes" + ); + db.insert_event(community, &attachment(&agent, &secret, None), None) + .await + .unwrap(); + assert!( + shares(&db, community, &secret).await.is_empty(), + "ordinary events cannot borrow the owner's uploads" + ); + } +} diff --git a/crates/buzz-db/src/store/media_report.rs b/crates/buzz-db/src/store/media_report.rs new file mode 100644 index 00000000000..b4d88349194 --- /dev/null +++ b/crates/buzz-db/src/store/media_report.rs @@ -0,0 +1,43 @@ +//! Private, temporary media access for review of an open event report. + +use buzz_core::CommunityId; +use buzz_datastore_tracing::datastore_span; +use uuid::Uuid; + +use crate::{Db, Result}; + +impl Db { + /// Find an open report of a live event that actually granted this file. + /// + /// The caller must authorize ViewQueue and durably audit the access before + /// serving bytes. Report notes, blob-only reports, and unrelated references + /// never supply attachment provenance. Reads use the writer for revocation. + #[datastore_span(name = "open_media_report", system = "postgresql")] + pub async fn open_media_report( + &self, + community: CommunityId, + sha256: &str, + ) -> Result)>> { + let mut conn = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + Ok(sqlx::query_as(concat!( + "SELECT r.id, r.target_event_id FROM moderation_reports r \ + JOIN media_shares s ON s.community_id = r.community_id \ + AND (s.event_id = r.target_event_id OR s.target_event_id = r.target_event_id) \ + AND s.sha256 = $2 \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + " \ + WHERE r.community_id = $1 AND r.target_kind = 'event' AND r.status = 'open' \ + ORDER BY r.created_at, r.id LIMIT 1", + )) + .bind(community.as_uuid()) + .bind(sha256) + .fetch_optional(&mut *conn) + .await?) + } +} diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs new file mode 100644 index 00000000000..4ced9087b59 --- /dev/null +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -0,0 +1,173 @@ +//! Who has uploaded each blob, as proof of possession for sharing it. + +use buzz_datastore_tracing::datastore_span; + +use crate::deletion::ServingWriteLease; +use crate::{error::Result, observability, Db}; +use buzz_core::CommunityId; + +impl Db { + /// Recover possession proof from retained server-authored upload audits. + /// Scan a finite sequence range using audit_log's tenant-leading primary + /// key; malformed or unattributed records never establish ownership. + #[datastore_span(name = "backfill_media_uploaders", system = "postgresql")] + pub(crate) async fn backfill_media_uploaders( + &self, + community: CommunityId, + batch_size: i64, + ) -> Result { + let mut conn = + observability::acquire_writer(&self.pool, observability::WriterOperation::Maintenance) + .await?; + let upper: Option = + sqlx::query_scalar("SELECT max(seq) FROM audit_log WHERE community_id = $1") + .bind(community.as_uuid()) + .fetch_one(&mut *conn) + .await?; + drop(conn); + let Some(upper) = upper else { + return Ok(0); + }; + let (mut cursor, mut added) = (0_i64, 0_u64); + while cursor < upper { + let mut tx = crate::begin_community_write_transaction( + &self.pool, + community, + observability::WriterOperation::Maintenance, + ) + .await?; + let page: Vec = sqlx::query_scalar( + "SELECT seq FROM audit_log WHERE community_id = $1 AND seq > $2 AND seq <= $3 \ + ORDER BY seq LIMIT $4", + ) + .bind(community.as_uuid()) + .bind(cursor) + .bind(upper) + .bind(batch_size) + .fetch_all(tx.conn()) + .await?; + let Some(&last) = page.last() else { + break; + }; + added += sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey, created_at) \ + SELECT community_id, object_id, actor_pubkey, created_at FROM audit_log \ + WHERE community_id = $1 AND seq > $2 AND seq <= $3 \ + AND action = 'media_uploaded' AND octet_length(actor_pubkey) = 32 \ + AND object_id ~ '^[0-9a-f]{64}$' \ + ORDER BY object_id, actor_pubkey ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(cursor) + .bind(last) + .execute(tx.conn()) + .await? + .rows_affected(); + tx.commit().await?; + cursor = last; + } + Ok(added) + } + + /// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are + /// no-ops. The write presents the upload's serving lease, so it may finish + /// while the community quiesces but never after the lease is gone. + #[datastore_span(name = "record_media_uploader", system = "postgresql")] + pub async fn record_media_uploader( + &self, + lease: &ServingWriteLease, + sha256: &str, + pubkey: &[u8], + ) -> Result<()> { + let connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::EventWrite) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + self.deletion_store() + .guard_transaction_with_serving_lease(&mut tx, lease) + .await?; + sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(lease.community_id.as_uuid()) + .bind(sha256) + .bind(pubkey) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } +} + +#[cfg(test)] +mod postgres_tests { + use std::time::Duration; + + use super::*; + use buzz_core::CommunityId; + use sqlx::PgPool; + use uuid::Uuid; + + const SHA: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + + async fn db_with_lease() -> (Db, CommunityId, ServingWriteLease) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let host = format!("uploader-{}.example", Uuid::new_v4().simple()); + let community_id = db + .ensure_configured_community(&host) + .await + .expect("seed community") + .id; + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community_id, + "media_upload", + "media-uploader-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + (db, community_id, lease) + } + + async fn uploader_rows(db: &Db, community_id: CommunityId) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1") + .bind(community_id.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("count uploaders") + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn repeat_upload_records_one_row() { + let (db, community_id, lease) = db_with_lease().await; + for _ in 0..2 { + db.record_media_uploader(&lease, SHA, &[7; 32]) + .await + .expect("record uploader"); + } + assert_eq!(uploader_rows(&db, community_id).await, 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn released_lease_refuses_the_write() { + let (db, community_id, lease) = db_with_lease().await; + db.deletion_store() + .release_serving_write_lease(&lease) + .await + .expect("release lease"); + assert!(db + .record_media_uploader(&lease, SHA, &[7; 32]) + .await + .is_err()); + assert_eq!(uploader_rows(&db, community_id).await, 0); + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 6850b625cf1..a643d1580a1 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,6 +32,15 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Bounded reclamation of inactive media grants. +pub mod media_cleanup; +/// Where each media blob is shared, recorded with the events that reference it. +mod media_grant; +pub mod media_reference; +/// Attachment provenance for scoped moderation report reads. +pub mod media_report; +/// Who has uploaded each blob, as proof of possession for sharing it. +pub mod media_uploader; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; /// Deployment-global operator-listener mention registrations and delivery queues. diff --git a/crates/buzz-db/src/store/push.rs b/crates/buzz-db/src/store/push.rs index 24ae312521e..d4ae0b86b3c 100644 --- a/crates/buzz-db/src/store/push.rs +++ b/crates/buzz-db/src/store/push.rs @@ -360,13 +360,7 @@ pub async fn accept_lease_event( return Err(error.into()); } // The row above is always kind 30350 with no channel. - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - 30350, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, None).await?; let (is_active, endpoint_hash, endpoint_grant, max_class, subscriptions) = active .map_or((false, None, None, None, None), |active| { diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 6bc4de1a959..43154fe4baf 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -1170,13 +1170,7 @@ pub(crate) async fn write_nip43_membership_snapshot( .bind(d_tag.as_deref()) .execute(tx.conn()) .await?; - crate::store::event_follow_up::after_admitted_insert( - tx, - event.id.as_bytes().as_slice(), - kind_i32, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, &event, None).await?; Ok(StoredEvent::with_received_at( event, diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 1ccee163320..4db5e590514 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -262,6 +262,57 @@ async fn replace_parameterized_event_in_transaction_impl( let tags_json = serde_json::to_value(&event.tags)?; crate::channel_members::guard_channel_roster_event(tx, kind_i32, channel_id, &tags_json) .await?; + + let sig_bytes = event.sig.serialize(); + let insert_result = sqlx::query( + "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \ + ON CONFLICT DO NOTHING", + ) + .bind(community_id.as_uuid()) + .bind(incoming_id) + .bind(pubkey_bytes.as_slice()) + .bind(created_at) + .bind(kind_i32) + .bind(&tags_json) + .bind(&event.content) + .bind(sig_bytes.as_slice()) + .bind(received_at) + .bind(channel_id) + .bind(d_tag) + .bind(crate::event::extract_not_before(event)) + .execute(tx.conn()) + .await?; + + if insert_result.rows_affected() == 0 { + return Ok(false); + } + + if is_nip_rs { + sqlx::query( + "INSERT INTO parameterized_event_watermarks \ + (community_id, kind, pubkey, d_tag, created_at, event_id) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (community_id, kind, pubkey, d_tag) DO UPDATE SET \ + created_at = EXCLUDED.created_at, event_id = EXCLUDED.event_id", + ) + .bind(community_id.as_uuid()) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .bind(created_at) + .bind(incoming_id) + .execute(tx.conn()) + .await?; + } + + crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; + // Recover eligible legacy references before retiring the old head. + if let Some((_, previous_id)) = &existing { + crate::store::media_grant::bootstrap_replacement(tx, event, previous_id).await?; + } + // Authorize retained references while the previous head is still live. + crate::store::event_follow_up::record_admitted_media(tx, event, channel_id).await?; if existing.is_some() { let previous_nip_rs_hard_delete: Option = if is_nip_rs { sqlx::query_scalar( @@ -279,16 +330,17 @@ async fn replace_parameterized_event_in_transaction_impl( } let statement = if hard_delete_superseded { "DELETE FROM events \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL" + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL AND id <> $5" } else { "UPDATE events SET deleted_at = NOW() \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL" + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL AND id <> $5" }; sqlx::query(statement) .bind(community_id.as_uuid()) .bind(kind_i32) .bind(pubkey_bytes.as_slice()) .bind(d_tag) + .bind(incoming_id) .execute(tx.conn()) .await?; @@ -311,50 +363,6 @@ async fn replace_parameterized_event_in_transaction_impl( } } - let sig_bytes = event.sig.serialize(); - let insert_result = sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \ - ON CONFLICT DO NOTHING", - ) - .bind(community_id.as_uuid()) - .bind(incoming_id) - .bind(pubkey_bytes.as_slice()) - .bind(created_at) - .bind(kind_i32) - .bind(&tags_json) - .bind(&event.content) - .bind(sig_bytes.as_slice()) - .bind(received_at) - .bind(channel_id) - .bind(d_tag) - .bind(crate::event::extract_not_before(event)) - .execute(tx.conn()) - .await?; - - if insert_result.rows_affected() == 0 { - return Ok(false); - } - - if is_nip_rs { - sqlx::query( - "INSERT INTO parameterized_event_watermarks \ - (community_id, kind, pubkey, d_tag, created_at, event_id) \ - VALUES ($1, $2, $3, $4, $5, $6) \ - ON CONFLICT (community_id, kind, pubkey, d_tag) DO UPDATE SET \ - created_at = EXCLUDED.created_at, event_id = EXCLUDED.event_id", - ) - .bind(community_id.as_uuid()) - .bind(kind_i32) - .bind(pubkey_bytes.as_slice()) - .bind(d_tag) - .bind(created_at) - .bind(incoming_id) - .execute(tx.conn()) - .await?; - } - - crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; Ok(true) } .await; @@ -383,8 +391,7 @@ async fn replace_parameterized_event_in_transaction_impl( .execute(tx.conn()) .await?; if status == ParameterizedReplaceStatus::Inserted { - crate::store::event_follow_up::after_admitted_insert(tx, incoming_id, kind_i32, channel_id) - .await?; + crate::store::event_follow_up::stage_admitted_insert(tx, event, channel_id); } Ok(ParameterizedReplaceResult::new( @@ -468,9 +475,9 @@ impl Db { // NIP-16: created_at is second-resolution. On same-second tie, lowest // event id (lexicographic) wins — deterministic across relays. let incoming_id = event.id.as_bytes().as_slice(); - if let Some((existing_ts, existing_id)) = existing { - let dominated = created_at < existing_ts - || (created_at == existing_ts + if let Some((existing_ts, existing_id)) = &existing { + let dominated = created_at < *existing_ts + || (created_at == *existing_ts && incoming_id >= existing_id.as_slice()); if dominated { tx.rollback().await?; @@ -493,20 +500,6 @@ impl Db { ) .await?; - // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. - sqlx::query( - "UPDATE events SET deleted_at = NOW() \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 \ - AND channel_id IS NOT DISTINCT FROM $4 \ - AND deleted_at IS NULL", - ) - .bind(community_id.as_uuid()) - .bind(kind_i32) - .bind(pubkey_bytes.as_slice()) - .bind(channel_id) - .execute(tx.conn()) - .await?; - // Insert the new event inside the same transaction. let sig_bytes = event.sig.serialize(); let received_at = chrono::Utc::now(); @@ -533,8 +526,7 @@ impl Db { let was_inserted = insert_result.rows_affected() > 0; if !was_inserted { - // ON CONFLICT fired — the event ID already exists. Rollback the - // soft-delete so we don't lose the previous replaceable event. + // ON CONFLICT fired — leave the previously-live event intact. tx.rollback().await?; return Ok(( StoredEvent::with_received_at( @@ -547,12 +539,27 @@ impl Db { )); } - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - kind_i32, - channel_id, + if let Some((_, previous_id)) = &existing { + crate::store::media_grant::bootstrap_replacement(&mut tx, event, previous_id) + .await?; + } + // Authorize retained references while the previous head is still live. + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) + .await?; + + // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. + sqlx::query( + "UPDATE events SET deleted_at = NOW() \ + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 \ + AND channel_id IS NOT DISTINCT FROM $4 \ + AND deleted_at IS NULL AND id <> $5", ) + .bind(community_id.as_uuid()) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(channel_id) + .bind(incoming_id) + .execute(tx.conn()) .await?; // The replaceable event and its denormalized mention index are one diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index b1c95d7f1c7..dbf96bd621b 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1643,12 +1643,14 @@ fn events_insert_tails(source: &str) -> Vec { /// An events insert must run both follow-ups: the push enqueue and the /// pre-commit TTL refresh. `after_admitted_insert` does both; a writer that -/// inserts inside a savepoint calls the two halves separately. +/// inserts inside a savepoint records media there and stages delivery after it. /// /// Only real calls count: a hook in a comment or a string literal does not. fn runs_event_follow_ups(function_source: &str) -> bool { let code = mask_comments_and_literals(function_source); code.contains("event_follow_up::after_admitted_insert(") + || (code.contains("event_follow_up::record_admitted_media(") + && code.contains("event_follow_up::stage_admitted_insert(")) || (code.contains("event_follow_up::enqueue_push_match(") && code.contains(".record_channel_event(")) } @@ -1696,6 +1698,12 @@ pub(crate) async fn hooked_writer(tx: &mut AdmittedTx) { .await .expect("follow up"); } +pub(crate) async fn media_savepoint_writer(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") + .execute(tx.conn()).await.expect("write"); + crate::store::event_follow_up::record_admitted_media(tx, event, channel).await.expect("media"); + crate::store::event_follow_up::stage_admitted_insert(tx, event, channel); +} pub(crate) async fn savepoint_writer(tx: &mut AdmittedTx) { sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") .execute(tx.conn()) diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 9c5f3dd93b3..9af57859158 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -579,6 +579,24 @@ async fn upload_blob_result( auth.tenant.host(), ); + state + .db + .record_media_uploader( + serving_write.lease(), + &descriptor.sha256, + auth.auth_event.pubkey.as_bytes(), + ) + .await + .map_err(|error| { + let error = anyhow::Error::from(error); + if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) { + serving_lease_lost(error) + } else { + tracing::error!(%error, "media uploader recording failed"); + MediaError::ServiceUnavailable + } + })?; + // Normalize MIME to a known set to bound label cardinality. let mime_label = match descriptor.mime_type.as_str() { "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "video/mp4" => { @@ -707,6 +725,24 @@ async fn enforce_blossom_read_membership( .map_err(|e| membership_denial(e, strictness)) } +/// Every read gate a Blossom GET/HEAD passes before any storage lookup: +/// community membership, then the channels the blob is attached in. A channel +/// refusal is `NotFound` so a non-member cannot probe whether a private +/// attachment exists. +async fn enforce_blossom_read_access( + state: &Arc, + tenant: &TenantContext, + auth_event: &nostr::Event, + headers: &HeaderMap, + sha256: &str, + strictness: BlossomStrictness, +) -> Result<(), MediaDenial> { + enforce_blossom_read_membership(state, tenant, auth_event, headers, strictness).await?; + super::media_access::enforce(state, tenant, auth_event.pubkey.as_bytes(), sha256) + .await + .map_err(|error| media_denial(error, strictness)) +} + fn blob_cache_control() -> &'static str { "private, max-age=31536000, immutable" } @@ -823,8 +859,9 @@ pub(crate) async fn get_blob( .map(drop) }) .await; + let proof_sha256 = sha256.clone(); let admission = match admit_nip_fi_http_on_state(&state, &req_headers, move || { - extract_blossom_read_proof(&headers_clone, &sha256, &tenant_host, strictness) + extract_blossom_read_proof(&headers_clone, &proof_sha256, &tenant_host, strictness) .map_err(|e| e.into_response()) }) .await @@ -833,8 +870,15 @@ pub(crate) async fn get_blob( Err(resp) => return Ok(resp), }; let auth_event = admission.into_extra(); - // Post-admission: membership gate. - enforce_blossom_read_membership(&state, &tenant, &auth_event, &req_headers, strictness).await?; + enforce_blossom_read_access( + &state, + &tenant, + &auth_event, + &req_headers, + &sha256, + strictness, + ) + .await?; serve_blob_for_tenant(&state, &tenant, &sha256_ext, &req_headers) .await .map_err(MediaDenial::from) @@ -1120,8 +1164,9 @@ pub(crate) async fn head_blob( .map(drop) }) .await; + let proof_sha256 = sha256.clone(); let admission = match admit_nip_fi_http_on_state(&state, &headers, move || { - extract_blossom_read_proof(&headers_clone, &sha256, &tenant_host, strictness) + extract_blossom_read_proof(&headers_clone, &proof_sha256, &tenant_host, strictness) .map_err(|e| e.into_response()) }) .await @@ -1130,7 +1175,8 @@ pub(crate) async fn head_blob( Err(resp) => return Ok(resp), }; let auth_event = admission.into_extra(); - enforce_blossom_read_membership(&state, &tenant, &auth_event, &headers, strictness).await?; + enforce_blossom_read_access(&state, &tenant, &auth_event, &headers, &sha256, strictness) + .await?; let cache_control = blob_cache_control(); // Sidecar gate FIRST — reject before any blob I/O. @@ -2144,11 +2190,13 @@ mod tests { const TEST_ISSUER: &str = "https://issuer.example"; const TEST_AUDIENCE: &str = "https://relay.example"; const TEST_KID: &str = "test-key-1"; - const TEST_EC_PKCS8_PEM: &str = "-----BEGIN PRIVATE KEY-----\n\ - MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n\ - WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n\ - zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n\ - -----END PRIVATE KEY-----\n"; + const TEST_EC_PKCS8_PEM: &str = concat!( + "-----BEGIN PRIVATE KEY-----\n", // sadscan:disable kingfisher.privkey.2 -- static test-only key + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n", + "WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n", + "zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n", + "-----END PRIVATE KEY-----\n", + ); /// Build an AppState with NIP-FI Enforce and a real injected P-256 verifier. async fn media_enforce_test_state() -> Option> { diff --git a/crates/buzz-relay/src/api/media_access.rs b/crates/buzz-relay/src/api/media_access.rs new file mode 100644 index 00000000000..c3f5e48e8b0 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access.rs @@ -0,0 +1,165 @@ +//! File-share rollout and the audited moderation exception. + +use std::sync::Arc; + +use buzz_audit::{AuditAction, NewAuditEntry}; +use buzz_core::TenantContext; +use buzz_media::MediaError; + +use crate::config::MediaReadMode; +use crate::handlers::moderation_authz::{ + authorize_moderation_action, ModerationAction, ModerationTarget, +}; +use crate::state::AppState; + +enum Access { + Shared, + Report(uuid::Uuid, Vec), + Denied, +} + +#[cfg(test)] +#[path = "media_access_postgres_tests.rs"] +mod postgres_tests; + +/// Called only after authentication and community admission, for GET and HEAD. +pub(super) async fn enforce( + state: &Arc, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, +) -> Result<(), MediaError> { + let mode = state.config.media_read_mode; + if mode == MediaReadMode::Off { + return Ok(()); + } + let mut decision = evaluate(state, tenant, actor, sha256).await; + if mode == MediaReadMode::Enforce { + if let Ok(Access::Report(report, event)) = &decision { + if let Err(error) = audit_access(state, tenant, actor, sha256, *report, event).await { + decision = Err(error); + } + } + } + let outcome = match &decision { + Ok(Access::Shared) => "allow", + Ok(Access::Report(..)) => "report", + Ok(Access::Denied) => "deny", + Err(_) => "error", + }; + metrics::counter!("buzz_media_read_access_total", "mode" => mode.as_str(), "outcome" => outcome) + .increment(1); + if mode == MediaReadMode::LogOnly { + // No identifiers in metrics; no report grant or audit write in shadow. + return Ok(()); + } + match decision? { + Access::Shared | Access::Report(..) => Ok(()), + Access::Denied => Err(MediaError::NotFound), + } +} + +async fn audit_access( + state: &AppState, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, + report: uuid::Uuid, + event: &[u8], +) -> Result<(), MediaError> { + let audit = state + .audit + .as_ref() + .ok_or(MediaError::AuthorizationUnavailable)?; + // Await durable audit, rather than the best-effort upload queue. + audit + .log(NewAuditEntry { + community_id: tenant.community(), + action: AuditAction::ModerationMediaRead, + actor_pubkey: Some(actor.to_vec()), + object_id: Some(sha256.to_owned()), + detail: serde_json::json!({"report_id": report, "event_id": hex::encode(event)}), + }) + .await + .map_err(|error| { + tracing::error!(%error, "Could not audit moderator media access"); + MediaError::AuthorizationUnavailable + })?; + Ok(()) +} + +async fn evaluate( + state: &Arc, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, +) -> Result { + evaluate_inner(state, tenant, actor, sha256) + .await + .map_err(|error| { + tracing::error!(%error, "Media read authorization lookup failed"); + MediaError::AuthorizationUnavailable + }) +} + +async fn evaluate_inner( + state: &Arc, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, +) -> anyhow::Result { + if state + .db + .can_read_blob(tenant.community(), sha256, actor) + .await? + { + return Ok(Access::Shared); + } + let Some((report, event)) = state + .db + .open_media_report(tenant.community(), sha256) + .await? + else { + return Ok(Access::Denied); + }; + match authorize_moderation_action( + tenant, + state, + actor, + None, + ModerationTarget::None, + ModerationAction::ViewQueue, + ) + .await + { + Ok(_) => {} + Err(error) if error.downcast_ref::().is_some() => return Err(error), + Err(_) => return Ok(Access::Denied), + } + anyhow::ensure!( + state.audit.is_some(), + "moderator media access requires durable auditing" + ); + Ok(Access::Report(report, event)) +} + +#[cfg(test)] +mod tests { + use crate::config::MediaReadMode; + + #[test] + fn rollout_modes_default_to_log_only_and_reject_unknown_values() { + assert_eq!(MediaReadMode::default(), MediaReadMode::LogOnly); + for (value, mode) in [ + ("", MediaReadMode::LogOnly), + ("log-only", MediaReadMode::LogOnly), + ("off", MediaReadMode::Off), + ("enforce", MediaReadMode::Enforce), + ] { + assert_eq!(value.parse::().unwrap(), mode); + } + for value in ["disabled", "Enforce", "log_only", "enforc"] { + assert!(value.parse::().is_err()); + } + } +} diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests.rs new file mode 100644 index 00000000000..07894296052 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access_postgres_tests.rs @@ -0,0 +1,501 @@ +use super::*; +use buzz_db::channel::{ChannelType, ChannelVisibility}; +use buzz_db::moderation::{NewReport, ReportTarget}; +use nostr::{EventBuilder, Keys, Kind, Tag}; +use sqlx::PgPool; +use std::time::Duration; +use uuid::Uuid; + +async fn fixture() -> (Arc, PgPool, TenantContext) { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + let db = buzz_db::Db::from_pool(pool.clone()); + let host = format!("media-review-{}.example", Uuid::new_v4().simple()); + let community = db.ensure_configured_community(&host).await.unwrap().id; + let mut config = crate::config::Config::for_test(); + config.require_relay_membership = false; + config.media_read_mode = MediaReadMode::Enforce; + let redis = deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .unwrap(); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new("redis://127.0.0.1:1", redis.clone()) + .await + .unwrap(), + ); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let storage = buzz_media::MediaStorage::new(&config.media).unwrap(); + let workflow = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let (state, _) = AppState::new( + config, + db, + redis, + buzz_audit::AuditService::new(pool.clone()), + pubsub, + auth, + buzz_search::SearchService::new(pool.clone()), + workflow, + Keys::generate(), + storage, + ); + ( + Arc::new(state), + pool, + TenantContext::resolved(community, host), + ) +} + +fn mode(state: &mut Arc, mode: MediaReadMode) { + Arc::make_mut(&mut Arc::get_mut(state).unwrap().config).media_read_mode = mode; +} + +async fn check( + state: &Arc, + tenant: &TenantContext, + keys: &Keys, + hash: &str, +) -> Result<(), MediaError> { + enforce(state, tenant, keys.public_key().as_bytes(), hash).await +} + +async fn post( + state: &AppState, + tenant: &TenantContext, + owner: &Keys, + channel: Uuid, +) -> (String, nostr::Event) { + let hash = format!("{:064x}", Uuid::new_v4().as_u128()); + let lease = state + .db + .deletion_store() + .acquire_serving_write_lease( + tenant.community(), + "media_upload", + "media-review-test", + Duration::from_secs(60), + ) + .await + .unwrap(); + state + .db + .record_media_uploader(&lease, &hash, owner.public_key().as_bytes()) + .await + .unwrap(); + let event = EventBuilder::new(Kind::from(9), format!("/media/{hash}.jpg")) + .sign_with_keys(owner) + .unwrap(); + state + .db + .insert_event(tenant.community(), &event, Some(channel)) + .await + .unwrap(); + (hash, event) +} + +async fn report( + state: &AppState, + tenant: &TenantContext, + reporter: &Keys, + target: ReportTarget, + note: Option<&str>, +) -> Uuid { + state + .db + .insert_moderation_report( + tenant.community(), + NewReport { + report_event_id: EventBuilder::new(Kind::from(1984), Uuid::new_v4().to_string()) + .sign_with_keys(reporter) + .unwrap() + .id + .as_bytes(), + reporter_pubkey: reporter.public_key().as_bytes(), + target, + channel_id: None, + report_type: "spam", + note, + }, + ) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn rollout_and_report_review_preserve_private_file_boundaries() { + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let (mut state, pool, tenant) = fixture().await; + let (owner, reviewer, stranger) = (Keys::generate(), Keys::generate(), Keys::generate()); + let community = tenant.community(); + let private = state + .db + .create_channel( + community, + "reported", + ChannelType::Stream, + ChannelVisibility::Private, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + let (hash, source) = post(&state, &tenant, &owner, private).await; + let (unrelated, _) = post(&state, &tenant, &owner, private).await; + let (edited_hash, seed) = post(&state, &tenant, &owner, private).await; + let edit = EventBuilder::new(Kind::from(40003), format!("/media/{edited_hash}")) + .tags([ + Tag::parse(["h", &private.to_string()]).unwrap(), + Tag::parse(["e", &source.id.to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + state + .db + .insert_event(community, &edit, Some(private)) + .await + .unwrap(); + state + .db + .soft_delete_event_and_update_thread(community, seed.id.as_bytes(), None, None) + .await + .unwrap(); + for policy in [ + MediaReadMode::Off, + MediaReadMode::LogOnly, + MediaReadMode::Enforce, + ] { + mode(&mut state, policy); + assert_eq!( + check(&state, &tenant, &stranger, &hash).await.is_ok(), + policy != MediaReadMode::Enforce + ); + assert!(check(&state, &tenant, &owner, &hash).await.is_ok()); + } + let _ = state + .db + .add_relay_member( + community, + &reviewer.public_key().to_hex(), + "admin", + None, + &state.relay_keypair, + ) + .await + .unwrap(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + ), + "role alone grants nothing" + ); + let report_id = report( + &state, + &tenant, + &owner, + ReportTarget::Event(source.id.to_bytes().to_vec()), + Some(&format!("/media/{unrelated}.jpg")), + ) + .await; + mode(&mut state, MediaReadMode::LogOnly); + assert!(check(&state, &tenant, &reviewer, &hash).await.is_ok()); + let before: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_log WHERE community_id = $1 AND action = 'moderation_media_read'") + .bind(community.as_uuid()).fetch_one(&pool).await.unwrap(); + assert_eq!( + before, 0, + "log-only must not claim an exceptional read occurred" + ); + mode(&mut state, MediaReadMode::Enforce); + assert!(check(&state, &tenant, &reviewer, &hash).await.is_ok()); + let audit: (Vec, String, serde_json::Value) = sqlx::query_as( + "SELECT actor_pubkey, object_id, detail FROM audit_log WHERE community_id = $1 AND action = 'moderation_media_read' ORDER BY seq DESC LIMIT 1") + .bind(community.as_uuid()).fetch_one(&pool).await.unwrap(); + assert_eq!(audit.0, reviewer.public_key().to_bytes()); + assert_eq!(audit.1, hash); + assert_eq!(audit.2["report_id"], report_id.to_string()); + assert_eq!(audit.2["event_id"], source.id.to_hex()); + assert!( + check(&state, &tenant, &reviewer, &edited_hash) + .await + .is_ok(), + "a report of the logical message covers its live edit attachment" + ); + assert!(matches!( + check(&state, &tenant, &stranger, &hash).await, + Err(MediaError::NotFound) + )); + assert!( + matches!( + check(&state, &tenant, &reviewer, &unrelated).await, + Err(MediaError::NotFound) + ), + "report notes are not attachments" + ); + report( + &state, + &tenant, + &owner, + ReportTarget::Blob(hex::decode(&unrelated).unwrap()), + None, + ) + .await; + assert!( + matches!( + check(&state, &tenant, &reviewer, &unrelated).await, + Err(MediaError::NotFound) + ), + "a raw reported hash grants nothing" + ); + for role in ["member", "owner", "admin"] { + let _ = state + .db + .update_relay_member_role( + community, + &reviewer.public_key().to_hex(), + role, + &state.relay_keypair, + ) + .await + .unwrap(); + assert_eq!( + check(&state, &tenant, &reviewer, &hash).await.is_ok(), + role != "member" + ); + } + // Other communities cannot supply the grant, report, or queue role. + let other = state + .db + .ensure_configured_community(&format!("other-{}.example", Uuid::new_v4())) + .await + .unwrap(); + let other_tenant = TenantContext::resolved(other.id, other.host); + let _ = state + .db + .add_relay_member( + other.id, + &reviewer.public_key().to_hex(), + "admin", + None, + &state.relay_keypair, + ) + .await + .unwrap(); + assert!(matches!( + check(&state, &other_tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + )); + state + .db + .resolve_moderation_report( + community, + report_id, + "dismissed", + reviewer.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + ), + "closing a report revokes the exception" + ); + assert!(matches!( + check(&state, &tenant, &reviewer, &edited_hash).await, + Err(MediaError::NotFound) + )); + report( + &state, + &tenant, + &owner, + ReportTarget::Event(source.id.to_bytes().to_vec()), + None, + ) + .await; + assert!(check(&state, &tenant, &reviewer, &hash).await.is_ok()); + let audit = Arc::get_mut(&mut state).unwrap().audit.take(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::AuthorizationUnavailable) + ), + "auditing is required" + ); + Arc::get_mut(&mut state).unwrap().audit = audit; + let broken_pool = PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + broken_pool.close().await; + let audit = Arc::get_mut(&mut state) + .unwrap() + .audit + .replace(Arc::new(buzz_audit::AuditService::new(broken_pool))); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::AuthorizationUnavailable) + ), + "audit commit failure must deny before serving bytes" + ); + Arc::get_mut(&mut state).unwrap().audit = audit; + assert!( + !state + .db + .can_read_blob(community, &hash, reviewer.public_key().as_bytes()) + .await + .unwrap(), + "report review creates no permanent share" + ); + state + .db + .soft_delete_event_and_update_thread(community, source.id.as_bytes(), None, None) + .await + .unwrap(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + ), + "deleted messages grant no exception" + ); + assert!( + matches!( + check(&state, &tenant, &reviewer, &edited_hash).await, + Err(MediaError::NotFound) + ), + "an open report cannot revive an edit of a deleted message" + ); + let mut cleanup_cursor = None; + loop { + let batch = state + .db + .reap_inactive_media_shares(cleanup_cursor.as_ref(), 1000) + .await + .unwrap(); + cleanup_cursor = batch.next_cursor; + if cleanup_cursor.is_none() { + break; + } + } + for cleaned_hash in [&hash, &edited_hash] { + assert!( + matches!( + check(&state, &tenant, &owner, cleaned_hash).await, + Err(MediaError::NotFound) + ), + "cleanup must not restore uploader preview for a deleted channel attachment" + ); + } + pool.close().await; + for policy in [ + MediaReadMode::Off, + MediaReadMode::LogOnly, + MediaReadMode::Enforce, + ] { + mode(&mut state, policy); + assert_eq!( + check(&state, &tenant, &stranger, &hash).await.is_ok(), + policy != MediaReadMode::Enforce, + "lookup failure follows rollout policy" + ); + } + let counts = snapshotter.snapshot().into_vec(); + for (policy, outcome) in [ + ("log-only", "deny"), + ("log-only", "error"), + ("enforce", "deny"), + ("enforce", "report"), + ] { + assert!( + counts.iter().any(|(key, _, _, value)| key.key().name() + == "buzz_media_read_access_total" + && key + .key() + .labels() + .any(|l| l.key() == "mode" && l.value() == policy) + && key + .key() + .labels() + .any(|l| l.key() == "outcome" && l.value() == outcome) + && matches!(value, metrics_util::debugging::DebugValue::Counter(n) if *n > 0)), + "missing {policy}/{outcome} rollout count" + ); + } + assert!(counts + .iter() + .filter(|(key, _, _, _)| key.key().name() == "buzz_media_read_access_total") + .all(|(key, _, _, _)| key + .key() + .labels() + .all(|l| matches!(l.key(), "mode" | "outcome")))); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn every_rollout_mode_keeps_http_authentication_and_community_admission() { + use axum::{body::Body, http::Request, routing::get, Router}; + use base64::Engine; + use nostr::{JsonUtil, Tag, Timestamp}; + use tower::ServiceExt; + + let (mut state, _, tenant) = fixture().await; + Arc::make_mut(&mut Arc::get_mut(&mut state).unwrap().config).require_relay_membership = true; + let stranger = Keys::generate(); + for policy in [ + MediaReadMode::Off, + MediaReadMode::LogOnly, + MediaReadMode::Enforce, + ] { + mode(&mut state, policy); + for method in ["GET", "HEAD"] { + for authenticated in [false, true] { + let auth = EventBuilder::new(Kind::from(24242), "read") + .tags([ + Tag::parse(["t", "get"]).unwrap(), + Tag::parse(["server", tenant.host()]).unwrap(), + Tag::parse(["expiration", &(Timestamp::now().as_secs() + 55).to_string()]) + .unwrap(), + ]) + .sign_with_keys(&stranger) + .unwrap(); + let mut request = Request::builder() + .method(method) + .uri(format!("/media/{}.jpg", "a".repeat(64))) + .header("host", tenant.host()); + if authenticated { + request = request.header( + "authorization", + format!( + "Nostr {}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(auth.as_json()) + ), + ); + } + let response = Router::new() + .route( + "/media/{sha256_ext}", + get(super::super::media::get_blob).head(super::super::media::head_blob), + ) + .with_state(state.clone()) + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!( + response.status().as_u16(), + if authenticated { 403 } else { 401 }, + "{policy:?} {method}" + ); + } + } + } +} diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index 89ff79f7e4c..f1df14da10a 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -8,6 +8,7 @@ pub mod gifs; pub mod git; pub mod invites; pub mod media; +mod media_access; pub mod mesh_demo; pub mod nip05; pub mod nip_fi; diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index c0e30144099..bb0b23560f4 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -24,6 +24,54 @@ pub enum ConfigError { InvalidValue(String), } +/// Rollout policy for file shares, independent of authentication and membership. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum MediaReadMode { + /// Preserve authenticated, community-scoped reads without checking shares. + Off, + /// Evaluate shares and record outcomes without denying reads. + #[default] + LogOnly, + /// Require a readable share, upload proof, or audited report access. + Enforce, +} + +impl MediaReadMode { + fn from_env() -> Result { + match std::env::var("BUZZ_MEDIA_READ_MODE") { + Ok(value) => value.parse(), + Err(std::env::VarError::NotPresent) => Ok(Self::default()), + Err(std::env::VarError::NotUnicode(_)) => Err(ConfigError::InvalidValue( + "BUZZ_MEDIA_READ_MODE must be valid Unicode".into(), + )), + } + } + + /// Stable configuration and metric label. + pub const fn as_str(self) -> &'static str { + match self { + Self::Off => "off", + Self::LogOnly => "log-only", + Self::Enforce => "enforce", + } + } +} + +impl std::str::FromStr for MediaReadMode { + type Err = ConfigError; + + fn from_str(value: &str) -> Result { + match value { + "off" => Ok(Self::Off), + "" | "log-only" => Ok(Self::LogOnly), + "enforce" => Ok(Self::Enforce), + _ => Err(ConfigError::InvalidValue( + "BUZZ_MEDIA_READ_MODE must be off, log-only, or enforce".into(), + )), + } + } +} + /// Authentication mode for the deployment-admin API. /// /// Configured by `BUZZ_ADMIN_AUTH`: unset/empty/`nip98` → `Nip98` (fail-secure @@ -399,6 +447,10 @@ pub struct Config { /// leaves all identity enforcement to NIP-42 alone. pub nip_fi: crate::nip_fi_config::NipFiRelayConfig, + /// `BUZZ_MEDIA_READ_MODE`: off, log-only (default), or enforce. + /// Loaded at startup; authentication and community membership always apply. + pub media_read_mode: MediaReadMode, + /// Issuer binding written to communities created under a host suffix /// (`BUZZ_NIP_FI_PROVISION_BINDING`). `None` binds nothing at creation. /// Parsed whatever the NIP-FI mode, since creation happens in every mode. @@ -1383,6 +1435,7 @@ impl Config { }; let nip_fi = crate::nip_fi_config::NipFiRelayConfig::from_env()?; + let media_read_mode = MediaReadMode::from_env()?; let nip_fi_provision_binding = crate::nip_fi_config::parse_provision_binding()?; let nip_fi_resolver_bounds = crate::nip_fi_resolver::ResolverBounds::from_env(db_pool_size)?; @@ -1463,6 +1516,7 @@ impl Config { nip_fi_provision_binding, nip_fi_resolver_bounds, nip_fi, + media_read_mode, }) } diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index fa181f2a473..1818c7fc0a4 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -134,6 +134,7 @@ const _: () = assert!(FLEET_STOCK_COLLECTION_DEADLINE.as_secs() > 5); const _: () = assert!(FLEET_ACTIVITY_COLLECTION_DEADLINE.as_secs() > 15); struct FleetUsageSchedule { + media_cleanup_cursor: Option, next_stock: std::time::Instant, next_activity: std::time::Instant, last_stock_success: Option, @@ -167,6 +168,7 @@ impl FleetUsageSchedule { retry_interval: std::time::Duration, ) -> Self { Self { + media_cleanup_cursor: None, next_stock: now, next_activity: now, last_stock_success: None, @@ -484,6 +486,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { metrics_port = config.metrics_port, max_frame_bytes = config.max_frame_bytes, audit_enabled = config.audit_enabled, + media_read_mode = config.media_read_mode.as_str(), push_enabled = config.push_enabled, "Config loaded" ); @@ -2505,6 +2508,20 @@ async fn run_usage_metrics_tick( } } if leader.is_some() { + // Run maintenance before optional telemetry collection. + match state + .db + .reap_inactive_media_shares(fleet_schedule.media_cleanup_cursor.as_ref(), 1000) + .await + { + Ok(batch) => { + fleet_schedule.media_cleanup_cursor = batch.next_cursor; + if batch.removed > 0 { + info!(removed = batch.removed, "reaped inactive media shares"); + } + } + Err(error) => warn!(error = %error, "failed to reap inactive media shares"), + } if emission_scope.includes_per_community() { let result = emit_db_usage_metrics(state, emission_scope, &host_map).await; record_legacy_usage_collection(result.is_ok(), fleet_schedule); diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index 67e4d3b35e2..a1ada5cebb9 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -297,6 +297,10 @@ pub fn try_install(port: u16, gauge_idle_timeout_secs: u64) -> Result<(), Metric describe_community_admission_metrics(); describe_db_pool_metrics(); describe_auth_metrics(); + metrics::describe_counter!( + "buzz_media_read_access_total", + "File authorization evaluations by rollout mode and bounded outcome" + ); crate::startup_steps::describe_metrics(); initialize_auth_metric_series(); initialize_partition_audit_metric_series(); diff --git a/crates/buzz-test-client/tests/conformance_multitenant.rs b/crates/buzz-test-client/tests/conformance_multitenant.rs index 4c8c8904ac5..583c0d43b44 100644 --- a/crates/buzz-test-client/tests/conformance_multitenant.rs +++ b/crates/buzz-test-client/tests/conformance_multitenant.rs @@ -2621,10 +2621,6 @@ mod media_blossom { /// the bytes: it is the metadata/descriptor/upload-auth/quota/audit layer plus /// the per-tenant read binding. B's private upload metadata and errors must not /// be observable from A even when the underlying blob is shared. - /// - /// Known limitation, deferred: relay membership plus knowledge of a hash is - /// sufficient to read a blob. Read auth binds host and tenant, not the channel - /// ACL of the message the blob was attached to. #[tokio::test] #[ignore] async fn media_metadata_boundary_holds_while_blob_bytes_shared() { diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 9bd73b13fcb..31d1ea8cdf2 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -1,7 +1,8 @@ //! Extended media upload integration tests — auth edge cases, content validation, //! multi-format uploads, WebSocket imeta validation. //! -//! Run: cargo test -p buzz-test-client --test e2e_media_extended -- --ignored --nocapture +//! Run: cargo test -p buzz-test-client --test e2e_media_extended -- --ignored --nocapture --test-threads=1 +//! The full suite expects BUZZ_MEDIA_READ_MODE=enforce on the relay. use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; @@ -37,6 +38,36 @@ fn http_client() -> Client { .expect("http client") } +async fn media_db_pool() -> sqlx::PgPool { + let database = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".into()); // sadscan:disable np.postgres.1 -- local test-only credentials + sqlx::PgPool::connect(&database).await.unwrap() +} + +async fn wait_for_share_cleanup(hash: &str, maximum: i64) { + let pool = media_db_pool().await; + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM media_shares s JOIN communities c ON c.id = s.community_id \ + WHERE c.host = $1 AND s.sha256 = $2", + ) + .bind(relay_server_authority()) + .bind(hash) + .fetch_one(&pool) + .await + .unwrap(); + if remaining <= maximum { + break; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + }) + .await + .expect("periodic cleanup must reclaim inactive grant rows"); + pool.close().await; +} + fn sign_blossom_auth(keys: &Keys, sha256: &str) -> nostr::Event { let now = Timestamp::now().as_secs(); let server = relay_server_authority(); @@ -98,6 +129,55 @@ async fn upload_to_path( .expect("upload request") } +/// Status of an authenticated Blossom read of `url` by `keys`. +async fn read_status(client: &Client, keys: &Keys, method: reqwest::Method, url: &str) -> u16 { + let sha256 = url + .rsplit('/') + .next() + .and_then(|file| file.split('.').next()) + .expect("media url ends in {sha256}.{ext}"); + client + .request(method, url) + .header( + "Authorization", + blossom_auth_header(&sign_blossom_get_auth(keys, sha256)), + ) + .send() + .await + .expect("read request") + .status() + .as_u16() +} + +/// Create a stream channel owned by `keys` via a signed kind:9007; returns its id. +async fn create_channel( + client: &Client, + keys: &Keys, + name_prefix: &str, + visibility: &str, +) -> String { + let channel_id = uuid::Uuid::new_v4().to_string(); + let create_event = EventBuilder::new(Kind::from(9007), "") + .tags(vec![ + Tag::parse(["h", &channel_id]).unwrap(), + Tag::parse(["name", &format!("{name_prefix}-{channel_id}")]).unwrap(), + Tag::parse(["channel_type", "stream"]).unwrap(), + Tag::parse(["visibility", visibility]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + let create_resp = client + .post(format!("{}/events", relay_http_url())) + .header("X-Pubkey", keys.public_key().to_hex()) + .header("Content-Type", "application/json") + .body(serde_json::to_string(&create_event).unwrap()) + .send() + .await + .unwrap(); + assert!(create_resp.status().is_success(), "channel creation failed"); + channel_id +} + fn tiny_jpeg() -> Vec { vec![ 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, @@ -200,6 +280,24 @@ async fn test_upload_png_roundtrip() { // GET back — reads are authenticated, so scope a token to the uploaded hash. let sha256 = desc["sha256"].as_str().expect("descriptor sha256"); + // Bind uploader recording to the real HTTP handler, independently of + // whether this relay currently enforces file reads. + let pool = media_db_pool().await; + let recorded: i64 = sqlx::query_scalar( + "SELECT count(*) FROM media_uploaders u JOIN communities c ON c.id = u.community_id \ + WHERE c.host = $1 AND u.sha256 = $2 AND u.pubkey = $3", + ) + .bind(relay_server_authority()) + .bind(sha256) + .bind(keys.public_key().to_bytes().as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + recorded, 1, + "successful upload must persist its authenticated uploader" + ); + pool.close().await; let get = client .get(desc["url"].as_str().unwrap()) .header( @@ -411,7 +509,7 @@ async fn test_auth_server_tag_correct() { Tag::parse(["t", "upload"]).unwrap(), Tag::parse(["x", &sha256]).unwrap(), Tag::parse(["expiration", &(now + 55).to_string()]).unwrap(), - Tag::parse(["server", "localhost:3000"]).unwrap(), + Tag::parse(["server", &relay_server_authority()]).unwrap(), ], ); let resp = upload_with_auth(&client, &auth, &sha256, &jpeg).await; @@ -631,38 +729,16 @@ async fn test_ws_valid_imeta() { use buzz_test_client::BuzzTestClient; let keys = Keys::generate(); - let pubkey_hex = keys.public_key().to_hex(); let http = http_client(); - // Create channel via signed kind:9007 event - let channel_uuid = uuid::Uuid::new_v4(); - let channel_name = format!("ws-imeta-test-{}", channel_uuid); - let create_event = EventBuilder::new(Kind::from(9007), "") - .tags(vec![ - Tag::parse(["h", &channel_uuid.to_string()]).unwrap(), - Tag::parse(["name", &channel_name]).unwrap(), - Tag::parse(["channel_type", "stream"]).unwrap(), - Tag::parse(["visibility", "open"]).unwrap(), - ]) - .sign_with_keys(&keys) - .unwrap(); - let create_resp = http - .post(format!("{}/events", relay_http_url())) - .header("X-Pubkey", &pubkey_hex) - .header("Content-Type", "application/json") - .body(serde_json::to_string(&create_event).unwrap()) - .send() - .await - .unwrap(); - assert!(create_resp.status().is_success(), "channel creation failed"); - let channel_id = channel_uuid.to_string(); + let channel_id = create_channel(&http, &keys, "ws-imeta-test", "open").await; println!("Channel: {channel_id}"); // Upload a JPEG to get a valid sha256 let jpeg = tiny_jpeg(); - let sha256 = hex::encode(Sha256::digest(&jpeg)); let resp = upload(&http, &keys, &jpeg).await; assert_eq!(resp.status(), 200); + let descriptor: serde_json::Value = resp.json().await.unwrap(); // Connect via WebSocket let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) @@ -673,14 +749,7 @@ async fn test_ws_valid_imeta() { let event = EventBuilder::new(Kind::from(9), "image via ws") .tags(vec![ Tag::parse(["h", &channel_id]).unwrap(), - Tag::parse([ - "imeta", - &format!("url http://localhost:3000/media/{sha256}.jpg"), - "m image/jpeg", - &format!("x {sha256}"), - "size 347", - ]) - .unwrap(), + imeta_for(&descriptor), ]) .sign_with_keys(&keys) .unwrap(); @@ -702,30 +771,9 @@ async fn test_ws_invalid_imeta_external_url() { use buzz_test_client::BuzzTestClient; let keys = Keys::generate(); - let pubkey_hex = keys.public_key().to_hex(); let http = http_client(); - let channel_uuid = uuid::Uuid::new_v4(); - let channel_name = format!("ws-imeta-bad-{}", channel_uuid); - let create_event = EventBuilder::new(Kind::from(9007), "") - .tags(vec![ - Tag::parse(["h", &channel_uuid.to_string()]).unwrap(), - Tag::parse(["name", &channel_name]).unwrap(), - Tag::parse(["channel_type", "stream"]).unwrap(), - Tag::parse(["visibility", "open"]).unwrap(), - ]) - .sign_with_keys(&keys) - .unwrap(); - let create_resp = http - .post(format!("{}/events", relay_http_url())) - .header("X-Pubkey", &pubkey_hex) - .header("Content-Type", "application/json") - .body(serde_json::to_string(&create_event).unwrap()) - .send() - .await - .unwrap(); - assert!(create_resp.status().is_success(), "channel creation failed"); - let channel_id = channel_uuid.to_string(); + let channel_id = create_channel(&http, &keys, "ws-imeta-bad", "open").await; let sha = "a".repeat(64); let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) @@ -768,30 +816,9 @@ async fn test_ws_invalid_imeta_missing_fields() { use buzz_test_client::BuzzTestClient; let keys = Keys::generate(); - let pubkey_hex = keys.public_key().to_hex(); let http = http_client(); - let channel_uuid = uuid::Uuid::new_v4(); - let channel_name = format!("ws-imeta-miss-{}", channel_uuid); - let create_event = EventBuilder::new(Kind::from(9007), "") - .tags(vec![ - Tag::parse(["h", &channel_uuid.to_string()]).unwrap(), - Tag::parse(["name", &channel_name]).unwrap(), - Tag::parse(["channel_type", "stream"]).unwrap(), - Tag::parse(["visibility", "open"]).unwrap(), - ]) - .sign_with_keys(&keys) - .unwrap(); - let create_resp = http - .post(format!("{}/events", relay_http_url())) - .header("X-Pubkey", &pubkey_hex) - .header("Content-Type", "application/json") - .body(serde_json::to_string(&create_event).unwrap()) - .send() - .await - .unwrap(); - assert!(create_resp.status().is_success(), "channel creation failed"); - let channel_id = channel_uuid.to_string(); + let channel_id = create_channel(&http, &keys, "ws-imeta-miss", "open").await; let sha = "b".repeat(64); let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) @@ -817,3 +844,448 @@ async fn test_ws_invalid_imeta_missing_fields() { client.disconnect().await.unwrap(); } + +#[tokio::test] +#[ignore] +async fn test_private_channel_attachment_unreadable_to_non_members() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let owner = Keys::generate(); + let outsider = Keys::generate(); + let http = http_client(); + let channel_id = create_channel(&http, &owner, "private-media", "private").await; + + let body = uuid::Uuid::new_v4().as_bytes().repeat(8); + let resp = upload(&http, &owner, &body).await; + assert_eq!(resp.status(), 200); + let desc: serde_json::Value = resp.json().await.unwrap(); + let url = desc["url"].as_str().expect("descriptor url"); + + assert_eq!( + read_status(&http, &owner, Method::GET, url).await, + 200, + "an uploader can preview an unposted upload" + ); + assert_eq!( + read_status(&http, &outsider, Method::GET, url).await, + 404, + "an unposted upload is readable only by its uploader" + ); + + let imeta = imeta_for(&desc); + + let mut owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let event = EventBuilder::new(Kind::from(9), "private attachment") + .tags(vec![Tag::parse(["h", &channel_id]).unwrap(), imeta.clone()]) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_client.send_event(event).await.unwrap(); + assert!(ok.accepted, "attachment rejected: {:?}", ok.message); + owner_client.disconnect().await.unwrap(); + + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &outsider, method.clone(), url).await, + 404, + "{method} by a non-member must not reveal a private-channel attachment" + ); + assert_eq!( + read_status(&http, &owner, method.clone(), url).await, + 200, + "{method} by a channel member must still succeed" + ); + } + + // Exercise the actual membership tombstone, not just never-joined readers. + let recipient = BuzzTestClient::connect(&relay_ws_url(), &outsider) + .await + .unwrap(); + let mut membership = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + for (kind, expected) in [(9000, 200), (9001, 404)] { + let change = EventBuilder::new(Kind::from(kind), "") + .tags([ + Tag::parse(["h", &channel_id]).unwrap(), + Tag::parse(["p", &outsider.public_key().to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = membership.send_event(change).await.unwrap(); + assert!( + result.accepted, + "membership change rejected: {}", + result.message + ); + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &outsider, method.clone(), url).await, + expected, + "removed private-channel members must lose media access" + ); + assert_eq!(read_status(&http, &owner, method, url).await, 200); + } + } + membership.disconnect().await.unwrap(); + recipient.disconnect().await.unwrap(); + + let outsider_channel = create_channel(&http, &outsider, "outsider-media", "open").await; + let mut outsider_client = BuzzTestClient::connect(&relay_ws_url(), &outsider) + .await + .unwrap(); + let mut post_in_outsider_channel = async |content: &str| { + let event = EventBuilder::new(Kind::from(9), content) + .tags(vec![ + Tag::parse(["h", &outsider_channel]).unwrap(), + imeta.clone(), + ]) + .sign_with_keys(&outsider) + .unwrap(); + let ok = outsider_client.send_event(event).await.unwrap(); + assert!(ok.accepted, "post rejected: {:?}", ok.message); + }; + let viewer = Keys::generate(); + + post_in_outsider_channel("reshare by hash").await; + assert_eq!( + read_status(&http, &viewer, Method::GET, url).await, + 404, + "referencing a known hash must not share the blob" + ); + + assert_eq!(upload(&http, &outsider, &body).await.status(), 200); + assert_eq!( + read_status(&http, &outsider, Method::GET, url).await, + 404, + "uploading grants no read access until the blob is shared" + ); + post_in_outsider_channel("my own copy").await; + assert_eq!( + read_status(&http, &viewer, Method::GET, url).await, + 200, + "uploading the same bytes lets the uploader share it into their channel" + ); + outsider_client.disconnect().await.unwrap(); +} + +/// NIP-92 `imeta` tag attaching the blob an upload `desc` describes. +fn imeta_for(desc: &serde_json::Value) -> Tag { + Tag::parse([ + "imeta", + &format!("url {}", desc["url"].as_str().unwrap()), + &format!("m {}", desc["type"].as_str().unwrap()), + &format!("x {}", desc["sha256"].as_str().unwrap()), + &format!("size {}", desc["size"]), + ]) + .unwrap() +} + +#[tokio::test] +#[ignore] +async fn test_deleted_messages_revoke_only_their_own_media_shares() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let (owner, viewer) = (Keys::generate(), Keys::generate()); + let http = http_client(); + let private = create_channel(&http, &owner, "retained-media", "private").await; + let open = create_channel(&http, &owner, "deleted-media", "open").await; + let body = uuid::Uuid::new_v4().as_bytes().repeat(8); + let response = upload(&http, &owner, &body).await; + assert_eq!(response.status(), 200); + let desc: serde_json::Value = response.json().await.unwrap(); + let url = desc["url"].as_str().unwrap(); + let mut client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let mut messages = Vec::new(); + for (name, channel) in [ + ("private copy", &private), + ("first", &open), + ("last", &open), + ] { + let event = EventBuilder::new(Kind::from(9), name) + .tags([Tag::parse(["h", channel]).unwrap(), imeta_for(&desc)]) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(event.clone()).await.unwrap(); + assert!(result.accepted, "attachment rejected: {}", result.message); + messages.push(event); + } + let added = upload(&http, &owner, &uuid::Uuid::new_v4().as_bytes().repeat(8)).await; + assert_eq!(added.status(), 200); + let added_desc: serde_json::Value = added.json().await.unwrap(); + let added_url = added_desc["url"].as_str().unwrap(); + for message in &messages[1..] { + let edit = EventBuilder::new(Kind::from(40003), "retain and add attachments") + .tags([ + Tag::parse(["h", &open]).unwrap(), + Tag::parse(["e", &message.id.to_hex()]).unwrap(), + imeta_for(&desc), + imeta_for(&added_desc), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(edit).await.unwrap(); + assert!(result.accepted, "edit rejected: {}", result.message); + } + for reader in [&owner, &viewer] { + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, reader, method, url).await, 200); + } + } + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method, added_url).await, 200); + } + // Author deletion removes the first post; the second must keep its grant. + // Moderator removal then removes the last open-channel grant. + for (kind, index, expected) in [(5, 1, 200), (9005, 2, 404)] { + let mut tags = vec![Tag::parse(["e", &messages[index].id.to_hex()]).unwrap()]; + if kind == 5 { + tags.push(Tag::parse(["k", "9"]).unwrap()); + } else { + tags.push(Tag::parse(["h", &open]).unwrap()); + } + let deletion = EventBuilder::new(Kind::from(kind), "remove attachment") + .tags(tags) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(deletion).await.unwrap(); + assert!(result.accepted, "deletion rejected: {}", result.message); + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &viewer, method.clone(), url).await, + expected + ); + assert_eq!(read_status(&http, &owner, method.clone(), url).await, 200); + assert_eq!( + read_status(&http, &viewer, method.clone(), added_url).await, + expected, + "deleting the logical message must revoke its edit-only attachment" + ); + } + } + + // Wait for the actual periodic worker to compact the two dead edit grants. + // Losing every channel row would incorrectly restore uploader preview. + wait_for_share_cleanup(added_desc["sha256"].as_str().unwrap(), 1).await; + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &owner, method.clone(), added_url).await, + 404 + ); + assert_eq!(read_status(&http, &viewer, method, added_url).await, 404); + } + + // A community-wide note grants access only for as long as it survives. + let note = EventBuilder::new(Kind::TextNote, format!("shared file {url}")) + .sign_with_keys(&owner) + .unwrap(); + assert!(client.send_event(note.clone()).await.unwrap().accepted); + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method, url).await, 200); + } + let deletion = EventBuilder::new(Kind::EventDeletion, "remove note") + .tags([ + Tag::parse(["e", ¬e.id.to_hex()]).unwrap(), + Tag::parse(["k", "1"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(deletion).await.unwrap(); + assert!( + result.accepted, + "note deletion rejected: {}", + result.message + ); + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method.clone(), url).await, 404); + assert_eq!(read_status(&http, &owner, method, url).await, 200); + } + client.disconnect().await.unwrap(); +} + +#[tokio::test] +#[ignore] +async fn test_private_attachment_cannot_hide_an_avatar() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let owner = Keys::generate(); + let griefer = Keys::generate(); + let viewer = Keys::generate(); + let http = http_client(); + + let body = uuid::Uuid::new_v4().as_bytes().repeat(8); + let resp = upload(&http, &owner, &body).await; + assert_eq!(resp.status(), 200); + let desc: serde_json::Value = resp.json().await.unwrap(); + let url = desc["url"].as_str().expect("descriptor url"); + + let mut owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({ "picture": url }).to_string(), + ) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_client.send_event(profile).await.unwrap(); + assert!(ok.accepted, "profile rejected: {:?}", ok.message); + owner_client.disconnect().await.unwrap(); + + let channel_id = create_channel(&http, &griefer, "hide-avatar", "private").await; + let mut griefer_client = BuzzTestClient::connect(&relay_ws_url(), &griefer) + .await + .unwrap(); + let event = EventBuilder::new(Kind::from(9), "attach someone's avatar") + .tags(vec![ + Tag::parse(["h", &channel_id]).unwrap(), + imeta_for(&desc), + ]) + .sign_with_keys(&griefer) + .unwrap(); + let ok = griefer_client.send_event(event).await.unwrap(); + assert!(ok.accepted, "attachment rejected: {:?}", ok.message); + griefer_client.disconnect().await.unwrap(); + + assert_eq!( + read_status(&http, &viewer, Method::GET, url).await, + 200, + "a private attachment must not hide a current avatar" + ); +} + +#[tokio::test] +#[ignore] +async fn test_profile_replacement_preserves_only_its_retained_avatar() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let (uploader, publisher, viewer) = (Keys::generate(), Keys::generate(), Keys::generate()); + let http = http_client(); + let mut image = tiny_gif(); + image[13..19].copy_from_slice(&uuid::Uuid::new_v4().as_bytes()[..6]); + let response = upload(&http, &uploader, &image).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + let base = Timestamp::now().as_secs(); + for (signer, keep_picture, step, expected) in [ + (&uploader, true, 0, 200), + (&publisher, true, 0, 200), + (&uploader, false, 1, 200), + (&publisher, true, 1, 200), + (&publisher, false, 2, 404), + (&publisher, true, 3, 404), + ] { + let mut client = BuzzTestClient::connect(&relay_ws_url(), signer) + .await + .unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({ + "name": format!("updated-profile-{step}"), + "picture": if keep_picture { url } else { "" }, + }) + .to_string(), + ) + .custom_created_at(Timestamp::from_secs(base + step)) + .sign_with_keys(signer) + .unwrap(); + let result = client.send_event(profile).await.unwrap(); + assert!(result.accepted, "profile rejected: {}", result.message); + client.disconnect().await.unwrap(); + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &viewer, method, url).await, + expected, + "retaining a picture preserves its grant; removing it revokes it" + ); + } + } + wait_for_share_cleanup(descriptor["sha256"].as_str().unwrap(), 0).await; + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method, url).await, 404); + } +} + +#[tokio::test] +#[ignore] +async fn test_agent_avatar_uploaded_by_owner_is_readable_to_other_members() { + use buzz_sdk::nip_oa; + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let (owner, agent, viewer) = (Keys::generate(), Keys::generate(), Keys::generate()); + let http = http_client(); + let owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let mut image = tiny_gif(); + image[13..19].copy_from_slice(&uuid::Uuid::new_v4().as_bytes()[..6]); + let response = upload(&http, &owner, &image).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + assert_eq!(read_status(&http, &viewer, Method::GET, url).await, 404); + + let tag = nip_oa::compute_auth_tag(&owner, &agent.public_key(), "kind=0").unwrap(); + let tag = nip_oa::parse_auth_tag(&tag).unwrap(); + let mut agent_client = BuzzTestClient::connect_unauthenticated(&relay_ws_url()) + .await + .unwrap(); + agent_client + .authenticate_with_nip_oa(&agent, &tag) + .await + .unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture": url}).to_string(), + ) + .sign_with_keys(&agent) + .unwrap(); + let result = agent_client.send_event(profile).await.unwrap(); + assert!( + result.accepted, + "agent profile rejected: {}", + result.message + ); + for reader in [&agent, &viewer] { + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, reader, method, url).await, 200); + } + } + agent_client.disconnect().await.unwrap(); + owner_client.disconnect().await.unwrap(); +} + +#[tokio::test] +#[ignore] +async fn test_media_read_rollout_policy() { + use reqwest::Method; + let policy = std::env::var("MEDIA_READ_TEST_MODE").unwrap_or_else(|_| "enforce".into()); + assert!(["off", "log-only", "enforce"].contains(&policy.as_str())); + let (owner, outsider) = (Keys::generate(), Keys::generate()); + let http = http_client(); + let response = upload(&http, &owner, &uuid::Uuid::new_v4().as_bytes().repeat(8)).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &owner, method.clone(), url).await, 200); + assert_eq!( + read_status(&http, &outsider, method.clone(), url).await, + if policy == "enforce" { 404 } else { 200 }, + "{policy} {method}" + ); + assert_eq!( + http.request(method, url).send().await.unwrap().status(), + 401 + ); + } +} diff --git a/crates/buzz-test-client/tests/e2e_media_review.rs b/crates/buzz-test-client/tests/e2e_media_review.rs new file mode 100644 index 00000000000..1578d222837 --- /dev/null +++ b/crates/buzz-test-client/tests/e2e_media_review.rs @@ -0,0 +1,186 @@ +//! Live report -> private file review -> report closure journey. +//! Requires the relay's database and BUZZ_MEDIA_READ_MODE=enforce. + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use buzz_test_client::BuzzTestClient; +use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; +use reqwest::{Client, Method}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +fn auth(keys: &Keys, host: &str, verb: &str, hash: &str) -> String { + let event = EventBuilder::new(Kind::from(24242), "media review test") + .tags([ + Tag::parse(["t", verb]).unwrap(), + Tag::parse(["server", host]).unwrap(), + Tag::parse(["x", hash]).unwrap(), + Tag::parse(["expiration", &(Timestamp::now().as_secs() + 55).to_string()]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + format!("Nostr {}", URL_SAFE_NO_PAD.encode(event.as_json())) +} + +async fn read(http: &Client, keys: &Keys, base: &str, host: &str, hash: &str, expected: u16) { + for method in [Method::GET, Method::HEAD] { + let response = http + .request(method.clone(), format!("{base}/media/{hash}")) + .header("authorization", auth(keys, host, "get", hash)) + .send() + .await + .unwrap(); + assert_eq!( + response.status().as_u16(), + expected, + "{method} for {}", + keys.public_key() + ); + } +} + +#[tokio::test] +#[ignore] +async fn open_report_grants_only_audited_moderator_attachment_access() { + let base = std::env::var("RELAY_HTTP_URL").unwrap_or_else(|_| "http://localhost:3000".into()); + let host = url::Url::parse(&base).unwrap().authority().to_owned(); + let ws = base + .replace("http://", "ws://") + .replace("https://", "wss://"); + let database = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".into()); // sadscan:disable np.postgres.1 -- local test-only credentials + let pool = sqlx::PgPool::connect(&database).await.unwrap(); + let community: Uuid = sqlx::query_scalar("SELECT id FROM communities WHERE host = $1") + .bind(&host) + .fetch_one(&pool) + .await + .unwrap(); + let (owner, moderator, stranger) = (Keys::generate(), Keys::generate(), Keys::generate()); + // Seed only the queue role; all upload, share, report, read, and closure I/O is real. + sqlx::query("INSERT INTO relay_members (community_id, pubkey, role) VALUES ($1,$2,'admin')") + .bind(community) + .bind(moderator.public_key().to_hex()) + .execute(&pool) + .await + .unwrap(); + let http = Client::new(); + let channel = Uuid::new_v4().to_string(); + let mut author = BuzzTestClient::connect(&ws, &owner).await.unwrap(); + let create = EventBuilder::new(Kind::from(9007), "") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["name", &channel]).unwrap(), + Tag::parse(["channel_type", "stream"]).unwrap(), + Tag::parse(["visibility", "private"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(create).await.unwrap().accepted); + let mut sources = Vec::new(); + for _ in 0..2 { + let bytes = Uuid::new_v4().as_bytes().repeat(8); + let hash = hex::encode(Sha256::digest(&bytes)); + let response = http + .put(format!("{base}/upload")) + .header("authorization", auth(&owner, &host, "upload", &hash)) + .header("x-sha-256", &hash) + .body(bytes) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + let event = EventBuilder::new(Kind::from(9), format!("{base}/media/{hash}")) + .tags([Tag::parse(["h", &channel]).unwrap()]) + .sign_with_keys(&owner) + .unwrap(); + let id = event.id; + assert!(author.send_event(event).await.unwrap().accepted); + sources.push((hash, id)); + } + let (hash, event_id) = &sources[0]; + read(&http, &moderator, &base, &host, hash, 404).await; + let report = EventBuilder::new( + Kind::from(1984), + format!("unrelated /media/{}", sources[1].0), + ) + .allow_self_tagging() + .tags([ + Tag::parse(["e", &event_id.to_hex(), "spam"]).unwrap(), + Tag::parse(["p", &owner.public_key().to_hex(), "spam"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let report_event_id = report.id; + let result = author.send_event(report).await.unwrap(); + assert!(result.accepted, "report rejected: {}", result.message); + read(&http, &moderator, &base, &host, hash, 200).await; + read(&http, &stranger, &base, &host, hash, 404).await; + read(&http, &moderator, &base, &host, &sources[1].0, 404).await; + let audited: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_log WHERE community_id=$1 AND action='moderation_media_read' AND actor_pubkey=$2 AND object_id=$3") + .bind(community).bind(moderator.public_key().to_bytes().to_vec()).bind(hash).fetch_one(&pool).await.unwrap(); + assert_eq!(audited, 2, "GET and HEAD must each commit an audit entry"); + let stored: i64 = + sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community) + .bind(report_event_id.to_bytes().to_vec()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, 0, + "reports must remain outside the public event log" + ); + let mut reviewer = BuzzTestClient::connect(&ws, &moderator).await.unwrap(); + let close = EventBuilder::new(Kind::from(9044), "") + .tags([ + Tag::parse(["report", &report_event_id.to_hex()]).unwrap(), + Tag::parse(["status", "dismissed"]).unwrap(), + Tag::parse(["action", "dismiss"]).unwrap(), + ]) + .sign_with_keys(&moderator) + .unwrap(); + let result = reviewer.send_event(close).await.unwrap(); + assert!( + result.accepted, + "report closure rejected: {}", + result.message + ); + read(&http, &moderator, &base, &host, hash, 404).await; + read(&http, &owner, &base, &host, hash, 200).await; + + // A later edit adds a different attachment. The report targets the logical + // message, and deleting it must revoke the exception even while open. + let edited_hash = &sources[1].0; + let edit = EventBuilder::new(Kind::from(40003), format!("{base}/media/{edited_hash}")) + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["e", &event_id.to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(edit).await.unwrap().accepted); + let reopened = EventBuilder::new(Kind::from(1984), "review edited attachment") + .allow_self_tagging() + .tags([ + Tag::parse(["e", &event_id.to_hex(), "spam"]).unwrap(), + Tag::parse(["p", &owner.public_key().to_hex(), "spam"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(reopened).await.unwrap().accepted); + read(&http, &moderator, &base, &host, edited_hash, 200).await; + read(&http, &stranger, &base, &host, edited_hash, 404).await; + let deletion = EventBuilder::new(Kind::EventDeletion, "delete edited message") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["e", &event_id.to_hex()]).unwrap(), + Tag::parse(["k", "9"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(deletion).await.unwrap().accepted); + read(&http, &moderator, &base, &host, edited_hash, 404).await; + // Its independent private post survives, so the channel owner still reads it. + read(&http, &owner, &base, &host, edited_hash, 200).await; + author.disconnect().await.unwrap(); + reviewer.disconnect().await.unwrap(); +} diff --git a/docs/database.md b/docs/database.md index 78ad9486dd4..0a3c9b58e70 100644 --- a/docs/database.md +++ b/docs/database.md @@ -100,7 +100,7 @@ writer of a fenced table already goes through admission. The remaining fence-retirement work drops the fence triggers and functions, and MUST also update ARCHITECTURE.md § *Dual enforcement*. -**Today this blocks new tenant-owned tables.** Startup and deletion validation +**New tenant-owned tables use application admission.** Startup and deletion validation no longer require a fence on any table; they pass with every universal fence object dropped (`migration_schema_fence_objects_are_optional_for_serving_and_deletion` in `crates/buzz-db/src/store/deletion.rs`). Two things still matter: @@ -110,21 +110,19 @@ in `crates/buzz-db/src/store/deletion.rs`). Two things still matter: (`community deletion catalog drift (missing=…, unknown=…)`), and deletion fails closed for every community, not only the new table's. Listing the table is enough to pass. -- Desired-state bootstraps run `schema/community_write_fence.sql`, which - attaches a fence trigger to every `community_id` table not excluded by - `community_write_fence_excluded_table`. A fresh desired-state database would - put a trigger on the new table and a migrated one would not, so the two - schema paths would diverge. - -So before a feature can add a tenant-owned `community_id` table, the -desired-state bootstrap must stop attaching fence triggers. That change belongs -to the fence-retirement work. Land it before the feature, or as the feature's -first PR. Do not work around it with a trigger, or with an entry in +- Desired-state bootstraps install only the legacy fences explicitly listed + under `schema/triggers/`. They do not discover and fence new tables. + Keep those existing backstops until their own retirement is verified; + new tables receive no fence in either schema path. + +The blanket desired-state fence attachment has been removed. A feature may +add a tenant-owned `community_id` table when it follows the checklist below. +Do not add a trigger, or an entry in `community_write_fence_excluded_table` for a table the community owns. #### Checklist: adding a tenant-owned `community_id` table -After the desired-state change above has landed, all in the same PR: +All in the same PR: - [ ] Every writer of the table goes through community admission (`begin_community_write_transaction` or another constructor that returns diff --git a/docs/multi-tenant-conformance.md b/docs/multi-tenant-conformance.md index 26b0af8cca5..09a1f6eef02 100644 --- a/docs/multi-tenant-conformance.md +++ b/docs/multi-tenant-conformance.md @@ -49,7 +49,7 @@ Conformance obligations: | Workflows, runs, approvals, webhooks, schedules | Workflows are channel-scoped or project/channel-global; triggers fire on matching stored events; schedule/webhook/manual triggers create runs; approval tokens are hashed. | Workflow definition's community from `req.community` at create/update; webhook/schedule/manual routes resolve workflow id inside host-derived community. | Community-global workflow namespace; runs/approvals inherit workflow community. | `workflows`, `workflow_runs`, `workflow_approvals` include `community_id`; workflow id/token hash lookups are scoped; trigger event ids are scoped. | Trigger evaluation only sees events in the same community. Webhook URLs include host-derived community; approval token grants cannot act on another community's same hash/id. | Existing workflow APIs and YAML remain unchanged in default community. | Add tests for identical workflow UUID/approval token hash in different communities and schedule execution isolation. | | Search / FTS | Postgres FTS over the `events.search_tsv` generated `tsvector` column (GIN-indexed); searchable rows expose `id`, `content`, `kind`, `pubkey`, optional `channel_id`, `created_at`, tag terms; channel-less scope is `ChannelScope::ChannelLessOnly`; the relay refetches canonical events from Postgres by hit id. | Search query carries `req.community`; searchable rows carry `community_id`. | Community-global search results; operator-global FTS index infrastructure may be shared. | Every search query filters by `community_id`, BitmapAnd-ed with the GIN `@@` probe; refetch by `(community_id, event_id)`. | Every query carries `community_id` plus channel scope. `ChannelLessOnly` means channel-less within the community, not platform global. | One community produces the same search results as today. | Tests for same event id/content in A and B, deletion in A not deleting B. | | Redis pub/sub, presence, typing, and cache invalidation | Event fan-out uses `buzz:channel:{uuid}`; presence uses `buzz:presence:{pubkey}`; typing uses `buzz:typing:{channel_id}`; cache invalidation uses `buzz:cache-invalidate`. | Pub/sub calls receive `TenantContext` and derive keys from `community_id` plus channel/pubkey. | Pub/sub and presence are community-global; Redis deployment is operator-global shared infrastructure. | Redis keys include community: `buzz:{community}:channel:{uuid}`, `buzz:{community}:presence:{pubkey}`, `buzz:{community}:typing:{channel_id}`, and community-aware cache invalidation payloads/channels. | Cross-node fan-out must not deliver events to subscriptions in another community. Same pubkey can be online/away differently in two communities. Cache drops only affect same-community membership/visibility caches unless explicitly all-community operator maintenance. | Single-community can preserve existing key names only if deployment is isolated; shared multi-tenant Redis must use the prefixed form. | Add tests for same pubkey presence in two communities and same channel UUID collision in two communities. | -| Media / Blossom / S3 | Authenticated uploads return content-addressed descriptors; `GET/HEAD /media/{sha256.ext}` requires a Blossom `t=get` auth event scoped to the serving host or the blob hash and binds the read to the header-resolved tenant, so a bare read is rejected before any storage lookup; upload audit has `channel_id = None`. | Upload and read request host provides `req.community`; Blossom/NIP-98 auth URL host must agree. | Blob CAS bytes may be operator-global shared storage; metadata, authorization, quotas, audit, and visibility are community-global. | Media metadata/audit rows include `community_id`; if object keys stay SHA-addressed, any per-community policy lives outside the raw blob key. | Upload/read authorization uses community context. Shared hash bytes are allowed only as dedup/storage optimization; metadata/errors must not reveal another community's private upload. | Existing media URLs keep working for default community, but clients must now present read auth; there is no config flag that restores unauthenticated reads. | Resolved: blob reads are authenticated and host/tenant-scoped, not public. Remaining gap, deferred: a read is not gated on the channel ACL of the message the blob was attached to, so relay membership plus a known hash is sufficient. | +| Media / Blossom / S3 | Authenticated uploads return content-addressed descriptors; `GET/HEAD /media/{sha256.ext}` requires a Blossom `t=get` auth event scoped to the serving host or the blob hash and binds the read to the header-resolved tenant, so a bare read is rejected before any storage lookup; upload audit has `channel_id = None`. | Upload and read request host provides `req.community`; Blossom/NIP-98 auth URL host must agree. | Blob CAS bytes may be operator-global shared storage; metadata, authorization, quotas, audit, and visibility are community-global. | Media metadata/audit rows include `community_id`; if object keys stay SHA-addressed, any per-community policy lives outside the raw blob key. | Upload/read authorization uses community context. Shared hash bytes are allowed only as dedup/storage optimization; metadata/errors must not reveal another community's private upload. | Existing media URLs keep working for default community, but clients must now present read auth; there is no config flag that restores unauthenticated reads. | Resolved: authenticated, host-bound file reads support `BUZZ_MEDIA_READ_MODE=off|log-only|enforce` (default `log-only`); authentication and community checks remain active in every mode. Enforce requires a live source-event share, upload proof, or audited open-report access by a community owner/admin. Deleted sources grant nothing. Complete backfill and inspect would-deny/error counters before enabling enforcement; see [media access rollout](operator-media-access.md). | | Git hosting / NIP-34 / object storage | Smart HTTP at `/git/{owner}/{repo}` hydrates from S3 object pointers; NIP-34 repo announcements use `d=repo-id`; pointer key is `repos/{owner}/{repo}/pointer`; git push emits kind:30618. | Git HTTP host gives `req.community`; NIP-98 URL and repo announcement community must agree. | Community-global repo namespace and NIP-34 state; pack/manifests CAS objects may be operator-global if pointers are scoped. | Pointer/name keys include community, e.g. `repos/{community}/{owner}/{repo}/pointer`; NIP-34 replaceable coords include `community_id`; any repo-name registry is `(community_id, owner, repo)` or `(community_id, repo)` per product rule. | Clone/push/read policy resolves repo and branch protections only inside the host community. Git hook policy callback carries community and rejects mismatches. | Existing clone URLs and repo ids work under the default community; object-store migration can move pointers under default prefix without changing git clients. | Add tests for same owner/repo in two communities and push in A not advancing B pointer. | | Mesh, agents, ACP/MCP, and CLI | Agents/CLI connect to a relay URL and use WS/REST; mesh/pairing/presence/status events are regular signed relay events. | The relay URL/host configured in the agent/CLI session selects community. | Agent membership, persona/profile, presence, jobs, memory events, and mesh status are community-global unless a future operator mesh plane is explicitly separate. | Any persisted agent profile/job/mesh status rows/events use `community_id`; Redis/presence/search keys follow the same community scoping. | A portable key may join multiple communities, but memberships, DMs, profiles, jobs, and presence do not bleed across them. | Existing `BUZZ_RELAY_URL` continues to select the one default community. | Add CLI/ACP smoke tests against two hosts using same key with different memberships/profile. | | Audit log and observability | One hash-chain audit log records event/channel/auth/media actions; errors are sanitized before reaching clients. | Every tenant-observable audit entry is labeled with `req.community` or inherited community from the object being acted on. | Community-global audit chains; operator metrics/log aggregation may be platform-global only if tenant labels are bounded and access-controlled. | `audit_log` key/sequence/head includes `community_id`; error/audit projection tables include `community_id`; uniqueness is `(community_id, seq)` and `(community_id, hash)` as appropriate. | Audit reads verify only one community chain. Error strings must not include cross-community IDs, constraint names, or existence facts. | Single-community audit verification still traverses one chain. | Eva owns model edits here; infra lane must ensure media/git/token/search rows emit community-labeled audit entries. | diff --git a/docs/nips/NIP-FI.md b/docs/nips/NIP-FI.md index 3d71a7c436b..6bacd01b395 100644 --- a/docs/nips/NIP-FI.md +++ b/docs/nips/NIP-FI.md @@ -704,7 +704,9 @@ mismatched `server` tag MUST be rejected as `evidence_rejected`. #### Read proofs Read proofs MAY be host-wide: no `x` tag is required for reads, and a valid -read proof authorizes reads of any blob on the one bound tenant host. +read proof authenticates the signing key on the one bound tenant host. +Proof validity does not itself grant access to every blob: community membership +and the relay's file-share policy apply separately. Read proofs MUST carry exactly one `server` tag whose value matches the request's already-resolved tenant host. A read proof with an absent or @@ -716,9 +718,12 @@ rejected as `evidence_rejected`. > **Named residual:** within a window of at most 60 seconds from minting (plus > a bounded 5-second future-skew allowance), a captured full header set allows -> reading any media blob on exactly one tenant host. This access is read-only, -> membership-checked, and revocable via assertion expiry or deny-map -> enforcement. It is not state-changing and does not cross tenant boundaries. +> impersonating the signing key for media reads on exactly one tenant host. +> Where the relay enforces its file-share policy, reads are limited to files +> that key may access through a live share, upload proof, or an audited +> open-report review; otherwise community membership is the read-access gate. +> This access is read-only and revocable via assertion expiry or deny-map +> enforcement; it does not cross tenant boundaries. #### Freshness diff --git a/docs/operator-media-access.md b/docs/operator-media-access.md new file mode 100644 index 00000000000..92c120d914b --- /dev/null +++ b/docs/operator-media-access.md @@ -0,0 +1,86 @@ +# Media access rollout + +File authorization is separate from authentication, NIP-FI, and community admission. +All three modes keep the existing authentication, host binding, ban, and membership checks. + +Set `BUZZ_MEDIA_READ_MODE` in the relay deployment environment and restart every pod: + +| Value | File-access behavior | +|---|---| +| `off` | Skip the new file-share check and its database lookups. | +| `log-only` (default) | Evaluate file access and count outcomes, but preserve existing reads, including on lookup errors. | +| `enforce` | Require file access; deny unauthorized reads with 404 and authorization failures with 503. | + +Unknown values fail startup. This setting loads at startup; it is not a live feature flag. +Changing modes requires a rolling restart, but no code rebuild. +During that restart, pods use their own configured mode until replaced. + +## Rollout + +1. Before any backfill, explicitly accept that reconsidered references can create lasting grants; otherwise defer the backfill. +2. Deploy share recording to every pod with reads in `log-only`. +3. Run `buzz-admin backfill-media-shares` after all pods record shares. +4. Inspect `buzz_media_read_access_total{mode="log-only",outcome="deny"}` and `outcome="error"` in Prometheus. +5. Investigate denied workflows and repair missing provenance before enforcing. +6. Set `BUZZ_MEDIA_READ_MODE=enforce` and restart every pod. + +The counter uses only fixed `mode` and `outcome` labels: `allow`, `report`, `deny`, or `error`. +It measures authorization evaluations on GET and HEAD, before checking whether storage contains the file. +It counts requests, not distinct files, and includes thumbnail requests checked against the original hash. +Use controlled client smoke tests to investigate counts; hashes, keys, and URLs are not metric labels. +Log-only GET/HEAD evaluations create no shares and do not grant future access. +Event recording and backfill still write grants in every read mode. + +Backfill reevaluates references received after recording began using their authors' current permissions. +It does not remember earlier refusals: any run, including the first, can grant an old reference after its author gains access. +For a community-wide note, that grant can survive removal of the author's channel access and deletion of the original private post. +The grant remains tied to the note and ends when that note is deleted. +Accept those resulting grants before the first backfill, or defer backfill until refusal-preserving behavior exists. +Changing read modes does not revoke grants, and preventing reevaluation later does not remove grants already created. +Any unwanted existing grants need a provenance-based repair before enforcement. +Every invocation restarts the scan from the beginning; rows are deduplicated, but earlier authorization decisions are not preserved. + +Before recording shares, the backfill restores uploader proof from retained server +`media_uploaded` audit rows with an attributed 32-byte pubkey and a valid SHA-256 object ID. +It scans each community's audit sequence in bounded batches and skips quiescing/deleting communities. +Missing, malformed, or unattributed audit evidence grants no ownership; those files still +need an upload of the exact bytes. Unparsed references need separate repair before enforcement. +To roll back file denials, set `off` or `log-only` and restart every pod. + +## Open reports + +A community owner or admin who can view the moderation queue may read an attachment +on a reported message outside their channels while that report has status `open`. +The report, message, and file grant must belong to the requesting community. +The source event and an edit's target message must still exist and be undeleted. +Report notes, arbitrary URLs, pubkey reports, and blob-only reports grant no extra access. +There is no role-based bypass for unrelated files, and reports create no public shares. + +In enforce mode, each exceptional GET or HEAD requires a durable `moderation_media_read` +audit entry containing the actor, report, source message, and file hash before storage access. +If auditing is disabled or fails, the exceptional read returns 503. +Log-only mode evaluates this permission without writing an exceptional-access audit entry. +Ordinary reads allowed by existing shares do not need the moderation exception. + +Closing the report, deleting the source message, or removing the queue role ends this exception +for subsequent requests; ordinary shares can still independently authorize access. +Escalated and processing reports do not qualify for this community-review exception. +Bytes already downloaded cannot be recalled. + +## Inactive grant cleanup + +The elected usage-poller leader inspects at most 1,000 media grants per tick, +in primary-key order, and removes inactive grants in community-admitted transactions. +The tick uses `BUZZ_USAGE_METRICS_INTERVAL_SECS` (300 seconds by default). +The scan wraps at the end; its cursor is held in memory and restarts after a relay restart. +Tenant-local failures retry on a later scan cycle, and quiescing/deleting communities are skipped. + +A grant is inactive when its source event is missing or deleted, or its edit target is no longer live. +Live grants survive cleanup. Inactive public profile/emoji grants are removed completely. +Cleanup compacts inactive channel history to at most one row per community and file: removing every +channel row would incorrectly restore uploader preview access after message deletion. +Whole-community purge removes that final row as well. + +Successful reclamation logs `reaped inactive media shares` with the number removed; +failures log `failed to reap inactive media shares` or a tenant-local sweep error. +This reclaims grant records; file retention follows the storage policy. diff --git a/migrations/0068_media_access.sql b/migrations/0068_media_access.sql new file mode 100644 index 00000000000..2e8fd7289c9 --- /dev/null +++ b/migrations/0068_media_access.sql @@ -0,0 +1,34 @@ +SET LOCAL lock_timeout = '5s'; + +-- Durable historical backfill cutoff, independent of the schema install tool. +ALTER TABLE communities + ADD COLUMN media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(); + +-- Channel-scoped media access: who may read each blob. +-- +-- media_uploaders: who has uploaded each blob. Uploading the exact bytes +-- proves possession, so an uploader may share a blob into their own channels +-- even when the same bytes are also shared somewhere they cannot read. +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); + +-- media_shares: one grant per referencing event and blob. A NULL channel +-- shares with the whole community. Deleted or purged source events grant +-- nothing; event_created_at locates the source in the partitioned table. +CREATE TABLE media_shares ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + event_id BYTEA NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + target_event_id BYTEA, + target_event_created_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((target_event_id IS NULL) = (target_event_created_at IS NULL)), + PRIMARY KEY (community_id, sha256, event_id) +); diff --git a/schema/community_write_fence.sql b/schema/community_write_fence.sql deleted file mode 100644 index 2ebc9fd1f98..00000000000 --- a/schema/community_write_fence.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Attach the universal fence to every existing table carrying community_id, --- including deployment-private sidecars whose community_id is provenance. --- --- Desired-state schema application does not replay migration history, so the --- explicit `SELECT attach_community_write_fence('')` calls in --- `triggers/
.sql` stay as first-class catalog declarations. They also --- make the fence contract visible to migration linting instead of hiding it --- only in this dynamic bootstrap loop. This loop runs last, so it fences every --- desired-state table regardless: a fenced desired state does not prove the --- migration path attaches the fence. Keep the explicit calls. -DO $$ -DECLARE - target REGCLASS; -BEGIN - FOR target IN - SELECT c.oid::REGCLASS - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - JOIN pg_attribute a ON a.attrelid = c.oid - WHERE n.nspname = current_schema() - AND c.relkind IN ('r', 'p') - AND NOT c.relispartition - AND a.attname = 'community_id' - AND NOT a.attisdropped - AND NOT community_write_fence_excluded_table(c.relname) - ORDER BY c.oid::REGCLASS::TEXT - LOOP - PERFORM attach_community_write_fence(target); - END LOOP; -END -$$; diff --git a/schema/schema.sql b/schema/schema.sql index d252cbdb7ea..1ee65c207b2 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -7,9 +7,9 @@ -- -- Everything SchemaBot cannot apply lives in this directory's other -- subdirectories: `types/`, `functions/`, `partitions/`, `triggers/` (including --- write-fence attachment), `seeds/`, and the `community_write_fence.sql` --- bootstrap. This file composes all of them into one fresh database, in load --- order: types → functions → tables → partitions → triggers → seeds → fence. +-- explicit legacy write-fence attachment), and `seeds/`. This file composes +-- them into one fresh database, in load order: +-- types → functions → tables → partitions → triggers → seeds. -- -- `./bin/pgschema apply --file schema/schema.sql` resolves the `\i` includes -- relative to this file (it refuses `..`). `pgschema` skips INSERTs and some @@ -107,6 +107,8 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i tables/public/moderation_reports.sql \i tables/public/community_bans.sql \i tables/public/community_nip_fi_issuers.sql +\i tables/public/media_shares.sql +\i tables/public/media_uploaders.sql \i tables/public/_operator_global_tables.sql \i tables/public/git_repo_names.sql \i tables/public/parameterized_event_watermarks.sql @@ -209,4 +211,3 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i seeds/storage_accounting_snapshots.sql -- Fence any community-scoped table that lacks an explicit attach. -\i community_write_fence.sql diff --git a/schema/tables/public/communities.sql b/schema/tables/public/communities.sql index 06f74c3f585..ea27392437b 100644 --- a/schema/tables/public/communities.sql +++ b/schema/tables/public/communities.sql @@ -20,6 +20,8 @@ CREATE TABLE communities ( -- Added by migration 0003; kept here so desired-state applies match. icon TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + -- The share-recording cutoff exists in both desired-state and migrated DBs. + media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(), archived_at TIMESTAMPTZ, deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql new file mode 100644 index 00000000000..600574f766c --- /dev/null +++ b/schema/tables/public/media_shares.sql @@ -0,0 +1,17 @@ +-- ── Media shares ────────────────────────────────────────────────────────────── +-- Each event's grant to a channel, or the whole community for a NULL channel. +-- Reads join the source event: deleted or purged events grant no access. +-- Keep its timestamp for an indexed lookup in the partitioned events table. + +CREATE TABLE media_shares ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + event_id BYTEA NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + target_event_id BYTEA, + target_event_created_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((target_event_id IS NULL) = (target_event_created_at IS NULL)), + PRIMARY KEY (community_id, sha256, event_id) +); diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql new file mode 100644 index 00000000000..1add946f3e2 --- /dev/null +++ b/schema/tables/public/media_uploaders.sql @@ -0,0 +1,11 @@ +-- ── Media uploaders ─────────────────────────────────────────────────────────── +-- Who has uploaded each blob in a community. Uploading the exact bytes proves +-- possession, so an uploader may share a blob into their own channels. + +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +);