From e0d28ed811be83c9664c59167b79af658b3a9ef6 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 22:23:55 +1100 Subject: [PATCH 01/48] feat(relay): record where media is shared and who uploaded it Add `media_shares` and `media_uploaders` (migration 0067) and start filling them, without changing who can read media yet. Each local `/media/` path in a stored event shares that blob into the event's channel (or with the whole community when channel-less), in the same transaction as the event, when its author can read the blob or uploaded those bytes; each upload records its uploader under the serving-write lease. `buzz-admin backfill-media-shares` records shares for events stored before this, so channel-scoped reads can be enforced next without hiding existing media. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 17 ++ crates/buzz-core/src/lib.rs | 2 + crates/buzz-core/src/media.rs | 130 +++++++++ crates/buzz-db/src/runtime/migration.rs | 21 +- crates/buzz-db/src/store/channel_members.rs | 9 +- crates/buzz-db/src/store/deletion.rs | 6 + crates/buzz-db/src/store/event.rs | 16 +- crates/buzz-db/src/store/event_follow_up.rs | 22 +- crates/buzz-db/src/store/media_reference.rs | 298 ++++++++++++++++++++ crates/buzz-db/src/store/media_uploader.rs | 110 ++++++++ crates/buzz-db/src/store/mod.rs | 4 + crates/buzz-db/src/store/push.rs | 8 +- crates/buzz-db/src/store/relay_members.rs | 8 +- crates/buzz-db/src/store/replaceable.rs | 12 +- crates/buzz-relay/src/api/media.rs | 22 +- migrations/0067_media_access.sql | 29 ++ schema/schema.sql | 4 + schema/tables/public/media_shares.sql | 15 + schema/tables/public/media_uploaders.sql | 11 + schema/triggers/media_shares.sql | 1 + schema/triggers/media_uploaders.sql | 1 + 21 files changed, 692 insertions(+), 54 deletions(-) create mode 100644 crates/buzz-core/src/media.rs create mode 100644 crates/buzz-db/src/store/media_reference.rs create mode 100644 crates/buzz-db/src/store/media_uploader.rs create mode 100644 migrations/0067_media_access.sql create mode 100644 schema/tables/public/media_shares.sql create mode 100644 schema/tables/public/media_uploaders.sql create mode 100644 schema/triggers/media_shares.sql create mode 100644 schema/triggers/media_uploaders.sql diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 4421f47e1fb..23ce4227488 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -95,6 +95,15 @@ enum Command { #[arg(long, default_value_t = 10_000_000)] max_objects: u64, }, + /// Record media shares for events stored before share recording existed. + /// + /// Idempotent and resumable; run once after deploying share recording and + /// before enforcing channel-scoped media reads. + BackfillMediaShares { + /// Events read and recorded per transaction. + #[arg(long, default_value_t = 1000)] + batch_size: i64, + }, /// Run the partition catalog audit using a read-only database session. PartitionAudit { /// Future months to include in the coverage check. @@ -188,6 +197,14 @@ async fn run(cli: Cli) -> Result { Ok(0) } Command::StorageSnapshot { max_objects } => cmd_storage_snapshot(max_objects).await, + Command::BackfillMediaShares { batch_size } => { + let added = connect_db() + .await? + .backfill_media_shares(batch_size) + .await?; + println!("Recorded {added} media shares."); + Ok(0) + } Command::PartitionAudit { months_ahead } => cmd_partition_audit(months_ahead).await, Command::AddMember { pubkey, role } => cmd_add_member(pubkey, role).await, Command::RemoveMember { pubkey, role } => cmd_remove_member(pubkey, role).await, diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index e7669c874b0..0c89cd62c0f 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -24,6 +24,8 @@ pub mod git_perms; pub mod invite; /// Buzz kind number registry — custom event type constants. pub mod kind; +/// Which media blobs an event references, for channel-scoped media access. +pub mod media; /// Network utilities — SSRF-safe IP classification. pub mod network; /// NIP-10 thread-marker parsing — shared `root`/`reply` marker resolver. diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs new file mode 100644 index 00000000000..29bd3a124ec --- /dev/null +++ b/crates/buzz-core/src/media.rs @@ -0,0 +1,130 @@ +//! Which media blobs an event shares, for channel-scoped media access. + +use std::collections::BTreeSet; + +use crate::kind::{ + event_is_shared, is_author_only_event_kind, is_shared_gated_kind, P_GATED_KINDS, +}; + +/// Hashes of every blob `event` shares into its channel, or with the whole +/// community when it is channel-less: each local media path `/media/` +/// anywhere in its content or tags (attachments, posters, thumbnails, link +/// previews, avatars, custom emoji, pasted links). +/// +/// Empty for an event only its author or tagged recipients can read, since +/// referencing a blob there shows it to no one new. +pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { + let kind = u32::from(event.kind.as_u16()); + if is_author_only_event_kind(event) + || P_GATED_KINDS.contains(&kind) + || (is_shared_gated_kind(kind) && !event_is_shared(event)) + { + return Vec::new(); + } + let mut hashes: BTreeSet = media_path_hashes(&event.content).collect(); + for tag in event.tags.iter() { + hashes.extend( + tag.as_slice() + .iter() + .flat_map(|part| media_path_hashes(part)), + ); + } + hashes.into_iter().collect() +} + +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) +} + +/// The hash in each `/media/` path in `text`, where the hash is not +/// followed by another hex digit. +fn media_path_hashes(text: &str) -> impl Iterator + '_ { + text.match_indices("/media/").filter_map(|(at, path)| { + let rest = &text[at + path.len()..]; + let hash = rest.get(..64)?; + let bounded = !rest[64..].starts_with(|c: char| matches!(c, '0'..='9' | 'a'..='f')); + (is_sha256_hex(hash) && bounded).then(|| hash.to_owned()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kind::{KIND_EVENT_REMINDER, KIND_PERSONA}; + use nostr::{EventBuilder, Keys, Kind, Tag}; + + fn media_url(sha256: &str, ext: &str) -> String { + format!("https://relay.example/media/{sha256}.{ext}") + } + + fn signed(builder: EventBuilder) -> nostr::Event { + builder.sign_with_keys(&Keys::generate()).unwrap() + } + + #[test] + fn shares_every_local_media_path() { + let (video, poster, embed) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); + let event = signed( + EventBuilder::new( + Kind::TextNote, + format!("look ![image]({})", media_url(&embed, "png")), + ) + .tags([ + Tag::parse([ + "imeta", + &format!("url {}", media_url(&video, "mp4")), + &format!("x {video}"), + &format!("image /media/{poster}.jpg"), + ]) + .unwrap(), + Tag::parse([ + "r", + &format!("https://elsewhere.example/{}", "e".repeat(64)), + ]) + .unwrap(), + Tag::parse(["r", &format!("/media/{}0", "f".repeat(64))]).unwrap(), + ]), + ); + assert_eq!(shared_blob_hashes(&event), vec![video, poster, embed]); + } + + #[test] + fn shares_avatars_emoji_and_shared_personas_only() { + let (avatar, emoji) = ("c".repeat(64), "d".repeat(64)); + let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar, "jpg")); + let profile = signed(EventBuilder::new(Kind::Metadata, &content)); + assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); + + let emoji_set = signed( + EventBuilder::new(Kind::from(30030), "").tags([Tag::parse([ + "emoji", + "party", + &media_url(&emoji, "png"), + ]) + .unwrap()]), + ); + assert_eq!(shared_blob_hashes(&emoji_set), vec![emoji]); + + let persona = |shared: bool| { + let mut tags = vec![Tag::parse(["d", "helper"]).unwrap()]; + if shared { + tags.push(Tag::parse(["shared", "true"]).unwrap()); + } + signed(EventBuilder::new(Kind::from(KIND_PERSONA as u16), &content).tags(tags)) + }; + assert_eq!(shared_blob_hashes(&persona(true)), vec![avatar]); + assert!(shared_blob_hashes(&persona(false)).is_empty()); + } + + #[test] + fn author_only_events_share_nothing() { + let reminder = signed( + EventBuilder::new(Kind::from(KIND_EVENT_REMINDER as u16), "").tags([Tag::parse([ + "imeta", + &format!("url /media/{}.jpg", "a".repeat(64)), + ]) + .unwrap()]), + ); + assert!(shared_blob_hashes(&reminder).is_empty()); + } +} diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 7cb0879ff87..b98addf01cf 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -709,7 +709,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 66); + assert_eq!(migrations.len(), 67); assert_eq!(migrations[64].version, 65); let retire_ttl = strip_sql_comments(migrations[64].sql.as_str()); for statement in [ @@ -746,6 +746,13 @@ mod postgres_tests { .sql .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); + assert_eq!(migrations[66].version, 67); + for table in ["media_uploaders", "media_shares"] { + assert!(migrations[66] + .sql + .as_str() + .contains(&format!("CREATE TABLE {table}"))); + } assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2317,6 +2324,18 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('community_nip_fi_issuers')")); expected_fences.insert("community_nip_fi_issuers".to_owned()); + // 0067 fences the media access tables. + let media_access = MIGRATOR + .iter() + .find(|m| m.version == 67) + .expect("media access migration"); + for table in ["media_uploaders", "media_shares"] { + assert!(media_access + .sql + .as_str() + .contains(&format!("attach_community_write_fence('{table}')"))); + expected_fences.insert(table.to_owned()); + } assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 4855d32cccf..a3034cbc2a7 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -503,13 +503,8 @@ impl LockedMemberSnapshot { "member snapshot event id already exists".into(), )); } - crate::store::event_follow_up::after_admitted_insert( - &mut self.tx, - event.id.as_bytes().as_slice(), - kind, - Some(channel_id), - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut self.tx, event, Some(channel_id)) + .await?; crate::insert_mentions_in_transaction(&mut self.tx, event, Some(channel_id)).await?; Ok(( buzz_core::StoredEvent::with_received_at( diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 1e1e68cb054..c48816a39d1 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -90,6 +90,8 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "media_shares", + "media_uploaders", "moderation_actions", "moderation_reports", "parameterized_event_watermarks", @@ -156,6 +158,8 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "audit_log", "community_bans", "community_nip_fi_issuers", + "media_shares", + "media_uploaders", "pubkey_allowlist", "relay_members", "users", @@ -4423,6 +4427,8 @@ DROP TRIGGER IF EXISTS community_write_fence_event_mentions ON event_mentions; DROP TRIGGER IF EXISTS community_write_fence_events ON events; DROP TRIGGER IF EXISTS community_write_fence_git_repo_names ON git_repo_names; DROP TRIGGER IF EXISTS community_write_fence_join_policy_acceptances ON join_policy_acceptances; +DROP TRIGGER IF EXISTS community_write_fence_media_shares ON media_shares; +DROP TRIGGER IF EXISTS community_write_fence_media_uploaders ON media_uploaders; DROP TRIGGER IF EXISTS community_write_fence_moderation_actions ON moderation_actions; DROP TRIGGER IF EXISTS community_write_fence_moderation_reports ON moderation_reports; DROP TRIGGER IF EXISTS community_write_fence_parameterized_event_watermarks ON parameterized_event_watermarks; diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 209f09e5733..82f5233d8dc 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -508,13 +508,7 @@ async fn insert_event_on( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(( @@ -1814,13 +1808,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; if let Some(ref meta) = thread_meta { let broadcast_val: bool = meta.broadcast; diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index d2853b4d89d..dd4e5aa9d19 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -37,15 +37,29 @@ const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; /// Follow-up for an event row inserted in an admitted transaction. /// /// Call only when the INSERT affected a row, after releasing any savepoint -/// that may roll it back. Stages push work for after commit and records TTL work. +/// that may roll it back. Stages push work for after commit, records TTL work, +/// and shares the blobs the event references into its channel in the same +/// transaction, so a stored reference always carries its share. pub(crate) async fn after_admitted_insert( tx: &mut AdmittedTx, - event_id: &[u8], - kind: i32, + event: &nostr::Event, channel_id: Option, ) -> Result<()> { - tx.record_push_event(event_id, kind); + let kind = i32::from(event.kind.as_u16()); + tx.record_push_event(event.id.as_bytes(), kind); tx.record_channel_event(channel_id, kind); + let hashes = buzz_core::media::shared_blob_hashes(event); + if !hashes.is_empty() { + let community = tx.community(); + crate::store::media_reference::record_media_shares( + tx.conn(), + community, + &hashes, + event.pubkey.as_bytes(), + channel_id, + ) + .await?; + } Ok(()) } diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs new file mode 100644 index 00000000000..c4aa9fb95e3 --- /dev/null +++ b/crates/buzz-db/src/store/media_reference.rs @@ -0,0 +1,298 @@ +//! Where each media blob is shared, recorded with the events that reference it. + +use buzz_datastore_tracing::datastore_span; +use chrono::{DateTime, Utc}; +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use crate::{error::Result, Db}; +use buzz_core::CommunityId; + +/// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. +/// +/// Everyone may when the blob is shared with the whole community or into no +/// channel; otherwise only readers of a channel it is shared into, open or one +/// they actively belong to. Shares are never withdrawn. +macro_rules! readable_sql { + () => { + "( \ + EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ + OR NOT EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND s.channel_id IS NOT NULL) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $3 AND cm.removed_at IS NULL))))" + }; +} + +/// Share into `channel_id` (or with the whole community when `None`) each blob +/// in `sha256s` that `author` can read or has uploaded (uploading the exact +/// bytes proves possession). Any other blob is left unshared, so a reference +/// built from a leaked hash shows it to no one new. Called with the +/// transaction that stores the referencing event. +pub(crate) async fn record_media_shares( + conn: &mut PgConnection, + community_id: CommunityId, + sha256s: &[String], + author: &[u8], + channel_id: Option, +) -> Result<()> { + sqlx::query(concat!( + "INSERT INTO media_shares (community_id, sha256, channel_id) \ + SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + WHERE EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR ", + readable_sql!(), + " ON CONFLICT DO NOTHING" + )) + .bind(community_id.as_uuid()) + .bind(sha256s) + .bind(author) + .bind(channel_id) + .execute(conn) + .await?; + Ok(()) +} + +impl Db { + /// Record the shares of every event already stored in an active community, + /// deleted events included, `batch_size` events per transaction; returns the + /// number of shares added. + /// + /// Shares come from the same rule as live writes. History predates the + /// readability check, so its references are recorded as they stand. + /// Idempotent and resumable: rerunning only fills shares still missing. + #[datastore_span(name = "backfill_media_shares", system = "postgresql")] + pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { + let communities: Vec = sqlx::query_scalar( + "SELECT id FROM communities WHERE deleted_at IS NULL AND deletion_state = 'active' \ + ORDER BY id", + ) + .fetch_all(&self.pool) + .await?; + let mut added = 0; + for community in communities { + let community = CommunityId::from_uuid(community); + let mut cursor: Option<(DateTime, Vec)> = None; + loop { + let rows = sqlx::query( + "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, \ + channel_id \ + FROM events \ + WHERE community_id = $1 \ + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) \ + ORDER BY created_at, id LIMIT $4", + ) + .bind(community.as_uuid()) + .bind(cursor.as_ref().map(|(at, _)| *at)) + .bind( + cursor + .as_ref() + .map(|(_, id)| id.clone()) + .unwrap_or_default(), + ) + .bind(batch_size) + .fetch_all(&self.pool) + .await?; + let Some(last) = rows.last() else { break }; + cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); + let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; + + let mut tx = crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await?; + for row in rows { + let Some(stored) = crate::event::row_to_stored_event(row)? else { + continue; + }; + let hashes = buzz_core::media::shared_blob_hashes(&stored.event); + if hashes.is_empty() { + continue; + } + added += sqlx::query( + "INSERT INTO media_shares (community_id, sha256, channel_id) \ + SELECT $1, unnest($2::text[]), $3 ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(&hashes) + .bind(stored.channel_id) + .execute(tx.conn()) + .await? + .rows_affected(); + } + tx.commit().await?; + if !full_batch { + break; + } + } + } + Ok(added) + } +} + +#[cfg(test)] +mod postgres_tests { + use std::time::Duration; + + use super::*; + use crate::channel::{ChannelType, ChannelVisibility}; + use nostr::{EventBuilder, Keys, Kind, Tag}; + use sqlx::PgPool; + + async fn fixture() -> (Db, CommunityId) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let host = format!("media-shares-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("seed community") + .id; + (db, community) + } + + async fn channel( + db: &Db, + community: CommunityId, + owner: &Keys, + visibility: ChannelVisibility, + ) -> Uuid { + db.create_channel( + community, + &format!("media-{}", Uuid::new_v4().simple()), + ChannelType::Stream, + visibility, + None, + &owner.public_key().to_bytes(), + None, + ) + .await + .expect("create channel") + .id + } + + fn attachment(keys: &Keys, sha256: &str, channel: Option) -> nostr::Event { + let mut tags = vec![Tag::parse(["imeta", &format!("url /media/{sha256}.jpg")]).unwrap()]; + if let Some(channel) = channel { + tags.push(Tag::parse(["h", &channel.to_string()]).unwrap()); + } + EventBuilder::new(Kind::from(9), "attachment") + .tags(tags) + .sign_with_keys(keys) + .unwrap() + } + + async fn shares(db: &Db, community: CommunityId, sha256: &str) -> Vec> { + let mut shares: Vec> = sqlx::query_scalar( + "SELECT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + ) + .bind(community.as_uuid()) + .bind(sha256) + .fetch_all(&db.pool) + .await + .expect("read shares"); + shares.sort(); + shares + } + + fn unique_sha() -> String { + format!("{:064x}", Uuid::new_v4().as_u128()) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stored_references_share_only_blobs_their_author_can_read() { + let (db, community) = fixture().await; + let (alice, bob) = (Keys::generate(), Keys::generate()); + let sha = unique_sha(); + let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; + + db.insert_event( + community, + &attachment(&alice, &sha, Some(private)), + Some(private), + ) + .await + .expect("alice posts privately"); + assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references the hash"); + assert_eq!( + shares(&db, community, &sha).await, + vec![Some(private)], + "a hash alone must not share a blob its author cannot read" + ); + + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community, + "media_upload", + "media-shares-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + db.record_media_uploader(&lease, &sha, &bob.public_key().to_bytes()) + .await + .expect("bob uploads the same bytes"); + db.insert_event(community, &attachment(&bob, &sha, Some(bobs)), Some(bobs)) + .await + .expect("bob posts his copy"); + let mut expected = vec![Some(private), Some(bobs)]; + expected.sort(); + assert_eq!( + shares(&db, community, &sha).await, + expected, + "an uploader may share their own bytes" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn backfill_records_shares_of_stored_events() { + let (db, community) = fixture().await; + let alice = Keys::generate(); + let sha = unique_sha(); + let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + db.insert_event( + community, + &attachment(&alice, &sha, Some(private)), + Some(private), + ) + .await + .expect("alice posts privately"); + let mut tx = crate::begin_community_write_transaction( + &db.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await + .expect("admit"); + sqlx::query("DELETE FROM media_shares WHERE community_id = $1") + .bind(community.as_uuid()) + .execute(tx.conn()) + .await + .expect("forget shares"); + tx.commit().await.expect("commit"); + + db.backfill_media_shares(100).await.expect("backfill"); + assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + } +} diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs new file mode 100644 index 00000000000..0039575af04 --- /dev/null +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -0,0 +1,110 @@ +//! Who has uploaded each blob, as proof of possession for sharing it. + +use buzz_datastore_tracing::datastore_span; + +use crate::deletion::ServingWriteLease; +use crate::{error::Result, observability, Db}; + +impl Db { + /// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are + /// no-ops. The write presents the upload's serving lease, so it may finish + /// while the community quiesces but never after the lease is gone. + #[datastore_span(name = "record_media_uploader", system = "postgresql")] + pub async fn record_media_uploader( + &self, + lease: &ServingWriteLease, + sha256: &str, + pubkey: &[u8], + ) -> Result<()> { + let connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::EventWrite) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + self.deletion_store() + .guard_transaction_with_serving_lease(&mut tx, lease) + .await?; + sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(lease.community_id.as_uuid()) + .bind(sha256) + .bind(pubkey) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } +} + +#[cfg(test)] +mod postgres_tests { + use std::time::Duration; + + use super::*; + use buzz_core::CommunityId; + use sqlx::PgPool; + use uuid::Uuid; + + const SHA: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + + async fn db_with_lease() -> (Db, CommunityId, ServingWriteLease) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let host = format!("uploader-{}.example", Uuid::new_v4().simple()); + let community_id = db + .ensure_configured_community(&host) + .await + .expect("seed community") + .id; + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community_id, + "media_upload", + "media-uploader-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + (db, community_id, lease) + } + + async fn uploader_rows(db: &Db, community_id: CommunityId) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1") + .bind(community_id.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("count uploaders") + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn repeat_upload_records_one_row() { + let (db, community_id, lease) = db_with_lease().await; + for _ in 0..2 { + db.record_media_uploader(&lease, SHA, &[7; 32]) + .await + .expect("record uploader"); + } + assert_eq!(uploader_rows(&db, community_id).await, 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn released_lease_refuses_the_write() { + let (db, community_id, lease) = db_with_lease().await; + db.deletion_store() + .release_serving_write_lease(&lease) + .await + .expect("release lease"); + assert!(db + .record_media_uploader(&lease, SHA, &[7; 32]) + .await + .is_err()); + assert_eq!(uploader_rows(&db, community_id).await, 0); + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index f759a770a25..b84f3903aff 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,6 +32,10 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Where each media blob is shared, recorded with the events that reference it. +pub mod media_reference; +/// Who has uploaded each blob, as proof of possession for sharing it. +pub mod media_uploader; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; /// Deployment-global operator-listener mention registrations and delivery queues. diff --git a/crates/buzz-db/src/store/push.rs b/crates/buzz-db/src/store/push.rs index 24ae312521e..d4ae0b86b3c 100644 --- a/crates/buzz-db/src/store/push.rs +++ b/crates/buzz-db/src/store/push.rs @@ -360,13 +360,7 @@ pub async fn accept_lease_event( return Err(error.into()); } // The row above is always kind 30350 with no channel. - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - 30350, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, None).await?; let (is_active, endpoint_hash, endpoint_grant, max_class, subscriptions) = active .map_or((false, None, None, None, None), |active| { diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 6b21c7ce01a..3f7887cd25e 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -1391,13 +1391,7 @@ impl Db { let was_inserted = insert_result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - kind_i32, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, &event, None).await?; tx.commit().await?; } else { tx.rollback().await?; diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 1ccee163320..59482e742c4 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -383,8 +383,7 @@ async fn replace_parameterized_event_in_transaction_impl( .execute(tx.conn()) .await?; if status == ParameterizedReplaceStatus::Inserted { - crate::store::event_follow_up::after_admitted_insert(tx, incoming_id, kind_i32, channel_id) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(ParameterizedReplaceResult::new( @@ -547,13 +546,8 @@ impl Db { )); } - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) + .await?; // The replaceable event and its denormalized mention index are one // authoritative discovery write. An indexing error must roll back the diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 6864cb15ae2..000ffaf6feb 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -576,6 +576,16 @@ async fn upload_blob_result( auth.tenant.host(), ); + state + .db + .record_media_uploader( + serving_write.lease(), + &descriptor.sha256, + auth.auth_event.pubkey.as_bytes(), + ) + .await + .map_err(|error| serving_lease_lost(error.into()))?; + // Normalize MIME to a known set to bound label cardinality. let mime_label = match descriptor.mime_type.as_str() { "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "video/mp4" => { @@ -2135,11 +2145,13 @@ mod tests { const TEST_ISSUER: &str = "https://issuer.example"; const TEST_AUDIENCE: &str = "https://relay.example"; const TEST_KID: &str = "test-key-1"; - const TEST_EC_PKCS8_PEM: &str = "-----BEGIN PRIVATE KEY-----\n\ - MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n\ - WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n\ - zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n\ - -----END PRIVATE KEY-----\n"; + const TEST_EC_PKCS8_PEM: &str = concat!( + "-----BEGIN PRIVATE KEY-----\n", // sadscan:disable kingfisher.privkey.2 -- static test-only key + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n", + "WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n", + "zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n", + "-----END PRIVATE KEY-----\n", + ); /// Build an AppState with NIP-FI Enforce and a real injected P-256 verifier. async fn media_enforce_test_state() -> Option> { diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql new file mode 100644 index 00000000000..c2f13825d63 --- /dev/null +++ b/migrations/0067_media_access.sql @@ -0,0 +1,29 @@ +-- Channel-scoped media access: who may read each blob. +-- +-- media_uploaders: who has uploaded each blob. Uploading the exact bytes +-- proves possession, so an uploader may share a blob into their own channels +-- even when the same bytes are also shared somewhere they cannot read. +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); + +-- media_shares: one row per channel a blob is shared into, written with the +-- event that references it, or a NULL channel for a channel-less reference +-- (avatars, custom emoji, notes), which shares it with the whole community. +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; + +SELECT attach_community_write_fence('media_uploaders'); +SELECT attach_community_write_fence('media_shares'); diff --git a/schema/schema.sql b/schema/schema.sql index d252cbdb7ea..9bdb5321b26 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -107,6 +107,8 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i tables/public/moderation_reports.sql \i tables/public/community_bans.sql \i tables/public/community_nip_fi_issuers.sql +\i tables/public/media_shares.sql +\i tables/public/media_uploaders.sql \i tables/public/_operator_global_tables.sql \i tables/public/git_repo_names.sql \i tables/public/parameterized_event_watermarks.sql @@ -159,6 +161,8 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i triggers/event_mentions.sql \i triggers/git_repo_names.sql \i triggers/join_policy_acceptances.sql +\i triggers/media_shares.sql +\i triggers/media_uploaders.sql \i triggers/moderation_actions.sql \i triggers/moderation_reports.sql \i triggers/parameterized_event_watermarks.sql diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql new file mode 100644 index 00000000000..18ab231b4bc --- /dev/null +++ b/schema/tables/public/media_shares.sql @@ -0,0 +1,15 @@ +-- ── Media shares ────────────────────────────────────────────────────────────── +-- Where each blob is shared: one row per channel an event referencing it was +-- stored in, or a NULL channel for a channel-less reference, which shares it +-- with the whole community. + +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql new file mode 100644 index 00000000000..5522ae84a7a --- /dev/null +++ b/schema/tables/public/media_uploaders.sql @@ -0,0 +1,11 @@ +-- ── Media uploaders ─────────────────────────────────────────────────────────── +-- Who has uploaded each blob in a community. Uploading the exact bytes proves +-- possession, so an uploader may share a blob into their own channels. + +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); diff --git a/schema/triggers/media_shares.sql b/schema/triggers/media_shares.sql new file mode 100644 index 00000000000..e105e557bf9 --- /dev/null +++ b/schema/triggers/media_shares.sql @@ -0,0 +1 @@ +SELECT attach_community_write_fence('media_shares'); diff --git a/schema/triggers/media_uploaders.sql b/schema/triggers/media_uploaders.sql new file mode 100644 index 00000000000..23367202e24 --- /dev/null +++ b/schema/triggers/media_uploaders.sql @@ -0,0 +1 @@ +SELECT attach_community_write_fence('media_uploaders'); From 86525c2f35aab42d12e1dbf5a1febc11c85079e1 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 22:26:50 +1100 Subject: [PATCH 02/48] test(db): upload media before posting it in share-recording tests Clients upload a file before attaching it, so record the uploader first and keep the test valid once reads deny files not yet shared into any channel. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 31 ++++++++++++--------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index c4aa9fb95e3..624e2bd4d0e 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -212,6 +212,22 @@ mod postgres_tests { format!("{:064x}", Uuid::new_v4().as_u128()) } + async fn upload(db: &Db, community: CommunityId, uploader: &Keys, sha256: &str) { + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community, + "media_upload", + "media-shares-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + db.record_media_uploader(&lease, sha256, &uploader.public_key().to_bytes()) + .await + .expect("record uploader"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn stored_references_share_only_blobs_their_author_can_read() { @@ -221,6 +237,7 @@ mod postgres_tests { let private = channel(&db, community, &alice, ChannelVisibility::Private).await; let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; + upload(&db, community, &alice, &sha).await; db.insert_event( community, &attachment(&alice, &sha, Some(private)), @@ -239,19 +256,7 @@ mod postgres_tests { "a hash alone must not share a blob its author cannot read" ); - let lease = db - .deletion_store() - .acquire_serving_write_lease( - community, - "media_upload", - "media-shares-test", - Duration::from_secs(60), - ) - .await - .expect("acquire lease"); - db.record_media_uploader(&lease, &sha, &bob.public_key().to_bytes()) - .await - .expect("bob uploads the same bytes"); + upload(&db, community, &bob, &sha).await; db.insert_event(community, &attachment(&bob, &sha, Some(bobs)), Some(bobs)) .await .expect("bob posts his copy"); From 35e67a227a7b2fb798602a583eb4fc6fcc8650be Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 08:32:30 +1100 Subject: [PATCH 03/48] fix(db): backfill only history unchecked; lock media shares in key order Trust references only from events received before migration 0067 was installed; later events get the live, checked recording, so a backfill run after rollout never shares a blob that live recording refused. Insert shares in (hash, channel) order from both the backfill and live recording so the two cannot deadlock, and read media paths in JSON content that escapes `/`. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-core/src/media.rs | 9 +- crates/buzz-db/src/store/media_reference.rs | 117 +++++++++++++++----- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index 29bd3a124ec..452238edba8 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -21,7 +21,9 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { { return Vec::new(); } - let mut hashes: BTreeSet = media_path_hashes(&event.content).collect(); + // JSON content may escape `/` as `\/`; scan the path as clients will see it. + let content = event.content.replace("\\/", "/"); + let mut hashes: BTreeSet = media_path_hashes(&content).collect(); for tag in event.tags.iter() { hashes.extend( tag.as_slice() @@ -94,6 +96,11 @@ mod tests { let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar, "jpg")); let profile = signed(EventBuilder::new(Kind::Metadata, &content)); assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); + let escaped = signed(EventBuilder::new( + Kind::Metadata, + content.replace('/', "\\/"), + )); + assert_eq!(shared_blob_hashes(&escaped), vec![avatar.clone()]); let emoji_set = signed( EventBuilder::new(Kind::from(30030), "").tags([Tag::parse([ diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 624e2bd4d0e..29de6e02989 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -35,42 +35,52 @@ macro_rules! readable_sql { /// in `sha256s` that `author` can read or has uploaded (uploading the exact /// bytes proves possession). Any other blob is left unshared, so a reference /// built from a leaked hash shows it to no one new. Called with the -/// transaction that stores the referencing event. +/// transaction that stores the referencing event; returns the shares added. pub(crate) async fn record_media_shares( conn: &mut PgConnection, community_id: CommunityId, sha256s: &[String], author: &[u8], channel_id: Option, -) -> Result<()> { - sqlx::query(concat!( +) -> Result { + let added = sqlx::query(concat!( "INSERT INTO media_shares (community_id, sha256, channel_id) \ SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ WHERE EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ OR ", readable_sql!(), - " ON CONFLICT DO NOTHING" + " ORDER BY b.sha256 ON CONFLICT DO NOTHING" )) .bind(community_id.as_uuid()) .bind(sha256s) .bind(author) .bind(channel_id) .execute(conn) - .await?; - Ok(()) + .await? + .rows_affected(); + Ok(added) } impl Db { /// Record the shares of every event already stored in an active community, - /// deleted events included, `batch_size` events per transaction; returns the - /// number of shares added. + /// deleted events included, reading `batch_size` events at a time; returns + /// the number of shares added. /// - /// Shares come from the same rule as live writes. History predates the - /// readability check, so its references are recorded as they stand. - /// Idempotent and resumable: rerunning only fills shares still missing. + /// Shares come from the same rule as live writes. Events received before + /// share recording was installed predate the readability check, so their + /// references are recorded as they stand; later events get the live, + /// checked recording, so the backfill never shares what live recording + /// refused. Idempotent and resumable: rerunning only fills missing shares. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { + let installed: Option> = + sqlx::query_scalar("SELECT installed_on FROM _sqlx_migrations WHERE version = 67") + .fetch_optional(&self.pool) + .await?; + let installed = installed.ok_or_else(|| { + crate::DbError::InvalidData("media access migration 0067 is not installed".into()) + })?; let communities: Vec = sqlx::query_scalar( "SELECT id FROM communities WHERE deleted_at IS NULL AND deletion_state = 'active' \ ORDER BY id", @@ -105,13 +115,10 @@ impl Db { cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; - let mut tx = crate::begin_community_write_transaction( - &self.pool, - community, - crate::observability::WriterOperation::Maintenance, - ) - .await?; + let mut history: Vec<(String, Option)> = Vec::new(); + let mut recorded_live = Vec::new(); for row in rows { + let received_at: DateTime = row.try_get("received_at")?; let Some(stored) = crate::event::row_to_stored_event(row)? else { continue; }; @@ -119,18 +126,46 @@ impl Db { if hashes.is_empty() { continue; } + if received_at < installed { + history.extend(hashes.into_iter().map(|hash| (hash, stored.channel_id))); + } else { + recorded_live.push((stored.event.pubkey, hashes, stored.channel_id)); + } + } + // Insert in key order, as live recording does, so a concurrent + // publish can never take the same share locks in reverse. + history.sort(); + history.dedup(); + if !history.is_empty() { + let (hashes, channels): (Vec, Vec>) = + history.into_iter().unzip(); + let mut tx = self.begin_media_share_transaction(community).await?; added += sqlx::query( "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, unnest($2::text[]), $3 ON CONFLICT DO NOTHING", + SELECT $1, h, c FROM unnest($2::text[], $3::uuid[]) AS t(h, c) \ + ORDER BY h, c NULLS FIRST \ + ON CONFLICT DO NOTHING", ) .bind(community.as_uuid()) - .bind(&hashes) - .bind(stored.channel_id) + .bind(hashes) + .bind(channels) .execute(tx.conn()) .await? .rows_affected(); + tx.commit().await?; + } + for (author, hashes, channel) in recorded_live { + let mut tx = self.begin_media_share_transaction(community).await?; + added += record_media_shares( + tx.conn(), + community, + &hashes, + author.as_bytes(), + channel, + ) + .await?; + tx.commit().await?; } - tx.commit().await?; if !full_batch { break; } @@ -138,6 +173,18 @@ impl Db { } Ok(added) } + + async fn begin_media_share_transaction( + &self, + community: CommunityId, + ) -> Result { + crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await + } } #[cfg(test)] @@ -271,11 +318,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn backfill_records_shares_of_stored_events() { + async fn backfill_trusts_history_and_rechecks_recorded_events() { let (db, community) = fixture().await; - let alice = Keys::generate(); - let sha = unique_sha(); + let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); + let (sha, historical) = (unique_sha(), unique_sha()); let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + upload(&db, community, &alice, &sha).await; db.insert_event( community, &attachment(&alice, &sha, Some(private)), @@ -283,6 +331,14 @@ mod postgres_tests { ) .await .expect("alice posts privately"); + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references the hash"); + let old = attachment(&carol, &historical, None); + db.insert_event(community, &old, None) + .await + .expect("carol posts before share recording"); + let mut tx = crate::begin_community_write_transaction( &db.pool, community, @@ -290,6 +346,12 @@ mod postgres_tests { ) .await .expect("admit"); + sqlx::query("UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = $2") + .bind(community.as_uuid()) + .bind(old.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .expect("age carol's event"); sqlx::query("DELETE FROM media_shares WHERE community_id = $1") .bind(community.as_uuid()) .execute(tx.conn()) @@ -298,6 +360,11 @@ mod postgres_tests { tx.commit().await.expect("commit"); db.backfill_media_shares(100).await.expect("backfill"); - assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + assert_eq!( + shares(&db, community, &sha).await, + vec![Some(private)], + "a reference live recording refused must stay unshared" + ); + assert_eq!(shares(&db, community, &historical).await, vec![None]); } } From 4275742fec45bb79468b07e03e8232935e96b3ae Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 10:41:35 +1100 Subject: [PATCH 04/48] fix(media): record shares safely before enforcing reads Require upload proof or a recorded share when recording new references. Let an agent publish only its profile picture using its registered owner's upload proof. Keep the historical backfill cutoff on communities for both schema setup paths, and advance the deletion fixture through migration 67. Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 5 +- crates/buzz-core/src/media.rs | 12 ++ crates/buzz-db/src/store/deletion.rs | 2 +- crates/buzz-db/src/store/event_follow_up.rs | 2 +- crates/buzz-db/src/store/media_reference.rs | 160 +++++++++++++++----- migrations/0067_media_access.sql | 4 + schema/tables/public/communities.sql | 2 + 7 files changed, 148 insertions(+), 39 deletions(-) diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 23ce4227488..f415f4c71f3 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -97,8 +97,9 @@ enum Command { }, /// Record media shares for events stored before share recording existed. /// - /// Idempotent and resumable; run once after deploying share recording and - /// before enforcing channel-scoped media reads. + /// Idempotent and resumable; run after every relay pod records shares and + /// before enforcing channel-scoped media reads. Historical references are + /// trusted only before each community's schema-recorded cutoff. BackfillMediaShares { /// Events read and recorded per transaction. #[arg(long, default_value_t = 1000)] diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index 452238edba8..bac5f48dceb 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -34,6 +34,18 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { hashes.into_iter().collect() } +/// The media blob used as a kind:0 profile's picture. A registered agent may +/// publish this one field using its owner's upload proof; arbitrary content +/// and tags must not inherit that authority. +pub fn profile_picture_blob_hash(event: &nostr::Event) -> Option { + if event.kind != nostr::Kind::Metadata { + return None; + } + let profile: serde_json::Value = serde_json::from_str(&event.content).ok()?; + let hash = media_path_hashes(profile.get("picture")?.as_str()?).next(); + hash +} + fn is_sha256_hex(value: &str) -> bool { value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) } diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index c48816a39d1..14033ee74d6 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -4399,7 +4399,7 @@ mod postgres_tests { }) .await .expect("connect isolated migration DB"); - crate::migration::run_migrations_through(&db.pool, 66) + crate::migration::run_migrations_through(&db.pool, 67) .await .expect("migrate with fence objects present"); let store = db.deletion_store(); diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index dd4e5aa9d19..6fd0de9fc27 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -55,7 +55,7 @@ pub(crate) async fn after_admitted_insert( tx.conn(), community, &hashes, - event.pubkey.as_bytes(), + event, channel_id, ) .await?; diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 29de6e02989..0be5c1d6737 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -10,17 +10,14 @@ use buzz_core::CommunityId; /// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. /// -/// Everyone may when the blob is shared with the whole community or into no -/// channel; otherwise only readers of a channel it is shared into, open or one -/// they actively belong to. Shares are never withdrawn. +/// A share admits everyone when it is community-wide, or readers of its +/// channel (open, or one they actively belong to). An unknown or unshared hash +/// grants nothing. Shares are never withdrawn. macro_rules! readable_sql { () => { "( \ EXISTS (SELECT 1 FROM media_shares s \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ - OR NOT EXISTS (SELECT 1 FROM media_shares s \ - WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ - AND s.channel_id IS NOT NULL) \ OR EXISTS (SELECT 1 FROM media_shares s \ JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ @@ -32,15 +29,16 @@ macro_rules! readable_sql { } /// Share into `channel_id` (or with the whole community when `None`) each blob -/// in `sha256s` that `author` can read or has uploaded (uploading the exact -/// bytes proves possession). Any other blob is left unshared, so a reference -/// built from a leaked hash shows it to no one new. Called with the +/// in `sha256s` that the event's author can read or has uploaded (uploading +/// the exact bytes proves possession). An agent's profile picture may also +/// use its registered owner's upload proof. Any other blob is left unshared, +/// so a reference built from a leaked hash shows it to no one new. Called with the /// transaction that stores the referencing event; returns the shares added. pub(crate) async fn record_media_shares( conn: &mut PgConnection, community_id: CommunityId, sha256s: &[String], - author: &[u8], + event: &nostr::Event, channel_id: Option, ) -> Result { let added = sqlx::query(concat!( @@ -48,14 +46,19 @@ pub(crate) async fn record_media_shares( SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ WHERE EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR (b.sha256 = $5 AND EXISTS ( \ + SELECT 1 FROM users agent JOIN media_uploaders u \ + ON u.community_id = agent.community_id AND u.pubkey = agent.agent_owner_pubkey \ + WHERE agent.community_id = $1 AND agent.pubkey = $3 AND u.sha256 = b.sha256)) \ OR ", readable_sql!(), " ORDER BY b.sha256 ON CONFLICT DO NOTHING" )) .bind(community_id.as_uuid()) .bind(sha256s) - .bind(author) + .bind(event.pubkey.as_bytes().as_slice()) .bind(channel_id) + .bind(buzz_core::media::profile_picture_blob_hash(event)) .execute(conn) .await? .rows_affected(); @@ -74,21 +77,19 @@ impl Db { /// refused. Idempotent and resumable: rerunning only fills missing shares. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { - let installed: Option> = - sqlx::query_scalar("SELECT installed_on FROM _sqlx_migrations WHERE version = 67") - .fetch_optional(&self.pool) - .await?; - let installed = installed.ok_or_else(|| { - crate::DbError::InvalidData("media access migration 0067 is not installed".into()) - })?; - let communities: Vec = sqlx::query_scalar( - "SELECT id FROM communities WHERE deleted_at IS NULL AND deletion_state = 'active' \ - ORDER BY id", + if batch_size <= 0 { + return Err(crate::DbError::InvalidData( + "batch size must be positive".into(), + )); + } + let communities: Vec<(Uuid, DateTime)> = sqlx::query_as( + "SELECT id, media_shares_recording_started_at FROM communities \ + WHERE deleted_at IS NULL AND deletion_state = 'active' ORDER BY id", ) .fetch_all(&self.pool) .await?; let mut added = 0; - for community in communities { + for (community, installed) in communities { let community = CommunityId::from_uuid(community); let mut cursor: Option<(DateTime, Vec)> = None; loop { @@ -129,7 +130,7 @@ impl Db { if received_at < installed { history.extend(hashes.into_iter().map(|hash| (hash, stored.channel_id))); } else { - recorded_live.push((stored.event.pubkey, hashes, stored.channel_id)); + recorded_live.push((stored.event, hashes, stored.channel_id)); } } // Insert in key order, as live recording does, so a concurrent @@ -154,16 +155,10 @@ impl Db { .rows_affected(); tx.commit().await?; } - for (author, hashes, channel) in recorded_live { + for (event, hashes, channel) in recorded_live { let mut tx = self.begin_media_share_transaction(community).await?; - added += record_media_shares( - tx.conn(), - community, - &hashes, - author.as_bytes(), - channel, - ) - .await?; + added += + record_media_shares(tx.conn(), community, &hashes, &event, channel).await?; tx.commit().await?; } if !full_batch { @@ -236,7 +231,7 @@ mod postgres_tests { if let Some(channel) = channel { tags.push(Tag::parse(["h", &channel.to_string()]).unwrap()); } - EventBuilder::new(Kind::from(9), "attachment") + EventBuilder::new(Kind::from(9), format!("attachment {}", Uuid::new_v4())) .tags(tags) .sign_with_keys(keys) .unwrap() @@ -284,7 +279,18 @@ mod postgres_tests { let private = channel(&db, community, &alice, ChannelVisibility::Private).await; let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; - upload(&db, community, &alice, &sha).await; + for uploader_present in [false, true] { + if uploader_present { + upload(&db, community, &alice, &sha).await; + } + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references an unshared hash"); + assert!( + shares(&db, community, &sha).await.is_empty(), + "a hash alone grants nothing, even before its uploader posts it" + ); + } db.insert_event( community, &attachment(&alice, &sha, Some(private)), @@ -319,6 +325,10 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn backfill_trusts_history_and_rechecks_recorded_events() { + check_backfill().await; + } + + async fn check_backfill() { let (db, community) = fixture().await; let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); let (sha, historical) = (unique_sha(), unique_sha()); @@ -359,7 +369,8 @@ mod postgres_tests { .expect("forget shares"); tx.commit().await.expect("commit"); - db.backfill_media_shares(100).await.expect("backfill"); + db.backfill_media_shares(1).await.expect("backfill"); + assert_eq!(db.backfill_media_shares(1).await.expect("rerun"), 0); assert_eq!( shares(&db, community, &sha).await, vec![Some(private)], @@ -367,4 +378,83 @@ mod postgres_tests { ); assert_eq!(shares(&db, community, &historical).await, vec![None]); } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn migration_schema_backfill_trusts_history_and_rechecks_recorded_events() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to migration DB"); + crate::migration::run_migrations_through(&pool, 67) + .await + .expect("migrate"); + pool.close().await; + check_backfill().await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn agent_profile_picture_uses_only_its_registered_owners_upload_proof() { + let (db, community) = fixture().await; + let (owner, agent) = (Keys::generate(), Keys::generate()); + let (avatar, secret, unknown) = (unique_sha(), unique_sha(), unique_sha()); + for sha in [&avatar, &secret] { + upload(&db, community, &owner, sha).await; + } + let profile = |picture: &str| { + EventBuilder::new( + Kind::Metadata, + serde_json::json!({ + "name": Uuid::new_v4().to_string(), + "picture": format!("/media/{picture}.jpg"), + "about": format!("/media/{secret}.jpg"), + }) + .to_string(), + ) + .tags([Tag::parse(["p", &owner.public_key().to_hex()]).unwrap()]) + .sign_with_keys(&agent) + .unwrap() + }; + db.insert_event(community, &profile(&avatar), None) + .await + .unwrap(); + assert!( + shares(&db, community, &avatar).await.is_empty(), + "an unverified owner tag grants nothing" + ); + for keys in [&owner, &agent] { + db.ensure_user(community, &keys.public_key().to_bytes()) + .await + .unwrap(); + } + db.set_agent_owner( + community, + &agent.public_key().to_bytes(), + &owner.public_key().to_bytes(), + ) + .await + .unwrap(); + db.insert_event(community, &profile(&avatar), None) + .await + .unwrap(); + assert_eq!(shares(&db, community, &avatar).await, vec![None]); + assert!( + shares(&db, community, &secret).await.is_empty(), + "owner proof applies only to picture" + ); + db.insert_event(community, &profile(&unknown), None) + .await + .unwrap(); + assert!( + shares(&db, community, &unknown).await.is_empty(), + "the owner must have uploaded these bytes" + ); + db.insert_event(community, &attachment(&agent, &secret, None), None) + .await + .unwrap(); + assert!( + shares(&db, community, &secret).await.is_empty(), + "ordinary events cannot borrow the owner's uploads" + ); + } } diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql index c2f13825d63..b70ac3530b2 100644 --- a/migrations/0067_media_access.sql +++ b/migrations/0067_media_access.sql @@ -1,3 +1,7 @@ +-- Durable historical backfill cutoff, independent of the schema install tool. +ALTER TABLE communities + ADD COLUMN media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(); + -- Channel-scoped media access: who may read each blob. -- -- media_uploaders: who has uploaded each blob. Uploading the exact bytes diff --git a/schema/tables/public/communities.sql b/schema/tables/public/communities.sql index 06f74c3f585..ea27392437b 100644 --- a/schema/tables/public/communities.sql +++ b/schema/tables/public/communities.sql @@ -20,6 +20,8 @@ CREATE TABLE communities ( -- Added by migration 0003; kept here so desired-state applies match. icon TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + -- The share-recording cutoff exists in both desired-state and migrated DBs. + media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(), archived_at TIMESTAMPTZ, deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), From 36c9cdae62b4939bb45e5ed2ae75c8a302542a85 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 13:46:41 +1100 Subject: [PATCH 05/48] fix(media): revoke grants from deleted messages Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 170 +++++++++++++++++--- migrations/0067_media_access.sql | 15 +- schema/tables/public/media_shares.sql | 15 +- 3 files changed, 161 insertions(+), 39 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 0be5c1d6737..dc776f72e6a 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -12,13 +12,17 @@ use buzz_core::CommunityId; /// /// A share admits everyone when it is community-wide, or readers of its /// channel (open, or one they actively belong to). An unknown or unshared hash -/// grants nothing. Shares are never withdrawn. +/// grants nothing. Deleted or purged source events grant nothing. macro_rules! readable_sql { () => { "( \ EXISTS (SELECT 1 FROM media_shares s \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ AND (c.visibility = 'open' \ @@ -41,10 +45,13 @@ pub(crate) async fn record_media_shares( event: &nostr::Event, channel_id: Option, ) -> Result { + let created_at_secs = event.created_at.as_secs() as i64; + let created_at = DateTime::from_timestamp(created_at_secs, 0) + .ok_or(crate::DbError::InvalidTimestamp(created_at_secs))?; let added = sqlx::query(concat!( - "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ - WHERE EXISTS (SELECT 1 FROM media_uploaders u \ + "INSERT INTO media_shares (community_id, sha256, channel_id, event_id, event_created_at) \ + SELECT $1, b.sha256, $4, $6, $7 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + WHERE (EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ OR (b.sha256 = $5 AND EXISTS ( \ SELECT 1 FROM users agent JOIN media_uploaders u \ @@ -52,13 +59,18 @@ pub(crate) async fn record_media_shares( WHERE agent.community_id = $1 AND agent.pubkey = $3 AND u.sha256 = b.sha256)) \ OR ", readable_sql!(), - " ORDER BY b.sha256 ON CONFLICT DO NOTHING" + ") AND EXISTS (SELECT 1 FROM events e \ + WHERE e.community_id = $1 AND e.id = $6 AND e.created_at = $7 \ + AND e.deleted_at IS NULL) \ + ORDER BY b.sha256 ON CONFLICT DO NOTHING" )) .bind(community_id.as_uuid()) .bind(sha256s) .bind(event.pubkey.as_bytes().as_slice()) .bind(channel_id) .bind(buzz_core::media::profile_picture_blob_hash(event)) + .bind(event.id.as_bytes().as_slice()) + .bind(created_at) .execute(conn) .await? .rows_affected(); @@ -66,8 +78,8 @@ pub(crate) async fn record_media_shares( } impl Db { - /// Record the shares of every event already stored in an active community, - /// deleted events included, reading `batch_size` events at a time; returns + /// Record the shares of undeleted events stored in an active community, + /// reading `batch_size` events at a time; returns /// the number of shares added. /// /// Shares come from the same rule as live writes. Events received before @@ -97,7 +109,7 @@ impl Db { "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, \ channel_id \ FROM events \ - WHERE community_id = $1 \ + WHERE community_id = $1 AND deleted_at IS NULL \ AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) \ ORDER BY created_at, id LIMIT $4", ) @@ -116,10 +128,11 @@ impl Db { cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; - let mut history: Vec<(String, Option)> = Vec::new(); + let mut history = Vec::new(); let mut recorded_live = Vec::new(); for row in rows { let received_at: DateTime = row.try_get("received_at")?; + let event_created_at: DateTime = row.try_get("created_at")?; let Some(stored) = crate::event::row_to_stored_event(row)? else { continue; }; @@ -128,7 +141,10 @@ impl Db { continue; } if received_at < installed { - history.extend(hashes.into_iter().map(|hash| (hash, stored.channel_id))); + let event_id = stored.event.id.to_bytes().to_vec(); + history.extend(hashes.into_iter().map(|hash| { + (hash, stored.channel_id, event_id.clone(), event_created_at) + })); } else { recorded_live.push((stored.event, hashes, stored.channel_id)); } @@ -138,18 +154,27 @@ impl Db { history.sort(); history.dedup(); if !history.is_empty() { - let (hashes, channels): (Vec, Vec>) = - history.into_iter().unzip(); + let hashes: Vec<_> = history.iter().map(|(h, _, _, _)| h).collect(); + let channels: Vec<_> = history.iter().map(|(_, c, _, _)| *c).collect(); + let event_ids: Vec<_> = history.iter().map(|(_, _, id, _)| id).collect(); + let event_times: Vec<_> = history.iter().map(|(_, _, _, at)| *at).collect(); let mut tx = self.begin_media_share_transaction(community).await?; added += sqlx::query( - "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, h, c FROM unnest($2::text[], $3::uuid[]) AS t(h, c) \ - ORDER BY h, c NULLS FIRST \ + "INSERT INTO media_shares \ + (community_id, sha256, channel_id, event_id, event_created_at) \ + SELECT $1, h, c, e.id, e.created_at \ + FROM unnest($2::text[], $3::uuid[], $4::bytea[], $5::timestamptz[]) \ + AS t(h, c, event_id, event_created_at) \ + JOIN events e ON e.community_id = $1 AND e.id = t.event_id \ + AND e.created_at = t.event_created_at AND e.deleted_at IS NULL \ + ORDER BY h, e.id \ ON CONFLICT DO NOTHING", ) .bind(community.as_uuid()) .bind(hashes) .bind(channels) + .bind(event_ids) + .bind(event_times) .execute(tx.conn()) .await? .rows_affected(); @@ -239,7 +264,7 @@ mod postgres_tests { async fn shares(db: &Db, community: CommunityId, sha256: &str) -> Vec> { let mut shares: Vec> = sqlx::query_scalar( - "SELECT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + "SELECT DISTINCT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", ) .bind(community.as_uuid()) .bind(sha256) @@ -328,10 +353,95 @@ mod postgres_tests { check_backfill().await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn deleting_a_reference_revokes_only_its_own_media_grant() { + let (db, community) = fixture().await; + let (owner, reader) = (Keys::generate(), Keys::generate()); + let open = channel(&db, community, &owner, ChannelVisibility::Open).await; + for channel_id in [Some(open), None] { + let sha = unique_sha(); + upload(&db, community, &owner, &sha).await; + let first = attachment(&owner, &sha, channel_id); + let last = attachment(&owner, &sha, channel_id); + for event in [&first, &last] { + db.insert_event(community, event, channel_id).await.unwrap(); + } + let sources: Vec> = sqlx::query_scalar( + "SELECT event_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + ) + .bind(community.as_uuid()) + .bind(&sha) + .fetch_all(&db.pool) + .await + .unwrap(); + assert_eq!(sources.len(), 2, "each message must retain its own grant"); + for event in [&first, &last] { + assert!(sources.contains(&event.id.to_bytes().to_vec())); + } + assert!(readable_share(&db, community, &sha, &reader).await); + for (event, still_shared) in [(&first, true), (&last, false)] { + assert!( + db.soft_delete_event_and_update_thread( + community, + event.id.as_bytes(), + None, + None, + ) + .await + .unwrap() + ); + assert_eq!( + readable_share(&db, community, &sha, &reader).await, + still_shared, + "only an undeleted source message can grant access" + ); + } + db.insert_event(community, &attachment(&reader, &sha, None), None) + .await + .unwrap(); + assert!(!readable_share(&db, community, &sha, &reader).await); + db.backfill_media_shares(1).await.unwrap(); + assert!(!readable_share(&db, community, &sha, &reader).await); + + let purged = attachment(&owner, &sha, channel_id); + db.insert_event(community, &purged, channel_id) + .await + .unwrap(); + assert!(readable_share(&db, community, &sha, &reader).await); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query("DELETE FROM events WHERE community_id = $1 AND id = $2") + .bind(community.as_uuid()) + .bind(purged.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert!( + !readable_share(&db, community, &sha, &reader).await, + "a purged source cannot leave a dangling grant" + ); + } + } + + async fn readable_share(db: &Db, community: CommunityId, sha: &str, reader: &Keys) -> bool { + sqlx::query_scalar(concat!( + "SELECT ", + readable_sql!(), + " FROM (SELECT $2::text AS sha256) b" + )) + .bind(community.as_uuid()) + .bind(sha) + .bind(reader.public_key().to_bytes().to_vec()) + .fetch_one(&db.pool) + .await + .unwrap() + } + async fn check_backfill() { let (db, community) = fixture().await; let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); - let (sha, historical) = (unique_sha(), unique_sha()); + let (sha, historical, deleted_history) = (unique_sha(), unique_sha(), unique_sha()); let private = channel(&db, community, &alice, ChannelVisibility::Private).await; upload(&db, community, &alice, &sha).await; db.insert_event( @@ -348,6 +458,11 @@ mod postgres_tests { db.insert_event(community, &old, None) .await .expect("carol posts before share recording"); + let deleted = attachment(&carol, &deleted_history, None); + db.insert_event(community, &deleted, None).await.unwrap(); + db.soft_delete_event_and_update_thread(community, deleted.id.as_bytes(), None, None) + .await + .unwrap(); let mut tx = crate::begin_community_write_transaction( &db.pool, @@ -356,12 +471,17 @@ mod postgres_tests { ) .await .expect("admit"); - sqlx::query("UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = $2") - .bind(community.as_uuid()) - .bind(old.id.as_bytes().as_slice()) - .execute(tx.conn()) - .await - .expect("age carol's event"); + sqlx::query( + "UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = ANY($2)", + ) + .bind(community.as_uuid()) + .bind(vec![ + old.id.to_bytes().to_vec(), + deleted.id.to_bytes().to_vec(), + ]) + .execute(tx.conn()) + .await + .expect("age carol's event"); sqlx::query("DELETE FROM media_shares WHERE community_id = $1") .bind(community.as_uuid()) .execute(tx.conn()) @@ -377,6 +497,10 @@ mod postgres_tests { "a reference live recording refused must stay unshared" ); assert_eq!(shares(&db, community, &historical).await, vec![None]); + assert!( + shares(&db, community, &deleted_history).await.is_empty(), + "legacy deleted messages must never gain a share" + ); } #[tokio::test] diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql index b70ac3530b2..9b4212113fa 100644 --- a/migrations/0067_media_access.sql +++ b/migrations/0067_media_access.sql @@ -15,19 +15,18 @@ CREATE TABLE media_uploaders ( PRIMARY KEY (community_id, sha256, pubkey) ); --- media_shares: one row per channel a blob is shared into, written with the --- event that references it, or a NULL channel for a channel-less reference --- (avatars, custom emoji, notes), which shares it with the whole community. +-- media_shares: one grant per referencing event and blob. A NULL channel +-- shares with the whole community. Deleted or purged source events grant +-- nothing; event_created_at locates the source in the partitioned table. CREATE TABLE media_shares ( community_id UUID NOT NULL REFERENCES communities(id), sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), channel_id UUID, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + event_id BYTEA NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, event_id) ); -CREATE UNIQUE INDEX idx_media_shares_channel - ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; -CREATE UNIQUE INDEX idx_media_shares_public - ON media_shares (community_id, sha256) WHERE channel_id IS NULL; SELECT attach_community_write_fence('media_uploaders'); SELECT attach_community_write_fence('media_shares'); diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql index 18ab231b4bc..e914069c963 100644 --- a/schema/tables/public/media_shares.sql +++ b/schema/tables/public/media_shares.sql @@ -1,15 +1,14 @@ -- ── Media shares ────────────────────────────────────────────────────────────── --- Where each blob is shared: one row per channel an event referencing it was --- stored in, or a NULL channel for a channel-less reference, which shares it --- with the whole community. +-- Each event's grant to a channel, or the whole community for a NULL channel. +-- Reads join the source event: deleted or purged events grant no access. +-- Keep its timestamp for an indexed lookup in the partitioned events table. CREATE TABLE media_shares ( community_id UUID NOT NULL REFERENCES communities(id), sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), channel_id UUID, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + event_id BYTEA NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, event_id) ); -CREATE UNIQUE INDEX idx_media_shares_channel - ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; -CREATE UNIQUE INDEX idx_media_shares_public - ON media_shares (community_id, sha256) WHERE channel_id IS NULL; From 7f7f21c7ea0a5c6ddd39d807dcadf13233ea6723 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 14:09:05 +1100 Subject: [PATCH 06/48] fix(media): preserve grants through profile updates Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/event_follow_up.rs | 32 ++++- crates/buzz-db/src/store/media_reference.rs | 65 +++++++++ crates/buzz-db/src/store/replaceable.rs | 131 ++++++++++--------- crates/buzz-db/tests/observability_source.rs | 10 +- 4 files changed, 170 insertions(+), 68 deletions(-) diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index 6fd0de9fc27..9079e496539 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -1,7 +1,10 @@ //! App-owned follow-up work for a newly inserted `events` row. //! -//! Two pieces of work follow every durable event insert: +//! Three pieces of work follow every durable event insert: //! +//! - **Media grants.** References grant access only after their author passes +//! the sharing check. Replacements record these before retiring the old head, +//! inside their savepoint; delivery is staged only after it succeeds. //! - **Push match enqueue.** Eligible event IDs are staged in memory. Commit //! submits them to a bounded best-effort worker using a dedicated connection. //! Push errors, delays, overload, and shutdown can drop wakes, never messages. @@ -45,9 +48,18 @@ pub(crate) async fn after_admitted_insert( event: &nostr::Event, channel_id: Option, ) -> Result<()> { - let kind = i32::from(event.kind.as_u16()); - tx.record_push_event(event.id.as_bytes(), kind); - tx.record_channel_event(channel_id, kind); + record_admitted_media(tx, event, channel_id).await?; + stage_admitted_insert(tx, event, channel_id); + Ok(()) +} + +/// Record media inside a replacement's savepoint, before retiring its old head. +/// This preserves authority carried by a reference the new head keeps. +pub(crate) async fn record_admitted_media( + tx: &mut AdmittedTx, + event: &nostr::Event, + channel_id: Option, +) -> Result<()> { let hashes = buzz_core::media::shared_blob_hashes(event); if !hashes.is_empty() { let community = tx.community(); @@ -63,6 +75,18 @@ pub(crate) async fn after_admitted_insert( Ok(()) } +/// Stage delivery only after the insertion's savepoint has succeeded. +/// Call after [`record_admitted_media`] when splitting the two phases. +pub(crate) fn stage_admitted_insert( + tx: &mut AdmittedTx, + event: &nostr::Event, + channel_id: Option, +) { + let kind = i32::from(event.kind.as_u16()); + tx.record_push_event(event.id.as_bytes(), kind); + tx.record_channel_event(channel_id, kind); +} + /// The channel whose TTL an event of `kind` in `channel_id` refreshes, if any. pub(crate) fn ttl_refresh_channel(channel_id: Option, kind: i32) -> Option { channel_id.filter(|_| kind != KIND_CHANNEL_CREATE) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index dc776f72e6a..f767db1efc7 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -438,6 +438,71 @@ mod postgres_tests { .unwrap() } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn replacement_preserves_only_the_media_it_continues_to_reference() { + let (db, community) = fixture().await; + let viewer = Keys::generate(); + for kind in [Kind::Metadata, Kind::from(30030)] { + let (uploader, publisher) = (Keys::generate(), Keys::generate()); + let sha = unique_sha(); + upload(&db, community, &uploader, &sha).await; + let base = nostr::Timestamp::now().as_secs(); + let make = |keys: &Keys, kind: Kind, step: u64, picture: bool| { + let url = if picture { + format!("/media/{sha}.jpg") + } else { + String::new() + }; + let mut tags = Vec::new(); + let content = if kind == Kind::Metadata { + serde_json::json!({"picture": url, "name": format!("profile-{step}")}) + .to_string() + } else { + tags.push(Tag::parse(["d", "media-continuity"]).unwrap()); + if picture { + tags.push(Tag::parse(["emoji", "sample", &url]).unwrap()); + } + String::new() + }; + EventBuilder::new(kind, content) + .tags(tags) + .custom_created_at(nostr::Timestamp::from_secs(base + step)) + .sign_with_keys(keys) + .unwrap() + }; + let publish = async |event: &nostr::Event| { + if event.kind == Kind::Metadata { + db.replace_addressable_event(community, event, None) + .await + .unwrap() + .1 + } else { + db.replace_parameterized_event(community, event, "media-continuity", None) + .await + .unwrap() + .1 + } + }; + assert!(publish(&make(&uploader, Kind::Metadata, 0, true)).await); + assert!(publish(&make(&publisher, kind, 0, true)).await); + assert!(publish(&make(&uploader, Kind::Metadata, 1, false)).await); + assert!(readable_share(&db, community, &sha, &viewer).await); + assert!(publish(&make(&publisher, kind, 1, true)).await); + assert!( + readable_share(&db, community, &sha, &viewer).await, + "a replacement must carry forward an authorized reference" + ); + assert!(publish(&make(&publisher, kind, 2, false)).await); + assert!(!readable_share(&db, community, &sha, &viewer).await); + assert!(publish(&make(&publisher, kind, 3, true)).await); + assert!( + !readable_share(&db, community, &sha, &viewer).await, + "a later replacement cannot revive a previously removed grant" + ); + } + } + async fn check_backfill() { let (db, community) = fixture().await; let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 59482e742c4..6f22d56fba2 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -262,6 +262,53 @@ async fn replace_parameterized_event_in_transaction_impl( let tags_json = serde_json::to_value(&event.tags)?; crate::channel_members::guard_channel_roster_event(tx, kind_i32, channel_id, &tags_json) .await?; + + let sig_bytes = event.sig.serialize(); + let insert_result = sqlx::query( + "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \ + ON CONFLICT DO NOTHING", + ) + .bind(community_id.as_uuid()) + .bind(incoming_id) + .bind(pubkey_bytes.as_slice()) + .bind(created_at) + .bind(kind_i32) + .bind(&tags_json) + .bind(&event.content) + .bind(sig_bytes.as_slice()) + .bind(received_at) + .bind(channel_id) + .bind(d_tag) + .bind(crate::event::extract_not_before(event)) + .execute(tx.conn()) + .await?; + + if insert_result.rows_affected() == 0 { + return Ok(false); + } + + if is_nip_rs { + sqlx::query( + "INSERT INTO parameterized_event_watermarks \ + (community_id, kind, pubkey, d_tag, created_at, event_id) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (community_id, kind, pubkey, d_tag) DO UPDATE SET \ + created_at = EXCLUDED.created_at, event_id = EXCLUDED.event_id", + ) + .bind(community_id.as_uuid()) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .bind(created_at) + .bind(incoming_id) + .execute(tx.conn()) + .await?; + } + + crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; + // Authorize retained references while the previous head is still live. + crate::store::event_follow_up::record_admitted_media(tx, event, channel_id).await?; if existing.is_some() { let previous_nip_rs_hard_delete: Option = if is_nip_rs { sqlx::query_scalar( @@ -279,16 +326,17 @@ async fn replace_parameterized_event_in_transaction_impl( } let statement = if hard_delete_superseded { "DELETE FROM events \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL" + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL AND id <> $5" } else { "UPDATE events SET deleted_at = NOW() \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL" + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL AND id <> $5" }; sqlx::query(statement) .bind(community_id.as_uuid()) .bind(kind_i32) .bind(pubkey_bytes.as_slice()) .bind(d_tag) + .bind(incoming_id) .execute(tx.conn()) .await?; @@ -311,50 +359,6 @@ async fn replace_parameterized_event_in_transaction_impl( } } - let sig_bytes = event.sig.serialize(); - let insert_result = sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \ - ON CONFLICT DO NOTHING", - ) - .bind(community_id.as_uuid()) - .bind(incoming_id) - .bind(pubkey_bytes.as_slice()) - .bind(created_at) - .bind(kind_i32) - .bind(&tags_json) - .bind(&event.content) - .bind(sig_bytes.as_slice()) - .bind(received_at) - .bind(channel_id) - .bind(d_tag) - .bind(crate::event::extract_not_before(event)) - .execute(tx.conn()) - .await?; - - if insert_result.rows_affected() == 0 { - return Ok(false); - } - - if is_nip_rs { - sqlx::query( - "INSERT INTO parameterized_event_watermarks \ - (community_id, kind, pubkey, d_tag, created_at, event_id) \ - VALUES ($1, $2, $3, $4, $5, $6) \ - ON CONFLICT (community_id, kind, pubkey, d_tag) DO UPDATE SET \ - created_at = EXCLUDED.created_at, event_id = EXCLUDED.event_id", - ) - .bind(community_id.as_uuid()) - .bind(kind_i32) - .bind(pubkey_bytes.as_slice()) - .bind(d_tag) - .bind(created_at) - .bind(incoming_id) - .execute(tx.conn()) - .await?; - } - - crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; Ok(true) } .await; @@ -383,7 +387,7 @@ async fn replace_parameterized_event_in_transaction_impl( .execute(tx.conn()) .await?; if status == ParameterizedReplaceStatus::Inserted { - crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; + crate::store::event_follow_up::stage_admitted_insert(tx, event, channel_id); } Ok(ParameterizedReplaceResult::new( @@ -492,20 +496,6 @@ impl Db { ) .await?; - // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. - sqlx::query( - "UPDATE events SET deleted_at = NOW() \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 \ - AND channel_id IS NOT DISTINCT FROM $4 \ - AND deleted_at IS NULL", - ) - .bind(community_id.as_uuid()) - .bind(kind_i32) - .bind(pubkey_bytes.as_slice()) - .bind(channel_id) - .execute(tx.conn()) - .await?; - // Insert the new event inside the same transaction. let sig_bytes = event.sig.serialize(); let received_at = chrono::Utc::now(); @@ -532,8 +522,7 @@ impl Db { let was_inserted = insert_result.rows_affected() > 0; if !was_inserted { - // ON CONFLICT fired — the event ID already exists. Rollback the - // soft-delete so we don't lose the previous replaceable event. + // ON CONFLICT fired — leave the previously-live event intact. tx.rollback().await?; return Ok(( StoredEvent::with_received_at( @@ -546,9 +535,25 @@ impl Db { )); } + // Authorize retained references while the previous head is still live. crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) .await?; + // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. + sqlx::query( + "UPDATE events SET deleted_at = NOW() \ + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 \ + AND channel_id IS NOT DISTINCT FROM $4 \ + AND deleted_at IS NULL AND id <> $5", + ) + .bind(community_id.as_uuid()) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(channel_id) + .bind(incoming_id) + .execute(tx.conn()) + .await?; + // The replaceable event and its denormalized mention index are one // authoritative discovery write. An indexing error must roll back the // new event and restore the previously-live event. diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 2b505e01ec8..f13e334394c 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1661,12 +1661,14 @@ fn inserts_events_row(source: &str) -> bool { /// An events insert must run both follow-ups: the push enqueue and the /// pre-commit TTL refresh. `after_admitted_insert` does both; a writer that -/// inserts inside a savepoint calls the two halves separately. +/// inserts inside a savepoint records media there and stages delivery after it. /// /// Only real calls count: a hook in a comment or a string literal does not. fn runs_event_follow_ups(function_source: &str) -> bool { let code = mask_comments_and_literals(function_source); code.contains("event_follow_up::after_admitted_insert(") + || (code.contains("event_follow_up::record_admitted_media(") + && code.contains("event_follow_up::stage_admitted_insert(")) || (code.contains("event_follow_up::enqueue_push_match(") && code.contains(".record_channel_event(")) } @@ -1714,6 +1716,12 @@ pub(crate) async fn hooked_writer(tx: &mut AdmittedTx) { .await .expect("follow up"); } +pub(crate) async fn media_savepoint_writer(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") + .execute(tx.conn()).await.expect("write"); + crate::store::event_follow_up::record_admitted_media(tx, event, channel).await.expect("media"); + crate::store::event_follow_up::stage_admitted_insert(tx, event, channel); +} pub(crate) async fn savepoint_writer(tx: &mut AdmittedTx) { sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") .execute(tx.conn()) From 6ca6acbd826b449a6efdbc71648f31bb8285917e Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Wed, 21 Oct 2026 11:30:33 +1100 Subject: [PATCH 07/48] docs(media): clarify backfill permission changes Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 4 +++- crates/buzz-db/src/store/media_reference.rs | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index f415f4c71f3..a3e5be3dabf 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -97,9 +97,11 @@ enum Command { }, /// Record media shares for events stored before share recording existed. /// - /// Idempotent and resumable; run after every relay pod records shares and + /// Run after every relay pod records shares and /// before enforcing channel-scoped media reads. Historical references are /// trusted only before each community's schema-recorded cutoff. + /// Later references use current permissions, so reruns can grant references + /// live recording previously refused if their authors have since gained access. BackfillMediaShares { /// Events read and recorded per transaction. #[arg(long, default_value_t = 1000)] diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index f767db1efc7..a6a552aae5e 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -85,8 +85,9 @@ impl Db { /// Shares come from the same rule as live writes. Events received before /// share recording was installed predate the readability check, so their /// references are recorded as they stand; later events get the live, - /// checked recording, so the backfill never shares what live recording - /// refused. Idempotent and resumable: rerunning only fills missing shares. + /// checked recording using the author's current permissions. Earlier refusals + /// are not remembered: reruns can grant old references after access changes. + /// Rows are deduplicated, but authorization decisions are not preserved. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { if batch_size <= 0 { From e539e8d6866218cb0adf30dacbafd0fac104b0e6 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 22:23:55 +1100 Subject: [PATCH 08/48] feat(relay): record where media is shared and who uploaded it Add `media_shares` and `media_uploaders` (migration 0067) and start filling them, without changing who can read media yet. Each local `/media/` path in a stored event shares that blob into the event's channel (or with the whole community when channel-less), in the same transaction as the event, when its author can read the blob or uploaded those bytes; each upload records its uploader under the serving-write lease. `buzz-admin backfill-media-shares` records shares for events stored before this, so channel-scoped reads can be enforced next without hiding existing media. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 17 ++ crates/buzz-core/src/lib.rs | 2 + crates/buzz-core/src/media.rs | 130 +++++++++ crates/buzz-db/src/runtime/migration.rs | 19 ++ crates/buzz-db/src/store/channel_members.rs | 9 +- crates/buzz-db/src/store/deletion.rs | 6 + crates/buzz-db/src/store/event.rs | 16 +- crates/buzz-db/src/store/event_follow_up.rs | 22 +- crates/buzz-db/src/store/media_reference.rs | 298 ++++++++++++++++++++ crates/buzz-db/src/store/media_uploader.rs | 110 ++++++++ crates/buzz-db/src/store/mod.rs | 4 + crates/buzz-db/src/store/push.rs | 8 +- crates/buzz-db/src/store/relay_members.rs | 8 +- crates/buzz-db/src/store/replaceable.rs | 12 +- crates/buzz-relay/src/api/media.rs | 22 +- migrations/0067_media_access.sql | 29 ++ schema/schema.sql | 4 + schema/tables/public/media_shares.sql | 15 + schema/tables/public/media_uploaders.sql | 11 + schema/triggers/media_shares.sql | 1 + schema/triggers/media_uploaders.sql | 1 + 21 files changed, 691 insertions(+), 53 deletions(-) create mode 100644 crates/buzz-core/src/media.rs create mode 100644 crates/buzz-db/src/store/media_reference.rs create mode 100644 crates/buzz-db/src/store/media_uploader.rs create mode 100644 migrations/0067_media_access.sql create mode 100644 schema/tables/public/media_shares.sql create mode 100644 schema/tables/public/media_uploaders.sql create mode 100644 schema/triggers/media_shares.sql create mode 100644 schema/triggers/media_uploaders.sql diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 4f91369d6df..7c78776c016 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -100,6 +100,15 @@ enum Command { #[arg(long, default_value_t = 10_000_000)] max_objects: u64, }, + /// Record media shares for events stored before share recording existed. + /// + /// Idempotent and resumable; run once after deploying share recording and + /// before enforcing channel-scoped media reads. + BackfillMediaShares { + /// Events read and recorded per transaction. + #[arg(long, default_value_t = 1000)] + batch_size: i64, + }, /// Run the partition catalog audit using a read-only database session. PartitionAudit { /// Future months to include in the coverage check. @@ -193,6 +202,14 @@ async fn run(cli: Cli) -> Result { Ok(0) } Command::StorageSnapshot { max_objects } => cmd_storage_snapshot(max_objects).await, + Command::BackfillMediaShares { batch_size } => { + let added = connect_db() + .await? + .backfill_media_shares(batch_size) + .await?; + println!("Recorded {added} media shares."); + Ok(0) + } Command::PartitionAudit { months_ahead } => cmd_partition_audit(months_ahead).await, Command::AddMember { pubkey, role } => cmd_add_member(pubkey, role).await, Command::RemoveMember { pubkey, role } => cmd_remove_member(pubkey, role).await, diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index e7669c874b0..0c89cd62c0f 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -24,6 +24,8 @@ pub mod git_perms; pub mod invite; /// Buzz kind number registry — custom event type constants. pub mod kind; +/// Which media blobs an event references, for channel-scoped media access. +pub mod media; /// Network utilities — SSRF-safe IP classification. pub mod network; /// NIP-10 thread-marker parsing — shared `root`/`reply` marker resolver. diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs new file mode 100644 index 00000000000..29bd3a124ec --- /dev/null +++ b/crates/buzz-core/src/media.rs @@ -0,0 +1,130 @@ +//! Which media blobs an event shares, for channel-scoped media access. + +use std::collections::BTreeSet; + +use crate::kind::{ + event_is_shared, is_author_only_event_kind, is_shared_gated_kind, P_GATED_KINDS, +}; + +/// Hashes of every blob `event` shares into its channel, or with the whole +/// community when it is channel-less: each local media path `/media/` +/// anywhere in its content or tags (attachments, posters, thumbnails, link +/// previews, avatars, custom emoji, pasted links). +/// +/// Empty for an event only its author or tagged recipients can read, since +/// referencing a blob there shows it to no one new. +pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { + let kind = u32::from(event.kind.as_u16()); + if is_author_only_event_kind(event) + || P_GATED_KINDS.contains(&kind) + || (is_shared_gated_kind(kind) && !event_is_shared(event)) + { + return Vec::new(); + } + let mut hashes: BTreeSet = media_path_hashes(&event.content).collect(); + for tag in event.tags.iter() { + hashes.extend( + tag.as_slice() + .iter() + .flat_map(|part| media_path_hashes(part)), + ); + } + hashes.into_iter().collect() +} + +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) +} + +/// The hash in each `/media/` path in `text`, where the hash is not +/// followed by another hex digit. +fn media_path_hashes(text: &str) -> impl Iterator + '_ { + text.match_indices("/media/").filter_map(|(at, path)| { + let rest = &text[at + path.len()..]; + let hash = rest.get(..64)?; + let bounded = !rest[64..].starts_with(|c: char| matches!(c, '0'..='9' | 'a'..='f')); + (is_sha256_hex(hash) && bounded).then(|| hash.to_owned()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kind::{KIND_EVENT_REMINDER, KIND_PERSONA}; + use nostr::{EventBuilder, Keys, Kind, Tag}; + + fn media_url(sha256: &str, ext: &str) -> String { + format!("https://relay.example/media/{sha256}.{ext}") + } + + fn signed(builder: EventBuilder) -> nostr::Event { + builder.sign_with_keys(&Keys::generate()).unwrap() + } + + #[test] + fn shares_every_local_media_path() { + let (video, poster, embed) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); + let event = signed( + EventBuilder::new( + Kind::TextNote, + format!("look ![image]({})", media_url(&embed, "png")), + ) + .tags([ + Tag::parse([ + "imeta", + &format!("url {}", media_url(&video, "mp4")), + &format!("x {video}"), + &format!("image /media/{poster}.jpg"), + ]) + .unwrap(), + Tag::parse([ + "r", + &format!("https://elsewhere.example/{}", "e".repeat(64)), + ]) + .unwrap(), + Tag::parse(["r", &format!("/media/{}0", "f".repeat(64))]).unwrap(), + ]), + ); + assert_eq!(shared_blob_hashes(&event), vec![video, poster, embed]); + } + + #[test] + fn shares_avatars_emoji_and_shared_personas_only() { + let (avatar, emoji) = ("c".repeat(64), "d".repeat(64)); + let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar, "jpg")); + let profile = signed(EventBuilder::new(Kind::Metadata, &content)); + assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); + + let emoji_set = signed( + EventBuilder::new(Kind::from(30030), "").tags([Tag::parse([ + "emoji", + "party", + &media_url(&emoji, "png"), + ]) + .unwrap()]), + ); + assert_eq!(shared_blob_hashes(&emoji_set), vec![emoji]); + + let persona = |shared: bool| { + let mut tags = vec![Tag::parse(["d", "helper"]).unwrap()]; + if shared { + tags.push(Tag::parse(["shared", "true"]).unwrap()); + } + signed(EventBuilder::new(Kind::from(KIND_PERSONA as u16), &content).tags(tags)) + }; + assert_eq!(shared_blob_hashes(&persona(true)), vec![avatar]); + assert!(shared_blob_hashes(&persona(false)).is_empty()); + } + + #[test] + fn author_only_events_share_nothing() { + let reminder = signed( + EventBuilder::new(Kind::from(KIND_EVENT_REMINDER as u16), "").tags([Tag::parse([ + "imeta", + &format!("url /media/{}.jpg", "a".repeat(64)), + ]) + .unwrap()]), + ); + assert!(shared_blob_hashes(&reminder).is_empty()); + } +} diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index af877332529..385ee4d2211 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -806,6 +806,13 @@ mod postgres_tests { .sql .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); + assert_eq!(migrations[66].version, 67); + for table in ["media_uploaders", "media_shares"] { + assert!(migrations[66] + .sql + .as_str() + .contains(&format!("CREATE TABLE {table}"))); + } assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2408,6 +2415,18 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('community_nip_fi_issuers')")); expected_fences.insert("community_nip_fi_issuers".to_owned()); + // 0067 fences the media access tables. + let media_access = MIGRATOR + .iter() + .find(|m| m.version == 67) + .expect("media access migration"); + for table in ["media_uploaders", "media_shares"] { + assert!(media_access + .sql + .as_str() + .contains(&format!("attach_community_write_fence('{table}')"))); + expected_fences.insert(table.to_owned()); + } assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 28e64ff61c8..111016ede1a 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -556,13 +556,8 @@ impl LockedMemberSnapshot { "member snapshot event id already exists".into(), )); } - crate::store::event_follow_up::after_admitted_insert( - &mut self.tx, - event.id.as_bytes().as_slice(), - kind, - Some(channel_id), - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut self.tx, event, Some(channel_id)) + .await?; crate::insert_mentions_in_transaction(&mut self.tx, event, Some(channel_id)).await?; Ok(( buzz_core::StoredEvent::with_received_at( diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index c5d62451278..2de124e5162 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -90,6 +90,8 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "media_shares", + "media_uploaders", "moderation_actions", "moderation_reports", "parameterized_event_watermarks", @@ -156,6 +158,8 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "audit_log", "community_bans", "community_nip_fi_issuers", + "media_shares", + "media_uploaders", "pubkey_allowlist", "relay_members", "users", @@ -4458,6 +4462,8 @@ DROP TRIGGER IF EXISTS community_write_fence_event_mentions ON event_mentions; DROP TRIGGER IF EXISTS community_write_fence_events ON events; DROP TRIGGER IF EXISTS community_write_fence_git_repo_names ON git_repo_names; DROP TRIGGER IF EXISTS community_write_fence_join_policy_acceptances ON join_policy_acceptances; +DROP TRIGGER IF EXISTS community_write_fence_media_shares ON media_shares; +DROP TRIGGER IF EXISTS community_write_fence_media_uploaders ON media_uploaders; DROP TRIGGER IF EXISTS community_write_fence_moderation_actions ON moderation_actions; DROP TRIGGER IF EXISTS community_write_fence_moderation_reports ON moderation_reports; DROP TRIGGER IF EXISTS community_write_fence_parameterized_event_watermarks ON parameterized_event_watermarks; diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index c279e4831d0..3aba787a523 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -515,13 +515,7 @@ async fn insert_event_on( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(( @@ -1905,13 +1899,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; if let Some(ref meta) = thread_meta { let broadcast_val: bool = meta.broadcast; diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index d2853b4d89d..dd4e5aa9d19 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -37,15 +37,29 @@ const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; /// Follow-up for an event row inserted in an admitted transaction. /// /// Call only when the INSERT affected a row, after releasing any savepoint -/// that may roll it back. Stages push work for after commit and records TTL work. +/// that may roll it back. Stages push work for after commit, records TTL work, +/// and shares the blobs the event references into its channel in the same +/// transaction, so a stored reference always carries its share. pub(crate) async fn after_admitted_insert( tx: &mut AdmittedTx, - event_id: &[u8], - kind: i32, + event: &nostr::Event, channel_id: Option, ) -> Result<()> { - tx.record_push_event(event_id, kind); + let kind = i32::from(event.kind.as_u16()); + tx.record_push_event(event.id.as_bytes(), kind); tx.record_channel_event(channel_id, kind); + let hashes = buzz_core::media::shared_blob_hashes(event); + if !hashes.is_empty() { + let community = tx.community(); + crate::store::media_reference::record_media_shares( + tx.conn(), + community, + &hashes, + event.pubkey.as_bytes(), + channel_id, + ) + .await?; + } Ok(()) } diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs new file mode 100644 index 00000000000..c4aa9fb95e3 --- /dev/null +++ b/crates/buzz-db/src/store/media_reference.rs @@ -0,0 +1,298 @@ +//! Where each media blob is shared, recorded with the events that reference it. + +use buzz_datastore_tracing::datastore_span; +use chrono::{DateTime, Utc}; +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use crate::{error::Result, Db}; +use buzz_core::CommunityId; + +/// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. +/// +/// Everyone may when the blob is shared with the whole community or into no +/// channel; otherwise only readers of a channel it is shared into, open or one +/// they actively belong to. Shares are never withdrawn. +macro_rules! readable_sql { + () => { + "( \ + EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ + OR NOT EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND s.channel_id IS NOT NULL) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $3 AND cm.removed_at IS NULL))))" + }; +} + +/// Share into `channel_id` (or with the whole community when `None`) each blob +/// in `sha256s` that `author` can read or has uploaded (uploading the exact +/// bytes proves possession). Any other blob is left unshared, so a reference +/// built from a leaked hash shows it to no one new. Called with the +/// transaction that stores the referencing event. +pub(crate) async fn record_media_shares( + conn: &mut PgConnection, + community_id: CommunityId, + sha256s: &[String], + author: &[u8], + channel_id: Option, +) -> Result<()> { + sqlx::query(concat!( + "INSERT INTO media_shares (community_id, sha256, channel_id) \ + SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + WHERE EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR ", + readable_sql!(), + " ON CONFLICT DO NOTHING" + )) + .bind(community_id.as_uuid()) + .bind(sha256s) + .bind(author) + .bind(channel_id) + .execute(conn) + .await?; + Ok(()) +} + +impl Db { + /// Record the shares of every event already stored in an active community, + /// deleted events included, `batch_size` events per transaction; returns the + /// number of shares added. + /// + /// Shares come from the same rule as live writes. History predates the + /// readability check, so its references are recorded as they stand. + /// Idempotent and resumable: rerunning only fills shares still missing. + #[datastore_span(name = "backfill_media_shares", system = "postgresql")] + pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { + let communities: Vec = sqlx::query_scalar( + "SELECT id FROM communities WHERE deleted_at IS NULL AND deletion_state = 'active' \ + ORDER BY id", + ) + .fetch_all(&self.pool) + .await?; + let mut added = 0; + for community in communities { + let community = CommunityId::from_uuid(community); + let mut cursor: Option<(DateTime, Vec)> = None; + loop { + let rows = sqlx::query( + "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, \ + channel_id \ + FROM events \ + WHERE community_id = $1 \ + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) \ + ORDER BY created_at, id LIMIT $4", + ) + .bind(community.as_uuid()) + .bind(cursor.as_ref().map(|(at, _)| *at)) + .bind( + cursor + .as_ref() + .map(|(_, id)| id.clone()) + .unwrap_or_default(), + ) + .bind(batch_size) + .fetch_all(&self.pool) + .await?; + let Some(last) = rows.last() else { break }; + cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); + let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; + + let mut tx = crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await?; + for row in rows { + let Some(stored) = crate::event::row_to_stored_event(row)? else { + continue; + }; + let hashes = buzz_core::media::shared_blob_hashes(&stored.event); + if hashes.is_empty() { + continue; + } + added += sqlx::query( + "INSERT INTO media_shares (community_id, sha256, channel_id) \ + SELECT $1, unnest($2::text[]), $3 ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(&hashes) + .bind(stored.channel_id) + .execute(tx.conn()) + .await? + .rows_affected(); + } + tx.commit().await?; + if !full_batch { + break; + } + } + } + Ok(added) + } +} + +#[cfg(test)] +mod postgres_tests { + use std::time::Duration; + + use super::*; + use crate::channel::{ChannelType, ChannelVisibility}; + use nostr::{EventBuilder, Keys, Kind, Tag}; + use sqlx::PgPool; + + async fn fixture() -> (Db, CommunityId) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let host = format!("media-shares-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("seed community") + .id; + (db, community) + } + + async fn channel( + db: &Db, + community: CommunityId, + owner: &Keys, + visibility: ChannelVisibility, + ) -> Uuid { + db.create_channel( + community, + &format!("media-{}", Uuid::new_v4().simple()), + ChannelType::Stream, + visibility, + None, + &owner.public_key().to_bytes(), + None, + ) + .await + .expect("create channel") + .id + } + + fn attachment(keys: &Keys, sha256: &str, channel: Option) -> nostr::Event { + let mut tags = vec![Tag::parse(["imeta", &format!("url /media/{sha256}.jpg")]).unwrap()]; + if let Some(channel) = channel { + tags.push(Tag::parse(["h", &channel.to_string()]).unwrap()); + } + EventBuilder::new(Kind::from(9), "attachment") + .tags(tags) + .sign_with_keys(keys) + .unwrap() + } + + async fn shares(db: &Db, community: CommunityId, sha256: &str) -> Vec> { + let mut shares: Vec> = sqlx::query_scalar( + "SELECT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + ) + .bind(community.as_uuid()) + .bind(sha256) + .fetch_all(&db.pool) + .await + .expect("read shares"); + shares.sort(); + shares + } + + fn unique_sha() -> String { + format!("{:064x}", Uuid::new_v4().as_u128()) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stored_references_share_only_blobs_their_author_can_read() { + let (db, community) = fixture().await; + let (alice, bob) = (Keys::generate(), Keys::generate()); + let sha = unique_sha(); + let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; + + db.insert_event( + community, + &attachment(&alice, &sha, Some(private)), + Some(private), + ) + .await + .expect("alice posts privately"); + assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references the hash"); + assert_eq!( + shares(&db, community, &sha).await, + vec![Some(private)], + "a hash alone must not share a blob its author cannot read" + ); + + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community, + "media_upload", + "media-shares-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + db.record_media_uploader(&lease, &sha, &bob.public_key().to_bytes()) + .await + .expect("bob uploads the same bytes"); + db.insert_event(community, &attachment(&bob, &sha, Some(bobs)), Some(bobs)) + .await + .expect("bob posts his copy"); + let mut expected = vec![Some(private), Some(bobs)]; + expected.sort(); + assert_eq!( + shares(&db, community, &sha).await, + expected, + "an uploader may share their own bytes" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn backfill_records_shares_of_stored_events() { + let (db, community) = fixture().await; + let alice = Keys::generate(); + let sha = unique_sha(); + let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + db.insert_event( + community, + &attachment(&alice, &sha, Some(private)), + Some(private), + ) + .await + .expect("alice posts privately"); + let mut tx = crate::begin_community_write_transaction( + &db.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await + .expect("admit"); + sqlx::query("DELETE FROM media_shares WHERE community_id = $1") + .bind(community.as_uuid()) + .execute(tx.conn()) + .await + .expect("forget shares"); + tx.commit().await.expect("commit"); + + db.backfill_media_shares(100).await.expect("backfill"); + assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + } +} diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs new file mode 100644 index 00000000000..0039575af04 --- /dev/null +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -0,0 +1,110 @@ +//! Who has uploaded each blob, as proof of possession for sharing it. + +use buzz_datastore_tracing::datastore_span; + +use crate::deletion::ServingWriteLease; +use crate::{error::Result, observability, Db}; + +impl Db { + /// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are + /// no-ops. The write presents the upload's serving lease, so it may finish + /// while the community quiesces but never after the lease is gone. + #[datastore_span(name = "record_media_uploader", system = "postgresql")] + pub async fn record_media_uploader( + &self, + lease: &ServingWriteLease, + sha256: &str, + pubkey: &[u8], + ) -> Result<()> { + let connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::EventWrite) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + self.deletion_store() + .guard_transaction_with_serving_lease(&mut tx, lease) + .await?; + sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(lease.community_id.as_uuid()) + .bind(sha256) + .bind(pubkey) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } +} + +#[cfg(test)] +mod postgres_tests { + use std::time::Duration; + + use super::*; + use buzz_core::CommunityId; + use sqlx::PgPool; + use uuid::Uuid; + + const SHA: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + + async fn db_with_lease() -> (Db, CommunityId, ServingWriteLease) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let host = format!("uploader-{}.example", Uuid::new_v4().simple()); + let community_id = db + .ensure_configured_community(&host) + .await + .expect("seed community") + .id; + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community_id, + "media_upload", + "media-uploader-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + (db, community_id, lease) + } + + async fn uploader_rows(db: &Db, community_id: CommunityId) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1") + .bind(community_id.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("count uploaders") + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn repeat_upload_records_one_row() { + let (db, community_id, lease) = db_with_lease().await; + for _ in 0..2 { + db.record_media_uploader(&lease, SHA, &[7; 32]) + .await + .expect("record uploader"); + } + assert_eq!(uploader_rows(&db, community_id).await, 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn released_lease_refuses_the_write() { + let (db, community_id, lease) = db_with_lease().await; + db.deletion_store() + .release_serving_write_lease(&lease) + .await + .expect("release lease"); + assert!(db + .record_media_uploader(&lease, SHA, &[7; 32]) + .await + .is_err()); + assert_eq!(uploader_rows(&db, community_id).await, 0); + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 6850b625cf1..670748df11c 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,6 +32,10 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Where each media blob is shared, recorded with the events that reference it. +pub mod media_reference; +/// Who has uploaded each blob, as proof of possession for sharing it. +pub mod media_uploader; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; /// Deployment-global operator-listener mention registrations and delivery queues. diff --git a/crates/buzz-db/src/store/push.rs b/crates/buzz-db/src/store/push.rs index 24ae312521e..d4ae0b86b3c 100644 --- a/crates/buzz-db/src/store/push.rs +++ b/crates/buzz-db/src/store/push.rs @@ -360,13 +360,7 @@ pub async fn accept_lease_event( return Err(error.into()); } // The row above is always kind 30350 with no channel. - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - 30350, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, None).await?; let (is_active, endpoint_hash, endpoint_grant, max_class, subscriptions) = active .map_or((false, None, None, None, None), |active| { diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 6bc4de1a959..43154fe4baf 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -1170,13 +1170,7 @@ pub(crate) async fn write_nip43_membership_snapshot( .bind(d_tag.as_deref()) .execute(tx.conn()) .await?; - crate::store::event_follow_up::after_admitted_insert( - tx, - event.id.as_bytes().as_slice(), - kind_i32, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, &event, None).await?; Ok(StoredEvent::with_received_at( event, diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 1ccee163320..59482e742c4 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -383,8 +383,7 @@ async fn replace_parameterized_event_in_transaction_impl( .execute(tx.conn()) .await?; if status == ParameterizedReplaceStatus::Inserted { - crate::store::event_follow_up::after_admitted_insert(tx, incoming_id, kind_i32, channel_id) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(ParameterizedReplaceResult::new( @@ -547,13 +546,8 @@ impl Db { )); } - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) + .await?; // The replaceable event and its denormalized mention index are one // authoritative discovery write. An indexing error must roll back the diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 9c5f3dd93b3..eead116018f 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -579,6 +579,16 @@ async fn upload_blob_result( auth.tenant.host(), ); + state + .db + .record_media_uploader( + serving_write.lease(), + &descriptor.sha256, + auth.auth_event.pubkey.as_bytes(), + ) + .await + .map_err(|error| serving_lease_lost(error.into()))?; + // Normalize MIME to a known set to bound label cardinality. let mime_label = match descriptor.mime_type.as_str() { "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "video/mp4" => { @@ -2144,11 +2154,13 @@ mod tests { const TEST_ISSUER: &str = "https://issuer.example"; const TEST_AUDIENCE: &str = "https://relay.example"; const TEST_KID: &str = "test-key-1"; - const TEST_EC_PKCS8_PEM: &str = "-----BEGIN PRIVATE KEY-----\n\ - MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n\ - WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n\ - zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n\ - -----END PRIVATE KEY-----\n"; + const TEST_EC_PKCS8_PEM: &str = concat!( + "-----BEGIN PRIVATE KEY-----\n", // sadscan:disable kingfisher.privkey.2 -- static test-only key + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n", + "WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n", + "zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n", + "-----END PRIVATE KEY-----\n", + ); /// Build an AppState with NIP-FI Enforce and a real injected P-256 verifier. async fn media_enforce_test_state() -> Option> { diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql new file mode 100644 index 00000000000..c2f13825d63 --- /dev/null +++ b/migrations/0067_media_access.sql @@ -0,0 +1,29 @@ +-- Channel-scoped media access: who may read each blob. +-- +-- media_uploaders: who has uploaded each blob. Uploading the exact bytes +-- proves possession, so an uploader may share a blob into their own channels +-- even when the same bytes are also shared somewhere they cannot read. +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); + +-- media_shares: one row per channel a blob is shared into, written with the +-- event that references it, or a NULL channel for a channel-less reference +-- (avatars, custom emoji, notes), which shares it with the whole community. +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; + +SELECT attach_community_write_fence('media_uploaders'); +SELECT attach_community_write_fence('media_shares'); diff --git a/schema/schema.sql b/schema/schema.sql index d252cbdb7ea..9bdb5321b26 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -107,6 +107,8 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i tables/public/moderation_reports.sql \i tables/public/community_bans.sql \i tables/public/community_nip_fi_issuers.sql +\i tables/public/media_shares.sql +\i tables/public/media_uploaders.sql \i tables/public/_operator_global_tables.sql \i tables/public/git_repo_names.sql \i tables/public/parameterized_event_watermarks.sql @@ -159,6 +161,8 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i triggers/event_mentions.sql \i triggers/git_repo_names.sql \i triggers/join_policy_acceptances.sql +\i triggers/media_shares.sql +\i triggers/media_uploaders.sql \i triggers/moderation_actions.sql \i triggers/moderation_reports.sql \i triggers/parameterized_event_watermarks.sql diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql new file mode 100644 index 00000000000..18ab231b4bc --- /dev/null +++ b/schema/tables/public/media_shares.sql @@ -0,0 +1,15 @@ +-- ── Media shares ────────────────────────────────────────────────────────────── +-- Where each blob is shared: one row per channel an event referencing it was +-- stored in, or a NULL channel for a channel-less reference, which shares it +-- with the whole community. + +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql new file mode 100644 index 00000000000..5522ae84a7a --- /dev/null +++ b/schema/tables/public/media_uploaders.sql @@ -0,0 +1,11 @@ +-- ── Media uploaders ─────────────────────────────────────────────────────────── +-- Who has uploaded each blob in a community. Uploading the exact bytes proves +-- possession, so an uploader may share a blob into their own channels. + +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); diff --git a/schema/triggers/media_shares.sql b/schema/triggers/media_shares.sql new file mode 100644 index 00000000000..e105e557bf9 --- /dev/null +++ b/schema/triggers/media_shares.sql @@ -0,0 +1 @@ +SELECT attach_community_write_fence('media_shares'); diff --git a/schema/triggers/media_uploaders.sql b/schema/triggers/media_uploaders.sql new file mode 100644 index 00000000000..23367202e24 --- /dev/null +++ b/schema/triggers/media_uploaders.sql @@ -0,0 +1 @@ +SELECT attach_community_write_fence('media_uploaders'); From aa6ba13a4cab1332ca624704b7f98a0a583130f8 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 22:26:50 +1100 Subject: [PATCH 09/48] test(db): upload media before posting it in share-recording tests Clients upload a file before attaching it, so record the uploader first and keep the test valid once reads deny files not yet shared into any channel. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 31 ++++++++++++--------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index c4aa9fb95e3..624e2bd4d0e 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -212,6 +212,22 @@ mod postgres_tests { format!("{:064x}", Uuid::new_v4().as_u128()) } + async fn upload(db: &Db, community: CommunityId, uploader: &Keys, sha256: &str) { + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community, + "media_upload", + "media-shares-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + db.record_media_uploader(&lease, sha256, &uploader.public_key().to_bytes()) + .await + .expect("record uploader"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn stored_references_share_only_blobs_their_author_can_read() { @@ -221,6 +237,7 @@ mod postgres_tests { let private = channel(&db, community, &alice, ChannelVisibility::Private).await; let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; + upload(&db, community, &alice, &sha).await; db.insert_event( community, &attachment(&alice, &sha, Some(private)), @@ -239,19 +256,7 @@ mod postgres_tests { "a hash alone must not share a blob its author cannot read" ); - let lease = db - .deletion_store() - .acquire_serving_write_lease( - community, - "media_upload", - "media-shares-test", - Duration::from_secs(60), - ) - .await - .expect("acquire lease"); - db.record_media_uploader(&lease, &sha, &bob.public_key().to_bytes()) - .await - .expect("bob uploads the same bytes"); + upload(&db, community, &bob, &sha).await; db.insert_event(community, &attachment(&bob, &sha, Some(bobs)), Some(bobs)) .await .expect("bob posts his copy"); From b34e8bbc59a0fe5b8ab869eba655a3769cc17d49 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 08:32:30 +1100 Subject: [PATCH 10/48] fix(db): backfill only history unchecked; lock media shares in key order Trust references only from events received before migration 0067 was installed; later events get the live, checked recording, so a backfill run after rollout never shares a blob that live recording refused. Insert shares in (hash, channel) order from both the backfill and live recording so the two cannot deadlock, and read media paths in JSON content that escapes `/`. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-core/src/media.rs | 9 +- crates/buzz-db/src/store/media_reference.rs | 117 +++++++++++++++----- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index 29bd3a124ec..452238edba8 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -21,7 +21,9 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { { return Vec::new(); } - let mut hashes: BTreeSet = media_path_hashes(&event.content).collect(); + // JSON content may escape `/` as `\/`; scan the path as clients will see it. + let content = event.content.replace("\\/", "/"); + let mut hashes: BTreeSet = media_path_hashes(&content).collect(); for tag in event.tags.iter() { hashes.extend( tag.as_slice() @@ -94,6 +96,11 @@ mod tests { let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar, "jpg")); let profile = signed(EventBuilder::new(Kind::Metadata, &content)); assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); + let escaped = signed(EventBuilder::new( + Kind::Metadata, + content.replace('/', "\\/"), + )); + assert_eq!(shared_blob_hashes(&escaped), vec![avatar.clone()]); let emoji_set = signed( EventBuilder::new(Kind::from(30030), "").tags([Tag::parse([ diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 624e2bd4d0e..29de6e02989 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -35,42 +35,52 @@ macro_rules! readable_sql { /// in `sha256s` that `author` can read or has uploaded (uploading the exact /// bytes proves possession). Any other blob is left unshared, so a reference /// built from a leaked hash shows it to no one new. Called with the -/// transaction that stores the referencing event. +/// transaction that stores the referencing event; returns the shares added. pub(crate) async fn record_media_shares( conn: &mut PgConnection, community_id: CommunityId, sha256s: &[String], author: &[u8], channel_id: Option, -) -> Result<()> { - sqlx::query(concat!( +) -> Result { + let added = sqlx::query(concat!( "INSERT INTO media_shares (community_id, sha256, channel_id) \ SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ WHERE EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ OR ", readable_sql!(), - " ON CONFLICT DO NOTHING" + " ORDER BY b.sha256 ON CONFLICT DO NOTHING" )) .bind(community_id.as_uuid()) .bind(sha256s) .bind(author) .bind(channel_id) .execute(conn) - .await?; - Ok(()) + .await? + .rows_affected(); + Ok(added) } impl Db { /// Record the shares of every event already stored in an active community, - /// deleted events included, `batch_size` events per transaction; returns the - /// number of shares added. + /// deleted events included, reading `batch_size` events at a time; returns + /// the number of shares added. /// - /// Shares come from the same rule as live writes. History predates the - /// readability check, so its references are recorded as they stand. - /// Idempotent and resumable: rerunning only fills shares still missing. + /// Shares come from the same rule as live writes. Events received before + /// share recording was installed predate the readability check, so their + /// references are recorded as they stand; later events get the live, + /// checked recording, so the backfill never shares what live recording + /// refused. Idempotent and resumable: rerunning only fills missing shares. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { + let installed: Option> = + sqlx::query_scalar("SELECT installed_on FROM _sqlx_migrations WHERE version = 67") + .fetch_optional(&self.pool) + .await?; + let installed = installed.ok_or_else(|| { + crate::DbError::InvalidData("media access migration 0067 is not installed".into()) + })?; let communities: Vec = sqlx::query_scalar( "SELECT id FROM communities WHERE deleted_at IS NULL AND deletion_state = 'active' \ ORDER BY id", @@ -105,13 +115,10 @@ impl Db { cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; - let mut tx = crate::begin_community_write_transaction( - &self.pool, - community, - crate::observability::WriterOperation::Maintenance, - ) - .await?; + let mut history: Vec<(String, Option)> = Vec::new(); + let mut recorded_live = Vec::new(); for row in rows { + let received_at: DateTime = row.try_get("received_at")?; let Some(stored) = crate::event::row_to_stored_event(row)? else { continue; }; @@ -119,18 +126,46 @@ impl Db { if hashes.is_empty() { continue; } + if received_at < installed { + history.extend(hashes.into_iter().map(|hash| (hash, stored.channel_id))); + } else { + recorded_live.push((stored.event.pubkey, hashes, stored.channel_id)); + } + } + // Insert in key order, as live recording does, so a concurrent + // publish can never take the same share locks in reverse. + history.sort(); + history.dedup(); + if !history.is_empty() { + let (hashes, channels): (Vec, Vec>) = + history.into_iter().unzip(); + let mut tx = self.begin_media_share_transaction(community).await?; added += sqlx::query( "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, unnest($2::text[]), $3 ON CONFLICT DO NOTHING", + SELECT $1, h, c FROM unnest($2::text[], $3::uuid[]) AS t(h, c) \ + ORDER BY h, c NULLS FIRST \ + ON CONFLICT DO NOTHING", ) .bind(community.as_uuid()) - .bind(&hashes) - .bind(stored.channel_id) + .bind(hashes) + .bind(channels) .execute(tx.conn()) .await? .rows_affected(); + tx.commit().await?; + } + for (author, hashes, channel) in recorded_live { + let mut tx = self.begin_media_share_transaction(community).await?; + added += record_media_shares( + tx.conn(), + community, + &hashes, + author.as_bytes(), + channel, + ) + .await?; + tx.commit().await?; } - tx.commit().await?; if !full_batch { break; } @@ -138,6 +173,18 @@ impl Db { } Ok(added) } + + async fn begin_media_share_transaction( + &self, + community: CommunityId, + ) -> Result { + crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::Maintenance, + ) + .await + } } #[cfg(test)] @@ -271,11 +318,12 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn backfill_records_shares_of_stored_events() { + async fn backfill_trusts_history_and_rechecks_recorded_events() { let (db, community) = fixture().await; - let alice = Keys::generate(); - let sha = unique_sha(); + let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); + let (sha, historical) = (unique_sha(), unique_sha()); let private = channel(&db, community, &alice, ChannelVisibility::Private).await; + upload(&db, community, &alice, &sha).await; db.insert_event( community, &attachment(&alice, &sha, Some(private)), @@ -283,6 +331,14 @@ mod postgres_tests { ) .await .expect("alice posts privately"); + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references the hash"); + let old = attachment(&carol, &historical, None); + db.insert_event(community, &old, None) + .await + .expect("carol posts before share recording"); + let mut tx = crate::begin_community_write_transaction( &db.pool, community, @@ -290,6 +346,12 @@ mod postgres_tests { ) .await .expect("admit"); + sqlx::query("UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = $2") + .bind(community.as_uuid()) + .bind(old.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .expect("age carol's event"); sqlx::query("DELETE FROM media_shares WHERE community_id = $1") .bind(community.as_uuid()) .execute(tx.conn()) @@ -298,6 +360,11 @@ mod postgres_tests { tx.commit().await.expect("commit"); db.backfill_media_shares(100).await.expect("backfill"); - assert_eq!(shares(&db, community, &sha).await, vec![Some(private)]); + assert_eq!( + shares(&db, community, &sha).await, + vec![Some(private)], + "a reference live recording refused must stay unshared" + ); + assert_eq!(shares(&db, community, &historical).await, vec![None]); } } From d4a6a1af76ec382d0fd2d3a047e07ef5efe360eb Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 10:41:35 +1100 Subject: [PATCH 11/48] fix(media): record shares safely before enforcing reads Require upload proof or a recorded share when recording new references. Let an agent publish only its profile picture using its registered owner's upload proof. Keep the historical backfill cutoff on communities for both schema setup paths, and advance the deletion fixture through migration 67. Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 5 +- crates/buzz-core/src/media.rs | 12 ++ crates/buzz-db/src/store/deletion.rs | 2 +- crates/buzz-db/src/store/event_follow_up.rs | 2 +- crates/buzz-db/src/store/media_reference.rs | 160 +++++++++++++++----- migrations/0067_media_access.sql | 4 + schema/tables/public/communities.sql | 2 + 7 files changed, 148 insertions(+), 39 deletions(-) diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 7c78776c016..6cfc4bdc241 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -102,8 +102,9 @@ enum Command { }, /// Record media shares for events stored before share recording existed. /// - /// Idempotent and resumable; run once after deploying share recording and - /// before enforcing channel-scoped media reads. + /// Idempotent and resumable; run after every relay pod records shares and + /// before enforcing channel-scoped media reads. Historical references are + /// trusted only before each community's schema-recorded cutoff. BackfillMediaShares { /// Events read and recorded per transaction. #[arg(long, default_value_t = 1000)] diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index 452238edba8..bac5f48dceb 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -34,6 +34,18 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { hashes.into_iter().collect() } +/// The media blob used as a kind:0 profile's picture. A registered agent may +/// publish this one field using its owner's upload proof; arbitrary content +/// and tags must not inherit that authority. +pub fn profile_picture_blob_hash(event: &nostr::Event) -> Option { + if event.kind != nostr::Kind::Metadata { + return None; + } + let profile: serde_json::Value = serde_json::from_str(&event.content).ok()?; + let hash = media_path_hashes(profile.get("picture")?.as_str()?).next(); + hash +} + fn is_sha256_hex(value: &str) -> bool { value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) } diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 2de124e5162..73b1e364717 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -4434,7 +4434,7 @@ mod postgres_tests { }) .await .expect("connect isolated migration DB"); - crate::migration::run_migrations_through(&db.pool, 66) + crate::migration::run_migrations_through(&db.pool, 67) .await .expect("migrate with fence objects present"); let store = db.deletion_store(); diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index dd4e5aa9d19..6fd0de9fc27 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -55,7 +55,7 @@ pub(crate) async fn after_admitted_insert( tx.conn(), community, &hashes, - event.pubkey.as_bytes(), + event, channel_id, ) .await?; diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 29de6e02989..0be5c1d6737 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -10,17 +10,14 @@ use buzz_core::CommunityId; /// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. /// -/// Everyone may when the blob is shared with the whole community or into no -/// channel; otherwise only readers of a channel it is shared into, open or one -/// they actively belong to. Shares are never withdrawn. +/// A share admits everyone when it is community-wide, or readers of its +/// channel (open, or one they actively belong to). An unknown or unshared hash +/// grants nothing. Shares are never withdrawn. macro_rules! readable_sql { () => { "( \ EXISTS (SELECT 1 FROM media_shares s \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ - OR NOT EXISTS (SELECT 1 FROM media_shares s \ - WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ - AND s.channel_id IS NOT NULL) \ OR EXISTS (SELECT 1 FROM media_shares s \ JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ @@ -32,15 +29,16 @@ macro_rules! readable_sql { } /// Share into `channel_id` (or with the whole community when `None`) each blob -/// in `sha256s` that `author` can read or has uploaded (uploading the exact -/// bytes proves possession). Any other blob is left unshared, so a reference -/// built from a leaked hash shows it to no one new. Called with the +/// in `sha256s` that the event's author can read or has uploaded (uploading +/// the exact bytes proves possession). An agent's profile picture may also +/// use its registered owner's upload proof. Any other blob is left unshared, +/// so a reference built from a leaked hash shows it to no one new. Called with the /// transaction that stores the referencing event; returns the shares added. pub(crate) async fn record_media_shares( conn: &mut PgConnection, community_id: CommunityId, sha256s: &[String], - author: &[u8], + event: &nostr::Event, channel_id: Option, ) -> Result { let added = sqlx::query(concat!( @@ -48,14 +46,19 @@ pub(crate) async fn record_media_shares( SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ WHERE EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR (b.sha256 = $5 AND EXISTS ( \ + SELECT 1 FROM users agent JOIN media_uploaders u \ + ON u.community_id = agent.community_id AND u.pubkey = agent.agent_owner_pubkey \ + WHERE agent.community_id = $1 AND agent.pubkey = $3 AND u.sha256 = b.sha256)) \ OR ", readable_sql!(), " ORDER BY b.sha256 ON CONFLICT DO NOTHING" )) .bind(community_id.as_uuid()) .bind(sha256s) - .bind(author) + .bind(event.pubkey.as_bytes().as_slice()) .bind(channel_id) + .bind(buzz_core::media::profile_picture_blob_hash(event)) .execute(conn) .await? .rows_affected(); @@ -74,21 +77,19 @@ impl Db { /// refused. Idempotent and resumable: rerunning only fills missing shares. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { - let installed: Option> = - sqlx::query_scalar("SELECT installed_on FROM _sqlx_migrations WHERE version = 67") - .fetch_optional(&self.pool) - .await?; - let installed = installed.ok_or_else(|| { - crate::DbError::InvalidData("media access migration 0067 is not installed".into()) - })?; - let communities: Vec = sqlx::query_scalar( - "SELECT id FROM communities WHERE deleted_at IS NULL AND deletion_state = 'active' \ - ORDER BY id", + if batch_size <= 0 { + return Err(crate::DbError::InvalidData( + "batch size must be positive".into(), + )); + } + let communities: Vec<(Uuid, DateTime)> = sqlx::query_as( + "SELECT id, media_shares_recording_started_at FROM communities \ + WHERE deleted_at IS NULL AND deletion_state = 'active' ORDER BY id", ) .fetch_all(&self.pool) .await?; let mut added = 0; - for community in communities { + for (community, installed) in communities { let community = CommunityId::from_uuid(community); let mut cursor: Option<(DateTime, Vec)> = None; loop { @@ -129,7 +130,7 @@ impl Db { if received_at < installed { history.extend(hashes.into_iter().map(|hash| (hash, stored.channel_id))); } else { - recorded_live.push((stored.event.pubkey, hashes, stored.channel_id)); + recorded_live.push((stored.event, hashes, stored.channel_id)); } } // Insert in key order, as live recording does, so a concurrent @@ -154,16 +155,10 @@ impl Db { .rows_affected(); tx.commit().await?; } - for (author, hashes, channel) in recorded_live { + for (event, hashes, channel) in recorded_live { let mut tx = self.begin_media_share_transaction(community).await?; - added += record_media_shares( - tx.conn(), - community, - &hashes, - author.as_bytes(), - channel, - ) - .await?; + added += + record_media_shares(tx.conn(), community, &hashes, &event, channel).await?; tx.commit().await?; } if !full_batch { @@ -236,7 +231,7 @@ mod postgres_tests { if let Some(channel) = channel { tags.push(Tag::parse(["h", &channel.to_string()]).unwrap()); } - EventBuilder::new(Kind::from(9), "attachment") + EventBuilder::new(Kind::from(9), format!("attachment {}", Uuid::new_v4())) .tags(tags) .sign_with_keys(keys) .unwrap() @@ -284,7 +279,18 @@ mod postgres_tests { let private = channel(&db, community, &alice, ChannelVisibility::Private).await; let bobs = channel(&db, community, &bob, ChannelVisibility::Open).await; - upload(&db, community, &alice, &sha).await; + for uploader_present in [false, true] { + if uploader_present { + upload(&db, community, &alice, &sha).await; + } + db.insert_event(community, &attachment(&bob, &sha, None), None) + .await + .expect("bob references an unshared hash"); + assert!( + shares(&db, community, &sha).await.is_empty(), + "a hash alone grants nothing, even before its uploader posts it" + ); + } db.insert_event( community, &attachment(&alice, &sha, Some(private)), @@ -319,6 +325,10 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn backfill_trusts_history_and_rechecks_recorded_events() { + check_backfill().await; + } + + async fn check_backfill() { let (db, community) = fixture().await; let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); let (sha, historical) = (unique_sha(), unique_sha()); @@ -359,7 +369,8 @@ mod postgres_tests { .expect("forget shares"); tx.commit().await.expect("commit"); - db.backfill_media_shares(100).await.expect("backfill"); + db.backfill_media_shares(1).await.expect("backfill"); + assert_eq!(db.backfill_media_shares(1).await.expect("rerun"), 0); assert_eq!( shares(&db, community, &sha).await, vec![Some(private)], @@ -367,4 +378,83 @@ mod postgres_tests { ); assert_eq!(shares(&db, community, &historical).await, vec![None]); } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn migration_schema_backfill_trusts_history_and_rechecks_recorded_events() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to migration DB"); + crate::migration::run_migrations_through(&pool, 67) + .await + .expect("migrate"); + pool.close().await; + check_backfill().await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn agent_profile_picture_uses_only_its_registered_owners_upload_proof() { + let (db, community) = fixture().await; + let (owner, agent) = (Keys::generate(), Keys::generate()); + let (avatar, secret, unknown) = (unique_sha(), unique_sha(), unique_sha()); + for sha in [&avatar, &secret] { + upload(&db, community, &owner, sha).await; + } + let profile = |picture: &str| { + EventBuilder::new( + Kind::Metadata, + serde_json::json!({ + "name": Uuid::new_v4().to_string(), + "picture": format!("/media/{picture}.jpg"), + "about": format!("/media/{secret}.jpg"), + }) + .to_string(), + ) + .tags([Tag::parse(["p", &owner.public_key().to_hex()]).unwrap()]) + .sign_with_keys(&agent) + .unwrap() + }; + db.insert_event(community, &profile(&avatar), None) + .await + .unwrap(); + assert!( + shares(&db, community, &avatar).await.is_empty(), + "an unverified owner tag grants nothing" + ); + for keys in [&owner, &agent] { + db.ensure_user(community, &keys.public_key().to_bytes()) + .await + .unwrap(); + } + db.set_agent_owner( + community, + &agent.public_key().to_bytes(), + &owner.public_key().to_bytes(), + ) + .await + .unwrap(); + db.insert_event(community, &profile(&avatar), None) + .await + .unwrap(); + assert_eq!(shares(&db, community, &avatar).await, vec![None]); + assert!( + shares(&db, community, &secret).await.is_empty(), + "owner proof applies only to picture" + ); + db.insert_event(community, &profile(&unknown), None) + .await + .unwrap(); + assert!( + shares(&db, community, &unknown).await.is_empty(), + "the owner must have uploaded these bytes" + ); + db.insert_event(community, &attachment(&agent, &secret, None), None) + .await + .unwrap(); + assert!( + shares(&db, community, &secret).await.is_empty(), + "ordinary events cannot borrow the owner's uploads" + ); + } } diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql index c2f13825d63..b70ac3530b2 100644 --- a/migrations/0067_media_access.sql +++ b/migrations/0067_media_access.sql @@ -1,3 +1,7 @@ +-- Durable historical backfill cutoff, independent of the schema install tool. +ALTER TABLE communities + ADD COLUMN media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(); + -- Channel-scoped media access: who may read each blob. -- -- media_uploaders: who has uploaded each blob. Uploading the exact bytes diff --git a/schema/tables/public/communities.sql b/schema/tables/public/communities.sql index 06f74c3f585..ea27392437b 100644 --- a/schema/tables/public/communities.sql +++ b/schema/tables/public/communities.sql @@ -20,6 +20,8 @@ CREATE TABLE communities ( -- Added by migration 0003; kept here so desired-state applies match. icon TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + -- The share-recording cutoff exists in both desired-state and migrated DBs. + media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(), archived_at TIMESTAMPTZ, deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), From ac39fcf180bf684c7a58ea5e14c27c10f615d821 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 13:46:41 +1100 Subject: [PATCH 12/48] fix(media): revoke grants from deleted messages Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 170 +++++++++++++++++--- migrations/0067_media_access.sql | 15 +- schema/tables/public/media_shares.sql | 15 +- 3 files changed, 161 insertions(+), 39 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 0be5c1d6737..dc776f72e6a 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -12,13 +12,17 @@ use buzz_core::CommunityId; /// /// A share admits everyone when it is community-wide, or readers of its /// channel (open, or one they actively belong to). An unknown or unshared hash -/// grants nothing. Shares are never withdrawn. +/// grants nothing. Deleted or purged source events grant nothing. macro_rules! readable_sql { () => { "( \ EXISTS (SELECT 1 FROM media_shares s \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ AND (c.visibility = 'open' \ @@ -41,10 +45,13 @@ pub(crate) async fn record_media_shares( event: &nostr::Event, channel_id: Option, ) -> Result { + let created_at_secs = event.created_at.as_secs() as i64; + let created_at = DateTime::from_timestamp(created_at_secs, 0) + .ok_or(crate::DbError::InvalidTimestamp(created_at_secs))?; let added = sqlx::query(concat!( - "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ - WHERE EXISTS (SELECT 1 FROM media_uploaders u \ + "INSERT INTO media_shares (community_id, sha256, channel_id, event_id, event_created_at) \ + SELECT $1, b.sha256, $4, $6, $7 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + WHERE (EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ OR (b.sha256 = $5 AND EXISTS ( \ SELECT 1 FROM users agent JOIN media_uploaders u \ @@ -52,13 +59,18 @@ pub(crate) async fn record_media_shares( WHERE agent.community_id = $1 AND agent.pubkey = $3 AND u.sha256 = b.sha256)) \ OR ", readable_sql!(), - " ORDER BY b.sha256 ON CONFLICT DO NOTHING" + ") AND EXISTS (SELECT 1 FROM events e \ + WHERE e.community_id = $1 AND e.id = $6 AND e.created_at = $7 \ + AND e.deleted_at IS NULL) \ + ORDER BY b.sha256 ON CONFLICT DO NOTHING" )) .bind(community_id.as_uuid()) .bind(sha256s) .bind(event.pubkey.as_bytes().as_slice()) .bind(channel_id) .bind(buzz_core::media::profile_picture_blob_hash(event)) + .bind(event.id.as_bytes().as_slice()) + .bind(created_at) .execute(conn) .await? .rows_affected(); @@ -66,8 +78,8 @@ pub(crate) async fn record_media_shares( } impl Db { - /// Record the shares of every event already stored in an active community, - /// deleted events included, reading `batch_size` events at a time; returns + /// Record the shares of undeleted events stored in an active community, + /// reading `batch_size` events at a time; returns /// the number of shares added. /// /// Shares come from the same rule as live writes. Events received before @@ -97,7 +109,7 @@ impl Db { "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, \ channel_id \ FROM events \ - WHERE community_id = $1 \ + WHERE community_id = $1 AND deleted_at IS NULL \ AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) \ ORDER BY created_at, id LIMIT $4", ) @@ -116,10 +128,11 @@ impl Db { cursor = Some((last.try_get("created_at")?, last.try_get("id")?)); let full_batch = i64::try_from(rows.len()).unwrap_or(i64::MAX) == batch_size; - let mut history: Vec<(String, Option)> = Vec::new(); + let mut history = Vec::new(); let mut recorded_live = Vec::new(); for row in rows { let received_at: DateTime = row.try_get("received_at")?; + let event_created_at: DateTime = row.try_get("created_at")?; let Some(stored) = crate::event::row_to_stored_event(row)? else { continue; }; @@ -128,7 +141,10 @@ impl Db { continue; } if received_at < installed { - history.extend(hashes.into_iter().map(|hash| (hash, stored.channel_id))); + let event_id = stored.event.id.to_bytes().to_vec(); + history.extend(hashes.into_iter().map(|hash| { + (hash, stored.channel_id, event_id.clone(), event_created_at) + })); } else { recorded_live.push((stored.event, hashes, stored.channel_id)); } @@ -138,18 +154,27 @@ impl Db { history.sort(); history.dedup(); if !history.is_empty() { - let (hashes, channels): (Vec, Vec>) = - history.into_iter().unzip(); + let hashes: Vec<_> = history.iter().map(|(h, _, _, _)| h).collect(); + let channels: Vec<_> = history.iter().map(|(_, c, _, _)| *c).collect(); + let event_ids: Vec<_> = history.iter().map(|(_, _, id, _)| id).collect(); + let event_times: Vec<_> = history.iter().map(|(_, _, _, at)| *at).collect(); let mut tx = self.begin_media_share_transaction(community).await?; added += sqlx::query( - "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, h, c FROM unnest($2::text[], $3::uuid[]) AS t(h, c) \ - ORDER BY h, c NULLS FIRST \ + "INSERT INTO media_shares \ + (community_id, sha256, channel_id, event_id, event_created_at) \ + SELECT $1, h, c, e.id, e.created_at \ + FROM unnest($2::text[], $3::uuid[], $4::bytea[], $5::timestamptz[]) \ + AS t(h, c, event_id, event_created_at) \ + JOIN events e ON e.community_id = $1 AND e.id = t.event_id \ + AND e.created_at = t.event_created_at AND e.deleted_at IS NULL \ + ORDER BY h, e.id \ ON CONFLICT DO NOTHING", ) .bind(community.as_uuid()) .bind(hashes) .bind(channels) + .bind(event_ids) + .bind(event_times) .execute(tx.conn()) .await? .rows_affected(); @@ -239,7 +264,7 @@ mod postgres_tests { async fn shares(db: &Db, community: CommunityId, sha256: &str) -> Vec> { let mut shares: Vec> = sqlx::query_scalar( - "SELECT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + "SELECT DISTINCT channel_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", ) .bind(community.as_uuid()) .bind(sha256) @@ -328,10 +353,95 @@ mod postgres_tests { check_backfill().await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn deleting_a_reference_revokes_only_its_own_media_grant() { + let (db, community) = fixture().await; + let (owner, reader) = (Keys::generate(), Keys::generate()); + let open = channel(&db, community, &owner, ChannelVisibility::Open).await; + for channel_id in [Some(open), None] { + let sha = unique_sha(); + upload(&db, community, &owner, &sha).await; + let first = attachment(&owner, &sha, channel_id); + let last = attachment(&owner, &sha, channel_id); + for event in [&first, &last] { + db.insert_event(community, event, channel_id).await.unwrap(); + } + let sources: Vec> = sqlx::query_scalar( + "SELECT event_id FROM media_shares WHERE community_id = $1 AND sha256 = $2", + ) + .bind(community.as_uuid()) + .bind(&sha) + .fetch_all(&db.pool) + .await + .unwrap(); + assert_eq!(sources.len(), 2, "each message must retain its own grant"); + for event in [&first, &last] { + assert!(sources.contains(&event.id.to_bytes().to_vec())); + } + assert!(readable_share(&db, community, &sha, &reader).await); + for (event, still_shared) in [(&first, true), (&last, false)] { + assert!( + db.soft_delete_event_and_update_thread( + community, + event.id.as_bytes(), + None, + None, + ) + .await + .unwrap() + ); + assert_eq!( + readable_share(&db, community, &sha, &reader).await, + still_shared, + "only an undeleted source message can grant access" + ); + } + db.insert_event(community, &attachment(&reader, &sha, None), None) + .await + .unwrap(); + assert!(!readable_share(&db, community, &sha, &reader).await); + db.backfill_media_shares(1).await.unwrap(); + assert!(!readable_share(&db, community, &sha, &reader).await); + + let purged = attachment(&owner, &sha, channel_id); + db.insert_event(community, &purged, channel_id) + .await + .unwrap(); + assert!(readable_share(&db, community, &sha, &reader).await); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query("DELETE FROM events WHERE community_id = $1 AND id = $2") + .bind(community.as_uuid()) + .bind(purged.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert!( + !readable_share(&db, community, &sha, &reader).await, + "a purged source cannot leave a dangling grant" + ); + } + } + + async fn readable_share(db: &Db, community: CommunityId, sha: &str, reader: &Keys) -> bool { + sqlx::query_scalar(concat!( + "SELECT ", + readable_sql!(), + " FROM (SELECT $2::text AS sha256) b" + )) + .bind(community.as_uuid()) + .bind(sha) + .bind(reader.public_key().to_bytes().to_vec()) + .fetch_one(&db.pool) + .await + .unwrap() + } + async fn check_backfill() { let (db, community) = fixture().await; let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); - let (sha, historical) = (unique_sha(), unique_sha()); + let (sha, historical, deleted_history) = (unique_sha(), unique_sha(), unique_sha()); let private = channel(&db, community, &alice, ChannelVisibility::Private).await; upload(&db, community, &alice, &sha).await; db.insert_event( @@ -348,6 +458,11 @@ mod postgres_tests { db.insert_event(community, &old, None) .await .expect("carol posts before share recording"); + let deleted = attachment(&carol, &deleted_history, None); + db.insert_event(community, &deleted, None).await.unwrap(); + db.soft_delete_event_and_update_thread(community, deleted.id.as_bytes(), None, None) + .await + .unwrap(); let mut tx = crate::begin_community_write_transaction( &db.pool, @@ -356,12 +471,17 @@ mod postgres_tests { ) .await .expect("admit"); - sqlx::query("UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = $2") - .bind(community.as_uuid()) - .bind(old.id.as_bytes().as_slice()) - .execute(tx.conn()) - .await - .expect("age carol's event"); + sqlx::query( + "UPDATE events SET received_at = 'epoch' WHERE community_id = $1 AND id = ANY($2)", + ) + .bind(community.as_uuid()) + .bind(vec![ + old.id.to_bytes().to_vec(), + deleted.id.to_bytes().to_vec(), + ]) + .execute(tx.conn()) + .await + .expect("age carol's event"); sqlx::query("DELETE FROM media_shares WHERE community_id = $1") .bind(community.as_uuid()) .execute(tx.conn()) @@ -377,6 +497,10 @@ mod postgres_tests { "a reference live recording refused must stay unshared" ); assert_eq!(shares(&db, community, &historical).await, vec![None]); + assert!( + shares(&db, community, &deleted_history).await.is_empty(), + "legacy deleted messages must never gain a share" + ); } #[tokio::test] diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql index b70ac3530b2..9b4212113fa 100644 --- a/migrations/0067_media_access.sql +++ b/migrations/0067_media_access.sql @@ -15,19 +15,18 @@ CREATE TABLE media_uploaders ( PRIMARY KEY (community_id, sha256, pubkey) ); --- media_shares: one row per channel a blob is shared into, written with the --- event that references it, or a NULL channel for a channel-less reference --- (avatars, custom emoji, notes), which shares it with the whole community. +-- media_shares: one grant per referencing event and blob. A NULL channel +-- shares with the whole community. Deleted or purged source events grant +-- nothing; event_created_at locates the source in the partitioned table. CREATE TABLE media_shares ( community_id UUID NOT NULL REFERENCES communities(id), sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), channel_id UUID, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + event_id BYTEA NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, event_id) ); -CREATE UNIQUE INDEX idx_media_shares_channel - ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; -CREATE UNIQUE INDEX idx_media_shares_public - ON media_shares (community_id, sha256) WHERE channel_id IS NULL; SELECT attach_community_write_fence('media_uploaders'); SELECT attach_community_write_fence('media_shares'); diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql index 18ab231b4bc..e914069c963 100644 --- a/schema/tables/public/media_shares.sql +++ b/schema/tables/public/media_shares.sql @@ -1,15 +1,14 @@ -- ── Media shares ────────────────────────────────────────────────────────────── --- Where each blob is shared: one row per channel an event referencing it was --- stored in, or a NULL channel for a channel-less reference, which shares it --- with the whole community. +-- Each event's grant to a channel, or the whole community for a NULL channel. +-- Reads join the source event: deleted or purged events grant no access. +-- Keep its timestamp for an indexed lookup in the partitioned events table. CREATE TABLE media_shares ( community_id UUID NOT NULL REFERENCES communities(id), sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), channel_id UUID, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() + event_id BYTEA NOT NULL, + event_created_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, event_id) ); -CREATE UNIQUE INDEX idx_media_shares_channel - ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; -CREATE UNIQUE INDEX idx_media_shares_public - ON media_shares (community_id, sha256) WHERE channel_id IS NULL; From 6c82228c3f7b66ea96d8875e4bda4fe75f069ab9 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 14:09:05 +1100 Subject: [PATCH 13/48] fix(media): preserve grants through profile updates Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/event_follow_up.rs | 32 ++++- crates/buzz-db/src/store/media_reference.rs | 65 +++++++++ crates/buzz-db/src/store/replaceable.rs | 131 ++++++++++--------- crates/buzz-db/tests/observability_source.rs | 10 +- 4 files changed, 170 insertions(+), 68 deletions(-) diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index 6fd0de9fc27..9079e496539 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -1,7 +1,10 @@ //! App-owned follow-up work for a newly inserted `events` row. //! -//! Two pieces of work follow every durable event insert: +//! Three pieces of work follow every durable event insert: //! +//! - **Media grants.** References grant access only after their author passes +//! the sharing check. Replacements record these before retiring the old head, +//! inside their savepoint; delivery is staged only after it succeeds. //! - **Push match enqueue.** Eligible event IDs are staged in memory. Commit //! submits them to a bounded best-effort worker using a dedicated connection. //! Push errors, delays, overload, and shutdown can drop wakes, never messages. @@ -45,9 +48,18 @@ pub(crate) async fn after_admitted_insert( event: &nostr::Event, channel_id: Option, ) -> Result<()> { - let kind = i32::from(event.kind.as_u16()); - tx.record_push_event(event.id.as_bytes(), kind); - tx.record_channel_event(channel_id, kind); + record_admitted_media(tx, event, channel_id).await?; + stage_admitted_insert(tx, event, channel_id); + Ok(()) +} + +/// Record media inside a replacement's savepoint, before retiring its old head. +/// This preserves authority carried by a reference the new head keeps. +pub(crate) async fn record_admitted_media( + tx: &mut AdmittedTx, + event: &nostr::Event, + channel_id: Option, +) -> Result<()> { let hashes = buzz_core::media::shared_blob_hashes(event); if !hashes.is_empty() { let community = tx.community(); @@ -63,6 +75,18 @@ pub(crate) async fn after_admitted_insert( Ok(()) } +/// Stage delivery only after the insertion's savepoint has succeeded. +/// Call after [`record_admitted_media`] when splitting the two phases. +pub(crate) fn stage_admitted_insert( + tx: &mut AdmittedTx, + event: &nostr::Event, + channel_id: Option, +) { + let kind = i32::from(event.kind.as_u16()); + tx.record_push_event(event.id.as_bytes(), kind); + tx.record_channel_event(channel_id, kind); +} + /// The channel whose TTL an event of `kind` in `channel_id` refreshes, if any. pub(crate) fn ttl_refresh_channel(channel_id: Option, kind: i32) -> Option { channel_id.filter(|_| kind != KIND_CHANNEL_CREATE) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index dc776f72e6a..f767db1efc7 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -438,6 +438,71 @@ mod postgres_tests { .unwrap() } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn replacement_preserves_only_the_media_it_continues_to_reference() { + let (db, community) = fixture().await; + let viewer = Keys::generate(); + for kind in [Kind::Metadata, Kind::from(30030)] { + let (uploader, publisher) = (Keys::generate(), Keys::generate()); + let sha = unique_sha(); + upload(&db, community, &uploader, &sha).await; + let base = nostr::Timestamp::now().as_secs(); + let make = |keys: &Keys, kind: Kind, step: u64, picture: bool| { + let url = if picture { + format!("/media/{sha}.jpg") + } else { + String::new() + }; + let mut tags = Vec::new(); + let content = if kind == Kind::Metadata { + serde_json::json!({"picture": url, "name": format!("profile-{step}")}) + .to_string() + } else { + tags.push(Tag::parse(["d", "media-continuity"]).unwrap()); + if picture { + tags.push(Tag::parse(["emoji", "sample", &url]).unwrap()); + } + String::new() + }; + EventBuilder::new(kind, content) + .tags(tags) + .custom_created_at(nostr::Timestamp::from_secs(base + step)) + .sign_with_keys(keys) + .unwrap() + }; + let publish = async |event: &nostr::Event| { + if event.kind == Kind::Metadata { + db.replace_addressable_event(community, event, None) + .await + .unwrap() + .1 + } else { + db.replace_parameterized_event(community, event, "media-continuity", None) + .await + .unwrap() + .1 + } + }; + assert!(publish(&make(&uploader, Kind::Metadata, 0, true)).await); + assert!(publish(&make(&publisher, kind, 0, true)).await); + assert!(publish(&make(&uploader, Kind::Metadata, 1, false)).await); + assert!(readable_share(&db, community, &sha, &viewer).await); + assert!(publish(&make(&publisher, kind, 1, true)).await); + assert!( + readable_share(&db, community, &sha, &viewer).await, + "a replacement must carry forward an authorized reference" + ); + assert!(publish(&make(&publisher, kind, 2, false)).await); + assert!(!readable_share(&db, community, &sha, &viewer).await); + assert!(publish(&make(&publisher, kind, 3, true)).await); + assert!( + !readable_share(&db, community, &sha, &viewer).await, + "a later replacement cannot revive a previously removed grant" + ); + } + } + async fn check_backfill() { let (db, community) = fixture().await; let (alice, bob, carol) = (Keys::generate(), Keys::generate(), Keys::generate()); diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 59482e742c4..6f22d56fba2 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -262,6 +262,53 @@ async fn replace_parameterized_event_in_transaction_impl( let tags_json = serde_json::to_value(&event.tags)?; crate::channel_members::guard_channel_roster_event(tx, kind_i32, channel_id, &tags_json) .await?; + + let sig_bytes = event.sig.serialize(); + let insert_result = sqlx::query( + "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \ + ON CONFLICT DO NOTHING", + ) + .bind(community_id.as_uuid()) + .bind(incoming_id) + .bind(pubkey_bytes.as_slice()) + .bind(created_at) + .bind(kind_i32) + .bind(&tags_json) + .bind(&event.content) + .bind(sig_bytes.as_slice()) + .bind(received_at) + .bind(channel_id) + .bind(d_tag) + .bind(crate::event::extract_not_before(event)) + .execute(tx.conn()) + .await?; + + if insert_result.rows_affected() == 0 { + return Ok(false); + } + + if is_nip_rs { + sqlx::query( + "INSERT INTO parameterized_event_watermarks \ + (community_id, kind, pubkey, d_tag, created_at, event_id) \ + VALUES ($1, $2, $3, $4, $5, $6) \ + ON CONFLICT (community_id, kind, pubkey, d_tag) DO UPDATE SET \ + created_at = EXCLUDED.created_at, event_id = EXCLUDED.event_id", + ) + .bind(community_id.as_uuid()) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .bind(created_at) + .bind(incoming_id) + .execute(tx.conn()) + .await?; + } + + crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; + // Authorize retained references while the previous head is still live. + crate::store::event_follow_up::record_admitted_media(tx, event, channel_id).await?; if existing.is_some() { let previous_nip_rs_hard_delete: Option = if is_nip_rs { sqlx::query_scalar( @@ -279,16 +326,17 @@ async fn replace_parameterized_event_in_transaction_impl( } let statement = if hard_delete_superseded { "DELETE FROM events \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL" + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL AND id <> $5" } else { "UPDATE events SET deleted_at = NOW() \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL" + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL AND id <> $5" }; sqlx::query(statement) .bind(community_id.as_uuid()) .bind(kind_i32) .bind(pubkey_bytes.as_slice()) .bind(d_tag) + .bind(incoming_id) .execute(tx.conn()) .await?; @@ -311,50 +359,6 @@ async fn replace_parameterized_event_in_transaction_impl( } } - let sig_bytes = event.sig.serialize(); - let insert_result = sqlx::query( - "INSERT INTO events (community_id, id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag, not_before) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) \ - ON CONFLICT DO NOTHING", - ) - .bind(community_id.as_uuid()) - .bind(incoming_id) - .bind(pubkey_bytes.as_slice()) - .bind(created_at) - .bind(kind_i32) - .bind(&tags_json) - .bind(&event.content) - .bind(sig_bytes.as_slice()) - .bind(received_at) - .bind(channel_id) - .bind(d_tag) - .bind(crate::event::extract_not_before(event)) - .execute(tx.conn()) - .await?; - - if insert_result.rows_affected() == 0 { - return Ok(false); - } - - if is_nip_rs { - sqlx::query( - "INSERT INTO parameterized_event_watermarks \ - (community_id, kind, pubkey, d_tag, created_at, event_id) \ - VALUES ($1, $2, $3, $4, $5, $6) \ - ON CONFLICT (community_id, kind, pubkey, d_tag) DO UPDATE SET \ - created_at = EXCLUDED.created_at, event_id = EXCLUDED.event_id", - ) - .bind(community_id.as_uuid()) - .bind(kind_i32) - .bind(pubkey_bytes.as_slice()) - .bind(d_tag) - .bind(created_at) - .bind(incoming_id) - .execute(tx.conn()) - .await?; - } - - crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; Ok(true) } .await; @@ -383,7 +387,7 @@ async fn replace_parameterized_event_in_transaction_impl( .execute(tx.conn()) .await?; if status == ParameterizedReplaceStatus::Inserted { - crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; + crate::store::event_follow_up::stage_admitted_insert(tx, event, channel_id); } Ok(ParameterizedReplaceResult::new( @@ -492,20 +496,6 @@ impl Db { ) .await?; - // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. - sqlx::query( - "UPDATE events SET deleted_at = NOW() \ - WHERE community_id = $1 AND kind = $2 AND pubkey = $3 \ - AND channel_id IS NOT DISTINCT FROM $4 \ - AND deleted_at IS NULL", - ) - .bind(community_id.as_uuid()) - .bind(kind_i32) - .bind(pubkey_bytes.as_slice()) - .bind(channel_id) - .execute(tx.conn()) - .await?; - // Insert the new event inside the same transaction. let sig_bytes = event.sig.serialize(); let received_at = chrono::Utc::now(); @@ -532,8 +522,7 @@ impl Db { let was_inserted = insert_result.rows_affected() > 0; if !was_inserted { - // ON CONFLICT fired — the event ID already exists. Rollback the - // soft-delete so we don't lose the previous replaceable event. + // ON CONFLICT fired — leave the previously-live event intact. tx.rollback().await?; return Ok(( StoredEvent::with_received_at( @@ -546,9 +535,25 @@ impl Db { )); } + // Authorize retained references while the previous head is still live. crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) .await?; + // Soft-delete the old event (if any). IS NOT DISTINCT FROM for NULL safety. + sqlx::query( + "UPDATE events SET deleted_at = NOW() \ + WHERE community_id = $1 AND kind = $2 AND pubkey = $3 \ + AND channel_id IS NOT DISTINCT FROM $4 \ + AND deleted_at IS NULL AND id <> $5", + ) + .bind(community_id.as_uuid()) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(channel_id) + .bind(incoming_id) + .execute(tx.conn()) + .await?; + // The replaceable event and its denormalized mention index are one // authoritative discovery write. An indexing error must roll back the // new event and restore the previously-live event. diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index b1c95d7f1c7..dbf96bd621b 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -1643,12 +1643,14 @@ fn events_insert_tails(source: &str) -> Vec { /// An events insert must run both follow-ups: the push enqueue and the /// pre-commit TTL refresh. `after_admitted_insert` does both; a writer that -/// inserts inside a savepoint calls the two halves separately. +/// inserts inside a savepoint records media there and stages delivery after it. /// /// Only real calls count: a hook in a comment or a string literal does not. fn runs_event_follow_ups(function_source: &str) -> bool { let code = mask_comments_and_literals(function_source); code.contains("event_follow_up::after_admitted_insert(") + || (code.contains("event_follow_up::record_admitted_media(") + && code.contains("event_follow_up::stage_admitted_insert(")) || (code.contains("event_follow_up::enqueue_push_match(") && code.contains(".record_channel_event(")) } @@ -1696,6 +1698,12 @@ pub(crate) async fn hooked_writer(tx: &mut AdmittedTx) { .await .expect("follow up"); } +pub(crate) async fn media_savepoint_writer(tx: &mut AdmittedTx) { + sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") + .execute(tx.conn()).await.expect("write"); + crate::store::event_follow_up::record_admitted_media(tx, event, channel).await.expect("media"); + crate::store::event_follow_up::stage_admitted_insert(tx, event, channel); +} pub(crate) async fn savepoint_writer(tx: &mut AdmittedTx) { sqlx::query("INSERT INTO events (community_id, id) VALUES ($1, $2)") .execute(tx.conn()) From a7cdc13cd73b978f996a51733790c8e45e2f4eab Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Wed, 21 Oct 2026 11:30:33 +1100 Subject: [PATCH 14/48] docs(media): clarify backfill permission changes Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 4 +++- crates/buzz-db/src/store/media_reference.rs | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 6cfc4bdc241..81dddfabfd6 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -102,9 +102,11 @@ enum Command { }, /// Record media shares for events stored before share recording existed. /// - /// Idempotent and resumable; run after every relay pod records shares and + /// Run after every relay pod records shares and /// before enforcing channel-scoped media reads. Historical references are /// trusted only before each community's schema-recorded cutoff. + /// Later references use current permissions, so reruns can grant references + /// live recording previously refused if their authors have since gained access. BackfillMediaShares { /// Events read and recorded per transaction. #[arg(long, default_value_t = 1000)] diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index f767db1efc7..a6a552aae5e 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -85,8 +85,9 @@ impl Db { /// Shares come from the same rule as live writes. Events received before /// share recording was installed predate the readability check, so their /// references are recorded as they stand; later events get the live, - /// checked recording, so the backfill never shares what live recording - /// refused. Idempotent and resumable: rerunning only fills missing shares. + /// checked recording using the author's current permissions. Earlier refusals + /// are not remembered: reruns can grant old references after access changes. + /// Rows are deduplicated, but authorization decisions are not preserved. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { if batch_size <= 0 { From 3d77f74cf236ba20fdffb8406a52f2a7f912e7d8 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 20:12:37 +1100 Subject: [PATCH 15/48] fix(db): renumber media migration after rebase Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/runtime/migration.rs | 10 +++++----- crates/buzz-db/src/store/deletion.rs | 2 +- crates/buzz-db/src/store/media_reference.rs | 2 +- .../{0067_media_access.sql => 0068_media_access.sql} | 0 4 files changed, 7 insertions(+), 7 deletions(-) rename migrations/{0067_media_access.sql => 0068_media_access.sql} (100%) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 385ee4d2211..97f58591e61 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -769,7 +769,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 67); + assert_eq!(migrations.len(), 68); assert_eq!(migrations[64].version, 65); let retire_ttl = strip_sql_comments(migrations[64].sql.as_str()); for statement in [ @@ -806,9 +806,9 @@ mod postgres_tests { .sql .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); - assert_eq!(migrations[66].version, 67); + assert_eq!(migrations[67].version, 68); for table in ["media_uploaders", "media_shares"] { - assert!(migrations[66] + assert!(migrations[67] .sql .as_str() .contains(&format!("CREATE TABLE {table}"))); @@ -2415,10 +2415,10 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('community_nip_fi_issuers')")); expected_fences.insert("community_nip_fi_issuers".to_owned()); - // 0067 fences the media access tables. + // 0068 fences the media access tables. let media_access = MIGRATOR .iter() - .find(|m| m.version == 67) + .find(|m| m.version == 68) .expect("media access migration"); for table in ["media_uploaders", "media_shares"] { assert!(media_access diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 73b1e364717..9a3b123cefa 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -4434,7 +4434,7 @@ mod postgres_tests { }) .await .expect("connect isolated migration DB"); - crate::migration::run_migrations_through(&db.pool, 67) + crate::migration::run_migrations_through(&db.pool, 68) .await .expect("migrate with fence objects present"); let store = db.deletion_store(); diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index a6a552aae5e..c5a846e8e23 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -575,7 +575,7 @@ mod postgres_tests { let pool = PgPool::connect(&crate::test_support::database_url()) .await .expect("connect to migration DB"); - crate::migration::run_migrations_through(&pool, 67) + crate::migration::run_migrations_through(&pool, 68) .await .expect("migrate"); pool.close().await; diff --git a/migrations/0067_media_access.sql b/migrations/0068_media_access.sql similarity index 100% rename from migrations/0067_media_access.sql rename to migrations/0068_media_access.sql From dd5a57a2f87fbc990b854b0d994a28fe097550a4 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 13:29:32 +1100 Subject: [PATCH 16/48] feat(relay): record who uploads each media blob Add a community-scoped `media_uploaders` table (migration 0067, write-fenced and purged with the community) and record the uploader on every successful upload, including deduplicated re-uploads. Uploading the exact bytes proves possession, which channel-scoped media reads (block/buzz#8444) will use to admit an uploader even when the same bytes are attached somewhere they cannot read. Nothing reads the table yet. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/runtime/migration.rs | 16 +++- crates/buzz-db/src/store/deletion.rs | 3 + crates/buzz-db/src/store/media_uploader.rs | 97 ++++++++++++++++++++++ crates/buzz-db/src/store/mod.rs | 2 + crates/buzz-relay/src/api/media.rs | 22 +++-- migrations/0067_media_uploaders.sql | 14 ++++ schema/schema.sql | 2 + schema/tables/public/media_uploaders.sql | 11 +++ schema/triggers/media_uploaders.sql | 1 + 9 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 crates/buzz-db/src/store/media_uploader.rs create mode 100644 migrations/0067_media_uploaders.sql create mode 100644 schema/tables/public/media_uploaders.sql create mode 100644 schema/triggers/media_uploaders.sql diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 7cb0879ff87..6f359ecfd5c 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -709,7 +709,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 66); + assert_eq!(migrations.len(), 67); assert_eq!(migrations[64].version, 65); let retire_ttl = strip_sql_comments(migrations[64].sql.as_str()); for statement in [ @@ -746,6 +746,11 @@ mod postgres_tests { .sql .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); + assert_eq!(migrations[66].version, 67); + assert!(migrations[66] + .sql + .as_str() + .contains("CREATE TABLE media_uploaders")); assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2317,6 +2322,15 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('community_nip_fi_issuers')")); expected_fences.insert("community_nip_fi_issuers".to_owned()); + // 0067 fences the media uploaders table. + assert!(MIGRATOR + .iter() + .find(|m| m.version == 67) + .expect("media uploaders migration") + .sql + .as_str() + .contains("attach_community_write_fence('media_uploaders')")); + expected_fences.insert("media_uploaders".to_owned()); assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 1e1e68cb054..1d910493a84 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -90,6 +90,7 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "media_uploaders", "moderation_actions", "moderation_reports", "parameterized_event_watermarks", @@ -156,6 +157,7 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "audit_log", "community_bans", "community_nip_fi_issuers", + "media_uploaders", "pubkey_allowlist", "relay_members", "users", @@ -4423,6 +4425,7 @@ DROP TRIGGER IF EXISTS community_write_fence_event_mentions ON event_mentions; DROP TRIGGER IF EXISTS community_write_fence_events ON events; DROP TRIGGER IF EXISTS community_write_fence_git_repo_names ON git_repo_names; DROP TRIGGER IF EXISTS community_write_fence_join_policy_acceptances ON join_policy_acceptances; +DROP TRIGGER IF EXISTS community_write_fence_media_uploaders ON media_uploaders; DROP TRIGGER IF EXISTS community_write_fence_moderation_actions ON moderation_actions; DROP TRIGGER IF EXISTS community_write_fence_moderation_reports ON moderation_reports; DROP TRIGGER IF EXISTS community_write_fence_parameterized_event_watermarks ON parameterized_event_watermarks; diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs new file mode 100644 index 00000000000..f4567a43682 --- /dev/null +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -0,0 +1,97 @@ +//! Who has uploaded each blob, as proof of possession for media reads. + +use buzz_datastore_tracing::datastore_span; +use sqlx::PgPool; + +use crate::{error::Result, Db}; +use buzz_core::CommunityId; + +/// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are no-ops. +pub async fn record_media_uploader( + pool: &PgPool, + community_id: CommunityId, + sha256: &str, + pubkey: &[u8], +) -> Result<()> { + sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(community_id.as_uuid()) + .bind(sha256) + .bind(pubkey) + .execute(pool) + .await?; + Ok(()) +} + +impl Db { + /// See [`record_media_uploader`]. + #[datastore_span(name = "record_media_uploader", system = "postgresql")] + pub async fn record_media_uploader( + &self, + community_id: CommunityId, + sha256: &str, + pubkey: &[u8], + ) -> Result<()> { + record_media_uploader(&self.pool, community_id, sha256, pubkey).await + } +} + +#[cfg(test)] +mod postgres_tests { + use super::*; + use uuid::Uuid; + + const SHA: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + + async fn community(db: &Db) -> CommunityId { + let host = format!("uploader-{}.example", Uuid::new_v4().simple()); + db.ensure_configured_community(&host) + .await + .expect("seed community") + .id + } + + async fn uploader_rows(db: &Db, community_id: CommunityId) -> i64 { + sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1") + .bind(community_id.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("count uploaders") + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn repeat_upload_records_one_row() { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let community_id = community(&db).await; + for _ in 0..2 { + db.record_media_uploader(community_id, SHA, &[7; 32]) + .await + .expect("record uploader"); + } + assert_eq!(uploader_rows(&db, community_id).await, 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn deleting_community_refuses_new_uploaders() { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); + let community_id = community(&db).await; + crate::test_support::quiesce_community_for_tests(&db.pool, community_id).await; + assert!(db + .record_media_uploader(community_id, SHA, &[7; 32]) + .await + .is_err()); + assert_eq!(uploader_rows(&db, community_id).await, 0); + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index f759a770a25..ccafad2f810 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,6 +32,8 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Who has uploaded each blob, as proof of possession for media reads. +pub mod media_uploader; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; /// Deployment-global operator-listener mention registrations and delivery queues. diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 6864cb15ae2..3182a22512d 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -576,6 +576,16 @@ async fn upload_blob_result( auth.tenant.host(), ); + state + .db + .record_media_uploader( + auth.tenant.community(), + &descriptor.sha256, + auth.auth_event.pubkey.as_bytes(), + ) + .await + .map_err(|_| MediaError::Internal)?; + // Normalize MIME to a known set to bound label cardinality. let mime_label = match descriptor.mime_type.as_str() { "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "video/mp4" => { @@ -2135,11 +2145,13 @@ mod tests { const TEST_ISSUER: &str = "https://issuer.example"; const TEST_AUDIENCE: &str = "https://relay.example"; const TEST_KID: &str = "test-key-1"; - const TEST_EC_PKCS8_PEM: &str = "-----BEGIN PRIVATE KEY-----\n\ - MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n\ - WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n\ - zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n\ - -----END PRIVATE KEY-----\n"; + const TEST_EC_PKCS8_PEM: &str = concat!( + "-----BEGIN PRIVATE KEY-----\n", // sadscan:disable kingfisher.privkey.2 -- static test-only key + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n", + "WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n", + "zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n", + "-----END PRIVATE KEY-----\n", + ); /// Build an AppState with NIP-FI Enforce and a real injected P-256 verifier. async fn media_enforce_test_state() -> Option> { diff --git a/migrations/0067_media_uploaders.sql b/migrations/0067_media_uploaders.sql new file mode 100644 index 00000000000..458381b7da3 --- /dev/null +++ b/migrations/0067_media_uploaders.sql @@ -0,0 +1,14 @@ +-- Who has uploaded each blob in a community. Uploading the exact bytes proves +-- possession, so channel-scoped media reads can always admit an uploader even +-- when the same bytes are also attached somewhere they cannot read. +-- +-- Additive. Nothing reads this table yet. +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); + +SELECT attach_community_write_fence('media_uploaders'); diff --git a/schema/schema.sql b/schema/schema.sql index d252cbdb7ea..47a06c8f178 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -107,6 +107,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i tables/public/moderation_reports.sql \i tables/public/community_bans.sql \i tables/public/community_nip_fi_issuers.sql +\i tables/public/media_uploaders.sql \i tables/public/_operator_global_tables.sql \i tables/public/git_repo_names.sql \i tables/public/parameterized_event_watermarks.sql @@ -159,6 +160,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i triggers/event_mentions.sql \i triggers/git_repo_names.sql \i triggers/join_policy_acceptances.sql +\i triggers/media_uploaders.sql \i triggers/moderation_actions.sql \i triggers/moderation_reports.sql \i triggers/parameterized_event_watermarks.sql diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql new file mode 100644 index 00000000000..7a1255ccdd3 --- /dev/null +++ b/schema/tables/public/media_uploaders.sql @@ -0,0 +1,11 @@ +-- ── Media uploaders ─────────────────────────────────────────────────────────── +-- Who has uploaded each blob in a community. Uploading the exact bytes proves +-- possession, so channel-scoped media reads can always admit an uploader. + +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); diff --git a/schema/triggers/media_uploaders.sql b/schema/triggers/media_uploaders.sql new file mode 100644 index 00000000000..23367202e24 --- /dev/null +++ b/schema/triggers/media_uploaders.sql @@ -0,0 +1 @@ +SELECT attach_community_write_fence('media_uploaders'); From ca631dae02fed37dedf1770e21784f110f73e717 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 13:36:27 +1100 Subject: [PATCH 17/48] fix(relay): bind uploader records to the upload's serving lease Record the uploader in a transaction admitted with the upload's serving-write lease, like other final writes of an external side effect, so an admitted upload can finish while its community quiesces and nothing lands after the lease is gone. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_uploader.rs | 101 ++++++++++++--------- crates/buzz-relay/src/api/media.rs | 4 +- 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs index f4567a43682..9a720aa6366 100644 --- a/crates/buzz-db/src/store/media_uploader.rs +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -1,56 +1,76 @@ //! Who has uploaded each blob, as proof of possession for media reads. use buzz_datastore_tracing::datastore_span; -use sqlx::PgPool; -use crate::{error::Result, Db}; -use buzz_core::CommunityId; - -/// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are no-ops. -pub async fn record_media_uploader( - pool: &PgPool, - community_id: CommunityId, - sha256: &str, - pubkey: &[u8], -) -> Result<()> { - sqlx::query( - "INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) \ - ON CONFLICT DO NOTHING", - ) - .bind(community_id.as_uuid()) - .bind(sha256) - .bind(pubkey) - .execute(pool) - .await?; - Ok(()) -} +use crate::deletion::ServingWriteLease; +use crate::{error::Result, observability, Db}; impl Db { - /// See [`record_media_uploader`]. + /// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are + /// no-ops. The write presents the upload's serving lease, so it may finish + /// while the community quiesces but never after the lease is gone. #[datastore_span(name = "record_media_uploader", system = "postgresql")] pub async fn record_media_uploader( &self, - community_id: CommunityId, + lease: &ServingWriteLease, sha256: &str, pubkey: &[u8], ) -> Result<()> { - record_media_uploader(&self.pool, community_id, sha256, pubkey).await + let connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::EventWrite) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + self.deletion_store() + .guard_transaction_with_serving_lease(&mut tx, lease) + .await?; + sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) \ + ON CONFLICT DO NOTHING", + ) + .bind(lease.community_id.as_uuid()) + .bind(sha256) + .bind(pubkey) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) } } #[cfg(test)] mod postgres_tests { + use std::time::Duration; + use super::*; + use buzz_core::CommunityId; + use sqlx::PgPool; use uuid::Uuid; const SHA: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; - async fn community(db: &Db) -> CommunityId { + async fn db_with_lease() -> (Db, CommunityId, ServingWriteLease) { + let db = Db::from_pool( + PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect to test DB"), + ); let host = format!("uploader-{}.example", Uuid::new_v4().simple()); - db.ensure_configured_community(&host) + let community_id = db + .ensure_configured_community(&host) .await .expect("seed community") - .id + .id; + let lease = db + .deletion_store() + .acquire_serving_write_lease( + community_id, + "media_upload", + "media-uploader-test", + Duration::from_secs(60), + ) + .await + .expect("acquire lease"); + (db, community_id, lease) } async fn uploader_rows(db: &Db, community_id: CommunityId) -> i64 { @@ -64,14 +84,9 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] async fn repeat_upload_records_one_row() { - let db = Db::from_pool( - PgPool::connect(&crate::test_support::database_url()) - .await - .expect("connect to test DB"), - ); - let community_id = community(&db).await; + let (db, community_id, lease) = db_with_lease().await; for _ in 0..2 { - db.record_media_uploader(community_id, SHA, &[7; 32]) + db.record_media_uploader(&lease, SHA, &[7; 32]) .await .expect("record uploader"); } @@ -80,16 +95,14 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn deleting_community_refuses_new_uploaders() { - let db = Db::from_pool( - PgPool::connect(&crate::test_support::database_url()) - .await - .expect("connect to test DB"), - ); - let community_id = community(&db).await; - crate::test_support::quiesce_community_for_tests(&db.pool, community_id).await; + async fn released_lease_refuses_the_write() { + let (db, community_id, lease) = db_with_lease().await; + db.deletion_store() + .release_serving_write_lease(&lease) + .await + .expect("release lease"); assert!(db - .record_media_uploader(community_id, SHA, &[7; 32]) + .record_media_uploader(&lease, SHA, &[7; 32]) .await .is_err()); assert_eq!(uploader_rows(&db, community_id).await, 0); diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 3182a22512d..000ffaf6feb 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -579,12 +579,12 @@ async fn upload_blob_result( state .db .record_media_uploader( - auth.tenant.community(), + serving_write.lease(), &descriptor.sha256, auth.auth_event.pubkey.as_bytes(), ) .await - .map_err(|_| MediaError::Internal)?; + .map_err(|error| serving_lease_lost(error.into()))?; // Normalize MIME to a known set to bound label cardinality. let mime_label = match descriptor.mime_type.as_str() { From 0309016ace0b212b5c46afe895da8a9ad87590a1 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 09:26:54 +1100 Subject: [PATCH 18/48] fix(relay): scope media reads to the channels a blob is attached in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET/HEAD /media only checked community membership, so anyone in a community who learned an attachment's hash could download a file posted in a private channel or DM they cannot read. Reads now also require that the reader can see at least one event attaching the blob (imeta `x` or link-preview hash) — in a channel they can read, or a channel-less event. Blobs never attached in a channel (avatars, custom emoji, unsent uploads) stay community-readable, and a refusal is a 404 so non-members cannot probe for private attachments. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 59 ++++++ crates/buzz-db/src/store/mod.rs | 2 + crates/buzz-relay/src/api/media.rs | 91 ++++++++- .../tests/conformance_multitenant.rs | 4 - .../tests/e2e_media_extended.rs | 177 +++++++++++------- 5 files changed, 252 insertions(+), 81 deletions(-) create mode 100644 crates/buzz-db/src/store/media_reference.rs diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs new file mode 100644 index 00000000000..d4f585307e9 --- /dev/null +++ b/crates/buzz-db/src/store/media_reference.rs @@ -0,0 +1,59 @@ +//! Which channels an uploaded blob is attached in. + +use buzz_datastore_tracing::datastore_span; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::{error::Result, Db}; +use buzz_core::CommunityId; + +/// Distinct `channel_id`s of every stored event, deleted or not, that attaches +/// the blob `sha256`; `None` is a channel-less event. +/// +/// A blob is attached by a NIP-92 `imeta` tag carrying `x ` (ingest +/// rejects imeta without it) or by a `link-preview` tag carrying it as the image +/// or favicon hash. Containment is the GIN-indexed prefilter; the positional +/// check keeps a hash in some other tag from counting as an attachment. +pub async fn get_blob_reference_channels( + pool: &PgPool, + community_id: CommunityId, + sha256: &str, +) -> Result>> { + let imeta_entry = format!("x {sha256}"); + let containments = vec![ + serde_json::json!([["imeta", &imeta_entry]]), + serde_json::json!([["link-preview", sha256]]), + ]; + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let channels = sqlx::query_scalar( + "SELECT DISTINCT channel_id FROM events \ + WHERE community_id = $1 \ + AND tags @> ANY($2::jsonb[]) \ + AND EXISTS (SELECT 1 FROM jsonb_array_elements(tags) AS tag \ + WHERE (tag->>0 = 'imeta' AND tag @> jsonb_build_array($3::text)) \ + OR (tag->>0 = 'link-preview' AND tag @> jsonb_build_array($4::text)))", + ) + .bind(community_id.as_uuid()) + .bind(containments) + .bind(imeta_entry) + .bind(sha256) + .fetch_all(&mut *connection) + .await?; + Ok(channels) +} + +impl Db { + /// Channels the blob `sha256` is attached in; see [`get_blob_reference_channels`]. + #[datastore_span(name = "get_blob_reference_channels", system = "postgresql")] + pub async fn get_blob_reference_channels( + &self, + community_id: CommunityId, + sha256: &str, + ) -> Result>> { + get_blob_reference_channels(&self.pool, community_id, sha256).await + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index f759a770a25..b3f74aa132e 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,6 +32,8 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Channels an uploaded blob is attached in, for media read authorization. +pub mod media_reference; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; /// Deployment-global operator-listener mention registrations and delivery queues. diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 6864cb15ae2..e1195a0f22b 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -704,6 +704,65 @@ async fn enforce_blossom_read_membership( .map_err(|e| membership_denial(e, strictness)) } +/// Channel gate for media reads, after the community membership gate. +/// +/// A blob attached in a channel is readable only through an attachment the +/// reader can see: one in a channel they can read, or a channel-less one. Blobs +/// never attached in a channel (avatars, custom emoji, unsent uploads) stay +/// readable to the whole community. A refusal is `NotFound` so a non-member +/// cannot probe whether a private attachment exists. +async fn enforce_blossom_read_channel_access( + state: &AppState, + tenant: &TenantContext, + reader: &nostr::PublicKey, + sha256: &str, + strictness: BlossomStrictness, +) -> Result<(), MediaDenial> { + let community = tenant.community(); + let unavailable = |_| media_denial(MediaError::AuthorizationUnavailable, strictness); + let references = state + .db + .get_blob_reference_channels(community, sha256) + .await + .map_err(unavailable)?; + let Some(channels) = references.into_iter().collect::>>() else { + return Ok(()); + }; + if channels.is_empty() { + return Ok(()); + } + let reader = reader.as_bytes(); + let accessible = state + .get_accessible_channel_ids_cached(community, reader) + .await + .map_err(unavailable)?; + if channels.iter().any(|id| accessible.contains(id)) { + return Ok(()); + } + let readable_system = state + .db + .get_readable_system_channel_ids(community, reader) + .await + .map_err(unavailable)?; + if channels.iter().any(|id| readable_system.contains(id)) { + return Ok(()); + } + Err(MediaError::NotFound.into()) +} + +/// Every read gate a Blossom GET/HEAD passes before any storage lookup. +async fn enforce_blossom_read_access( + state: &AppState, + tenant: &TenantContext, + auth_event: &nostr::Event, + headers: &HeaderMap, + sha256: &str, + strictness: BlossomStrictness, +) -> Result<(), MediaDenial> { + enforce_blossom_read_membership(state, tenant, auth_event, headers, strictness).await?; + enforce_blossom_read_channel_access(state, tenant, &auth_event.pubkey, sha256, strictness).await +} + fn blob_cache_control() -> &'static str { "private, max-age=31536000, immutable" } @@ -819,16 +878,24 @@ pub(crate) async fn get_blob( ) .map(drop) }); + let proof_sha256 = sha256.clone(); let admission = match admit_nip_fi_http_on_state(&state, &req_headers, move || { - extract_blossom_read_proof(&headers_clone, &sha256, &tenant_host, strictness) + extract_blossom_read_proof(&headers_clone, &proof_sha256, &tenant_host, strictness) .map_err(|e| e.into_response()) }) { Ok(a) => a, Err(resp) => return Ok(resp), }; let auth_event = admission.into_extra(); - // Post-admission: membership gate. - enforce_blossom_read_membership(&state, &tenant, &auth_event, &req_headers, strictness).await?; + enforce_blossom_read_access( + &state, + &tenant, + &auth_event, + &req_headers, + &sha256, + strictness, + ) + .await?; serve_blob_for_tenant(&state, &tenant, &sha256_ext, &req_headers) .await .map_err(MediaDenial::from) @@ -1113,15 +1180,17 @@ pub(crate) async fn head_blob( extract_blossom_read_proof(&headers, &sha256, &tenant_host, BlossomStrictness::Strict) .map(drop) }); + let proof_sha256 = sha256.clone(); let admission = match admit_nip_fi_http_on_state(&state, &headers, move || { - extract_blossom_read_proof(&headers_clone, &sha256, &tenant_host, strictness) + extract_blossom_read_proof(&headers_clone, &proof_sha256, &tenant_host, strictness) .map_err(|e| e.into_response()) }) { Ok(a) => a, Err(resp) => return Ok(resp), }; let auth_event = admission.into_extra(); - enforce_blossom_read_membership(&state, &tenant, &auth_event, &headers, strictness).await?; + enforce_blossom_read_access(&state, &tenant, &auth_event, &headers, &sha256, strictness) + .await?; let cache_control = blob_cache_control(); // Sidecar gate FIRST — reject before any blob I/O. @@ -2135,11 +2204,13 @@ mod tests { const TEST_ISSUER: &str = "https://issuer.example"; const TEST_AUDIENCE: &str = "https://relay.example"; const TEST_KID: &str = "test-key-1"; - const TEST_EC_PKCS8_PEM: &str = "-----BEGIN PRIVATE KEY-----\n\ - MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n\ - WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n\ - zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n\ - -----END PRIVATE KEY-----\n"; + const TEST_EC_PKCS8_PEM: &str = concat!( + "-----BEGIN PRIVATE KEY-----\n", // sadscan:disable kingfisher.privkey.2 -- static test-only key + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgcnxDM4EiirH9dHUE\n", + "WZc759TX4s5PAn8kO5ovXSnGxCWhRANCAARFb6ZnsfkqOOXyEhj3KBQphGKF4vTa\n", + "zhebbavbZ1ZoklqkF1cGg+jTO7rONAVEzXvXUWtV6CdDV+rybiVmFP2w\n", + "-----END PRIVATE KEY-----\n", + ); /// Build an AppState with NIP-FI Enforce and a real injected P-256 verifier. async fn media_enforce_test_state() -> Option> { diff --git a/crates/buzz-test-client/tests/conformance_multitenant.rs b/crates/buzz-test-client/tests/conformance_multitenant.rs index 4c8c8904ac5..583c0d43b44 100644 --- a/crates/buzz-test-client/tests/conformance_multitenant.rs +++ b/crates/buzz-test-client/tests/conformance_multitenant.rs @@ -2621,10 +2621,6 @@ mod media_blossom { /// the bytes: it is the metadata/descriptor/upload-auth/quota/audit layer plus /// the per-tenant read binding. B's private upload metadata and errors must not /// be observable from A even when the underlying blob is shared. - /// - /// Known limitation, deferred: relay membership plus knowledge of a hash is - /// sufficient to read a blob. Read auth binds host and tenant, not the channel - /// ACL of the message the blob was attached to. #[tokio::test] #[ignore] async fn media_metadata_boundary_holds_while_blob_bytes_shared() { diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 9bd73b13fcb..2b08ec6fcff 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -98,6 +98,55 @@ async fn upload_to_path( .expect("upload request") } +/// Status of an authenticated Blossom read of `url` by `keys`. +async fn read_status(client: &Client, keys: &Keys, method: reqwest::Method, url: &str) -> u16 { + let sha256 = url + .rsplit('/') + .next() + .and_then(|file| file.split('.').next()) + .expect("media url ends in {sha256}.{ext}"); + client + .request(method, url) + .header( + "Authorization", + blossom_auth_header(&sign_blossom_get_auth(keys, sha256)), + ) + .send() + .await + .expect("read request") + .status() + .as_u16() +} + +/// Create a stream channel owned by `keys` via a signed kind:9007; returns its id. +async fn create_channel( + client: &Client, + keys: &Keys, + name_prefix: &str, + visibility: &str, +) -> String { + let channel_id = uuid::Uuid::new_v4().to_string(); + let create_event = EventBuilder::new(Kind::from(9007), "") + .tags(vec![ + Tag::parse(["h", &channel_id]).unwrap(), + Tag::parse(["name", &format!("{name_prefix}-{channel_id}")]).unwrap(), + Tag::parse(["channel_type", "stream"]).unwrap(), + Tag::parse(["visibility", visibility]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + let create_resp = client + .post(format!("{}/events", relay_http_url())) + .header("X-Pubkey", keys.public_key().to_hex()) + .header("Content-Type", "application/json") + .body(serde_json::to_string(&create_event).unwrap()) + .send() + .await + .unwrap(); + assert!(create_resp.status().is_success(), "channel creation failed"); + channel_id +} + fn tiny_jpeg() -> Vec { vec![ 0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, @@ -631,31 +680,9 @@ async fn test_ws_valid_imeta() { use buzz_test_client::BuzzTestClient; let keys = Keys::generate(); - let pubkey_hex = keys.public_key().to_hex(); let http = http_client(); - // Create channel via signed kind:9007 event - let channel_uuid = uuid::Uuid::new_v4(); - let channel_name = format!("ws-imeta-test-{}", channel_uuid); - let create_event = EventBuilder::new(Kind::from(9007), "") - .tags(vec![ - Tag::parse(["h", &channel_uuid.to_string()]).unwrap(), - Tag::parse(["name", &channel_name]).unwrap(), - Tag::parse(["channel_type", "stream"]).unwrap(), - Tag::parse(["visibility", "open"]).unwrap(), - ]) - .sign_with_keys(&keys) - .unwrap(); - let create_resp = http - .post(format!("{}/events", relay_http_url())) - .header("X-Pubkey", &pubkey_hex) - .header("Content-Type", "application/json") - .body(serde_json::to_string(&create_event).unwrap()) - .send() - .await - .unwrap(); - assert!(create_resp.status().is_success(), "channel creation failed"); - let channel_id = channel_uuid.to_string(); + let channel_id = create_channel(&http, &keys, "ws-imeta-test", "open").await; println!("Channel: {channel_id}"); // Upload a JPEG to get a valid sha256 @@ -702,30 +729,9 @@ async fn test_ws_invalid_imeta_external_url() { use buzz_test_client::BuzzTestClient; let keys = Keys::generate(); - let pubkey_hex = keys.public_key().to_hex(); let http = http_client(); - let channel_uuid = uuid::Uuid::new_v4(); - let channel_name = format!("ws-imeta-bad-{}", channel_uuid); - let create_event = EventBuilder::new(Kind::from(9007), "") - .tags(vec![ - Tag::parse(["h", &channel_uuid.to_string()]).unwrap(), - Tag::parse(["name", &channel_name]).unwrap(), - Tag::parse(["channel_type", "stream"]).unwrap(), - Tag::parse(["visibility", "open"]).unwrap(), - ]) - .sign_with_keys(&keys) - .unwrap(); - let create_resp = http - .post(format!("{}/events", relay_http_url())) - .header("X-Pubkey", &pubkey_hex) - .header("Content-Type", "application/json") - .body(serde_json::to_string(&create_event).unwrap()) - .send() - .await - .unwrap(); - assert!(create_resp.status().is_success(), "channel creation failed"); - let channel_id = channel_uuid.to_string(); + let channel_id = create_channel(&http, &keys, "ws-imeta-bad", "open").await; let sha = "a".repeat(64); let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) @@ -768,30 +774,9 @@ async fn test_ws_invalid_imeta_missing_fields() { use buzz_test_client::BuzzTestClient; let keys = Keys::generate(); - let pubkey_hex = keys.public_key().to_hex(); let http = http_client(); - let channel_uuid = uuid::Uuid::new_v4(); - let channel_name = format!("ws-imeta-miss-{}", channel_uuid); - let create_event = EventBuilder::new(Kind::from(9007), "") - .tags(vec![ - Tag::parse(["h", &channel_uuid.to_string()]).unwrap(), - Tag::parse(["name", &channel_name]).unwrap(), - Tag::parse(["channel_type", "stream"]).unwrap(), - Tag::parse(["visibility", "open"]).unwrap(), - ]) - .sign_with_keys(&keys) - .unwrap(); - let create_resp = http - .post(format!("{}/events", relay_http_url())) - .header("X-Pubkey", &pubkey_hex) - .header("Content-Type", "application/json") - .body(serde_json::to_string(&create_event).unwrap()) - .send() - .await - .unwrap(); - assert!(create_resp.status().is_success(), "channel creation failed"); - let channel_id = channel_uuid.to_string(); + let channel_id = create_channel(&http, &keys, "ws-imeta-miss", "open").await; let sha = "b".repeat(64); let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) @@ -817,3 +802,61 @@ async fn test_ws_invalid_imeta_missing_fields() { client.disconnect().await.unwrap(); } + +#[tokio::test] +#[ignore] +async fn test_private_channel_attachment_unreadable_to_non_members() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let owner = Keys::generate(); + let outsider = Keys::generate(); + let http = http_client(); + let channel_id = create_channel(&http, &owner, "private-media", "private").await; + + let body = uuid::Uuid::new_v4().as_bytes().repeat(8); + let resp = upload(&http, &owner, &body).await; + assert_eq!(resp.status(), 200); + let desc: serde_json::Value = resp.json().await.unwrap(); + let url = desc["url"].as_str().expect("descriptor url"); + + assert_eq!( + read_status(&http, &outsider, Method::GET, url).await, + 200, + "a blob not yet attached anywhere stays community-readable" + ); + + let mut client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let event = EventBuilder::new(Kind::from(9), "private attachment") + .tags(vec![ + Tag::parse(["h", &channel_id]).unwrap(), + Tag::parse([ + "imeta", + &format!("url {url}"), + &format!("m {}", desc["type"].as_str().unwrap()), + &format!("x {}", desc["sha256"].as_str().unwrap()), + &format!("size {}", desc["size"]), + ]) + .unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let ok = client.send_event(event).await.unwrap(); + assert!(ok.accepted, "attachment rejected: {:?}", ok.message); + client.disconnect().await.unwrap(); + + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &outsider, method.clone(), url).await, + 404, + "{method} by a non-member must not reveal a private-channel attachment" + ); + assert_eq!( + read_status(&http, &owner, method.clone(), url).await, + 200, + "{method} by a channel member must still succeed" + ); + } +} From c22f768eec74942abc71d8d3050b1bd20287212e Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 12:30:54 +1100 Subject: [PATCH 19/48] fix(relay): cover posters and public media in channel-scoped reads Address review feedback on channel-scoped media reads: video posters (`imeta image`) now count as attachments, link-preview hashes match only their image and favicon positions, a cached channel miss is confirmed live before refusing, and current avatars and custom emoji stay readable to the whole community even when the same blob is also attached privately. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 80 +++++++++++++++---- crates/buzz-relay/src/api/media.rs | 32 ++++++-- .../tests/e2e_media_extended.rs | 16 +++- 3 files changed, 105 insertions(+), 23 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index d4f585307e9..3c4595abdab 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -1,4 +1,4 @@ -//! Which channels an uploaded blob is attached in. +//! Where an uploaded blob is referenced, for media read authorization. use buzz_datastore_tracing::datastore_span; use sqlx::PgPool; @@ -7,23 +7,35 @@ use uuid::Uuid; use crate::{error::Result, Db}; use buzz_core::CommunityId; +/// Image extensions a video poster (`imeta image`) may carry; ingest rejects others. +const POSTER_EXTS: [&str; 4] = ["jpg", "png", "gif", "webp"]; + /// Distinct `channel_id`s of every stored event, deleted or not, that attaches /// the blob `sha256`; `None` is a channel-less event. /// -/// A blob is attached by a NIP-92 `imeta` tag carrying `x ` (ingest -/// rejects imeta without it) or by a `link-preview` tag carrying it as the image -/// or favicon hash. Containment is the GIN-indexed prefilter; the positional -/// check keeps a hash in some other tag from counting as an attachment. +/// A blob is attached as an `imeta` file (`x `, required by ingest), as +/// a video poster (`image `, relative or on `media_base_url` as ingest +/// requires), or as a `link-preview` image or favicon hash. Containment is the +/// GIN-indexed prefilter; the positional check keeps the hash appearing in some +/// other field from counting as an attachment. pub async fn get_blob_reference_channels( pool: &PgPool, community_id: CommunityId, sha256: &str, + media_base_url: &str, ) -> Result>> { - let imeta_entry = format!("x {sha256}"); - let containments = vec![ - serde_json::json!([["imeta", &imeta_entry]]), - serde_json::json!([["link-preview", sha256]]), - ]; + let base = media_base_url.trim_end_matches('/'); + let mut imeta_entries = vec![format!("x {sha256}")]; + for ext in POSTER_EXTS { + imeta_entries.push(format!("image /media/{sha256}.{ext}")); + imeta_entries.push(format!("image {base}/{sha256}.{ext}")); + } + let mut containments: Vec = imeta_entries + .iter() + .map(|entry| serde_json::json!([["imeta", entry]])) + .collect(); + containments.push(serde_json::json!([["link-preview", sha256]])); + let mut connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::Authorization, @@ -34,18 +46,47 @@ pub async fn get_blob_reference_channels( WHERE community_id = $1 \ AND tags @> ANY($2::jsonb[]) \ AND EXISTS (SELECT 1 FROM jsonb_array_elements(tags) AS tag \ - WHERE (tag->>0 = 'imeta' AND tag @> jsonb_build_array($3::text)) \ - OR (tag->>0 = 'link-preview' AND tag @> jsonb_build_array($4::text)))", + WHERE (tag->>0 = 'imeta' AND tag ?| $3) \ + OR (tag->>0 = 'link-preview' AND $4 IN (tag->>8, tag->>10)))", ) .bind(community_id.as_uuid()) .bind(containments) - .bind(imeta_entry) + .bind(imeta_entries) .bind(sha256) .fetch_all(&mut *connection) .await?; Ok(channels) } +/// Whether the blob `sha256` is a current profile avatar or in a live custom +/// emoji set, both of which every community member must be able to load. +/// +/// Neither reference is indexed by hash, so callers should consult this only +/// after the channel references alone would refuse a read. +pub async fn is_blob_publicly_referenced( + pool: &PgPool, + community_id: CommunityId, + sha256: &str, +) -> Result { + let pattern = format!("%{sha256}%"); + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let referenced = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM users WHERE community_id = $1 AND avatar_url LIKE $2) \ + OR EXISTS (SELECT 1 FROM events WHERE community_id = $1 AND kind = $3 \ + AND deleted_at IS NULL AND tags::text LIKE $2)", + ) + .bind(community_id.as_uuid()) + .bind(pattern) + .bind(buzz_core::kind::KIND_EMOJI_SET as i32) + .fetch_one(&mut *connection) + .await?; + Ok(referenced) +} + impl Db { /// Channels the blob `sha256` is attached in; see [`get_blob_reference_channels`]. #[datastore_span(name = "get_blob_reference_channels", system = "postgresql")] @@ -53,7 +94,18 @@ impl Db { &self, community_id: CommunityId, sha256: &str, + media_base_url: &str, ) -> Result>> { - get_blob_reference_channels(&self.pool, community_id, sha256).await + get_blob_reference_channels(&self.pool, community_id, sha256, media_base_url).await + } + + /// Whether the blob is an avatar or custom emoji; see [`is_blob_publicly_referenced`]. + #[datastore_span(name = "is_blob_publicly_referenced", system = "postgresql")] + pub async fn is_blob_publicly_referenced( + &self, + community_id: CommunityId, + sha256: &str, + ) -> Result { + is_blob_publicly_referenced(&self.pool, community_id, sha256).await } } diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index e1195a0f22b..725796d6275 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -707,10 +707,10 @@ async fn enforce_blossom_read_membership( /// Channel gate for media reads, after the community membership gate. /// /// A blob attached in a channel is readable only through an attachment the -/// reader can see: one in a channel they can read, or a channel-less one. Blobs -/// never attached in a channel (avatars, custom emoji, unsent uploads) stay -/// readable to the whole community. A refusal is `NotFound` so a non-member -/// cannot probe whether a private attachment exists. +/// reader can see: one in a channel they can read, or a channel-less one. +/// Avatars, custom emoji, and blobs never attached in a channel (unsent +/// uploads) stay readable to the whole community. A refusal is `NotFound` so a +/// non-member cannot probe whether a private attachment exists. async fn enforce_blossom_read_channel_access( state: &AppState, tenant: &TenantContext, @@ -720,9 +720,10 @@ async fn enforce_blossom_read_channel_access( ) -> Result<(), MediaDenial> { let community = tenant.community(); let unavailable = |_| media_denial(MediaError::AuthorizationUnavailable, strictness); + let media_base = media_base_url_for_tenant(&state.config.relay_url, tenant.host()); let references = state .db - .get_blob_reference_channels(community, sha256) + .get_blob_reference_channels(community, sha256, &media_base) .await .map_err(unavailable)?; let Some(channels) = references.into_iter().collect::>>() else { @@ -732,19 +733,34 @@ async fn enforce_blossom_read_channel_access( return Ok(()); } let reader = reader.as_bytes(); - let accessible = state + let readable = |ids: &[uuid::Uuid]| channels.iter().any(|id| ids.contains(id)); + let cached = state .get_accessible_channel_ids_cached(community, reader) .await .map_err(unavailable)?; - if channels.iter().any(|id| accessible.contains(id)) { + if readable(&cached) { return Ok(()); } + // A cached miss may predate a join on another pod; confirm before refusing. + let live = state + .db + .get_accessible_channel_ids(community, reader) + .await + .map_err(unavailable)?; let readable_system = state .db .get_readable_system_channel_ids(community, reader) .await .map_err(unavailable)?; - if channels.iter().any(|id| readable_system.contains(id)) { + if readable(&live) || readable(&readable_system) { + return Ok(()); + } + if state + .db + .is_blob_publicly_referenced(community, sha256) + .await + .map_err(unavailable)? + { return Ok(()); } Err(MediaError::NotFound.into()) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 2b08ec6fcff..a7501eb63a5 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -845,7 +845,6 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { .unwrap(); let ok = client.send_event(event).await.unwrap(); assert!(ok.accepted, "attachment rejected: {:?}", ok.message); - client.disconnect().await.unwrap(); for method in [Method::GET, Method::HEAD] { assert_eq!( @@ -859,4 +858,19 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { "{method} by a channel member must still succeed" ); } + + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({ "picture": url }).to_string(), + ) + .sign_with_keys(&owner) + .unwrap(); + let ok = client.send_event(profile).await.unwrap(); + assert!(ok.accepted, "profile rejected: {:?}", ok.message); + client.disconnect().await.unwrap(); + assert_eq!( + read_status(&http, &outsider, Method::GET, url).await, + 200, + "an avatar stays community-readable even when also attached privately" + ); } From d8d166f146fefd9e998e6984d7bff76e4396fb01 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 13:01:07 +1100 Subject: [PATCH 20/48] refactor(relay): decide media reads in one query; index posters by hash Collapse the media channel gate into a single `can_read_blob` query: an attachment is readable when it is channel-less or in an open channel or one the reader is an active member of. This replaces the cached, live and system-channel lookups and has no cache staleness in either direction. Video posters are now found by an explicit `image-x ` imeta entry, validated to match the `image` url hash, instead of guessing every poster URL form. Clients emitting `image-x` follow once this relay is deployed; posters without it stay community-readable as before. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 114 +++++++++----------- crates/buzz-db/src/store/mod.rs | 2 +- crates/buzz-relay/src/api/media.rs | 78 +++----------- crates/buzz-relay/src/handlers/imeta.rs | 59 ++++++++-- 4 files changed, 112 insertions(+), 141 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 3c4595abdab..063f80a12a1 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -1,35 +1,30 @@ -//! Where an uploaded blob is referenced, for media read authorization. +//! Media read authorization from where a blob is referenced. use buzz_datastore_tracing::datastore_span; use sqlx::PgPool; -use uuid::Uuid; use crate::{error::Result, Db}; use buzz_core::CommunityId; -/// Image extensions a video poster (`imeta image`) may carry; ingest rejects others. -const POSTER_EXTS: [&str; 4] = ["jpg", "png", "gif", "webp"]; - -/// Distinct `channel_id`s of every stored event, deleted or not, that attaches -/// the blob `sha256`; `None` is a channel-less event. +/// Whether `reader` may read the blob `sha256`. +/// +/// A blob attached in a channel is readable only through an attachment the +/// reader can see: one in a channel they are an active member of or that is +/// open, or a channel-less one. Attachments count whether or not the event was +/// deleted. Blobs attached in no channel, current avatars, and custom emoji +/// stay readable to the whole community. /// -/// A blob is attached as an `imeta` file (`x `, required by ingest), as -/// a video poster (`image `, relative or on `media_base_url` as ingest -/// requires), or as a `link-preview` image or favicon hash. Containment is the -/// GIN-indexed prefilter; the positional check keeps the hash appearing in some -/// other field from counting as an attachment. -pub async fn get_blob_reference_channels( +/// A blob is attached as an `imeta` file (`x`) or video poster (`image-x`), or +/// as a `link-preview` image or favicon hash. Containment is the GIN-indexed +/// prefilter; the positional check keeps the hash in another field from +/// counting. +pub async fn can_read_blob( pool: &PgPool, community_id: CommunityId, sha256: &str, - media_base_url: &str, -) -> Result>> { - let base = media_base_url.trim_end_matches('/'); - let mut imeta_entries = vec![format!("x {sha256}")]; - for ext in POSTER_EXTS { - imeta_entries.push(format!("image /media/{sha256}.{ext}")); - imeta_entries.push(format!("image {base}/{sha256}.{ext}")); - } + reader: &[u8], +) -> Result { + let imeta_entries = vec![format!("x {sha256}"), format!("image-x {sha256}")]; let mut containments: Vec = imeta_entries .iter() .map(|entry| serde_json::json!([["imeta", entry]])) @@ -41,71 +36,60 @@ pub async fn get_blob_reference_channels( crate::observability::WriterOperation::Authorization, ) .await?; - let channels = sqlx::query_scalar( - "SELECT DISTINCT channel_id FROM events \ - WHERE community_id = $1 \ - AND tags @> ANY($2::jsonb[]) \ - AND EXISTS (SELECT 1 FROM jsonb_array_elements(tags) AS tag \ - WHERE (tag->>0 = 'imeta' AND tag ?| $3) \ - OR (tag->>0 = 'link-preview' AND $4 IN (tag->>8, tag->>10)))", + let readable = sqlx::query_scalar( + "WITH refs AS ( \ + SELECT DISTINCT channel_id FROM events \ + WHERE community_id = $1 \ + AND tags @> ANY($2::jsonb[]) \ + AND EXISTS (SELECT 1 FROM jsonb_array_elements(tags) AS tag \ + WHERE (tag->>0 = 'imeta' AND tag ?| $3) \ + OR (tag->>0 = 'link-preview' AND $4 IN (tag->>8, tag->>10))) \ + ) \ + SELECT NOT EXISTS (SELECT 1 FROM refs WHERE channel_id IS NOT NULL) \ + OR EXISTS (SELECT 1 FROM refs WHERE channel_id IS NULL) \ + OR EXISTS (SELECT 1 FROM refs r \ + JOIN channels c ON c.community_id = $1 AND c.id = r.channel_id \ + WHERE c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $5 AND cm.removed_at IS NULL)))", ) .bind(community_id.as_uuid()) .bind(containments) .bind(imeta_entries) .bind(sha256) - .fetch_all(&mut *connection) + .bind(reader) + .fetch_one(&mut *connection) .await?; - Ok(channels) -} + if readable { + return Ok(true); + } -/// Whether the blob `sha256` is a current profile avatar or in a live custom -/// emoji set, both of which every community member must be able to load. -/// -/// Neither reference is indexed by hash, so callers should consult this only -/// after the channel references alone would refuse a read. -pub async fn is_blob_publicly_referenced( - pool: &PgPool, - community_id: CommunityId, - sha256: &str, -) -> Result { - let pattern = format!("%{sha256}%"); - let mut connection = crate::observability::acquire_writer( - pool, - crate::observability::WriterOperation::Authorization, - ) - .await?; - let referenced = sqlx::query_scalar( + // Avatars and emoji are not indexed by hash, so only a would-be refusal pays + // for this scan. + let referenced_publicly = sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM users WHERE community_id = $1 AND avatar_url LIKE $2) \ OR EXISTS (SELECT 1 FROM events WHERE community_id = $1 AND kind = $3 \ AND deleted_at IS NULL AND tags::text LIKE $2)", ) .bind(community_id.as_uuid()) - .bind(pattern) + .bind(format!("%{sha256}%")) .bind(buzz_core::kind::KIND_EMOJI_SET as i32) .fetch_one(&mut *connection) .await?; - Ok(referenced) + Ok(referenced_publicly) } impl Db { - /// Channels the blob `sha256` is attached in; see [`get_blob_reference_channels`]. - #[datastore_span(name = "get_blob_reference_channels", system = "postgresql")] - pub async fn get_blob_reference_channels( - &self, - community_id: CommunityId, - sha256: &str, - media_base_url: &str, - ) -> Result>> { - get_blob_reference_channels(&self.pool, community_id, sha256, media_base_url).await - } - - /// Whether the blob is an avatar or custom emoji; see [`is_blob_publicly_referenced`]. - #[datastore_span(name = "is_blob_publicly_referenced", system = "postgresql")] - pub async fn is_blob_publicly_referenced( + /// Whether `reader` may read the blob; see [`can_read_blob`]. + #[datastore_span(name = "can_read_blob", system = "postgresql")] + pub async fn can_read_blob( &self, community_id: CommunityId, sha256: &str, + reader: &[u8], ) -> Result { - is_blob_publicly_referenced(&self.pool, community_id, sha256).await + can_read_blob(&self.pool, community_id, sha256, reader).await } } diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index b3f74aa132e..fda211b82ea 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,7 +32,7 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; -/// Channels an uploaded blob is attached in, for media read authorization. +/// Media read authorization from where a blob is referenced. pub mod media_reference; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 725796d6275..470e0e265c2 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -704,69 +704,10 @@ async fn enforce_blossom_read_membership( .map_err(|e| membership_denial(e, strictness)) } -/// Channel gate for media reads, after the community membership gate. -/// -/// A blob attached in a channel is readable only through an attachment the -/// reader can see: one in a channel they can read, or a channel-less one. -/// Avatars, custom emoji, and blobs never attached in a channel (unsent -/// uploads) stay readable to the whole community. A refusal is `NotFound` so a -/// non-member cannot probe whether a private attachment exists. -async fn enforce_blossom_read_channel_access( - state: &AppState, - tenant: &TenantContext, - reader: &nostr::PublicKey, - sha256: &str, - strictness: BlossomStrictness, -) -> Result<(), MediaDenial> { - let community = tenant.community(); - let unavailable = |_| media_denial(MediaError::AuthorizationUnavailable, strictness); - let media_base = media_base_url_for_tenant(&state.config.relay_url, tenant.host()); - let references = state - .db - .get_blob_reference_channels(community, sha256, &media_base) - .await - .map_err(unavailable)?; - let Some(channels) = references.into_iter().collect::>>() else { - return Ok(()); - }; - if channels.is_empty() { - return Ok(()); - } - let reader = reader.as_bytes(); - let readable = |ids: &[uuid::Uuid]| channels.iter().any(|id| ids.contains(id)); - let cached = state - .get_accessible_channel_ids_cached(community, reader) - .await - .map_err(unavailable)?; - if readable(&cached) { - return Ok(()); - } - // A cached miss may predate a join on another pod; confirm before refusing. - let live = state - .db - .get_accessible_channel_ids(community, reader) - .await - .map_err(unavailable)?; - let readable_system = state - .db - .get_readable_system_channel_ids(community, reader) - .await - .map_err(unavailable)?; - if readable(&live) || readable(&readable_system) { - return Ok(()); - } - if state - .db - .is_blob_publicly_referenced(community, sha256) - .await - .map_err(unavailable)? - { - return Ok(()); - } - Err(MediaError::NotFound.into()) -} - -/// Every read gate a Blossom GET/HEAD passes before any storage lookup. +/// Every read gate a Blossom GET/HEAD passes before any storage lookup: +/// community membership, then the channels the blob is attached in. A channel +/// refusal is `NotFound` so a non-member cannot probe whether a private +/// attachment exists. async fn enforce_blossom_read_access( state: &AppState, tenant: &TenantContext, @@ -776,7 +717,16 @@ async fn enforce_blossom_read_access( strictness: BlossomStrictness, ) -> Result<(), MediaDenial> { enforce_blossom_read_membership(state, tenant, auth_event, headers, strictness).await?; - enforce_blossom_read_channel_access(state, tenant, &auth_event.pubkey, sha256, strictness).await + let readable = state + .db + .can_read_blob(tenant.community(), sha256, auth_event.pubkey.as_bytes()) + .await + .map_err(|_| media_denial(MediaError::AuthorizationUnavailable, strictness))?; + if readable { + Ok(()) + } else { + Err(MediaError::NotFound.into()) + } } fn blob_cache_control() -> &'static str { diff --git a/crates/buzz-relay/src/handlers/imeta.rs b/crates/buzz-relay/src/handlers/imeta.rs index e3d564e448a..8ac401e0306 100644 --- a/crates/buzz-relay/src/handlers/imeta.rs +++ b/crates/buzz-relay/src/handlers/imeta.rs @@ -10,11 +10,11 @@ use buzz_media::validation::mime_to_ext; pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result<(), String> { const ALLOWED_IMETA_KEYS: &[&str] = &[ "url", "m", "x", "size", "dim", "blurhash", "alt", "thumb", "fallback", "duration", - "bitrate", "image", "filename", + "bitrate", "image", "image-x", "filename", ]; const SINGLETON_KEYS: &[&str] = &[ "url", "m", "x", "size", "dim", "blurhash", "thumb", "alt", "duration", "bitrate", "image", - "filename", + "image-x", "filename", ]; // Previewable media MIME types — these get the strict url-extension // consistency check below (their ext is derived from the MIME). Generic @@ -43,6 +43,8 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result let mut x_value = String::new(); let mut m_value = String::new(); let mut thumb_value = String::new(); + let mut image_value = String::new(); + let mut image_x_value = String::new(); for part in tag.iter().skip(1) { let mut parts = part.splitn(2, ' '); @@ -82,14 +84,18 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result has_m = true; } "x" => { - if value.len() != 64 - || !value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) - { + if !is_sha256_hex(value) { return Err("imeta x must be a 64-char lowercase hex SHA-256".into()); } x_value = value.to_string(); has_x = true; } + "image-x" => { + if !is_sha256_hex(value) { + return Err("imeta image-x must be a 64-char lowercase hex SHA-256".into()); + } + image_x_value = value.to_string(); + } "size" => { match value.parse::() { Ok(0) | Err(_) => { @@ -136,6 +142,7 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result .into(), ); } + image_value = value.to_string(); } "filename" => { // Original filename for the file-card label. Bounded length; @@ -165,7 +172,7 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result // Video-only NIP-71 fields must not appear on image blobs. let is_video = m_value == "video/mp4"; if !is_video { - for key in &["duration", "bitrate", "image"] { + for key in &["duration", "bitrate", "image", "image-x"] { if seen_keys.contains(*key) { return Err(format!( "imeta {key} is only valid for video/mp4, not {m_value}" @@ -200,6 +207,13 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result } } } + // `image-x` lets media reads find a poster by hash; it must name the + // poster that `image` points at. + if !image_x_value.is_empty() + && extract_hash_from_media_url(&image_value) != Some(image_x_value.as_str()) + { + return Err("imeta image-x must match the image url hash".into()); + } } Ok(()) } @@ -347,15 +361,15 @@ fn is_well_formed_mime(mime: &str) -> bool { && !mime.chars().any(|c| c.is_whitespace() || c.is_control()) } +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) +} + /// Extract the 64-char hex hash from a `/media/{hash}.{ext}` URL. fn extract_hash_from_media_url(url: &str) -> Option<&str> { let after = url.rsplit("/media/").next()?; let hash = after.split('.').next()?; - if hash.len() == 64 && hash.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) { - Some(hash) - } else { - None - } + is_sha256_hex(hash).then_some(hash) } /// Extract the primary extension from a `/media/{hash}.{ext}` URL (not thumb). @@ -572,6 +586,29 @@ mod tests { assert!(err.contains("url hash does not match x"), "{err}"); } + fn video_with_poster(poster_x: Option<&str>) -> Vec { + let poster = "c".repeat(64); + let mut tag = vec![ + "imeta".into(), + format!("url /media/{HASH}.mp4"), + "m video/mp4".into(), + format!("x {HASH}"), + "size 100".into(), + format!("image /media/{poster}.jpg"), + ]; + tag.extend(poster_x.map(|x| format!("image-x {x}"))); + tag + } + + #[test] + fn test_imeta_image_x_must_name_the_poster() { + let poster = "c".repeat(64); + assert!(validate_imeta_tags(&[video_with_poster(Some(&poster))], BASE).is_ok()); + assert!(validate_imeta_tags(&[video_with_poster(None)], BASE).is_ok()); + let err = validate_imeta_tags(&[video_with_poster(Some(HASH))], BASE).unwrap_err(); + assert!(err.contains("image-x must match"), "{err}"); + } + #[test] fn test_imeta_generic_file_with_filename_passes() { // Generic file attachment: non-media MIME, arbitrary ext, filename label. From 8cefaca38e6972ca660dfeeca00fc2c8f8f1f779 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 13:33:40 +1100 Subject: [PATCH 21/48] fix(relay): only readers may attach media; uploaders may always read Knowing a blob's hash (it is in every media URL) was enough for any community member to attach it in a channel-less note, link preview, avatar or emoji set and make a private attachment community-readable. Ingest now admits an attachment (imeta `x`/`image-x`, link-preview image/favicon) only when its author can read the blob, and reads always admit the blob's uploader, since uploading the exact bytes proves possession. The avatar and custom-emoji exemption is removed: it could be triggered by any text containing the hash. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 34 ++++-------- crates/buzz-relay/src/handlers/imeta.rs | 45 ++++++++++++++++ crates/buzz-relay/src/handlers/ingest.rs | 15 ++++++ .../tests/e2e_media_extended.rs | 53 +++++++++++-------- 4 files changed, 101 insertions(+), 46 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 063f80a12a1..7132aaffa2e 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -8,11 +8,13 @@ use buzz_core::CommunityId; /// Whether `reader` may read the blob `sha256`. /// -/// A blob attached in a channel is readable only through an attachment the -/// reader can see: one in a channel they are an active member of or that is -/// open, or a channel-less one. Attachments count whether or not the event was -/// deleted. Blobs attached in no channel, current avatars, and custom emoji -/// stay readable to the whole community. +/// The uploader can always read it: uploading the exact bytes proves +/// possession. Otherwise a blob attached in a channel is readable only through +/// an attachment the reader can see: one in a channel they are an active member +/// of or that is open, or a channel-less one. Attachments count whether or not +/// the event was deleted; blobs attached in no channel stay readable to the +/// whole community. Ingest only admits an attachment whose author can read the +/// blob, so knowing a hash is not enough to widen who can read it. /// /// A blob is attached as an `imeta` file (`x`) or video poster (`image-x`), or /// as a `link-preview` image or favicon hash. Containment is the GIN-indexed @@ -45,7 +47,9 @@ pub async fn can_read_blob( WHERE (tag->>0 = 'imeta' AND tag ?| $3) \ OR (tag->>0 = 'link-preview' AND $4 IN (tag->>8, tag->>10))) \ ) \ - SELECT NOT EXISTS (SELECT 1 FROM refs WHERE channel_id IS NOT NULL) \ + SELECT EXISTS (SELECT 1 FROM media_uploaders \ + WHERE community_id = $1 AND sha256 = $4 AND pubkey = $5) \ + OR NOT EXISTS (SELECT 1 FROM refs WHERE channel_id IS NOT NULL) \ OR EXISTS (SELECT 1 FROM refs WHERE channel_id IS NULL) \ OR EXISTS (SELECT 1 FROM refs r \ JOIN channels c ON c.community_id = $1 AND c.id = r.channel_id \ @@ -62,23 +66,7 @@ pub async fn can_read_blob( .bind(reader) .fetch_one(&mut *connection) .await?; - if readable { - return Ok(true); - } - - // Avatars and emoji are not indexed by hash, so only a would-be refusal pays - // for this scan. - let referenced_publicly = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM users WHERE community_id = $1 AND avatar_url LIKE $2) \ - OR EXISTS (SELECT 1 FROM events WHERE community_id = $1 AND kind = $3 \ - AND deleted_at IS NULL AND tags::text LIKE $2)", - ) - .bind(community_id.as_uuid()) - .bind(format!("%{sha256}%")) - .bind(buzz_core::kind::KIND_EMOJI_SET as i32) - .fetch_one(&mut *connection) - .await?; - Ok(referenced_publicly) + Ok(readable) } impl Db { diff --git a/crates/buzz-relay/src/handlers/imeta.rs b/crates/buzz-relay/src/handlers/imeta.rs index 8ac401e0306..a5136eacf36 100644 --- a/crates/buzz-relay/src/handlers/imeta.rs +++ b/crates/buzz-relay/src/handlers/imeta.rs @@ -361,6 +361,27 @@ fn is_well_formed_mime(mime: &str) -> bool { && !mime.chars().any(|c| c.is_whitespace() || c.is_control()) } +/// Hashes of every blob `event` attaches, as `buzz_db::store::media_reference` +/// counts them: imeta `x` and `image-x`, and link-preview image and favicon. +pub fn attached_blob_hashes(event: &nostr::Event) -> std::collections::BTreeSet { + let mut hashes = std::collections::BTreeSet::new(); + for tag in event.tags.iter() { + let parts = tag.as_slice(); + match parts.first().map(String::as_str) { + Some("imeta") => hashes.extend(parts.iter().skip(1).filter_map(|part| { + let (key, value) = part.split_once(' ')?; + matches!(key, "x" | "image-x").then(|| value.to_owned()) + })), + Some("link-preview") => { + hashes.extend([8, 10].iter().filter_map(|&i| parts.get(i).cloned())); + } + _ => {} + } + } + hashes.retain(|hash| is_sha256_hex(hash)); + hashes +} + fn is_sha256_hex(value: &str) -> bool { value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) } @@ -609,6 +630,30 @@ mod tests { assert!(err.contains("image-x must match"), "{err}"); } + #[test] + fn attached_blob_hashes_reads_only_attachment_positions() { + let poster = "c".repeat(64); + let preview_image = "d".repeat(64); + let mut preview = vec!["link-preview".to_owned(); 11]; + preview[4] = "f".repeat(64); + preview[8] = preview_image.clone(); + preview[10] = String::new(); + let event = nostr::EventBuilder::new(nostr::Kind::TextNote, "") + .tags([ + nostr::Tag::parse(video_with_poster(Some(&poster))).unwrap(), + nostr::Tag::parse(preview).unwrap(), + nostr::Tag::parse(["x", &"e".repeat(64)]).unwrap(), + ]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + assert_eq!( + attached_blob_hashes(&event), + [HASH.to_owned(), poster, preview_image] + .into_iter() + .collect() + ); + } + #[test] fn test_imeta_generic_file_with_filename_passes() { // Generic file attachment: non-media MIME, arbitrary ext, filename label. diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 0b2e8d8f096..d1cbb55982a 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -2554,6 +2554,21 @@ async fn ingest_event_inner( // (commands, feedback, reports, moderation) so no write returns ahead of it. enforce_write_restriction(state, tenant, kind_u32, auth.pubkey()).await?; + // An attachment widens who can read its blob, so only an author who can + // already read the blob may attach it. Also before every kind branch. + for sha256 in super::imeta::attached_blob_hashes(&event) { + let readable = state + .db + .can_read_blob(tenant.community(), &sha256, event.pubkey.as_bytes()) + .await + .map_err(|e| IngestError::Internal(format!("error: checking attachment: {e}")))?; + if !readable { + return Err(IngestError::Rejected( + "invalid: attached media is not readable by the author".into(), + )); + } + } + // Command kinds are routed AFTER signature verification, timestamp check, // pubkey/auth match, and scope validation — never before. if buzz_core::kind::is_command_kind(kind_u32) { diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index a7501eb63a5..be9a9026618 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -826,25 +826,25 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { "a blob not yet attached anywhere stays community-readable" ); - let mut client = BuzzTestClient::connect(&relay_ws_url(), &owner) + let imeta = Tag::parse([ + "imeta", + &format!("url {url}"), + &format!("m {}", desc["type"].as_str().unwrap()), + &format!("x {}", desc["sha256"].as_str().unwrap()), + &format!("size {}", desc["size"]), + ]) + .unwrap(); + + let mut owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) .await .unwrap(); let event = EventBuilder::new(Kind::from(9), "private attachment") - .tags(vec![ - Tag::parse(["h", &channel_id]).unwrap(), - Tag::parse([ - "imeta", - &format!("url {url}"), - &format!("m {}", desc["type"].as_str().unwrap()), - &format!("x {}", desc["sha256"].as_str().unwrap()), - &format!("size {}", desc["size"]), - ]) - .unwrap(), - ]) + .tags(vec![Tag::parse(["h", &channel_id]).unwrap(), imeta.clone()]) .sign_with_keys(&owner) .unwrap(); - let ok = client.send_event(event).await.unwrap(); + let ok = owner_client.send_event(event).await.unwrap(); assert!(ok.accepted, "attachment rejected: {:?}", ok.message); + owner_client.disconnect().await.unwrap(); for method in [Method::GET, Method::HEAD] { assert_eq!( @@ -859,18 +859,25 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { ); } - let profile = EventBuilder::new( - Kind::Metadata, - serde_json::json!({ "picture": url }).to_string(), - ) - .sign_with_keys(&owner) - .unwrap(); - let ok = client.send_event(profile).await.unwrap(); - assert!(ok.accepted, "profile rejected: {:?}", ok.message); - client.disconnect().await.unwrap(); + let mut outsider_client = BuzzTestClient::connect(&relay_ws_url(), &outsider) + .await + .unwrap(); + let reshare = EventBuilder::new(Kind::TextNote, "reshare by hash") + .tags(vec![imeta]) + .sign_with_keys(&outsider) + .unwrap(); + let ok = outsider_client.send_event(reshare).await.unwrap(); + assert!( + !ok.accepted, + "knowing a hash must not let a non-member attach the blob publicly" + ); + outsider_client.disconnect().await.unwrap(); + assert_eq!(read_status(&http, &outsider, Method::GET, url).await, 404); + + assert_eq!(upload(&http, &outsider, &body).await.status(), 200); assert_eq!( read_status(&http, &outsider, Method::GET, url).await, 200, - "an avatar stays community-readable even when also attached privately" + "uploading the same bytes proves possession" ); } From 7efeb6912e3f631a089b261c6144ba600196c77a Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 13:45:16 +1100 Subject: [PATCH 22/48] fix(relay): keep avatars and emoji public; check all media references in one query Any member could hide someone else's avatar or custom emoji by attaching the public blob in their own private channel. Current avatars and live emoji sets and lists now keep a blob community-readable, and ingest applies the same author-must-read rule to them (any standalone 64-hex token in a profile or emoji event), so a public reference can neither hide nor leak a blob. Ingest checks every hash an event references in one batched query instead of one query per hash. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 103 ++++++++++-------- crates/buzz-relay/src/api/media.rs | 6 +- crates/buzz-relay/src/handlers/imeta.rs | 60 ++++++++-- crates/buzz-relay/src/handlers/ingest.rs | 23 ++-- .../tests/e2e_media_extended.rs | 73 +++++++++++-- 5 files changed, 190 insertions(+), 75 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 7132aaffa2e..0fe1a62f185 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -4,80 +4,91 @@ use buzz_datastore_tracing::datastore_span; use sqlx::PgPool; use crate::{error::Result, Db}; +use buzz_core::kind::{KIND_EMOJI_LIST, KIND_EMOJI_SET}; use buzz_core::CommunityId; -/// Whether `reader` may read the blob `sha256`. +/// Whether `reader` may read every blob in `sha256s` (vacuously true if empty). /// -/// The uploader can always read it: uploading the exact bytes proves -/// possession. Otherwise a blob attached in a channel is readable only through -/// an attachment the reader can see: one in a channel they are an active member -/// of or that is open, or a channel-less one. Attachments count whether or not -/// the event was deleted; blobs attached in no channel stay readable to the -/// whole community. Ingest only admits an attachment whose author can read the -/// blob, so knowing a hash is not enough to widen who can read it. +/// A blob is readable by its uploader (uploading the exact bytes proves +/// possession), and by everyone while it is a current avatar or in a live custom +/// emoji set or list. Otherwise a blob attached in a channel is readable only +/// through an attachment the reader can see: one in a channel they are an +/// active member of or that is open, or a channel-less one. Attachments count +/// whether or not the event was deleted; a blob attached nowhere stays readable +/// to the whole community. /// /// A blob is attached as an `imeta` file (`x`) or video poster (`image-x`), or -/// as a `link-preview` image or favicon hash. Containment is the GIN-indexed -/// prefilter; the positional check keeps the hash in another field from -/// counting. -pub async fn can_read_blob( +/// as a `link-preview` image or favicon hash; containment is the GIN-indexed +/// prefilter and the positional check keeps the hash in another field from +/// counting. Avatars and emoji name a blob by a standalone 64-hex token. Ingest +/// admits every such reference only if its author can already read the blob, so +/// knowing a hash is not enough to change who can read it. +pub async fn can_read_blobs( pool: &PgPool, community_id: CommunityId, - sha256: &str, + sha256s: &[String], reader: &[u8], ) -> Result { - let imeta_entries = vec![format!("x {sha256}"), format!("image-x {sha256}")]; - let mut containments: Vec = imeta_entries - .iter() - .map(|entry| serde_json::json!([["imeta", entry]])) - .collect(); - containments.push(serde_json::json!([["link-preview", sha256]])); - let mut connection = crate::observability::acquire_writer( pool, crate::observability::WriterOperation::Authorization, ) .await?; let readable = sqlx::query_scalar( - "WITH refs AS ( \ - SELECT DISTINCT channel_id FROM events \ - WHERE community_id = $1 \ - AND tags @> ANY($2::jsonb[]) \ - AND EXISTS (SELECT 1 FROM jsonb_array_elements(tags) AS tag \ - WHERE (tag->>0 = 'imeta' AND tag ?| $3) \ - OR (tag->>0 = 'link-preview' AND $4 IN (tag->>8, tag->>10))) \ + "WITH blobs AS (SELECT DISTINCT unnest($2::text[]) AS sha256), \ + refs AS ( \ + SELECT b.sha256, e.channel_id FROM blobs b \ + JOIN events e ON e.community_id = $1 \ + AND (e.tags @> jsonb_build_array(jsonb_build_array('imeta', 'x ' || b.sha256)) \ + OR e.tags @> jsonb_build_array(jsonb_build_array('imeta', 'image-x ' || b.sha256)) \ + OR e.tags @> jsonb_build_array(jsonb_build_array('link-preview', b.sha256))) \ + AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) AS tag \ + WHERE (tag->>0 = 'imeta' \ + AND tag ?| ARRAY['x ' || b.sha256, 'image-x ' || b.sha256]) \ + OR (tag->>0 = 'link-preview' \ + AND b.sha256 IN (tag->>8, tag->>10))) \ ) \ - SELECT EXISTS (SELECT 1 FROM media_uploaders \ - WHERE community_id = $1 AND sha256 = $4 AND pubkey = $5) \ - OR NOT EXISTS (SELECT 1 FROM refs WHERE channel_id IS NOT NULL) \ - OR EXISTS (SELECT 1 FROM refs WHERE channel_id IS NULL) \ - OR EXISTS (SELECT 1 FROM refs r \ - JOIN channels c ON c.community_id = $1 AND c.id = r.channel_id \ - WHERE c.deleted_at IS NULL \ - AND (c.visibility = 'open' \ - OR EXISTS (SELECT 1 FROM channel_members cm \ - WHERE cm.community_id = $1 AND cm.channel_id = c.id \ - AND cm.pubkey = $5 AND cm.removed_at IS NULL)))", + SELECT coalesce(bool_and(CASE \ + WHEN EXISTS (SELECT 1 FROM media_uploaders u WHERE u.community_id = $1 \ + AND u.sha256 = b.sha256 AND u.pubkey = $3) THEN true \ + WHEN NOT EXISTS (SELECT 1 FROM refs r \ + WHERE r.sha256 = b.sha256 AND r.channel_id IS NOT NULL) THEN true \ + WHEN EXISTS (SELECT 1 FROM refs r \ + WHERE r.sha256 = b.sha256 AND r.channel_id IS NULL) THEN true \ + WHEN EXISTS (SELECT 1 FROM refs r \ + JOIN channels c ON c.community_id = $1 AND c.id = r.channel_id \ + WHERE r.sha256 = b.sha256 AND c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $3 AND cm.removed_at IS NULL))) \ + THEN true \ + ELSE EXISTS (SELECT 1 FROM users WHERE community_id = $1 \ + AND avatar_url ~ ('(^|[^0-9a-f])' || b.sha256 || '([^0-9a-f]|$)')) \ + OR EXISTS (SELECT 1 FROM events WHERE community_id = $1 AND kind IN ($4, $5) \ + AND deleted_at IS NULL \ + AND tags::text ~ ('(^|[^0-9a-f])' || b.sha256 || '([^0-9a-f]|$)')) \ + END), true) FROM blobs b", ) .bind(community_id.as_uuid()) - .bind(containments) - .bind(imeta_entries) - .bind(sha256) + .bind(sha256s) .bind(reader) + .bind(KIND_EMOJI_SET as i32) + .bind(KIND_EMOJI_LIST as i32) .fetch_one(&mut *connection) .await?; Ok(readable) } impl Db { - /// Whether `reader` may read the blob; see [`can_read_blob`]. - #[datastore_span(name = "can_read_blob", system = "postgresql")] - pub async fn can_read_blob( + /// Whether `reader` may read every listed blob; see [`can_read_blobs`]. + #[datastore_span(name = "can_read_blobs", system = "postgresql")] + pub async fn can_read_blobs( &self, community_id: CommunityId, - sha256: &str, + sha256s: &[String], reader: &[u8], ) -> Result { - can_read_blob(&self.pool, community_id, sha256, reader).await + can_read_blobs(&self.pool, community_id, sha256s, reader).await } } diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 210ba49fe1d..d5264ee062c 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -729,7 +729,11 @@ async fn enforce_blossom_read_access( enforce_blossom_read_membership(state, tenant, auth_event, headers, strictness).await?; let readable = state .db - .can_read_blob(tenant.community(), sha256, auth_event.pubkey.as_bytes()) + .can_read_blobs( + tenant.community(), + &[sha256.to_owned()], + auth_event.pubkey.as_bytes(), + ) .await .map_err(|_| media_denial(MediaError::AuthorizationUnavailable, strictness))?; if readable { diff --git a/crates/buzz-relay/src/handlers/imeta.rs b/crates/buzz-relay/src/handlers/imeta.rs index a5136eacf36..abbd4f7d49a 100644 --- a/crates/buzz-relay/src/handlers/imeta.rs +++ b/crates/buzz-relay/src/handlers/imeta.rs @@ -361,12 +361,22 @@ fn is_well_formed_mime(mime: &str) -> bool { && !mime.chars().any(|c| c.is_whitespace() || c.is_control()) } -/// Hashes of every blob `event` attaches, as `buzz_db::store::media_reference` -/// counts them: imeta `x` and `image-x`, and link-preview image and favicon. -pub fn attached_blob_hashes(event: &nostr::Event) -> std::collections::BTreeSet { +/// Hashes of every blob `event` references, as `buzz_db::store::media_reference` +/// counts them: imeta `x` and `image-x`, link-preview image and favicon, and any +/// standalone 64-hex token in a profile's content or an emoji set or list's tags. +pub fn referenced_blob_hashes(event: &nostr::Event) -> Vec { + use buzz_core::kind::{KIND_EMOJI_LIST, KIND_EMOJI_SET, KIND_PROFILE}; + let kind = u32::from(event.kind.as_u16()); + let public_reference = kind == KIND_EMOJI_SET || kind == KIND_EMOJI_LIST; let mut hashes = std::collections::BTreeSet::new(); + if kind == KIND_PROFILE { + hashes.extend(hex64_tokens(&event.content)); + } for tag in event.tags.iter() { let parts = tag.as_slice(); + if public_reference { + hashes.extend(parts.iter().flat_map(|part| hex64_tokens(part))); + } match parts.first().map(String::as_str) { Some("imeta") => hashes.extend(parts.iter().skip(1).filter_map(|part| { let (key, value) = part.split_once(' ')?; @@ -378,8 +388,17 @@ pub fn attached_blob_hashes(event: &nostr::Event) -> std::collections::BTreeSet< _ => {} } } - hashes.retain(|hash| is_sha256_hex(hash)); hashes + .into_iter() + .filter(|hash| is_sha256_hex(hash)) + .collect() +} + +/// Maximal runs of exactly 64 lowercase hex characters in `text`. +fn hex64_tokens(text: &str) -> impl Iterator + '_ { + text.split(|c: char| !matches!(c, '0'..='9' | 'a'..='f')) + .filter(|token| token.len() == 64) + .map(str::to_owned) } fn is_sha256_hex(value: &str) -> bool { @@ -631,7 +650,7 @@ mod tests { } #[test] - fn attached_blob_hashes_reads_only_attachment_positions() { + fn referenced_blob_hashes_reads_only_attachment_positions() { let poster = "c".repeat(64); let preview_image = "d".repeat(64); let mut preview = vec!["link-preview".to_owned(); 11]; @@ -647,13 +666,36 @@ mod tests { .sign_with_keys(&nostr::Keys::generate()) .unwrap(); assert_eq!( - attached_blob_hashes(&event), - [HASH.to_owned(), poster, preview_image] - .into_iter() - .collect() + referenced_blob_hashes(&event), + vec![HASH.to_owned(), poster, preview_image] ); } + #[test] + fn referenced_blob_hashes_covers_avatars_and_emoji() { + let avatar = "c".repeat(64); + let emoji = "d".repeat(64); + let keys = nostr::Keys::generate(); + let profile = nostr::EventBuilder::new( + nostr::Kind::Metadata, + format!(r#"{{"picture":"https://relay.example/media/{avatar}.jpg"}}"#), + ) + .sign_with_keys(&keys) + .unwrap(); + assert_eq!(referenced_blob_hashes(&profile), vec![avatar]); + + let emoji_set = nostr::EventBuilder::new(nostr::Kind::from(30030), "") + .tags([nostr::Tag::parse([ + "emoji", + "party", + &format!("https://relay.example/media/{emoji}.png"), + ]) + .unwrap()]) + .sign_with_keys(&keys) + .unwrap(); + assert_eq!(referenced_blob_hashes(&emoji_set), vec![emoji]); + } + #[test] fn test_imeta_generic_file_with_filename_passes() { // Generic file attachment: non-media MIME, arbitrary ext, filename label. diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index d1cbb55982a..8b137499da4 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -2554,19 +2554,20 @@ async fn ingest_event_inner( // (commands, feedback, reports, moderation) so no write returns ahead of it. enforce_write_restriction(state, tenant, kind_u32, auth.pubkey()).await?; - // An attachment widens who can read its blob, so only an author who can - // already read the blob may attach it. Also before every kind branch. - for sha256 in super::imeta::attached_blob_hashes(&event) { - let readable = state + // A media reference changes who can read its blob, so only an author who can + // already read every referenced blob may publish it. Also before every kind + // branch, and one query however many blobs the event names. + let referenced = super::imeta::referenced_blob_hashes(&event); + if !referenced.is_empty() + && !state .db - .can_read_blob(tenant.community(), &sha256, event.pubkey.as_bytes()) + .can_read_blobs(tenant.community(), &referenced, event.pubkey.as_bytes()) .await - .map_err(|e| IngestError::Internal(format!("error: checking attachment: {e}")))?; - if !readable { - return Err(IngestError::Rejected( - "invalid: attached media is not readable by the author".into(), - )); - } + .map_err(|e| IngestError::Internal(format!("error: checking media: {e}")))? + { + return Err(IngestError::Rejected( + "invalid: referenced media is not readable by the author".into(), + )); } // Command kinds are routed AFTER signature verification, timestamp check, diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index be9a9026618..ea70a8bef5a 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -826,14 +826,7 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { "a blob not yet attached anywhere stays community-readable" ); - let imeta = Tag::parse([ - "imeta", - &format!("url {url}"), - &format!("m {}", desc["type"].as_str().unwrap()), - &format!("x {}", desc["sha256"].as_str().unwrap()), - &format!("size {}", desc["size"]), - ]) - .unwrap(); + let imeta = imeta_for(&desc); let mut owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) .await @@ -881,3 +874,67 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { "uploading the same bytes proves possession" ); } + +/// NIP-92 `imeta` tag attaching the blob an upload `desc` describes. +fn imeta_for(desc: &serde_json::Value) -> Tag { + Tag::parse([ + "imeta", + &format!("url {}", desc["url"].as_str().unwrap()), + &format!("m {}", desc["type"].as_str().unwrap()), + &format!("x {}", desc["sha256"].as_str().unwrap()), + &format!("size {}", desc["size"]), + ]) + .unwrap() +} + +#[tokio::test] +#[ignore] +async fn test_private_attachment_cannot_hide_an_avatar() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let owner = Keys::generate(); + let griefer = Keys::generate(); + let viewer = Keys::generate(); + let http = http_client(); + + let body = uuid::Uuid::new_v4().as_bytes().repeat(8); + let resp = upload(&http, &owner, &body).await; + assert_eq!(resp.status(), 200); + let desc: serde_json::Value = resp.json().await.unwrap(); + let url = desc["url"].as_str().expect("descriptor url"); + + let mut owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({ "picture": url }).to_string(), + ) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_client.send_event(profile).await.unwrap(); + assert!(ok.accepted, "profile rejected: {:?}", ok.message); + owner_client.disconnect().await.unwrap(); + + let channel_id = create_channel(&http, &griefer, "hide-avatar", "private").await; + let mut griefer_client = BuzzTestClient::connect(&relay_ws_url(), &griefer) + .await + .unwrap(); + let event = EventBuilder::new(Kind::from(9), "attach someone's avatar") + .tags(vec![ + Tag::parse(["h", &channel_id]).unwrap(), + imeta_for(&desc), + ]) + .sign_with_keys(&griefer) + .unwrap(); + let ok = griefer_client.send_event(event).await.unwrap(); + assert!(ok.accepted, "attachment rejected: {:?}", ok.message); + griefer_client.disconnect().await.unwrap(); + + assert_eq!( + read_status(&http, &viewer, Method::GET, url).await, + 200, + "a private attachment must not hide a current avatar" + ); +} From 784f6ae70678158b0594b7183856d79b975f3833 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 16:31:01 +1100 Subject: [PATCH 23/48] refactor(relay): authorize media reads from an explicit media_shares table Replace the per-read scan of event tags, avatars and emoji sets with a `media_shares` table (migration 0068): every blob an event references is shared into the event's channel, or with the whole community when it is channel-less, written by the shared event follow-up in the same transaction as the event insert. A read is now indexed lookups: the uploader, a community-wide share, no channel share at all, or a share in a channel the reader can see. Reference extraction moves to `buzz_core::media` so ingest's author-can-read check and the share writer use one definition. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-core/src/lib.rs | 2 + crates/buzz-core/src/media.rs | 107 ++++++++++++++++++++ crates/buzz-db/src/runtime/migration.rs | 16 ++- crates/buzz-db/src/store/channel_members.rs | 9 +- crates/buzz-db/src/store/deletion.rs | 3 + crates/buzz-db/src/store/event.rs | 16 +-- crates/buzz-db/src/store/event_follow_up.rs | 21 +++- crates/buzz-db/src/store/media_reference.rs | 99 +++++++++--------- crates/buzz-db/src/store/push.rs | 8 +- crates/buzz-db/src/store/relay_members.rs | 8 +- crates/buzz-db/src/store/replaceable.rs | 12 +-- crates/buzz-relay/src/handlers/imeta.rs | 92 +---------------- crates/buzz-relay/src/handlers/ingest.rs | 2 +- migrations/0068_media_shares.sql | 16 +++ schema/schema.sql | 2 + schema/tables/public/media_shares.sql | 15 +++ schema/triggers/media_shares.sql | 1 + 17 files changed, 236 insertions(+), 193 deletions(-) create mode 100644 crates/buzz-core/src/media.rs create mode 100644 migrations/0068_media_shares.sql create mode 100644 schema/tables/public/media_shares.sql create mode 100644 schema/triggers/media_shares.sql diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index e7669c874b0..0c89cd62c0f 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -24,6 +24,8 @@ pub mod git_perms; pub mod invite; /// Buzz kind number registry — custom event type constants. pub mod kind; +/// Which media blobs an event references, for channel-scoped media access. +pub mod media; /// Network utilities — SSRF-safe IP classification. pub mod network; /// NIP-10 thread-marker parsing — shared `root`/`reply` marker resolver. diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs new file mode 100644 index 00000000000..ede14def0c7 --- /dev/null +++ b/crates/buzz-core/src/media.rs @@ -0,0 +1,107 @@ +//! Which media blobs an event references, for channel-scoped media access. + +use std::collections::BTreeSet; + +use crate::kind::{KIND_EMOJI_LIST, KIND_EMOJI_SET, KIND_PROFILE}; + +/// Hashes of every blob `event` references, each of which shares the blob into +/// the event's channel (or with the whole community for a channel-less event): +/// imeta `x` and `image-x`, link-preview image and favicon, and any standalone +/// 64-hex token in a profile's content or an emoji set or list's tags. +pub fn referenced_blob_hashes(event: &nostr::Event) -> Vec { + let kind = u32::from(event.kind.as_u16()); + let emoji = kind == KIND_EMOJI_SET || kind == KIND_EMOJI_LIST; + let mut hashes = BTreeSet::new(); + if kind == KIND_PROFILE { + hashes.extend(hex64_tokens(&event.content)); + } + for tag in event.tags.iter() { + let parts = tag.as_slice(); + if emoji { + hashes.extend(parts.iter().flat_map(|part| hex64_tokens(part))); + } + match parts.first().map(String::as_str) { + Some("imeta") => hashes.extend(parts.iter().skip(1).filter_map(|part| { + let (key, value) = part.split_once(' ')?; + matches!(key, "x" | "image-x").then(|| value.to_owned()) + })), + Some("link-preview") => { + hashes.extend([8, 10].iter().filter_map(|&i| parts.get(i).cloned())); + } + _ => {} + } + } + hashes + .into_iter() + .filter(|hash| is_sha256_hex(hash)) + .collect() +} + +/// Whether `value` is a 64-char lowercase hex SHA-256. +pub fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) +} + +/// Maximal runs of exactly 64 lowercase hex characters in `text`. +fn hex64_tokens(text: &str) -> impl Iterator + '_ { + text.split(|c: char| !matches!(c, '0'..='9' | 'a'..='f')) + .filter(|token| token.len() == 64) + .map(str::to_owned) +} + +#[cfg(test)] +mod tests { + use super::*; + use nostr::{EventBuilder, Keys, Kind, Tag}; + + #[test] + fn reads_only_attachment_positions() { + let (video, poster, preview_image) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); + let mut preview = vec!["link-preview".to_owned(); 11]; + preview[4] = "f".repeat(64); + preview[8] = preview_image.clone(); + preview[10] = String::new(); + let event = EventBuilder::new(Kind::TextNote, "") + .tags([ + Tag::parse([ + "imeta", + &format!("url /media/{video}.mp4"), + &format!("x {video}"), + &format!("image-x {poster}"), + ]) + .unwrap(), + Tag::parse(preview).unwrap(), + Tag::parse(["x", &"e".repeat(64)]).unwrap(), + ]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + assert_eq!( + referenced_blob_hashes(&event), + vec![video, poster, preview_image] + ); + } + + #[test] + fn covers_avatars_and_emoji() { + let (avatar, emoji) = ("c".repeat(64), "d".repeat(64)); + let keys = Keys::generate(); + let profile = EventBuilder::new( + Kind::Metadata, + format!(r#"{{"picture":"https://relay.example/media/{avatar}.jpg"}}"#), + ) + .sign_with_keys(&keys) + .unwrap(); + assert_eq!(referenced_blob_hashes(&profile), vec![avatar]); + + let emoji_set = EventBuilder::new(Kind::from(30030), "") + .tags([Tag::parse([ + "emoji", + "party", + &format!("https://relay.example/media/{emoji}.png"), + ]) + .unwrap()]) + .sign_with_keys(&keys) + .unwrap(); + assert_eq!(referenced_blob_hashes(&emoji_set), vec![emoji]); + } +} diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 6f359ecfd5c..61ce323bc1c 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -709,7 +709,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 67); + assert_eq!(migrations.len(), 68); assert_eq!(migrations[64].version, 65); let retire_ttl = strip_sql_comments(migrations[64].sql.as_str()); for statement in [ @@ -751,6 +751,11 @@ mod postgres_tests { .sql .as_str() .contains("CREATE TABLE media_uploaders")); + assert_eq!(migrations[67].version, 68); + assert!(migrations[67] + .sql + .as_str() + .contains("CREATE TABLE media_shares")); assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2331,6 +2336,15 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('media_uploaders')")); expected_fences.insert("media_uploaders".to_owned()); + // 0068 fences the media shares table. + assert!(MIGRATOR + .iter() + .find(|m| m.version == 68) + .expect("media shares migration") + .sql + .as_str() + .contains("attach_community_write_fence('media_shares')")); + expected_fences.insert("media_shares".to_owned()); assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index 4855d32cccf..a3034cbc2a7 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -503,13 +503,8 @@ impl LockedMemberSnapshot { "member snapshot event id already exists".into(), )); } - crate::store::event_follow_up::after_admitted_insert( - &mut self.tx, - event.id.as_bytes().as_slice(), - kind, - Some(channel_id), - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut self.tx, event, Some(channel_id)) + .await?; crate::insert_mentions_in_transaction(&mut self.tx, event, Some(channel_id)).await?; Ok(( buzz_core::StoredEvent::with_received_at( diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 1d910493a84..c48816a39d1 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -90,6 +90,7 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "media_shares", "media_uploaders", "moderation_actions", "moderation_reports", @@ -157,6 +158,7 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "audit_log", "community_bans", "community_nip_fi_issuers", + "media_shares", "media_uploaders", "pubkey_allowlist", "relay_members", @@ -4425,6 +4427,7 @@ DROP TRIGGER IF EXISTS community_write_fence_event_mentions ON event_mentions; DROP TRIGGER IF EXISTS community_write_fence_events ON events; DROP TRIGGER IF EXISTS community_write_fence_git_repo_names ON git_repo_names; DROP TRIGGER IF EXISTS community_write_fence_join_policy_acceptances ON join_policy_acceptances; +DROP TRIGGER IF EXISTS community_write_fence_media_shares ON media_shares; DROP TRIGGER IF EXISTS community_write_fence_media_uploaders ON media_uploaders; DROP TRIGGER IF EXISTS community_write_fence_moderation_actions ON moderation_actions; DROP TRIGGER IF EXISTS community_write_fence_moderation_reports ON moderation_reports; diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index 209f09e5733..82f5233d8dc 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -508,13 +508,7 @@ async fn insert_event_on( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(( @@ -1814,13 +1808,7 @@ pub(crate) async fn insert_event_with_thread_metadata_tx( let was_inserted = result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - tx, - id_bytes.as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; if let Some(ref meta) = thread_meta { let broadcast_val: bool = meta.broadcast; diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index d2853b4d89d..e639fc87a68 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -37,15 +37,28 @@ const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; /// Follow-up for an event row inserted in an admitted transaction. /// /// Call only when the INSERT affected a row, after releasing any savepoint -/// that may roll it back. Stages push work for after commit and records TTL work. +/// that may roll it back. Stages push work for after commit, records TTL work, +/// and shares every blob the event references into its channel in the same +/// transaction, so a stored reference always carries its share. pub(crate) async fn after_admitted_insert( tx: &mut AdmittedTx, - event_id: &[u8], - kind: i32, + event: &nostr::Event, channel_id: Option, ) -> Result<()> { - tx.record_push_event(event_id, kind); + let kind = i32::from(event.kind.as_u16()); + tx.record_push_event(event.id.as_bytes(), kind); tx.record_channel_event(channel_id, kind); + let hashes = buzz_core::media::referenced_blob_hashes(event); + if !hashes.is_empty() { + let community = tx.community(); + crate::store::media_reference::record_media_shares( + tx.conn(), + community, + &hashes, + channel_id, + ) + .await?; + } Ok(()) } diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 0fe1a62f185..9486e332617 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -1,28 +1,41 @@ -//! Media read authorization from where a blob is referenced. +//! Media read authorization from where each blob is shared. use buzz_datastore_tracing::datastore_span; -use sqlx::PgPool; +use sqlx::{PgConnection, PgPool}; +use uuid::Uuid; use crate::{error::Result, Db}; -use buzz_core::kind::{KIND_EMOJI_LIST, KIND_EMOJI_SET}; use buzz_core::CommunityId; +/// Share each blob in `sha256s` into `channel_id`, or with the whole community +/// when `None`. Called with the transaction that stores the referencing event. +pub(crate) async fn record_media_shares( + conn: &mut PgConnection, + community_id: CommunityId, + sha256s: &[String], + channel_id: Option, +) -> Result<()> { + sqlx::query( + "INSERT INTO media_shares (community_id, sha256, channel_id) \ + SELECT $1, unnest($2::text[]), $3 \ + ON CONFLICT DO NOTHING", + ) + .bind(community_id.as_uuid()) + .bind(sha256s) + .bind(channel_id) + .execute(conn) + .await?; + Ok(()) +} + /// Whether `reader` may read every blob in `sha256s` (vacuously true if empty). /// /// A blob is readable by its uploader (uploading the exact bytes proves -/// possession), and by everyone while it is a current avatar or in a live custom -/// emoji set or list. Otherwise a blob attached in a channel is readable only -/// through an attachment the reader can see: one in a channel they are an -/// active member of or that is open, or a channel-less one. Attachments count -/// whether or not the event was deleted; a blob attached nowhere stays readable -/// to the whole community. -/// -/// A blob is attached as an `imeta` file (`x`) or video poster (`image-x`), or -/// as a `link-preview` image or favicon hash; containment is the GIN-indexed -/// prefilter and the positional check keeps the hash in another field from -/// counting. Avatars and emoji name a blob by a standalone 64-hex token. Ingest -/// admits every such reference only if its author can already read the blob, so -/// knowing a hash is not enough to change who can read it. +/// possession), by everyone when it is shared with the whole community or not +/// shared into any channel, and otherwise only by readers of a channel it is +/// shared into: an open channel or one they actively belong to. Shares are never +/// withdrawn when the referencing event is deleted, and ingest only stores a +/// reference whose author can already read the blob. pub async fn can_read_blobs( pool: &PgPool, community_id: CommunityId, @@ -35,46 +48,28 @@ pub async fn can_read_blobs( ) .await?; let readable = sqlx::query_scalar( - "WITH blobs AS (SELECT DISTINCT unnest($2::text[]) AS sha256), \ - refs AS ( \ - SELECT b.sha256, e.channel_id FROM blobs b \ - JOIN events e ON e.community_id = $1 \ - AND (e.tags @> jsonb_build_array(jsonb_build_array('imeta', 'x ' || b.sha256)) \ - OR e.tags @> jsonb_build_array(jsonb_build_array('imeta', 'image-x ' || b.sha256)) \ - OR e.tags @> jsonb_build_array(jsonb_build_array('link-preview', b.sha256))) \ - AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) AS tag \ - WHERE (tag->>0 = 'imeta' \ - AND tag ?| ARRAY['x ' || b.sha256, 'image-x ' || b.sha256]) \ - OR (tag->>0 = 'link-preview' \ - AND b.sha256 IN (tag->>8, tag->>10))) \ - ) \ - SELECT coalesce(bool_and(CASE \ - WHEN EXISTS (SELECT 1 FROM media_uploaders u WHERE u.community_id = $1 \ - AND u.sha256 = b.sha256 AND u.pubkey = $3) THEN true \ - WHEN NOT EXISTS (SELECT 1 FROM refs r \ - WHERE r.sha256 = b.sha256 AND r.channel_id IS NOT NULL) THEN true \ - WHEN EXISTS (SELECT 1 FROM refs r \ - WHERE r.sha256 = b.sha256 AND r.channel_id IS NULL) THEN true \ - WHEN EXISTS (SELECT 1 FROM refs r \ - JOIN channels c ON c.community_id = $1 AND c.id = r.channel_id \ - WHERE r.sha256 = b.sha256 AND c.deleted_at IS NULL \ - AND (c.visibility = 'open' \ - OR EXISTS (SELECT 1 FROM channel_members cm \ - WHERE cm.community_id = $1 AND cm.channel_id = c.id \ - AND cm.pubkey = $3 AND cm.removed_at IS NULL))) \ - THEN true \ - ELSE EXISTS (SELECT 1 FROM users WHERE community_id = $1 \ - AND avatar_url ~ ('(^|[^0-9a-f])' || b.sha256 || '([^0-9a-f]|$)')) \ - OR EXISTS (SELECT 1 FROM events WHERE community_id = $1 AND kind IN ($4, $5) \ - AND deleted_at IS NULL \ - AND tags::text ~ ('(^|[^0-9a-f])' || b.sha256 || '([^0-9a-f]|$)')) \ - END), true) FROM blobs b", + "SELECT coalesce(bool_and( \ + EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND s.channel_id IS NULL) \ + OR NOT EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $3 AND cm.removed_at IS NULL))) \ + ), true) \ + FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b", ) .bind(community_id.as_uuid()) .bind(sha256s) .bind(reader) - .bind(KIND_EMOJI_SET as i32) - .bind(KIND_EMOJI_LIST as i32) .fetch_one(&mut *connection) .await?; Ok(readable) diff --git a/crates/buzz-db/src/store/push.rs b/crates/buzz-db/src/store/push.rs index 24ae312521e..d4ae0b86b3c 100644 --- a/crates/buzz-db/src/store/push.rs +++ b/crates/buzz-db/src/store/push.rs @@ -360,13 +360,7 @@ pub async fn accept_lease_event( return Err(error.into()); } // The row above is always kind 30350 with no channel. - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - 30350, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, None).await?; let (is_active, endpoint_hash, endpoint_grant, max_class, subscriptions) = active .map_or((false, None, None, None, None), |active| { diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 6b21c7ce01a..3f7887cd25e 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -1391,13 +1391,7 @@ impl Db { let was_inserted = insert_result.rows_affected() > 0; if was_inserted { - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - kind_i32, - None, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, &event, None).await?; tx.commit().await?; } else { tx.rollback().await?; diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 1ccee163320..59482e742c4 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -383,8 +383,7 @@ async fn replace_parameterized_event_in_transaction_impl( .execute(tx.conn()) .await?; if status == ParameterizedReplaceStatus::Inserted { - crate::store::event_follow_up::after_admitted_insert(tx, incoming_id, kind_i32, channel_id) - .await?; + crate::store::event_follow_up::after_admitted_insert(tx, event, channel_id).await?; } Ok(ParameterizedReplaceResult::new( @@ -547,13 +546,8 @@ impl Db { )); } - crate::store::event_follow_up::after_admitted_insert( - &mut tx, - event.id.as_bytes().as_slice(), - kind_i32, - channel_id, - ) - .await?; + crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) + .await?; // The replaceable event and its denormalized mention index are one // authoritative discovery write. An indexing error must roll back the diff --git a/crates/buzz-relay/src/handlers/imeta.rs b/crates/buzz-relay/src/handlers/imeta.rs index abbd4f7d49a..4b7a35c96ef 100644 --- a/crates/buzz-relay/src/handlers/imeta.rs +++ b/crates/buzz-relay/src/handlers/imeta.rs @@ -1,5 +1,6 @@ //! imeta tag validation helpers — shared between ingest pipeline and bridge. +use buzz_core::media::is_sha256_hex; use buzz_core::tenant::TenantContext; use buzz_media::validation::mime_to_ext; @@ -361,50 +362,6 @@ fn is_well_formed_mime(mime: &str) -> bool { && !mime.chars().any(|c| c.is_whitespace() || c.is_control()) } -/// Hashes of every blob `event` references, as `buzz_db::store::media_reference` -/// counts them: imeta `x` and `image-x`, link-preview image and favicon, and any -/// standalone 64-hex token in a profile's content or an emoji set or list's tags. -pub fn referenced_blob_hashes(event: &nostr::Event) -> Vec { - use buzz_core::kind::{KIND_EMOJI_LIST, KIND_EMOJI_SET, KIND_PROFILE}; - let kind = u32::from(event.kind.as_u16()); - let public_reference = kind == KIND_EMOJI_SET || kind == KIND_EMOJI_LIST; - let mut hashes = std::collections::BTreeSet::new(); - if kind == KIND_PROFILE { - hashes.extend(hex64_tokens(&event.content)); - } - for tag in event.tags.iter() { - let parts = tag.as_slice(); - if public_reference { - hashes.extend(parts.iter().flat_map(|part| hex64_tokens(part))); - } - match parts.first().map(String::as_str) { - Some("imeta") => hashes.extend(parts.iter().skip(1).filter_map(|part| { - let (key, value) = part.split_once(' ')?; - matches!(key, "x" | "image-x").then(|| value.to_owned()) - })), - Some("link-preview") => { - hashes.extend([8, 10].iter().filter_map(|&i| parts.get(i).cloned())); - } - _ => {} - } - } - hashes - .into_iter() - .filter(|hash| is_sha256_hex(hash)) - .collect() -} - -/// Maximal runs of exactly 64 lowercase hex characters in `text`. -fn hex64_tokens(text: &str) -> impl Iterator + '_ { - text.split(|c: char| !matches!(c, '0'..='9' | 'a'..='f')) - .filter(|token| token.len() == 64) - .map(str::to_owned) -} - -fn is_sha256_hex(value: &str) -> bool { - value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) -} - /// Extract the 64-char hex hash from a `/media/{hash}.{ext}` URL. fn extract_hash_from_media_url(url: &str) -> Option<&str> { let after = url.rsplit("/media/").next()?; @@ -649,53 +606,6 @@ mod tests { assert!(err.contains("image-x must match"), "{err}"); } - #[test] - fn referenced_blob_hashes_reads_only_attachment_positions() { - let poster = "c".repeat(64); - let preview_image = "d".repeat(64); - let mut preview = vec!["link-preview".to_owned(); 11]; - preview[4] = "f".repeat(64); - preview[8] = preview_image.clone(); - preview[10] = String::new(); - let event = nostr::EventBuilder::new(nostr::Kind::TextNote, "") - .tags([ - nostr::Tag::parse(video_with_poster(Some(&poster))).unwrap(), - nostr::Tag::parse(preview).unwrap(), - nostr::Tag::parse(["x", &"e".repeat(64)]).unwrap(), - ]) - .sign_with_keys(&nostr::Keys::generate()) - .unwrap(); - assert_eq!( - referenced_blob_hashes(&event), - vec![HASH.to_owned(), poster, preview_image] - ); - } - - #[test] - fn referenced_blob_hashes_covers_avatars_and_emoji() { - let avatar = "c".repeat(64); - let emoji = "d".repeat(64); - let keys = nostr::Keys::generate(); - let profile = nostr::EventBuilder::new( - nostr::Kind::Metadata, - format!(r#"{{"picture":"https://relay.example/media/{avatar}.jpg"}}"#), - ) - .sign_with_keys(&keys) - .unwrap(); - assert_eq!(referenced_blob_hashes(&profile), vec![avatar]); - - let emoji_set = nostr::EventBuilder::new(nostr::Kind::from(30030), "") - .tags([nostr::Tag::parse([ - "emoji", - "party", - &format!("https://relay.example/media/{emoji}.png"), - ]) - .unwrap()]) - .sign_with_keys(&keys) - .unwrap(); - assert_eq!(referenced_blob_hashes(&emoji_set), vec![emoji]); - } - #[test] fn test_imeta_generic_file_with_filename_passes() { // Generic file attachment: non-media MIME, arbitrary ext, filename label. diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 8b137499da4..0c5b4214816 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -2557,7 +2557,7 @@ async fn ingest_event_inner( // A media reference changes who can read its blob, so only an author who can // already read every referenced blob may publish it. Also before every kind // branch, and one query however many blobs the event names. - let referenced = super::imeta::referenced_blob_hashes(&event); + let referenced = buzz_core::media::referenced_blob_hashes(&event); if !referenced.is_empty() && !state .db diff --git a/migrations/0068_media_shares.sql b/migrations/0068_media_shares.sql new file mode 100644 index 00000000000..67318d7802a --- /dev/null +++ b/migrations/0068_media_shares.sql @@ -0,0 +1,16 @@ +-- Where each blob is shared: one row per channel an event referencing it was +-- stored in, or a NULL channel for a channel-less reference (avatars, custom +-- emoji, notes), which shares it with the whole community. Media reads admit a +-- reader who can see any share; rows are written with the referencing event. +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; + +SELECT attach_community_write_fence('media_shares'); diff --git a/schema/schema.sql b/schema/schema.sql index 47a06c8f178..9bdb5321b26 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -107,6 +107,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i tables/public/moderation_reports.sql \i tables/public/community_bans.sql \i tables/public/community_nip_fi_issuers.sql +\i tables/public/media_shares.sql \i tables/public/media_uploaders.sql \i tables/public/_operator_global_tables.sql \i tables/public/git_repo_names.sql @@ -160,6 +161,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i triggers/event_mentions.sql \i triggers/git_repo_names.sql \i triggers/join_policy_acceptances.sql +\i triggers/media_shares.sql \i triggers/media_uploaders.sql \i triggers/moderation_actions.sql \i triggers/moderation_reports.sql diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql new file mode 100644 index 00000000000..18ab231b4bc --- /dev/null +++ b/schema/tables/public/media_shares.sql @@ -0,0 +1,15 @@ +-- ── Media shares ────────────────────────────────────────────────────────────── +-- Where each blob is shared: one row per channel an event referencing it was +-- stored in, or a NULL channel for a channel-less reference, which shares it +-- with the whole community. + +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; diff --git a/schema/triggers/media_shares.sql b/schema/triggers/media_shares.sql new file mode 100644 index 00000000000..e105e557bf9 --- /dev/null +++ b/schema/triggers/media_shares.sql @@ -0,0 +1 @@ +SELECT attach_community_write_fence('media_shares'); From 42feb178ba38bed67cd0ec41f286e63e7b35fe84 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 16:50:33 +1100 Subject: [PATCH 24/48] fix(relay): share only readable media instead of rejecting references Rejecting an event whose author could not read a referenced blob exposed whether a guessed hash was a private attachment and broke authorized edits that kept an existing attachment. The share writer now records only blobs the author can read, in the event's transaction, so such an event is stored but shares nothing new. Events only their author or tagged recipients can read no longer share their blobs community-wide, shared personas and teams share their avatars like profiles, and the unshared-blob check uses the channel share index. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-core/src/media.rs | 81 +++++++++++----- crates/buzz-db/src/store/event_follow_up.rs | 5 +- crates/buzz-db/src/store/media_reference.rs | 94 ++++++++++--------- crates/buzz-relay/src/api/media.rs | 6 +- crates/buzz-relay/src/handlers/ingest.rs | 16 ---- .../tests/e2e_media_extended.rs | 11 ++- 6 files changed, 119 insertions(+), 94 deletions(-) diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index ede14def0c7..75835bccef6 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -1,18 +1,30 @@ -//! Which media blobs an event references, for channel-scoped media access. +//! Which media blobs an event shares, for channel-scoped media access. use std::collections::BTreeSet; -use crate::kind::{KIND_EMOJI_LIST, KIND_EMOJI_SET, KIND_PROFILE}; +use crate::kind::{ + event_is_shared, is_author_only_event_kind, is_shared_gated_kind, KIND_EMOJI_LIST, + KIND_EMOJI_SET, KIND_PROFILE, P_GATED_KINDS, +}; -/// Hashes of every blob `event` references, each of which shares the blob into -/// the event's channel (or with the whole community for a channel-less event): -/// imeta `x` and `image-x`, link-preview image and favicon, and any standalone -/// 64-hex token in a profile's content or an emoji set or list's tags. -pub fn referenced_blob_hashes(event: &nostr::Event) -> Vec { +/// Hashes of every blob `event` shares into its channel, or with the whole +/// community when it is channel-less: imeta `x` and `image-x`, link-preview +/// image and favicon, and any standalone 64-hex token in the content of a +/// profile or shared persona or team, or in an emoji set or list's tags. +/// +/// Empty for an event only its author or tagged recipients can read, since +/// referencing a blob there shows it to no one new. +pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { let kind = u32::from(event.kind.as_u16()); + if is_author_only_event_kind(event) + || P_GATED_KINDS.contains(&kind) + || (is_shared_gated_kind(kind) && !event_is_shared(event)) + { + return Vec::new(); + } let emoji = kind == KIND_EMOJI_SET || kind == KIND_EMOJI_LIST; let mut hashes = BTreeSet::new(); - if kind == KIND_PROFILE { + if kind == KIND_PROFILE || is_shared_gated_kind(kind) { hashes.extend(hex64_tokens(&event.content)); } for tag in event.tags.iter() { @@ -52,8 +64,13 @@ fn hex64_tokens(text: &str) -> impl Iterator + '_ { #[cfg(test)] mod tests { use super::*; + use crate::kind::{KIND_EVENT_REMINDER, KIND_PERSONA}; use nostr::{EventBuilder, Keys, Kind, Tag}; + fn media_url(sha256: &str) -> String { + format!("https://relay.example/media/{sha256}.jpg") + } + #[test] fn reads_only_attachment_positions() { let (video, poster, preview_image) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); @@ -76,32 +93,48 @@ mod tests { .sign_with_keys(&Keys::generate()) .unwrap(); assert_eq!( - referenced_blob_hashes(&event), + shared_blob_hashes(&event), vec![video, poster, preview_image] ); } #[test] - fn covers_avatars_and_emoji() { + fn covers_avatars_emoji_and_shared_personas_only() { let (avatar, emoji) = ("c".repeat(64), "d".repeat(64)); let keys = Keys::generate(); - let profile = EventBuilder::new( - Kind::Metadata, - format!(r#"{{"picture":"https://relay.example/media/{avatar}.jpg"}}"#), - ) - .sign_with_keys(&keys) - .unwrap(); - assert_eq!(referenced_blob_hashes(&profile), vec![avatar]); + let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar)); + let profile = EventBuilder::new(Kind::Metadata, &content) + .sign_with_keys(&keys) + .unwrap(); + assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); let emoji_set = EventBuilder::new(Kind::from(30030), "") - .tags([Tag::parse([ - "emoji", - "party", - &format!("https://relay.example/media/{emoji}.png"), - ]) - .unwrap()]) + .tags([Tag::parse(["emoji", "party", &media_url(&emoji)]).unwrap()]) .sign_with_keys(&keys) .unwrap(); - assert_eq!(referenced_blob_hashes(&emoji_set), vec![emoji]); + assert_eq!(shared_blob_hashes(&emoji_set), vec![emoji]); + + let persona = |shared: bool| { + let mut tags = vec![Tag::parse(["d", "helper"]).unwrap()]; + if shared { + tags.push(Tag::parse(["shared", "true"]).unwrap()); + } + EventBuilder::new(Kind::from(KIND_PERSONA as u16), &content) + .tags(tags) + .sign_with_keys(&keys) + .unwrap() + }; + assert_eq!(shared_blob_hashes(&persona(true)), vec![avatar]); + assert!(shared_blob_hashes(&persona(false)).is_empty()); + } + + #[test] + fn author_only_events_share_nothing() { + let sha = "a".repeat(64); + let reminder = EventBuilder::new(Kind::from(KIND_EVENT_REMINDER as u16), "") + .tags([Tag::parse(["imeta", &format!("x {sha}")]).unwrap()]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + assert!(shared_blob_hashes(&reminder).is_empty()); } } diff --git a/crates/buzz-db/src/store/event_follow_up.rs b/crates/buzz-db/src/store/event_follow_up.rs index e639fc87a68..dd4e5aa9d19 100644 --- a/crates/buzz-db/src/store/event_follow_up.rs +++ b/crates/buzz-db/src/store/event_follow_up.rs @@ -38,7 +38,7 @@ const SQLSTATE_LOCK_NOT_AVAILABLE: &str = "55P03"; /// /// Call only when the INSERT affected a row, after releasing any savepoint /// that may roll it back. Stages push work for after commit, records TTL work, -/// and shares every blob the event references into its channel in the same +/// and shares the blobs the event references into its channel in the same /// transaction, so a stored reference always carries its share. pub(crate) async fn after_admitted_insert( tx: &mut AdmittedTx, @@ -48,13 +48,14 @@ pub(crate) async fn after_admitted_insert( let kind = i32::from(event.kind.as_u16()); tx.record_push_event(event.id.as_bytes(), kind); tx.record_channel_event(channel_id, kind); - let hashes = buzz_core::media::referenced_blob_hashes(event); + let hashes = buzz_core::media::shared_blob_hashes(event); if !hashes.is_empty() { let community = tx.community(); crate::store::media_reference::record_media_shares( tx.conn(), community, &hashes, + event.pubkey.as_bytes(), channel_id, ) .await?; diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 9486e332617..f126b420d8a 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -7,39 +7,64 @@ use uuid::Uuid; use crate::{error::Result, Db}; use buzz_core::CommunityId; -/// Share each blob in `sha256s` into `channel_id`, or with the whole community -/// when `None`. Called with the transaction that stores the referencing event. +/// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. +/// +/// The uploader may (uploading the exact bytes proves possession), as may +/// everyone when the blob is shared with the whole community or into no +/// channel; otherwise only readers of a channel it is shared into, open or one +/// they actively belong to. Shares are never withdrawn. +macro_rules! readable_sql { + () => { + "( \ + EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ + OR NOT EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND s.channel_id IS NOT NULL) \ + OR EXISTS (SELECT 1 FROM media_shares s \ + JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ + AND (c.visibility = 'open' \ + OR EXISTS (SELECT 1 FROM channel_members cm \ + WHERE cm.community_id = $1 AND cm.channel_id = c.id \ + AND cm.pubkey = $3 AND cm.removed_at IS NULL))))" + }; +} + +/// Share into `channel_id` (or with the whole community when `None`) each blob +/// in `sha256s` that `author` can read. A blob the author cannot read is left +/// unshared, so a reference built from a leaked hash shows it to no one new. +/// Called with the transaction that stores the referencing event. pub(crate) async fn record_media_shares( conn: &mut PgConnection, community_id: CommunityId, sha256s: &[String], + author: &[u8], channel_id: Option, ) -> Result<()> { - sqlx::query( + sqlx::query(concat!( "INSERT INTO media_shares (community_id, sha256, channel_id) \ - SELECT $1, unnest($2::text[]), $3 \ - ON CONFLICT DO NOTHING", - ) + SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + WHERE ", + readable_sql!(), + " ON CONFLICT DO NOTHING" + )) .bind(community_id.as_uuid()) .bind(sha256s) + .bind(author) .bind(channel_id) .execute(conn) .await?; Ok(()) } -/// Whether `reader` may read every blob in `sha256s` (vacuously true if empty). -/// -/// A blob is readable by its uploader (uploading the exact bytes proves -/// possession), by everyone when it is shared with the whole community or not -/// shared into any channel, and otherwise only by readers of a channel it is -/// shared into: an open channel or one they actively belong to. Shares are never -/// withdrawn when the referencing event is deleted, and ingest only stores a -/// reference whose author can already read the blob. -pub async fn can_read_blobs( +/// Whether `reader` may read the blob `sha256`. +pub async fn can_read_blob( pool: &PgPool, community_id: CommunityId, - sha256s: &[String], + sha256: &str, reader: &[u8], ) -> Result { let mut connection = crate::observability::acquire_writer( @@ -47,28 +72,13 @@ pub async fn can_read_blobs( crate::observability::WriterOperation::Authorization, ) .await?; - let readable = sqlx::query_scalar( - "SELECT coalesce(bool_and( \ - EXISTS (SELECT 1 FROM media_uploaders u \ - WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ - OR EXISTS (SELECT 1 FROM media_shares s \ - WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ - AND s.channel_id IS NULL) \ - OR NOT EXISTS (SELECT 1 FROM media_shares s \ - WHERE s.community_id = $1 AND s.sha256 = b.sha256) \ - OR EXISTS (SELECT 1 FROM media_shares s \ - JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ - WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ - AND c.deleted_at IS NULL \ - AND (c.visibility = 'open' \ - OR EXISTS (SELECT 1 FROM channel_members cm \ - WHERE cm.community_id = $1 AND cm.channel_id = c.id \ - AND cm.pubkey = $3 AND cm.removed_at IS NULL))) \ - ), true) \ - FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b", - ) + let readable = sqlx::query_scalar(concat!( + "SELECT ", + readable_sql!(), + " FROM (SELECT $2::text AS sha256) b" + )) .bind(community_id.as_uuid()) - .bind(sha256s) + .bind(sha256) .bind(reader) .fetch_one(&mut *connection) .await?; @@ -76,14 +86,14 @@ pub async fn can_read_blobs( } impl Db { - /// Whether `reader` may read every listed blob; see [`can_read_blobs`]. - #[datastore_span(name = "can_read_blobs", system = "postgresql")] - pub async fn can_read_blobs( + /// Whether `reader` may read the blob; see [`can_read_blob`]. + #[datastore_span(name = "can_read_blob", system = "postgresql")] + pub async fn can_read_blob( &self, community_id: CommunityId, - sha256s: &[String], + sha256: &str, reader: &[u8], ) -> Result { - can_read_blobs(&self.pool, community_id, sha256s, reader).await + can_read_blob(&self.pool, community_id, sha256, reader).await } } diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index d5264ee062c..210ba49fe1d 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -729,11 +729,7 @@ async fn enforce_blossom_read_access( enforce_blossom_read_membership(state, tenant, auth_event, headers, strictness).await?; let readable = state .db - .can_read_blobs( - tenant.community(), - &[sha256.to_owned()], - auth_event.pubkey.as_bytes(), - ) + .can_read_blob(tenant.community(), sha256, auth_event.pubkey.as_bytes()) .await .map_err(|_| media_denial(MediaError::AuthorizationUnavailable, strictness))?; if readable { diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 0c5b4214816..0b2e8d8f096 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -2554,22 +2554,6 @@ async fn ingest_event_inner( // (commands, feedback, reports, moderation) so no write returns ahead of it. enforce_write_restriction(state, tenant, kind_u32, auth.pubkey()).await?; - // A media reference changes who can read its blob, so only an author who can - // already read every referenced blob may publish it. Also before every kind - // branch, and one query however many blobs the event names. - let referenced = buzz_core::media::referenced_blob_hashes(&event); - if !referenced.is_empty() - && !state - .db - .can_read_blobs(tenant.community(), &referenced, event.pubkey.as_bytes()) - .await - .map_err(|e| IngestError::Internal(format!("error: checking media: {e}")))? - { - return Err(IngestError::Rejected( - "invalid: referenced media is not readable by the author".into(), - )); - } - // Command kinds are routed AFTER signature verification, timestamp check, // pubkey/auth match, and scope validation — never before. if buzz_core::kind::is_command_kind(kind_u32) { diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index ea70a8bef5a..42ec2c011fb 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -860,12 +860,13 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { .sign_with_keys(&outsider) .unwrap(); let ok = outsider_client.send_event(reshare).await.unwrap(); - assert!( - !ok.accepted, - "knowing a hash must not let a non-member attach the blob publicly" - ); + assert!(ok.accepted, "reshare rejected: {:?}", ok.message); outsider_client.disconnect().await.unwrap(); - assert_eq!(read_status(&http, &outsider, Method::GET, url).await, 404); + assert_eq!( + read_status(&http, &outsider, Method::GET, url).await, + 404, + "referencing a known hash must not share the blob" + ); assert_eq!(upload(&http, &outsider, &body).await.status(), 200); assert_eq!( From af6554a71805d95ef2d1666d1c8ca1c99d46693a Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 17:00:04 +1100 Subject: [PATCH 25/48] refactor(relay): share every local media path an event names Derive shares from each `/media/` path anywhere in an event's content or tags rather than per-field rules. Current clients' video posters (`image` urls) and media embedded only in message text now get channel shares, and the `image-x` imeta key, needed only to look posters up on read, is dropped. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-core/src/media.rs | 136 +++++++++++------------- crates/buzz-relay/src/handlers/imeta.rs | 56 ++-------- 2 files changed, 74 insertions(+), 118 deletions(-) diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index 75835bccef6..29bd3a124ec 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -3,14 +3,13 @@ use std::collections::BTreeSet; use crate::kind::{ - event_is_shared, is_author_only_event_kind, is_shared_gated_kind, KIND_EMOJI_LIST, - KIND_EMOJI_SET, KIND_PROFILE, P_GATED_KINDS, + event_is_shared, is_author_only_event_kind, is_shared_gated_kind, P_GATED_KINDS, }; /// Hashes of every blob `event` shares into its channel, or with the whole -/// community when it is channel-less: imeta `x` and `image-x`, link-preview -/// image and favicon, and any standalone 64-hex token in the content of a -/// profile or shared persona or team, or in an emoji set or list's tags. +/// community when it is channel-less: each local media path `/media/` +/// anywhere in its content or tags (attachments, posters, thumbnails, link +/// previews, avatars, custom emoji, pasted links). /// /// Empty for an event only its author or tagged recipients can read, since /// referencing a blob there shows it to no one new. @@ -22,43 +21,30 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { { return Vec::new(); } - let emoji = kind == KIND_EMOJI_SET || kind == KIND_EMOJI_LIST; - let mut hashes = BTreeSet::new(); - if kind == KIND_PROFILE || is_shared_gated_kind(kind) { - hashes.extend(hex64_tokens(&event.content)); - } + let mut hashes: BTreeSet = media_path_hashes(&event.content).collect(); for tag in event.tags.iter() { - let parts = tag.as_slice(); - if emoji { - hashes.extend(parts.iter().flat_map(|part| hex64_tokens(part))); - } - match parts.first().map(String::as_str) { - Some("imeta") => hashes.extend(parts.iter().skip(1).filter_map(|part| { - let (key, value) = part.split_once(' ')?; - matches!(key, "x" | "image-x").then(|| value.to_owned()) - })), - Some("link-preview") => { - hashes.extend([8, 10].iter().filter_map(|&i| parts.get(i).cloned())); - } - _ => {} - } + hashes.extend( + tag.as_slice() + .iter() + .flat_map(|part| media_path_hashes(part)), + ); } - hashes - .into_iter() - .filter(|hash| is_sha256_hex(hash)) - .collect() + hashes.into_iter().collect() } -/// Whether `value` is a 64-char lowercase hex SHA-256. -pub fn is_sha256_hex(value: &str) -> bool { +fn is_sha256_hex(value: &str) -> bool { value.len() == 64 && value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) } -/// Maximal runs of exactly 64 lowercase hex characters in `text`. -fn hex64_tokens(text: &str) -> impl Iterator + '_ { - text.split(|c: char| !matches!(c, '0'..='9' | 'a'..='f')) - .filter(|token| token.len() == 64) - .map(str::to_owned) +/// The hash in each `/media/` path in `text`, where the hash is not +/// followed by another hex digit. +fn media_path_hashes(text: &str) -> impl Iterator + '_ { + text.match_indices("/media/").filter_map(|(at, path)| { + let rest = &text[at + path.len()..]; + let hash = rest.get(..64)?; + let bounded = !rest[64..].starts_with(|c: char| matches!(c, '0'..='9' | 'a'..='f')); + (is_sha256_hex(hash) && bounded).then(|| hash.to_owned()) + }) } #[cfg(test)] @@ -67,51 +53,56 @@ mod tests { use crate::kind::{KIND_EVENT_REMINDER, KIND_PERSONA}; use nostr::{EventBuilder, Keys, Kind, Tag}; - fn media_url(sha256: &str) -> String { - format!("https://relay.example/media/{sha256}.jpg") + fn media_url(sha256: &str, ext: &str) -> String { + format!("https://relay.example/media/{sha256}.{ext}") + } + + fn signed(builder: EventBuilder) -> nostr::Event { + builder.sign_with_keys(&Keys::generate()).unwrap() } #[test] - fn reads_only_attachment_positions() { - let (video, poster, preview_image) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); - let mut preview = vec!["link-preview".to_owned(); 11]; - preview[4] = "f".repeat(64); - preview[8] = preview_image.clone(); - preview[10] = String::new(); - let event = EventBuilder::new(Kind::TextNote, "") + fn shares_every_local_media_path() { + let (video, poster, embed) = ("a".repeat(64), "c".repeat(64), "d".repeat(64)); + let event = signed( + EventBuilder::new( + Kind::TextNote, + format!("look ![image]({})", media_url(&embed, "png")), + ) .tags([ Tag::parse([ "imeta", - &format!("url /media/{video}.mp4"), + &format!("url {}", media_url(&video, "mp4")), &format!("x {video}"), - &format!("image-x {poster}"), + &format!("image /media/{poster}.jpg"), ]) .unwrap(), - Tag::parse(preview).unwrap(), - Tag::parse(["x", &"e".repeat(64)]).unwrap(), - ]) - .sign_with_keys(&Keys::generate()) - .unwrap(); - assert_eq!( - shared_blob_hashes(&event), - vec![video, poster, preview_image] + Tag::parse([ + "r", + &format!("https://elsewhere.example/{}", "e".repeat(64)), + ]) + .unwrap(), + Tag::parse(["r", &format!("/media/{}0", "f".repeat(64))]).unwrap(), + ]), ); + assert_eq!(shared_blob_hashes(&event), vec![video, poster, embed]); } #[test] - fn covers_avatars_emoji_and_shared_personas_only() { + fn shares_avatars_emoji_and_shared_personas_only() { let (avatar, emoji) = ("c".repeat(64), "d".repeat(64)); - let keys = Keys::generate(); - let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar)); - let profile = EventBuilder::new(Kind::Metadata, &content) - .sign_with_keys(&keys) - .unwrap(); + let content = format!(r#"{{"picture":"{}"}}"#, media_url(&avatar, "jpg")); + let profile = signed(EventBuilder::new(Kind::Metadata, &content)); assert_eq!(shared_blob_hashes(&profile), vec![avatar.clone()]); - let emoji_set = EventBuilder::new(Kind::from(30030), "") - .tags([Tag::parse(["emoji", "party", &media_url(&emoji)]).unwrap()]) - .sign_with_keys(&keys) - .unwrap(); + let emoji_set = signed( + EventBuilder::new(Kind::from(30030), "").tags([Tag::parse([ + "emoji", + "party", + &media_url(&emoji, "png"), + ]) + .unwrap()]), + ); assert_eq!(shared_blob_hashes(&emoji_set), vec![emoji]); let persona = |shared: bool| { @@ -119,10 +110,7 @@ mod tests { if shared { tags.push(Tag::parse(["shared", "true"]).unwrap()); } - EventBuilder::new(Kind::from(KIND_PERSONA as u16), &content) - .tags(tags) - .sign_with_keys(&keys) - .unwrap() + signed(EventBuilder::new(Kind::from(KIND_PERSONA as u16), &content).tags(tags)) }; assert_eq!(shared_blob_hashes(&persona(true)), vec![avatar]); assert!(shared_blob_hashes(&persona(false)).is_empty()); @@ -130,11 +118,13 @@ mod tests { #[test] fn author_only_events_share_nothing() { - let sha = "a".repeat(64); - let reminder = EventBuilder::new(Kind::from(KIND_EVENT_REMINDER as u16), "") - .tags([Tag::parse(["imeta", &format!("x {sha}")]).unwrap()]) - .sign_with_keys(&Keys::generate()) - .unwrap(); + let reminder = signed( + EventBuilder::new(Kind::from(KIND_EVENT_REMINDER as u16), "").tags([Tag::parse([ + "imeta", + &format!("url /media/{}.jpg", "a".repeat(64)), + ]) + .unwrap()]), + ); assert!(shared_blob_hashes(&reminder).is_empty()); } } diff --git a/crates/buzz-relay/src/handlers/imeta.rs b/crates/buzz-relay/src/handlers/imeta.rs index 4b7a35c96ef..e3d564e448a 100644 --- a/crates/buzz-relay/src/handlers/imeta.rs +++ b/crates/buzz-relay/src/handlers/imeta.rs @@ -1,6 +1,5 @@ //! imeta tag validation helpers — shared between ingest pipeline and bridge. -use buzz_core::media::is_sha256_hex; use buzz_core::tenant::TenantContext; use buzz_media::validation::mime_to_ext; @@ -11,11 +10,11 @@ use buzz_media::validation::mime_to_ext; pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result<(), String> { const ALLOWED_IMETA_KEYS: &[&str] = &[ "url", "m", "x", "size", "dim", "blurhash", "alt", "thumb", "fallback", "duration", - "bitrate", "image", "image-x", "filename", + "bitrate", "image", "filename", ]; const SINGLETON_KEYS: &[&str] = &[ "url", "m", "x", "size", "dim", "blurhash", "thumb", "alt", "duration", "bitrate", "image", - "image-x", "filename", + "filename", ]; // Previewable media MIME types — these get the strict url-extension // consistency check below (their ext is derived from the MIME). Generic @@ -44,8 +43,6 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result let mut x_value = String::new(); let mut m_value = String::new(); let mut thumb_value = String::new(); - let mut image_value = String::new(); - let mut image_x_value = String::new(); for part in tag.iter().skip(1) { let mut parts = part.splitn(2, ' '); @@ -85,18 +82,14 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result has_m = true; } "x" => { - if !is_sha256_hex(value) { + if value.len() != 64 + || !value.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) + { return Err("imeta x must be a 64-char lowercase hex SHA-256".into()); } x_value = value.to_string(); has_x = true; } - "image-x" => { - if !is_sha256_hex(value) { - return Err("imeta image-x must be a 64-char lowercase hex SHA-256".into()); - } - image_x_value = value.to_string(); - } "size" => { match value.parse::() { Ok(0) | Err(_) => { @@ -143,7 +136,6 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result .into(), ); } - image_value = value.to_string(); } "filename" => { // Original filename for the file-card label. Bounded length; @@ -173,7 +165,7 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result // Video-only NIP-71 fields must not appear on image blobs. let is_video = m_value == "video/mp4"; if !is_video { - for key in &["duration", "bitrate", "image", "image-x"] { + for key in &["duration", "bitrate", "image"] { if seen_keys.contains(*key) { return Err(format!( "imeta {key} is only valid for video/mp4, not {m_value}" @@ -208,13 +200,6 @@ pub fn validate_imeta_tags(tags: &[Vec], media_base_url: &str) -> Result } } } - // `image-x` lets media reads find a poster by hash; it must name the - // poster that `image` points at. - if !image_x_value.is_empty() - && extract_hash_from_media_url(&image_value) != Some(image_x_value.as_str()) - { - return Err("imeta image-x must match the image url hash".into()); - } } Ok(()) } @@ -366,7 +351,11 @@ fn is_well_formed_mime(mime: &str) -> bool { fn extract_hash_from_media_url(url: &str) -> Option<&str> { let after = url.rsplit("/media/").next()?; let hash = after.split('.').next()?; - is_sha256_hex(hash).then_some(hash) + if hash.len() == 64 && hash.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')) { + Some(hash) + } else { + None + } } /// Extract the primary extension from a `/media/{hash}.{ext}` URL (not thumb). @@ -583,29 +572,6 @@ mod tests { assert!(err.contains("url hash does not match x"), "{err}"); } - fn video_with_poster(poster_x: Option<&str>) -> Vec { - let poster = "c".repeat(64); - let mut tag = vec![ - "imeta".into(), - format!("url /media/{HASH}.mp4"), - "m video/mp4".into(), - format!("x {HASH}"), - "size 100".into(), - format!("image /media/{poster}.jpg"), - ]; - tag.extend(poster_x.map(|x| format!("image-x {x}"))); - tag - } - - #[test] - fn test_imeta_image_x_must_name_the_poster() { - let poster = "c".repeat(64); - assert!(validate_imeta_tags(&[video_with_poster(Some(&poster))], BASE).is_ok()); - assert!(validate_imeta_tags(&[video_with_poster(None)], BASE).is_ok()); - let err = validate_imeta_tags(&[video_with_poster(Some(HASH))], BASE).unwrap_err(); - assert!(err.contains("image-x must match"), "{err}"); - } - #[test] fn test_imeta_generic_file_with_filename_passes() { // Generic file attachment: non-media MIME, arbitrary ext, filename label. From 65c24f90c8becdad7f3856d2b5e51a348c6c2ccd Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 21:53:53 +1100 Subject: [PATCH 26/48] refactor(db): create media access tables in one migration `media_uploaders` and `media_shares` only make sense together, so create both in migration 0067 now that they ship in one change. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/runtime/migration.rs | 43 ++++++++++--------------- migrations/0067_media_access.sql | 29 +++++++++++++++++ migrations/0067_media_uploaders.sql | 14 -------- migrations/0068_media_shares.sql | 16 --------- 4 files changed, 46 insertions(+), 56 deletions(-) create mode 100644 migrations/0067_media_access.sql delete mode 100644 migrations/0067_media_uploaders.sql delete mode 100644 migrations/0068_media_shares.sql diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 61ce323bc1c..b98addf01cf 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -709,7 +709,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 68); + assert_eq!(migrations.len(), 67); assert_eq!(migrations[64].version, 65); let retire_ttl = strip_sql_comments(migrations[64].sql.as_str()); for statement in [ @@ -747,15 +747,12 @@ mod postgres_tests { .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); assert_eq!(migrations[66].version, 67); - assert!(migrations[66] - .sql - .as_str() - .contains("CREATE TABLE media_uploaders")); - assert_eq!(migrations[67].version, 68); - assert!(migrations[67] - .sql - .as_str() - .contains("CREATE TABLE media_shares")); + for table in ["media_uploaders", "media_shares"] { + assert!(migrations[66] + .sql + .as_str() + .contains(&format!("CREATE TABLE {table}"))); + } assert_eq!(migrations[55].version, 56); assert!(migrations[55] .sql @@ -2327,24 +2324,18 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('community_nip_fi_issuers')")); expected_fences.insert("community_nip_fi_issuers".to_owned()); - // 0067 fences the media uploaders table. - assert!(MIGRATOR + // 0067 fences the media access tables. + let media_access = MIGRATOR .iter() .find(|m| m.version == 67) - .expect("media uploaders migration") - .sql - .as_str() - .contains("attach_community_write_fence('media_uploaders')")); - expected_fences.insert("media_uploaders".to_owned()); - // 0068 fences the media shares table. - assert!(MIGRATOR - .iter() - .find(|m| m.version == 68) - .expect("media shares migration") - .sql - .as_str() - .contains("attach_community_write_fence('media_shares')")); - expected_fences.insert("media_shares".to_owned()); + .expect("media access migration"); + for table in ["media_uploaders", "media_shares"] { + assert!(media_access + .sql + .as_str() + .contains(&format!("attach_community_write_fence('{table}')"))); + expected_fences.insert(table.to_owned()); + } assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql new file mode 100644 index 00000000000..7202685741c --- /dev/null +++ b/migrations/0067_media_access.sql @@ -0,0 +1,29 @@ +-- Channel-scoped media access: who may read each blob. +-- +-- media_uploaders: who has uploaded each blob. Uploading the exact bytes +-- proves possession, so an uploader can always read a blob even when the same +-- bytes are also shared somewhere they cannot read. +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, sha256, pubkey) +); + +-- media_shares: one row per channel a blob is shared into, written with the +-- event that references it, or a NULL channel for a channel-less reference +-- (avatars, custom emoji, notes), which shares it with the whole community. +CREATE TABLE media_shares ( + community_id UUID NOT NULL REFERENCES communities(id), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + channel_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_media_shares_channel + ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; +CREATE UNIQUE INDEX idx_media_shares_public + ON media_shares (community_id, sha256) WHERE channel_id IS NULL; + +SELECT attach_community_write_fence('media_uploaders'); +SELECT attach_community_write_fence('media_shares'); diff --git a/migrations/0067_media_uploaders.sql b/migrations/0067_media_uploaders.sql deleted file mode 100644 index 458381b7da3..00000000000 --- a/migrations/0067_media_uploaders.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Who has uploaded each blob in a community. Uploading the exact bytes proves --- possession, so channel-scoped media reads can always admit an uploader even --- when the same bytes are also attached somewhere they cannot read. --- --- Additive. Nothing reads this table yet. -CREATE TABLE media_uploaders ( - community_id UUID NOT NULL REFERENCES communities(id), - sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), - pubkey BYTEA NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (community_id, sha256, pubkey) -); - -SELECT attach_community_write_fence('media_uploaders'); diff --git a/migrations/0068_media_shares.sql b/migrations/0068_media_shares.sql deleted file mode 100644 index 67318d7802a..00000000000 --- a/migrations/0068_media_shares.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Where each blob is shared: one row per channel an event referencing it was --- stored in, or a NULL channel for a channel-less reference (avatars, custom --- emoji, notes), which shares it with the whole community. Media reads admit a --- reader who can see any share; rows are written with the referencing event. -CREATE TABLE media_shares ( - community_id UUID NOT NULL REFERENCES communities(id), - sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), - channel_id UUID, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); -CREATE UNIQUE INDEX idx_media_shares_channel - ON media_shares (community_id, sha256, channel_id) WHERE channel_id IS NOT NULL; -CREATE UNIQUE INDEX idx_media_shares_public - ON media_shares (community_id, sha256) WHERE channel_id IS NULL; - -SELECT attach_community_write_fence('media_shares'); From 147eb25e70f7c74a376fc6c925156d559b8956f2 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 22:09:00 +1100 Subject: [PATCH 27/48] fix(relay): uploading lets you share a blob, not read it An uploader kept read access to their own upload after losing every channel it was shared into. Reads are now decided by shares alone; having uploaded the exact bytes only lets the uploader share them into a channel they post in, so identical uploads still work and losing a channel revokes access to everything in it. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 18 +++++----- crates/buzz-db/src/store/media_uploader.rs | 2 +- crates/buzz-db/src/store/mod.rs | 2 +- .../tests/e2e_media_extended.rs | 33 ++++++++++++++----- migrations/0067_media_access.sql | 4 +-- schema/tables/public/media_uploaders.sql | 2 +- 6 files changed, 38 insertions(+), 23 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index f126b420d8a..a219bc54269 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -9,16 +9,13 @@ use buzz_core::CommunityId; /// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. /// -/// The uploader may (uploading the exact bytes proves possession), as may -/// everyone when the blob is shared with the whole community or into no +/// Everyone may when the blob is shared with the whole community or into no /// channel; otherwise only readers of a channel it is shared into, open or one /// they actively belong to. Shares are never withdrawn. macro_rules! readable_sql { () => { "( \ - EXISTS (SELECT 1 FROM media_uploaders u \ - WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ - OR EXISTS (SELECT 1 FROM media_shares s \ + EXISTS (SELECT 1 FROM media_shares s \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ OR NOT EXISTS (SELECT 1 FROM media_shares s \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ @@ -34,9 +31,10 @@ macro_rules! readable_sql { } /// Share into `channel_id` (or with the whole community when `None`) each blob -/// in `sha256s` that `author` can read. A blob the author cannot read is left -/// unshared, so a reference built from a leaked hash shows it to no one new. -/// Called with the transaction that stores the referencing event. +/// in `sha256s` that `author` can read or has uploaded (uploading the exact +/// bytes proves possession). Any other blob is left unshared, so a reference +/// built from a leaked hash shows it to no one new. Called with the +/// transaction that stores the referencing event. pub(crate) async fn record_media_shares( conn: &mut PgConnection, community_id: CommunityId, @@ -47,7 +45,9 @@ pub(crate) async fn record_media_shares( sqlx::query(concat!( "INSERT INTO media_shares (community_id, sha256, channel_id) \ SELECT $1, b.sha256, $4 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ - WHERE ", + WHERE EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ + OR ", readable_sql!(), " ON CONFLICT DO NOTHING" )) diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs index 9a720aa6366..0039575af04 100644 --- a/crates/buzz-db/src/store/media_uploader.rs +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -1,4 +1,4 @@ -//! Who has uploaded each blob, as proof of possession for media reads. +//! Who has uploaded each blob, as proof of possession for sharing it. use buzz_datastore_tracing::datastore_span; diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 48efd56d57e..76bd8f7be99 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -34,7 +34,7 @@ pub mod feed; pub mod git_repo; /// Media read authorization from where a blob is referenced. pub mod media_reference; -/// Who has uploaded each blob, as proof of possession for media reads. +/// Who has uploaded each blob, as proof of possession for sharing it. pub mod media_uploader; /// Community moderation: reports, bans/timeouts, audit actions. pub mod moderation; diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 42ec2c011fb..04078f1ef25 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -852,18 +852,26 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { ); } + let outsider_channel = create_channel(&http, &outsider, "outsider-media", "open").await; let mut outsider_client = BuzzTestClient::connect(&relay_ws_url(), &outsider) .await .unwrap(); - let reshare = EventBuilder::new(Kind::TextNote, "reshare by hash") - .tags(vec![imeta]) - .sign_with_keys(&outsider) - .unwrap(); - let ok = outsider_client.send_event(reshare).await.unwrap(); - assert!(ok.accepted, "reshare rejected: {:?}", ok.message); - outsider_client.disconnect().await.unwrap(); + let mut post_in_outsider_channel = async |content: &str| { + let event = EventBuilder::new(Kind::from(9), content) + .tags(vec![ + Tag::parse(["h", &outsider_channel]).unwrap(), + imeta.clone(), + ]) + .sign_with_keys(&outsider) + .unwrap(); + let ok = outsider_client.send_event(event).await.unwrap(); + assert!(ok.accepted, "post rejected: {:?}", ok.message); + }; + let viewer = Keys::generate(); + + post_in_outsider_channel("reshare by hash").await; assert_eq!( - read_status(&http, &outsider, Method::GET, url).await, + read_status(&http, &viewer, Method::GET, url).await, 404, "referencing a known hash must not share the blob" ); @@ -871,9 +879,16 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { assert_eq!(upload(&http, &outsider, &body).await.status(), 200); assert_eq!( read_status(&http, &outsider, Method::GET, url).await, + 404, + "uploading grants no read access until the blob is shared" + ); + post_in_outsider_channel("my own copy").await; + assert_eq!( + read_status(&http, &viewer, Method::GET, url).await, 200, - "uploading the same bytes proves possession" + "uploading the same bytes lets the uploader share it into their channel" ); + outsider_client.disconnect().await.unwrap(); } /// NIP-92 `imeta` tag attaching the blob an upload `desc` describes. diff --git a/migrations/0067_media_access.sql b/migrations/0067_media_access.sql index 7202685741c..c2f13825d63 100644 --- a/migrations/0067_media_access.sql +++ b/migrations/0067_media_access.sql @@ -1,8 +1,8 @@ -- Channel-scoped media access: who may read each blob. -- -- media_uploaders: who has uploaded each blob. Uploading the exact bytes --- proves possession, so an uploader can always read a blob even when the same --- bytes are also shared somewhere they cannot read. +-- proves possession, so an uploader may share a blob into their own channels +-- even when the same bytes are also shared somewhere they cannot read. CREATE TABLE media_uploaders ( community_id UUID NOT NULL REFERENCES communities(id), sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql index 7a1255ccdd3..5522ae84a7a 100644 --- a/schema/tables/public/media_uploaders.sql +++ b/schema/tables/public/media_uploaders.sql @@ -1,6 +1,6 @@ -- ── Media uploaders ─────────────────────────────────────────────────────────── -- Who has uploaded each blob in a community. Uploading the exact bytes proves --- possession, so channel-scoped media reads can always admit an uploader. +-- possession, so an uploader may share a blob into their own channels. CREATE TABLE media_uploaders ( community_id UUID NOT NULL REFERENCES communities(id), From 865f390d2445f21e7f6053633147f9e2f5fc9864 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Mon, 19 Oct 2026 22:27:09 +1100 Subject: [PATCH 28/48] fix(relay): deny media reads outside the channels a file is shared in Enforce the recorded shares on GET/HEAD /media: a file shared with the whole community is readable by everyone, a file shared into channels by readers of any of them, and a file not yet shared into any channel only by its uploader. Anything else is a 404 so non-members cannot probe for private attachments. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_reference.rs | 55 ++++++++++++++++--- .../tests/e2e_media_extended.rs | 9 ++- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index c4aa9fb95e3..efd43b2e5fd 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -1,8 +1,8 @@ -//! Where each media blob is shared, recorded with the events that reference it. +//! Where each media blob is shared, and media read authorization from it. use buzz_datastore_tracing::datastore_span; use chrono::{DateTime, Utc}; -use sqlx::{PgConnection, Row}; +use sqlx::{PgConnection, PgPool, Row}; use uuid::Uuid; use crate::{error::Result, Db}; @@ -10,17 +10,20 @@ use buzz_core::CommunityId; /// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. /// -/// Everyone may when the blob is shared with the whole community or into no -/// channel; otherwise only readers of a channel it is shared into, open or one -/// they actively belong to. Shares are never withdrawn. +/// Everyone may when the blob is shared with the whole community, and readers +/// of a channel it is shared into (open, or one they actively belong to). A +/// blob not yet shared into any channel is readable only by its uploader. +/// Shares are never withdrawn. macro_rules! readable_sql { () => { "( \ EXISTS (SELECT 1 FROM media_shares s \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ - OR NOT EXISTS (SELECT 1 FROM media_shares s \ - WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ - AND s.channel_id IS NOT NULL) \ + OR (NOT EXISTS (SELECT 1 FROM media_shares s \ + WHERE s.community_id = $1 AND s.sha256 = b.sha256 \ + AND s.channel_id IS NOT NULL) \ + AND EXISTS (SELECT 1 FROM media_uploaders u \ + WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3)) \ OR EXISTS (SELECT 1 FROM media_shares s \ JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ @@ -61,7 +64,43 @@ pub(crate) async fn record_media_shares( Ok(()) } +/// Whether `reader` may read the blob `sha256`; see `readable_sql`. +pub async fn can_read_blob( + pool: &PgPool, + community_id: CommunityId, + sha256: &str, + reader: &[u8], +) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let readable = sqlx::query_scalar(concat!( + "SELECT ", + readable_sql!(), + " FROM (SELECT $2::text AS sha256) b" + )) + .bind(community_id.as_uuid()) + .bind(sha256) + .bind(reader) + .fetch_one(&mut *connection) + .await?; + Ok(readable) +} + impl Db { + /// Whether `reader` may read the blob; see [`can_read_blob`]. + #[datastore_span(name = "can_read_blob", system = "postgresql")] + pub async fn can_read_blob( + &self, + community_id: CommunityId, + sha256: &str, + reader: &[u8], + ) -> Result { + can_read_blob(&self.pool, community_id, sha256, reader).await + } + /// Record the shares of every event already stored in an active community, /// deleted events included, `batch_size` events per transaction; returns the /// number of shares added. diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 04078f1ef25..a3d8e06facd 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -821,9 +821,14 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { let url = desc["url"].as_str().expect("descriptor url"); assert_eq!( - read_status(&http, &outsider, Method::GET, url).await, + read_status(&http, &owner, Method::GET, url).await, 200, - "a blob not yet attached anywhere stays community-readable" + "an uploader can preview an unposted upload" + ); + assert_eq!( + read_status(&http, &outsider, Method::GET, url).await, + 404, + "an unposted upload is readable only by its uploader" ); let imeta = imeta_for(&desc); From 53f6a1c08c72332abd1d322ceea4d891e3aac0a2 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 10:45:29 +1100 Subject: [PATCH 29/48] test(media): cover avatars uploaded by agent owners Signed-off-by: Nikolaj Goodger --- .../tests/e2e_media_extended.rs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index a3d8e06facd..a16285df63c 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -959,3 +959,53 @@ async fn test_private_attachment_cannot_hide_an_avatar() { "a private attachment must not hide a current avatar" ); } + +#[tokio::test] +#[ignore] +async fn test_agent_avatar_uploaded_by_owner_is_readable_to_other_members() { + use buzz_sdk::nip_oa; + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let (owner, agent, viewer) = (Keys::generate(), Keys::generate(), Keys::generate()); + let http = http_client(); + let owner_client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let mut image = tiny_gif(); + image[13..19].copy_from_slice(&uuid::Uuid::new_v4().as_bytes()[..6]); + let response = upload(&http, &owner, &image).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + assert_eq!(read_status(&http, &viewer, Method::GET, url).await, 404); + + let tag = nip_oa::compute_auth_tag(&owner, &agent.public_key(), "kind=0").unwrap(); + let tag = nip_oa::parse_auth_tag(&tag).unwrap(); + let mut agent_client = BuzzTestClient::connect_unauthenticated(&relay_ws_url()) + .await + .unwrap(); + agent_client + .authenticate_with_nip_oa(&agent, &tag) + .await + .unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture": url}).to_string(), + ) + .sign_with_keys(&agent) + .unwrap(); + let result = agent_client.send_event(profile).await.unwrap(); + assert!( + result.accepted, + "agent profile rejected: {}", + result.message + ); + for reader in [&agent, &viewer] { + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, reader, method, url).await, 200); + } + } + agent_client.disconnect().await.unwrap(); + owner_client.disconnect().await.unwrap(); +} From d078ce217c6405c66182561adcfbda3148bf6e8c Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 10:53:21 +1100 Subject: [PATCH 30/48] test(media): use the configured relay in upload fixtures Signed-off-by: Nikolaj Goodger --- .../buzz-test-client/tests/e2e_media_extended.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index a16285df63c..d1c216b2e3e 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -1,7 +1,7 @@ //! Extended media upload integration tests — auth edge cases, content validation, //! multi-format uploads, WebSocket imeta validation. //! -//! Run: cargo test -p buzz-test-client --test e2e_media_extended -- --ignored --nocapture +//! Run: cargo test -p buzz-test-client --test e2e_media_extended -- --ignored --nocapture --test-threads=1 use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; @@ -460,7 +460,7 @@ async fn test_auth_server_tag_correct() { Tag::parse(["t", "upload"]).unwrap(), Tag::parse(["x", &sha256]).unwrap(), Tag::parse(["expiration", &(now + 55).to_string()]).unwrap(), - Tag::parse(["server", "localhost:3000"]).unwrap(), + Tag::parse(["server", &relay_server_authority()]).unwrap(), ], ); let resp = upload_with_auth(&client, &auth, &sha256, &jpeg).await; @@ -687,9 +687,9 @@ async fn test_ws_valid_imeta() { // Upload a JPEG to get a valid sha256 let jpeg = tiny_jpeg(); - let sha256 = hex::encode(Sha256::digest(&jpeg)); let resp = upload(&http, &keys, &jpeg).await; assert_eq!(resp.status(), 200); + let descriptor: serde_json::Value = resp.json().await.unwrap(); // Connect via WebSocket let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) @@ -700,14 +700,7 @@ async fn test_ws_valid_imeta() { let event = EventBuilder::new(Kind::from(9), "image via ws") .tags(vec![ Tag::parse(["h", &channel_id]).unwrap(), - Tag::parse([ - "imeta", - &format!("url http://localhost:3000/media/{sha256}.jpg"), - "m image/jpeg", - &format!("x {sha256}"), - "size 347", - ]) - .unwrap(), + imeta_for(&descriptor), ]) .sign_with_keys(&keys) .unwrap(); From 1b2cb9e5c19cbe325635326d60c1cf2b1b3547e5 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 13:52:22 +1100 Subject: [PATCH 31/48] test(media): cover author and moderator deletion Signed-off-by: Nikolaj Goodger --- .../tests/e2e_media_extended.rs | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index d1c216b2e3e..3c067c581c0 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -901,6 +901,95 @@ fn imeta_for(desc: &serde_json::Value) -> Tag { .unwrap() } +#[tokio::test] +#[ignore] +async fn test_deleted_messages_revoke_only_their_own_media_shares() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let (owner, viewer) = (Keys::generate(), Keys::generate()); + let http = http_client(); + let private = create_channel(&http, &owner, "retained-media", "private").await; + let open = create_channel(&http, &owner, "deleted-media", "open").await; + let body = uuid::Uuid::new_v4().as_bytes().repeat(8); + let response = upload(&http, &owner, &body).await; + assert_eq!(response.status(), 200); + let desc: serde_json::Value = response.json().await.unwrap(); + let url = desc["url"].as_str().unwrap(); + let mut client = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let mut messages = Vec::new(); + for (name, channel) in [ + ("private copy", &private), + ("first", &open), + ("last", &open), + ] { + let event = EventBuilder::new(Kind::from(9), name) + .tags([Tag::parse(["h", channel]).unwrap(), imeta_for(&desc)]) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(event.clone()).await.unwrap(); + assert!(result.accepted, "attachment rejected: {}", result.message); + messages.push(event); + } + for reader in [&owner, &viewer] { + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, reader, method, url).await, 200); + } + } + // Author deletion removes the first post; the second must keep its grant. + // Moderator removal then removes the last open-channel grant. + for (kind, index, expected) in [(5, 1, 200), (9005, 2, 404)] { + let mut tags = vec![Tag::parse(["e", &messages[index].id.to_hex()]).unwrap()]; + if kind == 5 { + tags.push(Tag::parse(["k", "9"]).unwrap()); + } else { + tags.push(Tag::parse(["h", &open]).unwrap()); + } + let deletion = EventBuilder::new(Kind::from(kind), "remove attachment") + .tags(tags) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(deletion).await.unwrap(); + assert!(result.accepted, "deletion rejected: {}", result.message); + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &viewer, method.clone(), url).await, + expected + ); + assert_eq!(read_status(&http, &owner, method, url).await, 200); + } + } + + // A community-wide note grants access only for as long as it survives. + let note = EventBuilder::new(Kind::TextNote, format!("shared file {url}")) + .sign_with_keys(&owner) + .unwrap(); + assert!(client.send_event(note.clone()).await.unwrap().accepted); + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method, url).await, 200); + } + let deletion = EventBuilder::new(Kind::EventDeletion, "remove note") + .tags([ + Tag::parse(["e", ¬e.id.to_hex()]).unwrap(), + Tag::parse(["k", "1"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(deletion).await.unwrap(); + assert!( + result.accepted, + "note deletion rejected: {}", + result.message + ); + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method.clone(), url).await, 404); + assert_eq!(read_status(&http, &owner, method, url).await, 200); + } + client.disconnect().await.unwrap(); +} + #[tokio::test] #[ignore] async fn test_private_attachment_cannot_hide_an_avatar() { From 583c54f89d97d409b02ec0424c657980407cae64 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 14:12:14 +1100 Subject: [PATCH 32/48] test(media): cover avatar grant replacement Signed-off-by: Nikolaj Goodger --- .../tests/e2e_media_extended.rs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 3c067c581c0..410478d3f79 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -1042,6 +1042,56 @@ async fn test_private_attachment_cannot_hide_an_avatar() { ); } +#[tokio::test] +#[ignore] +async fn test_profile_replacement_preserves_only_its_retained_avatar() { + use buzz_test_client::BuzzTestClient; + use reqwest::Method; + + let (uploader, publisher, viewer) = (Keys::generate(), Keys::generate(), Keys::generate()); + let http = http_client(); + let mut image = tiny_gif(); + image[13..19].copy_from_slice(&uuid::Uuid::new_v4().as_bytes()[..6]); + let response = upload(&http, &uploader, &image).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + let base = Timestamp::now().as_secs(); + for (signer, keep_picture, step, expected) in [ + (&uploader, true, 0, 200), + (&publisher, true, 0, 200), + (&uploader, false, 1, 200), + (&publisher, true, 1, 200), + (&publisher, false, 2, 404), + (&publisher, true, 3, 404), + ] { + let mut client = BuzzTestClient::connect(&relay_ws_url(), signer) + .await + .unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({ + "name": format!("updated-profile-{step}"), + "picture": if keep_picture { url } else { "" }, + }) + .to_string(), + ) + .custom_created_at(Timestamp::from_secs(base + step)) + .sign_with_keys(signer) + .unwrap(); + let result = client.send_event(profile).await.unwrap(); + assert!(result.accepted, "profile rejected: {}", result.message); + client.disconnect().await.unwrap(); + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &viewer, method, url).await, + expected, + "retaining a picture preserves its grant; removing it revokes it" + ); + } + } +} + #[tokio::test] #[ignore] async fn test_agent_avatar_uploaded_by_owner_is_readable_to_other_members() { From 17cc26880ea0106cc12b7f7ad344b217ff2e2d28 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Tue, 20 Oct 2026 15:22:45 +1100 Subject: [PATCH 33/48] fix(media): stage read enforcement and audit report access Signed-off-by: Nikolaj Goodger --- .env.example | 6 + .github/workflows/_ci-relay.yml | 4 +- crates/buzz-audit/src/action.rs | 4 + crates/buzz-db/src/store/media_report.rs | 40 ++ crates/buzz-db/src/store/mod.rs | 2 + crates/buzz-relay/src/api/media.rs | 13 +- crates/buzz-relay/src/api/media_access.rs | 165 +++++++ .../src/api/media_access_postgres_tests.rs | 428 ++++++++++++++++++ crates/buzz-relay/src/api/mod.rs | 1 + crates/buzz-relay/src/config.rs | 54 +++ crates/buzz-relay/src/main.rs | 1 + crates/buzz-relay/src/metrics.rs | 4 + .../tests/e2e_media_extended.rs | 27 ++ .../tests/e2e_media_review.rs | 151 ++++++ docs/multi-tenant-conformance.md | 2 +- docs/nips/NIP-FI.md | 13 +- docs/operator-media-access.md | 54 +++ 17 files changed, 953 insertions(+), 16 deletions(-) create mode 100644 crates/buzz-db/src/store/media_report.rs create mode 100644 crates/buzz-relay/src/api/media_access.rs create mode 100644 crates/buzz-relay/src/api/media_access_postgres_tests.rs create mode 100644 crates/buzz-test-client/tests/e2e_media_review.rs create mode 100644 docs/operator-media-access.md diff --git a/.env.example b/.env.example index 2af5dc36bb1..0c979f35809 100644 --- a/.env.example +++ b/.env.example @@ -41,6 +41,12 @@ PGDATABASE=buzz # Redis 7 # ----------------------------------------------------------------------------- REDIS_URL=redis://localhost:6379 + +# File-share authorization rollout (independent of NIP-FI). +# off: skip share checks; log-only (default): count would-deny/errors but allow; +# enforce: deny unauthorized reads. Authentication/community checks always apply. +# Read at startup: change deployment config and restart every relay pod. +BUZZ_MEDIA_READ_MODE=log-only # Max connections in the relay's shared Redis pool (default 16). # BUZZ_REDIS_POOL_SIZE=16 diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index a0fa0a4ae83..ec8140058a4 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -598,6 +598,8 @@ jobs: relay-e2e: name: Relay E2E + env: + BUZZ_MEDIA_READ_MODE: enforce runs-on: ubuntu-latest timeout-minutes: 20 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.rust) @@ -639,7 +641,7 @@ jobs: # --no-fail-fast: without it cargo stops after the first failing binary, # so one broken case hides every later binary's result. run: | - cargo test -p buzz-test-client --no-fail-fast --test e2e_media --test e2e_media_extended --test e2e_media_video -- --ignored --nocapture + cargo test -p buzz-test-client --no-fail-fast --test e2e_media --test e2e_media_extended --test e2e_media_review --test e2e_media_video -- --ignored --nocapture env: RELAY_URL: ws://localhost:3000 RELAY_HTTP_URL: http://localhost:3000 diff --git a/crates/buzz-audit/src/action.rs b/crates/buzz-audit/src/action.rs index be7ccc35454..78acb15f2fc 100644 --- a/crates/buzz-audit/src/action.rs +++ b/crates/buzz-audit/src/action.rs @@ -28,6 +28,8 @@ pub enum AuditAction { RateLimitExceeded, /// A media file was uploaded via the Blossom endpoint. MediaUploaded, + /// A moderator received temporary access to a reported message's file. + ModerationMediaRead, } impl AuditAction { @@ -45,6 +47,7 @@ impl AuditAction { Self::AuthFailure => "auth_failure", Self::RateLimitExceeded => "rate_limit_exceeded", Self::MediaUploaded => "media_uploaded", + Self::ModerationMediaRead => "moderation_media_read", } } @@ -60,6 +63,7 @@ impl AuditAction { Self::AuthFailure, Self::RateLimitExceeded, Self::MediaUploaded, + Self::ModerationMediaRead, ]; } diff --git a/crates/buzz-db/src/store/media_report.rs b/crates/buzz-db/src/store/media_report.rs new file mode 100644 index 00000000000..43c3639b80e --- /dev/null +++ b/crates/buzz-db/src/store/media_report.rs @@ -0,0 +1,40 @@ +//! Private, temporary media access for review of an open event report. + +use buzz_core::CommunityId; +use buzz_datastore_tracing::datastore_span; +use uuid::Uuid; + +use crate::{Db, Result}; + +impl Db { + /// Find an open report of a live event that actually granted this file. + /// + /// The caller must authorize ViewQueue and durably audit the access before + /// serving bytes. Report notes, blob-only reports, and unrelated references + /// never supply attachment provenance. Reads use the writer for revocation. + #[datastore_span(name = "open_media_report", system = "postgresql")] + pub async fn open_media_report( + &self, + community: CommunityId, + sha256: &str, + ) -> Result)>> { + let mut conn = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + Ok(sqlx::query_as( + "SELECT r.id, r.target_event_id FROM moderation_reports r \ + JOIN media_shares s ON s.community_id = r.community_id \ + AND s.event_id = r.target_event_id AND s.sha256 = $2 \ + JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ + WHERE r.community_id = $1 AND r.target_kind = 'event' AND r.status = 'open' \ + ORDER BY r.created_at, r.id LIMIT 1", + ) + .bind(community.as_uuid()) + .bind(sha256) + .fetch_optional(&mut *conn) + .await?) + } +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index b84f3903aff..397c95bac23 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -34,6 +34,8 @@ pub mod feed; pub mod git_repo; /// Where each media blob is shared, recorded with the events that reference it. pub mod media_reference; +/// Attachment provenance for scoped moderation report reads. +pub mod media_report; /// Who has uploaded each blob, as proof of possession for sharing it. pub mod media_uploader; /// Community moderation: reports, bans/timeouts, audit actions. diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 643e8babc2b..c5a53bcaf46 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -722,7 +722,7 @@ async fn enforce_blossom_read_membership( /// refusal is `NotFound` so a non-member cannot probe whether a private /// attachment exists. async fn enforce_blossom_read_access( - state: &AppState, + state: &Arc, tenant: &TenantContext, auth_event: &nostr::Event, headers: &HeaderMap, @@ -730,16 +730,9 @@ async fn enforce_blossom_read_access( strictness: BlossomStrictness, ) -> Result<(), MediaDenial> { enforce_blossom_read_membership(state, tenant, auth_event, headers, strictness).await?; - let readable = state - .db - .can_read_blob(tenant.community(), sha256, auth_event.pubkey.as_bytes()) + super::media_access::enforce(state, tenant, auth_event.pubkey.as_bytes(), sha256) .await - .map_err(|_| media_denial(MediaError::AuthorizationUnavailable, strictness))?; - if readable { - Ok(()) - } else { - Err(MediaError::NotFound.into()) - } + .map_err(|error| media_denial(error, strictness)) } fn blob_cache_control() -> &'static str { diff --git a/crates/buzz-relay/src/api/media_access.rs b/crates/buzz-relay/src/api/media_access.rs new file mode 100644 index 00000000000..c3f5e48e8b0 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access.rs @@ -0,0 +1,165 @@ +//! File-share rollout and the audited moderation exception. + +use std::sync::Arc; + +use buzz_audit::{AuditAction, NewAuditEntry}; +use buzz_core::TenantContext; +use buzz_media::MediaError; + +use crate::config::MediaReadMode; +use crate::handlers::moderation_authz::{ + authorize_moderation_action, ModerationAction, ModerationTarget, +}; +use crate::state::AppState; + +enum Access { + Shared, + Report(uuid::Uuid, Vec), + Denied, +} + +#[cfg(test)] +#[path = "media_access_postgres_tests.rs"] +mod postgres_tests; + +/// Called only after authentication and community admission, for GET and HEAD. +pub(super) async fn enforce( + state: &Arc, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, +) -> Result<(), MediaError> { + let mode = state.config.media_read_mode; + if mode == MediaReadMode::Off { + return Ok(()); + } + let mut decision = evaluate(state, tenant, actor, sha256).await; + if mode == MediaReadMode::Enforce { + if let Ok(Access::Report(report, event)) = &decision { + if let Err(error) = audit_access(state, tenant, actor, sha256, *report, event).await { + decision = Err(error); + } + } + } + let outcome = match &decision { + Ok(Access::Shared) => "allow", + Ok(Access::Report(..)) => "report", + Ok(Access::Denied) => "deny", + Err(_) => "error", + }; + metrics::counter!("buzz_media_read_access_total", "mode" => mode.as_str(), "outcome" => outcome) + .increment(1); + if mode == MediaReadMode::LogOnly { + // No identifiers in metrics; no report grant or audit write in shadow. + return Ok(()); + } + match decision? { + Access::Shared | Access::Report(..) => Ok(()), + Access::Denied => Err(MediaError::NotFound), + } +} + +async fn audit_access( + state: &AppState, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, + report: uuid::Uuid, + event: &[u8], +) -> Result<(), MediaError> { + let audit = state + .audit + .as_ref() + .ok_or(MediaError::AuthorizationUnavailable)?; + // Await durable audit, rather than the best-effort upload queue. + audit + .log(NewAuditEntry { + community_id: tenant.community(), + action: AuditAction::ModerationMediaRead, + actor_pubkey: Some(actor.to_vec()), + object_id: Some(sha256.to_owned()), + detail: serde_json::json!({"report_id": report, "event_id": hex::encode(event)}), + }) + .await + .map_err(|error| { + tracing::error!(%error, "Could not audit moderator media access"); + MediaError::AuthorizationUnavailable + })?; + Ok(()) +} + +async fn evaluate( + state: &Arc, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, +) -> Result { + evaluate_inner(state, tenant, actor, sha256) + .await + .map_err(|error| { + tracing::error!(%error, "Media read authorization lookup failed"); + MediaError::AuthorizationUnavailable + }) +} + +async fn evaluate_inner( + state: &Arc, + tenant: &TenantContext, + actor: &[u8], + sha256: &str, +) -> anyhow::Result { + if state + .db + .can_read_blob(tenant.community(), sha256, actor) + .await? + { + return Ok(Access::Shared); + } + let Some((report, event)) = state + .db + .open_media_report(tenant.community(), sha256) + .await? + else { + return Ok(Access::Denied); + }; + match authorize_moderation_action( + tenant, + state, + actor, + None, + ModerationTarget::None, + ModerationAction::ViewQueue, + ) + .await + { + Ok(_) => {} + Err(error) if error.downcast_ref::().is_some() => return Err(error), + Err(_) => return Ok(Access::Denied), + } + anyhow::ensure!( + state.audit.is_some(), + "moderator media access requires durable auditing" + ); + Ok(Access::Report(report, event)) +} + +#[cfg(test)] +mod tests { + use crate::config::MediaReadMode; + + #[test] + fn rollout_modes_default_to_log_only_and_reject_unknown_values() { + assert_eq!(MediaReadMode::default(), MediaReadMode::LogOnly); + for (value, mode) in [ + ("", MediaReadMode::LogOnly), + ("log-only", MediaReadMode::LogOnly), + ("off", MediaReadMode::Off), + ("enforce", MediaReadMode::Enforce), + ] { + assert_eq!(value.parse::().unwrap(), mode); + } + for value in ["disabled", "Enforce", "log_only", "enforc"] { + assert!(value.parse::().is_err()); + } + } +} diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests.rs new file mode 100644 index 00000000000..f1f99447316 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access_postgres_tests.rs @@ -0,0 +1,428 @@ +use super::*; +use buzz_db::channel::{ChannelType, ChannelVisibility}; +use buzz_db::moderation::{NewReport, ReportTarget}; +use nostr::{EventBuilder, Keys, Kind}; +use sqlx::PgPool; +use std::time::Duration; +use uuid::Uuid; + +async fn fixture() -> (Arc, PgPool, TenantContext) { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + let db = buzz_db::Db::from_pool(pool.clone()); + let host = format!("media-review-{}.example", Uuid::new_v4().simple()); + let community = db.ensure_configured_community(&host).await.unwrap().id; + let mut config = crate::config::Config::for_test(); + config.require_relay_membership = false; + config.media_read_mode = MediaReadMode::Enforce; + let redis = deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .unwrap(); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new("redis://127.0.0.1:1", redis.clone()) + .await + .unwrap(), + ); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let storage = buzz_media::MediaStorage::new(&config.media).unwrap(); + let workflow = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let (state, _) = AppState::new( + config, + db, + redis, + buzz_audit::AuditService::new(pool.clone()), + pubsub, + auth, + buzz_search::SearchService::new(pool.clone()), + workflow, + Keys::generate(), + storage, + ); + ( + Arc::new(state), + pool, + TenantContext::resolved(community, host), + ) +} + +fn mode(state: &mut Arc, mode: MediaReadMode) { + Arc::make_mut(&mut Arc::get_mut(state).unwrap().config).media_read_mode = mode; +} + +async fn check( + state: &Arc, + tenant: &TenantContext, + keys: &Keys, + hash: &str, +) -> Result<(), MediaError> { + enforce(state, tenant, keys.public_key().as_bytes(), hash).await +} + +async fn post( + state: &AppState, + tenant: &TenantContext, + owner: &Keys, + channel: Uuid, +) -> (String, nostr::Event) { + let hash = format!("{:064x}", Uuid::new_v4().as_u128()); + let lease = state + .db + .deletion_store() + .acquire_serving_write_lease( + tenant.community(), + "media_upload", + "media-review-test", + Duration::from_secs(60), + ) + .await + .unwrap(); + state + .db + .record_media_uploader(&lease, &hash, owner.public_key().as_bytes()) + .await + .unwrap(); + let event = EventBuilder::new(Kind::from(9), format!("/media/{hash}.jpg")) + .sign_with_keys(owner) + .unwrap(); + state + .db + .insert_event(tenant.community(), &event, Some(channel)) + .await + .unwrap(); + (hash, event) +} + +async fn report( + state: &AppState, + tenant: &TenantContext, + reporter: &Keys, + target: ReportTarget, + note: Option<&str>, +) -> Uuid { + state + .db + .insert_moderation_report( + tenant.community(), + NewReport { + report_event_id: EventBuilder::new(Kind::from(1984), Uuid::new_v4().to_string()) + .sign_with_keys(reporter) + .unwrap() + .id + .as_bytes(), + reporter_pubkey: reporter.public_key().as_bytes(), + target, + channel_id: None, + report_type: "spam", + note, + }, + ) + .await + .unwrap() +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn rollout_and_report_review_preserve_private_file_boundaries() { + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let (mut state, pool, tenant) = fixture().await; + let (owner, reviewer, stranger) = (Keys::generate(), Keys::generate(), Keys::generate()); + let community = tenant.community(); + let private = state + .db + .create_channel( + community, + "reported", + ChannelType::Stream, + ChannelVisibility::Private, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + let (hash, source) = post(&state, &tenant, &owner, private).await; + let (unrelated, _) = post(&state, &tenant, &owner, private).await; + for policy in [ + MediaReadMode::Off, + MediaReadMode::LogOnly, + MediaReadMode::Enforce, + ] { + mode(&mut state, policy); + assert_eq!( + check(&state, &tenant, &stranger, &hash).await.is_ok(), + policy != MediaReadMode::Enforce + ); + assert!(check(&state, &tenant, &owner, &hash).await.is_ok()); + } + state + .db + .add_relay_member(community, &reviewer.public_key().to_hex(), "admin", None) + .await + .unwrap(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + ), + "role alone grants nothing" + ); + let report_id = report( + &state, + &tenant, + &owner, + ReportTarget::Event(source.id.to_bytes().to_vec()), + Some(&format!("/media/{unrelated}.jpg")), + ) + .await; + mode(&mut state, MediaReadMode::LogOnly); + assert!(check(&state, &tenant, &reviewer, &hash).await.is_ok()); + let before: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_log WHERE community_id = $1 AND action = 'moderation_media_read'") + .bind(community.as_uuid()).fetch_one(&pool).await.unwrap(); + assert_eq!( + before, 0, + "log-only must not claim an exceptional read occurred" + ); + mode(&mut state, MediaReadMode::Enforce); + assert!(check(&state, &tenant, &reviewer, &hash).await.is_ok()); + let audit: (Vec, String, serde_json::Value) = sqlx::query_as( + "SELECT actor_pubkey, object_id, detail FROM audit_log WHERE community_id = $1 AND action = 'moderation_media_read' ORDER BY seq DESC LIMIT 1") + .bind(community.as_uuid()).fetch_one(&pool).await.unwrap(); + assert_eq!(audit.0, reviewer.public_key().to_bytes()); + assert_eq!(audit.1, hash); + assert_eq!(audit.2["report_id"], report_id.to_string()); + assert_eq!(audit.2["event_id"], source.id.to_hex()); + assert!(matches!( + check(&state, &tenant, &stranger, &hash).await, + Err(MediaError::NotFound) + )); + assert!( + matches!( + check(&state, &tenant, &reviewer, &unrelated).await, + Err(MediaError::NotFound) + ), + "report notes are not attachments" + ); + report( + &state, + &tenant, + &owner, + ReportTarget::Blob(hex::decode(&unrelated).unwrap()), + None, + ) + .await; + assert!( + matches!( + check(&state, &tenant, &reviewer, &unrelated).await, + Err(MediaError::NotFound) + ), + "a raw reported hash grants nothing" + ); + for role in ["member", "owner", "admin"] { + state + .db + .update_relay_member_role(community, &reviewer.public_key().to_hex(), role) + .await + .unwrap(); + assert_eq!( + check(&state, &tenant, &reviewer, &hash).await.is_ok(), + role != "member" + ); + } + // Other communities cannot supply the grant, report, or queue role. + let other = state + .db + .ensure_configured_community(&format!("other-{}.example", Uuid::new_v4())) + .await + .unwrap(); + let other_tenant = TenantContext::resolved(other.id, other.host); + state + .db + .add_relay_member(other.id, &reviewer.public_key().to_hex(), "admin", None) + .await + .unwrap(); + assert!(matches!( + check(&state, &other_tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + )); + state + .db + .resolve_moderation_report( + community, + report_id, + "dismissed", + reviewer.public_key().as_bytes(), + None, + ) + .await + .unwrap(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + ), + "closing a report revokes the exception" + ); + report( + &state, + &tenant, + &owner, + ReportTarget::Event(source.id.to_bytes().to_vec()), + None, + ) + .await; + assert!(check(&state, &tenant, &reviewer, &hash).await.is_ok()); + let audit = Arc::get_mut(&mut state).unwrap().audit.take(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::AuthorizationUnavailable) + ), + "auditing is required" + ); + Arc::get_mut(&mut state).unwrap().audit = audit; + let broken_pool = PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + broken_pool.close().await; + let audit = Arc::get_mut(&mut state) + .unwrap() + .audit + .replace(Arc::new(buzz_audit::AuditService::new(broken_pool))); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::AuthorizationUnavailable) + ), + "audit commit failure must deny before serving bytes" + ); + Arc::get_mut(&mut state).unwrap().audit = audit; + assert!( + !state + .db + .can_read_blob(community, &hash, reviewer.public_key().as_bytes()) + .await + .unwrap(), + "report review creates no permanent share" + ); + state + .db + .soft_delete_event_and_update_thread(community, source.id.as_bytes(), None, None) + .await + .unwrap(); + assert!( + matches!( + check(&state, &tenant, &reviewer, &hash).await, + Err(MediaError::NotFound) + ), + "deleted messages grant no exception" + ); + pool.close().await; + for policy in [ + MediaReadMode::Off, + MediaReadMode::LogOnly, + MediaReadMode::Enforce, + ] { + mode(&mut state, policy); + assert_eq!( + check(&state, &tenant, &stranger, &hash).await.is_ok(), + policy != MediaReadMode::Enforce, + "lookup failure follows rollout policy" + ); + } + let counts = snapshotter.snapshot().into_vec(); + for (policy, outcome) in [ + ("log-only", "deny"), + ("log-only", "error"), + ("enforce", "deny"), + ("enforce", "report"), + ] { + assert!( + counts.iter().any(|(key, _, _, value)| key.key().name() + == "buzz_media_read_access_total" + && key + .key() + .labels() + .any(|l| l.key() == "mode" && l.value() == policy) + && key + .key() + .labels() + .any(|l| l.key() == "outcome" && l.value() == outcome) + && matches!(value, metrics_util::debugging::DebugValue::Counter(n) if *n > 0)), + "missing {policy}/{outcome} rollout count" + ); + } + assert!(counts + .iter() + .filter(|(key, _, _, _)| key.key().name() == "buzz_media_read_access_total") + .all(|(key, _, _, _)| key + .key() + .labels() + .all(|l| matches!(l.key(), "mode" | "outcome")))); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn every_rollout_mode_keeps_http_authentication_and_community_admission() { + use axum::{body::Body, http::Request, routing::get, Router}; + use base64::Engine; + use nostr::{JsonUtil, Tag, Timestamp}; + use tower::ServiceExt; + + let (mut state, _, tenant) = fixture().await; + Arc::make_mut(&mut Arc::get_mut(&mut state).unwrap().config).require_relay_membership = true; + let stranger = Keys::generate(); + for policy in [ + MediaReadMode::Off, + MediaReadMode::LogOnly, + MediaReadMode::Enforce, + ] { + mode(&mut state, policy); + for method in ["GET", "HEAD"] { + for authenticated in [false, true] { + let auth = EventBuilder::new(Kind::from(24242), "read") + .tags([ + Tag::parse(["t", "get"]).unwrap(), + Tag::parse(["server", tenant.host()]).unwrap(), + Tag::parse(["expiration", &(Timestamp::now().as_secs() + 55).to_string()]) + .unwrap(), + ]) + .sign_with_keys(&stranger) + .unwrap(); + let mut request = Request::builder() + .method(method) + .uri(format!("/media/{}.jpg", "a".repeat(64))) + .header("host", tenant.host()); + if authenticated { + request = request.header( + "authorization", + format!( + "Nostr {}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(auth.as_json()) + ), + ); + } + let response = Router::new() + .route( + "/media/{sha256_ext}", + get(super::super::media::get_blob).head(super::super::media::head_blob), + ) + .with_state(state.clone()) + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!( + response.status().as_u16(), + if authenticated { 403 } else { 401 }, + "{policy:?} {method}" + ); + } + } + } +} diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index 89ff79f7e4c..f1df14da10a 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -8,6 +8,7 @@ pub mod gifs; pub mod git; pub mod invites; pub mod media; +mod media_access; pub mod mesh_demo; pub mod nip05; pub mod nip_fi; diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 4c9e257318b..90937d4ffe5 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -24,6 +24,54 @@ pub enum ConfigError { InvalidValue(String), } +/// Rollout policy for file shares, independent of authentication and membership. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum MediaReadMode { + /// Preserve authenticated, community-scoped reads without checking shares. + Off, + /// Evaluate shares and record outcomes without denying reads. + #[default] + LogOnly, + /// Require a readable share, upload proof, or audited report access. + Enforce, +} + +impl MediaReadMode { + fn from_env() -> Result { + match std::env::var("BUZZ_MEDIA_READ_MODE") { + Ok(value) => value.parse(), + Err(std::env::VarError::NotPresent) => Ok(Self::default()), + Err(std::env::VarError::NotUnicode(_)) => Err(ConfigError::InvalidValue( + "BUZZ_MEDIA_READ_MODE must be valid Unicode".into(), + )), + } + } + + /// Stable configuration and metric label. + pub const fn as_str(self) -> &'static str { + match self { + Self::Off => "off", + Self::LogOnly => "log-only", + Self::Enforce => "enforce", + } + } +} + +impl std::str::FromStr for MediaReadMode { + type Err = ConfigError; + + fn from_str(value: &str) -> Result { + match value { + "off" => Ok(Self::Off), + "" | "log-only" => Ok(Self::LogOnly), + "enforce" => Ok(Self::Enforce), + _ => Err(ConfigError::InvalidValue( + "BUZZ_MEDIA_READ_MODE must be off, log-only, or enforce".into(), + )), + } + } +} + /// Authentication mode for the deployment-admin API. /// /// Configured by `BUZZ_ADMIN_AUTH`: unset/empty/`nip98` → `Nip98` (fail-secure @@ -399,6 +447,10 @@ pub struct Config { /// leaves all identity enforcement to NIP-42 alone. pub nip_fi: crate::nip_fi_config::NipFiRelayConfig, + /// `BUZZ_MEDIA_READ_MODE`: off, log-only (default), or enforce. + /// Loaded at startup; authentication and community membership always apply. + pub media_read_mode: MediaReadMode, + /// Issuer binding written to communities created under a host suffix /// (`BUZZ_NIP_FI_PROVISION_BINDING`). `None` binds nothing at creation. /// Parsed whatever the NIP-FI mode, since creation happens in every mode. @@ -1382,6 +1434,7 @@ impl Config { }; let nip_fi = crate::nip_fi_config::NipFiRelayConfig::from_env()?; + let media_read_mode = MediaReadMode::from_env()?; let nip_fi_provision_binding = crate::nip_fi_config::parse_provision_binding()?; let nip_fi_resolver_bounds = crate::nip_fi_resolver::ResolverBounds::from_env(db_pool_size)?; @@ -1462,6 +1515,7 @@ impl Config { nip_fi_provision_binding, nip_fi_resolver_bounds, nip_fi, + media_read_mode, }) } diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index a154f707574..7d15766234d 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -453,6 +453,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> { metrics_port = config.metrics_port, max_frame_bytes = config.max_frame_bytes, audit_enabled = config.audit_enabled, + media_read_mode = config.media_read_mode.as_str(), push_enabled = config.push_enabled, "Config loaded" ); diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index 67e4d3b35e2..a1ada5cebb9 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -297,6 +297,10 @@ pub fn try_install(port: u16, gauge_idle_timeout_secs: u64) -> Result<(), Metric describe_community_admission_metrics(); describe_db_pool_metrics(); describe_auth_metrics(); + metrics::describe_counter!( + "buzz_media_read_access_total", + "File authorization evaluations by rollout mode and bounded outcome" + ); crate::startup_steps::describe_metrics(); initialize_auth_metric_series(); initialize_partition_audit_metric_series(); diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 410478d3f79..2ff4977f5d2 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -2,6 +2,7 @@ //! multi-format uploads, WebSocket imeta validation. //! //! Run: cargo test -p buzz-test-client --test e2e_media_extended -- --ignored --nocapture --test-threads=1 +//! The full suite expects BUZZ_MEDIA_READ_MODE=enforce on the relay. use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; @@ -1141,3 +1142,29 @@ async fn test_agent_avatar_uploaded_by_owner_is_readable_to_other_members() { agent_client.disconnect().await.unwrap(); owner_client.disconnect().await.unwrap(); } + +#[tokio::test] +#[ignore] +async fn test_media_read_rollout_policy() { + use reqwest::Method; + let policy = std::env::var("MEDIA_READ_TEST_MODE").unwrap_or_else(|_| "enforce".into()); + assert!(["off", "log-only", "enforce"].contains(&policy.as_str())); + let (owner, outsider) = (Keys::generate(), Keys::generate()); + let http = http_client(); + let response = upload(&http, &owner, &uuid::Uuid::new_v4().as_bytes().repeat(8)).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &owner, method.clone(), url).await, 200); + assert_eq!( + read_status(&http, &outsider, method.clone(), url).await, + if policy == "enforce" { 404 } else { 200 }, + "{policy} {method}" + ); + assert_eq!( + http.request(method, url).send().await.unwrap().status(), + 401 + ); + } +} diff --git a/crates/buzz-test-client/tests/e2e_media_review.rs b/crates/buzz-test-client/tests/e2e_media_review.rs new file mode 100644 index 00000000000..346414cb32e --- /dev/null +++ b/crates/buzz-test-client/tests/e2e_media_review.rs @@ -0,0 +1,151 @@ +//! Live report -> private file review -> report closure journey. +//! Requires the relay's database and BUZZ_MEDIA_READ_MODE=enforce. + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use buzz_test_client::BuzzTestClient; +use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; +use reqwest::{Client, Method}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +fn auth(keys: &Keys, host: &str, verb: &str, hash: &str) -> String { + let event = EventBuilder::new(Kind::from(24242), "media review test") + .tags([ + Tag::parse(["t", verb]).unwrap(), + Tag::parse(["server", host]).unwrap(), + Tag::parse(["x", hash]).unwrap(), + Tag::parse(["expiration", &(Timestamp::now().as_secs() + 55).to_string()]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + format!("Nostr {}", URL_SAFE_NO_PAD.encode(event.as_json())) +} + +async fn read(http: &Client, keys: &Keys, base: &str, host: &str, hash: &str, expected: u16) { + for method in [Method::GET, Method::HEAD] { + let response = http + .request(method.clone(), format!("{base}/media/{hash}")) + .header("authorization", auth(keys, host, "get", hash)) + .send() + .await + .unwrap(); + assert_eq!( + response.status().as_u16(), + expected, + "{method} for {}", + keys.public_key() + ); + } +} + +#[tokio::test] +#[ignore] +async fn open_report_grants_only_audited_moderator_attachment_access() { + let base = std::env::var("RELAY_HTTP_URL").unwrap_or_else(|_| "http://localhost:3000".into()); + let host = url::Url::parse(&base).unwrap().authority().to_owned(); + let ws = base + .replace("http://", "ws://") + .replace("https://", "wss://"); + let database = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".into()); // sadscan:disable np.postgres.1 -- local test-only credentials + let pool = sqlx::PgPool::connect(&database).await.unwrap(); + let community: Uuid = sqlx::query_scalar("SELECT id FROM communities WHERE host = $1") + .bind(&host) + .fetch_one(&pool) + .await + .unwrap(); + let (owner, moderator, stranger) = (Keys::generate(), Keys::generate(), Keys::generate()); + // Seed only the queue role; all upload, share, report, read, and closure I/O is real. + sqlx::query("INSERT INTO relay_members (community_id, pubkey, role) VALUES ($1,$2,'admin')") + .bind(community) + .bind(moderator.public_key().to_hex()) + .execute(&pool) + .await + .unwrap(); + let http = Client::new(); + let channel = Uuid::new_v4().to_string(); + let mut author = BuzzTestClient::connect(&ws, &owner).await.unwrap(); + let create = EventBuilder::new(Kind::from(9007), "") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["name", &channel]).unwrap(), + Tag::parse(["channel_type", "stream"]).unwrap(), + Tag::parse(["visibility", "private"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(create).await.unwrap().accepted); + let mut sources = Vec::new(); + for _ in 0..2 { + let bytes = Uuid::new_v4().as_bytes().repeat(8); + let hash = hex::encode(Sha256::digest(&bytes)); + let response = http + .put(format!("{base}/upload")) + .header("authorization", auth(&owner, &host, "upload", &hash)) + .header("x-sha-256", &hash) + .body(bytes) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + let event = EventBuilder::new(Kind::from(9), format!("{base}/media/{hash}")) + .tags([Tag::parse(["h", &channel]).unwrap()]) + .sign_with_keys(&owner) + .unwrap(); + let id = event.id; + assert!(author.send_event(event).await.unwrap().accepted); + sources.push((hash, id)); + } + let (hash, event_id) = &sources[0]; + read(&http, &moderator, &base, &host, hash, 404).await; + let report = EventBuilder::new( + Kind::from(1984), + format!("unrelated /media/{}", sources[1].0), + ) + .allow_self_tagging() + .tags([ + Tag::parse(["e", &event_id.to_hex(), "spam"]).unwrap(), + Tag::parse(["p", &owner.public_key().to_hex(), "spam"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let report_event_id = report.id; + let result = author.send_event(report).await.unwrap(); + assert!(result.accepted, "report rejected: {}", result.message); + read(&http, &moderator, &base, &host, hash, 200).await; + read(&http, &stranger, &base, &host, hash, 404).await; + read(&http, &moderator, &base, &host, &sources[1].0, 404).await; + let audited: i64 = sqlx::query_scalar("SELECT count(*) FROM audit_log WHERE community_id=$1 AND action='moderation_media_read' AND actor_pubkey=$2 AND object_id=$3") + .bind(community).bind(moderator.public_key().to_bytes().to_vec()).bind(hash).fetch_one(&pool).await.unwrap(); + assert_eq!(audited, 2, "GET and HEAD must each commit an audit entry"); + let stored: i64 = + sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND id=$2") + .bind(community) + .bind(report_event_id.to_bytes().to_vec()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, 0, + "reports must remain outside the public event log" + ); + let mut reviewer = BuzzTestClient::connect(&ws, &moderator).await.unwrap(); + let close = EventBuilder::new(Kind::from(9044), "") + .tags([ + Tag::parse(["report", &report_event_id.to_hex()]).unwrap(), + Tag::parse(["status", "dismissed"]).unwrap(), + Tag::parse(["action", "dismiss"]).unwrap(), + ]) + .sign_with_keys(&moderator) + .unwrap(); + let result = reviewer.send_event(close).await.unwrap(); + assert!( + result.accepted, + "report closure rejected: {}", + result.message + ); + read(&http, &moderator, &base, &host, hash, 404).await; + read(&http, &owner, &base, &host, hash, 200).await; + author.disconnect().await.unwrap(); + reviewer.disconnect().await.unwrap(); +} diff --git a/docs/multi-tenant-conformance.md b/docs/multi-tenant-conformance.md index 26b0af8cca5..09a1f6eef02 100644 --- a/docs/multi-tenant-conformance.md +++ b/docs/multi-tenant-conformance.md @@ -49,7 +49,7 @@ Conformance obligations: | Workflows, runs, approvals, webhooks, schedules | Workflows are channel-scoped or project/channel-global; triggers fire on matching stored events; schedule/webhook/manual triggers create runs; approval tokens are hashed. | Workflow definition's community from `req.community` at create/update; webhook/schedule/manual routes resolve workflow id inside host-derived community. | Community-global workflow namespace; runs/approvals inherit workflow community. | `workflows`, `workflow_runs`, `workflow_approvals` include `community_id`; workflow id/token hash lookups are scoped; trigger event ids are scoped. | Trigger evaluation only sees events in the same community. Webhook URLs include host-derived community; approval token grants cannot act on another community's same hash/id. | Existing workflow APIs and YAML remain unchanged in default community. | Add tests for identical workflow UUID/approval token hash in different communities and schedule execution isolation. | | Search / FTS | Postgres FTS over the `events.search_tsv` generated `tsvector` column (GIN-indexed); searchable rows expose `id`, `content`, `kind`, `pubkey`, optional `channel_id`, `created_at`, tag terms; channel-less scope is `ChannelScope::ChannelLessOnly`; the relay refetches canonical events from Postgres by hit id. | Search query carries `req.community`; searchable rows carry `community_id`. | Community-global search results; operator-global FTS index infrastructure may be shared. | Every search query filters by `community_id`, BitmapAnd-ed with the GIN `@@` probe; refetch by `(community_id, event_id)`. | Every query carries `community_id` plus channel scope. `ChannelLessOnly` means channel-less within the community, not platform global. | One community produces the same search results as today. | Tests for same event id/content in A and B, deletion in A not deleting B. | | Redis pub/sub, presence, typing, and cache invalidation | Event fan-out uses `buzz:channel:{uuid}`; presence uses `buzz:presence:{pubkey}`; typing uses `buzz:typing:{channel_id}`; cache invalidation uses `buzz:cache-invalidate`. | Pub/sub calls receive `TenantContext` and derive keys from `community_id` plus channel/pubkey. | Pub/sub and presence are community-global; Redis deployment is operator-global shared infrastructure. | Redis keys include community: `buzz:{community}:channel:{uuid}`, `buzz:{community}:presence:{pubkey}`, `buzz:{community}:typing:{channel_id}`, and community-aware cache invalidation payloads/channels. | Cross-node fan-out must not deliver events to subscriptions in another community. Same pubkey can be online/away differently in two communities. Cache drops only affect same-community membership/visibility caches unless explicitly all-community operator maintenance. | Single-community can preserve existing key names only if deployment is isolated; shared multi-tenant Redis must use the prefixed form. | Add tests for same pubkey presence in two communities and same channel UUID collision in two communities. | -| Media / Blossom / S3 | Authenticated uploads return content-addressed descriptors; `GET/HEAD /media/{sha256.ext}` requires a Blossom `t=get` auth event scoped to the serving host or the blob hash and binds the read to the header-resolved tenant, so a bare read is rejected before any storage lookup; upload audit has `channel_id = None`. | Upload and read request host provides `req.community`; Blossom/NIP-98 auth URL host must agree. | Blob CAS bytes may be operator-global shared storage; metadata, authorization, quotas, audit, and visibility are community-global. | Media metadata/audit rows include `community_id`; if object keys stay SHA-addressed, any per-community policy lives outside the raw blob key. | Upload/read authorization uses community context. Shared hash bytes are allowed only as dedup/storage optimization; metadata/errors must not reveal another community's private upload. | Existing media URLs keep working for default community, but clients must now present read auth; there is no config flag that restores unauthenticated reads. | Resolved: blob reads are authenticated and host/tenant-scoped, not public. Remaining gap, deferred: a read is not gated on the channel ACL of the message the blob was attached to, so relay membership plus a known hash is sufficient. | +| Media / Blossom / S3 | Authenticated uploads return content-addressed descriptors; `GET/HEAD /media/{sha256.ext}` requires a Blossom `t=get` auth event scoped to the serving host or the blob hash and binds the read to the header-resolved tenant, so a bare read is rejected before any storage lookup; upload audit has `channel_id = None`. | Upload and read request host provides `req.community`; Blossom/NIP-98 auth URL host must agree. | Blob CAS bytes may be operator-global shared storage; metadata, authorization, quotas, audit, and visibility are community-global. | Media metadata/audit rows include `community_id`; if object keys stay SHA-addressed, any per-community policy lives outside the raw blob key. | Upload/read authorization uses community context. Shared hash bytes are allowed only as dedup/storage optimization; metadata/errors must not reveal another community's private upload. | Existing media URLs keep working for default community, but clients must now present read auth; there is no config flag that restores unauthenticated reads. | Resolved: authenticated, host-bound file reads support `BUZZ_MEDIA_READ_MODE=off|log-only|enforce` (default `log-only`); authentication and community checks remain active in every mode. Enforce requires a live source-event share, upload proof, or audited open-report access by a community owner/admin. Deleted sources grant nothing. Complete backfill and inspect would-deny/error counters before enabling enforcement; see [media access rollout](operator-media-access.md). | | Git hosting / NIP-34 / object storage | Smart HTTP at `/git/{owner}/{repo}` hydrates from S3 object pointers; NIP-34 repo announcements use `d=repo-id`; pointer key is `repos/{owner}/{repo}/pointer`; git push emits kind:30618. | Git HTTP host gives `req.community`; NIP-98 URL and repo announcement community must agree. | Community-global repo namespace and NIP-34 state; pack/manifests CAS objects may be operator-global if pointers are scoped. | Pointer/name keys include community, e.g. `repos/{community}/{owner}/{repo}/pointer`; NIP-34 replaceable coords include `community_id`; any repo-name registry is `(community_id, owner, repo)` or `(community_id, repo)` per product rule. | Clone/push/read policy resolves repo and branch protections only inside the host community. Git hook policy callback carries community and rejects mismatches. | Existing clone URLs and repo ids work under the default community; object-store migration can move pointers under default prefix without changing git clients. | Add tests for same owner/repo in two communities and push in A not advancing B pointer. | | Mesh, agents, ACP/MCP, and CLI | Agents/CLI connect to a relay URL and use WS/REST; mesh/pairing/presence/status events are regular signed relay events. | The relay URL/host configured in the agent/CLI session selects community. | Agent membership, persona/profile, presence, jobs, memory events, and mesh status are community-global unless a future operator mesh plane is explicitly separate. | Any persisted agent profile/job/mesh status rows/events use `community_id`; Redis/presence/search keys follow the same community scoping. | A portable key may join multiple communities, but memberships, DMs, profiles, jobs, and presence do not bleed across them. | Existing `BUZZ_RELAY_URL` continues to select the one default community. | Add CLI/ACP smoke tests against two hosts using same key with different memberships/profile. | | Audit log and observability | One hash-chain audit log records event/channel/auth/media actions; errors are sanitized before reaching clients. | Every tenant-observable audit entry is labeled with `req.community` or inherited community from the object being acted on. | Community-global audit chains; operator metrics/log aggregation may be platform-global only if tenant labels are bounded and access-controlled. | `audit_log` key/sequence/head includes `community_id`; error/audit projection tables include `community_id`; uniqueness is `(community_id, seq)` and `(community_id, hash)` as appropriate. | Audit reads verify only one community chain. Error strings must not include cross-community IDs, constraint names, or existence facts. | Single-community audit verification still traverses one chain. | Eva owns model edits here; infra lane must ensure media/git/token/search rows emit community-labeled audit entries. | diff --git a/docs/nips/NIP-FI.md b/docs/nips/NIP-FI.md index 3d71a7c436b..2387b606a56 100644 --- a/docs/nips/NIP-FI.md +++ b/docs/nips/NIP-FI.md @@ -704,7 +704,9 @@ mismatched `server` tag MUST be rejected as `evidence_rejected`. #### Read proofs Read proofs MAY be host-wide: no `x` tag is required for reads, and a valid -read proof authorizes reads of any blob on the one bound tenant host. +read proof authenticates the signing key on the one bound tenant host. +Proof validity does not itself grant access to every blob: community membership +and the relay's file-share policy apply separately. Read proofs MUST carry exactly one `server` tag whose value matches the request's already-resolved tenant host. A read proof with an absent or @@ -716,9 +718,12 @@ rejected as `evidence_rejected`. > **Named residual:** within a window of at most 60 seconds from minting (plus > a bounded 5-second future-skew allowance), a captured full header set allows -> reading any media blob on exactly one tenant host. This access is read-only, -> membership-checked, and revocable via assertion expiry or deny-map -> enforcement. It is not state-changing and does not cross tenant boundaries. +> impersonating the signing key for media reads on exactly one tenant host. +> With `BUZZ_MEDIA_READ_MODE=enforce`, reads are limited to files that key may +> access through a live share, upload proof, or an audited open-report review. +> In `off` or `log-only`, community membership remains the read-access gate. +> This access is read-only and revocable via assertion expiry or deny-map +> enforcement; it does not cross tenant boundaries. #### Freshness diff --git a/docs/operator-media-access.md b/docs/operator-media-access.md new file mode 100644 index 00000000000..ca317a8ffa7 --- /dev/null +++ b/docs/operator-media-access.md @@ -0,0 +1,54 @@ +# Media access rollout + +File authorization is separate from authentication, NIP-FI, and community admission. +All three modes keep the existing authentication, host binding, ban, and membership checks. + +Set `BUZZ_MEDIA_READ_MODE` in the relay deployment environment and restart every pod: + +| Value | File-access behavior | +|---|---| +| `off` | Skip the new file-share check and its database lookups. | +| `log-only` (default) | Evaluate file access and count outcomes, but preserve existing reads, including on lookup errors. | +| `enforce` | Require file access; deny unauthorized reads with 404 and authorization failures with 503. | + +Unknown values fail startup. This setting loads at startup; it is not a live feature flag. +Changing modes requires a rolling restart, but no code rebuild. +During that restart, pods use their own configured mode until replaced. + +## Rollout + +1. Deploy share recording to every pod with reads in `log-only`. +2. Run `buzz-admin backfill-media-shares` after all pods record shares. +3. Inspect `buzz_media_read_access_total{mode="log-only",outcome="deny"}` and `outcome="error"` in Prometheus. +4. Investigate denied workflows and repair missing provenance before enforcing. +5. Set `BUZZ_MEDIA_READ_MODE=enforce` and restart every pod. + +The counter uses only fixed `mode` and `outcome` labels: `allow`, `report`, `deny`, or `error`. +It measures authorization evaluations on GET and HEAD, before checking whether storage contains the file. +It counts requests, not distinct files, and includes thumbnail requests checked against the original hash. +Use controlled client smoke tests to investigate counts; hashes, keys, and URLs are not metric labels. +Log-only evaluations create no shares and do not grant future access. + +Previously unshared uploads without uploader proof still need an upload of the exact bytes before they can be shared. +Historical uploader backfill and unparsed-reference recovery remain separate work. +To roll back file denials, set `off` or `log-only` and restart every pod. + +## Open reports + +A community owner or admin who can view the moderation queue may read an attachment +on a reported message outside their channels while that report has status `open`. +The report, message, and file grant must belong to the requesting community. +The source message must still exist and be undeleted. +Report notes, arbitrary URLs, pubkey reports, and blob-only reports grant no extra access. +There is no role-based bypass for unrelated files, and reports create no public shares. + +In enforce mode, each exceptional GET or HEAD requires a durable `moderation_media_read` +audit entry containing the actor, report, source message, and file hash before storage access. +If auditing is disabled or fails, the exceptional read returns 503. +Log-only mode evaluates this permission without writing an exceptional-access audit entry. +Ordinary reads allowed by existing shares do not need the moderation exception. + +Closing the report, deleting the source message, or removing the queue role ends this exception +for subsequent requests; ordinary shares can still independently authorize access. +Escalated and processing reports do not qualify for this community-review exception. +Bytes already downloaded cannot be recalled. From e1dbda76beb6e4d8befc82f367bc88d6c71b1554 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Wed, 21 Oct 2026 11:30:59 +1100 Subject: [PATCH 34/48] docs(media): separate protocol policy from rollout settings Signed-off-by: Nikolaj Goodger --- docs/nips/NIP-FI.md | 6 +++--- docs/operator-media-access.md | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/nips/NIP-FI.md b/docs/nips/NIP-FI.md index 2387b606a56..6bacd01b395 100644 --- a/docs/nips/NIP-FI.md +++ b/docs/nips/NIP-FI.md @@ -719,9 +719,9 @@ rejected as `evidence_rejected`. > **Named residual:** within a window of at most 60 seconds from minting (plus > a bounded 5-second future-skew allowance), a captured full header set allows > impersonating the signing key for media reads on exactly one tenant host. -> With `BUZZ_MEDIA_READ_MODE=enforce`, reads are limited to files that key may -> access through a live share, upload proof, or an audited open-report review. -> In `off` or `log-only`, community membership remains the read-access gate. +> Where the relay enforces its file-share policy, reads are limited to files +> that key may access through a live share, upload proof, or an audited +> open-report review; otherwise community membership is the read-access gate. > This access is read-only and revocable via assertion expiry or deny-map > enforcement; it does not cross tenant boundaries. diff --git a/docs/operator-media-access.md b/docs/operator-media-access.md index ca317a8ffa7..30a8dc76128 100644 --- a/docs/operator-media-access.md +++ b/docs/operator-media-access.md @@ -29,6 +29,13 @@ It counts requests, not distinct files, and includes thumbnail requests checked Use controlled client smoke tests to investigate counts; hashes, keys, and URLs are not metric labels. Log-only evaluations create no shares and do not grant future access. +Backfill reevaluates references received after recording began using their authors' current permissions. +It does not remember earlier refusals: a rerun can grant an old reference after its author gains access. +For a community-wide note, that grant can survive removal of the author's channel access and deletion of the original private post. +The grant remains tied to the note and ends when that note is deleted. +This migration limitation is accepted for recording and log-only rollout; before enabling enforcement, explicitly accept it or prevent reevaluation. +Reruns deduplicate rows but do not preserve earlier authorization decisions. + Previously unshared uploads without uploader proof still need an upload of the exact bytes before they can be shared. Historical uploader backfill and unparsed-reference recovery remain separate work. To roll back file denials, set `off` or `log-only` and restart every pod. From b267129ff7d64e87b23cda3a89977e8eeadb4a3e Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Wed, 21 Oct 2026 11:35:46 +1100 Subject: [PATCH 35/48] test(media): adapt fixtures to membership writer API Signed-off-by: Nikolaj Goodger --- .../src/api/media_access_postgres_tests.rs | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests.rs index f1f99447316..4570cd7f2aa 100644 --- a/crates/buzz-relay/src/api/media_access_postgres_tests.rs +++ b/crates/buzz-relay/src/api/media_access_postgres_tests.rs @@ -161,9 +161,15 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { ); assert!(check(&state, &tenant, &owner, &hash).await.is_ok()); } - state + let _ = state .db - .add_relay_member(community, &reviewer.public_key().to_hex(), "admin", None) + .add_relay_member( + community, + &reviewer.public_key().to_hex(), + "admin", + None, + &state.relay_keypair, + ) .await .unwrap(); assert!( @@ -225,9 +231,14 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { "a raw reported hash grants nothing" ); for role in ["member", "owner", "admin"] { - state + let _ = state .db - .update_relay_member_role(community, &reviewer.public_key().to_hex(), role) + .update_relay_member_role( + community, + &reviewer.public_key().to_hex(), + role, + &state.relay_keypair, + ) .await .unwrap(); assert_eq!( @@ -242,9 +253,15 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { .await .unwrap(); let other_tenant = TenantContext::resolved(other.id, other.host); - state + let _ = state .db - .add_relay_member(other.id, &reviewer.public_key().to_hex(), "admin", None) + .add_relay_member( + other.id, + &reviewer.public_key().to_hex(), + "admin", + None, + &state.relay_keypair, + ) .await .unwrap(); assert!(matches!( From bdfb63016dd95c74b184afcd9887b583b34b4257 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 20:42:58 +1100 Subject: [PATCH 36/48] fix(db): bootstrap only declared legacy fences Signed-off-by: Nikolaj Goodger --- ARCHITECTURE.md | 2 ++ crates/buzz-db/src/runtime/migration.rs | 4 +++ .../schema_fence_bootstrap_postgres_tests.rs | 36 +++++++++++++++++++ docs/database.md | 22 ++++++------ schema/community_write_fence.sql | 31 ---------------- schema/schema.sql | 7 ++-- 6 files changed, 55 insertions(+), 47 deletions(-) create mode 100644 crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs delete mode 100644 schema/community_write_fence.sql diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7ff26b4ef74..34f2afa2d5a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -504,6 +504,8 @@ shape is an explicit transition, not a flag day: never spans communities. Serving leases use the same application admission. Startup and deletion catalog validation tolerate the universal fence objects being absent; inventories still record live fences for older executors. + Desired-state bootstraps install only explicitly declared legacy fences. + New tenant-owned tables rely on application admission, without a fence trigger. - **Replica floor (transition foundation):** runtime now provides a shared replica-floor lock helper for compliant channel-event writers, and the writer probe handshake takes the exclusive counterpart before sampling `S`, scanning diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index af877332529..66cc1092d5d 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -14,6 +14,10 @@ use crate::Result; static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations"); +#[cfg(test)] +#[path = "tests/schema_fence_bootstrap_postgres_tests.rs"] +mod schema_fence_bootstrap_postgres_tests; + /// Run all pending Buzz database migrations. /// /// The entire run holds the exclusive [`SCHEMA_DESTRUCTION_LOCK_KEY`] session diff --git a/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs new file mode 100644 index 00000000000..022403ef699 --- /dev/null +++ b/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs @@ -0,0 +1,36 @@ +use sqlx::PgPool; + +/// A fresh bootstrap keeps the declared backstops but never discovers new +/// community tables and silently attaches a trigger to them. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_bootstrap_keeps_only_explicit_legacy_fences() { + let pool = PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + sqlx::raw_sql("CREATE TABLE new_tenant_probe (community_id UUID NOT NULL)") + .execute(&pool) + .await + .unwrap(); + sqlx::raw_sql(&crate::test_support::desired_state_schema_sql()) + .execute(&pool) + .await + .unwrap(); + + for (table, expected) in [ + ("new_tenant_probe", false), + ("events", true), + ("users", true), + ] { + let fenced: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_trigger \ + WHERE tgrelid = $1::regclass \ + AND tgfoid = 'enforce_community_write_fence()'::regprocedure)", + ) + .bind(table) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(fenced, expected, "unexpected fence on {table}"); + } +} diff --git a/docs/database.md b/docs/database.md index 78ad9486dd4..0a3c9b58e70 100644 --- a/docs/database.md +++ b/docs/database.md @@ -100,7 +100,7 @@ writer of a fenced table already goes through admission. The remaining fence-retirement work drops the fence triggers and functions, and MUST also update ARCHITECTURE.md § *Dual enforcement*. -**Today this blocks new tenant-owned tables.** Startup and deletion validation +**New tenant-owned tables use application admission.** Startup and deletion validation no longer require a fence on any table; they pass with every universal fence object dropped (`migration_schema_fence_objects_are_optional_for_serving_and_deletion` in `crates/buzz-db/src/store/deletion.rs`). Two things still matter: @@ -110,21 +110,19 @@ in `crates/buzz-db/src/store/deletion.rs`). Two things still matter: (`community deletion catalog drift (missing=…, unknown=…)`), and deletion fails closed for every community, not only the new table's. Listing the table is enough to pass. -- Desired-state bootstraps run `schema/community_write_fence.sql`, which - attaches a fence trigger to every `community_id` table not excluded by - `community_write_fence_excluded_table`. A fresh desired-state database would - put a trigger on the new table and a migrated one would not, so the two - schema paths would diverge. - -So before a feature can add a tenant-owned `community_id` table, the -desired-state bootstrap must stop attaching fence triggers. That change belongs -to the fence-retirement work. Land it before the feature, or as the feature's -first PR. Do not work around it with a trigger, or with an entry in +- Desired-state bootstraps install only the legacy fences explicitly listed + under `schema/triggers/`. They do not discover and fence new tables. + Keep those existing backstops until their own retirement is verified; + new tables receive no fence in either schema path. + +The blanket desired-state fence attachment has been removed. A feature may +add a tenant-owned `community_id` table when it follows the checklist below. +Do not add a trigger, or an entry in `community_write_fence_excluded_table` for a table the community owns. #### Checklist: adding a tenant-owned `community_id` table -After the desired-state change above has landed, all in the same PR: +All in the same PR: - [ ] Every writer of the table goes through community admission (`begin_community_write_transaction` or another constructor that returns diff --git a/schema/community_write_fence.sql b/schema/community_write_fence.sql deleted file mode 100644 index 2ebc9fd1f98..00000000000 --- a/schema/community_write_fence.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Attach the universal fence to every existing table carrying community_id, --- including deployment-private sidecars whose community_id is provenance. --- --- Desired-state schema application does not replay migration history, so the --- explicit `SELECT attach_community_write_fence('')` calls in --- `triggers/
.sql` stay as first-class catalog declarations. They also --- make the fence contract visible to migration linting instead of hiding it --- only in this dynamic bootstrap loop. This loop runs last, so it fences every --- desired-state table regardless: a fenced desired state does not prove the --- migration path attaches the fence. Keep the explicit calls. -DO $$ -DECLARE - target REGCLASS; -BEGIN - FOR target IN - SELECT c.oid::REGCLASS - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - JOIN pg_attribute a ON a.attrelid = c.oid - WHERE n.nspname = current_schema() - AND c.relkind IN ('r', 'p') - AND NOT c.relispartition - AND a.attname = 'community_id' - AND NOT a.attisdropped - AND NOT community_write_fence_excluded_table(c.relname) - ORDER BY c.oid::REGCLASS::TEXT - LOOP - PERFORM attach_community_write_fence(target); - END LOOP; -END -$$; diff --git a/schema/schema.sql b/schema/schema.sql index d252cbdb7ea..68f09de3262 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -7,9 +7,9 @@ -- -- Everything SchemaBot cannot apply lives in this directory's other -- subdirectories: `types/`, `functions/`, `partitions/`, `triggers/` (including --- write-fence attachment), `seeds/`, and the `community_write_fence.sql` --- bootstrap. This file composes all of them into one fresh database, in load --- order: types → functions → tables → partitions → triggers → seeds → fence. +-- explicit legacy write-fence attachment), and `seeds/`. This file composes +-- them into one fresh database, in load order: +-- types → functions → tables → partitions → triggers → seeds. -- -- `./bin/pgschema apply --file schema/schema.sql` resolves the `\i` includes -- relative to this file (it refuses `..`). `pgschema` skips INSERTs and some @@ -209,4 +209,3 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i seeds/storage_accounting_snapshots.sql -- Fence any community-scoped table that lacks an explicit attach. -\i community_write_fence.sql From 7892f0bddde44385c79ddc475dd3f9bc0c3ca7aa Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 20:52:21 +1100 Subject: [PATCH 37/48] test(db): audit the schema bootstrap fixture Signed-off-by: Nikolaj Goodger --- .../tests/schema_fence_bootstrap_postgres_tests.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs index 022403ef699..67eb9a9a990 100644 --- a/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/schema_fence_bootstrap_postgres_tests.rs @@ -12,10 +12,13 @@ async fn migration_schema_bootstrap_keeps_only_explicit_legacy_fences() { .execute(&pool) .await .unwrap(); - sqlx::raw_sql(&crate::test_support::desired_state_schema_sql()) - .execute(&pool) - .await - .unwrap(); + // Only checked-in schema files contribute SQL to this test bootstrap. + sqlx::raw_sql(sqlx::AssertSqlSafe( + crate::test_support::desired_state_schema_sql(), + )) + .execute(&pool) + .await + .unwrap(); for (table, expected) in [ ("new_tenant_probe", false), From 53073b1965581b7a8aaff55ff3efc8eb3475605e Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 20:57:33 +1100 Subject: [PATCH 38/48] fix(media): bind grants to message lifetimes Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/runtime/migration.rs | 12 -- crates/buzz-db/src/store/deletion.rs | 57 +++++++ crates/buzz-db/src/store/media_grant.rs | 143 ++++++++++++++++++ crates/buzz-db/src/store/media_reference.rs | 155 +++++++++++++++----- crates/buzz-db/src/store/mod.rs | 1 + crates/buzz-db/src/store/replaceable.rs | 14 +- migrations/0068_media_access.sql | 10 +- schema/schema.sql | 2 - schema/tables/public/media_shares.sql | 5 +- schema/tables/public/media_uploaders.sql | 2 +- schema/triggers/media_shares.sql | 1 - schema/triggers/media_uploaders.sql | 1 - 12 files changed, 338 insertions(+), 65 deletions(-) create mode 100644 crates/buzz-db/src/store/media_grant.rs delete mode 100644 schema/triggers/media_shares.sql delete mode 100644 schema/triggers/media_uploaders.sql diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 509490dd018..701a36fb639 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -2419,18 +2419,6 @@ mod postgres_tests { .as_str() .contains("attach_community_write_fence('community_nip_fi_issuers')")); expected_fences.insert("community_nip_fi_issuers".to_owned()); - // 0068 fences the media access tables. - let media_access = MIGRATOR - .iter() - .find(|m| m.version == 68) - .expect("media access migration"); - for table in ["media_uploaders", "media_shares"] { - assert!(media_access - .sql - .as_str() - .contains(&format!("attach_community_write_fence('{table}')"))); - expected_fences.insert(table.to_owned()); - } assert_eq!( expected_fences, schema.fence_attachments, "write-fence attachment targets differ after recovery policy" diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 9a3b123cefa..777562106d4 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -8495,6 +8495,31 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("count relay admin rows") } + async fn seed_unfenced_media(db: &Db, community: Uuid) -> nostr::Event { + let community = CommunityId::from_uuid(community); + let keys = nostr::Keys::generate(); + let sha = "ab".repeat(32); + let store = db.deletion_store(); + let lease = store + .acquire_serving_write_lease( + community, + "media_upload", + "purge-test", + Duration::from_secs(60), + ) + .await + .unwrap(); + db.record_media_uploader(&lease, &sha, keys.public_key().as_bytes()) + .await + .unwrap(); + store.release_serving_write_lease(&lease).await.unwrap(); + let event = nostr::EventBuilder::new(nostr::Kind::TextNote, format!("/media/{sha}")) + .sign_with_keys(&keys) + .unwrap(); + db.insert_event(community, &event, None).await.unwrap(); + event + } + /// Report-backed staff actions FK `moderation_reports`, and outbox rows FK /// their action. Purge must delete the community's report-backed and direct /// rows in FK order and leave other communities' rows alone. @@ -8504,6 +8529,21 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); let (db, store) = store().await; let (request, inventory) = inventoried_request(&db, &store).await; let community = *request.community_id.as_uuid(); + let media = seed_unfenced_media(&db, community).await; + let backstops: i64 = sqlx::query_scalar( + "SELECT (SELECT count(*) FROM pg_constraint WHERE contype = 'f' \ + AND conrelid IN ('media_shares'::regclass, 'media_uploaders'::regclass)) \ + + (SELECT count(*) FROM pg_trigger \ + WHERE tgrelid IN ('media_shares'::regclass, 'media_uploaders'::regclass) \ + AND tgfoid = 'enforce_community_write_fence()'::regprocedure)", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!( + backstops, 0, + "new media tables rely on application admission" + ); seed_relay_admin_rows(&db, community).await; let bystander = db .ensure_configured_community(&format!("bystander-{}.example", Uuid::new_v4().simple())) @@ -8513,6 +8553,7 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .as_uuid() .to_owned(); seed_relay_admin_rows(&db, bystander).await; + seed_unfenced_media(&db, bystander).await; let live = store .inventory_schema(request.community_id) .await @@ -8530,6 +8571,10 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("claim") .expect("won claim"); store.begin_quiescing(&claim.lease).await.expect("quiesce"); + assert!(db + .insert_event(request.community_id, &media, None) + .await + .is_err()); let generation = store.fence(&claim.lease).await.expect("fence"); let token = LeaseToken { fence_generation: Some(generation), @@ -8546,6 +8591,18 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("bindings"); let deleted = store.purge_postgres(&token).await.expect("purge postgres"); + for table in ["media_shares", "media_uploaders"] { + assert_eq!(deleted.get(table), Some(&1)); + } + let media_rows: (i64, i64) = sqlx::query_as( + "SELECT (SELECT count(*) FROM media_shares WHERE community_id = $1), \ + (SELECT count(*) FROM media_uploaders WHERE community_id = $1)", + ) + .bind(bystander) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!(media_rows, (1, 1), "another community's media must survive"); assert_eq!(deleted.get("relay_admin_outbox"), Some(&2)); assert_eq!(deleted.get("relay_admin_actions"), Some(&2)); assert_eq!(relay_admin_row_counts(&db, community).await, (0, 0)); diff --git a/crates/buzz-db/src/store/media_grant.rs b/crates/buzz-db/src/store/media_grant.rs new file mode 100644 index 00000000000..b1aa93977a2 --- /dev/null +++ b/crates/buzz-db/src/store/media_grant.rs @@ -0,0 +1,143 @@ +//! Provenance shared by live media recording, legacy replacement, and backfill. + +use buzz_core::{CommunityId, StoredEvent}; +use chrono::{DateTime, Utc}; +use nostr::Event; +use sqlx::PgConnection; +use uuid::Uuid; + +use crate::{AdmittedTx, Result}; + +/// Resolve the first valid edit target, as the relay's edit validator does. +/// Nested edits and missing/deleted/cross-channel targets grant no file access. +pub(crate) async fn edit_target( + conn: &mut PgConnection, + community: CommunityId, + event: &Event, + channel: Option, +) -> Result, DateTime)>> { + if event.kind.as_u16() != buzz_core::kind::KIND_STREAM_MESSAGE_EDIT as u16 { + return Ok(None); + } + let Some(channel) = channel else { + return Ok(None); + }; + let Some(target) = event.tags.iter().find_map(|tag| { + let parts = tag.as_slice(); + (parts.first().is_some_and(|name| name == "e")) + .then(|| { + parts + .get(1) + .and_then(|id| nostr::EventId::from_hex(id).ok()) + }) + .flatten() + }) else { + return Ok(None); + }; + Ok(sqlx::query_as( + "SELECT id, created_at FROM events \ + WHERE community_id = $1 AND id = $2 AND channel_id = $3 \ + AND deleted_at IS NULL AND kind <> 40003 \ + ORDER BY created_at DESC LIMIT 1", + ) + .bind(community.as_uuid()) + .bind(target.as_bytes().as_slice()) + .bind(channel) + .fetch_optional(conn) + .await?) +} + +/// Record only server-received pre-cutoff references, with their live source +/// and (for edits) target. The receipt check is repeated in the INSERT. +pub(crate) async fn record_history( + conn: &mut PgConnection, + community: CommunityId, + events: &[StoredEvent], +) -> Result { + let mut hashes = Vec::new(); + let mut channels = Vec::new(); + let mut ids = Vec::new(); + let mut times = Vec::new(); + let mut targets = Vec::new(); + let mut target_times = Vec::new(); + for stored in events { + let event = &stored.event; + let blobs = buzz_core::media::shared_blob_hashes(event); + if blobs.is_empty() { + continue; + } + let target = edit_target(conn, community, event, stored.channel_id).await?; + if event.kind.as_u16() == 40003 && target.is_none() { + continue; + } + let seconds = event.created_at.as_secs() as i64; + let created_at = DateTime::from_timestamp(seconds, 0) + .ok_or(crate::DbError::InvalidTimestamp(seconds))?; + for hash in blobs { + hashes.push(hash); + channels.push(stored.channel_id); + ids.push(event.id.to_bytes().to_vec()); + times.push(created_at); + targets.push(target.as_ref().map(|(id, _)| id.clone())); + target_times.push(target.as_ref().map(|(_, at)| *at)); + } + } + if hashes.is_empty() { + return Ok(0); + } + Ok(sqlx::query( + "INSERT INTO media_shares \ + (community_id, sha256, channel_id, event_id, event_created_at, \ + target_event_id, target_event_created_at) \ + SELECT $1, t.h, t.c, e.id, e.created_at, t.target_id, t.target_at \ + FROM unnest($2::text[], $3::uuid[], $4::bytea[], $5::timestamptz[], \ + $6::bytea[], $7::timestamptz[]) \ + AS t(h, c, event_id, event_created_at, target_id, target_at) \ + JOIN events e ON e.community_id = $1 AND e.id = t.event_id \ + AND e.created_at = t.event_created_at AND e.deleted_at IS NULL \ + JOIN communities c ON c.id = e.community_id \ + AND e.received_at < c.media_shares_recording_started_at \ + ORDER BY t.h, e.id ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(hashes) + .bind(channels) + .bind(ids) + .bind(times) + .bind(targets) + .bind(target_times) + .execute(conn) + .await? + .rows_affected()) +} + +/// Recover an eligible legacy head inside its replacement transaction, while +/// it still grants access. Only references the replacement keeps survive it. +pub(crate) async fn bootstrap_replacement( + tx: &mut AdmittedTx, + incoming: &Event, + previous_id: &[u8], +) -> Result<()> { + if buzz_core::media::shared_blob_hashes(incoming).is_empty() { + return Ok(()); + } + let community = tx.community(); + let row = sqlx::query( + "SELECT e.id, e.pubkey, e.created_at, e.kind, e.tags, e.content, e.sig, \ + e.received_at, e.channel_id \ + FROM events e JOIN communities c ON c.id = e.community_id \ + WHERE e.community_id = $1 AND e.id = $2 AND e.deleted_at IS NULL \ + AND e.received_at < c.media_shares_recording_started_at \ + ORDER BY e.created_at DESC LIMIT 1", + ) + .bind(community.as_uuid()) + .bind(previous_id) + .fetch_optional(tx.conn()) + .await?; + if let Some(row) = row { + if let Some(stored) = crate::event::row_to_stored_event(row)? { + record_history(tx.conn(), community, &[stored]).await?; + } + } + Ok(()) +} diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index c5a846e8e23..00fbd919c6c 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -8,6 +8,19 @@ use uuid::Uuid; use crate::{error::Result, Db}; use buzz_core::CommunityId; +/// Both the reference and its edited message must survive. The target lookup +/// uses its recorded timestamp to reach the original event's partition. +macro_rules! live_source_sql { + () => { + "e.deleted_at IS NULL AND (e.kind <> 40003 OR EXISTS ( \ + SELECT 1 FROM events target \ + WHERE target.community_id = s.community_id AND target.id = s.target_event_id \ + AND target.created_at = s.target_event_created_at AND target.deleted_at IS NULL \ + AND target.kind <> 40003 AND target.channel_id IS NOT DISTINCT FROM e.channel_id))" + }; +} +pub(crate) use live_source_sql; + /// SQL predicate: whether the reader `$3` may read the blob `b.sha256` in community `$1`. /// /// A share admits everyone when it is community-wide, or readers of its @@ -15,20 +28,26 @@ use buzz_core::CommunityId; /// grants nothing. Deleted or purged source events grant nothing. macro_rules! readable_sql { () => { - "( \ + concat!( + "( \ EXISTS (SELECT 1 FROM media_shares s \ JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ - AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + " \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND s.channel_id IS NULL) \ OR EXISTS (SELECT 1 FROM media_shares s \ JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ - AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + " \ JOIN channels c ON c.community_id = $1 AND c.id = s.channel_id \ WHERE s.community_id = $1 AND s.sha256 = b.sha256 AND c.deleted_at IS NULL \ AND (c.visibility = 'open' \ OR EXISTS (SELECT 1 FROM channel_members cm \ WHERE cm.community_id = $1 AND cm.channel_id = c.id \ AND cm.pubkey = $3 AND cm.removed_at IS NULL))))" + ) }; } @@ -48,9 +67,14 @@ pub(crate) async fn record_media_shares( let created_at_secs = event.created_at.as_secs() as i64; let created_at = DateTime::from_timestamp(created_at_secs, 0) .ok_or(crate::DbError::InvalidTimestamp(created_at_secs))?; + let target = super::media_grant::edit_target(conn, community_id, event, channel_id).await?; + if event.kind.as_u16() == 40003 && target.is_none() { + return Ok(0); + } let added = sqlx::query(concat!( - "INSERT INTO media_shares (community_id, sha256, channel_id, event_id, event_created_at) \ - SELECT $1, b.sha256, $4, $6, $7 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ + "INSERT INTO media_shares (community_id, sha256, channel_id, event_id, event_created_at, \ + target_event_id, target_event_created_at) \ + SELECT $1, b.sha256, $4, $6, $7, $8, $9 FROM (SELECT DISTINCT unnest($2::text[]) AS sha256) b \ WHERE (EXISTS (SELECT 1 FROM media_uploaders u \ WHERE u.community_id = $1 AND u.sha256 = b.sha256 AND u.pubkey = $3) \ OR (b.sha256 = $5 AND EXISTS ( \ @@ -71,6 +95,8 @@ pub(crate) async fn record_media_shares( .bind(buzz_core::media::profile_picture_blob_hash(event)) .bind(event.id.as_bytes().as_slice()) .bind(created_at) + .bind(target.as_ref().map(|(id, _)| id)) + .bind(target.as_ref().map(|(_, at)| *at)) .execute(conn) .await? .rows_affected(); @@ -86,7 +112,7 @@ impl Db { /// share recording was installed predate the readability check, so their /// references are recorded as they stand; later events get the live, /// checked recording using the author's current permissions. Earlier refusals - /// are not remembered: reruns can grant old references after access changes. + /// are not remembered: any run can grant old references after access changes. /// Rows are deduplicated, but authorization decisions are not preserved. #[datastore_span(name = "backfill_media_shares", system = "postgresql")] pub async fn backfill_media_shares(&self, batch_size: i64) -> Result { @@ -133,7 +159,6 @@ impl Db { let mut recorded_live = Vec::new(); for row in rows { let received_at: DateTime = row.try_get("received_at")?; - let event_created_at: DateTime = row.try_get("created_at")?; let Some(stored) = crate::event::row_to_stored_event(row)? else { continue; }; @@ -142,43 +167,15 @@ impl Db { continue; } if received_at < installed { - let event_id = stored.event.id.to_bytes().to_vec(); - history.extend(hashes.into_iter().map(|hash| { - (hash, stored.channel_id, event_id.clone(), event_created_at) - })); + history.push(stored); } else { recorded_live.push((stored.event, hashes, stored.channel_id)); } } - // Insert in key order, as live recording does, so a concurrent - // publish can never take the same share locks in reverse. - history.sort(); - history.dedup(); if !history.is_empty() { - let hashes: Vec<_> = history.iter().map(|(h, _, _, _)| h).collect(); - let channels: Vec<_> = history.iter().map(|(_, c, _, _)| *c).collect(); - let event_ids: Vec<_> = history.iter().map(|(_, _, id, _)| id).collect(); - let event_times: Vec<_> = history.iter().map(|(_, _, _, at)| *at).collect(); let mut tx = self.begin_media_share_transaction(community).await?; - added += sqlx::query( - "INSERT INTO media_shares \ - (community_id, sha256, channel_id, event_id, event_created_at) \ - SELECT $1, h, c, e.id, e.created_at \ - FROM unnest($2::text[], $3::uuid[], $4::bytea[], $5::timestamptz[]) \ - AS t(h, c, event_id, event_created_at) \ - JOIN events e ON e.community_id = $1 AND e.id = t.event_id \ - AND e.created_at = t.event_created_at AND e.deleted_at IS NULL \ - ORDER BY h, e.id \ - ON CONFLICT DO NOTHING", - ) - .bind(community.as_uuid()) - .bind(hashes) - .bind(channels) - .bind(event_ids) - .bind(event_times) - .execute(tx.conn()) - .await? - .rows_affected(); + added += + super::media_grant::record_history(tx.conn(), community, &history).await?; tx.commit().await?; } for (event, hashes, channel) in recorded_live { @@ -380,6 +377,41 @@ mod postgres_tests { for event in [&first, &last] { assert!(sources.contains(&event.id.to_bytes().to_vec())); } + let edit_only = if let Some(channel) = channel_id { + let edit_sha = unique_sha(); + upload(&db, community, &owner, &edit_sha).await; + for target in [&first, &last] { + let edit = EventBuilder::new( + Kind::from(40003), + format!("retained /media/{sha}, added /media/{edit_sha}"), + ) + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &target.id.to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &edit, channel_id).await.unwrap(); + // The backfill must also reject surviving legacy edits + // once their logical message has disappeared. + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query( + "UPDATE events SET received_at = (SELECT media_shares_recording_started_at \ + FROM communities WHERE id = $1) - interval '1 second' \ + WHERE community_id = $1 AND id = $2", + ) + .bind(community.as_uuid()) + .bind(edit.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + } + assert!(readable_share(&db, community, &edit_sha, &reader).await); + Some(edit_sha) + } else { + None + }; assert!(readable_share(&db, community, &sha, &reader).await); for (event, still_shared) in [(&first, true), (&last, false)] { assert!( @@ -397,6 +429,13 @@ mod postgres_tests { still_shared, "only an undeleted source message can grant access" ); + if let Some(edit_sha) = &edit_only { + assert_eq!( + readable_share(&db, community, edit_sha, &reader).await, + still_shared, + "edit-only attachments must share the target message's lifetime" + ); + } } db.insert_event(community, &attachment(&reader, &sha, None), None) .await @@ -404,6 +443,9 @@ mod postgres_tests { assert!(!readable_share(&db, community, &sha, &reader).await); db.backfill_media_shares(1).await.unwrap(); assert!(!readable_share(&db, community, &sha, &reader).await); + if let Some(edit_sha) = &edit_only { + assert!(!readable_share(&db, community, edit_sha, &reader).await); + } let purged = attachment(&owner, &sha, channel_id); db.insert_event(community, &purged, channel_id) @@ -449,7 +491,7 @@ mod postgres_tests { let sha = unique_sha(); upload(&db, community, &uploader, &sha).await; let base = nostr::Timestamp::now().as_secs(); - let make = |keys: &Keys, kind: Kind, step: u64, picture: bool| { + let make_for = |sha: &str, keys: &Keys, kind: Kind, step: u64, picture: bool| { let url = if picture { format!("/media/{sha}.jpg") } else { @@ -472,6 +514,9 @@ mod postgres_tests { .sign_with_keys(keys) .unwrap() }; + let make = |keys: &Keys, kind: Kind, step: u64, picture: bool| { + make_for(&sha, keys, kind, step, picture) + }; let publish = async |event: &nostr::Event| { if event.kind == Kind::Metadata { db.replace_addressable_event(community, event, None) @@ -501,6 +546,38 @@ mod postgres_tests { !readable_share(&db, community, &sha, &viewer).await, "a later replacement cannot revive a previously removed grant" ); + + // An unbackfilled legacy head has neither proof nor shares. A + // normal name-only update must retain the eligible old reference. + let legacy_keys = Keys::generate(); + let legacy_sha = unique_sha(); + let legacy_make = + |step, picture| make_for(&legacy_sha, &legacy_keys, kind, step, picture); + let legacy = legacy_make(0, true); + assert!(publish(&legacy).await); + assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + sqlx::query( + "UPDATE events SET received_at = (SELECT media_shares_recording_started_at \ + FROM communities WHERE id = $1) - interval '1 second' \ + WHERE community_id = $1 AND id = $2", + ) + .bind(community.as_uuid()) + .bind(legacy.id.as_bytes().as_slice()) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + assert!(publish(&legacy_make(1, true)).await); + db.backfill_media_shares(1).await.unwrap(); + assert!( + readable_share(&db, community, &legacy_sha, &viewer).await, + "a retained legacy avatar/emoji must survive replacement before backfill" + ); + assert!(publish(&legacy_make(2, false)).await); + assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); + assert!(publish(&legacy_make(3, true)).await); + assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); } } diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 670748df11c..6cf1e942a07 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -33,6 +33,7 @@ pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; /// Where each media blob is shared, recorded with the events that reference it. +mod media_grant; pub mod media_reference; /// Who has uploaded each blob, as proof of possession for sharing it. pub mod media_uploader; diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 6f22d56fba2..4db5e590514 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -307,6 +307,10 @@ async fn replace_parameterized_event_in_transaction_impl( } crate::insert_mentions_in_transaction(&mut *tx, event, channel_id).await?; + // Recover eligible legacy references before retiring the old head. + if let Some((_, previous_id)) = &existing { + crate::store::media_grant::bootstrap_replacement(tx, event, previous_id).await?; + } // Authorize retained references while the previous head is still live. crate::store::event_follow_up::record_admitted_media(tx, event, channel_id).await?; if existing.is_some() { @@ -471,9 +475,9 @@ impl Db { // NIP-16: created_at is second-resolution. On same-second tie, lowest // event id (lexicographic) wins — deterministic across relays. let incoming_id = event.id.as_bytes().as_slice(); - if let Some((existing_ts, existing_id)) = existing { - let dominated = created_at < existing_ts - || (created_at == existing_ts + if let Some((existing_ts, existing_id)) = &existing { + let dominated = created_at < *existing_ts + || (created_at == *existing_ts && incoming_id >= existing_id.as_slice()); if dominated { tx.rollback().await?; @@ -535,6 +539,10 @@ impl Db { )); } + if let Some((_, previous_id)) = &existing { + crate::store::media_grant::bootstrap_replacement(&mut tx, event, previous_id) + .await?; + } // Authorize retained references while the previous head is still live. crate::store::event_follow_up::after_admitted_insert(&mut tx, event, channel_id) .await?; diff --git a/migrations/0068_media_access.sql b/migrations/0068_media_access.sql index 9b4212113fa..7d062fd9025 100644 --- a/migrations/0068_media_access.sql +++ b/migrations/0068_media_access.sql @@ -8,7 +8,7 @@ ALTER TABLE communities -- proves possession, so an uploader may share a blob into their own channels -- even when the same bytes are also shared somewhere they cannot read. CREATE TABLE media_uploaders ( - community_id UUID NOT NULL REFERENCES communities(id), + community_id UUID NOT NULL, sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), pubkey BYTEA NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), @@ -19,14 +19,14 @@ CREATE TABLE media_uploaders ( -- shares with the whole community. Deleted or purged source events grant -- nothing; event_created_at locates the source in the partitioned table. CREATE TABLE media_shares ( - community_id UUID NOT NULL REFERENCES communities(id), + community_id UUID NOT NULL, sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), channel_id UUID, event_id BYTEA NOT NULL, event_created_at TIMESTAMPTZ NOT NULL, + target_event_id BYTEA, + target_event_created_at TIMESTAMPTZ, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((target_event_id IS NULL) = (target_event_created_at IS NULL)), PRIMARY KEY (community_id, sha256, event_id) ); - -SELECT attach_community_write_fence('media_uploaders'); -SELECT attach_community_write_fence('media_shares'); diff --git a/schema/schema.sql b/schema/schema.sql index 8d4cac4c54e..1ee65c207b2 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -161,8 +161,6 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i triggers/event_mentions.sql \i triggers/git_repo_names.sql \i triggers/join_policy_acceptances.sql -\i triggers/media_shares.sql -\i triggers/media_uploaders.sql \i triggers/moderation_actions.sql \i triggers/moderation_reports.sql \i triggers/parameterized_event_watermarks.sql diff --git a/schema/tables/public/media_shares.sql b/schema/tables/public/media_shares.sql index e914069c963..600574f766c 100644 --- a/schema/tables/public/media_shares.sql +++ b/schema/tables/public/media_shares.sql @@ -4,11 +4,14 @@ -- Keep its timestamp for an indexed lookup in the partitioned events table. CREATE TABLE media_shares ( - community_id UUID NOT NULL REFERENCES communities(id), + community_id UUID NOT NULL, sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), channel_id UUID, event_id BYTEA NOT NULL, event_created_at TIMESTAMPTZ NOT NULL, + target_event_id BYTEA, + target_event_created_at TIMESTAMPTZ, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK ((target_event_id IS NULL) = (target_event_created_at IS NULL)), PRIMARY KEY (community_id, sha256, event_id) ); diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql index 5522ae84a7a..1add946f3e2 100644 --- a/schema/tables/public/media_uploaders.sql +++ b/schema/tables/public/media_uploaders.sql @@ -3,7 +3,7 @@ -- possession, so an uploader may share a blob into their own channels. CREATE TABLE media_uploaders ( - community_id UUID NOT NULL REFERENCES communities(id), + community_id UUID NOT NULL, sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), pubkey BYTEA NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), diff --git a/schema/triggers/media_shares.sql b/schema/triggers/media_shares.sql deleted file mode 100644 index e105e557bf9..00000000000 --- a/schema/triggers/media_shares.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT attach_community_write_fence('media_shares'); diff --git a/schema/triggers/media_uploaders.sql b/schema/triggers/media_uploaders.sql deleted file mode 100644 index 23367202e24..00000000000 --- a/schema/triggers/media_uploaders.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT attach_community_write_fence('media_uploaders'); From 8868e2874e3f496fb7e149b96316659dd3d30ef9 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 20:57:34 +1100 Subject: [PATCH 39/48] docs(media): decide on grants before backfill Signed-off-by: Nikolaj Goodger --- docs/operator-media-access.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/operator-media-access.md b/docs/operator-media-access.md index 30a8dc76128..5c7a56933f7 100644 --- a/docs/operator-media-access.md +++ b/docs/operator-media-access.md @@ -17,24 +17,28 @@ During that restart, pods use their own configured mode until replaced. ## Rollout -1. Deploy share recording to every pod with reads in `log-only`. -2. Run `buzz-admin backfill-media-shares` after all pods record shares. -3. Inspect `buzz_media_read_access_total{mode="log-only",outcome="deny"}` and `outcome="error"` in Prometheus. -4. Investigate denied workflows and repair missing provenance before enforcing. -5. Set `BUZZ_MEDIA_READ_MODE=enforce` and restart every pod. +1. Before any backfill, explicitly accept that reconsidered references can create lasting grants; otherwise defer the backfill. +2. Deploy share recording to every pod with reads in `log-only`. +3. Run `buzz-admin backfill-media-shares` after all pods record shares. +4. Inspect `buzz_media_read_access_total{mode="log-only",outcome="deny"}` and `outcome="error"` in Prometheus. +5. Investigate denied workflows and repair missing provenance before enforcing. +6. Set `BUZZ_MEDIA_READ_MODE=enforce` and restart every pod. The counter uses only fixed `mode` and `outcome` labels: `allow`, `report`, `deny`, or `error`. It measures authorization evaluations on GET and HEAD, before checking whether storage contains the file. It counts requests, not distinct files, and includes thumbnail requests checked against the original hash. Use controlled client smoke tests to investigate counts; hashes, keys, and URLs are not metric labels. -Log-only evaluations create no shares and do not grant future access. +Log-only GET/HEAD evaluations create no shares and do not grant future access. +Event recording and backfill still write grants in every read mode. Backfill reevaluates references received after recording began using their authors' current permissions. -It does not remember earlier refusals: a rerun can grant an old reference after its author gains access. +It does not remember earlier refusals: any run, including the first, can grant an old reference after its author gains access. For a community-wide note, that grant can survive removal of the author's channel access and deletion of the original private post. The grant remains tied to the note and ends when that note is deleted. -This migration limitation is accepted for recording and log-only rollout; before enabling enforcement, explicitly accept it or prevent reevaluation. -Reruns deduplicate rows but do not preserve earlier authorization decisions. +Accept those resulting grants before the first backfill, or defer backfill until refusal-preserving behavior exists. +Changing read modes does not revoke grants, and preventing reevaluation later does not remove grants already created. +Any unwanted existing grants need a provenance-based repair before enforcement. +Every invocation restarts the scan from the beginning; rows are deduplicated, but earlier authorization decisions are not preserved. Previously unshared uploads without uploader proof still need an upload of the exact bytes before they can be shared. Historical uploader backfill and unparsed-reference recovery remain separate work. @@ -45,7 +49,7 @@ To roll back file denials, set `off` or `log-only` and restart every pod. A community owner or admin who can view the moderation queue may read an attachment on a reported message outside their channels while that report has status `open`. The report, message, and file grant must belong to the requesting community. -The source message must still exist and be undeleted. +The source event and an edit's target message must still exist and be undeleted. Report notes, arbitrary URLs, pubkey reports, and blob-only reports grant no extra access. There is no role-based bypass for unrelated files, and reports create no public shares. From 658dbbb82a1388397fd512981311167e48e6eda7 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 20:58:13 +1100 Subject: [PATCH 40/48] docs(media): require acceptance before first backfill Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 81dddfabfd6..00b64721266 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -105,8 +105,10 @@ enum Command { /// Run after every relay pod records shares and /// before enforcing channel-scoped media reads. Historical references are /// trusted only before each community's schema-recorded cutoff. - /// Later references use current permissions, so reruns can grant references + /// Later references use current permissions, so any run can grant references /// live recording previously refused if their authors have since gained access. + /// Accept the resulting lasting grants before the first backfill; otherwise + /// defer it. Each invocation restarts the scan and deduplicates existing rows. BackfillMediaShares { /// Events read and recorded per transaction. #[arg(long, default_value_t = 1000)] From 7b8bee1e07f43ff4b7653a480865766197ce1007 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 21:07:22 +1100 Subject: [PATCH 41/48] fix(media): honor edited message lifetime in report reads Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/media_report.rs | 11 ++++-- .../src/api/media_access_postgres_tests.rs | 37 ++++++++++++++++++- .../tests/e2e_media_extended.rs | 27 +++++++++++++- .../tests/e2e_media_review.rs | 35 ++++++++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/crates/buzz-db/src/store/media_report.rs b/crates/buzz-db/src/store/media_report.rs index 43c3639b80e..b4d88349194 100644 --- a/crates/buzz-db/src/store/media_report.rs +++ b/crates/buzz-db/src/store/media_report.rs @@ -23,15 +23,18 @@ impl Db { crate::observability::WriterOperation::Authorization, ) .await?; - Ok(sqlx::query_as( + Ok(sqlx::query_as(concat!( "SELECT r.id, r.target_event_id FROM moderation_reports r \ JOIN media_shares s ON s.community_id = r.community_id \ - AND s.event_id = r.target_event_id AND s.sha256 = $2 \ + AND (s.event_id = r.target_event_id OR s.target_event_id = r.target_event_id) \ + AND s.sha256 = $2 \ JOIN events e ON e.community_id = s.community_id AND e.id = s.event_id \ - AND e.created_at = s.event_created_at AND e.deleted_at IS NULL \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + " \ WHERE r.community_id = $1 AND r.target_kind = 'event' AND r.status = 'open' \ ORDER BY r.created_at, r.id LIMIT 1", - ) + )) .bind(community.as_uuid()) .bind(sha256) .fetch_optional(&mut *conn) diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests.rs index 4570cd7f2aa..3ae9665ac3f 100644 --- a/crates/buzz-relay/src/api/media_access_postgres_tests.rs +++ b/crates/buzz-relay/src/api/media_access_postgres_tests.rs @@ -1,7 +1,7 @@ use super::*; use buzz_db::channel::{ChannelType, ChannelVisibility}; use buzz_db::moderation::{NewReport, ReportTarget}; -use nostr::{EventBuilder, Keys, Kind}; +use nostr::{EventBuilder, Keys, Kind, Tag}; use sqlx::PgPool; use std::time::Duration; use uuid::Uuid; @@ -149,6 +149,24 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { .id; let (hash, source) = post(&state, &tenant, &owner, private).await; let (unrelated, _) = post(&state, &tenant, &owner, private).await; + let (edited_hash, seed) = post(&state, &tenant, &owner, private).await; + let edit = EventBuilder::new(Kind::from(40003), format!("/media/{edited_hash}")) + .tags([ + Tag::parse(["h", &private.to_string()]).unwrap(), + Tag::parse(["e", &source.id.to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + state + .db + .insert_event(community, &edit, Some(private)) + .await + .unwrap(); + state + .db + .soft_delete_event_and_update_thread(community, seed.id.as_bytes(), None, None) + .await + .unwrap(); for policy in [ MediaReadMode::Off, MediaReadMode::LogOnly, @@ -204,6 +222,12 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { assert_eq!(audit.1, hash); assert_eq!(audit.2["report_id"], report_id.to_string()); assert_eq!(audit.2["event_id"], source.id.to_hex()); + assert!( + check(&state, &tenant, &reviewer, &edited_hash) + .await + .is_ok(), + "a report of the logical message covers its live edit attachment" + ); assert!(matches!( check(&state, &tenant, &stranger, &hash).await, Err(MediaError::NotFound) @@ -286,6 +310,10 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { ), "closing a report revokes the exception" ); + assert!(matches!( + check(&state, &tenant, &reviewer, &edited_hash).await, + Err(MediaError::NotFound) + )); report( &state, &tenant, @@ -340,6 +368,13 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { ), "deleted messages grant no exception" ); + assert!( + matches!( + check(&state, &tenant, &reviewer, &edited_hash).await, + Err(MediaError::NotFound) + ), + "an open report cannot revive an edit of a deleted message" + ); pool.close().await; for policy in [ MediaReadMode::Off, diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 2ff4977f5d2..2f1fbc74dd7 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -934,11 +934,31 @@ async fn test_deleted_messages_revoke_only_their_own_media_shares() { assert!(result.accepted, "attachment rejected: {}", result.message); messages.push(event); } + let added = upload(&http, &owner, &uuid::Uuid::new_v4().as_bytes().repeat(8)).await; + assert_eq!(added.status(), 200); + let added_desc: serde_json::Value = added.json().await.unwrap(); + let added_url = added_desc["url"].as_str().unwrap(); + for message in &messages[1..] { + let edit = EventBuilder::new(Kind::from(40003), "retain and add attachments") + .tags([ + Tag::parse(["h", &open]).unwrap(), + Tag::parse(["e", &message.id.to_hex()]).unwrap(), + imeta_for(&desc), + imeta_for(&added_desc), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = client.send_event(edit).await.unwrap(); + assert!(result.accepted, "edit rejected: {}", result.message); + } for reader in [&owner, &viewer] { for method in [Method::GET, Method::HEAD] { assert_eq!(read_status(&http, reader, method, url).await, 200); } } + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method, added_url).await, 200); + } // Author deletion removes the first post; the second must keep its grant. // Moderator removal then removes the last open-channel grant. for (kind, index, expected) in [(5, 1, 200), (9005, 2, 404)] { @@ -959,7 +979,12 @@ async fn test_deleted_messages_revoke_only_their_own_media_shares() { read_status(&http, &viewer, method.clone(), url).await, expected ); - assert_eq!(read_status(&http, &owner, method, url).await, 200); + assert_eq!(read_status(&http, &owner, method.clone(), url).await, 200); + assert_eq!( + read_status(&http, &viewer, method.clone(), added_url).await, + expected, + "deleting the logical message must revoke its edit-only attachment" + ); } } diff --git a/crates/buzz-test-client/tests/e2e_media_review.rs b/crates/buzz-test-client/tests/e2e_media_review.rs index 346414cb32e..1578d222837 100644 --- a/crates/buzz-test-client/tests/e2e_media_review.rs +++ b/crates/buzz-test-client/tests/e2e_media_review.rs @@ -146,6 +146,41 @@ async fn open_report_grants_only_audited_moderator_attachment_access() { ); read(&http, &moderator, &base, &host, hash, 404).await; read(&http, &owner, &base, &host, hash, 200).await; + + // A later edit adds a different attachment. The report targets the logical + // message, and deleting it must revoke the exception even while open. + let edited_hash = &sources[1].0; + let edit = EventBuilder::new(Kind::from(40003), format!("{base}/media/{edited_hash}")) + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["e", &event_id.to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(edit).await.unwrap().accepted); + let reopened = EventBuilder::new(Kind::from(1984), "review edited attachment") + .allow_self_tagging() + .tags([ + Tag::parse(["e", &event_id.to_hex(), "spam"]).unwrap(), + Tag::parse(["p", &owner.public_key().to_hex(), "spam"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(reopened).await.unwrap().accepted); + read(&http, &moderator, &base, &host, edited_hash, 200).await; + read(&http, &stranger, &base, &host, edited_hash, 404).await; + let deletion = EventBuilder::new(Kind::EventDeletion, "delete edited message") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["e", &event_id.to_hex()]).unwrap(), + Tag::parse(["k", "9"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + assert!(author.send_event(deletion).await.unwrap().accepted); + read(&http, &moderator, &base, &host, edited_hash, 404).await; + // Its independent private post survives, so the channel owner still reads it. + read(&http, &owner, &base, &host, edited_hash, 200).await; author.disconnect().await.unwrap(); reviewer.disconnect().await.unwrap(); } From 11f017f234a38a21f12ba1964cc3ff2f9247d060 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 21:24:18 +1100 Subject: [PATCH 42/48] fix(db): bound the media migration lock wait Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/runtime/migration.rs | 3 +++ migrations/0068_media_access.sql | 2 ++ 2 files changed, 5 insertions(+) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 701a36fb639..12f89c0504c 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -811,6 +811,9 @@ mod postgres_tests { .as_str() .contains("CREATE TABLE community_nip_fi_issuers")); assert_eq!(migrations[67].version, 68); + assert!(strip_sql_comments(migrations[67].sql.as_str()) + .trim_start() + .starts_with("SET LOCAL lock_timeout = '5s';")); for table in ["media_uploaders", "media_shares"] { assert!(migrations[67] .sql diff --git a/migrations/0068_media_access.sql b/migrations/0068_media_access.sql index 7d062fd9025..2e8fd7289c9 100644 --- a/migrations/0068_media_access.sql +++ b/migrations/0068_media_access.sql @@ -1,3 +1,5 @@ +SET LOCAL lock_timeout = '5s'; + -- Durable historical backfill cutoff, independent of the schema install tool. ALTER TABLE communities ADD COLUMN media_shares_recording_started_at TIMESTAMPTZ NOT NULL DEFAULT now(); From f67333b2005f457713e35864d5958dc385b61595 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 23:24:06 +1100 Subject: [PATCH 43/48] fix(media): reclaim inactive grants and cover upload attribution Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/lib.rs | 8 +- crates/buzz-db/src/store/media_cleanup.rs | 120 ++++++++++++++++++ crates/buzz-db/src/store/media_reference.rs | 95 ++++++++++++++ crates/buzz-db/src/store/mod.rs | 2 + crates/buzz-relay/src/main.rs | 15 +++ .../tests/e2e_media_extended.rs | 24 ++++ 6 files changed, 260 insertions(+), 4 deletions(-) create mode 100644 crates/buzz-db/src/store/media_cleanup.rs diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f495983a28d..7fa87929f45 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -66,10 +66,10 @@ pub(crate) use runtime::{ }; pub use store::{ admin_moderation, allowlist, api_token, archived_identities, artifact, channel, - channel_members, community, deletion, dm, event, feed, git_repo, moderation, nip_fi_binding, - operator_listener, partition, personal_read, product_feedback, push, reaction, read_state, - relay_admin_actions, relay_invite, relay_members, relay_operators, reminder, replaceable, - storage_accounting, thread, thread_window, usage, user, workflow, + channel_members, community, deletion, dm, event, feed, git_repo, media_cleanup, moderation, + nip_fi_binding, operator_listener, partition, personal_read, product_feedback, push, reaction, + read_state, relay_admin_actions, relay_invite, relay_members, relay_operators, reminder, + replaceable, storage_accounting, thread, thread_window, usage, user, workflow, }; pub use allowlist::AllowlistEntry; diff --git a/crates/buzz-db/src/store/media_cleanup.rs b/crates/buzz-db/src/store/media_cleanup.rs new file mode 100644 index 00000000000..a07c51b52b4 --- /dev/null +++ b/crates/buzz-db/src/store/media_cleanup.rs @@ -0,0 +1,120 @@ +//! Bounded, tenant-admitted reclamation of grants whose sources are gone. + +use std::collections::BTreeMap; + +use buzz_datastore_tracing::datastore_span; +use sqlx::FromRow; +use uuid::Uuid; + +use crate::{Db, Result}; + +/// Volatile primary-key position for the periodic media-share reaper. +/// Restarting a scan is safe: cleanup never creates an authorization grant. +#[derive(Clone, Debug, FromRow)] +pub struct MediaShareCleanupCursor { + community_id: Uuid, + sha256: String, + event_id: Vec, +} + +/// Committed cleanup and the next scan position; `None` starts another cycle. +pub struct MediaShareCleanupBatch { + /// Number of inactive grant rows removed. + pub removed: u64, + /// Continue after this key, or wrap to the beginning on the next tick. + pub next_cursor: Option, +} + +impl Db { + /// Inspect at most `batch_size` grants in primary-key order and remove + /// inactive ones under each community's write admission. + /// + /// Keep the first channel grant per blob, even if inactive: otherwise + /// removing its last historical channel row would restore uploader preview + /// access. This bounds dead channel history to one row per blob; public + /// profile/emoji grants need no such marker. Live sources are never removed. + /// Tenant-local errors retry in a later scan cycle, as do skipped locks. + #[datastore_span(name = "reap_inactive_media_shares", system = "postgresql")] + pub async fn reap_inactive_media_shares( + &self, + after: Option<&MediaShareCleanupCursor>, + batch_size: u16, + ) -> Result { + if !(1..=1000).contains(&batch_size) { + return Err(crate::DbError::InvalidData( + "media cleanup batch size must be between 1 and 1000".into(), + )); + } + let mut conn = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; + // Intentional fleet-wide candidate discovery, bounded by the leading + // primary key. It reads only keys; every mutation is admitted below. + let rows = sqlx::query_as::<_, MediaShareCleanupCursor>( + "SELECT community_id, sha256, event_id FROM media_shares \ + WHERE (community_id, sha256, event_id) > ($1, $2, $3) \ + ORDER BY community_id, sha256, event_id LIMIT $4", + ) + .bind(after.map_or(Uuid::nil(), |key| key.community_id)) + .bind(after.map_or("", |key| key.sha256.as_str())) + .bind(after.map_or(&[][..], |key| key.event_id.as_slice())) + .bind(i64::from(batch_size)) + .fetch_all(&mut *conn) + .await?; + drop(conn); + let next_cursor = (rows.len() == usize::from(batch_size)) + .then(|| rows.last().cloned()) + .flatten(); + let mut groups = BTreeMap::, Vec>)>::new(); + for row in rows { + let (hashes, ids) = groups.entry(row.community_id).or_default(); + hashes.push(row.sha256); + ids.push(row.event_id); + } + let groups = &groups; + let removed = crate::sweep_admitted_communities( + &self.pool, + groups.keys().copied(), + crate::observability::WriterOperation::Maintenance, + "media_share_reap", + |community, mut tx| async move { + sqlx::raw_sql("SET LOCAL lock_timeout = '1s'; SET LOCAL statement_timeout = '5s'") + .execute(tx.conn()) + .await?; + let (hashes, ids) = &groups[&community]; + let removed = sqlx::query(concat!( + "DELETE FROM media_shares s \ + USING unnest($2::text[], $3::bytea[]) AS batch(sha256, event_id) \ + WHERE s.community_id = $1 AND s.sha256 = batch.sha256 \ + AND s.event_id = batch.event_id \ + AND NOT EXISTS (SELECT 1 FROM events e \ + WHERE e.community_id = s.community_id AND e.id = s.event_id \ + AND e.created_at = s.event_created_at AND ", + crate::store::media_reference::live_source_sql!(), + ") AND (s.channel_id IS NULL OR EXISTS ( \ + SELECT 1 FROM media_shares previous \ + WHERE previous.community_id = s.community_id \ + AND previous.sha256 = s.sha256 AND previous.channel_id IS NOT NULL \ + AND previous.event_id < s.event_id))", + )) + .bind(community) + .bind(hashes) + .bind(ids) + .execute(tx.conn()) + .await? + .rows_affected(); + tx.commit().await?; + Ok(removed) + }, + ) + .await + .into_iter() + .sum(); + Ok(MediaShareCleanupBatch { + removed, + next_cursor, + }) + } +} diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 00fbd919c6c..875ba582e9d 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -464,7 +464,69 @@ mod postgres_tests { !readable_share(&db, community, &sha, &reader).await, "a purged source cannot leave a dangling grant" ); + assert!(reap_all(&db).await > 0); + assert_eq!( + shares(&db, community, &sha).await.len(), + usize::from(channel_id.is_some()), + "cleanup retains at most one channel boundary, never public history" + ); + if let Some(edit_sha) = &edit_only { + assert_eq!( + shares(&db, community, edit_sha).await.len(), + 1, + "inactive edit grants compact to one channel marker" + ); + } + assert!(!readable_share(&db, community, &sha, &reader).await); + assert_eq!(reap_all(&db).await, 0, "cleanup is idempotent"); } + + // Fleet cleanup skips a quiescing community while reclaiming another + // community's dead copy of the same hash and keeping its live copy. + let sha = unique_sha(); + upload(&db, community, &owner, &sha).await; + let retired = attachment(&owner, &sha, None); + db.insert_event(community, &retired, None).await.unwrap(); + db.soft_delete_event_and_update_thread(community, retired.id.as_bytes(), None, None) + .await + .unwrap(); + let mut lifecycle = db.pool.begin().await.unwrap(); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.as_uuid().to_string()) + .execute(&mut *lifecycle) + .await + .unwrap(); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(community.as_uuid()) + .execute(&mut *lifecycle) + .await + .unwrap(); + lifecycle.commit().await.unwrap(); + let (other, other_community) = fixture().await; + upload(&other, other_community, &owner, &sha).await; + let old = attachment(&owner, &sha, None); + let current = attachment(&owner, &sha, None); + for event in [&old, ¤t] { + other + .insert_event(other_community, event, None) + .await + .unwrap(); + } + other + .soft_delete_event_and_update_thread(other_community, old.id.as_bytes(), None, None) + .await + .unwrap(); + assert!(reap_all(&db).await > 0); + assert_eq!( + shares(&db, community, &sha).await.len(), + 1, + "cleanup may not write into a quiescing community" + ); + assert_eq!(shares(&other, other_community, &sha).await.len(), 1); + assert!(readable_share(&other, other_community, &sha, &reader).await); } async fn readable_share(db: &Db, community: CommunityId, sha: &str, reader: &Keys) -> bool { @@ -481,6 +543,25 @@ mod postgres_tests { .unwrap() } + async fn reap_all(db: &Db) -> u64 { + let mut cursor = None; + let mut removed = 0; + // One row per page exercises progress across live rows and retained + // channel markers as well as rows deleted by a previous page. + for _ in 0..10_000 { + let batch = db + .reap_inactive_media_shares(cursor.as_ref(), 1) + .await + .unwrap(); + removed += batch.removed; + cursor = batch.next_cursor; + if cursor.is_none() { + return removed; + } + } + panic!("cleanup must finish its bounded primary-key scan"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn replacement_preserves_only_the_media_it_continues_to_reference() { @@ -535,6 +616,12 @@ mod postgres_tests { assert!(publish(&make(&uploader, Kind::Metadata, 1, false)).await); assert!(readable_share(&db, community, &sha, &viewer).await); assert!(publish(&make(&publisher, kind, 1, true)).await); + assert!(reap_all(&db).await > 0); + assert_eq!( + shares(&db, community, &sha).await.len(), + 1, + "cleanup keeps the surviving profile/emoji head" + ); assert!( readable_share(&db, community, &sha, &viewer).await, "a replacement must carry forward an authorized reference" @@ -569,6 +656,8 @@ mod postgres_tests { .unwrap(); tx.commit().await.unwrap(); assert!(publish(&legacy_make(1, true)).await); + assert!(reap_all(&db).await > 0); + assert_eq!(shares(&db, community, &legacy_sha).await.len(), 1); db.backfill_media_shares(1).await.unwrap(); assert!( readable_share(&db, community, &legacy_sha, &viewer).await, @@ -578,6 +667,12 @@ mod postgres_tests { assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); assert!(publish(&legacy_make(3, true)).await); assert!(!readable_share(&db, community, &legacy_sha, &viewer).await); + assert!(reap_all(&db).await > 0); + assert!( + shares(&db, community, &legacy_sha).await.is_empty(), + "replaced avatar/emoji heads must not accumulate inactive rows" + ); + assert!(shares(&db, community, &sha).await.is_empty()); } } diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 6cf1e942a07..f1e290fb15b 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -32,6 +32,8 @@ mod event_follow_up_postgres_tests; pub mod feed; /// Git repository name registry (NIP-34 kind:30617). pub mod git_repo; +/// Bounded reclamation of inactive media grants. +pub mod media_cleanup; /// Where each media blob is shared, recorded with the events that reference it. mod media_grant; pub mod media_reference; diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index fa181f2a473..0e2167b5e22 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -134,6 +134,7 @@ const _: () = assert!(FLEET_STOCK_COLLECTION_DEADLINE.as_secs() > 5); const _: () = assert!(FLEET_ACTIVITY_COLLECTION_DEADLINE.as_secs() > 15); struct FleetUsageSchedule { + media_cleanup_cursor: Option, next_stock: std::time::Instant, next_activity: std::time::Instant, last_stock_success: Option, @@ -167,6 +168,7 @@ impl FleetUsageSchedule { retry_interval: std::time::Duration, ) -> Self { Self { + media_cleanup_cursor: None, next_stock: now, next_activity: now, last_stock_success: None, @@ -2517,6 +2519,19 @@ async fn run_usage_metrics_tick( emit_fleet_db_usage_metrics(state, fleet_schedule).await; } let invite_retention_cutoff = chrono::Utc::now() - chrono::Duration::days(30); + match state + .db + .reap_inactive_media_shares(fleet_schedule.media_cleanup_cursor.as_ref(), 1000) + .await + { + Ok(batch) => { + fleet_schedule.media_cleanup_cursor = batch.next_cursor; + if batch.removed > 0 { + info!(removed = batch.removed, "reaped inactive media shares"); + } + } + Err(error) => warn!(error = %error, "failed to reap inactive media shares"), + } match state .db .reap_expired_relay_invites(invite_retention_cutoff) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 9bd73b13fcb..aefee104e4a 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -37,6 +37,12 @@ fn http_client() -> Client { .expect("http client") } +async fn media_db_pool() -> sqlx::PgPool { + let database = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".into()); // sadscan:disable np.postgres.1 -- local test-only credentials + sqlx::PgPool::connect(&database).await.unwrap() +} + fn sign_blossom_auth(keys: &Keys, sha256: &str) -> nostr::Event { let now = Timestamp::now().as_secs(); let server = relay_server_authority(); @@ -200,6 +206,24 @@ async fn test_upload_png_roundtrip() { // GET back — reads are authenticated, so scope a token to the uploaded hash. let sha256 = desc["sha256"].as_str().expect("descriptor sha256"); + // Bind uploader recording to the real HTTP handler, independently of + // whether this relay currently enforces file reads. + let pool = media_db_pool().await; + let recorded: i64 = sqlx::query_scalar( + "SELECT count(*) FROM media_uploaders u JOIN communities c ON c.id = u.community_id \ + WHERE c.host = $1 AND u.sha256 = $2 AND u.pubkey = $3", + ) + .bind(relay_server_authority()) + .bind(sha256) + .bind(keys.public_key().to_bytes().as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + recorded, 1, + "successful upload must persist its authenticated uploader" + ); + pool.close().await; let get = client .get(desc["url"].as_str().unwrap()) .header( From 42e69e65733fee4526d7aef90ec8f2149b916cf6 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 23:29:35 +1100 Subject: [PATCH 44/48] fix(media): run cleanup before telemetry collection Signed-off-by: Nikolaj Goodger --- crates/buzz-relay/src/main.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 0e2167b5e22..725ca1d8d61 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -2507,18 +2507,7 @@ async fn run_usage_metrics_tick( } } if leader.is_some() { - if emission_scope.includes_per_community() { - let result = emit_db_usage_metrics(state, emission_scope, &host_map).await; - record_legacy_usage_collection(result.is_ok(), fleet_schedule); - if let Err(error) = result { - warn!("Usage metrics leader demoting: DB collection failed"); - *leader = None; - return Err(error); - } - } else { - emit_fleet_db_usage_metrics(state, fleet_schedule).await; - } - let invite_retention_cutoff = chrono::Utc::now() - chrono::Duration::days(30); + // Run maintenance before optional telemetry collection. match state .db .reap_inactive_media_shares(fleet_schedule.media_cleanup_cursor.as_ref(), 1000) @@ -2532,6 +2521,18 @@ async fn run_usage_metrics_tick( } Err(error) => warn!(error = %error, "failed to reap inactive media shares"), } + if emission_scope.includes_per_community() { + let result = emit_db_usage_metrics(state, emission_scope, &host_map).await; + record_legacy_usage_collection(result.is_ok(), fleet_schedule); + if let Err(error) = result { + warn!("Usage metrics leader demoting: DB collection failed"); + *leader = None; + return Err(error); + } + } else { + emit_fleet_db_usage_metrics(state, fleet_schedule).await; + } + let invite_retention_cutoff = chrono::Utc::now() - chrono::Duration::days(30); match state .db .reap_expired_relay_invites(invite_retention_cutoff) From 4c74b29e6669518842463bc3b4240a6ce6a3a0e7 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 23:33:01 +1100 Subject: [PATCH 45/48] test(media): verify cleanup preserves file access boundaries Signed-off-by: Nikolaj Goodger --- .github/workflows/_ci-relay.yml | 1 + .../src/api/media_access_postgres_tests.rs | 21 ++++++++++ .../tests/e2e_media_extended.rs | 39 +++++++++++++++++++ docs/operator-media-access.md | 18 +++++++++ 4 files changed, 79 insertions(+) diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index 109e8726f89..9dafd9624cd 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -600,6 +600,7 @@ jobs: name: Relay E2E env: BUZZ_MEDIA_READ_MODE: enforce + BUZZ_USAGE_METRICS_INTERVAL_SECS: "5" runs-on: ubuntu-latest timeout-minutes: 20 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.rust) diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests.rs index 3ae9665ac3f..07894296052 100644 --- a/crates/buzz-relay/src/api/media_access_postgres_tests.rs +++ b/crates/buzz-relay/src/api/media_access_postgres_tests.rs @@ -375,6 +375,27 @@ async fn rollout_and_report_review_preserve_private_file_boundaries() { ), "an open report cannot revive an edit of a deleted message" ); + let mut cleanup_cursor = None; + loop { + let batch = state + .db + .reap_inactive_media_shares(cleanup_cursor.as_ref(), 1000) + .await + .unwrap(); + cleanup_cursor = batch.next_cursor; + if cleanup_cursor.is_none() { + break; + } + } + for cleaned_hash in [&hash, &edited_hash] { + assert!( + matches!( + check(&state, &tenant, &owner, cleaned_hash).await, + Err(MediaError::NotFound) + ), + "cleanup must not restore uploader preview for a deleted channel attachment" + ); + } pool.close().await; for policy in [ MediaReadMode::Off, diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 514cb22a1b5..498600cf4dd 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -44,6 +44,30 @@ async fn media_db_pool() -> sqlx::PgPool { sqlx::PgPool::connect(&database).await.unwrap() } +async fn wait_for_share_cleanup(hash: &str, maximum: i64) { + let pool = media_db_pool().await; + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM media_shares s JOIN communities c ON c.id = s.community_id \ + WHERE c.host = $1 AND s.sha256 = $2", + ) + .bind(relay_server_authority()) + .bind(hash) + .fetch_one(&pool) + .await + .unwrap(); + if remaining <= maximum { + break; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + }) + .await + .expect("periodic cleanup must reclaim inactive grant rows"); + pool.close().await; +} + fn sign_blossom_auth(keys: &Keys, sha256: &str) -> nostr::Event { let now = Timestamp::now().as_secs(); let server = relay_server_authority(); @@ -1012,6 +1036,17 @@ async fn test_deleted_messages_revoke_only_their_own_media_shares() { } } + // Wait for the actual periodic worker to compact the two dead edit grants. + // Losing every channel row would incorrectly restore uploader preview. + wait_for_share_cleanup(added_desc["sha256"].as_str().unwrap(), 1).await; + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &owner, method.clone(), added_url).await, + 404 + ); + assert_eq!(read_status(&http, &viewer, method, added_url).await, 404); + } + // A community-wide note grants access only for as long as it survives. let note = EventBuilder::new(Kind::TextNote, format!("shared file {url}")) .sign_with_keys(&owner) @@ -1140,6 +1175,10 @@ async fn test_profile_replacement_preserves_only_its_retained_avatar() { ); } } + wait_for_share_cleanup(descriptor["sha256"].as_str().unwrap(), 0).await; + for method in [Method::GET, Method::HEAD] { + assert_eq!(read_status(&http, &viewer, method, url).await, 404); + } } #[tokio::test] diff --git a/docs/operator-media-access.md b/docs/operator-media-access.md index 5c7a56933f7..eb341960ab8 100644 --- a/docs/operator-media-access.md +++ b/docs/operator-media-access.md @@ -63,3 +63,21 @@ Closing the report, deleting the source message, or removing the queue role ends for subsequent requests; ordinary shares can still independently authorize access. Escalated and processing reports do not qualify for this community-review exception. Bytes already downloaded cannot be recalled. + +## Inactive grant cleanup + +The elected usage-poller leader inspects at most 1,000 media grants per tick, +in primary-key order, and removes inactive grants in community-admitted transactions. +The tick uses `BUZZ_USAGE_METRICS_INTERVAL_SECS` (300 seconds by default). +The scan wraps at the end; its cursor is held in memory and restarts after a relay restart. +Tenant-local failures retry on a later scan cycle, and quiescing/deleting communities are skipped. + +A grant is inactive when its source event is missing or deleted, or its edit target is no longer live. +Live grants survive cleanup. Inactive public profile/emoji grants are removed completely. +Cleanup compacts inactive channel history to at most one row per community and file: removing every +channel row would incorrectly restore uploader preview access after message deletion. +Whole-community purge removes that final row as well. + +Successful reclamation logs `reaped inactive media shares` with the number removed; +failures log `failed to reap inactive media shares` or a tenant-local sweep error. +This reclaims grant records; file retention follows the storage policy. From 2a6765309e48f9a01b983df7f388a29057084276 Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Thu, 22 Oct 2026 23:53:22 +1100 Subject: [PATCH 46/48] fix(media): finish deferred provenance and access feedback Signed-off-by: Nikolaj Goodger --- crates/buzz-admin/src/main.rs | 4 +- crates/buzz-core/src/media.rs | 4 +- crates/buzz-db/src/store/media_reference.rs | 168 ++++++++++++++++++-- crates/buzz-db/src/store/media_uploader.rs | 63 ++++++++ crates/buzz-relay/src/api/media.rs | 10 +- 5 files changed, 231 insertions(+), 18 deletions(-) diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index 00b64721266..9d7f396dbf5 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -105,13 +105,15 @@ enum Command { /// Run after every relay pod records shares and /// before enforcing channel-scoped media reads. Historical references are /// trusted only before each community's schema-recorded cutoff. + /// Restores uploader proof from surviving server media_uploaded audit rows; + /// files without that evidence still need an upload of the exact bytes. /// Later references use current permissions, so any run can grant references /// live recording previously refused if their authors have since gained access. /// Accept the resulting lasting grants before the first backfill; otherwise /// defer it. Each invocation restarts the scan and deduplicates existing rows. BackfillMediaShares { /// Events read and recorded per transaction. - #[arg(long, default_value_t = 1000)] + #[arg(long, default_value_t = 1000, value_parser = clap::value_parser!(i64).range(1..))] batch_size: i64, }, /// Run the partition catalog audit using a read-only database session. diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs index bac5f48dceb..1685ae5f566 100644 --- a/crates/buzz-core/src/media.rs +++ b/crates/buzz-core/src/media.rs @@ -3,7 +3,8 @@ use std::collections::BTreeSet; use crate::kind::{ - event_is_shared, is_author_only_event_kind, is_shared_gated_kind, P_GATED_KINDS, + event_is_shared, is_author_only_event_kind, is_shared_gated_kind, KIND_AGENT_ENGRAM, + P_GATED_KINDS, }; /// Hashes of every blob `event` shares into its channel, or with the whole @@ -17,6 +18,7 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec { let kind = u32::from(event.kind.as_u16()); if is_author_only_event_kind(event) || P_GATED_KINDS.contains(&kind) + || kind == KIND_AGENT_ENGRAM || (is_shared_gated_kind(kind) && !event_is_shared(event)) { return Vec::new(); diff --git a/crates/buzz-db/src/store/media_reference.rs b/crates/buzz-db/src/store/media_reference.rs index 875ba582e9d..3bd57f746a2 100644 --- a/crates/buzz-db/src/store/media_reference.rs +++ b/crates/buzz-db/src/store/media_reference.rs @@ -108,6 +108,10 @@ impl Db { /// reading `batch_size` events at a time; returns /// the number of shares added. /// + /// Retained server upload audits restore uploader proof before references + /// are evaluated. Missing, malformed, or unattributed audit rows establish + /// no ownership; those uploads still need the exact bytes uploaded again. + /// /// Shares come from the same rule as live writes. Events received before /// share recording was installed predate the readability check, so their /// references are recorded as they stand; later events get the live, @@ -130,6 +134,7 @@ impl Db { let mut added = 0; for (community, installed) in communities { let community = CommunityId::from_uuid(community); + self.backfill_media_uploaders(community, batch_size).await?; let mut cursor: Option<(DateTime, Vec)> = None; loop { let rows = sqlx::query( @@ -343,6 +348,44 @@ mod postgres_tests { expected, "an uploader may share their own bytes" ); + + let excluded = unique_sha(); + upload(&db, community, &alice, &excluded).await; + use buzz_core::kind::{ + AUTHOR_ONLY_KINDS, KIND_AGENT_ENGRAM, KIND_DELETION, KIND_EVENT_REMINDER, + P_GATED_KINDS, SHARED_GATED_KINDS, + }; + let kinds = AUTHOR_ONLY_KINDS + .iter() + .chain(P_GATED_KINDS) + .chain(SHARED_GATED_KINDS) + .copied() + .chain([KIND_AGENT_ENGRAM, KIND_DELETION]); + for kind in kinds { + if (20_000..30_000).contains(&kind) { + continue; // Ephemeral frames have no stored reference to backfill. + } + let mut tags = vec![Tag::parse(["d", "private-media"]).unwrap()]; + if kind == KIND_DELETION { + tags.push(Tag::parse(["k", &KIND_EVENT_REMINDER.to_string()]).unwrap()); + } + let event = EventBuilder::new(Kind::from(kind as u16), format!("/media/{excluded}")) + .tags(tags) + .sign_with_keys(&alice) + .unwrap(); + db.insert_event(community, &event, None).await.unwrap(); + assert!( + db.get_event_by_id(community, event.id.as_bytes()) + .await + .unwrap() + .is_some(), + "kind {kind} must exercise a stored event" + ); + assert!( + shares(&db, community, &excluded).await.is_empty(), + "private kind {kind} must create no grant even for its uploader" + ); + } } #[tokio::test] @@ -490,21 +533,7 @@ mod postgres_tests { db.soft_delete_event_and_update_thread(community, retired.id.as_bytes(), None, None) .await .unwrap(); - let mut lifecycle = db.pool.begin().await.unwrap(); - sqlx::query( - "SELECT set_config('buzz.deletion_executor_community', $1, true), \ - set_config('buzz.deletion_fence_generation', '0', true)", - ) - .bind(community.as_uuid().to_string()) - .execute(&mut *lifecycle) - .await - .unwrap(); - sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") - .bind(community.as_uuid()) - .execute(&mut *lifecycle) - .await - .unwrap(); - lifecycle.commit().await.unwrap(); + quiesce(&db, community).await; let (other, other_community) = fixture().await; upload(&other, other_community, &owner, &sha).await; let old = attachment(&owner, &sha, None); @@ -543,6 +572,24 @@ mod postgres_tests { .unwrap() } + async fn quiesce(db: &Db, community: CommunityId) { + let mut lifecycle = db.pool.begin().await.unwrap(); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.as_uuid().to_string()) + .execute(&mut *lifecycle) + .await + .unwrap(); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(community.as_uuid()) + .execute(&mut *lifecycle) + .await + .unwrap(); + lifecycle.commit().await.unwrap(); + } + async fn reap_all(db: &Db) -> u64 { let mut cursor = None; let mut removed = 0; @@ -727,6 +774,73 @@ mod postgres_tests { .expect("forget shares"); tx.commit().await.expect("commit"); + // Surviving server upload audits recover legacy ownership; malformed, + // missing, or unrelated audit evidence never creates uploader proof. + let (recovered, unproven) = (unique_sha(), unique_sha()); + let (uploader, stranger) = (Keys::generate(), Keys::generate()); + db.insert_event(community, &attachment(&uploader, &recovered, None), None) + .await + .unwrap(); + db.insert_event(community, &attachment(&stranger, &unproven, None), None) + .await + .unwrap(); + let mut tx = db.begin_media_share_transaction(community).await.unwrap(); + for (seq, action, actor, object) in [ + ( + 1_i64, + "media_uploaded", + Some(uploader.public_key().to_bytes().to_vec()), + recovered.clone(), + ), + (2, "media_uploaded", None, unproven.clone()), + (3, "media_uploaded", Some(vec![1; 31]), unproven.clone()), + ( + 4, + "event_created", + Some(stranger.public_key().to_bytes().to_vec()), + unproven.clone(), + ), + ( + 5, + "media_uploaded", + Some(stranger.public_key().to_bytes().to_vec()), + "bad-hash".into(), + ), + ] { + sqlx::query( + "INSERT INTO audit_log (community_id, seq, hash, action, actor_pubkey, object_id) \ + VALUES ($1,$2,$3,$4,$5,$6)", + ) + .bind(community.as_uuid()) + .bind(seq) + .bind(vec![seq as u8; 32]) + .bind(action) + .bind(actor) + .bind(object) + .execute(tx.conn()) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + let (fenced, fenced_community) = fixture().await; + let mut tx = fenced + .begin_media_share_transaction(fenced_community) + .await + .unwrap(); + sqlx::query( + "INSERT INTO audit_log (community_id, seq, hash, action, actor_pubkey, object_id) \ + VALUES ($1,1,$2,'media_uploaded',$3,$4)", + ) + .bind(fenced_community.as_uuid()) + .bind(vec![1_u8; 32]) + .bind(stranger.public_key().to_bytes().as_slice()) + .bind(&unproven) + .execute(tx.conn()) + .await + .unwrap(); + tx.commit().await.unwrap(); + quiesce(&fenced, fenced_community).await; + db.backfill_media_shares(1).await.expect("backfill"); assert_eq!(db.backfill_media_shares(1).await.expect("rerun"), 0); assert_eq!( @@ -735,6 +849,30 @@ mod postgres_tests { "a reference live recording refused must stay unshared" ); assert_eq!(shares(&db, community, &historical).await, vec![None]); + let fenced_proofs: i64 = + sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1") + .bind(fenced_community.as_uuid()) + .fetch_one(&db.pool) + .await + .unwrap(); + assert_eq!( + fenced_proofs, 0, + "audit recovery must not write across a fenced tenant boundary" + ); + assert_eq!( + shares(&db, community, &recovered).await, + vec![None], + "retained upload evidence must recover an uploader's intended share" + ); + assert!(shares(&db, community, &unproven).await.is_empty()); + let proofs: Vec<(String, Vec)> = sqlx::query_as( + "SELECT sha256, pubkey FROM media_uploaders WHERE community_id = $1 AND sha256 = ANY($2)", + ).bind(community.as_uuid()).bind(vec![recovered.clone(), unproven]) + .fetch_all(&db.pool).await.unwrap(); + assert_eq!( + proofs, + vec![(recovered, uploader.public_key().to_bytes().to_vec())] + ); assert!( shares(&db, community, &deleted_history).await.is_empty(), "legacy deleted messages must never gain a share" diff --git a/crates/buzz-db/src/store/media_uploader.rs b/crates/buzz-db/src/store/media_uploader.rs index 0039575af04..4ced9087b59 100644 --- a/crates/buzz-db/src/store/media_uploader.rs +++ b/crates/buzz-db/src/store/media_uploader.rs @@ -4,8 +4,71 @@ use buzz_datastore_tracing::datastore_span; use crate::deletion::ServingWriteLease; use crate::{error::Result, observability, Db}; +use buzz_core::CommunityId; impl Db { + /// Recover possession proof from retained server-authored upload audits. + /// Scan a finite sequence range using audit_log's tenant-leading primary + /// key; malformed or unattributed records never establish ownership. + #[datastore_span(name = "backfill_media_uploaders", system = "postgresql")] + pub(crate) async fn backfill_media_uploaders( + &self, + community: CommunityId, + batch_size: i64, + ) -> Result { + let mut conn = + observability::acquire_writer(&self.pool, observability::WriterOperation::Maintenance) + .await?; + let upper: Option = + sqlx::query_scalar("SELECT max(seq) FROM audit_log WHERE community_id = $1") + .bind(community.as_uuid()) + .fetch_one(&mut *conn) + .await?; + drop(conn); + let Some(upper) = upper else { + return Ok(0); + }; + let (mut cursor, mut added) = (0_i64, 0_u64); + while cursor < upper { + let mut tx = crate::begin_community_write_transaction( + &self.pool, + community, + observability::WriterOperation::Maintenance, + ) + .await?; + let page: Vec = sqlx::query_scalar( + "SELECT seq FROM audit_log WHERE community_id = $1 AND seq > $2 AND seq <= $3 \ + ORDER BY seq LIMIT $4", + ) + .bind(community.as_uuid()) + .bind(cursor) + .bind(upper) + .bind(batch_size) + .fetch_all(tx.conn()) + .await?; + let Some(&last) = page.last() else { + break; + }; + added += sqlx::query( + "INSERT INTO media_uploaders (community_id, sha256, pubkey, created_at) \ + SELECT community_id, object_id, actor_pubkey, created_at FROM audit_log \ + WHERE community_id = $1 AND seq > $2 AND seq <= $3 \ + AND action = 'media_uploaded' AND octet_length(actor_pubkey) = 32 \ + AND object_id ~ '^[0-9a-f]{64}$' \ + ORDER BY object_id, actor_pubkey ON CONFLICT DO NOTHING", + ) + .bind(community.as_uuid()) + .bind(cursor) + .bind(last) + .execute(tx.conn()) + .await? + .rows_affected(); + tx.commit().await?; + cursor = last; + } + Ok(added) + } + /// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are /// no-ops. The write presents the upload's serving lease, so it may finish /// while the community quiesces but never after the lease is gone. diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index eead116018f..137f5acf01c 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -587,7 +587,15 @@ async fn upload_blob_result( auth.auth_event.pubkey.as_bytes(), ) .await - .map_err(|error| serving_lease_lost(error.into()))?; + .map_err(|error| { + let error = anyhow::Error::from(error); + if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) { + serving_lease_lost(error) + } else { + tracing::error!(%error, "media uploader recording failed"); + MediaError::ServiceUnavailable + } + })?; // Normalize MIME to a known set to bound label cardinality. let mime_label = match descriptor.mime_type.as_str() { From 648358800bf07fea2ce2c6984d163396740f498d Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Fri, 23 Oct 2026 00:02:14 +1100 Subject: [PATCH 47/48] test(media): cover removed members and document audit recovery Signed-off-by: Nikolaj Goodger --- .../tests/e2e_media_extended.rs | 33 +++++++++++++++++++ docs/operator-media-access.md | 7 ++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 498600cf4dd..31d1ea8cdf2 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -899,6 +899,39 @@ async fn test_private_channel_attachment_unreadable_to_non_members() { ); } + // Exercise the actual membership tombstone, not just never-joined readers. + let recipient = BuzzTestClient::connect(&relay_ws_url(), &outsider) + .await + .unwrap(); + let mut membership = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + for (kind, expected) in [(9000, 200), (9001, 404)] { + let change = EventBuilder::new(Kind::from(kind), "") + .tags([ + Tag::parse(["h", &channel_id]).unwrap(), + Tag::parse(["p", &outsider.public_key().to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = membership.send_event(change).await.unwrap(); + assert!( + result.accepted, + "membership change rejected: {}", + result.message + ); + for method in [Method::GET, Method::HEAD] { + assert_eq!( + read_status(&http, &outsider, method.clone(), url).await, + expected, + "removed private-channel members must lose media access" + ); + assert_eq!(read_status(&http, &owner, method, url).await, 200); + } + } + membership.disconnect().await.unwrap(); + recipient.disconnect().await.unwrap(); + let outsider_channel = create_channel(&http, &outsider, "outsider-media", "open").await; let mut outsider_client = BuzzTestClient::connect(&relay_ws_url(), &outsider) .await diff --git a/docs/operator-media-access.md b/docs/operator-media-access.md index eb341960ab8..92c120d914b 100644 --- a/docs/operator-media-access.md +++ b/docs/operator-media-access.md @@ -40,8 +40,11 @@ Changing read modes does not revoke grants, and preventing reevaluation later do Any unwanted existing grants need a provenance-based repair before enforcement. Every invocation restarts the scan from the beginning; rows are deduplicated, but earlier authorization decisions are not preserved. -Previously unshared uploads without uploader proof still need an upload of the exact bytes before they can be shared. -Historical uploader backfill and unparsed-reference recovery remain separate work. +Before recording shares, the backfill restores uploader proof from retained server +`media_uploaded` audit rows with an attributed 32-byte pubkey and a valid SHA-256 object ID. +It scans each community's audit sequence in bounded batches and skips quiescing/deleting communities. +Missing, malformed, or unattributed audit evidence grants no ownership; those files still +need an upload of the exact bytes. Unparsed references need separate repair before enforcement. To roll back file denials, set `off` or `log-only` and restart every pod. ## Open reports From bb447892964c4b208416b722700489dd55b3e6bf Mon Sep 17 00:00:00 2001 From: Nikolaj Goodger Date: Fri, 23 Oct 2026 13:41:52 +1100 Subject: [PATCH 48/48] test(db): expect only legacy tables to carry community write fences The optional-fence test assumed every scoped table is fenced, so the first tenant tables added under docs/database.md (scoped but admitted only in the app) broke it. Assert that fences exist exactly when the fence objects do and cover only scoped tables. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Nikolaj Goodger --- crates/buzz-db/src/store/deletion.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 777562106d4..c236d4158c4 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -4507,14 +4507,13 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); store.validate_catalog().await.expect("deletion catalog"); let inventory = store.inventory_schema(community).await.expect("inventory"); assert_eq!(inventory.scoped_tables.len(), EXPECTED_SCOPED_TABLES.len()); - assert_eq!( - inventory.fenced_tables.len(), - if objects_present { - EXPECTED_SCOPED_TABLES.len() - } else { - 0 - } - ); + // Tables added after fence retirement are admitted in the app and + // carry no fence, so only legacy tables are fenced. + assert_eq!(inventory.fenced_tables.is_empty(), !objects_present); + assert!(inventory + .fenced_tables + .iter() + .all(|table| inventory.scoped_tables.contains(table))); let lease = store .acquire_serving_write_lease( community,