diff --git a/.env.example b/.env.example index 98916ec2793..eb9a91a259b 100644 --- a/.env.example +++ b/.env.example @@ -185,6 +185,10 @@ BUZZ_PUSH_GATEWAY_DELIVERY_URL=https://push.buzz.xyz/v1/deliveries/apns # BUZZ_GIT_PACK_CACHE_MAX_BYTES=5368709120 # BUZZ_GIT_PACK_CACHE_MAX_CONCURRENT_POPULATIONS=2 +# Channel media revocation: off (default), shadow, enforce. Restart to change. +# Apply migration 0086 first; see docs/operator-private-media.md. +BUZZ_MEDIA_READ_MODE=off + # ----------------------------------------------------------------------------- # S3-Compatible Object Storage (media + Git/CAS) # ----------------------------------------------------------------------------- diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index 7652bab800e..3cd039baa0c 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -663,6 +663,8 @@ jobs: merge-multiple: true path: target/ci - name: Start relay + env: + BUZZ_MEDIA_READ_MODE: enforce run: | chmod +x ./target/ci/buzz-relay ./target/ci/git-credential-nostr ./scripts/start-relay-for-tests.sh --no-build diff --git a/Justfile b/Justfile index f595fab05bb..66fe30e07b4 100644 --- a/Justfile +++ b/Justfile @@ -560,7 +560,7 @@ test-unit: # cross-node fan-out loss handling; its stalled-delivery case fakes a # database that never answers on loopback, and none use DATABASE_URL. cargo nextest run -p buzz-relay --lib --bin buzz-relay \ - -E 'test(/^subscription::tests::/) + test(/^handlers::multinode_fanout::tests::/) + test(/^api::admin::/) + test(/^access::tests::/) + test(/^api::git::transport::access_log_tests::/) + test(=api::bridge::postgres_tests::nip98_replay_check_logs_only_its_refusal) + test(=api::bridge::postgres_tests::warn_only_artifact_read_lines_keep_the_filter_summary) + test(=connection::tests::close_totals_count_frames_the_writer_flushed) + test(=connection::tests::dispatcher_counts_each_req_and_count_frame_once) + test(=handlers::req::tests::unqueued_eose_logs_aborted_not_served) + test(=api::bridge::postgres_tests::reject_code_keeps_only_a_known_prefix) + test(=api::bridge::postgres_tests::nip98_replay_check_fails_closed_when_guard_errors) + test(=api::bridge::postgres_tests::verify_bridge_auth_rejects_nip98_event_signed_for_wrong_communitys_host) + test(=api::bridge::postgres_tests::verify_bridge_auth_can_require_payload_tag_for_json_body_endpoints) + test(=api::bridge::postgres_tests::verify_bridge_auth_accepts_nip98_event_signed_for_matching_host) + test(=api::bridge::postgres_tests::moderation_read_query_bearing_nip98_event_verifies_with_matching_query) + test(=api::bridge::postgres_tests::moderation_read_query_bearing_nip98_event_rejected_against_bare_path) + test(=api::bridge::postgres_tests::moderation_read_audit_query_bearing_nip98_event_verifies) + test(=api::bridge::postgres_tests::moderation_read_restricted_no_query_still_verifies) + test(=connection::tests::aborting_lifecycle_owner_is_drop_safe) + test(=connection::tests::shutdown_terminalizes_auth_stalled_on_database_before_late_success) + test(=connection::terminal_close_delivery_tests::upgrade_would_deny_line_without_auth_logs_absent_at_close) + test(=handlers::event::tests::observer_owner_cache_is_scoped_to_community) + test(=state::tests::shared_counter_between_direct_and_fanout) + test(=connection::tests::startup_cancellation_writes_one_close_line) + test(=connection::tests::connection_limit_refusal_writes_one_close_line) + test(=connection::tests::dropped_authenticated_handler_writes_one_close_line) + test(=connection::tests::close_totals_count_a_batch_flushed_by_the_drain_after_cancellation) + test(=connection::tests::close_totals_skip_a_frame_the_socket_rejects) + test(=connection::tests::close_totals_skip_frames_stranded_by_a_failed_flush) + test(=connection::tests::read_before_login_logs_one_denial_from_the_handler) + test(=connection::tests::read_refused_by_a_full_handler_semaphore_logs_one_denial) + test(=connection::tests::saturated_handler_write_line_records_whether_its_ok_was_queued) + test(=connection::tests::read_refused_by_unavailable_admission_logs_one_denial) + test(=handlers::auth::tests::repeat_auth_after_a_terminal_logs_one_denial) + test(=nip_fi_upgrade::tests::deny_protected_logs_one_denied_auth_line) + test(=api::bridge::moderation_refusal_tests::moderation_lookup_failure_is_unavailable) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::coalesced_refresh::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_policy::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::db_read_error_tests::/) + test(/^api::profiles::/) + test(/^api::bridge::search_slots::tests::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^api::git::store::tests::/) + test(/^api::git::store::fault_probe::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^mesh_boot::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::accessible_channel_invalidation_is_scoped_to_community) + test(=state::tests::accessible_channels_join_member_and_open_halves) + test(=state::tests::channel_deleted_invalidation_is_scoped_to_community) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::failed_search_closes_without_eose_or_trying_later_filters) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::count_fallback_fetches_one_extra_candidate) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + test(=api::invites::postgres_tests::restriction_tests::postgres_tests::destructive_fixture_rejects_shared_database_urls) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' + -E 'test(/^api::media_access::tests::/) + test(/^handlers::media_access::tests::/) + test(/^subscription::tests::/) + test(/^handlers::multinode_fanout::tests::/) + test(/^api::admin::/) + test(/^access::tests::/) + test(/^api::git::transport::access_log_tests::/) + test(=api::bridge::postgres_tests::nip98_replay_check_logs_only_its_refusal) + test(=api::bridge::postgres_tests::warn_only_artifact_read_lines_keep_the_filter_summary) + test(=connection::tests::close_totals_count_frames_the_writer_flushed) + test(=connection::tests::dispatcher_counts_each_req_and_count_frame_once) + test(=handlers::req::tests::unqueued_eose_logs_aborted_not_served) + test(=api::bridge::postgres_tests::reject_code_keeps_only_a_known_prefix) + test(=api::bridge::postgres_tests::nip98_replay_check_fails_closed_when_guard_errors) + test(=api::bridge::postgres_tests::verify_bridge_auth_rejects_nip98_event_signed_for_wrong_communitys_host) + test(=api::bridge::postgres_tests::verify_bridge_auth_can_require_payload_tag_for_json_body_endpoints) + test(=api::bridge::postgres_tests::verify_bridge_auth_accepts_nip98_event_signed_for_matching_host) + test(=api::bridge::postgres_tests::moderation_read_query_bearing_nip98_event_verifies_with_matching_query) + test(=api::bridge::postgres_tests::moderation_read_query_bearing_nip98_event_rejected_against_bare_path) + test(=api::bridge::postgres_tests::moderation_read_audit_query_bearing_nip98_event_verifies) + test(=api::bridge::postgres_tests::moderation_read_restricted_no_query_still_verifies) + test(=connection::tests::aborting_lifecycle_owner_is_drop_safe) + test(=connection::tests::shutdown_terminalizes_auth_stalled_on_database_before_late_success) + test(=connection::terminal_close_delivery_tests::upgrade_would_deny_line_without_auth_logs_absent_at_close) + test(=handlers::event::tests::observer_owner_cache_is_scoped_to_community) + test(=state::tests::shared_counter_between_direct_and_fanout) + test(=connection::tests::startup_cancellation_writes_one_close_line) + test(=connection::tests::connection_limit_refusal_writes_one_close_line) + test(=connection::tests::dropped_authenticated_handler_writes_one_close_line) + test(=connection::tests::close_totals_count_a_batch_flushed_by_the_drain_after_cancellation) + test(=connection::tests::close_totals_skip_a_frame_the_socket_rejects) + test(=connection::tests::close_totals_skip_frames_stranded_by_a_failed_flush) + test(=connection::tests::read_before_login_logs_one_denial_from_the_handler) + test(=connection::tests::read_refused_by_a_full_handler_semaphore_logs_one_denial) + test(=connection::tests::saturated_handler_write_line_records_whether_its_ok_was_queued) + test(=connection::tests::read_refused_by_unavailable_admission_logs_one_denial) + test(=handlers::auth::tests::repeat_auth_after_a_terminal_logs_one_denial) + test(=nip_fi_upgrade::tests::deny_protected_logs_one_denied_auth_line) + test(=api::bridge::moderation_refusal_tests::moderation_lookup_failure_is_unavailable) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::coalesced_refresh::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_policy::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::db_read_error_tests::/) + test(/^api::profiles::/) + test(/^api::bridge::search_slots::tests::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^api::git::store::tests::/) + test(/^api::git::store::fault_probe::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^mesh_boot::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::accessible_channel_invalidation_is_scoped_to_community) + test(=state::tests::accessible_channels_join_member_and_open_halves) + test(=state::tests::channel_deleted_invalidation_is_scoped_to_community) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::failed_search_closes_without_eose_or_trying_later_filters) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::count_fallback_fetches_one_extra_candidate) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + test(=api::invites::postgres_tests::restriction_tests::postgres_tests::destructive_fixture_rejects_shared_database_urls) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' # Note on audio::join::tests scope: the full suite is infra-free (no DB, # no Redis). The infra-free audio/FI regression witnesses — bootstrap # ordering barrier, CommitConfirmed arm, pending-close invisibility, @@ -679,6 +679,11 @@ test-unit: test-integration: ./scripts/run-tests.sh integration +# Enforce-mode upload/publication/read journey (requires PostgreSQL and Redis). +# Uses the isolated PostgreSQL runner, including emoji entry replacement checks. +test-media-access: + ./scripts/postgres-test-run.sh -p buzz-relay --lib -E 'test(=api::media_access_postgres_tests::private_media_revocation_http_journey)' + # Regenerate the model-capability normative corpus from the production Rust # resolver. The corpus is a golden snapshot, never hand-edited: this runs the # `#[ignore]`d writer test in buzz-agent, which serializes `resolve()` over the diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index 99b4aa68032..f1330a65655 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -86,3 +86,6 @@ pub mod test_helpers { pub mod artifact; /// NIP-AR sync filter parsing and limits. pub mod artifact_sync; + +/// Local media reference extraction. +pub mod media; diff --git a/crates/buzz-core/src/media.rs b/crates/buzz-core/src/media.rs new file mode 100644 index 00000000000..1247f15719d --- /dev/null +++ b/crates/buzz-core/src/media.rs @@ -0,0 +1,343 @@ +//! A shared vocabulary for local media references on reads and publications. +use nostr::Event; +use std::collections::BTreeSet; + +/// Parse a local media URL, returning its parent hash (also for thumbnails). +/// Only the tenant's canonical base and root-relative paths count. +pub fn local_hash(value: &str, base: &str) -> Option { + let alternate = if let Some(rest) = base.strip_prefix("https://") { + format!("http://{rest}") + } else if let Some(rest) = base.strip_prefix("http://") { + format!("https://{rest}") + } else { + base.to_owned() + }; + let prefix = format!("{}/", base.trim_end_matches('/')); + let alternate_prefix = format!("{}/", alternate.trim_end_matches('/')); + let rest = value + .strip_prefix("/media/") + .or_else(|| value.strip_prefix(&prefix)) + .or_else(|| value.strip_prefix(&alternate_prefix))?; + let mut parts = rest.split('.'); + let hash = parts.next()?; + if hash.len() != 64 + || !hash + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return None; + } + let safe_ext = |s: &str| { + !s.is_empty() + && s.len() <= 8 + && s.bytes() + .all(|b| b.is_ascii_digit() || b.is_ascii_lowercase()) + }; + match (parts.next(), parts.next(), parts.next()) { + (None, None, None) => {} + (Some(ext), None, None) if safe_ext(ext) => {} + (Some("thumb"), Some("jpg" | "webp"), None) => {} + _ => return None, + } + Some(hash.to_owned()) +} + +/// Decode the desktop's animated-avatar descriptor into its poster and animation. +/// Each URL still needs its own tenant check. Malformed escapes never become URLs. +pub fn animated_avatar_urls(value: &str) -> Option<(String, String)> { + let (poster, encoded) = value.split_once("#buzz-anim=")?; + if encoded.is_empty() { + return None; + } + let bytes = encoded.as_bytes(); + for (i, byte) in bytes.iter().enumerate() { + if *byte == b'%' + && (i + 2 >= bytes.len() + || !bytes[i + 1].is_ascii_hexdigit() + || !bytes[i + 2].is_ascii_hexdigit()) + { + return None; + } + } + let animation = percent_encoding::percent_decode_str(encoded) + .decode_utf8() + .ok()? + .into_owned(); + for candidate in [poster, animation.as_str()] { + let parsed = url::Url::parse(candidate).ok()?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return None; + } + } + Some((poster.to_owned(), animation)) +} + +fn avatar_hashes(value: &str, base: &str, out: &mut BTreeSet) { + if let Some((poster, animation)) = animated_avatar_urls(value) { + out.extend(local_hash(&poster, base)); + out.extend(local_hash(&animation, base)); + } else { + out.extend(local_hash(value, base)); + } +} + +/// Extract counted references. imeta URL/image fields cover attachments, +/// previews and posters. Profile pictures/banners, group pictures and emoji +/// are the explicitly supported non-message references. Free text grants nothing. +pub fn reference_hashes(event: &Event, base: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + let preview_suppressed = event + .tags + .iter() + .any(|tag| matches!(tag.as_slice(),[name,value] if name=="link-preview" && value=="none")); + for tag in event.tags.iter() { + let parts = tag.as_slice(); + match parts.first().map(String::as_str) { + Some("imeta") => { + for part in &parts[1..] { + if let Some(("url" | "image", value)) = part.split_once(' ') { + out.extend(local_hash(value, base)); + } + } + } + Some("link-preview") + if !preview_suppressed + && parts.len() == 11 + && parts[1] == "snapshot" + && parts[2] == "1" => + { + for value in [&parts[7], &parts[9]] { + out.extend(local_hash(value, base)); + } + } + Some("picture") if event.kind.as_u16() == 39000 => { + if let Some(value) = parts.get(1) { + out.extend(local_hash(value, base)); + } + } + Some("emoji") if matches!(event.kind.as_u16(), 10030 | 30030 | 30180) => { + if let Some(value) = parts.get(2) { + out.extend(local_hash(value, base)); + } + } + _ => {} + } + } + if matches!(event.kind.as_u16(), 0 | 30175) { + if let Ok(profile) = serde_json::from_str::(&event.content) { + let fields = if event.kind.as_u16() == 30175 { + [profile.get("avatar_url"), None] + } else { + [ + profile.get("picture").or_else(|| profile.get("image")), + profile.get("banner"), + ] + }; + for value in fields.into_iter().flatten() { + if let Some(value) = value.as_str() { + avatar_hashes(value, base, &mut out); + } + } + } + } + out +} + +/// All local URLs in structured command payloads as well as counted event +/// references must be authorized before dispatch. This also covers metadata +/// commands whose accepted payload later becomes relay-signed state. +pub fn publication_hashes(event: &Event, base: &str) -> BTreeSet { + fn visit(value: &serde_json::Value, base: &str, out: &mut BTreeSet) { + match value { + serde_json::Value::String(value) => { + avatar_hashes(value, base, out); + } + serde_json::Value::Array(values) => { + for value in values { + visit(value, base, out); + } + } + serde_json::Value::Object(values) => { + for value in values.values() { + visit(value, base, out); + } + } + _ => {} + } + } + let mut out = reference_hashes(event, base); + if let Ok(value) = serde_json::from_str(&event.content) { + visit(&value, base, &mut out); + } + for tag in event.tags.iter() { + for part in tag.as_slice().iter().skip(1) { + out.extend(local_hash(part, base)); + } + } + out +} + +/// First valid edit target in signed tag order, accepting all ASCII hex casing. +pub fn first_edit_target(event: &nostr::Event) -> Option> { + event.tags.iter().find_map(|tag| { + let parts = tag.as_slice(); + let value = parts.get(1)?; + if parts.first().map(String::as_str) == Some("e") + && value.len() == 64 + && value.bytes().all(|c| c.is_ascii_hexdigit()) + { + hex::decode(value).ok() + } else { + None + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use nostr::{EventBuilder, Keys, Kind, Tag}; + #[test] + fn aliases_and_foreign_hosts() { + let hash = "a".repeat(64); + for suffix in ["", ".png", ".thumb.jpg", ".thumb.webp"] { + assert_eq!( + local_hash(&format!("/media/{hash}{suffix}"), "https://local/media"), + Some(hash.clone()) + ); + } + for value in [ + format!("https://foreign/media/{hash}.jpg"), + format!("/media/{hash}0"), + format!("/media/{hash}.png?x=1"), + ] { + assert_eq!(local_hash(&value, "https://local/media"), None); + } + } + #[test] + fn animated_profiles_check_each_url_and_reject_malformed_descriptors() { + let a = "a".repeat(64); + let b = "b".repeat(64); + for (poster, animation, expected) in [ + ( + format!("https://local/media/{a}.png"), + format!("https://local/media/{b}.png"), + BTreeSet::from([a.clone(), b.clone()]), + ), + ( + format!("https://foreign/media/{a}.png"), + format!("https://local/media/{b}.png"), + BTreeSet::from([b]), + ), + ( + format!("https://local/media/{a}.png"), + format!("https://foreign/media/{}.png", "c".repeat(64)), + BTreeSet::from([a.clone()]), + ), + ] { + let descriptor = format!( + "{poster}#buzz-anim={}", + percent_encoding::utf8_percent_encode( + &animation, + percent_encoding::NON_ALPHANUMERIC + ) + ); + let event = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":descriptor}).to_string(), + ) + .sign_with_keys(&Keys::generate()) + .unwrap(); + assert_eq!(reference_hashes(&event, "https://local/media"), expected); + assert_eq!(publication_hashes(&event, "https://local/media"), expected); + } + for value in [ + "https://local/a#buzz-anim=%xx", + "https://local/a#buzz-anim=%ff", + "https://local/a#buzz-anim=", + "https://local/a#buzz-anim=javascript:foo", + ] { + assert!(animated_avatar_urls(value).is_none(), "{value}"); + } + } + #[test] + fn snapshot_media_counts_only_image_and_favicon_urls() { + let a = "a".repeat(64); + let b = "b".repeat(64); + let c = "c".repeat(64); + for (image, favicon, expected) in [ + ( + format!("/media/{a}.png"), + format!("https://local/media/{b}.png"), + BTreeSet::from([a.clone(), b.clone()]), + ), + ( + format!("https://foreign/media/{a}.png"), + String::new(), + BTreeSet::new(), + ), + ] { + let event = EventBuilder::new(Kind::Custom(9), "") + .tags([Tag::parse([ + "link-preview", + "snapshot", + "1", + &format!("https://local/media/{c}.png"), + "title", + "site", + "description", + &image, + &a, + &favicon, + &b, + ]) + .unwrap()]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + assert_eq!(reference_hashes(&event, "https://local/media"), expected); + } + } + #[test] + fn profile_image_fallback_matches_materialization() { + let a = "a".repeat(64); + let b = "b".repeat(64); + for (content, expected) in [ + ( + serde_json::json!({"image":format!("/media/{a}.png")}), + BTreeSet::from([a.clone()]), + ), + ( + serde_json::json!({"picture":format!("/media/{b}.png"),"image":format!("/media/{a}.png")}), + BTreeSet::from([b]), + ), + ( + serde_json::json!({"picture":null,"image":format!("/media/{a}.png")}), + BTreeSet::new(), + ), + ] { + let event = EventBuilder::new(Kind::Metadata, content.to_string()) + .sign_with_keys(&Keys::generate()) + .unwrap(); + assert_eq!(reference_hashes(&event, "https://local/media"), expected); + } + } + #[test] + fn profile_and_imeta_share_the_publication_extractor() { + let hash = "b".repeat(64); + let event = EventBuilder::new( + Kind::Metadata, + format!(r#"{{"picture":"https:\/\/local\/media\/{hash}.jpg"}}"#), + ) + .tags([Tag::parse(["imeta", &format!("url /media/{hash}.jpg")]).unwrap()]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + assert_eq!( + reference_hashes(&event, "https://local/media"), + BTreeSet::from([hash]) + ); + assert_eq!( + publication_hashes(&event, "https://local/media"), + reference_hashes(&event, "https://local/media") + ); + } +} diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index 9ce2cb04057..19f3d1057ab 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -5,6 +5,10 @@ use thiserror::Error; /// Errors produced by database operations. #[derive(Debug, Error)] pub enum DbError { + /// Media authorization could not reserve capacity within its bounded wait. + #[error("media authorization capacity exceeded")] + MediaAuthorizationCapacity, + /// A SQLx driver-level error. #[error("database error: {0}")] Sqlx(#[from] sqlx::Error), diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index b362b9fca5e..92eeb42f882 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -92,10 +92,10 @@ pub(crate) use runtime::{ pub use store::{ admin_moderation, allowlist, api_token, archived_identities, artifact, artifact_sync, channel, channel_members, community, community_auth, deletion, dm, entity_deletion, event, feed, - git_repo, leader_lock, membership_snapshot, moderation, nip_fi_binding, operator_listener, - partition, personal_read, product_feedback, push, reaction, read_state, relay_admin_actions, - relay_invite, relay_members, relay_operators, reminder, replaceable, storage_accounting, - thread, thread_window, user, user_deletion, user_deletion_fences, workflow, + git_repo, leader_lock, media, membership_snapshot, moderation, nip_fi_binding, + operator_listener, partition, personal_read, product_feedback, push, reaction, read_state, + relay_admin_actions, relay_invite, relay_members, relay_operators, reminder, replaceable, + storage_accounting, thread, thread_window, user, user_deletion, user_deletion_fences, workflow, }; pub use allowlist::AllowlistEntry; diff --git a/crates/buzz-db/src/runtime/media_index_prebuild_postgres_tests.rs b/crates/buzz-db/src/runtime/media_index_prebuild_postgres_tests.rs new file mode 100644 index 00000000000..e42a737ff5c --- /dev/null +++ b/crates/buzz-db/src/runtime/media_index_prebuild_postgres_tests.rs @@ -0,0 +1,104 @@ +//! Exercise the shipped psql procedure with real old-snapshot waits. +use sqlx::Executor; +use std::{path::Path, process::Stdio, time::Duration}; + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn migration_schema_media_prebuild_waits_for_old_transactions_during_creation_and_cleanup() { + let url = crate::test_support::database_url(); + let pool = sqlx::PgPool::connect(&url).await.unwrap(); + // This migration-mode fixture owns an empty per-test database. Keep only + // the columns the operator procedure uses, including its final 0086 guard. + pool.execute("CREATE TABLE events (community_id bytea, channel_id bytea, tags jsonb, created_at bigint, id bytea, kind int, deleted_at bigint) PARTITION BY RANGE(created_at); CREATE TABLE events_p PARTITION OF events FOR VALUES FROM (0) TO (100); INSERT INTO events(tags,created_at,kind) VALUES ('[]',1,40003)").await.unwrap(); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let expression = crate::store::media::MEDIA_EDIT_TARGET_SQL; + for cleanup in [false, true] { + if cleanup { + pool.execute("DROP INDEX idx_events_media_edit_target") + .await + .unwrap(); + sqlx::raw_sql(sqlx::AssertSqlSafe(format!("CREATE INDEX idx_events_media_edit_target ON ONLY events(community_id,channel_id,({expression}),created_at DESC,id DESC) WHERE kind=40003 AND deleted_at IS NULL"))).execute(&pool).await.unwrap(); + } + let mut old = pool.begin().await.unwrap(); + old.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .await + .unwrap(); + old.execute("SELECT * FROM events").await.unwrap(); + if cleanup { + let mut conn = pool.acquire().await.unwrap(); + conn.execute("SET lock_timeout = '50ms'").await.unwrap(); + let error = sqlx::raw_sql(sqlx::AssertSqlSafe(format!("CREATE INDEX CONCURRENTLY ON events_p(community_id,channel_id,({expression}),created_at DESC,id DESC) WHERE kind=40003 AND deleted_at IS NULL"))).execute(&mut *conn).await.unwrap_err(); + assert!(error.to_string().contains("lock timeout"), "{error}"); + conn.execute("RESET lock_timeout").await.unwrap(); + let invalid: i64 = sqlx::query_scalar("SELECT count(*) FROM pg_index WHERE indrelid='events_p'::regclass AND NOT indisvalid").fetch_one(&pool).await.unwrap(); + assert_eq!( + invalid, 1, + "cleanup starts with a real failed concurrent build" + ); + } + let name = format!("media_prebuild_{}", uuid::Uuid::new_v4().simple()); + let mut script = tokio::process::Command::new("psql") + .args(["-X", "-v", "ON_ERROR_STOP=1", "-f"]) + .arg(root.join("scripts/maintenance/prebuild-media-edit-index.sql")) + .arg("--dbname") + .arg(&url) + .env("PGAPPNAME", &name) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let waiting = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let waiting: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND application_name=$1 AND wait_event_type='Lock' AND wait_event='virtualxid')") + .bind(&name).fetch_one(&pool).await.unwrap(); + if waiting { break; } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }).await; + if waiting.is_err() { + old.rollback().await.unwrap(); + let result = script.wait_with_output().await.unwrap(); + panic!( + "cleanup={cleanup}: script did not reach old-transaction wait: {} {}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + } + // Keep the old snapshot until the original one-second deadline has + // passed. An early exit here catches both CREATE and DROP regressions. + assert!( + tokio::time::timeout(Duration::from_millis(1300), script.wait()) + .await + .is_err(), + "cleanup={cleanup}: concurrent operation must outlive the short metadata lock budget" + ); + old.commit().await.unwrap(); + let result = tokio::time::timeout(Duration::from_secs(10), script.wait_with_output()) + .await + .unwrap() + .unwrap(); + assert!( + result.status.success(), + "cleanup={cleanup}: {}", + String::from_utf8_lossy(&result.stderr) + ); + let valid: i64 = sqlx::query_scalar("SELECT count(*) FROM pg_index WHERE indrelid IN ('events'::regclass,'events_p'::regclass) AND indisvalid AND indisready").fetch_one(&pool).await.unwrap(); + assert_eq!(valid, 2); + let invalid: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_index WHERE indrelid='events_p'::regclass AND NOT indisvalid", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + invalid, 0, + "retry must remove the unusable child, not leave it beside a replacement" + ); + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM events") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 1); + } +} diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 1264ea9e8f8..f7f0888e96f 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -312,6 +312,10 @@ async fn reject_legacy_nip_rs_cardinality_ambiguity(conn: &mut PgConnection) -> #[path = "follow_up_retirement_tests.rs"] mod retirement_postgres_tests; +#[cfg(test)] +#[path = "media_index_prebuild_postgres_tests.rs"] +mod media_index_prebuild_postgres_tests; + #[cfg(test)] mod postgres_tests { use super::*; @@ -898,7 +902,7 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 85); + assert_eq!(migrations.len(), 86); assert!(migrations[83] .sql .as_str() @@ -907,6 +911,10 @@ mod postgres_tests { .sql .as_str() .contains("ADD COLUMN requested_requires_nip_fi BOOLEAN NOT NULL DEFAULT true")); + assert!(migrations[85] + .sql + .as_str() + .contains("CREATE TABLE media_uploaders")); for (index, migration) in migrations.iter().enumerate() { assert_eq!(migration.version, (index + 1) as i64); } @@ -3034,6 +3042,8 @@ mod postgres_tests { .expect("apply migrations for schema parity"); for table in [ + "media_uploaders", + "media_publications", "relay_members", "relay_admin_actions", "relay_admin_outbox", @@ -3063,6 +3073,14 @@ mod postgres_tests { migration and schema.sql must both use a representable shape." ); } + async fn media_edit_index(pool: &PgPool) -> (String, String, String) { + sqlx::query_as("SELECT pg_get_indexdef(indexrelid),indoption::text,pg_get_expr(indexprs,indrelid) FROM pg_index WHERE indexrelid='public.idx_events_media_edit_target'::regclass AND indisvalid AND indisready AND indislive") + .fetch_one(pool).await.expect("valid media edit index") + } + assert_eq!( + media_edit_index(&desired).await, + media_edit_index(&migrated).await + ); // `channels` columns only: pgschema re-emits `idx_channels_id_live` // without its `INCLUDE (community_id)`, an index drift that predates // this check. @@ -3566,6 +3584,7 @@ mod postgres_tests { .execute(&pool) .await .expect("repair ambiguous row"); + crate::test_support::prebuild_media_edit_index(&pool).await; run_migrations(&pool) .await .expect("retry succeeds after operator repair"); @@ -3645,6 +3664,7 @@ mod postgres_tests { vec![(1, Some(true)), (30_179, Some(true)), (30_350, None)] ); + crate::test_support::prebuild_media_edit_index(&pool).await; run_migrations(&pool) .await .expect("apply remaining migrations to populated database"); @@ -4072,4 +4092,112 @@ mod postgres_tests { .await .is_err()); } + /// Small populated upgrades build inline; contention stays bounded and a + /// verified prebuild avoids requesting the writer-conflicting index lock. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn media_edit_index_upgrade_builds_inline_or_verifies_prebuild() { + let pool = connect_test_pool().await; + for prebuilt in [false, true] { + reset_public_schema(&pool).await; + run_migrations_through(&pool, 85).await.unwrap(); + let community = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO communities(id,host) VALUES($1,'media-index-upgrade.example')", + ) + .bind(community) + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig) VALUES($1,decode(repeat('a',64),'hex'),decode(repeat('b',64),'hex'),now(),9,'[]','media',decode(repeat('c',128),'hex'))") + .bind(community).execute(&pool).await.unwrap(); + if prebuilt { + sqlx::query("CREATE INDEX idx_events_media_edit_target ON events(community_id,id)") + .execute(&pool) + .await + .unwrap(); + let error = run_migrations(&pool).await.unwrap_err(); + assert!( + error.to_string().contains("invalid or wrong definition"), + "{error}" + ); + sqlx::query("DROP INDEX idx_events_media_edit_target") + .execute(&pool) + .await + .unwrap(); + crate::test_support::prebuild_media_edit_index(&pool).await; + } + let mut writer = pool.begin().await.unwrap(); + sqlx::query("LOCK TABLE public.events IN ROW EXCLUSIVE MODE") + .execute(&mut *writer) + .await + .unwrap(); + if prebuilt { + run_migrations(&pool) + .await + .expect("verified prebuild does not request an index-build lock"); + writer.rollback().await.unwrap(); + } else { + let error = run_migrations(&pool).await.unwrap_err(); + assert!( + error.to_string().contains("lock timeout"), + "index build must obey its lock budget: {error}" + ); + let applied: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM _sqlx_migrations WHERE version>=86)", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert!( + !applied, + "failed first new migration must leave version 85 intact" + ); + let old_migrator = sqlx::migrate::Migrator { + migrations: std::borrow::Cow::Owned( + MIGRATOR + .iter() + .filter(|migration| migration.version <= 85) + .cloned() + .collect(), + ), + ..sqlx::migrate::Migrator::DEFAULT + }; + with_exclusive_schema_destruction_lock(&pool, |mut conn| async move { + let result = old_migrator + .run(&mut conn) + .await + .map_err(crate::DbError::from); + (conn, result) + }) + .await + .expect("old binary's migrator must still restart after the failed index build"); + let tables: bool = sqlx::query_scalar("SELECT to_regclass('public.media_uploaders') IS NOT NULL OR to_regclass('public.media_publications') IS NOT NULL") + .fetch_one(&pool).await.unwrap(); + assert!( + !tables, + "media table expansion must follow the fallible index build" + ); + let recovery_column: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='relay_members' AND column_name='auto_join_pending')") + .fetch_one(&pool).await.unwrap(); + assert!(recovery_column, "main's migration 84 must remain intact"); + let policy_columns: i64 = sqlx::query_scalar("SELECT count(*) FROM information_schema.columns WHERE table_schema='public' AND table_name='community_auth_upgrades' AND column_name IN ('requested_requires_nip_fi','expected_policy_generation','expected_requires_nip_fi')") + .fetch_one(&pool).await.unwrap(); + assert_eq!(policy_columns, 3, "base migration 85 must remain intact"); + writer.rollback().await.unwrap(); + run_migrations(&pool).await.expect( + "small populated database builds its index without operator intervention", + ); + } + run_migrations(&pool).await.unwrap(); + let valid: bool = sqlx::query_scalar("SELECT indisvalid AND indisready FROM pg_index WHERE indexrelid='public.idx_events_media_edit_target'::regclass") + .fetch_one(&pool).await.unwrap(); + assert!(valid, "both upgrade paths must leave a valid index"); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM events") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 1, "upgrade preserves old relay rows"); + } + } } diff --git a/crates/buzz-db/src/runtime/mod.rs b/crates/buzz-db/src/runtime/mod.rs index 9f78fde1d85..f6f34615583 100644 --- a/crates/buzz-db/src/runtime/mod.rs +++ b/crates/buzz-db/src/runtime/mod.rs @@ -264,6 +264,9 @@ where #[derive(Clone, Debug)] pub struct Db { pub(crate) pool: PgPool, + /// Shared across clones; media authorization may use at most a quarter of + /// the writer pool (capped at eight), leaving capacity for ordinary writes. + pub(crate) media_authorization_slots: std::sync::Arc, /// Optional bounded post-commit producer, shared across handle clones. pub(crate) push_enqueue: Option, /// Maximum connections configured for this pool (from [`DbConfig::max_connections`]). @@ -819,6 +822,7 @@ impl Db { .push_enabled .then(|| push_enqueue::PushEnqueue::new(&pool)); Ok(Self { + media_authorization_slots: media_authorization_slots(&pool), push_enqueue, pool, max_connections: config.max_connections, @@ -1377,6 +1381,7 @@ impl Db { /// that need push must use [`Db::new`] with `push_enabled` instead. pub fn from_pool(pool: PgPool) -> Self { Self { + media_authorization_slots: media_authorization_slots(&pool), push_enqueue: None, max_connections: pool.options().get_max_connections(), read_max_connections: pool.options().get_max_connections(), @@ -1398,6 +1403,7 @@ impl Db { /// [`Db::fence`]). pub fn from_pools(pool: PgPool, read_pool: PgPool) -> Self { Self { + media_authorization_slots: media_authorization_slots(&pool), push_enqueue: None, max_connections: pool.options().get_max_connections(), read_max_connections: read_pool.options().get_max_connections(), @@ -2172,3 +2178,9 @@ mod pool_role_tests { #[cfg(test)] #[path = "tests.rs"] mod postgres_tests; + +fn media_authorization_slots(pool: &PgPool) -> std::sync::Arc { + std::sync::Arc::new(tokio::sync::Semaphore::new( + (pool.options().get_max_connections() as usize / 4).clamp(1, 8), + )) +} diff --git a/crates/buzz-db/src/runtime/tests/listed_channel_name_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/listed_channel_name_postgres_tests.rs index a07a3858ef6..096eee36f5f 100644 --- a/crates/buzz-db/src/runtime/tests/listed_channel_name_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/listed_channel_name_postgres_tests.rs @@ -533,7 +533,9 @@ async fn migration_schema_listed_name_index_needs_the_renames_then_builds_inline seed.open(&pool, newer, "Square-iOS", 10).await; let before = seed.names(&pool).await; - let error = migration::run_migrations(&pool).await.unwrap_err(); + let error = migration::run_migrations_through(&pool, BEFORE_INDEX + 1) + .await + .unwrap_err(); assert!( error.to_string().contains("duplicate names remain"), "must name the duplicate rows, not merely fail: {error}" @@ -552,7 +554,9 @@ async fn migration_schema_listed_name_index_needs_the_renames_then_builds_inline .execute(&mut *writer) .await .unwrap(); - let error = migration::run_migrations(&pool).await.unwrap_err(); + let error = migration::run_migrations_through(&pool, BEFORE_INDEX + 1) + .await + .unwrap_err(); assert!( error.to_string().contains("lock timeout"), "the build must not queue behind a writer: {error}" @@ -563,7 +567,7 @@ async fn migration_schema_listed_name_index_needs_the_renames_then_builds_inline // With no writer in the way, the populated table migrates with no // operator step: the migration builds the index itself. - migration::run_migrations(&pool) + migration::run_migrations_through(&pool, BEFORE_INDEX + 1) .await .expect("migrate a populated table after the renames"); assert!(listed_name_migration_recorded(&pool).await); @@ -615,7 +619,9 @@ async fn migration_schema_listed_name_index_validates_a_prebuild() { .execute(&pool) .await .unwrap(); - let error = migration::run_migrations(&pool).await.unwrap_err(); + let error = migration::run_migrations_through(&pool, BEFORE_INDEX + 1) + .await + .unwrap_err(); assert!( error .to_string() @@ -639,7 +645,7 @@ async fn migration_schema_listed_name_index_validates_a_prebuild() { .execute(&mut *writer) .await .unwrap(); - migration::run_migrations(&pool) + migration::run_migrations_through(&pool, BEFORE_INDEX + 1) .await .expect("accept the prebuild without waiting for channel writers"); writer.rollback().await.unwrap(); diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index c326fe81dce..3a3d4e6e4fd 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -93,6 +93,8 @@ pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ "events", "git_repo_names", "join_policy_acceptances", + "media_uploaders", + "media_publications", "moderation_actions", "moderation_reports", "parameterized_event_watermarks", @@ -138,6 +140,8 @@ pub const PURGE_SCOPED_TABLES: &[&str] = &[ "workflow_runs", "push_wake_outbox", "join_policy_acceptances", + "media_uploaders", + "media_publications", "moderation_reports", "subscriptions", "api_tokens", @@ -8089,6 +8093,13 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .execute(&db.pool) .await .expect("insert guarded NIP-RS row"); + let media_hash = "a".repeat(64); + db.record_media_upload(request.community_id, &media_hash, &[7; 32]) + .await + .expect("upload proof"); + db.record_media_publications(request.community_id, vec![media_hash.clone()]) + .await + .expect("publication boundary"); store .approve(request.id, "approver", None) .await @@ -8099,6 +8110,14 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .expect("claim") .expect("won claim"); store.begin_quiescing(&claim.lease).await.expect("quiesce"); + assert!(db + .record_media_upload(request.community_id, &media_hash, &[8; 32]) + .await + .is_err()); + assert!(db + .record_media_publications(request.community_id, vec!["b".repeat(64)]) + .await + .is_err()); let generation = store.fence(&claim.lease).await.expect("fence"); let token = LeaseToken { fence_generation: Some(generation), @@ -8154,6 +8173,8 @@ DROP FUNCTION community_write_fence_excluded_table(NAME); .await .expect("bindings"); let first = store.purge_postgres(&token).await.expect("purge postgres"); + assert_eq!(first["media_uploaders"], 1); + assert_eq!(first["media_publications"], 1); let expected_tables: BTreeSet<&str> = EXPECTED_SCOPED_TABLES .iter() .chain(RELAY_ADMIN_TABLES.iter().map(|(table, _)| table)) diff --git a/crates/buzz-db/src/store/media.rs b/crates/buzz-db/src/store/media.rs new file mode 100644 index 00000000000..e6218ae2c3f --- /dev/null +++ b/crates/buzz-db/src/store/media.rs @@ -0,0 +1,435 @@ +//! Fresh media reads use primary-database event and channel state. +use crate::{Db, DbError, Result}; +use buzz_core::{kind, media::reference_hashes, CommunityId}; +use buzz_datastore_tracing::datastore_span; +use futures_util::TryStreamExt; +use sqlx::{Postgres, QueryBuilder, Row}; + +// Unqualified tags lets this exact expression serve indexes and aliased probes. +// Keep DDL/catalog and Rust target parity covered by the PostgreSQL tests. +pub(crate) const MEDIA_EDIT_TARGET_SQL: &str = r#"decode(lower(jsonb_path_query_first(tags, 'strict $[*] ? (@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]', '{}'::jsonb, true) #>> '{}'), 'hex')"#; + +// Short indexed authorization probes cannot amortize LLVM compilation. +// Partitioned permission plans can compile for seconds before reading any row. +// Ordinary database work retains its configured JIT policy. +pub(super) const MEDIA_LOOKUP_SETTINGS_SQL: &str = + "SET LOCAL statement_timeout = '2s'; SET LOCAL jit = off"; + +fn edit_target(alias: &str) -> String { + MEDIA_EDIT_TARGET_SQL.replacen("(tags,", &format!("({alias}.tags,"), 1) +} + +/// Result of a complete live-reference lookup, for bounded denial metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MediaAccess { + /// A current readable reference. + Allowed, + /// Uploaded bytes with no past publication boundary or current reference. + UploaderPreview, + /// No published reference or upload proof exists. + NoReference, + /// References exist but none is currently readable. + Restricted, +} + +impl Db { + /// Record possession after the upload pipeline verified the actual bytes. + /// Independent of optional audit and moderation records; admission fences deletion. + #[datastore_span(name = "record_media_upload", system = "postgresql")] + pub async fn record_media_upload( + &self, + community: CommunityId, + sha256: &str, + pubkey: &[u8], + ) -> Result<()> { + let mut tx = crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + sqlx::query("INSERT INTO media_uploaders (community_id, sha256, pubkey) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING") + .bind(community.as_uuid()).bind(sha256).bind(pubkey).execute(tx.conn()).await?; + tx.commit().await?; + Ok(()) + } + + /// Record a sticky publication boundary before dispatch, even if the later + /// command fails. This grants no read access and survives source purging. + pub async fn record_media_publications( + &self, + community: CommunityId, + hashes: Vec, + ) -> Result<()> { + let mut tx = crate::begin_community_write_transaction( + &self.pool, + community, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + sqlx::query("INSERT INTO media_publications (community_id,sha256) SELECT $1, unnest($2::text[]) ON CONFLICT DO NOTHING") + .bind(community.as_uuid()).bind(hashes).execute(tx.conn()).await?; + tx.commit().await?; + Ok(()) + } + + /// Resolve an attachment through live references, never a membership cache + /// or read replica. A repeatable-read snapshot covers both indexed references + /// and profile fallback, including the absence check for uploader previews. + /// `visible` is the ordinary relay REQ visibility predicate. + #[allow(clippy::too_many_arguments)] // Complete tenant/media/reader authority stays explicit. + #[datastore_span(name = "media_access", system = "postgresql")] + pub async fn media_access( + &self, + community: CommunityId, + sha256: &str, + ext: &str, + base: &str, + pubkey: &[u8], + for_publication: bool, + visible: fn(&nostr::Event, &[u8]) -> bool, + ) -> Result { + let _permit = tokio::time::timeout( + std::time::Duration::from_millis(500), + self.media_authorization_slots.acquire(), + ) + .await + .map_err(|_| DbError::MediaAuthorizationCapacity)? + .map_err(|_| DbError::MediaAuthorizationCapacity)?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + let result = async { + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *tx) + .await?; + sqlx::raw_sql(MEDIA_LOOKUP_SETTINGS_SQL) + .execute(&mut *tx) + .await?; + let uploaded: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM media_uploaders WHERE community_id=$1 AND sha256=$2 AND pubkey=$3)") + .bind(community.as_uuid()).bind(sha256).bind(pubkey).fetch_one(&mut *tx).await?; + if uploaded && for_publication { + return Ok(MediaAccess::Allowed); + } + + let needles = media_reference_needles(sha256, ext, base); + let mut referenced = false; + let mut scanned_bytes = 0usize; + // Prefer readable live references before counting any inaccessible rows. + // The second pass only classifies absence; it can stop at the first real reference. + for readable_only in [true, false] { + for probe in 0..4 { + let mut query = media_reference_query( + community, + sha256, + pubkey, + &needles, + probe, + readable_only, + ); + // Stream candidates: a frequently shared hash must not allocate + // 1,025 full message bodies before finding the first readable one. + let statement = query.build(); + let mut rows = statement.fetch(&mut *tx); + let mut count = 0; + while let Some(row) = rows.try_next().await? { + count += 1; + let content: &str = row.try_get("content")?; + let tags: serde_json::Value = row.try_get("tags")?; + scanned_bytes = scanned_bytes + .saturating_add(content.len()) + .saturating_add(tags.to_string().len()); + if count > 1024 || scanned_bytes > 16 * 1024 * 1024 { + return Err(DbError::InvalidData( + "media reference work budget exceeded".into(), + )); + } + let readable: bool = row.try_get("readable")?; + let stored = crate::event::row_to_stored_event(row)?.ok_or_else(|| { + DbError::InvalidData("malformed media reference event".into()) + })?; + let event = &stored.event; + if !reference_hashes(event, base).contains(sha256) { + continue; + } + referenced = true; + if !readable_only { + break; + } + let kind = event.kind.as_u16() as u32; + let tagged = event.tags.iter().any(|tag| matches!(tag.as_slice(), [name, value, ..] if name == "p" && value == &hex::encode(pubkey))); + let private_ok = (!kind::P_GATED_KINDS.contains(&kind) || tagged) + && (kind != kind::KIND_AGENT_ENGRAM + || event.pubkey.as_bytes() == pubkey + || tagged); + if readable && private_ok && visible(event, pubkey) { + return Ok(MediaAccess::Allowed); + } + } + if referenced && !readable_only { + break; + } + } + if referenced && !readable_only { + break; + } + } + // Community icons are current community state; commands validate their + // structured URLs before updating this row. They grant community access. + let icon: Option = sqlx::query_scalar("SELECT icon FROM communities WHERE id=$1") + .bind(community.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if icon + .as_deref() + .and_then(|value| buzz_core::media::local_hash(value, base)) + .as_deref() + == Some(sha256) + { + return Ok(MediaAccess::Allowed); + } + let published: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM media_publications WHERE community_id=$1 AND sha256=$2)", + ) + .bind(community.as_uuid()) + .bind(sha256) + .fetch_one(&mut *tx) + .await?; + if referenced || published { + Ok(MediaAccess::Restricted) + } else if uploaded { + Ok(MediaAccess::UploaderPreview) + } else { + Ok(MediaAccess::NoReference) + } + }.await; + // Dropping a streaming lookup queues rollback asynchronously. Await it + // while the permit is held so response completion cannot free capacity + // while the writer is still draining the query/transaction. + tx.rollback().await?; + result + } +} + +pub(super) fn media_reference_query( + community: CommunityId, + sha256: &str, + pubkey: &[u8], + needles: &[serde_json::Value], + probe: usize, + readable_only: bool, +) -> QueryBuilder { + let mut query = QueryBuilder::::new(""); + query.push("SELECT id,pubkey,created_at,kind,tags,content,sig,received_at,channel_id,"); + query.push(if readable_only { "true" } else { "false" }); + query.push(" AS readable FROM ("); + query.push("SELECT e.id,e.pubkey,e.created_at,e.kind,"); + query.push(if probe == 3 { + "jsonb_path_query_array(e.tags, 'strict $[*] ? (@[0] == \"link-preview\")') AS tags," + } else { + "e.tags," + }); + query.push( + "e.content,e.sig,e.received_at,e.channel_id, \ + (e.deleted_at IS NULL AND (e.channel_id IS NULL OR EXISTS ( \ + SELECT 1 FROM channels c WHERE c.community_id=e.community_id AND c.id=e.channel_id \ + AND c.deleted_at IS NULL AND (c.visibility='open' OR EXISTS ( \ + SELECT 1 FROM channel_members m WHERE m.community_id=c.community_id AND m.channel_id=c.id \ + AND m.removed_at IS NULL AND m.pubkey=" + ); + query + .push_bind(pubkey) + .push( + ")))) AND (e.kind <> 40003 OR EXISTS (SELECT 1 FROM events original \ + WHERE original.community_id=e.community_id AND original.channel_id=e.channel_id \ + AND original.deleted_at IS NULL AND original.kind IN (9,40002) \ + AND ", + ) + .push(edit_target("e")) + .push( + "=original.id AND NOT EXISTS (SELECT 1 FROM events newer \ + WHERE newer.community_id=e.community_id AND newer.channel_id=e.channel_id \ + AND newer.kind=40003 AND newer.deleted_at IS NULL \ + AND (newer.created_at,newer.id) > (e.created_at,e.id) AND ", + ) + .push(edit_target("newer")) + .push("=original.id))) "); + if probe == 3 { + // The renderer retains snapshots unless the original or any live + // edit suppresses them; later body edits never undo suppression. + query.push("AND e.kind IN (9,40002) AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) marker(value) WHERE marker.value='[\"link-preview\",\"none\"]'::jsonb) AND EXISTS (SELECT 1 FROM events edit WHERE edit.community_id=e.community_id AND edit.channel_id=e.channel_id AND edit.kind=40003 AND edit.deleted_at IS NULL AND ") + .push(edit_target("edit")).push("=e.id) AND NOT EXISTS (SELECT 1 FROM events suppressed WHERE suppressed.community_id=e.community_id AND suppressed.channel_id=e.channel_id AND suppressed.kind=40003 AND suppressed.deleted_at IS NULL AND ") + .push(edit_target("suppressed")).push("=e.id AND EXISTS (SELECT 1 FROM jsonb_array_elements(suppressed.tags) marker(value) WHERE marker.value='[\"link-preview\",\"none\"]'::jsonb))"); + } else { + query.push("AND (e.kind NOT IN (9,40002) OR NOT EXISTS (SELECT 1 FROM events edit WHERE edit.community_id=e.community_id AND edit.channel_id=e.channel_id AND edit.kind=40003 AND edit.deleted_at IS NULL AND ") + .push(edit_target("edit")).push("=e.id))"); + } + query + .push(") AS readable FROM events e WHERE e.community_id=") + .push_bind(*community.as_uuid()) + .push(" AND e.deleted_at IS NULL"); + if probe == 2 { + // Never scan message bodies. Only profiles and personas have content references; + // current profiles are indexed by community/kind/deleted_at. + query + .push(" AND e.kind IN (0,30175) AND e.deleted_at IS NULL AND (strpos(e.content, ") + .push_bind(sha256) + .push(") > 0) AND CASE WHEN e.content IS JSON OBJECT THEN ((e.kind=0 AND ((jsonb_typeof(COALESCE(e.content::jsonb->'picture', e.content::jsonb->'image'))='string' AND strpos(COALESCE(e.content::jsonb->'picture',e.content::jsonb->'image') #>> '{}', ") + .push_bind(sha256) + .push(") > 0) OR (jsonb_typeof(e.content::jsonb->'banner')='string' AND strpos(e.content::jsonb->>'banner', ") + .push_bind(sha256) + .push(") > 0))) OR (e.kind=30175 AND jsonb_typeof(e.content::jsonb->'avatar_url')='string' AND strpos(e.content::jsonb->>'avatar_url', ") + .push_bind(sha256) + .push(") > 0)) ELSE false END"); + } else { + query.push(" AND ("); + let selected = if probe == 0 { + &needles[..1] + } else { + &needles[1..] + }; + for (i, tag) in ["imeta", "picture", "emoji", "link-preview"] + .into_iter() + .enumerate() + { + let matching: Vec = selected + .iter() + .filter(|needle| { + needle[0][0].as_str() == Some(tag) && (probe != 3 || tag == "link-preview") + }) + .cloned() + .collect(); + if i > 0 { + query.push(" OR "); + } + query.push("("); + if tag == "picture" { + query.push("e.kind=39000 AND "); + } + if tag == "emoji" { + query.push("e.kind IN (10030,30030,30180) AND "); + } + let preview_urls = (tag == "link-preview").then(|| { + matching + .iter() + .filter_map(|needle| needle[0][1].as_str().map(str::to_owned)) + .collect::>() + }); + query + .push("e.tags @> ANY(") + .push_bind(matching) + .push("::jsonb[])"); + if tag == "imeta" && probe == 0 { + // A bare x marker is not a reference. Require the matching URL + // in that same tag before charging this row to the work budget. + let urls: Vec = needles + .iter() + .filter_map(|needle| needle[0][1].as_str()) + .filter(|value| value.starts_with("url ")) + .map(|value| serde_json::json!([value])) + .collect(); + query.push(" AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) AS imeta(value) WHERE imeta.value @> ") + .push_bind(serde_json::json!(["imeta", format!("x {sha256}")])) + .push(" AND imeta.value @> ANY(").push_bind(urls).push("::jsonb[]))"); + } + if let Some(urls) = preview_urls { + // Containment ignores array positions. Recheck the actual + // snapshot fields before non-counting rows consume the budget. + query.push(" AND EXISTS (SELECT 1 FROM jsonb_array_elements(e.tags) AS preview(value) \ + WHERE preview.value->>0='link-preview' AND preview.value->>1='snapshot' \ + AND preview.value->>2='1' AND CASE WHEN jsonb_typeof(preview.value)='array' \ + THEN jsonb_array_length(preview.value)=11 ELSE false END \ + AND (preview.value->>7 = ANY(") + .push_bind(urls.clone()).push("::text[]) OR preview.value->>9 = ANY(") + .push_bind(urls).push("::text[])))"); + } + query.push(")"); + } + query.push(")"); + } + query.push(") candidate"); + if readable_only { + query.push(" WHERE readable"); + // Match the ordinary visibility classes before budgeting candidates. + // The relay predicate still performs its authoritative final check. + let gated: Vec = kind::P_GATED_KINDS + .iter() + .chain(kind::RESULT_GATED_KINDS) + .map(|kind| *kind as i32) + .collect(); + let author_only: Vec = kind::AUTHOR_ONLY_KINDS + .iter() + .map(|kind| *kind as i32) + .collect(); + let shared: Vec = kind::SHARED_GATED_KINDS + .iter() + .map(|kind| *kind as i32) + .collect(); + query + .push(" AND (kind <> ALL(") + .push_bind(gated) + .push(") OR tags @> ") + .push_bind(serde_json::json!([["p", hex::encode(pubkey)]])) + .push(")"); + query + .push(" AND (kind <> ALL(") + .push_bind(author_only) + .push(") OR pubkey=") + .push_bind(pubkey) + .push(")"); + query + .push(" AND (kind <> ") + .push_bind(kind::KIND_AGENT_ENGRAM as i32) + .push(" OR pubkey=") + .push_bind(pubkey) + .push(" OR tags @> ") + .push_bind(serde_json::json!([["p", hex::encode(pubkey)]])) + .push(")"); + query.push(" AND (kind <> ALL(").push_bind(shared).push(") OR pubkey=").push_bind(pubkey) + .push(" OR jsonb_path_query_array(tags, 'strict $[*] ? (@[0] == \"shared\")') = '[[\"shared\",\"true\"]]'::jsonb)"); + } + if probe == 2 { + query + .push(" ORDER BY (strpos(content, ") + .push_bind(sha256) + .push(") > 0) DESC, created_at ASC, id ASC"); + } + query.push(" LIMIT 1025"); + query +} + +pub(super) fn media_reference_needles( + sha256: &str, + ext: &str, + base: &str, +) -> Vec { + let mut needles = vec![serde_json::json!([["imeta", format!("x {sha256}")]])]; + let mut bases = vec!["/media".to_owned(), base.trim_end_matches('/').to_owned()]; + // Both schemes have historically appeared in tenant descriptor URLs. + if let Some(rest) = base.strip_prefix("https://") { + bases.push(format!("http://{rest}")); + } + if let Some(rest) = base.strip_prefix("http://") { + bases.push(format!("https://{rest}")); + } + for prefix in bases { + for suffix in [ + String::new(), + format!(".{ext}"), + ".thumb.jpg".into(), + ".thumb.webp".into(), + ] { + let url = format!("{prefix}/{sha256}{suffix}"); + for field in ["url", "image"] { + needles.push(serde_json::json!([["imeta", format!("{field} {url}")]])); + } + needles.push(serde_json::json!([["link-preview", url]])); + needles.push(serde_json::json!([["picture", url]])); + needles.push(serde_json::json!([["emoji", url]])); + } + } + needles +} diff --git a/crates/buzz-db/src/store/media_plan_fixture.sql b/crates/buzz-db/src/store/media_plan_fixture.sql new file mode 100644 index 00000000000..0282a9bb8ab --- /dev/null +++ b/crates/buzz-db/src/store/media_plan_fixture.sql @@ -0,0 +1,52 @@ +CREATE SCHEMA media_access_plan; +SET LOCAL search_path TO media_access_plan,public; +CREATE TABLE media_access_plan.channels(community_id uuid,id uuid,deleted_at timestamptz,visibility text); +CREATE TABLE media_access_plan.channel_members(community_id uuid,channel_id uuid,pubkey bytea,removed_at timestamptz); +CREATE INDEX plan_members ON media_access_plan.channel_members(community_id,channel_id,pubkey); +CREATE INDEX plan_channels ON media_access_plan.channels(community_id,id); +INSERT INTO media_access_plan.channels SELECT '00000000-0000-0000-0000-000000000001',md5(g::text)::uuid,NULL,CASE WHEN g%2=0 THEN 'open' ELSE 'private' END FROM generate_series(0,255) g; +INSERT INTO media_access_plan.channel_members SELECT community_id,id,decode(repeat('1',64),'hex'),NULL FROM media_access_plan.channels WHERE visibility='private'; +CREATE TABLE media_access_plan.events (LIKE public.events INCLUDING DEFAULTS INCLUDING GENERATED) PARTITION BY RANGE (created_at); +CREATE TABLE media_access_plan.events_2025_01 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-01-01') TO ('2025-02-01'); +CREATE TABLE media_access_plan.events_2025_02 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-02-01') TO ('2025-03-01'); +CREATE TABLE media_access_plan.events_2025_03 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-03-01') TO ('2025-04-01'); +CREATE TABLE media_access_plan.events_2025_04 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-04-01') TO ('2025-05-01'); +CREATE TABLE media_access_plan.events_2025_05 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-05-01') TO ('2025-06-01'); +CREATE TABLE media_access_plan.events_2025_06 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-06-01') TO ('2025-07-01'); +CREATE TABLE media_access_plan.events_2025_07 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-07-01') TO ('2025-08-01'); +CREATE TABLE media_access_plan.events_2025_08 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-08-01') TO ('2025-09-01'); +CREATE TABLE media_access_plan.events_2025_09 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-09-01') TO ('2025-10-01'); +CREATE TABLE media_access_plan.events_2025_10 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-10-01') TO ('2025-11-01'); +CREATE TABLE media_access_plan.events_2025_11 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-11-01') TO ('2025-12-01'); +CREATE TABLE media_access_plan.events_2025_12 PARTITION OF media_access_plan.events FOR VALUES FROM ('2025-12-01') TO ('2026-01-01'); +CREATE TABLE media_access_plan.events_2026_01 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); +CREATE TABLE media_access_plan.events_2026_02 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-02-01') TO ('2026-03-01'); +CREATE TABLE media_access_plan.events_2026_03 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-03-01') TO ('2026-04-01'); +CREATE TABLE media_access_plan.events_2026_04 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-04-01') TO ('2026-05-01'); +CREATE TABLE media_access_plan.events_2026_05 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-05-01') TO ('2026-06-01'); +CREATE TABLE media_access_plan.events_2026_06 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-06-01') TO ('2026-07-01'); +CREATE TABLE media_access_plan.events_2026_07 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-07-01') TO ('2026-08-01'); +CREATE TABLE media_access_plan.events_2026_08 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-08-01') TO ('2026-09-01'); +CREATE TABLE media_access_plan.events_2026_09 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-09-01') TO ('2026-10-01'); +CREATE TABLE media_access_plan.events_2026_10 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-10-01') TO ('2026-11-01'); +CREATE TABLE media_access_plan.events_2026_11 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-11-01') TO ('2026-12-01'); +CREATE TABLE media_access_plan.events_2026_12 PARTITION OF media_access_plan.events FOR VALUES FROM ('2026-12-01') TO ('2027-01-01'); +CREATE INDEX media_access_plan_tags ON media_access_plan.events USING gin(tags jsonb_path_ops); +CREATE INDEX media_access_plan_id ON media_access_plan.events(community_id,id,created_at DESC); +CREATE INDEX media_access_plan_channel ON media_access_plan.events(community_id,channel_id,created_at DESC,id); +CREATE INDEX media_access_plan_kind ON media_access_plan.events(community_id,kind,created_at DESC,id); +CREATE INDEX media_access_plan_addressable ON media_access_plan.events(community_id,kind,pubkey,channel_id,deleted_at); +INSERT INTO media_access_plan.events(community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id) +SELECT '00000000-0000-0000-0000-000000000001', decode(md5(g::text)||md5(g::text),'hex'), decode(repeat('1',64),'hex'), + '2025-01-01'::timestamptz + (g/50000) * interval '1 month', CASE WHEN g%100=2 THEN 0 WHEN g%4=3 THEN 40003 ELSE 9 END, + CASE WHEN g%50000=0 THEN jsonb_build_array(jsonb_build_array('imeta','x '||repeat('a',64),'url https://plan.example/media/'||repeat('a',64)||'.bin')) + WHEN g%4=3 THEN jsonb_build_array(jsonb_build_array('e',upper(md5((g-2)::text)||md5((g-2)::text))),jsonb_build_array('imeta','x '||CASE WHEN g%50000=3 THEN repeat('a',64) ELSE md5(g::text)||md5(g::text) END,'url https://plan.example/media/'||CASE WHEN g%50000=7 THEN repeat('a',64) ELSE md5(g::text)||md5(g::text) END||'.bin')) + WHEN g%50000=1 THEN jsonb_build_array(jsonb_build_array('imeta','url https://plan.example/media/'||repeat('a',64)||'.bin')) + ELSE jsonb_build_array(jsonb_build_array('imeta','x '||encode(decode(md5(g::text)||md5(g::text),'hex'),'hex'))) END, + CASE WHEN g=2 THEN '{"picture":"https://plan.example/media/'||repeat('a',64)||'.bin"}' ELSE 'ordinary content' END, + decode(repeat('2',128),'hex'), md5(((g/4)%256)::text)::uuid FROM generate_series(0,1199999) g; +-- Finish bulk-load GIN maintenance as autovacuum does in steady state. +-- Otherwise every URL alternative repeatedly scans the bulk insert pending list. +SELECT gin_clean_pending_list(inhrelid) FROM pg_inherits +WHERE inhparent='media_access_plan.media_access_plan_tags'::regclass; +ANALYZE media_access_plan.events; diff --git a/crates/buzz-db/src/store/media_postgres_tests.rs b/crates/buzz-db/src/store/media_postgres_tests.rs new file mode 100644 index 00000000000..5c57e3b4021 --- /dev/null +++ b/crates/buzz-db/src/store/media_postgres_tests.rs @@ -0,0 +1,760 @@ +use super::media::MediaAccess; +use crate::{ + channel::{ChannelType, ChannelVisibility}, + Db, +}; +use buzz_core::CommunityId; +use nostr::{Event, EventBuilder, Keys, Kind, Tag, Timestamp}; + +async fn allowed( + db: &Db, + community: CommunityId, + hash: &str, + base: &str, + keys: &Keys, +) -> MediaAccess { + db.media_access( + community, + hash, + "bin", + base, + keys.public_key().as_bytes(), + false, + |_, _| true, + ) + .await + .unwrap() +} +fn attachment( + keys: &Keys, + hash: &str, + channel: uuid::Uuid, + kind: u16, + target: Option<&Event>, + at: u64, +) -> Event { + let mut tags = vec![Tag::parse(["h", &channel.to_string()]).unwrap()]; + if !hash.is_empty() { + tags.push( + Tag::parse([ + "imeta", + &format!("url /media/{hash}.bin"), + &format!("x {hash}"), + ]) + .unwrap(), + ); + } + if let Some(target) = target { + tags.push(Tag::parse(["e", &target.id.to_hex().to_uppercase()]).unwrap()); + } + EventBuilder::new(Kind::Custom(kind), "media") + .tags(tags) + .custom_created_at(Timestamp::from(at)) + .sign_with_keys(keys) + .unwrap() +} +#[tokio::test] +#[ignore = "requires Postgres"] +async fn edits_deletion_legacy_profiles_and_publication_boundaries() { + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + let db = Db::from_pool(pool); + let host = format!("media-db-{}.example", uuid::Uuid::new_v4()); + let community = db.ensure_configured_community(&host).await.unwrap().id; + let base = format!("https://{host}/media"); + let owner = Keys::generate(); + let outsider = Keys::generate(); + db.ensure_user(community, owner.public_key().as_bytes()) + .await + .unwrap(); + let channel = db + .create_channel( + community, + "media", + ChannelType::Stream, + ChannelVisibility::Private, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + let a = "a".repeat(64); + let b = "b".repeat(64); + db.record_media_upload(community, &a, owner.public_key().as_bytes()) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &owner).await, + MediaAccess::UploaderPreview + ); + assert_eq!( + allowed(&db, community, &a, &base, &outsider).await, + MediaAccess::NoReference + ); + let now = Timestamp::now().as_secs(); + let original = attachment(&owner, &a, channel, 9, None, now); + db.record_media_publications(community, vec![a.clone(), b.clone()]) + .await + .unwrap(); + db.insert_event(community, &original, Some(channel)) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &owner).await, + MediaAccess::Allowed + ); + assert_eq!( + allowed(&db, community, &a, &base, &outsider).await, + MediaAccess::Restricted + ); + let decoy = attachment(&owner, "", channel, 9, None, now + 1); + db.insert_event(community, &decoy, Some(channel)) + .await + .unwrap(); + let edit = attachment(&owner, &b, channel, 40003, Some(&original), now + 1); + let edit = EventBuilder::new(Kind::Custom(40003), &edit.content) + .tags(edit.tags.iter().cloned()) + .tags([Tag::parse(["e", &decoy.id.to_hex()]).unwrap()]) + .custom_created_at(Timestamp::from(now + 1)) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &edit, Some(channel)) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &owner).await, + MediaAccess::Restricted, + "edit drops original attachment" + ); + assert_eq!( + allowed(&db, community, &b, &base, &owner).await, + MediaAccess::Allowed + ); + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(original.id.as_bytes().as_slice()) + .execute(db.pool()) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &b, &base, &owner).await, + MediaAccess::Restricted, + "a second e tag cannot substitute for the deleted edit target" + ); + sqlx::query("UPDATE events SET deleted_at=NULL WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(original.id.as_bytes().as_slice()) + .execute(db.pool()) + .await + .unwrap(); + let edit2 = attachment(&owner, "", channel, 40003, Some(&original), now + 2); + db.insert_event(community, &edit2, Some(channel)) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &b, &base, &owner).await, + MediaAccess::Restricted, + "newest edit alone supplies references" + ); + sqlx::query("DELETE FROM events WHERE community_id=$1") + .bind(community.as_uuid()) + .execute(db.pool()) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &owner).await, + MediaAccess::Restricted, + "purge must not resurrect uploader preview" + ); + // Profiles predate recording and therefore need no grant backfill. + sqlx::query(r#"INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig) + SELECT $1,decode(md5(g::text)||md5(g::text),'hex'),decode(md5(g::text)||md5(g::text),'hex'),now(),0,'[]', + '{"name":"\u00e9"}',decode(repeat('a',128),'hex') FROM generate_series(1,1100) g"#) + .bind(community.as_uuid()).execute(db.pool()).await.unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":format!("{base}/{a}.bin")}).to_string(), + ) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &profile, None).await.unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &outsider).await, + MediaAccess::Allowed + ); + sqlx::query("DELETE FROM events WHERE community_id=$1 AND kind=0 AND pubkey<>$2") + .bind(community.as_uuid()) + .bind(owner.public_key().as_bytes().as_slice()) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(profile.id.as_bytes().as_slice()) + .execute(db.pool()) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &outsider).await, + MediaAccess::Restricted, + "superseded avatar grants nothing" + ); + // Image-only profiles remain readable. Over-encoded animation hashes do not + // widen profile candidates beyond direct hash matches. + let encoded = format!("{base}/{a}.bin") + .replace(':', "%3A") + .replace('/', "%2F") + .replace('a', "%61"); + for (content, expected) in [ + ( + serde_json::json!({"image":format!("{base}/{a}.bin")}), + MediaAccess::Allowed, + ), + ( + serde_json::json!({"picture":format!("https://foreign/poster.png#buzz-anim={encoded}")}), + MediaAccess::Restricted, + ), + ] { + let current = EventBuilder::new(Kind::Metadata, content.to_string()) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, ¤t, None).await.unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &outsider).await, + expected + ); + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(current.id.as_bytes().as_slice()) + .execute(db.pool()) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &a, &base, &owner).await, + MediaAccess::Restricted + ); + } + // A tenant never borrows another tenant's proof or publication. + let other = db + .ensure_configured_community(&format!("other-{}.example", uuid::Uuid::new_v4())) + .await + .unwrap() + .id; + assert_eq!( + allowed(&db, other, &a, &base, &owner).await, + MediaAccess::NoReference + ); +} + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn popular_media_prefers_live_readable_references_and_bounds_work() { + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + let db = Db::from_pool(pool); + let host = format!("popular-media-{}.example", uuid::Uuid::new_v4()); + let community = db.ensure_configured_community(&host).await.unwrap().id; + let base = format!("https://{host}/media"); + let owner = Keys::generate(); + let reader = Keys::generate(); + db.ensure_user(community, owner.public_key().as_bytes()) + .await + .unwrap(); + let channel = db + .create_channel( + community, + "popular", + ChannelType::Stream, + ChannelVisibility::Private, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + let hash = "c".repeat(64); + let emoji_url = format!("{base}/{hash}.bin"); + // Non-counting reactions and superseded sets must never consume the budget. + sqlx::query("INSERT INTO events (community_id,id,pubkey,created_at,kind,tags,content,sig,deleted_at) SELECT $1,decode(lpad(to_hex(n),64,'0'),'hex'),$2,now(),CASE WHEN n%2=0 THEN 7 ELSE 30030 END,$3,'emoji',decode(repeat('00',64),'hex'),CASE WHEN n%2=1 THEN now() ELSE NULL END FROM generate_series(1,2052) n") + .bind(community.as_uuid()).bind(owner.public_key().as_bytes().as_slice()) + .bind(serde_json::json!([["emoji","popular",emoji_url]])) + .execute(db.pool()).await.unwrap(); + let live = EventBuilder::new(Kind::Custom(30030), "") + .tags([ + Tag::parse(["d", "current"]).unwrap(), + Tag::parse(["emoji", "popular", &emoji_url]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &live, None).await.unwrap(); + assert_eq!( + allowed(&db, community, &hash, &base, &reader).await, + MediaAccess::Allowed + ); + let shared_hash = "d".repeat(64); + sqlx::query("INSERT INTO events (community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id) SELECT $1,decode(lpad(to_hex(n),64,'0'),'hex'),$2,now(),9,$3,'private',decode(repeat('00',64),'hex'),$4 FROM generate_series(3000,4050) n") + .bind(community.as_uuid()).bind(owner.public_key().as_bytes().as_slice()) + .bind(serde_json::json!([["imeta",format!("x {shared_hash}"),format!("url /media/{shared_hash}.bin")]])) + .bind(channel).execute(db.pool()).await.unwrap(); + let public = attachment( + &owner, + &shared_hash, + channel, + 1, + None, + Timestamp::now().as_secs(), + ); + db.insert_event(community, &public, None).await.unwrap(); + assert_eq!( + allowed(&db, community, &shared_hash, &base, &reader).await, + MediaAccess::Allowed + ); + // Page-position preview URLs never count, even when they fill a whole + // older partition before the real image-position reference. + let open = db + .create_channel( + community, + "preview-decoys", + ChannelType::Stream, + ChannelVisibility::Open, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + let snapshot_hash = "f".repeat(64); + let snapshot_url = format!("{base}/{snapshot_hash}.png"); + sqlx::query("INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id) SELECT $1,decode(lpad(to_hex(n),64,'0'),'hex'),$2,'2025-01-01'::timestamptz,9,$3,$4,decode(repeat('00',64),'hex'),$5 FROM generate_series(6000,7024) n") + .bind(community.as_uuid()).bind(owner.public_key().as_bytes().as_slice()) + .bind(serde_json::json!([["h",open.to_string()],["link-preview","snapshot","1",snapshot_url,"title","site","description","","","",""]])) + .bind(&snapshot_url).bind(open).execute(db.pool()).await.unwrap(); + let snapshot = EventBuilder::new(Kind::Custom(9), "https://page.example/") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse([ + "link-preview", + "snapshot", + "1", + "https://page.example/", + "title", + "site", + "description", + &snapshot_url, + &snapshot_hash, + "", + "", + ]) + .unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &snapshot, Some(channel)) + .await + .unwrap(); + assert_eq!( + db.media_access( + community, + &snapshot_hash, + "png", + &base, + owner.public_key().as_bytes(), + false, + |_, _| true + ) + .await + .expect("page-position decoys must not exhaust the image-reference budget"), + MediaAccess::Allowed + ); + // More than a full budget of unrelated, ordinary animated profiles must + // not turn a private denial or an absent hash into authorization-unavailable. + let poster = "2".repeat(64); + let animation = format!("{base}/{}.png", "3".repeat(64)) + .replace(':', "%3A") + .replace('/', "%2F"); + // Python-style ensure_ascii JSON leaves ordinary non-ASCII names escaped, + // while the unrelated URL hash remains plain ASCII. Keep literal escapes in + // the signed/stored content, rather than serializing a decoded Rust string. + let profile_content = format!( + r#"{{"name":"Jos\u00e9 \ud83d\udc1d","picture":"{base}/{poster}.png#buzz-anim={animation}"}}"# + ); + assert!(profile_content.contains(r"\u00e9")); + assert_eq!( + serde_json::from_str::(&profile_content).unwrap()["name"], + "José 🐝" + ); + let profiles: Vec<_> = (0..1100) + .map(|_| { + let event = EventBuilder::new(Kind::Metadata, &profile_content) + .sign_with_keys(&Keys::generate()) + .unwrap(); + serde_json::to_value(event).unwrap() + }) + .collect(); + sqlx::query("INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig) SELECT $1,decode(p->>'id','hex'),decode(p->>'pubkey','hex'),to_timestamp((p->>'created_at')::bigint),0,p->'tags',p->>'content',decode(p->>'sig','hex') FROM jsonb_array_elements($2) p") + .bind(community.as_uuid()).bind(serde_json::Value::Array(profiles)).execute(db.pool()).await.unwrap(); + let private_denial = db + .media_access( + community, + &snapshot_hash, + "png", + &base, + reader.public_key().as_bytes(), + false, + |_, _| true, + ) + .await; + let absent = db + .media_access( + community, + &"1".repeat(64), + "bin", + &base, + reader.public_key().as_bytes(), + false, + |_, _| true, + ) + .await; + assert!( + matches!(private_denial, Ok(MediaAccess::Restricted)) + && matches!(absent, Ok(MediaAccess::NoReference)), + "escaped-name decoys: private={private_denial:?}, absent={absent:?}" + ); + // A direct ASCII hash remains readable among escaped-name decoys. A hash + // whose own ASCII characters are Unicode-escaped needs republication. + let canonical_hash = "4".repeat(64); + for (profile_hash, expected) in [ + (canonical_hash.clone(), MediaAccess::Allowed), + (r"\u0034".repeat(64), MediaAccess::Restricted), + ] { + let profile = EventBuilder::new( + Kind::Metadata, + format!(r#"{{"name":"Jos\u00e9","picture":"{base}/{profile_hash}.bin"}}"#), + ) + .sign_with_keys(&Keys::generate()) + .unwrap(); + db.insert_event(community, &profile, None).await.unwrap(); + db.record_media_publications(community, vec![canonical_hash.clone()]) + .await + .unwrap(); + assert_eq!( + allowed(&db, community, &canonical_hash, &base, &reader).await, + expected + ); + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(community.as_uuid()) + .bind(profile.id.as_bytes().as_slice()) + .execute(db.pool()) + .await + .unwrap(); + } + // An oversized readable row must fail closed without buffering all candidates. + let huge_hash = "e".repeat(64); + let huge = EventBuilder::new(Kind::Custom(1), "z".repeat(16 * 1024 * 1024 + 1)) + .tags([Tag::parse([ + "imeta", + &format!("x {huge_hash}"), + &format!("url /media/{huge_hash}.bin"), + ]) + .unwrap()]) + .sign_with_keys(&owner) + .unwrap(); + db.insert_event(community, &huge, None).await.unwrap(); + let error = db + .media_access( + community, + &huge_hash, + "bin", + &base, + reader.public_key().as_bytes(), + false, + |_, _| true, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("work budget"), "{error}"); + // Capacity saturation has a bounded wait before attempting to acquire a writer. + let permits = db.media_authorization_slots.available_permits(); + let _held = db + .media_authorization_slots + .acquire_many(permits as u32) + .await + .unwrap(); + let error = db + .media_access( + community, + &hash, + "bin", + &base, + reader.public_key().as_bytes(), + false, + |_, _| true, + ) + .await + .unwrap_err(); + assert!( + matches!(error, crate::DbError::MediaAuthorizationCapacity), + "{error}" + ); + // Poll until the request is queued, then release capacity without a delay. + let reader_pubkey = reader.public_key(); + let waiting = db.media_access( + community, + &hash, + "bin", + &base, + reader_pubkey.as_bytes(), + false, + |_, _| true, + ); + tokio::pin!(waiting); + assert!(futures_util::poll!(waiting.as_mut()).is_pending()); + drop(_held); + assert_eq!(waiting.await.unwrap(), MediaAccess::Allowed); + + assert_eq!( + sqlx::query_scalar::<_, i32>("SELECT 1") + .fetch_one(db.pool()) + .await + .unwrap(), + 1 + ); +} + +/// Explain the actual complete statement, including channel and edit joins. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn media_reference_full_query_plans() { + use sqlx::{Execute, Row}; + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + let mut tx = pool.begin().await.unwrap(); + let fixture = include_str!("media_plan_fixture.sql"); + sqlx::raw_sql(fixture).execute(&mut *tx).await.unwrap(); + sqlx::raw_sql(sqlx::AssertSqlSafe(format!("CREATE INDEX idx_events_media_edit_target ON media_access_plan.events (community_id,channel_id, ({}),created_at DESC,id DESC) WHERE kind=40003 AND deleted_at IS NULL", super::media::MEDIA_EDIT_TARGET_SQL))) + .execute(&mut *tx).await.unwrap(); + sqlx::query("SET LOCAL jit = on") + .execute(&mut *tx) + .await + .unwrap(); + let jit_available: bool = sqlx::query_scalar("SELECT pg_jit_available()") + .fetch_one(&mut *tx) + .await + .unwrap(); + sqlx::raw_sql(super::media::MEDIA_LOOKUP_SETTINGS_SQL) + .execute(&mut *tx) + .await + .unwrap(); + let jit: String = sqlx::query_scalar("SHOW jit") + .fetch_one(&mut *tx) + .await + .unwrap(); + assert_eq!( + jit, "off", + "the production transaction must not compile partitioned authorization probes" + ); + println!( + "PostgreSQL: {}; JIT available: {}; authorization jit={jit}", + sqlx::query_scalar::<_, String>("SELECT version()") + .fetch_one(&mut *tx) + .await + .unwrap(), + jit_available + ); + let community = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); + let hash = "a".repeat(64); + let reader = [0x11; 32]; + let needles = super::media::media_reference_needles(&hash, "bin", "https://plan.example/media"); + for readable_only in [true, false] { + for probe in 0..4 { + let mut builder = super::media::media_reference_query( + community, + &hash, + &reader, + &needles, + probe, + readable_only, + ); + let mut statement = builder.build(); + let args = statement.take_arguments().unwrap().unwrap(); + let sql = format!( + "EXPLAIN (ANALYZE, BUFFERS, COSTS ON) {}", + statement.sql().as_str() + ); + // Only the production QueryBuilder SQL is prefixed; all values stay bound. + let rows = sqlx::query_with(sqlx::AssertSqlSafe(sql), args) + .fetch_all(&mut *tx) + .await + .unwrap(); + println!("FULL_QUERY_PROBE_{probe}_READABLE_{readable_only}: 24 partitions, 1200000 rows including 300000 live edits across 256 channels"); + let plan = rows + .iter() + .map(|row| row.get::(0)) + .collect::>() + .join("\n"); + if readable_only && probe < 2 { + let names: Vec = sqlx::query_scalar("SELECT c.relname::text FROM pg_inherits h JOIN pg_class c ON c.oid=h.inhrelid WHERE h.inhparent='media_access_plan.idx_events_media_edit_target'::regclass") + .fetch_all(&mut *tx).await.unwrap(); + assert!( + names.iter().any(|name| plan.contains(name)), + "edit-target index absent: {plan}" + ); + } + let execution_ms: f64 = plan + .lines() + .find_map(|line| line.strip_prefix("Execution Time: ")) + .and_then(|value| value.strip_suffix(" ms")) + .unwrap() + .parse() + .unwrap(); + assert!( + execution_ms < 1000.0, + "authorization probe must retain headroom under its 2s timeout: {plan}" + ); + for row in rows { + println!("{}", row.get::(0)); + } + } + } + // The isolation wrapper owns this database. Commit the fixture so a fresh + // production Db handle can read it, with server-side JIT enabled by default. + tx.commit().await.unwrap(); + let production_pool = sqlx::postgres::PgPoolOptions::new().max_connections(1) + .after_connect(|connection,_| Box::pin(async move { + sqlx::raw_sql("SET search_path=media_access_plan,public; SET jit=on; SET statement_timeout='2s'") + .execute(connection).await?; + Ok(()) + })) + .connect(&crate::test_support::database_url()).await.unwrap(); + let channel: uuid::Uuid = sqlx::query_scalar( + "SELECT id FROM media_access_plan.channels WHERE visibility='open' LIMIT 1", + ) + .fetch_one(&production_pool) + .await + .unwrap(); + let url_hash = "b".repeat(64); + let event = EventBuilder::new(Kind::Custom(9), "URL-only legacy attachment") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse([ + "imeta", + &format!("url https://plan.example/media/{url_hash}.bin"), + ]) + .unwrap(), + ]) + .sign_with_keys(&Keys::generate()) + .unwrap(); + sqlx::query("INSERT INTO media_access_plan.events(community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id) VALUES($1,$2,$3,$4,9,$5,$6,$7,$8)") + .bind(community.as_uuid()).bind(event.id.as_bytes().as_slice()).bind(event.pubkey.to_bytes().as_slice()) + .bind(chrono::DateTime::from_timestamp(event.created_at.as_secs() as i64,0).unwrap()) + .bind(serde_json::to_value(&event.tags).unwrap()).bind(&event.content) + .bind(event.sig.serialize().as_slice()).bind(channel).execute(&production_pool).await.unwrap(); + let db = Db::from_pool(production_pool.clone()); + let production_started = std::time::Instant::now(); + let access = db + .media_access( + community, + &url_hash, + "bin", + "https://plan.example/media", + &reader, + false, + |_, _| true, + ) + .await + .expect("production URL-only lookup must apply the bounded non-JIT policy"); + println!( + "PRODUCTION_URL_ONLY_LOOKUP_MS: {}", + production_started.elapsed().as_secs_f64() * 1000.0 + ); + assert_eq!(access, MediaAccess::Allowed); + assert_eq!( + sqlx::query_scalar::<_, String>("SHOW jit") + .fetch_one(&production_pool) + .await + .unwrap(), + "on", + "lookup rollback must restore the pooled connection's JIT setting" + ); + production_pool.close().await; +} + +/// Compare the production Rust target rule to SQL, the catalog and new partitions. +#[tokio::test] +#[ignore = "requires Postgres"] +async fn media_edit_target_sql_parity_and_partition_inheritance() { + use serde_json::json; + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .unwrap(); + let mut tx = pool.begin().await.unwrap(); + let expression = super::media::MEDIA_EDIT_TARGET_SQL; + let a = "aB".repeat(32); + let b = "C".repeat(64); + let cases = [ + json!([["e", a.to_uppercase()]]), + json!([["e", a]]), + json!([["e", "bad"], ["e", a], ["e", b]]), + json!([["e", a], ["e", b]]), + json!([["e"], ["e", 23], ["e", a]]), + json!([["e"]]), + json!([["e", 23]]), + json!([null, ["e", a]]), + json!(null), + json!([]), + ]; + let absent: Option> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( + "SELECT {expression} FROM (SELECT NULL::jsonb AS tags) input" + ))) + .fetch_one(&mut *tx) + .await + .unwrap(); + assert!(absent.is_none()); + let keys = Keys::generate(); + for tags in cases { + // Non-string historical tags cannot be signed by the Rust SDK; omit + // malformed tags when constructing its reference input. + let valid_tags: Vec = tags + .as_array() + .into_iter() + .flatten() + .filter_map(|tag| { + let values: Option> = + tag.as_array()?.iter().map(|v| v.as_str()).collect(); + Tag::parse(values?).ok() + }) + .collect(); + let event = EventBuilder::new(Kind::Custom(40003), "") + .tags(valid_tags) + .sign_with_keys(&keys) + .unwrap(); + let expected = buzz_core::media::first_edit_target(&event); + let actual: Option> = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( + "SELECT {expression} FROM (SELECT $1::jsonb AS tags) input" + ))) + .bind(&tags) + .fetch_one(&mut *tx) + .await + .unwrap(); + assert_eq!(actual, expected, "tags={tags}"); + } + sqlx::raw_sql(sqlx::AssertSqlSafe(format!("CREATE TEMP TABLE media_expected_target(tags jsonb); CREATE INDEX media_expected_target_idx ON media_expected_target (({expression}));"))) + .execute(&mut *tx).await.unwrap(); + let expressions: Vec = sqlx::query_scalar("SELECT pg_get_expr(indexprs,indrelid) FROM pg_index WHERE indexrelid IN ('media_expected_target_idx'::regclass,'public.idx_events_media_edit_target'::regclass) ORDER BY indexrelid") + .fetch_all(&mut *tx).await.unwrap(); + assert_eq!(expressions.len(), 2); + assert_eq!(expressions[0], expressions[1]); + sqlx::raw_sql("CREATE TABLE media_partition_parent (LIKE public.events INCLUDING ALL) PARTITION BY RANGE (created_at); CREATE TABLE media_future_partition PARTITION OF media_partition_parent FOR VALUES FROM ('2099-01-01') TO ('2099-02-01');") + .execute(&mut *tx).await.unwrap(); + let inherited: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM pg_index i JOIN pg_inherits h ON h.inhrelid=i.indexrelid JOIN pg_index parent ON parent.indexrelid=h.inhparent WHERE i.indrelid='media_future_partition'::regclass AND parent.indexprs IS NOT NULL AND i.indisvalid AND pg_get_expr(i.indexprs,i.indrelid)=(SELECT pg_get_expr(indexprs,indrelid) FROM pg_index WHERE indexrelid='public.idx_events_media_edit_target'::regclass))") + .fetch_one(&mut *tx).await.unwrap(); + assert!(inherited); + tx.rollback().await.unwrap(); +} diff --git a/crates/buzz-db/src/store/mod.rs b/crates/buzz-db/src/store/mod.rs index 017f1ba4da4..f9448aae513 100644 --- a/crates/buzz-db/src/store/mod.rs +++ b/crates/buzz-db/src/store/mod.rs @@ -113,3 +113,9 @@ mod admission_postgres_tests; /// Durable NIP-FI verified-profile projection state. pub mod verified_profiles; + +/// Live reference authorization and upload possession. +pub mod media; + +#[cfg(test)] +mod media_postgres_tests; diff --git a/crates/buzz-db/src/store/relay_invite/upgrade_postgres_tests.rs b/crates/buzz-db/src/store/relay_invite/upgrade_postgres_tests.rs index 7a8b8fb3442..b9cb3bce3e5 100644 --- a/crates/buzz-db/src/store/relay_invite/upgrade_postgres_tests.rs +++ b/crates/buzz-db/src/store/relay_invite/upgrade_postgres_tests.rs @@ -637,6 +637,9 @@ async fn migration_schema_desired_pgschema_reconcile_matches_populated_upgrade() // Exercise the deployment path against populated data at the complete // pinned-main schema through 0074, rather than reproducing its DDL by hand. + // Existing partitioned tables require the operator prebuild before pgschema, + // which otherwise tries unsupported CREATE INDEX CONCURRENTLY on the parent. + crate::test_support::prebuild_media_edit_index(&pool).await; apply_pgschema_to_existing_database(&pool).await; assert_admission_column(&pool).await; assert_join_role_preserved(&pool, cases[0].community).await; diff --git a/crates/buzz-db/src/test_support.rs b/crates/buzz-db/src/test_support.rs index e5caa9473b2..80d43afef78 100644 --- a/crates/buzz-db/src/test_support.rs +++ b/crates/buzz-db/src/test_support.rs @@ -8,6 +8,14 @@ pub(crate) fn database_url() -> String { .unwrap_or_else(|_| DEFAULT_DATABASE_URL.to_owned()) } +/// Exercise the operator prebuild path on a small isolated upgrade fixture. +/// Large deployments use concurrent child builds; tests own their entire database. +pub(crate) async fn prebuild_media_edit_index(pool: &sqlx::PgPool) { + let expression = crate::store::media::MEDIA_EDIT_TARGET_SQL; + sqlx::raw_sql(sqlx::AssertSqlSafe(format!("CREATE INDEX idx_events_media_edit_target ON events(community_id,channel_id,({expression}),created_at DESC,id DESC) WHERE kind=40003 AND deleted_at IS NULL"))) + .execute(pool).await.expect("operator media index prebuild on isolated fixture"); +} + /// A uniquely named database on the test server that is dropped when the /// guard goes out of scope, including when the test panics. /// diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index abe6bdd40ea..f12c0f00519 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -459,7 +459,11 @@ impl MediaStorage { sha256: &str, ) -> Result { let key = Self::ctx_sidecar_key(ctx, sha256); - let resp = self.bucket.get_object(&key).await?; + let resp = match self.bucket.get_object(&key).await { + Ok(response) => response, + Err(s3::error::S3Error::HttpFailWithBody(404, _)) => return Err(MediaError::NotFound), + Err(error) => return Err(error.into()), + }; let meta: BlobMeta = serde_json::from_slice(&resp.to_vec())?; Ok(meta) } diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 17f992b9a80..b1e58b0f6a2 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -90,7 +90,7 @@ enum UploadRouteMode { /// /// Non-Blossom errors (`blossom_denial_kind()` returns `None`) always fall /// through to `MediaError::into_response()` regardless of mode. -pub(crate) struct MediaDenial(MediaError, BlossomStrictness); +pub(crate) struct MediaDenial(pub(super) MediaError, pub(super) BlossomStrictness); impl IntoResponse for MediaDenial { fn into_response(self) -> Response { @@ -112,7 +112,8 @@ impl IntoResponse for MediaDenial { ); let mut builder = axum::http::Response::builder() .status(class.http_status()) - .header(header::CONTENT_TYPE, class.content_type()); + .header(header::CONTENT_TYPE, class.content_type()) + .header(header::CACHE_CONTROL, "private, no-store"); if let Some(challenge) = class.www_authenticate() { builder = builder.header("WWW-Authenticate", challenge); } @@ -121,7 +122,12 @@ impl IntoResponse for MediaDenial { .expect("NIP-FI denial response is always valid"); } } - error.into_response() + let mut response = error.into_response(); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response } } @@ -609,6 +615,69 @@ async fn upload_blob_result( // get the same NIP-FI shape as pre-body ones. .map_err(|e| media_denial(e, strictness))?; + if state.config.media_read_mode != super::media_access::MediaReadMode::Off { + let mut recording_reason = "lookup_error"; + let recording: Result<(), MediaDenial> = async { + // A reupload of a legacy attachment must remember that it has been + // published before recording new preview authority. Otherwise purging + // the legacy source later could resurrect uploader preview access. + let base = media_base_url_for_tenant(&state.config.relay_url, auth.tenant.host()); + let ext = descriptor + .url + .rsplit('.') + .next() + .ok_or_else(|| media_denial(MediaError::Internal, strictness))?; + let existing = state + .db + .media_access( + auth.tenant.community(), + &descriptor.sha256, + ext, + &base, + auth.auth_event.pubkey.as_bytes(), + false, + crate::handlers::req::event_visible_to_reader, + ) + .await + .map_err(|error| { + if matches!(error, buzz_db::DbError::MediaAuthorizationCapacity) { + recording_reason = "capacity"; + } + media_denial(MediaError::AuthorizationUnavailable, strictness) + })?; + if matches!( + existing, + buzz_db::media::MediaAccess::Allowed | buzz_db::media::MediaAccess::Restricted + ) { + state + .db + .record_media_publications( + auth.tenant.community(), + vec![descriptor.sha256.clone()], + ) + .await + .map_err(|_| media_denial(MediaError::AuthorizationUnavailable, strictness))?; + } + state + .db + .record_media_upload( + auth.tenant.community(), + &descriptor.sha256, + auth.auth_event.pubkey.as_bytes(), + ) + .await + .map_err(|_| media_denial(MediaError::AuthorizationUnavailable, strictness))?; + Ok(()) + } + .await; + if let Err(error) = recording { + metrics::counter!("buzz_media_upload_access_total", "mode" => state.config.media_read_mode.label(), "reason" => recording_reason).increment(1); + if state.config.media_read_mode == super::media_access::MediaReadMode::Enforce { + return Err(error); + } + } + } + rewrite_descriptor_urls_for_tenant( &mut descriptor, &state.config.relay_url, @@ -757,7 +826,7 @@ async fn enforce_blossom_read_membership( } fn blob_cache_control() -> &'static str { - "private, max-age=31536000, immutable" + "private, no-store" } /// Whether a path-segment extension is a safe token. @@ -908,6 +977,14 @@ pub(crate) async fn get_blob( &slot, ) .await?; + super::media_access::enforce_read( + &state, + &tenant, + &sha256_ext, + auth_event.pubkey.as_bytes(), + strictness, + ) + .await?; serve_blob_for_tenant(&state, &tenant, &sha256_ext, &req_headers) .await .map_err(MediaDenial::from) @@ -1013,6 +1090,7 @@ pub(crate) async fn serve_blob_for_tenant( if start >= total { return axum::response::Response::builder() .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(header::CACHE_CONTROL, cache_control) .header(header::CONTENT_RANGE, format!("bytes */{total}")) .body(axum::body::Body::empty()) .map_err(|_| MediaError::Internal); @@ -1040,6 +1118,7 @@ pub(crate) async fn serve_blob_for_tenant( } None => Ok(axum::response::Response::builder() .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(header::CACHE_CONTROL, cache_control) .header(header::CONTENT_RANGE, format!("bytes */{total}")) .body(axum::body::Body::empty()) .map_err(|_| MediaError::Internal)?), @@ -1224,6 +1303,14 @@ pub(crate) async fn head_blob( &slot, ) .await?; + super::media_access::enforce_read( + &state, + &tenant, + &sha256_ext, + auth_event.pubkey.as_bytes(), + strictness, + ) + .await?; let cache_control = blob_cache_control(); // Sidecar gate FIRST — reject before any blob I/O. diff --git a/crates/buzz-relay/src/api/media_access.rs b/crates/buzz-relay/src/api/media_access.rs new file mode 100644 index 00000000000..d6a6b21df44 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access.rs @@ -0,0 +1,126 @@ +//! Channel media policy, separate from Blossom proof/community admission. +use super::media::MediaDenial; +use crate::state::AppState; +use buzz_core::TenantContext; +use buzz_media::{auth::BlossomStrictness, MediaError}; + +/// Startup-selected rollout policy. Off preserves the current community gate. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MediaReadMode { + /// Skip reference authorization (does not provide channel revocation). + #[default] + Off, + /// Evaluate and count the policy without refusing reads/publications. + Shadow, + /// Enforce fresh channel and event authorization. + Enforce, +} +impl std::str::FromStr for MediaReadMode { + type Err = String; + fn from_str(value: &str) -> Result { + match value { + "off" => Ok(Self::Off), + "shadow" => Ok(Self::Shadow), + "enforce" => Ok(Self::Enforce), + _ => Err("BUZZ_MEDIA_READ_MODE must be off, shadow, or enforce".into()), + } + } +} +impl MediaReadMode { + pub(crate) fn label(self) -> &'static str { + match self { + Self::Off => "off", + Self::Shadow => "shadow", + Self::Enforce => "enforce", + } + } +} + +/// Authorize the parent hash before any blob/thumbnail/range information. +/// The tenant sidecar is a cheap first gate before the new DB lookup. +pub(super) async fn enforce_read( + state: &AppState, + tenant: &TenantContext, + path: &str, + pubkey: &[u8], + strictness: BlossomStrictness, +) -> Result<(), MediaDenial> { + let mode = state.config.media_read_mode; + if mode == MediaReadMode::Off { + return Ok(()); + } + let hash = path + .split('.') + .next() + .ok_or(MediaDenial(MediaError::NotFound, strictness))?; + let meta = match state.media_storage.get_sidecar(tenant, hash).await { + Ok(meta) => meta, + Err(error) => { + let missing = matches!(&error, MediaError::NotFound); + metrics::counter!("buzz_media_read_access_total","mode"=>mode.label(),"reason"=>if missing {"no_ref"} else {"lookup_error"}).increment(1); + if mode == MediaReadMode::Shadow { + return Ok(()); + } + return Err(MediaDenial( + if missing { + MediaError::NotFound + } else { + MediaError::ServiceUnavailable + }, + strictness, + )); + } + }; + let base = super::media::media_base_url_for_tenant(&state.config.relay_url, tenant.host()); + let result = state + .db + .media_access( + tenant.community(), + hash, + &meta.ext, + &base, + pubkey, + false, + crate::handlers::req::event_visible_to_reader, + ) + .await; + let (reason, error) = match result { + Ok(buzz_db::media::MediaAccess::Allowed | buzz_db::media::MediaAccess::UploaderPreview) => { + ("allow", None) + } + Ok(buzz_db::media::MediaAccess::NoReference) => ("no_ref", Some(MediaError::NotFound)), + Ok(buzz_db::media::MediaAccess::Restricted) => ("not_member", Some(MediaError::NotFound)), + Err(error) => { + tracing::warn!(error_kind = %error.kind(), "media authorization lookup failed"); + ( + if matches!(error, buzz_db::DbError::MediaAuthorizationCapacity) { + "capacity" + } else { + "lookup_error" + }, + Some(MediaError::AuthorizationUnavailable), + ) + } + }; + metrics::counter!("buzz_media_read_access_total", "mode" => mode.label(), "reason" => reason) + .increment(1); + if mode == MediaReadMode::Enforce { + if let Some(error) = error { + return Err(MediaDenial(error, strictness)); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rollout_mode_is_explicit() { + for mode in ["off", "shadow", "enforce"] { + assert!(mode.parse::().is_ok()); + } + assert!("typo".parse::().is_err()); + assert_eq!(MediaReadMode::default(), MediaReadMode::Off); + } +} diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests.rs new file mode 100644 index 00000000000..0fe6b7b9852 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access_postgres_tests.rs @@ -0,0 +1,968 @@ +//! Real media handlers and PostgreSQL, with an in-process S3 HTTP fixture. +use super::media::{get_blob, head_blob, upload_blob}; +use crate::state::AppState; +use axum::{ + body::{to_bytes, Body, Bytes}, + http::{Method, Request}, + routing::any, + Router, +}; +use axum::{ + extract::{Path, State}, + http::{header, StatusCode}, + response::{IntoResponse, Response}, +}; +use base64::Engine; +use buzz_db::channel::{ChannelType, ChannelVisibility, MemberRole}; +use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::sync::Mutex; +use tower::ServiceExt; +use uuid::Uuid; + +#[derive(Clone, Default)] +struct Bucket { + objects: Arc>>, + reads: Arc, +} +async fn object( + State(bucket): State, + Path(key): Path, + request: Request, +) -> Response { + if request.method() == Method::PUT { + let data = to_bytes(request.into_body(), 2_000_000).await.unwrap(); + bucket.objects.lock().await.insert(key, data); + return StatusCode::OK.into_response(); + } + bucket.reads.fetch_add(1, Ordering::SeqCst); + let Some(mut data) = bucket.objects.lock().await.get(&key).cloned() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let mut status = StatusCode::OK; + if let Some(range) = request.headers().get(header::RANGE) { + let (start, end) = range + .to_str() + .unwrap() + .strip_prefix("bytes=") + .unwrap() + .split_once('-') + .unwrap(); + let start: usize = start.parse().unwrap(); + let end: usize = end.parse().unwrap_or(data.len() - 1).min(data.len() - 1); + data = data.slice(start..=end); + status = StatusCode::PARTIAL_CONTENT; + } + Response::builder() + .status(status) + .header(header::CONTENT_LENGTH, data.len()) + .body(if request.method() == Method::HEAD { + Body::empty() + } else { + Body::from(data) + }) + .unwrap() +} +fn proof(keys: &Keys, host: &str, hash: &str, verb: &str) -> String { + let event = EventBuilder::new(Kind::Custom(24242), "media fixture") + .tags([ + Tag::parse(["t", verb]).unwrap(), + Tag::parse(["x", hash]).unwrap(), + Tag::parse(["server", host]).unwrap(), + Tag::parse(["expiration", &(Timestamp::now().as_secs() + 55).to_string()]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + format!( + "Nostr {}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(event.as_json()) + ) +} +fn get( + keys: &Keys, + host: &str, + hash: &str, + path: &str, + method: &str, + range: Option<&str>, +) -> Request { + let mut req = Request::builder() + .method(method) + .uri(format!("/media/{path}")) + .header(header::HOST, host) + .header(header::AUTHORIZATION, proof(keys, host, hash, "get")); + if let Some(range) = range { + req = req.header(header::RANGE, range); + } + req.body(Body::empty()).unwrap() +} +async fn fixture(endpoint: &str, mode: super::media_access::MediaReadMode) -> Arc { + let mut config = crate::config::Config::for_test(); + config.database_url = crate::test_support::database_url(); + config.require_relay_membership = false; + config.media_read_mode = mode; + config.media.s3_endpoint = endpoint.to_owned(); + config.media.s3_access_key = "fixture".into(); + config.media.s3_secret_key = "fixture".into(); + config.media_uploads_per_minute = 100; + config.redis_url = + std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".into()); + let pool = sqlx::PgPool::connect(&config.database_url).await.unwrap(); + let db = buzz_db::Db::from_pool(pool.clone()); + let redis = buzz_pubsub::redis_connection(&config.redis_url).unwrap(); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis.clone()) + .await + .unwrap(), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool); + let workflow = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media = buzz_media::MediaStorage::new(&config.media).unwrap(); + let (state, _) = AppState::new( + config, + db, + redis, + audit, + pubsub, + auth, + search, + workflow, + Keys::generate(), + media, + ); + Arc::new(state) +} +fn router(state: Arc) -> Router { + Router::new() + .route("/upload", axum::routing::put(upload_blob)) + .route( + "/media/{sha256_ext}", + axum::routing::get(get_blob).head(head_blob), + ) + .with_state(state) +} +async fn post( + state: &Arc, + tenant: &buzz_core::TenantContext, + keys: &Keys, + event: nostr::Event, +) -> Result { + crate::handlers::ingest::ingest_event( + state, + tenant, + event, + crate::handlers::ingest::IngestAuth::Nip42 { + pubkey: keys.public_key(), + scopes: buzz_auth::Scope::all_known(), + channel_ids: None, + conn_id: Uuid::new_v4(), + }, + ) + .await +} + +mod emoji_reference_postgres_tests; +mod reference_budget_postgres_tests; +mod reference_formats_postgres_tests; + +#[tokio::test] +#[ignore = "requires Postgres"] +async fn private_media_revocation_http_journey() { + let bucket = Bucket::default(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let service = Router::new() + .route("/{*key}", any(object)) + .with_state(bucket.clone()); + let server = tokio::spawn(async move { + axum::serve(listener, service).await.unwrap(); + }); + let state = fixture(&endpoint, super::media_access::MediaReadMode::Enforce).await; + let host = format!("private-media-{}.example", Uuid::new_v4()); + state.db.ensure_configured_community(&host).await.unwrap(); + let tenant = crate::tenant::bind_community(&state.db, &host) + .await + .unwrap(); + let owner = Keys::generate(); + let member = Keys::generate(); + let survivor = Keys::generate(); + for keys in [&owner, &member, &survivor] { + state + .db + .ensure_user(tenant.community(), keys.public_key().as_bytes()) + .await + .unwrap(); + } + let channel = state + .db + .create_channel( + tenant.community(), + "private-media", + ChannelType::Stream, + ChannelVisibility::Private, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + for keys in [&member, &survivor] { + state + .db + .add_member( + tenant.community(), + channel, + keys.public_key().as_bytes(), + MemberRole::Member, + Some(owner.public_key().as_bytes()), + ) + .await + .unwrap(); + } + let bytes = b"channel-secret attachment bytes"; + let hash = hex::encode(Sha256::digest(bytes)); + let upload = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, &host) + .header(header::AUTHORIZATION, proof(&owner, &host, &hash, "upload")) + .header("X-SHA-256", &hash) + .body(Body::from(bytes.as_slice())) + .unwrap(); + let response = router(state.clone()).oneshot(upload).await.unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), 100_000).await.unwrap(); + assert_eq!( + status, + StatusCode::OK, + "upload: {}", + String::from_utf8_lossy(&body) + ); + let descriptor: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let url = descriptor["url"].as_str().unwrap(); + let ext = url.rsplit('.').next().unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(get(&owner, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::OK, + "uploader preview" + ); + assert_eq!( + router(state.clone()) + .oneshot(get(&member, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::NOT_FOUND, + "unposted is uploader-only" + ); + let message = EventBuilder::new(Kind::Custom(9), "private file") + .tags([ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse([ + "imeta", + &format!("url {url}"), + &format!("x {hash}"), + "m application/octet-stream", + &format!("size {}", bytes.len()), + ]) + .unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let result = post(&state, &tenant, &owner, message.clone()).await; + assert!(result.is_ok(), "post failed: {:?}", result.err()); + let response = router(state.clone()) + .oneshot(get(&member, &host, &hash, &hash, "GET", None)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for range in ["bytes=99999999-", "bytes=invalid"] { + let response = router(state.clone()) + .oneshot(get(&survivor, &host, &hash, &hash, "GET", Some(range))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + } + // Model an attachment posted before recording existed. A real reupload + // must retain its legacy publication boundary even after source purging. + sqlx::query("DELETE FROM media_publications WHERE community_id=$1 AND sha256=$2") + .bind(tenant.community().as_uuid()) + .bind(&hash) + .execute(state.db.pool()) + .await + .unwrap(); + sqlx::query("DELETE FROM media_uploaders WHERE community_id=$1 AND sha256=$2") + .bind(tenant.community().as_uuid()) + .bind(&hash) + .execute(state.db.pool()) + .await + .unwrap(); + let reupload = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, &host) + .header(header::AUTHORIZATION, proof(&owner, &host, &hash, "upload")) + .header("X-SHA-256", &hash) + .body(Body::from(bytes.as_slice())) + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(reupload) + .await + .unwrap() + .status(), + StatusCode::OK + ); + sqlx::query("DELETE FROM events WHERE community_id=$1 AND id=$2") + .bind(tenant.community().as_uuid()) + .bind(message.id.as_bytes().as_slice()) + .execute(state.db.pool()) + .await + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(get(&owner, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::NOT_FOUND, + "legacy reupload cannot resurrect preview after purging" + ); + state + .db + .insert_event(tenant.community(), &message, Some(channel)) + .await + .unwrap(); + // A normal channel view can request attachments and avatars in parallel. + let requests = (0..32) + .map(|_| router(state.clone()).oneshot(get(&member, &host, &hash, &hash, "GET", None))); + for response in futures_util::future::join_all(requests).await { + assert_eq!( + response.unwrap().status(), + StatusCode::OK, + "32 concurrent authorized reads" + ); + } + // Real thumbnail keys ensure alias positives cannot pass vacuously. + for variant in ["jpg", "webp"] { + bucket.objects.lock().await.insert( + format!("{}/{hash}.thumb.{variant}", state.config.media.s3_bucket), + Bytes::from_static(b"thumb bytes"), + ); + } + for path in [ + hash.clone(), + format!("{hash}.{ext}"), + format!("{hash}.thumb.jpg"), + format!("{hash}.thumb.webp"), + ] { + for (method, range, expected) in [ + ("GET", None, StatusCode::OK), + ("HEAD", None, StatusCode::OK), + ("GET", Some("bytes=0-3"), StatusCode::PARTIAL_CONTENT), + ] { + assert_eq!( + router(state.clone()) + .oneshot(get(&member, &host, &hash, &path, method, range)) + .await + .unwrap() + .status(), + expected, + "authorized {method} {path}" + ); + } + } + state + .db + .remove_member( + tenant.community(), + channel, + member.public_key().as_bytes(), + owner.public_key().as_bytes(), + ) + .await + .unwrap(); + for path in [ + hash.clone(), + format!("{hash}.{ext}"), + format!("{hash}.thumb.jpg"), + format!("{hash}.thumb.webp"), + ] { + for (method, range) in [ + ("GET", None), + ("HEAD", None), + ("GET", Some("bytes=0-3")), + ("GET", Some("bytes=99999999-")), + ] { + let before = bucket.reads.load(Ordering::SeqCst); + let response = router(state.clone()) + .oneshot(get(&member, &host, &hash, &path, method, range)) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "{method} {path} {range:?}" + ); + assert!(!response.headers().contains_key(header::CONTENT_RANGE)); + assert_eq!( + bucket.reads.load(Ordering::SeqCst), + before + 1, + "denial reads only the tenant sidecar" + ); + } + } + for keys in [&owner, &survivor] { + assert_eq!( + router(state.clone()) + .oneshot(get(keys, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::OK + ); + } + let wrong = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, &host) + .header( + header::AUTHORIZATION, + proof(&member, &host, &hash, "upload"), + ) + .header("X-SHA-256", &hash) + .body(Body::from("wrong bytes")) + .unwrap(); + assert_eq!( + router(state.clone()).oneshot(wrong).await.unwrap().status(), + StatusCode::UNAUTHORIZED, + "deduplicated upload still verifies bytes before recording possession" + ); + for mode in [ + super::media_access::MediaReadMode::Off, + super::media_access::MediaReadMode::Shadow, + ] { + let rollout = fixture(&endpoint, mode).await; + assert_eq!( + router(rollout) + .oneshot(get(&member, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::OK, + "rollout mode {mode:?}" + ); + } + let open = state + .db + .create_channel( + tenant.community(), + "open-media", + ChannelType::Stream, + ChannelVisibility::Open, + None, + owner.public_key().as_bytes(), + None, + ) + .await + .unwrap() + .id; + for event in [ + EventBuilder::new(Kind::Custom(9033), "") + .tags([Tag::parse(["icon", url]).unwrap()]) + .sign_with_keys(&member) + .unwrap(), + EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":url}).to_string(), + ) + .sign_with_keys(&member) + .unwrap(), + EventBuilder::new(Kind::Custom(9), "copy") + .tags([ + Tag::parse(["h", &open.to_string()]).unwrap(), + Tag::parse([ + "imeta", + &format!("url {url}"), + &format!("x {hash}"), + "m application/octet-stream", + &format!("size {}", bytes.len()), + ]) + .unwrap(), + ]) + .sign_with_keys(&member) + .unwrap(), + ] { + let error = post(&state, &tenant, &member, event) + .await + .err() + .expect("copied URL denied"); + assert!( + matches!(error, crate::handlers::ingest::IngestError::Rejected(ref reason) if reason == "restricted: attachment is not accessible") + ); + } + // First-party owners upload avatars before the registered agent signs kind:0. + let agent = Keys::generate(); + state + .db + .ensure_user(tenant.community(), agent.public_key().as_bytes()) + .await + .unwrap(); + state + .db + .set_agent_owner( + tenant.community(), + agent.public_key().as_bytes(), + owner.public_key().as_bytes(), + ) + .await + .unwrap(); + state + .db + .add_member( + tenant.community(), + channel, + agent.public_key().as_bytes(), + MemberRole::Member, + Some(owner.public_key().as_bytes()), + ) + .await + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(get(&agent, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::OK, + "owned agent has access before removal" + ); + state + .db + .remove_member( + tenant.community(), + channel, + agent.public_key().as_bytes(), + owner.public_key().as_bytes(), + ) + .await + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(get(&agent, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::NOT_FOUND, + "owned agent loses access after removal" + ); + let stolen_profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":url}).to_string(), + ) + .sign_with_keys(&agent) + .unwrap(); + assert!( + post(&state, &tenant, &agent, stolen_profile).await.is_err(), + "removed owned agent cannot turn owner-uploaded private media into its avatar" + ); + let avatar_bytes = b"new agent avatar"; + let avatar = hex::encode(Sha256::digest(avatar_bytes)); + state + .db + .record_media_upload(tenant.community(), &avatar, owner.public_key().as_bytes()) + .await + .unwrap(); + let sidecar = bucket + .objects + .lock() + .await + .get(&format!( + "{}/_meta/{}/{hash}.json", + state.config.media.s3_bucket, + tenant.community() + )) + .cloned() + .unwrap(); + bucket.objects.lock().await.insert( + format!( + "{}/_meta/{}/{avatar}.json", + state.config.media.s3_bucket, + tenant.community() + ), + sidecar, + ); + let picture = format!("http://{host}/media/{avatar}.{ext}"); + let forged = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":picture}).to_string(), + ) + .tags([Tag::parse([ + "imeta", + &format!("url {url}"), + &format!("x {hash}"), + "m application/octet-stream", + &format!("size {}", bytes.len()), + ]) + .unwrap()]) + .sign_with_keys(&agent) + .unwrap(); + assert!( + post(&state, &tenant, &agent, forged).await.is_err(), + "owner avatar delegation must not authorize unrelated attachments" + ); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":picture}).to_string(), + ) + .sign_with_keys(&agent) + .unwrap(); + assert!( + post(&state, &tenant, &agent, profile.clone()) + .await + .is_err(), + "agent must not borrow owner upload proof" + ); + // The creation path transfers actual bytes with an agent-signed upload. + let upload = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, &host) + .header( + header::AUTHORIZATION, + proof(&agent, &host, &avatar, "upload"), + ) + .header("X-SHA-256", &avatar) + .header(header::CONTENT_TYPE, "application/octet-stream") + .body(Body::from(avatar_bytes.as_slice())) + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(upload) + .await + .unwrap() + .status(), + StatusCode::OK + ); + let created_profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":picture}).to_string(), + ) + .sign_with_keys(&agent) + .unwrap(); + post(&state, &tenant, &agent, created_profile) + .await + .expect("agent's own upload permits creation under enforce"); + let copy = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":url}).to_string(), + ) + .sign_with_keys(&member) + .unwrap(); + let request = Request::builder() + .method("POST") + .uri("/events") + .header(header::HOST, &host) + .header("x-pubkey", member.public_key().to_hex()) + .body(Body::from(copy.as_json())) + .unwrap(); + let response = crate::router::build_router(state.clone()) + .oneshot(request) + .await + .unwrap(); + let body = to_bytes(response.into_body(), 100_000).await.unwrap(); + assert!( + String::from_utf8_lossy(&body).contains("attachment is not accessible"), + "REST denial: {}", + String::from_utf8_lossy(&body) + ); + use buzz_workflow::action_sink::ActionSink; + let sink = crate::workflow_sink::RelayActionSink::new(&state); + let text = serde_json::json!({"picture":url}).to_string(); + let result = sink + .send_message( + tenant.community(), + &open.to_string(), + &text, + &text, + &member.public_key().to_hex(), + None, + ) + .await; + assert!( + matches!(result, Err(buzz_workflow::action_sink::ActionSinkError::InvalidInput(ref reason)) if reason.contains("attachment is not accessible")), + "workflow copy must be denied" + ); + // A legitimate readable second share preserves access after one removal. + let shared = EventBuilder::new(Kind::Custom(9), "share") + .tags([ + Tag::parse(["h", &open.to_string()]).unwrap(), + Tag::parse(["imeta", &format!("url {url}"), "m application/octet-stream"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + state + .db + .insert_event(tenant.community(), &shared, Some(open)) + .await + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(get(&member, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::OK, + "legacy URL-only share" + ); + reference_budget_postgres_tests::run(&state, &tenant, &host, &owner, &member, channel).await; + emoji_reference_postgres_tests::run(&state, &tenant, &host, &owner, &survivor).await; + reference_formats_postgres_tests::run( + &state, &tenant, &host, &owner, &member, &survivor, channel, + ) + .await; + // Every event kind has bounded publication work; shadow still accepts. + let mut emoji_tags = vec![Tag::parse(["d", "large"]).unwrap()]; + for n in 1000..1513 { + let emoji_hash = format!("{n:064x}"); + state + .db + .record_media_upload( + tenant.community(), + &emoji_hash, + owner.public_key().as_bytes(), + ) + .await + .unwrap(); + let sidecar = bucket + .objects + .lock() + .await + .get(&format!( + "{}/_meta/{}/{hash}.json", + state.config.media.s3_bucket, + tenant.community() + )) + .cloned() + .unwrap(); + bucket.objects.lock().await.insert( + format!( + "{}/_meta/{}/{emoji_hash}.json", + state.config.media.s3_bucket, + tenant.community() + ), + sidecar, + ); + emoji_tags.push( + Tag::parse([ + "emoji", + &format!("emoji{n}"), + &format!("http://{host}/media/{emoji_hash}.{ext}"), + ]) + .unwrap(), + ); + } + let shadow = fixture(&endpoint, super::media_access::MediaReadMode::Shadow).await; + // Shadow counts sidecar lookup errors and lets the serving path decide + // whether storage can supply bytes. Enforce still refuses these checks. + let broken_hash = "8".repeat(64); + let broken_key = format!( + "{}/_meta/{}/{broken_hash}.json", + state.config.media.s3_bucket, + tenant.community() + ); + for malformed in [false, true] { + if malformed { + bucket + .objects + .lock() + .await + .insert(broken_key.clone(), Bytes::from_static(b"not JSON")); + } + assert!( + super::media_access::enforce_read( + &shadow, + &tenant, + &broken_hash, + owner.public_key().as_bytes(), + buzz_media::auth::BlossomStrictness::Permissive, + ) + .await + .is_ok(), + "shadow must not refuse a sidecar lookup error" + ); + assert!(super::media_access::enforce_read( + &state, + &tenant, + &broken_hash, + owner.public_key().as_bytes(), + buzz_media::auth::BlossomStrictness::Permissive + ) + .await + .is_err()); + } + for kind in [10030, 30030] { + let large_set = EventBuilder::new(Kind::Custom(kind), "") + .tags(emoji_tags.clone()) + .sign_with_keys(&owner) + .unwrap(); + let reads = bucket.reads.load(Ordering::SeqCst); + let error = + crate::handlers::media_access::validate_media_references(&tenant, &state, &large_set) + .await + .unwrap_err(); + assert!( + matches!(error,crate::handlers::ingest::IngestError::Rejected(message) if message.contains("too many attachment")) + ); + assert_eq!( + bucket.reads.load(Ordering::SeqCst), + reads, + "over-limit emoji must stop before sidecar fan-out" + ); + crate::handlers::media_access::validate_media_references(&tenant, &shadow, &large_set) + .await + .unwrap(); + } + let bounded_set = EventBuilder::new(Kind::Custom(30030), "") + .tags(emoji_tags.into_iter().take(513)) + .sign_with_keys(&owner) + .unwrap(); + crate::handlers::media_access::validate_media_references(&tenant, &state, &bounded_set) + .await + .expect("512 emoji assets remain publishable"); + let many_tags: Vec = (0..513) + .map(|n| Tag::parse(["imeta", &format!("url http://{host}/media/{n:064x}.bin")]).unwrap()) + .collect(); + let oversized = EventBuilder::new(Kind::Custom(9), "") + .tags(many_tags) + .sign_with_keys(&owner) + .unwrap(); + let error = + crate::handlers::media_access::validate_media_references(&tenant, &state, &oversized) + .await + .unwrap_err(); + assert!( + matches!(error, crate::handlers::ingest::IngestError::Rejected(message) if message.contains("too many attachment")) + ); + crate::handlers::media_access::validate_media_references(&tenant, &shadow, &oversized) + .await + .expect("shadow counts and accepts publication cap"); + + let stale_profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":format!("http://{host}/media/{}.bin", "f".repeat(64))}) + .to_string(), + ) + .sign_with_keys(&owner) + .unwrap(); + crate::handlers::media_access::validate_media_references(&tenant, &shadow, &stale_profile) + .await + .expect("shadow accepts a missing sidecar"); + assert!( + crate::handlers::media_access::validate_media_references(&tenant, &state, &stale_profile) + .await + .is_err(), + "enforce rejects missing sidecars" + ); + // Inject a lookup error after a real existing sidecar: authorization fails closed. + sqlx::query("ALTER TABLE media_uploaders RENAME TO unavailable_media_uploaders") + .execute(state.db.pool()) + .await + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(get(&owner, &host, &hash, &hash, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::SERVICE_UNAVAILABLE + ); + crate::handlers::media_access::validate_media_references(&tenant, &shadow, &bounded_set) + .await + .expect("shadow accepts database lookup failures"); + let upload = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, &host) + .header(header::AUTHORIZATION, proof(&owner, &host, &hash, "upload")) + .header("X-SHA-256", &hash) + .header(header::CONTENT_TYPE, "application/octet-stream") + .body(Body::from(bytes.as_slice())) + .unwrap(); + assert_eq!( + router(shadow.clone()) + .oneshot(upload) + .await + .unwrap() + .status(), + StatusCode::OK, + "shadow upload accepts proof lookup failure" + ); + sqlx::query("ALTER TABLE unavailable_media_uploaders RENAME TO media_uploaders") + .execute(state.db.pool()) + .await + .unwrap(); + sqlx::query("ALTER TABLE media_publications RENAME TO unavailable_media_publications") + .execute(state.db.pool()) + .await + .unwrap(); + crate::handlers::media_access::validate_media_references(&tenant, &shadow, &bounded_set) + .await + .expect("shadow accepts publication recording failure"); + assert!( + crate::handlers::media_access::validate_media_references(&tenant, &state, &bounded_set) + .await + .is_err(), + "enforce rejects recording failure" + ); + let upload = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, &host) + .header(header::AUTHORIZATION, proof(&owner, &host, &hash, "upload")) + .header("X-SHA-256", &hash) + .header(header::CONTENT_TYPE, "application/octet-stream") + .body(Body::from(bytes.as_slice())) + .unwrap(); + assert_eq!( + router(shadow).oneshot(upload).await.unwrap().status(), + StatusCode::OK, + "shadow upload accepts marker recording failure" + ); + let absent = "f".repeat(64); + assert_eq!( + router(state.clone()) + .oneshot(get(&owner, &host, &absent, &absent, "GET", None)) + .await + .unwrap() + .status(), + StatusCode::NOT_FOUND, + "missing sidecar precedes broken DB" + ); + server.abort(); +} diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests/emoji_reference_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests/emoji_reference_postgres_tests.rs new file mode 100644 index 00000000000..17d2f78db6a --- /dev/null +++ b/crates/buzz-relay/src/api/media_access_postgres_tests/emoji_reference_postgres_tests.rs @@ -0,0 +1,77 @@ +use super::*; + +// Extend the enforce-mode HTTP journey, including replacement of a shortcode. +pub(super) async fn run( + state: &Arc, + tenant: &buzz_core::TenantContext, + host: &str, + owner: &Keys, + member: &Keys, +) { + let mut hashes = std::collections::HashSet::new(); + for kind in [30180, 30030, 10030] { + let mut previous: Option = None; + for revision in 0..2 { + // Unique payloads cannot borrow another journey step's reference. + let bytes = + format!("emoji-media-authorization-{kind}-revision-{revision}").into_bytes(); + let hash = hex::encode(Sha256::digest(&bytes)); + assert!( + hashes.insert(hash.clone()), + "emoji fixtures must have distinct hashes" + ); + let request = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, host) + .header(header::AUTHORIZATION, proof(owner, host, &hash, "upload")) + .header("X-SHA-256", &hash) + .body(Body::from(bytes)) + .unwrap(); + let response = router(state.clone()).oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "upload kind {kind}"); + let body = to_bytes(response.into_body(), 100_000).await.unwrap(); + let descriptor: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let url = descriptor["url"].as_str().unwrap(); + // The asset is fresh: only the uploader can preview it. + for (keys, expected) in [(owner, StatusCode::OK), (member, StatusCode::NOT_FOUND)] { + check(state, host, keys, &hash, expected).await; + } + let builder = if kind == 30180 { + buzz_sdk::build_emoji_entry("media_test", url).unwrap() + } else { + EventBuilder::new(Kind::Custom(kind), "").tags([ + Tag::parse(["d", "media_test"]).unwrap(), + Tag::parse(["emoji", "media_test", url]).unwrap(), + ]) + }; + let event = builder + .custom_created_at(Timestamp::from( + Timestamp::now().as_secs() + revision as u64, + )) + .sign_with_keys(owner) + .unwrap(); + post(state, tenant, owner, event).await.unwrap(); + for keys in [owner, member] { + check(state, host, keys, &hash, StatusCode::OK).await; + if let Some(old_hash) = &previous { + // Replacing an entry/set retires the old reference and cannot + // revive uploader preview after publication consumed it. + check(state, host, keys, old_hash, StatusCode::NOT_FOUND).await; + } + } + previous = Some(hash); + } + } + assert_eq!(hashes.len(), 6); +} + +async fn check(state: &Arc, host: &str, keys: &Keys, hash: &str, status: StatusCode) { + for method in ["GET", "HEAD"] { + let response = router(state.clone()) + .oneshot(get(keys, host, hash, hash, method, None)) + .await + .unwrap(); + assert_eq!(response.status(), status, "emoji {method} {hash}"); + } +} diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests/reference_budget_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests/reference_budget_postgres_tests.rs new file mode 100644 index 00000000000..82da98b503d --- /dev/null +++ b/crates/buzz-relay/src/api/media_access_postgres_tests/reference_budget_postgres_tests.rs @@ -0,0 +1,97 @@ +use super::*; + +pub(super) async fn run( + state: &Arc, + tenant: &buzz_core::TenantContext, + host: &str, + owner: &Keys, + member: &Keys, + channel: Uuid, +) { + let bytes = b"reference-budget-avatar-review21"; + let hash = hex::encode(Sha256::digest(bytes)); + let request = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, host) + .header(header::AUTHORIZATION, proof(owner, host, &hash, "upload")) + .header("X-SHA-256", &hash) + .body(Body::from(bytes.as_slice())) + .unwrap(); + let response = router(state.clone()).oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let descriptor: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), 100_000).await.unwrap()).unwrap(); + let url = descriptor["url"].as_str().unwrap(); + let profile = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":url}).to_string(), + ) + .sign_with_keys(owner) + .unwrap(); + post(state, tenant, owner, profile.clone()).await.unwrap(); + + // New artifacts must reject x-only imeta rather than accepting more decoys. + let artifact = EventBuilder::new(Kind::Custom(45010), "budget fixture") + .tags([ + Tag::parse(["ar", "1"]).unwrap(), + Tag::parse(["d", &Uuid::new_v4().to_string()]).unwrap(), + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["type", "buzz.task"]).unwrap(), + Tag::parse(["op", "create"]).unwrap(), + Tag::parse(["title", "Budget fixture"]).unwrap(), + Tag::parse(["imeta", &format!("x {hash}")]).unwrap(), + ]) + .sign_with_keys(owner) + .unwrap(); + let Err(crate::handlers::ingest::IngestError::Rejected(message)) = + post(state, tenant, owner, artifact).await + else { + panic!("expected invalid artifact imeta rejection"); + }; + assert!(message.contains("url"), "{message}"); + + // Simulate historical artifact rows, including containment decoys where the + // URL is on a separate tag. Neither is a same-tag media reference. + sqlx::query("INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig,channel_id) SELECT $1,decode(lpad(to_hex(n),64,'0'),'hex'),$2,'2025-01-01'::timestamptz,45010,$3,'decoy',decode(repeat('00',64),'hex'),$4 FROM generate_series(21000,22025) n") + .bind(tenant.community().as_uuid()).bind(owner.public_key().as_bytes().as_slice()) + .bind(serde_json::json!([["imeta",format!("x {hash}")],["other",format!("url {url}")]])) + .bind(channel).execute(state.db.pool()).await.unwrap(); + // Raw-content matches in malformed or large non-image fields must not + // consume the byte budget before the real profile is examined. + let padded = serde_json::json!({"about":format!("{hash}{}","a".repeat(256*1024)),"picture":null,"image":url}).to_string(); + let profiles: Vec<_> = (0..71) + .map(|_| { + let event = EventBuilder::new(Kind::Metadata, &padded) + .custom_created_at(Timestamp::from(1_735_689_600)) + .sign_with_keys(&Keys::generate()) + .unwrap(); + serde_json::to_value(event).unwrap() + }) + .collect(); + sqlx::query("INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig) SELECT $1,decode(p->>'id','hex'),decode(p->>'pubkey','hex'),to_timestamp((p->>'created_at')::bigint),0,p->'tags',p->>'content',decode(p->>'sig','hex') FROM jsonb_array_elements($2) p") + .bind(tenant.community().as_uuid()).bind(serde_json::Value::Array(profiles)) + .execute(state.db.pool()).await.unwrap(); + sqlx::query("INSERT INTO events(community_id,id,pubkey,created_at,kind,tags,content,sig) VALUES($1,decode(repeat('ab',32),'hex'),$2,now(),0,'[]',$3,decode(repeat('00',64),'hex'))") + .bind(tenant.community().as_uuid()).bind(member.public_key().as_bytes().as_slice()) + .bind(format!("not JSON {hash}")).execute(state.db.pool()).await.unwrap(); + for keys in [owner, member] { + for method in ["GET", "HEAD"] { + let response = router(state.clone()) + .oneshot(get(keys, host, &hash, &hash, method, None)) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "avatar remains readable beside >1024 x-only artifacts" + ); + } + } + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(tenant.community().as_uuid()) + .bind(profile.id.as_bytes().as_slice()) + .execute(state.db.pool()) + .await + .unwrap(); +} diff --git a/crates/buzz-relay/src/api/media_access_postgres_tests/reference_formats_postgres_tests.rs b/crates/buzz-relay/src/api/media_access_postgres_tests/reference_formats_postgres_tests.rs new file mode 100644 index 00000000000..1985509bd68 --- /dev/null +++ b/crates/buzz-relay/src/api/media_access_postgres_tests/reference_formats_postgres_tests.rs @@ -0,0 +1,254 @@ +use super::*; + +// Exercise the client wire formats through real upload, ingest and download paths. +pub(super) async fn run( + state: &Arc, + tenant: &buzz_core::TenantContext, + host: &str, + owner: &Keys, + member: &Keys, + survivor: &Keys, + channel: Uuid, +) { + let images: [&[u8]; 2] = [ + include_bytes!("../../../../buzz-media/tests/fixtures/ios/uikit-sanitized.png"), + include_bytes!( + "../../../../buzz-media/tests/fixtures/android/sanitized/bitmap-srgb-sanitized.png" + ), + ]; + for (index, format) in ["snapshot", "animated", "image"].into_iter().enumerate() { + let mut assets = Vec::new(); + for bytes in images { + let hash = hex::encode(Sha256::digest(bytes)); + let request = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, host) + .header(header::AUTHORIZATION, proof(owner, host, &hash, "upload")) + .header("X-SHA-256", &hash) + .body(Body::from(bytes)) + .unwrap(); + let response = router(state.clone()).oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), 100_000).await.unwrap(); + let descriptor: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assets.push((hash, descriptor["url"].as_str().unwrap().to_owned())); + } + let [(image_hash, image), (favicon_hash, favicon)] = assets.as_slice() else { + unreachable!() + }; + let (kind,content,tags) = match format { + "snapshot" => (Kind::Custom(9),"https://example.com/page".to_owned(),vec![ + Tag::parse(["h",&channel.to_string()]).unwrap(), + Tag::parse(["link-preview","snapshot","1","https://example.com/page","title","site","description",image,image_hash,favicon,favicon_hash]).unwrap(), + ]), + "animated" => (Kind::Metadata,serde_json::json!({"picture":format!("{image}#buzz-anim={}",url::form_urlencoded::byte_serialize(favicon.as_bytes()).collect::())}).to_string(),vec![]), + _ => (Kind::Metadata,serde_json::json!({"image":image,"banner":favicon}).to_string(),vec![]), + }; + // Owning an account or knowing a descriptor never substitutes for bytes. + let copied = EventBuilder::new(kind, &content) + .tags(tags.clone()) + .sign_with_keys(member) + .unwrap(); + assert!( + crate::handlers::media_access::validate_media_references(tenant, state, &copied) + .await + .is_err(), + "copied {format}" + ); + if format == "animated" { + let request = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, host) + .header( + header::AUTHORIZATION, + proof(member, host, image_hash, "upload"), + ) + .header("X-SHA-256", image_hash) + .body(Body::from(images[0])) + .unwrap(); + assert_eq!( + router(state.clone()) + .oneshot(request) + .await + .unwrap() + .status(), + StatusCode::OK + ); + assert!( + crate::handlers::media_access::validate_media_references(tenant, state, &copied) + .await + .is_err(), + "owning only the poster cannot publish the private animation" + ); + } + let event = EventBuilder::new(kind, content) + .tags(tags) + .custom_created_at(Timestamp::from( + Timestamp::now().as_secs() + 30 + index as u64, + )) + .sign_with_keys(owner) + .unwrap(); + post(state, tenant, owner, event.clone()).await.unwrap(); + for (hash, _) in &assets { + for keys in [owner, survivor] { + let response = router(state.clone()) + .oneshot(get(keys, host, hash, hash, "GET", None)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "posted {format}"); + } + let response = router(state.clone()) + .oneshot(get(member, host, hash, hash, "GET", None)) + .await + .unwrap(); + assert_eq!( + response.status(), + if format == "snapshot" { + StatusCode::NOT_FOUND + } else { + StatusCode::OK + } + ); + } + if format == "snapshot" { + for (revision, suppress) in [(1, false), (2, true), (3, false)] { + let mut tags = vec![ + Tag::parse(["h", &channel.to_string()]).unwrap(), + Tag::parse(["e", &event.id.to_hex()]).unwrap(), + ]; + if suppress { + tags.push(Tag::parse(["link-preview", "none"]).unwrap()); + } + let edit = + EventBuilder::new(Kind::Custom(40003), "edited https://example.com/page") + .tags(tags) + .custom_created_at(Timestamp::from(event.created_at.as_secs() + revision)) + .sign_with_keys(owner) + .unwrap(); + post(state, tenant, owner, edit).await.unwrap(); + for (hash, _) in &assets { + for keys in [owner, survivor, member] { + for method in ["GET", "HEAD"] { + let response = router(state.clone()) + .oneshot(get(keys, host, hash, hash, method, None)) + .await + .unwrap(); + assert_eq!( + response.status(), + if revision >= 2 || keys.public_key() == member.public_key() { + StatusCode::NOT_FOUND + } else { + StatusCode::OK + }, + "latest edit preview suppression={suppress}" + ); + } + } + } + } + } + sqlx::query("UPDATE events SET deleted_at=now() WHERE community_id=$1 AND id=$2") + .bind(tenant.community().as_uuid()) + .bind(event.id.as_bytes().as_slice()) + .execute(state.db.pool()) + .await + .unwrap(); + for (hash, _) in &assets { + let response = router(state.clone()) + .oneshot(get(owner, host, hash, hash, "GET", None)) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "deleting {format} must not revive uploader preview" + ); + } + } + // Persona definitions are readable only by their owner until shared. Each + // gets fresh bytes so another profile cannot hide a missing reference. + for shared in [false, true] { + let bytes = format!("persona-media-review21-shared-{shared}"); + let hash = hex::encode(Sha256::digest(bytes.as_bytes())); + let request = Request::builder() + .method("PUT") + .uri("/upload") + .header(header::HOST, host) + .header(header::AUTHORIZATION, proof(owner, host, &hash, "upload")) + .header("X-SHA-256", &hash) + .body(Body::from(bytes)) + .unwrap(); + let response = router(state.clone()).oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let descriptor: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), 100_000).await.unwrap()) + .unwrap(); + let mut tags = vec![Tag::parse([ + "d", + if shared { + "shared-media" + } else { + "private-media" + }, + ]) + .unwrap()]; + if shared { + tags.push(Tag::parse(["shared", "true"]).unwrap()); + } + let persona = EventBuilder::new( + Kind::Custom(30175), + serde_json::json!({"name":"Media persona","avatar_url":descriptor["url"]}).to_string(), + ) + .tags(tags) + .sign_with_keys(owner) + .unwrap(); + post(state, tenant, owner, persona).await.unwrap(); + for keys in [owner, member] { + let report = EventBuilder::new(Kind::Custom(1984), "media report") + .allow_self_tagging() + .tags([ + Tag::parse(["p", &owner.public_key().to_hex()]).unwrap(), + Tag::parse(["x", &hash, "spam"]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + let result = post(state, tenant, keys, report).await; + if shared || keys.public_key() == owner.public_key() { + if let Err(error) = result { + panic!("readable persona blob report shared={shared}: {error:?}"); + } + } else { + assert!( + matches!(result,Err(crate::handlers::ingest::IngestError::Rejected(message)) if message.contains("report target blob not found")) + ); + let absent = EventBuilder::new(Kind::Custom(1984), "missing report") + .tags([ + Tag::parse(["p", &owner.public_key().to_hex()]).unwrap(), + Tag::parse(["x", &"9".repeat(64), "spam"]).unwrap(), + ]) + .sign_with_keys(keys) + .unwrap(); + assert!( + matches!(post(state,tenant,keys,absent).await,Err(crate::handlers::ingest::IngestError::Rejected(message)) if message.contains("report target blob not found")) + ); + } + for method in ["GET", "HEAD"] { + let response = router(state.clone()) + .oneshot(get(keys, host, &hash, &hash, method, None)) + .await + .unwrap(); + assert_eq!( + response.status(), + if shared || keys.public_key() == owner.public_key() { + StatusCode::OK + } else { + StatusCode::NOT_FOUND + }, + "persona shared={shared}" + ); + } + } + } +} diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index e7ab1622b68..b0641a09ce9 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -8,6 +8,8 @@ pub mod gifs; pub mod git; pub mod invites; pub mod media; +/// Channel-aware media authorization. +pub mod media_access; pub mod mesh_demo; pub mod nip05; pub mod nip_fi; @@ -944,3 +946,6 @@ mod parse_query_tests { mod artifact; pub(crate) mod artifact_sync; + +#[cfg(test)] +mod media_access_postgres_tests; diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 3c2deba25f6..eb4fd6ab556 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -313,6 +313,8 @@ pub struct Config { /// Media storage configuration (S3/MinIO). pub media: buzz_media::MediaConfig, + /// Channel media authorization rollout: off, shadow, or enforce. + pub media_read_mode: crate::api::media_access::MediaReadMode, /// Maximum concurrent media uploads handled by one relay process. pub media_max_concurrent_uploads: usize, /// Maximum concurrent media uploads accepted from one pubkey. @@ -1010,6 +1012,15 @@ impl Config { )); } }; + let media_read_mode = match std::env::var("BUZZ_MEDIA_READ_MODE") { + Ok(value) => value.parse().map_err(ConfigError::InvalidValue)?, + Err(std::env::VarError::NotPresent) => crate::api::media_access::MediaReadMode::Off, + Err(_) => { + return Err(ConfigError::InvalidValue( + "BUZZ_MEDIA_READ_MODE must be valid Unicode".into(), + )) + } + }; let media = buzz_media::MediaConfig { s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") .unwrap_or_else(|_| "http://localhost:9000".to_string()), @@ -1483,6 +1494,7 @@ impl Config { allow_nip_oa_auth, klipy, media, + media_read_mode, media_max_concurrent_uploads, media_max_concurrent_uploads_per_pubkey, media_uploads_per_minute, diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index fcafc186ef1..46a1c81f282 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -1460,26 +1460,8 @@ async fn validate_edit_ownership( event: &Event, state: &AppState, ) -> Result<(), String> { - let target_hex = event - .tags - .iter() - .find_map(|t| { - if t.kind().to_string() == "e" { - t.content().and_then(|v| { - if v.len() == 64 && v.chars().all(|c| c.is_ascii_hexdigit()) { - Some(v.to_string()) - } else { - None - } - }) - } else { - None - } - }) + let target_bytes = buzz_core::media::first_edit_target(event) .ok_or_else(|| "missing e tag for edit target".to_string())?; - - let target_bytes = - hex::decode(&target_hex).map_err(|_| "invalid target event ID".to_string())?; // Kind 40003 skips the generic membership gate, so this validator is the // only authority: a target outside the edit's `h` channel, unreadable, or // not the actor's to edit is denied exactly like a missing one. @@ -2566,6 +2548,28 @@ struct ChannelCreateRequest { members: Vec>, } +async fn validate_event_imeta( + tenant: &TenantContext, + state: &AppState, + event: &Event, + tenant_media_base: &str, +) -> Result<(), IngestError> { + let imeta_tags: Vec> = event + .tags + .iter() + .filter(|t| t.kind().to_string() == "imeta") + .map(|t| t.as_slice().iter().map(|s| s.to_string()).collect()) + .collect(); + if !imeta_tags.is_empty() { + crate::api::validate_imeta_tags(&imeta_tags, tenant_media_base) + .map_err(|e| IngestError::Rejected(format!("invalid: {e}")))?; + crate::api::verify_imeta_blobs(tenant, &imeta_tags, &state.media_storage) + .await + .map_err(|e| IngestError::Rejected(format!("invalid: {e}")))?; + } + Ok(()) +} + async fn ingest_event_inner( state: &Arc, tracer: &Arc, @@ -2727,6 +2731,8 @@ async fn ingest_event_inner( // (commands, feedback, reports, moderation) so no write returns ahead of it. enforce_write_restriction(state, tenant, kind_u32, auth.pubkey()).await?; + super::media_access::validate_media_references(tenant, state, &event).await?; + // Command kinds are routed AFTER signature verification, timestamp check, // pubkey/auth match, and scope validation — never before. if buzz_core::kind::is_command_kind(kind_u32) { @@ -3155,6 +3161,9 @@ async fn ingest_event_inner( // Artifact revisions passed the same home-channel write gates as kind 9 // above; they are stored and published without conversation side effects. if kind_u32 == buzz_core::kind::KIND_ARTIFACT { + let base = + crate::api::media::media_base_url_for_tenant(&state.config.relay_url, tenant.host()); + validate_event_imeta(tenant, state, &event, &base).await?; let result = super::artifact::accept(state, tenant, &event, &auth).await?; if let Some(ch_id) = channel_id { emit( @@ -3413,19 +3422,7 @@ async fn ingest_event_inner( .map_err(|e| IngestError::Rejected(format!("invalid: {e}")))?; } - let imeta_tags: Vec> = event - .tags - .iter() - .filter(|t| t.kind().to_string() == "imeta") - .map(|t| t.as_slice().iter().map(|s| s.to_string()).collect()) - .collect(); - if !imeta_tags.is_empty() { - crate::api::validate_imeta_tags(&imeta_tags, &tenant_media_base) - .map_err(|e| IngestError::Rejected(format!("invalid: {e}")))?; - crate::api::verify_imeta_blobs(tenant, &imeta_tags, &state.media_storage) - .await - .map_err(|e| IngestError::Rejected(format!("invalid: {e}")))?; - } + validate_event_imeta(tenant, state, &event, &tenant_media_base).await?; let thread_meta = if requires_h_channel_scope(kind_u32) { if let Some(ch_id) = channel_id { diff --git a/crates/buzz-relay/src/handlers/media_access.rs b/crates/buzz-relay/src/handlers/media_access.rs new file mode 100644 index 00000000000..626c45c7ac1 --- /dev/null +++ b/crates/buzz-relay/src/handlers/media_access.rs @@ -0,0 +1,165 @@ +//! Prevent a copied inaccessible URL from granting access through a new event. +use super::ingest::IngestError; +use crate::{api::media_access::MediaReadMode, state::AppState}; +use buzz_core::TenantContext; +use nostr::Event; + +pub(crate) async fn validate_media_references( + tenant: &TenantContext, + state: &AppState, + event: &Event, +) -> Result<(), IngestError> { + let mode = state.config.media_read_mode; + if mode == MediaReadMode::Off { + return Ok(()); + } + let base = crate::api::media::media_base_url_for_tenant(&state.config.relay_url, tenant.host()); + let hashes = buzz_core::media::publication_hashes(event, &base); + // The bounded profile read probe searches the original signed JSON for the + // literal hash. Do not accept an encoded-only reference that probe cannot find. + if !profile_hashes_are_literal(event, &hashes) { + metrics::counter!("buzz_media_publication_access_total", "mode" => mode.label(), "reason" => "noncanonical_profile_hash").increment(1); + if mode == MediaReadMode::Enforce { + return Err(IngestError::Rejected( + "restricted: profile media hashes must appear literally in profile content".into(), + )); + } + } + if hashes.len() > 512 { + metrics::counter!("buzz_media_publication_access_total", "mode" => mode.label(), "reason" => "limit").increment(1); + return if mode == MediaReadMode::Enforce { + Err(IngestError::Rejected( + "restricted: too many attachment references".into(), + )) + } else { + // Skip per-reference fan-out, but retain a single best-effort + // publication boundary batch for accepted shadow events. + if state + .db + .record_media_publications(tenant.community(), hashes.into_iter().collect()) + .await + .is_err() + { + metrics::counter!("buzz_media_publication_access_total", "mode" => mode.label(), "reason" => "record_error").increment(1); + } + Ok(()) + }; + } + let author = super::ingest::effective_message_author(event, &state.relay_keypair.public_key()); + for hash in &hashes { + let checked = async { + let meta = state + .media_storage + .get_sidecar(tenant, hash) + .await + .map_err(|error| { + if matches!(error, buzz_media::MediaError::NotFound) { + ( + "missing_sidecar", + IngestError::Rejected( + "restricted: attachment is not accessible".into(), + ), + ) + } else { + ( + "lookup_error", + IngestError::Internal("error: media reference lookup failed".into()), + ) + } + })?; + let access = state + .db + .media_access( + tenant.community(), + hash, + &meta.ext, + &base, + &author, + true, + super::req::event_visible_to_reader, + ) + .await + .map_err(|error| { + ( + if matches!(error, buzz_db::DbError::MediaAuthorizationCapacity) { + "capacity" + } else { + "lookup_error" + }, + IngestError::Internal("error: media authorization unavailable".into()), + ) + })?; + if matches!( + access, + buzz_db::media::MediaAccess::Allowed | buzz_db::media::MediaAccess::UploaderPreview + ) { + Ok(()) + } else { + Err(( + "denied", + IngestError::Rejected("restricted: attachment is not accessible".into()), + )) + } + } + .await; + let reason = checked + .as_ref() + .err() + .map_or("allow", |(reason, _)| *reason); + metrics::counter!("buzz_media_publication_access_total", "mode" => mode.label(), "reason" => reason).increment(1); + if let Err((_, error)) = checked { + if mode == MediaReadMode::Enforce { + return Err(error); + } + } + } + // A marker consumes previews but never grants download access. Shadow + // recording is best effort and cannot change event acceptance on failure. + if !hashes.is_empty() + && state + .db + .record_media_publications(tenant.community(), hashes.into_iter().collect()) + .await + .is_err() + { + metrics::counter!("buzz_media_publication_access_total", "mode" => mode.label(), "reason" => "record_error").increment(1); + if mode == MediaReadMode::Enforce { + return Err(IngestError::Internal( + "error: media publication recording failed".into(), + )); + } + } + Ok(()) +} + +fn profile_hashes_are_literal(event: &Event, hashes: &std::collections::BTreeSet) -> bool { + !matches!(event.kind.as_u16(), 0 | 30175) + || hashes.iter().all(|hash| event.content.contains(hash)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn profile_publication_matches_the_bounded_literal_read_probe() { + let keys = nostr::Keys::generate(); + let hash = "f".repeat(64); + let poster = "a".repeat(64); + let base = "https://example.org/media"; + let plain = format!(r#"{{"picture":"{base}/{hash}.png"}}"#); + let escaped = plain.replace(&hash, &format!("\\u0066{}", &hash[1..])); + let animated = format!( + r#"{{"picture":"{base}/{poster}.png#buzz-anim=https%3A%2F%2Fexample.org%2Fmedia%2F%66{}.gif"}}"#, + &hash[1..] + ); + for (content, accepted) in [(plain, true), (escaped, false), (animated, false)] { + let event = nostr::EventBuilder::new(nostr::Kind::Metadata, content) + .sign_with_keys(&keys) + .unwrap(); + let hashes = buzz_core::media::publication_hashes(&event, base); + assert!(hashes.contains(&hash)); + assert_eq!(profile_hashes_are_literal(&event, &hashes), accepted); + } + } +} diff --git a/crates/buzz-relay/src/handlers/mod.rs b/crates/buzz-relay/src/handlers/mod.rs index 7f5d11082b4..3a0216ecdd3 100644 --- a/crates/buzz-relay/src/handlers/mod.rs +++ b/crates/buzz-relay/src/handlers/mod.rs @@ -34,6 +34,8 @@ pub mod identity_archive; pub mod imeta; /// Transport-neutral event ingestion pipeline. pub mod ingest; +/// Local media publication authorization. +pub mod media_access; #[cfg(test)] mod membership_notice_postgres_tests; /// Community moderation authorization seam (capability helper). diff --git a/crates/buzz-relay/src/handlers/report.rs b/crates/buzz-relay/src/handlers/report.rs index 32395a23582..50217f21fc1 100644 --- a/crates/buzz-relay/src/handlers/report.rs +++ b/crates/buzz-relay/src/handlers/report.rs @@ -80,11 +80,36 @@ pub async fn handle_report_event( // Known Phase-1 limitation: the media sidecar API does not expose a // cheap typed not-found vs transient-storage distinction here, so // all lookup failures surface as a missing blob to the reporter. - state + let meta = state .media_storage .get_sidecar(tenant, &sha_hex) .await .map_err(|_| "invalid: report target blob not found".to_string())?; + if state.config.media_read_mode == crate::api::media_access::MediaReadMode::Enforce { + let base = crate::api::media::media_base_url_for_tenant( + &state.config.relay_url, + tenant.host(), + ); + let access = state + .db + .media_access( + tenant.community(), + &sha_hex, + &meta.ext, + &base, + &reporter_pubkey, + false, + super::req::event_visible_to_reader, + ) + .await; + if !matches!( + access, + Ok(buzz_db::media::MediaAccess::Allowed + | buzz_db::media::MediaAccess::UploaderPreview) + ) { + return Err("invalid: report target blob not found".to_owned()); + } + } (ReportTarget::Blob(sha256), None) } ParsedReportTarget::Pubkey { pubkey } => (ReportTarget::Pubkey(pubkey), None), diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 4208b5b0e42..905d1c7b080 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -402,6 +402,12 @@ impl ActionSink for RelayActionSink { .sign_with_keys(&state.relay_keypair) .map_err(|e| ActionSinkError::EventBuild(format!("signing: {e}")))?; + crate::handlers::media_access::validate_media_references(&tenant, &state, &event) + .await + .map_err(|error| { + ActionSinkError::InvalidInput(format!("media reference denied: {error:?}")) + })?; + let event_id_hex = event.id.to_hex(); let event_id_bytes = event.id.as_bytes().to_vec(); let kind_u32 = KIND_STREAM_MESSAGE; diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index e9929b848d9..56b11b90d1e 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -882,3 +882,128 @@ async fn test_ws_invalid_imeta_missing_fields() { client.disconnect().await.unwrap(); } + +/// Run against an isolated relay with BUZZ_MEDIA_READ_MODE=enforce. +#[tokio::test] +#[ignore = "requires a relay with channel media enforcement"] +async fn test_private_channel_media_revocation() { + use buzz_test_client::BuzzTestClient; + let owner = Keys::generate(); + let member = Keys::generate(); + let survivor = Keys::generate(); + let mut owner_ws = BuzzTestClient::connect(&relay_ws_url(), &owner) + .await + .unwrap(); + let mut member_ws = BuzzTestClient::connect(&relay_ws_url(), &member) + .await + .unwrap(); + let survivor_ws = BuzzTestClient::connect(&relay_ws_url(), &survivor) + .await + .unwrap(); + let channel = uuid::Uuid::new_v4().to_string(); + let create = EventBuilder::new(Kind::Custom(9007), "") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["name", &format!("private-{channel}")]).unwrap(), + Tag::parse(["channel_type", "stream"]).unwrap(), + Tag::parse(["visibility", "private"]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_ws.send_event(create).await.unwrap(); + assert!(ok.accepted, "create: {}", ok.message); + for keys in [&member, &survivor] { + let add = EventBuilder::new(Kind::Custom(9000), "") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["p", &keys.public_key().to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_ws.send_event(add).await.unwrap(); + assert!(ok.accepted, "add: {}", ok.message); + } + let http = http_client(); + let bytes = format!("private attachment {channel}").into_bytes(); + let hash = hex::encode(Sha256::digest(&bytes)); + let response = upload(&http, &owner, &bytes).await; + assert_eq!(response.status(), 200); + let descriptor: serde_json::Value = response.json().await.unwrap(); + let url = descriptor["url"].as_str().unwrap(); + let post = EventBuilder::new(Kind::Custom(9), "private media") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse([ + "imeta", + &format!("url {url}"), + &format!("x {hash}"), + "m application/octet-stream", + &format!("size {}", bytes.len()), + ]) + .unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_ws.send_event(post).await.unwrap(); + assert!(ok.accepted, "post: {}", ok.message); + let auth = || blossom_auth_header(&sign_blossom_get_auth(&member, &hash)); + let response = http + .get(url) + .header("Authorization", auth()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!(response.headers()["cache-control"], "private, no-store"); + let remove = EventBuilder::new(Kind::Custom(9001), "") + .tags([ + Tag::parse(["h", &channel]).unwrap(), + Tag::parse(["p", &member.public_key().to_hex()]).unwrap(), + ]) + .sign_with_keys(&owner) + .unwrap(); + let ok = owner_ws.send_event(remove).await.unwrap(); + assert!(ok.accepted, "remove: {}", ok.message); + for (method, range) in [ + (reqwest::Method::GET, None), + (reqwest::Method::HEAD, None), + (reqwest::Method::GET, Some("bytes=0-3")), + (reqwest::Method::GET, Some("bytes=999999-")), + ] { + let mut request = http + .request(method.clone(), url) + .header("Authorization", auth()); + if let Some(range) = range { + request = request.header("Range", range); + } + let response = request.send().await.unwrap(); + assert_eq!(response.status(), 404, "{method} {range:?}"); + assert!(!response.headers().contains_key("content-range")); + } + for keys in [&owner, &survivor] { + let response = http + .get(url) + .header( + "Authorization", + blossom_auth_header(&sign_blossom_get_auth(keys, &hash)), + ) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200, "positive control"); + } + let copy = EventBuilder::new( + Kind::Metadata, + serde_json::json!({"picture":url}).to_string(), + ) + .sign_with_keys(&member) + .unwrap(); + let ok = member_ws.send_event(copy).await.unwrap(); + assert!( + !ok.accepted && ok.message.contains("attachment is not accessible"), + "copy: {ok:?}" + ); + owner_ws.disconnect().await.unwrap(); + member_ws.disconnect().await.unwrap(); + survivor_ws.disconnect().await.unwrap(); +} diff --git a/desktop/src-tauri/src/commands/agents.rs b/desktop/src-tauri/src/commands/agents.rs index ba28ba67c03..c80ef617c46 100644 --- a/desktop/src-tauri/src/commands/agents.rs +++ b/desktop/src-tauri/src/commands/agents.rs @@ -493,8 +493,8 @@ async fn create_managed_agent_in( // ── Phase 2: compute NIP-OA auth tag (sync) ────────────────────────────── // Agents authenticate via the auth tag in their kind:0 profile event. // No tokens are minted. Fail closed: bad auth tag → don't create agent. + let (owner_keys, creation_relay) = profile::creation_authority(state, &resolved_relay_url)?; let auth_tag = { - let owner_keys = state.signing_keys()?; // Bridge nostr 0.37 → 0.36 (buzz-sdk) via hex round-trip. let compat_owner = nostr::Keys::parse(&owner_keys.secret_key().to_secret_hex()) .map_err(|e| format!("failed to bridge owner keys: {e}"))?; @@ -809,11 +809,11 @@ async fn create_managed_agent_in( try_regenerate_nest(&app); // ── Phase 4: sync agent profile on relay (async, outside lock) ─────────── - // Use the avatar persisted on the record so the published profile and any - // later reconciliation agree on the same value. - let mut profile_sync_error = profile::publish_agent_profile_with_about( + // Publish the saved avatar after transferring its bytes to the new signer. + let mut profile_sync_error = profile::publish_created_agent_profile( state, - &resolved_relay_url, + &creation_relay, + &owner_keys, &agent_keys, &name, resolved_avatar_url.as_deref(), diff --git a/desktop/src-tauri/src/commands/agents_profile.rs b/desktop/src-tauri/src/commands/agents_profile.rs index 909606c38be..a3081449126 100644 --- a/desktop/src-tauri/src/commands/agents_profile.rs +++ b/desktop/src-tauri/src/commands/agents_profile.rs @@ -349,44 +349,49 @@ pub(super) fn profile_needs_sync( } } -/// Publish a managed agent's kind:0 profile with the authored public -/// description as `about`, resolving the effective -/// relay URL from the record's stored value. Returns the sync error (if any) -/// rather than failing the caller — profile publish is best-effort in the -/// create and snapshot-import flows that share this helper. +/// Capture authority while the creating workspace is still active, before awaits. +pub(crate) fn creation_authority( + state: &AppState, + relay: &str, +) -> Result<(nostr::Keys, String), String> { + Ok(( + state.signing_keys()?, + crate::relay::effective_agent_relay_url(relay, &relay_ws_url_with_override(state)), + )) +} + +/// Explicit owner action transfers verified bytes; reconcile has no owner signer. #[allow(clippy::too_many_arguments)] -pub(crate) async fn publish_agent_profile_with_about( +pub(crate) async fn publish_created_agent_profile( state: &AppState, - record_relay_url: &str, - agent_keys: &nostr::Keys, - display_name: &str, - avatar_url: Option<&str>, + relay: &str, + owner: &nostr::Keys, + agent: &nostr::Keys, + name: &str, + avatar: Option<&str>, about: Option<&str>, auth_tag: Option<&str>, ) -> Option { - let relay_url = crate::relay::effective_agent_relay_url( - record_relay_url, - &relay_ws_url_with_override(state), - ); - crate::relay::sync_managed_agent_profile( - state, - &relay_url, - agent_keys, - display_name, - avatar_url, - about, - auth_tag, + if let Err(error) = crate::relay::profile_avatar::transfer_created_avatar( + state, relay, owner, agent, avatar, auth_tag, ) .await - .err() + { + return Some(error); + } + crate::relay::sync_managed_agent_profile(state, relay, agent, name, avatar, about, auth_tag) + .await + .err() } /// Publish a fresh persona-backed agent's kind:0 profile, computing the /// effective public `about` from the persona itself. /// Shared by flows in files at the size ratchet (snapshot import). +#[allow(clippy::too_many_arguments)] // Complete captured authority stays explicit. pub(crate) async fn publish_persona_profile( state: &AppState, - record_relay_url: &str, + relay_url: &str, + owner_keys: &nostr::Keys, agent_keys: &nostr::Keys, display_name: &str, avatar_url: Option<&str>, @@ -394,9 +399,10 @@ pub(crate) async fn publish_persona_profile( auth_tag: Option<&str>, ) -> Option { let about = crate::managed_agents::effective_agent_description(persona.description.as_deref()); - publish_agent_profile_with_about( + publish_created_agent_profile( state, - record_relay_url, + relay_url, + owner_keys, agent_keys, display_name, avatar_url, diff --git a/desktop/src-tauri/src/commands/personas/mod.rs b/desktop/src-tauri/src/commands/personas/mod.rs index ac43a4719ab..9a8668e4059 100644 --- a/desktop/src-tauri/src/commands/personas/mod.rs +++ b/desktop/src-tauri/src/commands/personas/mod.rs @@ -70,6 +70,8 @@ pub use sharing::set_persona_shared; pub use sharing::update_persona_and_publish; mod update; pub use update::update_persona; +#[cfg(test)] +pub(crate) use update::update_persona_with; mod inbound; pub use inbound::reconcile_inbound_persona_event; #[cfg(test)] diff --git a/desktop/src-tauri/src/commands/personas/pending.rs b/desktop/src-tauri/src/commands/personas/pending.rs index 47836cd30cd..d83ce345bc3 100644 --- a/desktop/src-tauri/src/commands/personas/pending.rs +++ b/desktop/src-tauri/src/commands/personas/pending.rs @@ -71,6 +71,14 @@ pub(super) fn prepare_persona_publication( shared_override: Option, ) -> Result { let scope = crate::managed_agents::retention::active_retention_scope(app, state)?; + prepare_persona_publication_in_scope(scope, persona, shared_override) +} + +pub(super) fn prepare_persona_publication_in_scope( + scope: RetentionScope, + persona: &AgentDefinition, + shared_override: Option, +) -> Result { let (event, retained, persona) = prepare_persona_publication_at( &scope.db_path, &scope.owner_keys, @@ -104,8 +112,8 @@ fn retained_persona_is_shared(row: Option<&RetainedEvent>) -> bool { /// never present an unshared persona as published. The durable share state /// lives in the retention head, so nothing is lost: the true value reappears /// once the identity is signable again. -pub(super) fn project_active_persona_sharing( - app: &AppHandle, +pub(super) fn project_active_persona_sharing( + app: &AppHandle, state: &AppState, personas: &mut [AgentDefinition], ) { @@ -132,7 +140,7 @@ fn project_scoped_persona_sharing( } } -fn project_persona_sharing_at( +pub(super) fn project_persona_sharing_at( db_path: &std::path::Path, owner_pubkey: &str, personas: &mut [AgentDefinition], diff --git a/desktop/src-tauri/src/commands/personas/sharing.rs b/desktop/src-tauri/src/commands/personas/sharing.rs index 9df82517613..1644fd30b27 100644 --- a/desktop/src-tauri/src/commands/personas/sharing.rs +++ b/desktop/src-tauri/src/commands/personas/sharing.rs @@ -78,13 +78,19 @@ pub async fn update_persona_and_publish( app: AppHandle, ) -> Result { let (_, prepared) = - super::update::update_persona_with(input, app.clone(), |app, state, persona| { + super::update::update_persona_with(input, app.clone(), |app, _state, persona, scope| { // Strict path: this command's contract is to report the publication // outcome, so an enqueue failure must reach the UI rather than being // logged and swallowed. - let result = prepare_persona_publication(app, state, persona, None)?; + let scope = scope.ok_or("Signing keys unavailable for persona publication")?; + let result = + super::pending::prepare_persona_publication_in_scope(scope.clone(), persona, None)?; // F2: refresh any shared 30178 heads that include this persona. - crate::commands::refresh_team_catalog_heads_for_persona(app, state, &persona.id); + crate::commands::refresh_team_catalog_heads_for_persona_in_scope( + app, + scope, + &persona.id, + ); Ok(result) }) .await?; diff --git a/desktop/src-tauri/src/commands/personas/snapshot/import.rs b/desktop/src-tauri/src/commands/personas/snapshot/import.rs index 0c9f1709165..ffec154f792 100644 --- a/desktop/src-tauri/src/commands/personas/snapshot/import.rs +++ b/desktop/src-tauri/src/commands/personas/snapshot/import.rs @@ -443,7 +443,7 @@ pub(crate) fn build_agent_snapshot_import_preview( /// `AgentDefinition` + `ManagedAgentRecord` through the same primitives /// used by the normal create flow. /// 3. Publish — kind:30175 definition via retention path; kind:0 profile -/// via `sync_managed_agent_profile`. +/// via `publish_created_agent_profile`. /// 4. Memory — for each opted-in entry, build a fresh `kind:30174` event /// with `engram::build_event` under the new agent↔owner conversation /// key and POST it to the relay. Failures are collected and returned as @@ -458,6 +458,8 @@ pub async fn confirm_agent_snapshot_import( app: AppHandle, state: State<'_, AppState>, ) -> Result { + let owner_keys = state.signing_keys()?; + let captured_relay = relay_ws_url_with_override(&state); // ── Phase 1: validate (no writes) ──────────────────────────────────────── // Locked cards unlock only via this machine's exact key endpoints; // anything else fails closed here, before key generation. @@ -512,8 +514,12 @@ pub async fn confirm_agent_snapshot_import( }; // ── Phase 2: mint keys + auth tag (sync, outside lock) ─────────────────── + if state.signing_keys()?.public_key() != owner_keys.public_key() + || relay_ws_url_with_override(&state) != captured_relay + { + return Err("Workspace changed during import; retry the import".into()); + } let (agent_keys, private_key_nsec, pubkey, auth_tag, owner_pubkey_hex) = { - let owner_keys = state.signing_keys()?; let agent_keys = nostr::Keys::generate(); let pubkey = agent_keys.public_key().to_hex(); let private_key_nsec = agent_keys @@ -690,11 +696,11 @@ pub async fn confirm_agent_snapshot_import( }; // ── Phase 3b: publish kind:0 profile (async, outside lock) ─────────────── - let relay_url = - effective_agent_relay_url(&record.relay_url, &relay_ws_url_with_override(&state)); + let relay_url = effective_agent_relay_url(&record.relay_url, &captured_relay); let profile_sync_error = crate::commands::agents::publish_persona_profile( &state, - &record.relay_url, + &relay_url, + &owner_keys, &agent_keys, &display_name, effective_avatar.as_deref(), diff --git a/desktop/src-tauri/src/commands/personas/update.rs b/desktop/src-tauri/src/commands/personas/update.rs index 76939f75d04..33d11ca3968 100644 --- a/desktop/src-tauri/src/commands/personas/update.rs +++ b/desktop/src-tauri/src/commands/personas/update.rs @@ -14,7 +14,7 @@ use crate::{ util::now_iso, }; -use super::{normalize_description, pending, retain_persona_pending, trim_optional, trim_required}; +use super::{normalize_description, pending, trim_optional, trim_required}; #[cfg(test)] mod name_propagation_tests; @@ -121,17 +121,110 @@ type ProfileSyncParams = Vec<( Option, )>; +pub(crate) type PersonaAvatarTransfers = Vec<(nostr::Keys, String, Option, Option)>; + +/// Only settings used by the target avatar plan participate in its fence. +/// Runtime churn and unrelated definitions cannot invalidate the transfer. +fn avatar_settings_snapshot( + persona: &AgentDefinition, + records: &[ManagedAgentRecord], +) -> Result<(serde_json::Value, serde_json::Value), String> { + let mut linked: Vec<_> = records + .iter() + .filter(|r| r.persona_id.as_deref() == Some(&persona.id)) + .map(|r| { + serde_json::json!({ + "pubkey":r.pubkey, "private_key_nsec":r.private_key_nsec, + "relay_url":r.relay_url, "auth_tag":r.auth_tag, "avatar_url":r.avatar_url, + "agent_command_override":r.agent_command_override, + }) + }) + .collect(); + linked.sort_by(|a, b| a["pubkey"].as_str().cmp(&b["pubkey"].as_str())); + Ok(( + serde_json::to_value(persona).map_err(|e| e.to_string())?, + serde_json::Value::Array(linked), + )) +} + +/// Build the complete plan before any transfer or local write. A missing signer +/// must leave the old desired avatar in place so key recovery can retry it. +fn plan_persona_avatar_transfers( + persona: &AgentDefinition, + records: &[ManagedAgentRecord], + workspace_relay: &str, +) -> Result { + let linked: Vec<_> = records + .iter() + .filter(|r| r.persona_id.as_deref() == Some(&persona.id)) + .collect(); + let unavailable: Vec<_> = linked + .iter() + .filter(|r| nostr::Keys::parse(&r.private_key_nsec).is_err()) + .map(|r| format!("{} ({})", r.name, r.pubkey)) + .collect(); + if !unavailable.is_empty() { + return Err(format!("Cannot change the avatar: signing keys are unavailable for {}. Restore their keys, or remove or unlink those instances, then retry. Changes were not saved.", unavailable.join(", "))); + } + linked + .into_iter() + .map(|record| { + let keys = nostr::Keys::parse(&record.private_key_nsec).map_err(|e| e.to_string())?; + let mut updated = record.clone(); + let update = prepare_linked_profile_update(&mut updated, persona, false, true, false); + Ok(( + keys, + crate::relay::effective_agent_relay_url(&record.relay_url, workspace_relay), + update.profile_avatar, + record.auth_tag.clone(), + )) + }) + .collect() +} + +/// Transfer explicit persona selections before local writes. Empty plans (an +/// unchanged avatar) do not acquire upload proof with the owner's authority. +pub(crate) async fn transfer_persona_avatars_before_save( + state: &AppState, + owner: &nostr::Keys, + transfers: &PersonaAvatarTransfers, +) -> Result<(), String> { + for (keys, relay, avatar, tag) in transfers { + crate::relay::profile_avatar::transfer_created_avatar( + state, + relay, + owner, + keys, + avatar.as_deref(), + tag.as_deref(), + ) + .await + .map_err(|e| { + format!("Could not transfer the persona avatar; changes were not saved: {e}") + })?; + } + Ok(()) +} + #[tauri::command] pub async fn update_persona( input: UpdatePersonaRequest, app: AppHandle, ) -> Result { - let (persona, ()) = update_persona_with(input, app, |app, state, persona| { - retain_persona_pending(app, state, persona); + let (persona, ()) = update_persona_with(input, app, |app, _state, persona, scope| { + if let Some(scope) = scope { + if let Err(e) = pending::retain_persona_pending_at(scope, persona) { + eprintln!("buzz-desktop: persona-retain: {e}"); + } + crate::commands::refresh_team_catalog_heads_for_persona_in_scope( + app, + scope, + &persona.id, + ); + } // F2: immediately refresh any shared 30178 heads that include this // persona as a member. Best-effort inside retain so a hiccup cannot // fail the persona edit itself. - crate::commands::refresh_team_catalog_heads_for_persona(app, state, &persona.id); Ok(()) }) .await?; @@ -145,13 +238,68 @@ pub async fn update_persona( /// [`update_persona`] enqueues best-effort, while /// [`sharing::update_persona_and_publish`] prepares a strict publication and /// returns the event so the caller can await relay acceptance. -pub(super) async fn update_persona_with( +pub(crate) async fn update_persona_with( input: UpdatePersonaRequest, - app: AppHandle, - retain: impl FnOnce(&AppHandle, &AppState, &AgentDefinition) -> Result + Send + 'static, + app: AppHandle, + retain: impl FnOnce( + &AppHandle, + &AppState, + &AgentDefinition, + Option<&crate::managed_agents::retention::RetentionScope>, + ) -> Result + + Send + + 'static, ) -> Result<(AgentDefinition, R), String> { use tauri::Manager; + // Capture explicit owner authority before any await. Transfer changed avatar + // bytes before saving, so a failed transfer leaves the old local avatar and + // a normal retry still recognizes the change. + let state = app.state::(); + let retention_scope = { + let _workspace = state.workspace_apply_lock.lock().await; + crate::managed_agents::retention::active_retention_scope(&app, &state).ok() + }; + let mut owner_keys = None; + let workspace_relay = retention_scope + .as_ref() + .map(|scope| scope.relay_url.clone()) + .unwrap_or_else(|| crate::relay::relay_ws_url_with_override(&state)); + let (avatar_snapshot, transfers) = { + let _guard = state + .managed_agents_store_lock + .lock() + .map_err(|e| e.to_string())?; + let personas = load_personas(&app)?; + let records = load_managed_agents(&app)?; + let persona = personas + .iter() + .find(|p| p.id == input.id) + .ok_or_else(|| format!("agent {} not found", input.id))?; + let avatar = trim_optional(input.avatar_url.clone()); + let mut transfers = Vec::new(); + let snapshot = if persona.avatar_url != avatar { + let mut changed = persona.clone(); + changed.avatar_url = avatar; + changed.acp_command = trim_optional(input.acp_command.clone()); + transfers = plan_persona_avatar_transfers(&changed, &records, &workspace_relay)?; + owner_keys = Some( + retention_scope + .as_ref() + .ok_or("Signing keys unavailable for avatar transfer")? + .owner_keys + .clone(), + ); + Some(avatar_settings_snapshot(persona, &records)?) + } else { + None + }; + (snapshot, transfers) + }; + if let Some(owner) = &owner_keys { + transfer_persona_avatars_before_save(&state, owner, &transfers).await?; + } + // Phase 1: synchronous save (persona record + linked agent avatar updates) let (result, retained, profile_sync_params) = tokio::task::spawn_blocking({ let app = app.clone(); @@ -172,7 +320,23 @@ pub(super) async fn update_persona_with( .lock() .map_err(|error| error.to_string())?; let mut personas = load_personas(&app)?; - pending::project_active_persona_sharing(&app, &state, &mut personas); + if let Some(expected) = avatar_snapshot { + let current = personas.iter().find(|p| p.id == input.id).ok_or("Agent changed during avatar transfer; retry the edit")?; + if avatar_settings_snapshot(current, &load_managed_agents(&app)?)? != expected + || state.signing_keys()?.public_key() != owner_keys.as_ref().ok_or("Missing avatar transfer authority")?.public_key() + || crate::relay::relay_ws_url_with_override(&state) != workspace_relay + { + return Err("Agent settings or workspace changed during avatar transfer; retry the edit".into()); + } + } + if let Some(scope) = &retention_scope { + if let Err(e) = pending::project_persona_sharing_at(&scope.db_path, &scope.owner_keys.public_key().to_hex(), &mut personas) { + eprintln!("buzz-desktop: persona-share-projection: {e}"); + for persona in &mut personas { persona.shared = false; } + } + } else { + for persona in &mut personas { persona.shared = false; } + } let persona = personas .iter_mut() .find(|record| record.id == input.id) @@ -214,7 +378,7 @@ pub(super) async fn update_persona_with( let result = persona.clone(); save_personas(&app, &personas)?; - let retained = retain(&app, &state, &result)?; + let retained = retain(&app, &state, &result, retention_scope.as_ref())?; try_regenerate_nest(&app); // If the avatar, display_name, or effective description changed, @@ -226,7 +390,6 @@ pub(super) async fn update_persona_with( let mut records = load_managed_agents(&app)?; let mut params: ProfileSyncParams = Vec::new(); let mut agents_modified = false; - let workspace_relay = crate::relay::relay_ws_url_with_override(&state); // Propagate the display_name rename to instances that still // carry the old definition display_name (pool-named instances diff --git a/desktop/src-tauri/src/commands/personas/update/name_propagation_tests.rs b/desktop/src-tauri/src/commands/personas/update/name_propagation_tests.rs index e75e88edfe1..89f93e8d15b 100644 --- a/desktop/src-tauri/src/commands/personas/update/name_propagation_tests.rs +++ b/desktop/src-tauri/src/commands/personas/update/name_propagation_tests.rs @@ -209,3 +209,24 @@ fn test_rename_renames_all_matching_instances_in_one_pass() { assert_eq!(records[1].name, "Duncan Idaho"); assert_eq!(records[2].name, "Birch", "pool-named instance untouched"); } + +#[test] +fn missing_signer_refuses_the_whole_avatar_plan_and_key_recovery_can_retry() { + let mut record = agent("persona-1", "Keyless", None); + record.slug = Some("persona-1".into()); + record.avatar_url = Some("https://example.org/old.png".into()); + let mut persona = record.clone().to_definition_view().unwrap(); + persona.id = "persona-1".into(); + let previous = serde_json::to_value(&record).unwrap(); + persona.avatar_url = Some("https://example.org/new.png".into()); + let error = plan_persona_avatar_transfers(&persona, &[record.clone()], "wss://example.org") + .err() + .unwrap(); + assert!(error.contains("Keyless") && error.contains("Restore") && error.contains("unlink")); + assert_eq!(serde_json::to_value(&record).unwrap(), previous); + let keys = nostr::Keys::generate(); + record.private_key_nsec = keys.secret_key().to_secret_hex(); + let plan = plan_persona_avatar_transfers(&persona, &[record], "wss://example.org").unwrap(); + assert_eq!(plan.len(), 1); + assert_eq!(plan[0].2, persona.avatar_url); +} diff --git a/desktop/src-tauri/src/commands/team_snapshot.rs b/desktop/src-tauri/src/commands/team_snapshot.rs index 8f43e4f8ce5..9cd2d0fa1f1 100644 --- a/desktop/src-tauri/src/commands/team_snapshot.rs +++ b/desktop/src-tauri/src/commands/team_snapshot.rs @@ -20,10 +20,13 @@ use crate::{ load_managed_agents, load_personas, load_teams, load_teams_readonly, save_managed_agents, save_personas, save_teams, AgentDefinition, ManagedAgentRecord, TeamRecord, }, - relay::{effective_agent_relay_url, relay_ws_url_with_override, sync_managed_agent_profile}, + relay::{effective_agent_relay_url, relay_ws_url_with_override}, util::now_iso, }; +mod profile; +pub(crate) use profile::publish_imported_team_member_profile; + /// Team snapshots have a combined 25 MiB JSON / 50 MiB PNG payload cap. /// Every member is validated before any persistent write. pub(crate) const MAX_TEAM_SNAPSHOT_JSON_BYTES: usize = 25 * 1024 * 1024; @@ -500,7 +503,7 @@ pub async fn preview_team_snapshot_import( /// surfacing rollback failures alongside the original error. This makes /// the store phase all-or-none for ordinary application errors; a process /// crash between atomic file commits is NOT covered. -/// 4. Profile sync — for each member, call `sync_managed_agent_profile`. +/// 4. Profile sync — for each member, call `publish_created_agent_profile`. /// Best-effort; errors are collected per member. /// 5. Memory restore — for each member with non-empty snapshot memory, /// publish each entry as a `kind:30174` engram event. Best-effort. @@ -524,10 +527,9 @@ pub async fn confirm_team_snapshot_import( // ── Phase 2: mint keys + auth tags (sync, outside lock) ───────────────── // All mints must succeed before we enter the store. If any fails, zero writes. - let owner_pubkey_hex = { - let keys = state.signing_keys()?; - keys.public_key().to_hex() - }; + let owner_keys = state.signing_keys()?; + let relay_ws = relay_ws_url_with_override(&state); + let owner_pubkey_hex = owner_keys.public_key().to_hex(); let mut minted: Vec = Vec::with_capacity(snapshot.members.len()); for (member, definition) in snapshot.members.iter().zip(definitions) { @@ -537,7 +539,6 @@ pub async fn confirm_team_snapshot_import( let minted_parallelism = definition.parallelism; let (agent_keys, private_key_nsec, pubkey, auth_tag) = { - let owner_keys = state.signing_keys()?; let agent_keys = nostr::Keys::generate(); let pubkey = agent_keys.public_key().to_hex(); let private_key_nsec = { @@ -774,7 +775,6 @@ pub async fn confirm_team_snapshot_import( }; // ── Phase 4 & 5: profile sync + memory restore (async, outside lock) ──── - let relay_ws = relay_ws_url_with_override(&state); let mut member_results: Vec = Vec::with_capacity(minted.len()); for (m, snap_member) in minted.iter().zip(snapshot.members.iter()) { @@ -783,17 +783,17 @@ pub async fn confirm_team_snapshot_import( // Phase 4: profile sync (best-effort). let profile_about = crate::managed_agents::effective_agent_description(m.definition.description.as_deref()); - let profile_sync_error = sync_managed_agent_profile( + let profile_sync_error = publish_imported_team_member_profile( &state, &relay_url, + &owner_keys, &m.agent_keys, &m.display_name, m.effective_avatar.as_deref(), profile_about.as_deref(), m.auth_tag.as_deref(), ) - .await - .err(); + .await; // Phase 5: memory restore (best-effort). let memory_total = snap_member.memory.entries.len(); diff --git a/desktop/src-tauri/src/commands/team_snapshot/profile.rs b/desktop/src-tauri/src/commands/team_snapshot/profile.rs new file mode 100644 index 00000000000..1249be575e2 --- /dev/null +++ b/desktop/src-tauri/src/commands/team_snapshot/profile.rs @@ -0,0 +1,19 @@ +use crate::app_state::AppState; + +/// Publish an imported member only after its own signer holds the selected +/// target-local avatar bytes. The caller captures owner and relay before I/O; +/// failures remain in the import result's profile_sync_error for the user. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn publish_imported_team_member_profile( + state: &AppState, + relay: &str, + owner: &nostr::Keys, + agent: &nostr::Keys, + name: &str, + avatar: Option<&str>, + about: Option<&str>, + auth_tag: Option<&str>, +) -> Option { + crate::publish_created_agent_profile(state, relay, owner, agent, name, avatar, about, auth_tag) + .await +} diff --git a/desktop/src-tauri/src/commands/teams/mod.rs b/desktop/src-tauri/src/commands/teams/mod.rs index 208ac3a7117..61ee952ceea 100644 --- a/desktop/src-tauri/src/commands/teams/mod.rs +++ b/desktop/src-tauri/src/commands/teams/mod.rs @@ -214,6 +214,15 @@ pub(crate) fn refresh_team_catalog_heads_for_persona( pending::refresh_shared_team_catalog_heads_for_persona(app, state, persona_id); } +/// Refresh persona-dependent catalog heads in an edit's captured community. +pub(crate) fn refresh_team_catalog_heads_for_persona_in_scope( + app: &AppHandle, + scope: &crate::managed_agents::retention::RetentionScope, + persona_id: &str, +) { + pending::refresh_shared_team_catalog_heads_for_persona_in_scope(app, scope, persona_id); +} + /// Refresh (or retract) one team's shared 30178 catalog head after an inbound /// 30176 team edit landed on this device. /// diff --git a/desktop/src-tauri/src/commands/teams/pending.rs b/desktop/src-tauri/src/commands/teams/pending.rs index 9967e590fb7..e1e4f66d4bb 100644 --- a/desktop/src-tauri/src/commands/teams/pending.rs +++ b/desktop/src-tauri/src/commands/teams/pending.rs @@ -393,13 +393,25 @@ pub(super) fn refresh_shared_team_catalog_heads_for_persona( app: &AppHandle, state: &AppState, persona_id: &str, +) { + match crate::managed_agents::retention::active_retention_scope(app, state) { + Ok(scope) => { + refresh_shared_team_catalog_heads_for_persona_in_scope(app, &scope, persona_id) + } + Err(e) => eprintln!("buzz-desktop: team-catalog-refresh-for-persona: {e}"), + } +} + +pub(super) fn refresh_shared_team_catalog_heads_for_persona_in_scope( + app: &AppHandle, + scope: &RetentionScope, + persona_id: &str, ) { let result = (|| -> Result<(), String> { use crate::managed_agents::{load_personas, load_teams}; let teams = load_teams(app)?; let personas = load_personas(app)?; - let scope = crate::managed_agents::retention::active_retention_scope(app, state)?; for team in &teams { if team.is_builtin || !team.persona_ids.iter().any(|id| id == persona_id) { diff --git a/desktop/src-tauri/src/managed_agents/retention.rs b/desktop/src-tauri/src/managed_agents/retention.rs index 88278288c16..44e5a479612 100644 --- a/desktop/src-tauri/src/managed_agents/retention.rs +++ b/desktop/src-tauri/src/managed_agents/retention.rs @@ -23,6 +23,7 @@ pub use legacy_migration::migrate_legacy_retention_db; /// their relay heads and pending publications are not. Keeping a separate /// database per `(relay_url, owner_pubkey)` prevents a pending write created in /// community A from being drained into community B after a workspace switch. +#[derive(Clone)] pub struct RetentionScope { pub db_path: PathBuf, pub relay_url: String, diff --git a/desktop/src-tauri/src/relay/profile_avatar.rs b/desktop/src-tauri/src/relay/profile_avatar.rs index 7d6240134c9..998db9e78f9 100644 --- a/desktop/src-tauri/src/relay/profile_avatar.rs +++ b/desktop/src-tauri/src/relay/profile_avatar.rs @@ -62,6 +62,28 @@ pub(crate) async fn localize_avatar( keys: &Keys, avatar: Option<&str>, auth_tag: Option<&str>, +) -> Result, String> { + if let Some((poster, animation)) = avatar.and_then(buzz_core_pkg::media::animated_avatar_urls) { + let poster = localize_avatar_url(state, relay, keys, Some(&poster), auth_tag) + .await? + .ok_or("missing avatar poster")?; + let animation = localize_avatar_url(state, relay, keys, Some(&animation), auth_tag) + .await? + .ok_or("missing avatar animation")?; + return Ok(Some(format!( + "{poster}#buzz-anim={}", + percent_encoding::utf8_percent_encode(&animation, percent_encoding::NON_ALPHANUMERIC) + ))); + } + localize_avatar_url(state, relay, keys, avatar, auth_tag).await +} + +async fn localize_avatar_url( + state: &AppState, + relay: &str, + keys: &Keys, + avatar: Option<&str>, + auth_tag: Option<&str>, ) -> Result, String> { let Some(avatar) = avatar else { return Ok(None); @@ -158,6 +180,74 @@ pub(crate) async fn localize_avatar( if !mime.starts_with("image/") { return Err("agent avatar must be an image".into()); } + upload_avatar_bytes(state, &target, keys, hash, bytes, &mime, auth_tag).await +} + +/// Explicit creation-time byte transfer. Only the chosen target-local image is +/// fetched as the creating owner; actual bytes are then uploaded as the new +/// agent. Reconciliation never inherits this owner capability. +pub(crate) async fn transfer_created_avatar( + state: &AppState, + relay: &str, + owner: &Keys, + agent: &Keys, + avatar: Option<&str>, + auth_tag: Option<&str>, +) -> Result<(), String> { + if let Some((poster, animation)) = avatar.and_then(buzz_core_pkg::media::animated_avatar_urls) { + transfer_created_avatar_url(state, relay, owner, agent, Some(&poster), auth_tag).await?; + transfer_created_avatar_url(state, relay, owner, agent, Some(&animation), auth_tag).await + } else { + transfer_created_avatar_url(state, relay, owner, agent, avatar, auth_tag).await + } +} + +async fn transfer_created_avatar_url( + state: &AppState, + relay: &str, + owner: &Keys, + agent: &Keys, + avatar: Option<&str>, + auth_tag: Option<&str>, +) -> Result<(), String> { + let Some(source) = avatar.and_then(|value| url::Url::parse(value).ok()) else { + return Ok(()); + }; + let target = community_base(relay)?; + if source.origin() != target.origin() || !source.path().starts_with("/media/") { + return Ok(()); + } + let hash = media_hash(&source)?; + let auth = sign_blossom_get_auth_header(owner, target.as_str(), 60)?; + let response = state + .media_fetch_client + .get(source.clone()) + .timeout(TIMEOUT) + .header("Authorization", auth) + .send() + .await + .map_err(|e| classify_request_error(&e))?; + let bytes = read_bounded(response, MAX_AVATAR_BYTES).await?; + if hex::encode(Sha256::digest(&bytes)) != hash { + return Err("agent avatar content hash does not match its URL".into()); + } + let mime = detect_and_validate_mime(&bytes)?; + if !mime.starts_with("image/") { + return Err("agent avatar must be an image".into()); + } + upload_avatar_bytes(state, &target, agent, hash, bytes, &mime, auth_tag).await?; + Ok(()) +} + +async fn upload_avatar_bytes( + state: &AppState, + target: &url::Url, + keys: &Keys, + hash: &str, + bytes: Vec, + mime: &str, + auth_tag: Option<&str>, +) -> Result, String> { // Preserve the already-uploaded bytes (including animation). Re-encoding // changes the content hash and needlessly destroys the portable identity. let upload_auth = sign_blossom_upload_auth(keys, hash, 60, target.as_str())?; @@ -174,7 +264,7 @@ pub(crate) async fn localize_avatar( .put(url) .timeout(TIMEOUT) .header("Authorization", &auth) - .header("Content-Type", &mime) + .header("Content-Type", mime) .header("X-SHA-256", hash) .body(body.clone()); if let Some(tag) = auth_tag { @@ -247,15 +337,21 @@ async fn existing_projection( } let profile: serde_json::Value = serde_json::from_str(&event.content).map_err(|e| e.to_string())?; - let Some(picture) = profile["picture"] - .as_str() - .and_then(|p| url::Url::parse(p).ok()) - else { + let Some(picture) = profile["picture"].as_str() else { return Ok(None); }; - Ok((picture.origin() == target.origin() - && media_hash(&picture).ok() == Some(media_hash(source)?)) - .then(|| picture.into())) + let candidates = match buzz_core_pkg::media::animated_avatar_urls(picture) { + Some((poster, animation)) => vec![poster, animation], + None => vec![picture.to_owned()], + }; + let source_hash = media_hash(source)?; + Ok(candidates + .into_iter() + .filter_map(|value| url::Url::parse(&value).ok()) + .find(|picture| { + picture.origin() == target.origin() && media_hash(picture).ok() == Some(source_hash) + }) + .map(Into::into)) } async fn read_bounded(response: reqwest::Response, cap: usize) -> Result, String> { diff --git a/desktop/src-tauri/src/relay/profile_avatar/tests.rs b/desktop/src-tauri/src/relay/profile_avatar/tests.rs index 7778f638018..67934ec50df 100644 --- a/desktop/src-tauri/src/relay/profile_avatar/tests.rs +++ b/desktop/src-tauri/src/relay/profile_avatar/tests.rs @@ -18,6 +18,8 @@ struct Data { profiles: Vec, requests: Vec<(String, nostr::Event)>, fail_upload: bool, + enforce_possession: bool, + uploaders: std::collections::HashSet<(String, nostr::PublicKey)>, reject_profile: bool, redirect: Option, upload_url: Option, @@ -92,6 +94,8 @@ async fn upload(State(state): State, headers: HeaderMap, bytes: Byt } let hash = hex::encode(Sha256::digest(&bytes)); assert_eq!(headers["x-sha-256"].to_str().unwrap(), hash); + let signer = data.requests.last().unwrap().1.pubkey; + data.uploaders.insert((hash.clone(), signer)); data.blobs.insert(format!("{hash}.png"), bytes.to_vec()); Json(serde_json::json!({ "url": data.upload_url.clone().unwrap_or_else(|| format!("{}/media/{hash}.png", state.base)), @@ -106,7 +110,24 @@ async fn publish( auth(&headers, "profile", &state); event.verify().unwrap(); let mut data = state.data.lock().unwrap(); - let accepted = !data.reject_profile; + let owns_avatar = if data.enforce_possession { + let profile: serde_json::Value = serde_json::from_str(&event.content).unwrap(); + profile["picture"].as_str().is_none_or(|picture| { + let urls = buzz_core_pkg::media::animated_avatar_urls(picture) + .map(|(poster, animation)| vec![poster, animation]) + .unwrap_or_else(|| vec![picture.to_owned()]); + urls.into_iter().all(|value| { + let url = url::Url::parse(&value).unwrap(); + url.origin() != url::Url::parse(&state.base).unwrap().origin() + || data + .uploaders + .contains(&(media_hash(&url).unwrap().to_owned(), event.pubkey)) + }) + }) + } else { + true + }; + let accepted = !data.reject_profile && owns_avatar; let id = event.id.to_hex(); if accepted { data.profiles.push(event); @@ -225,6 +246,37 @@ async fn two_communities_add_edit_restart_keeps_local_images_and_fixed_agent_sig } let path = projected.split("/media/").nth(1).unwrap(); assert_eq!(a.state.data.lock().unwrap().blobs[path], image(2)); + // Animated profiles localize both assets, and retain both projections after + // the source community is removed without authenticating back to it. + let poster = seed(&b, image(6)); + let animation = seed(&b, image(7)); + let descriptor = format!( + "{poster}#buzz-anim={}", + percent_encoding::utf8_percent_encode(&animation, percent_encoding::NON_ALPHANUMERIC) + ); + sync(&state, &a, &keys, &descriptor).await.unwrap(); + let (projected_poster, projected_animation) = + buzz_core_pkg::media::animated_avatar_urls(&picture(&a)).unwrap(); + assert_eq!( + projected_poster, + poster.replace(&b.state.base, &a.state.base) + ); + assert_eq!( + projected_animation, + animation.replace(&b.state.base, &a.state.base) + ); + for (value, bytes) in [ + (projected_poster, image(6)), + (projected_animation, image(7)), + ] { + let path = value.split("/media/").nth(1).unwrap(); + assert_eq!(a.state.data.lock().unwrap().blobs[path], bytes); + } + let source_requests = b.state.data.lock().unwrap().requests.len(); + b.state.data.lock().unwrap().blobs.clear(); + *state.agent_avatar_communities.lock().unwrap() = vec![a.state.base.clone()]; + sync(&state, &a, &keys, &descriptor).await.unwrap(); + assert_eq!(b.state.data.lock().unwrap().requests.len(), source_requests); } #[tokio::test] @@ -550,3 +602,218 @@ async fn removed_source_projection_query_uses_fixed_signer_and_cannot_redirect() assert!(trap.state.data.lock().unwrap().requests.is_empty()); assert_eq!(picture(&b), projected); } + +#[tokio::test] +async fn creation_transfers_only_chosen_bytes_from_owner_to_agent() { + let target = server().await; + let state = state(&[&target]); + let owner = Keys::generate(); + let agent = Keys::generate(); + let poster = seed(&target, image(3)); + let animation = seed(&target, image(5)); + let avatar = format!( + "{poster}#buzz-anim={}", + percent_encoding::utf8_percent_encode(&animation, percent_encoding::NON_ALPHANUMERIC) + ); + let external = server().await; + // A workspace switch after creation captured its target must not retarget + // either the owner download or the agent's profile publication. + *state.relay_url_override.lock().unwrap() = Some(external.state.base.clone()); + assert_eq!( + crate::publish_created_agent_profile( + &state, + &target.state.base, + &owner, + &agent, + "Carl", + Some(&avatar), + None, + None + ) + .await, + None + ); + { + let data = target.state.data.lock().unwrap(); + for (method, signer) in [("read", owner.public_key()), ("upload", agent.public_key())] { + let requests: Vec<_> = data + .requests + .iter() + .filter(|(verb, _)| verb == method) + .collect(); + assert_eq!( + requests.len(), + 2, + "both animated avatar assets must transfer" + ); + assert!(requests.iter().all(|(_, event)| event.pubkey == signer)); + } + let uploads: std::collections::BTreeSet = data + .requests + .iter() + .filter(|(method, _)| method == "upload") + .map(|(_, event)| { + event + .tags + .iter() + .find(|tag| tag.as_slice().first().map(String::as_str) == Some("x")) + .unwrap() + .as_slice()[1] + .clone() + }) + .collect(); + assert_eq!( + uploads, + std::collections::BTreeSet::from([ + media_hash(&url::Url::parse(&poster).unwrap()) + .unwrap() + .to_owned(), + media_hash(&url::Url::parse(&animation).unwrap()) + .unwrap() + .to_owned(), + ]), + "agent possession must cover both distinct avatar assets" + ); + assert_eq!(data.profiles.last().unwrap().pubkey, agent.public_key()); + } + // Arbitrary external URLs never receive the owner's credentials. + transfer_created_avatar( + &state, + &target.state.base, + &owner, + &agent, + Some(&seed(&external, image(4))), + None, + ) + .await + .unwrap(); + assert!(external.state.data.lock().unwrap().requests.is_empty()); +} + +#[derive(Clone, Copy, Debug)] +enum ExplicitAvatarFlow { + TeamImport, + AgentImport, +} + +async fn explicit_avatar_flow( + flow: ExplicitAvatarFlow, + state: &AppState, + relay: &str, + owner: &Keys, + agent: &Keys, + avatar: &str, +) -> Result<(), String> { + match flow { + ExplicitAvatarFlow::TeamImport => { + match crate::publish_imported_team_member_profile( + state, + relay, + owner, + agent, + "Imported Agent", + Some(avatar), + None, + None, + ) + .await + { + Some(error) => Err(error), + None => Ok(()), + } + } + ExplicitAvatarFlow::AgentImport => { + let persona = serde_json::from_value(serde_json::json!({ + "id":"imported", "display_name":"Imported Agent", "system_prompt":"Be helpful", + "created_at":"2026-11-06T00:00:00Z", "updated_at":"2026-11-06T00:00:00Z" + })) + .unwrap(); + match crate::publish_persona_profile( + state, + relay, + owner, + agent, + "Imported Agent", + Some(avatar), + &persona, + None, + ) + .await + { + Some(error) => Err(error), + None => Ok(()), + } + } + } +} + +async fn explicit_avatar_journey(flow: ExplicitAvatarFlow) { + let target = server().await; + target.state.data.lock().unwrap().enforce_possession = true; + let external = server().await; + let state = state(&[&target]); + let owner = Keys::generate(); + let agent = Keys::generate(); + let poster = seed(&target, image(21)); + let animation = seed(&target, image(22)); + let avatar = format!( + "{poster}#buzz-anim={}", + percent_encoding::utf8_percent_encode(&animation, percent_encoding::NON_ALPHANUMERIC) + ); + // Mutable workspace state cannot replace the explicitly captured signer + // or destination while either production path awaits network I/O. + *state.relay_url_override.lock().unwrap() = Some(external.state.base.clone()); + *state.keys.lock().unwrap() = Keys::generate(); + explicit_avatar_flow(flow, &state, &target.state.base, &owner, &agent, &avatar) + .await + .unwrap(); + { + let data = target.state.data.lock().unwrap(); + for (method, signer) in [("read", owner.public_key()), ("upload", agent.public_key())] { + let requests: Vec<_> = data + .requests + .iter() + .filter(|(verb, _)| verb == method) + .collect(); + assert_eq!(requests.len(), 2, "{flow:?}: both assets must transfer"); + assert!(requests.iter().all(|(_, event)| event.pubkey == signer)); + } + assert_eq!(data.uploaders.len(), 2); + assert_eq!(data.profiles.last().unwrap().pubkey, agent.public_key()); + } + let public_avatar = seed(&external, image(23)); + explicit_avatar_flow( + flow, + &state, + &target.state.base, + &owner, + &agent, + &public_avatar, + ) + .await + .unwrap(); + assert!(external.state.data.lock().unwrap().requests.is_empty()); + let before = target.state.data.lock().unwrap().profiles.len(); + target.state.data.lock().unwrap().fail_upload = true; + let error = explicit_avatar_flow(flow, &state, &target.state.base, &owner, &agent, &avatar) + .await + .unwrap_err(); + assert!(error.contains("503"), "{flow:?}: {error}"); + assert_eq!( + target.state.data.lock().unwrap().profiles.len(), + before, + "failed transfer cannot publish a new profile" + ); +} + +#[tokio::test] +async fn team_import_transfers_both_assets_and_reports_failure() { + explicit_avatar_journey(ExplicitAvatarFlow::TeamImport).await; +} + +#[tokio::test] +async fn individual_import_transfers_both_assets_and_reports_failure() { + explicit_avatar_journey(ExplicitAvatarFlow::AgentImport).await; +} + +mod persona_edit; diff --git a/desktop/src-tauri/src/relay/profile_avatar/tests/persona_edit.rs b/desktop/src-tauri/src/relay/profile_avatar/tests/persona_edit.rs new file mode 100644 index 00000000000..d24d60c855c --- /dev/null +++ b/desktop/src-tauri/src/relay/profile_avatar/tests/persona_edit.rs @@ -0,0 +1,287 @@ +use super::*; + +// The real edit writes agent keys and schedules nest work. Confine its process, +// key store, app data and background nest directory on every supported platform. +#[test] +fn persona_avatar_edit_transfers_both_assets_and_reports_failure() { + const CHILD: &str = "BUZZ_PERSONA_AVATAR_TEST_HOME"; + if let Some(home) = std::env::var_os(CHILD) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(edit_in_confined_child(std::path::Path::new(&home))); + return; + } + let home = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "relay::profile_avatar::tests::persona_edit::persona_avatar_edit_transfers_both_assets_and_reports_failure", "--nocapture"]) + .env(CHILD,home.path()).env("HOME",home.path()).env("XDG_DATA_HOME",home.path()) + .env("APPDATA",home.path()).env("LOCALAPPDATA",home.path()) + .env_remove("BUZZ_PRIVATE_KEY").env_remove("BUZZ_AUTH_TAG").env_remove("BUZZ_RELAY_URL").env_remove("BUZZ_NETWORK_TRACE") + .output().unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed"), + "child failed: {stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +async fn edit_in_confined_child(home: &std::path::Path) { + use crate::managed_agents::{ + load_managed_agents, load_personas, save_managed_agents, save_personas, + }; + use tauri::Manager; + crate::managed_agents::storage::NO_KEYCHAIN_FOR_TEST + .store(true, std::sync::atomic::Ordering::SeqCst); + crate::managed_agents::pin_nest_dir_for_test(home.join("nest")); + let target = server().await; + target.state.data.lock().unwrap().enforce_possession = true; + let external = server().await; + let owner = Keys::generate(); + let agent = Keys::generate(); + let state = state(&[&target]); + *state.keys.lock().unwrap() = owner.clone(); + *state.relay_url_override.lock().unwrap() = Some(target.state.base.clone()); + let mut context = tauri::test::mock_context(tauri::test::noop_assets()); + context.config_mut().identifier = home.join("app-data").to_str().unwrap().to_owned(); + let app = tauri::test::mock_builder() + .manage(state) + .build(context) + .unwrap(); + assert!(crate::managed_agents::managed_agents_base_dir(app.handle()) + .unwrap() + .starts_with(home)); + let persona: crate::managed_agents::AgentDefinition = serde_json::from_value(serde_json::json!({ + "id":"edited", "display_name":"Edited Agent", "system_prompt":"Be helpful", + "avatar_url":null,"created_at":"2026-11-06T00:00:00Z", "updated_at":"2026-11-06T00:00:00Z" + })).unwrap(); + let record: crate::managed_agents::ManagedAgentRecord = serde_json::from_value(serde_json::json!({ + "pubkey":agent.public_key().to_hex(),"private_key_nsec":agent.secret_key().to_secret_hex(), + "persona_id":"edited","name":"Edited Agent","relay_url":"", "acp_command":"", "agent_command":"", + "agent_args":[],"mcp_command":"","turn_timeout_seconds":0,"system_prompt":null, + "created_at":"2026-11-06T00:00:00Z", "updated_at":"2026-11-06T00:00:00Z", + "last_started_at":null,"last_stopped_at":null,"last_exit_code":null,"last_error":null + })).unwrap(); + let mut unavailable = record.clone(); + unavailable.pubkey = Keys::generate().public_key().to_hex(); + unavailable.private_key_nsec.clear(); + unavailable.name = "Unavailable key".into(); + save_personas(app.handle(), &[persona]).unwrap(); + save_managed_agents(app.handle(), &[record, unavailable]).unwrap(); + let request = |avatar: &str, name: &str| { + serde_json::from_value(serde_json::json!({ + "id":"edited","displayName":name,"systemPrompt":"Be helpful","avatarUrl":avatar + })) + .unwrap() + }; + let poster = seed(&target, image(31)); + let animation = seed(&target, image(32)); + let avatar = format!( + "{poster}#buzz-anim={}", + percent_encoding::utf8_percent_encode(&animation, percent_encoding::NON_ALPHANUMERIC) + ); + let before_personas = serde_json::to_value(load_personas(app.handle()).unwrap()).unwrap(); + let before_records = serde_json::to_value(load_managed_agents(app.handle()).unwrap()).unwrap(); + let error = crate::update_persona_with( + request(&avatar, "Edited Agent"), + app.handle().clone(), + |_, _, _, _| Ok(()), + ) + .await + .unwrap_err(); + assert!( + error.contains("Unavailable key") && error.contains("Restore") && error.contains("unlink") + ); + assert_eq!( + serde_json::to_value(load_personas(app.handle()).unwrap()).unwrap(), + before_personas + ); + assert_eq!( + serde_json::to_value(load_managed_agents(app.handle()).unwrap()).unwrap(), + before_records + ); + assert!( + target.state.data.lock().unwrap().requests.is_empty(), + "refusal happens before any transfer" + ); + let recovered = Keys::generate(); + let mut records = load_managed_agents(app.handle()).unwrap(); + records[1].private_key_nsec = recovered.secret_key().to_secret_hex(); + records[1].pubkey = recovered.public_key().to_hex(); + save_managed_agents(app.handle(), &records).unwrap(); + crate::update_persona_with( + request(&avatar, "Edited Agent"), + app.handle().clone(), + |_, _, _, _| Ok(()), + ) + .await + .unwrap(); + assert_eq!( + load_personas(app.handle()) + .unwrap() + .into_iter() + .find(|p| p.id == "edited") + .unwrap() + .avatar_url + .as_deref(), + Some(avatar.as_str()) + ); + assert_eq!( + load_managed_agents(app.handle()).unwrap()[0] + .avatar_url + .as_deref(), + Some(avatar.as_str()) + ); + { + let data = target.state.data.lock().unwrap(); + for (method, signer) in [("read", owner.public_key()), ("upload", agent.public_key())] { + let requests: Vec<_> = data + .requests + .iter() + .filter(|(verb, _)| verb == method) + .collect(); + assert_eq!(requests.len(), 4); + assert!(requests.iter().all(|(_, event)| event.pubkey == signer + || (method == "upload" && event.pubkey == recovered.public_key()))); + } + assert!(data + .profiles + .iter() + .any(|event| event.pubkey == agent.public_key())); + } + // An unchanged selection can rename the profile with agent authority only. + app.state::() + .identity_lost + .store(true, std::sync::atomic::Ordering::Release); + let mut records = load_managed_agents(app.handle()).unwrap(); + records[1].private_key_nsec.clear(); + save_managed_agents(app.handle(), &records).unwrap(); + target.state.data.lock().unwrap().requests.clear(); + crate::update_persona_with( + request(&avatar, "Renamed Agent"), + app.handle().clone(), + |_, _, _, _| Ok(()), + ) + .await + .unwrap(); + assert!(target + .state + .data + .lock() + .unwrap() + .requests + .iter() + .all(|(_, event)| event.pubkey == agent.public_key())); + app.state::() + .identity_lost + .store(false, std::sync::atomic::Ordering::Release); + let mut records = load_managed_agents(app.handle()).unwrap(); + records[1].private_key_nsec = recovered.secret_key().to_secret_hex(); + save_managed_agents(app.handle(), &records).unwrap(); + let next_avatar = seed(&target, image(33)); + let before_personas = serde_json::to_value(load_personas(app.handle()).unwrap()).unwrap(); + let before_records = serde_json::to_value(load_managed_agents(app.handle()).unwrap()).unwrap(); + target.state.data.lock().unwrap().fail_upload = true; + let error = crate::update_persona_with( + request(&next_avatar, "Renamed Agent"), + app.handle().clone(), + |_, _, _, _| Ok(()), + ) + .await + .unwrap_err(); + assert!(error.contains("changes were not saved") && error.contains("503")); + assert_eq!( + serde_json::to_value(load_personas(app.handle()).unwrap()).unwrap(), + before_personas + ); + assert_eq!( + serde_json::to_value(load_managed_agents(app.handle()).unwrap()).unwrap(), + before_records + ); + target.state.data.lock().unwrap().fail_upload = false; + for switch_workspace in [false, true] { + let state = app.state::(); + *state.keys.lock().unwrap() = owner.clone(); + *state.relay_url_override.lock().unwrap() = Some(target.state.base.clone()); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + target.state.data.lock().unwrap().pause_read = Some((entered.clone(), release.clone())); + let edit = crate::update_persona_with( + request(&next_avatar, "Renamed Agent"), + app.handle().clone(), + |_, _, _, _| Ok(()), + ); + tokio::pin!(edit); + tokio::select! { _=entered.notified()=>{}, result=&mut edit=>panic!("edit completed before transfer pause: {result:?}") } + if switch_workspace { + *state.relay_url_override.lock().unwrap() = Some(external.state.base.clone()); + *state.keys.lock().unwrap() = Keys::generate(); + } else { + let mut records = load_managed_agents(app.handle()).unwrap(); + records[0].name = "Concurrent edit".into(); + records[0].avatar_url = Some("https://example.org/concurrent.png".into()); + save_managed_agents(app.handle(), &records).unwrap(); + } + target.state.data.lock().unwrap().pause_read = None; + release.notify_one(); + assert!(edit + .await + .unwrap_err() + .contains("changed during avatar transfer")); + assert_eq!( + serde_json::to_value(load_personas(app.handle()).unwrap()).unwrap(), + before_personas + ); + assert_eq!( + load_managed_agents(app.handle()).unwrap()[0].name, + "Concurrent edit" + ); + assert!(external.state.data.lock().unwrap().requests.is_empty()); + } + // A workspace switch in the last synchronous retention window must not + // retarget the captured enqueue authority. + let state = app.state::(); + *state.keys.lock().unwrap() = owner.clone(); + *state.relay_url_override.lock().unwrap() = Some(target.state.base.clone()); + let expected_relay = target.state.base.clone(); + let (_, retained_scope) = crate::update_persona_with( + request(&avatar, "Scoped Rename"), + app.handle().clone(), + move |_, state, _, scope| { + *state.relay_url_override.lock().unwrap() = Some("https://switched.example".into()); + *state.keys.lock().unwrap() = Keys::generate(); + let scope = scope.unwrap(); + assert_eq!(scope.relay_url, expected_relay); + Ok(scope.clone()) + }, + ) + .await + .unwrap(); + assert_eq!(retained_scope.owner_keys.public_key(), owner.public_key()); + let state = app.state::(); + *state.keys.lock().unwrap() = owner; + *state.relay_url_override.lock().unwrap() = Some(target.state.base.clone()); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + target.state.data.lock().unwrap().pause_read = Some((entered.clone(), release.clone())); + let edit = crate::update_persona_with( + request(&next_avatar, "Runtime-safe edit"), + app.handle().clone(), + |_, _, _, _| Ok(()), + ); + tokio::pin!(edit); + tokio::select! { _=entered.notified()=>{}, result=&mut edit=>panic!("edit completed before pause: {result:?}") } + let mut records = load_managed_agents(app.handle()).unwrap(); + records[0].runtime_pid = Some(12345); + records[1].last_started_at = Some("2026-11-06T20:00:00Z".into()); + save_managed_agents(app.handle(), &records).unwrap(); + target.state.data.lock().unwrap().pause_read = None; + release.notify_one(); + edit.await.unwrap(); + assert_eq!( + load_managed_agents(app.handle()).unwrap()[0].runtime_pid, + Some(12345) + ); +} diff --git a/desktop/src/features/agents/AGENTS.md b/desktop/src/features/agents/AGENTS.md index a9e6a73dd14..a4be78584fa 100644 --- a/desktop/src/features/agents/AGENTS.md +++ b/desktop/src/features/agents/AGENTS.md @@ -435,6 +435,20 @@ keeps the source URL, not the target projection. Transfer or kind:0 rejection leaves the previous profile intact so normal reconciliation can retry. The Agent-managed profiles opt-out still disables automatic reconciliation. +Creation explicitly transfers a chosen target-local avatar: the captured creating +owner downloads only that image, then the new agent signs an upload of the +verified bytes before its profile is published. The relay never lends an +owner's upload proof to the agent. Automatic reconciliation has no owner signer. +The captured community stays pinned through download, upload and profile publication, even if the active +workspace changes while the request is awaiting I/O. + +Changed persona avatars transfer before local saves; a failed transfer returns +an error and leaves the old local avatar intact. A concurrent settings/workspace +change fences the save and requires retry. Team and individual snapshot imports +transfer target-local poster and animation bytes before profile publication and +report errors in the import result. All explicit flows capture owner/community +before awaits. Unchanged-avatar reconciliation never borrows owner authority. + The configured community origin set is refreshed through narrow workspace IPC before startup restore and when inactive communities change, without resetting active community state. After source removal, the shared writer may reuse an @@ -452,3 +466,9 @@ matches the code is worse than no rule; a new pattern that isn't written down here will be broken by the next agent that never learns it existed. Reviewers: treat a config-behavior diff without a matching AGENTS.md diff (or an explicit "no rules changed" note) as incomplete. + +Changed-avatar edits refuse the entire save before any transfer when a linked +instance has no usable signing key. The error names affected instances; restore +their keys, or remove/unlink those instances, then retry. The previous persona +and instance avatars remain unchanged, so the same requested avatar is retryable. +Keys may be permanently lost; waiting alone does not repair a lost key. diff --git a/docs/operator-private-media.md b/docs/operator-private-media.md new file mode 100644 index 00000000000..dc37d37d52b --- /dev/null +++ b/docs/operator-private-media.md @@ -0,0 +1,191 @@ +# Private-channel media revocation + +`BUZZ_MEDIA_READ_MODE` is a startup setting: `off` (default), `shadow`, or +`enforce`. Unknown values fail startup. Channel revocation is effective only +in `enforce`. Blossom authentication, tenant binding, and community admission +remain mandatory in every mode. + +## Read policy + +Every GET and HEAD, including each Range request, authorizes the parent hash +before returning blob bytes, thumbnail bytes, existence, size, or range errors. +A tenant sidecar lookup precedes the new database lookup. Authorization uses +a fresh primary-database repeatable-read snapshot, without membership caches +or replicas. A removal committed before that snapshot excludes the member. +A request authorized before removal may finish afterward. + +A file is readable through any current readable reference. Shared bytes may +therefore remain readable through another open channel or a DM the person +still belongs to. Channel owners/admins have no blanket exception to private +channel membership. Ordinary REQ event visibility, recipient-gated kinds and +encrypted engrams retain their restrictions. Gift-wrap ciphertext itself +cannot supply hidden attachment references. + +Counted references are imeta `url` and `image` fields, link-preview snapshot image/favicon fields, profile `picture` (or `image` when +`picture` is absent) and `banner` fields, group metadata `picture` tags, and emoji tags on emoji lists +and sets and per-emoji entries (kind 30180). Persona definitions (kind 30175) +count `avatar_url`, with the same owner-only/shared visibility as their events. +Root-relative paths and the tenant's HTTP/HTTPS media bases count. +Animated profile descriptors count their poster and decoded animation URL +independently, with the same tenant check on each. The indexed profile lookup +expects hash characters to remain literal, as in the desktop descriptor. +Republish over-encoded hashes such as `%61` or JSON Unicode-escaped hash characters such as `\u0061` using canonical URLs. Ordinary Unicode escapes in profile names are supported; the URL hash must remain plain ASCII. A pasted URL in free text alone grants nothing; messages must publish imeta. +Deleted events/channels grant nothing. Only the newest live edit supplies a +message's attachments; its original must remain live. Original link-preview +snapshots remain readable through an edit unless the original or any live edit carries the exact +`["link-preview","none"]` marker; later edits do not undo suppression; this does not retain removed attachments. +Superseded profiles +are excluded. Current community icons grant community-wide access. + +The read path probes the existing tags GIN index for imeta hashes and exact +local URL aliases. It scans content only for current kind-0 profiles and kind-30175 persona avatar fields in the +community, never all message bodies. LLVM/JIT is disabled only in this short authorization transaction: compiling +partitioned permission probes costs seconds without amortizing that cost. +Other database work retains its configured JIT policy. A two-second statement deadline and a +1,024-candidate budget per probe and a 16 MiB aggregate work budget bound each lookup. An accessible candidate can succeed +before an overflow refusal. Candidates stream one at a time rather than allocating an entire page of message bodies. Budget exhaustion or a database failure returns +503; an inaccessible/missing reference returns 404. NIP-FI strict mode keeps +its existing authorization-unavailable response contract. + +## Uploads and publication + +After the upload pipeline verifies the supplied bytes, the relay records +`media_uploaders` through community write admission. This is independent of +optional audit/upload moderation records. An unposted file is previewable by +its uploader only. Real reuploads may prove possession for another uploader. +Possession authorizes publication; a known URL/hash alone does not. A registered +agent must hold its own upload proof or readable reference; owner proof is never inherited. + +The shared publication check runs before common ingestion dispatch (WS and +HTTP, including metadata commands) and before direct workflow persistence. +It uses the same reference extractor as reads, plus structured command URLs. +A durable `media_publications` marker consumes preview authority before +publication dispatch and survives source deletion. A later rejected operation +may conservatively consume a preview; reupload still proves publication +rights. Reuploads also record a boundary when a legacy reference already exists, so later purging that reference cannot revive previews. The marker grants no download access. Both tables are purged during +community deletion, and every writer uses admission. + +Deployment-admin feedback attachment reads retain their independent admin +and feedback-row provenance checks. Feedback does not grant community-wide +media access through the ordinary media endpoint. + +## Rollout and recovery + +1. Before any pod enters shadow, [build and verify the edit-target index](private-media-index-deployment.md). + Small databases may auto-migrate through 0086: a missing index builds + within a one-second lock and five-second statement budget, even with data. + Large deployments disable auto-migration and prebuild concurrently per + partition before applying 0086 or desired-state schema. A budget failure + leaves migration retryable; a valid prebuild skips the blocking build. + Bypassing the schema step can cause lookup timeouts/503 in enforce. Restore + off, finish index construction/verification, then begin shadow. + The migrations add plain tenant tables and an index, with no SQL functions, + triggers, foreign keys or data rewrite. +2. Deploy the new binary on every writer with mode `off`. +3. Restart all writers in `shadow`; inspect + `buzz_media_read_access_total{mode="shadow",reason="no_ref"}`, + `reason="not_member"`, and `reason="lookup_error"`, plus + `buzz_media_publication_access_total`. Exercise avatars, icons, emoji, + uploads, private messages and old attachments during a staging soak. +4. Repair unsupported/historical URLs or republish them through an authorized + uploader before restarting every serving pod in `enforce`. + +**Enforce blocker: SDK/CLI attachment edits remain unsupported.** +`buzz_sdk::build_edit` and `buzz messages edit` currently omit attachment imeta. +Editing an attachment message through these clients can make its file unreadable, +including to its sender. This review item is deferred, not fixed. Enforce stays blocked unless the operator can rule out SDK/CLI attachment edits +in the community. The agent harness edit producer is unverified and remains +inside this blocker until its metadata behavior is traced and tested. To remove +the blocker, update both SDK/CLI paths to preserve the +latest effective attachment metadata across successive edits without restoring +files removed by an earlier edit. Require an upload → successive edits → GET/HEAD +regression proving retained files stay readable and removed files stay denied. +Keeping a URL in the edited body does not preserve access. + +There is no grant backfill: existing stored references are evaluated directly, +and signed event tags are never rewritten. Legacy unreferenced/orphan blobs +have no upload proof and are denied in enforce mode; their owner must reupload. +Pre-enforcement references remain authoritative, including a preexisting avatar +that points to the same bytes as a private attachment. Custom historical media +hosts beyond the tenant's current authority require explicit repair. + +Roll back policy by restarting with `off`; retain the additive schema and +publication markers. If rolling back to a pre-0086 binary, keep auto-migration +disabled and suspend community deletion: its exact-set deletion catalog does +not recognize the two added media tables, so deletion fails closed. Resume +deletion after deploying a binary whose deletion catalog includes them. Mixed off/enforce serving pods cannot provide revocation. +Shadow/off do not close the original exposure. Shadow authorization lookup failures, including sidecar failures, are counted +without refusing reads. The serving path can still fail if storage cannot supply +bytes. Enforcement fails closed. + +Successful media responses, authorized range errors and media denials use +`private, no-store` in all modes, including off. Off preserves the previous +authorization behavior; cache headers intentionally change before enforce to +avoid creating more year-long immutable entries during the preparatory rollout. New +responses therefore increase repeat-download bandwidth. The desktop proxy +passes this policy to the WebView; its local file cache and the mobile client's +local copies can still display previously fetched bytes. Old immutable HTTP +cache entries, files already downloaded, screenshots and copied bytes cannot +be recalled. The guarantee applies to requests reaching the relay; migrating +existing client caches is a separate client rollout concern. + +Any authorized member may deliberately republish or reupload bytes into a +readable channel or avatar. Revoking one channel cannot revoke that independent +share. This policy does not prevent authorized recipients from copying data. + +Media authorization uses a shared semaphore across database-handle clones, +limited to one quarter of the writer pool, capped at eight lookups. Saturation +waits up to 500 ms before returning authorization-unavailable, counted with +`reason="capacity"`, before pool acquisition. Every Range +request reauthorizes; all events, including emoji lists and sets, have a 512-reference cap, and checks run +sequentially within that connection bound. Missing sidecars reject publication +in enforce mode; shadow counts failures and accepts the write, including stale +avatar metadata edits and upload-proof/publication-marker errors. + +Eight slots provide an upper bound of `8 / mean_lookup_seconds` lookups per +second per pod before queueing (for example, 20 ms means about 400/s; 100 ms +means about 80/s). A smaller writer pool reduces that bound proportionally. +These are capacity calculations, not measured throughput targets. Measure full +request latency and concurrent image/avatar bursts, including all probes, +sidecars and pool acquisition. Keep sustained traffic below the measured bound; +the 500 ms queue absorbs bursts but cannot increase throughput. Each lookup +uses a writer connection and several statements; no-store repeats that work on +every view. The synthetic query-plan times exclude storage/network/pool costs. + +During staging soak, abort enforcement if writer authorization acquire p95 +exceeds 50 ms for five minutes, event-write acquire p95 doubles its pre-rollout +baseline, or media lookup_error/capacity refusal exceeds 0.1% of requests for +five minutes. Investigate query latency, pool utilization and storage errors +before retrying. Shadow failures must be repaired before enforce, since a failed +recording does not establish possession. Uploads made in off mode record no +proof: uploads not yet posted when enforce starts require reuploading. + +Before enforce, every client that uploads avatars with one key and publishes +profiles with another must transfer the actual poster/animation bytes to an +upload signed by the profile key. Owner relationships never inherit possession. +Validate this in each supported client during shadow; upgrading only this +repository's desktop does not migrate other clients. This includes the new +desktop/web app in `block/buzz-app`, mobile, and CLI/plugin integrations that +publish agent profiles: verify their signer and upload flow before enabling +enforce for their users. + +The legacy desktop transfers both poster and animation bytes for changed +persona avatars before saving the edit; transfer failures leave the old local +avatar intact. Team and individual agent snapshot imports also transfer selected +target-local avatars before publishing each new agent profile. Imports report +transfer failures in their profile-sync results. Validate all three workflows +against enforce mode before rollout. Unchanged-avatar reconciliation never +borrows the owner's signer. + +New managed-agent creation explicitly downloads the chosen target-local avatar +with the captured owner's signer, verifies its image bytes and hash, then uploads +those bytes with the new agent's signer. The agent holds its own possession +proof. The captured community stays pinned through profile publication, even +if the owner changes workspaces during transfer. Automatic reconciliation never +borrows the owner's proof or signer. + +Changed-avatar edits refuse the entire save before any transfer when a linked +instance has no usable signing key. The error names affected instances; restore +their keys, or remove/unlink those instances, then retry. The previous persona +and instance avatars remain unchanged, so the same requested avatar is retryable. +Keys may be permanently lost; waiting alone does not repair a lost key. diff --git a/docs/private-media-index-deployment.md b/docs/private-media-index-deployment.md new file mode 100644 index 00000000000..0edad57c11a --- /dev/null +++ b/docs/private-media-index-deployment.md @@ -0,0 +1,71 @@ +# Media edit target index deployment + +Before enabling shadow, build and verify the partial btree on the normalized +first valid edit target. Small databases may use migration 0086 with auto-migration; +large deployments with auto-migration off prebuild concurrently before applying +0086 or desired-state schema. Signed event tags are never rewritten. +The immutable built-in JSONPath expression preserves tag order and hex casing. +No SQL function is added. + +For a concurrent prebuild, use the same approach as [thread windows](thread-window-deployment.md). +Run from this checkout, outside a transaction: + +```sh +psql "$DATABASE_URL" -X -v ON_ERROR_STOP=1 -f scripts/maintenance/prebuild-media-edit-index.sql +``` + +The script creates an `ON ONLY` parent, builds each immediate events partition +index `CONCURRENTLY` with PostgreSQL-generated names (matching pgschema), then +attaches it. pgschema cannot build a partitioned parent concurrently, so this +prebuild is required for large existing databases under either schema deployment +path. The final bounded migration guard checks +readiness, validity, access method, key order, predicate and exact expression. +A valid parent proves every existing partition has an attached matching index. +New monthly partitions inherit the index through ordinary `PARTITION OF` creation; +the PostgreSQL regression creates a future partition and verifies attachment. +Keep partition management paused while running the prebuild; if a new partition +appears before verification, rerun the idempotent script and verify again. + +A failed concurrent child build leaves an invalid index. Run only one prebuild +at a time; ensure an earlier build has ended before retrying. A retry drops +only invalid, unattached child indexes whose definitions match the parent, +using `DROP INDEX CONCURRENTLY` outside a transaction. It then rebuilds them +concurrently with the same generated names expected by desired-state tooling. +Concurrent child creation and cleanup disable `lock_timeout` so older +transactions can finish; each statement retains its 30-minute timeout. Parent +creation and partition attachment retain the one-second lock timeout. +Valid or attached indexes are retained. Wrong definitions fail attachment or +migration verification and need operator correction before proceeding. Large +deployments must retain the concurrent build procedure. + +Migration 0086 builds a missing index inline on empty or populated databases, +with a one-second lock timeout and five-second statement timeout. Contention or +a build exceeding those budgets aborts the migration without recording 0086; +retry after the contention clears, or disable auto-migration and prebuild +concurrently for a large deployment. A prebuilt index skips CREATE entirely, +then passes the same exact catalog validation without an index-build lock. +With auto-migration disabled, operators must apply and verify 0086 before deploying +this binary, even in off mode. Bypassing that ordering and enabling enforce can +cause lookup timeouts and authorization-unavailable responses; restore off and +complete prebuild/verification before shadow. Do not use that bypass as a soak. + +This is an additive expand/deploy change: existing relay reads and writes ignore +the new index, which adds only normal index maintenance. No row backfill, trigger, +function or foreign key is added. New and old binaries may serve against the +expanded schema. Keep auto-migration disabled on old binaries/restarts/rollback +because sqlx rejects unknown migration versions. Do not run community deletion +between 0086's table expansion and the new binary's deletion-catalog deployment. + +Migration 0086 builds the index and expands the media ownership tables in one +transaction, with the fallible index build first. An index timeout leaves the previously applied migration +version unchanged, so an older auto-migrating binary can still restart. After +successful expansion, the usual rollback requirement above still applies. + +The earlier PR snapshot split this unmerged expansion across migrations 0084 +and 0085. They were consolidated during review, then renumbered to 0086 after the base added +its unrelated 0084 default-channel recovery and 0085 community-policy migrations. +Both base migrations are unchanged. +A disposable database that already applied an earlier PR snapshot must be +recreated before testing this revision; do not rewrite its sqlx migration rows +or checksums. Keep any ongoing old-snapshot trial pinned to its matching binary. +Merged migration history remains unchanged. diff --git a/migrations/0086_private_media.sql b/migrations/0086_private_media.sql new file mode 100644 index 00000000000..c8372bc5a03 --- /dev/null +++ b/migrations/0086_private_media.sql @@ -0,0 +1,44 @@ +-- Atomic expand: a bounded index failure records no new migration version. +-- Build within bounded startup budgets; large deployments prebuild concurrently. +-- See docs/private-media-index-deployment.md. +SET LOCAL lock_timeout = '1s'; +SET LOCAL statement_timeout = '5s'; +DO $migration$ +BEGIN + IF to_regclass('public.idx_events_media_edit_target') IS NULL THEN + CREATE INDEX idx_events_media_edit_target ON public.events + (community_id, channel_id, (decode(lower(jsonb_path_query_first(tags, 'strict $[*] ? (@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]', '{}'::jsonb, true) #>> '{}'), 'hex')), created_at DESC, id DESC) + WHERE kind = 40003 AND deleted_at IS NULL; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_index i + WHERE i.indexrelid = 'public.idx_events_media_edit_target'::regclass + AND i.indrelid = 'public.events'::regclass + AND i.indisvalid AND i.indisready AND i.indislive + AND NOT i.indisunique AND i.indnkeyatts = 5 AND i.indnatts = 5 + AND pg_get_indexdef(i.indexrelid,1,true) = 'community_id' + AND pg_get_indexdef(i.indexrelid,2,true) = 'channel_id' + AND pg_get_indexdef(i.indexrelid,4,true) = 'created_at' + AND pg_get_indexdef(i.indexrelid,5,true) = 'id' + AND i.indoption::text = '0 0 0 3 3' + AND pg_get_expr(i.indexprs, i.indrelid) = $expr$decode(lower((jsonb_path_query_first(tags, 'strict $[*]?(@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]'::jsonpath, '{}'::jsonb, true) #>> '{}'::text[])), 'hex'::text)$expr$ + AND pg_get_expr(i.indpred, i.indrelid) = '((kind = 40003) AND (deleted_at IS NULL))' + AND (SELECT amname FROM pg_am WHERE oid=(SELECT relam FROM pg_class WHERE oid=i.indexrelid))='btree' + ) THEN + RAISE EXCEPTION 'idx_events_media_edit_target invalid or wrong definition; see docs/private-media-index-deployment.md'; + END IF; +END $migration$; + +-- Independent of optional upload moderation records. All writers use admission. +SET LOCAL lock_timeout = '5s'; +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL CHECK (length(pubkey) = 32), + PRIMARY KEY (community_id, sha256, pubkey) +); +CREATE TABLE media_publications ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (community_id, sha256) +); diff --git a/schema/schema.sql b/schema/schema.sql index 84d06d7b402..f804fb0bc4a 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -220,3 +220,6 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto; \i tables/public/community_auth_upgrades.sql \i seeds/community_auth_upgrades.sql + +\i tables/public/media_uploaders.sql +\i tables/public/media_publications.sql diff --git a/schema/tables/public/events.sql b/schema/tables/public/events.sql index 10c1adc19e9..6056fc556bd 100644 --- a/schema/tables/public/events.sql +++ b/schema/tables/public/events.sql @@ -69,3 +69,8 @@ CREATE INDEX idx_events_search_tsv ON events USING GIN (search_tsv); -- e-tag containment (`tags @> '[["e",""]]'`) for the aux closure and #e -- reads. Mirrors migrations/0004; jsonb_path_ops supports exactly @>. CREATE INDEX idx_events_tags_gin ON events USING GIN (tags jsonb_path_ops); + +-- First valid edit target, matching the relay Rust rule and indexed probes. +CREATE INDEX idx_events_media_edit_target ON events + (community_id, channel_id, (decode(lower(jsonb_path_query_first(tags, 'strict $[*] ? (@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]', '{}'::jsonb, true) #>> '{}'), 'hex')), created_at DESC, id DESC) + WHERE kind = 40003 AND deleted_at IS NULL; diff --git a/schema/tables/public/media_publications.sql b/schema/tables/public/media_publications.sql new file mode 100644 index 00000000000..a6544d2f73b --- /dev/null +++ b/schema/tables/public/media_publications.sql @@ -0,0 +1,5 @@ +CREATE TABLE media_publications ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (community_id, sha256) +); diff --git a/schema/tables/public/media_uploaders.sql b/schema/tables/public/media_uploaders.sql new file mode 100644 index 00000000000..82b7fd765d0 --- /dev/null +++ b/schema/tables/public/media_uploaders.sql @@ -0,0 +1,6 @@ +CREATE TABLE media_uploaders ( + community_id UUID NOT NULL, + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + pubkey BYTEA NOT NULL CHECK (length(pubkey) = 32), + PRIMARY KEY (community_id, sha256, pubkey) +); diff --git a/scripts/maintenance/prebuild-media-edit-index.sql b/scripts/maintenance/prebuild-media-edit-index.sql new file mode 100644 index 00000000000..f762baeb572 --- /dev/null +++ b/scripts/maintenance/prebuild-media-edit-index.sql @@ -0,0 +1,55 @@ +-- Run in psql, outside a transaction, before deploying migration 0086. +-- Let PostgreSQL generate child names, matching desired-state partition indexes. +\set ON_ERROR_STOP on +SET lock_timeout = '1s'; +SET statement_timeout = '30min'; +SELECT $ddl$CREATE INDEX idx_events_media_edit_target ON ONLY public.events + (community_id,channel_id, (decode(lower(jsonb_path_query_first(tags, 'strict $[*] ? (@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]', '{}'::jsonb, true) #>> '{}'), 'hex')),created_at DESC,id DESC) + WHERE kind=40003 AND deleted_at IS NULL$ddl$ +WHERE to_regclass('public.idx_events_media_edit_target') IS NULL +\gexec +-- Concurrent operations wait for older transactions/snapshots. Keep the overall +-- statement budget, but do not cancel those waits after one second. +SET lock_timeout = 0; +-- A failed generated child keeps its name. Remove only unusable, unattached, +-- definition-matching builds so retries reuse desired-state child names. +SELECT format('DROP INDEX CONCURRENTLY %s', i.indexrelid::regclass) +FROM pg_index i JOIN pg_inherits h ON h.inhrelid=i.indrelid +WHERE h.inhparent='public.events'::regclass AND NOT i.indisvalid AND NOT i.indisunique + AND NOT EXISTS (SELECT 1 FROM pg_inherits attached WHERE attached.inhrelid=i.indexrelid) + AND substring(pg_get_indexdef(i.indexrelid) FROM ' USING .*') = + substring(pg_get_indexdef('public.idx_events_media_edit_target'::regclass) FROM ' USING .*') +ORDER BY i.indexrelid +\gexec +SELECT format($ddl$CREATE INDEX CONCURRENTLY ON %s + (community_id,channel_id, (decode(lower(jsonb_path_query_first(tags, 'strict $[*] ? (@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]', '{}'::jsonb, true) #>> '{}'), 'hex')),created_at DESC,id DESC) + WHERE kind=40003 AND deleted_at IS NULL$ddl$, + c.oid::regclass) +FROM pg_inherits h JOIN pg_class c ON c.oid=h.inhrelid +WHERE h.inhparent='public.events'::regclass + AND NOT EXISTS (SELECT 1 FROM pg_inherits ix JOIN pg_index i ON i.indexrelid=ix.inhrelid + WHERE ix.inhparent='public.idx_events_media_edit_target'::regclass AND i.indrelid=c.oid) + AND NOT EXISTS (SELECT 1 FROM pg_index i WHERE i.indrelid=c.oid + AND i.indisvalid AND i.indisready AND i.indislive + AND substring(pg_get_indexdef(i.indexrelid) FROM ' USING .*') = + substring(pg_get_indexdef('public.idx_events_media_edit_target'::regclass) FROM ' USING .*')) +ORDER BY c.oid +\gexec +-- Parent metadata operations must still acquire their locks promptly. +SET lock_timeout = '1s'; +SELECT format('ALTER INDEX public.idx_events_media_edit_target ATTACH PARTITION %s', child.indexrelid::regclass) +FROM pg_inherits h JOIN pg_class c ON c.oid=h.inhrelid +CROSS JOIN LATERAL (SELECT i.indexrelid FROM pg_index i WHERE i.indrelid=c.oid + AND i.indisvalid AND i.indisready AND i.indislive + AND substring(pg_get_indexdef(i.indexrelid) FROM ' USING .*') = + substring(pg_get_indexdef('public.idx_events_media_edit_target'::regclass) FROM ' USING .*') + ORDER BY i.indexrelid LIMIT 1) child +WHERE h.inhparent='public.events'::regclass + AND NOT EXISTS (SELECT 1 FROM pg_inherits ix JOIN pg_index i ON i.indexrelid=ix.inhrelid + WHERE ix.inhparent='public.idx_events_media_edit_target'::regclass AND i.indrelid=c.oid) +ORDER BY c.oid +\gexec +-- Validate using the exact same bounded guard as startup (no table build here). +BEGIN; +\ir verify-media-edit-index.sql +COMMIT; diff --git a/scripts/maintenance/verify-media-edit-index.sql b/scripts/maintenance/verify-media-edit-index.sql new file mode 100644 index 00000000000..f4c94ff14ef --- /dev/null +++ b/scripts/maintenance/verify-media-edit-index.sql @@ -0,0 +1,29 @@ +-- Build within bounded startup budgets; large deployments prebuild concurrently. +-- See docs/private-media-index-deployment.md. +SET LOCAL lock_timeout = '1s'; +SET LOCAL statement_timeout = '5s'; +DO $migration$ +BEGIN + IF to_regclass('public.idx_events_media_edit_target') IS NULL THEN + CREATE INDEX idx_events_media_edit_target ON public.events + (community_id, channel_id, (decode(lower(jsonb_path_query_first(tags, 'strict $[*] ? (@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]', '{}'::jsonb, true) #>> '{}'), 'hex')), created_at DESC, id DESC) + WHERE kind = 40003 AND deleted_at IS NULL; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_index i + WHERE i.indexrelid = 'public.idx_events_media_edit_target'::regclass + AND i.indrelid = 'public.events'::regclass + AND i.indisvalid AND i.indisready AND i.indislive + AND NOT i.indisunique AND i.indnkeyatts = 5 AND i.indnatts = 5 + AND pg_get_indexdef(i.indexrelid,1,true) = 'community_id' + AND pg_get_indexdef(i.indexrelid,2,true) = 'channel_id' + AND pg_get_indexdef(i.indexrelid,4,true) = 'created_at' + AND pg_get_indexdef(i.indexrelid,5,true) = 'id' + AND i.indoption::text = '0 0 0 3 3' + AND pg_get_expr(i.indexprs, i.indrelid) = $expr$decode(lower((jsonb_path_query_first(tags, 'strict $[*]?(@[0] == "e" && @[1] like_regex "^[0-9A-Fa-f]{64}$")[1]'::jsonpath, '{}'::jsonb, true) #>> '{}'::text[])), 'hex'::text)$expr$ + AND pg_get_expr(i.indpred, i.indrelid) = '((kind = 40003) AND (deleted_at IS NULL))' + AND (SELECT amname FROM pg_am WHERE oid=(SELECT relam FROM pg_class WHERE oid=i.indexrelid))='btree' + ) THEN + RAISE EXCEPTION 'idx_events_media_edit_target invalid or wrong definition; see docs/private-media-index-deployment.md'; + END IF; +END $migration$;