From 349be4f164b36400ef5fb0039f954586cfea8db0 Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 10 Nov 2026 13:52:10 -0500 Subject: [PATCH 1/2] feat(integrations): ingest verified provider activity into Buzz threads Signed-off-by: Tee8z --- .github/workflows/voltage-agent-images.yml | 25 +- Cargo.lock | 23 + Cargo.toml | 1 + crates/buzz-core/src/integration.rs | 210 ++++++ crates/buzz-core/src/kind.rs | 14 + crates/buzz-core/src/lib.rs | 2 + crates/buzz-ingest/Cargo.toml | 29 + crates/buzz-ingest/README.md | 105 +++ crates/buzz-ingest/src/config.rs | 122 ++++ crates/buzz-ingest/src/http.rs | 286 ++++++++ crates/buzz-ingest/src/main.rs | 66 ++ crates/buzz-ingest/src/normalize.rs | 471 ++++++++++++ crates/buzz-ingest/src/plan.rs | 273 +++++++ crates/buzz-ingest/src/relay.rs | 111 +++ crates/buzz-ingest/src/tests.rs | 689 ++++++++++++++++++ crates/buzz-relay/src/handlers/ingest.rs | 4 +- crates/buzz-workflow/src/integration_tests.rs | 101 +++ crates/buzz-workflow/src/lib.rs | 39 +- crates/buzz-workflow/src/schema.rs | 41 ++ docs/nips/NIP-IG.md | 75 ++ mobile/lib/shared/relay/nostr_models.dart | 1 + voltage/images/README.md | 7 +- voltage/images/buzz-ingest/Dockerfile | 16 + .../buzz-ingest/Dockerfile.dockerignore | 82 +++ voltage/images/buzz-ingest/test-image.sh | 31 + 25 files changed, 2815 insertions(+), 9 deletions(-) create mode 100644 crates/buzz-core/src/integration.rs create mode 100644 crates/buzz-ingest/Cargo.toml create mode 100644 crates/buzz-ingest/README.md create mode 100644 crates/buzz-ingest/src/config.rs create mode 100644 crates/buzz-ingest/src/http.rs create mode 100644 crates/buzz-ingest/src/main.rs create mode 100644 crates/buzz-ingest/src/normalize.rs create mode 100644 crates/buzz-ingest/src/plan.rs create mode 100644 crates/buzz-ingest/src/relay.rs create mode 100644 crates/buzz-ingest/src/tests.rs create mode 100644 crates/buzz-workflow/src/integration_tests.rs create mode 100644 docs/nips/NIP-IG.md create mode 100644 voltage/images/buzz-ingest/Dockerfile create mode 100644 voltage/images/buzz-ingest/Dockerfile.dockerignore create mode 100755 voltage/images/buzz-ingest/test-image.sh diff --git a/.github/workflows/voltage-agent-images.yml b/.github/workflows/voltage-agent-images.yml index c482d3e2c12..6021ab8b436 100644 --- a/.github/workflows/voltage-agent-images.yml +++ b/.github/workflows/voltage-agent-images.yml @@ -9,9 +9,10 @@ name: Voltage images # buzz-agent voltage/images/buzz-agent/Dockerfile → voltage-buzz-agent-staging # buzz-sandbox-reaper voltage/images/buzz-sandbox-reaper/Dockerfile → voltage-buzz-sandbox-reaper-staging # buzz-relay the upstream root Dockerfile (target runtime) → buzz-relay +# buzz-ingest voltage/images/buzz-ingest/Dockerfile → buzz-ingest # # The changes job selects only the images whose inputs changed. Manual runs -# (workflow_dispatch) build all three, or only buzz-agent with agent_only. +# (workflow_dispatch) build all images, or only buzz-agent with agent_only. # # Trust model: this repository and its logs are public. Pull requests run # without AWS access. Only the publish and merge jobs request an OIDC token, @@ -131,6 +132,15 @@ jobs: - 'examples/**' - 'voltage/images/buzz-sandbox-reaper/**' - '.github/workflows/voltage-agent-images.yml' + ingest: + - '.cargo/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - 'crates/**' + - 'examples/**' + - 'voltage/images/buzz-ingest/**' + - '.github/workflows/voltage-agent-images.yml' relay: - 'Dockerfile' - '.dockerignore' @@ -157,16 +167,19 @@ jobs: AGENT: ${{ steps.filter.outputs.agent }} REAPER: ${{ steps.filter.outputs.reaper }} RELAY: ${{ steps.filter.outputs.relay }} + INGEST: ${{ steps.filter.outputs.ingest }} run: | set -euo pipefail selected=() if [ "$ALL" = true ] || [ "$AGENT_ONLY" = true ] || [ "$AGENT" = true ]; then selected+=(buzz-agent); fi if [ "$ALL" = true ] || [ "$REAPER" = true ]; then selected+=(buzz-sandbox-reaper); fi if [ "$ALL" = true ] || [ "$RELAY" = true ]; then selected+=(buzz-relay); fi + if [ "$ALL" = true ] || [ "$INGEST" = true ]; then selected+=(buzz-ingest); fi images='{ "buzz-agent": {"file": "voltage/images/buzz-agent/Dockerfile", "target": "", "repository": "voltage-buzz-agent-staging"}, "buzz-sandbox-reaper": {"file": "voltage/images/buzz-sandbox-reaper/Dockerfile", "target": "", "repository": "voltage-buzz-sandbox-reaper-staging"}, - "buzz-relay": {"file": "Dockerfile", "target": "runtime", "repository": "buzz-relay"} + "buzz-relay": {"file": "Dockerfile", "target": "runtime", "repository": "buzz-relay"}, + "buzz-ingest": {"file": "voltage/images/buzz-ingest/Dockerfile", "target": "", "repository": "buzz-ingest"} }' platforms='[{"arch": "amd64", "runner": "ubuntu-latest"}, {"arch": "arm64", "runner": "ubuntu-24.04-arm"}]' build=$(jq -cn --argjson images "$images" --argjson platforms "$platforms" \ @@ -208,7 +221,7 @@ jobs: run: | set -euo pipefail sh voltage/images/buzz-agent/select-skills.sh --none "$RUNNER_TEMP/engineering-skills" - for file in voltage/images/buzz-agent/Dockerfile voltage/images/buzz-sandbox-reaper/Dockerfile Dockerfile; do + for file in voltage/images/buzz-agent/Dockerfile voltage/images/buzz-sandbox-reaper/Dockerfile voltage/images/buzz-ingest/Dockerfile Dockerfile; do echo "==> $file" docker buildx build --check --build-context engineering-skills="$RUNNER_TEMP/engineering-skills" -f "$file" . done @@ -261,6 +274,9 @@ jobs: docker run --rm "$IMAGE" --help docker run --rm "$IMAGE" reconcile --help docker run --rm "$IMAGE" serve --help | grep -- '--mention-launch' + - name: Ingest smoke test + if: matrix.image == 'buzz-ingest' + run: bash voltage/images/buzz-ingest/test-image.sh "$IMAGE" - name: Relay smoke test if: matrix.image == 'buzz-relay' run: bash voltage/images/test-relay-image.sh "$IMAGE" @@ -414,6 +430,9 @@ jobs: run: | docker run --rm "$IMAGE" --help docker run --rm "$IMAGE" serve --help | grep -- '--mention-launch' + - name: Ingest smoke test + if: matrix.image == 'buzz-ingest' + run: bash voltage/images/buzz-ingest/test-image.sh "$IMAGE" - name: Relay smoke test if: matrix.image == 'buzz-relay' run: bash voltage/images/test-relay-image.sh "$IMAGE" diff --git a/Cargo.lock b/Cargo.lock index b7e7e325c85..d1270c3a11e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1812,6 +1812,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "buzz-ingest" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "base64 0.22.1", + "buzz-core", + "chrono", + "hex", + "hmac 0.13.0", + "nostr 0.44.7", + "reqwest 0.13.4", + "serde", + "serde_json", + "sha2 0.11.0", + "tokio", + "tokio-util", + "tower 0.5.3", + "url", + "uuid", +] + [[package]] name = "buzz-media" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 90fbc6d96ff..bb19b20e68f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/git-credential-nostr", "crates/git-sign-nostr", "crates/buzz-pair-relay", + "crates/buzz-ingest", "crates/buzz-relay-mesh", "crates/buzz-mesh-smoke", "crates/buzz-dev-mcp", diff --git a/crates/buzz-core/src/integration.rs b/crates/buzz-core/src/integration.rs new file mode 100644 index 00000000000..1ef6b3dc0ed --- /dev/null +++ b/crates/buzz-core/src/integration.rs @@ -0,0 +1,210 @@ +//! NIP-IG integration event tags — the shared contract between integration +//! publishers (such as `buzz-ingest`) and consumers (the workflow engine). +//! +//! An integration event is an ordinary channel event (a kind:9 message, or a +//! kind:30623 [`crate::kind::KIND_INTEGRATION_RECORD`]) that mirrors an +//! occurrence in an external system: a Linear issue comment, a GitHub check +//! run, a Slack message. Its typed metadata lives in `ext-*` tags: +//! +//! ```text +//! ["ext-source", "linear"] provider +//! ["ext-action", "issue.labeled"] normalized "." +//! ["ext-id", ""] +//! ["ext-url", "https://linear.app/..."] +//! ["ext-actor", ""] +//! ["ext-thread", ""] +//! ["ext-key", ""] +//! ["ext-labels", "agent"] repeatable +//! ``` +//! +//! Consumers read these generically: every `ext-` tag becomes the field +//! `` (dashes folded to underscores), and repeated tags join with `,`. +//! Nothing here knows about any provider. See `docs/nips/NIP-IG.md`. +//! +//! The tags are signer-controlled metadata. A consumer that acts on them must +//! first check that the signer is a trusted integration publisher. + +use std::collections::BTreeMap; + +/// Prefix shared by every integration metadata tag. +pub const EXT_TAG_PREFIX: &str = "ext-"; +/// Provider name, for example `github`, `linear`, or `slack`. +pub const TAG_SOURCE: &str = "ext-source"; +/// Normalized action, `.`, for example `pull_request.opened`. +pub const TAG_ACTION: &str = "ext-action"; +/// Provider-side identifier of the object the occurrence is about. +pub const TAG_ID: &str = "ext-id"; +/// Canonical provider URL of the object. +pub const TAG_URL: &str = "ext-url"; +/// Provider-side actor (login, user id) who caused the occurrence. +pub const TAG_ACTOR: &str = "ext-actor"; +/// Key of the external thread (issue, pull request, Slack thread). +pub const TAG_THREAD: &str = "ext-thread"; +/// Stable dedup key of the occurrence; equal to the record's `d` tag. +pub const TAG_KEY: &str = "ext-key"; +/// Repository, as `owner/name` (GitHub). +pub const TAG_REPO: &str = "ext-repo"; +/// Team or workspace key (Linear team key, Slack team id). +pub const TAG_TEAM: &str = "ext-team"; +/// Human reference, for example `ENG-123` or `owner/repo#42`. +pub const TAG_REF: &str = "ext-ref"; +/// Title of the object the thread mirrors. +pub const TAG_TITLE: &str = "ext-title"; +/// Current workflow state (Linear state name, GitHub open/closed/merged). +pub const TAG_STATE: &str = "ext-state"; +/// A label currently on the object. Repeatable. +pub const TAG_LABEL: &str = "ext-labels"; +/// A label this occurrence added. Repeatable. +pub const TAG_ADDED_LABEL: &str = "ext-added-labels"; +/// CI conclusion (`success`, `failure`, ...). +pub const TAG_CONCLUSION: &str = "ext-conclusion"; +/// Review state (`approved`, `changes_requested`, `commented`). +pub const TAG_REVIEW_STATE: &str = "ext-review-state"; +/// Provider channel id (Slack channel). +pub const TAG_CHANNEL: &str = "ext-channel"; + +/// On a kind:30623 record: the id of the kind:9 message that renders it. +/// +/// Deliberately not an `e` tag. The record is written before its message +/// (so a retry can never orphan a message), and an `e` tag would make the +/// relay resolve NIP-10 thread metadata against an event that does not exist +/// yet. +pub const TAG_MESSAGE: &str = "message"; + +/// Most `ext-*` fields read from one event. Bounds work on a hostile event. +pub const MAX_FIELDS: usize = 32; + +/// Most bytes kept per field value, after joining repeated tags. +pub const MAX_FIELD_BYTES: usize = 4096; + +/// Read every `ext-*` tag of `tags` into a field map. +/// +/// - `ext-added-labels` becomes the key `added_labels`. +/// - Repeated tags join their values with `,` in tag order. +/// - Names that would not be a plain identifier (ASCII letters, digits, +/// underscore after folding) are skipped, so every key is usable as an +/// evalexpr variable suffix. +/// - At most [`MAX_FIELDS`] keys; values are cut at [`MAX_FIELD_BYTES`] on a +/// character boundary. +pub fn ext_fields(tags: &nostr::Tags) -> BTreeMap { + let mut fields: BTreeMap = BTreeMap::new(); + for tag in tags.iter() { + let parts = tag.as_slice(); + let (Some(name), Some(value)) = (parts.first(), parts.get(1)) else { + continue; + }; + let Some(raw_key) = name.strip_prefix(EXT_TAG_PREFIX) else { + continue; + }; + let key = raw_key.replace('-', "_"); + if key.is_empty() || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + continue; + } + if !fields.contains_key(&key) && fields.len() >= MAX_FIELDS { + continue; + } + let slot = fields.entry(key).or_default(); + if !slot.is_empty() { + slot.push(','); + } + slot.push_str(value); + truncate_on_char_boundary(slot, MAX_FIELD_BYTES); + } + fields +} + +/// True when `tags` carry both an `ext-source` and an `ext-action` value. +pub fn is_integration_event(tags: &nostr::Tags) -> bool { + let has = |name: &str| { + tags.iter().any(|t| { + let parts = t.as_slice(); + parts.first().map(String::as_str) == Some(name) + && parts.get(1).is_some_and(|v| !v.is_empty()) + }) + }; + has(TAG_SOURCE) && has(TAG_ACTION) +} + +fn truncate_on_char_boundary(s: &mut String, max: usize) { + if s.len() <= max { + return; + } + let mut cut = max; + while !s.is_char_boundary(cut) { + cut -= 1; + } + s.truncate(cut); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tags(raw: &[&[&str]]) -> nostr::Tags { + nostr::Tags::from_list( + raw.iter() + .map(|t| nostr::Tag::parse(t.iter().copied()).unwrap()) + .collect(), + ) + } + + #[test] + fn ext_fields_folds_names_and_joins_repeats() { + let t = tags(&[ + &["h", "chan"], + &["ext-source", "linear"], + &["ext-action", "issue.labeled"], + &["ext-labels", "agent"], + &["ext-labels", "bug"], + &["ext-added-labels", "agent"], + &["message", "abc"], + ]); + let f = ext_fields(&t); + assert_eq!(f.get("source").map(String::as_str), Some("linear")); + assert_eq!(f.get("action").map(String::as_str), Some("issue.labeled")); + assert_eq!(f.get("labels").map(String::as_str), Some("agent,bug")); + assert_eq!(f.get("added_labels").map(String::as_str), Some("agent")); + assert!(!f.contains_key("h")); + assert!(!f.contains_key("message")); + } + + #[test] + fn ext_fields_skips_non_identifier_names_and_bounds_output() { + let mut raw: Vec> = vec![ + vec!["ext-".into(), "empty".into()], + vec!["ext-a.b".into(), "dot".into()], + vec!["ext-bad name".into(), "space".into()], + ]; + for i in 0..(MAX_FIELDS + 5) { + raw.push(vec![format!("ext-f{i}"), "v".into()]); + } + raw.push(vec!["ext-big".into(), "é".repeat(MAX_FIELD_BYTES)]); + let t = nostr::Tags::from_list( + raw.iter() + .map(|t| nostr::Tag::parse(t.iter().map(String::as_str)).unwrap()) + .collect(), + ); + let f = ext_fields(&t); + assert_eq!(f.len(), MAX_FIELDS); + assert!(f.keys().all(|k| !k.is_empty() && !k.contains('.'))); + + let big = tags(&[&["ext-big", &"é".repeat(MAX_FIELD_BYTES)]]); + let f = ext_fields(&big); + let v = f.get("big").unwrap(); + assert!(v.len() <= MAX_FIELD_BYTES); + assert!(v.chars().all(|c| c == 'é')); + } + + #[test] + fn is_integration_event_requires_source_and_action() { + assert!(is_integration_event(&tags(&[ + &["ext-source", "github"], + &["ext-action", "pull_request.opened"], + ]))); + assert!(!is_integration_event(&tags(&[&["ext-source", "github"]]))); + assert!(!is_integration_event(&tags(&[ + &["ext-source", "github"], + &["ext-action", ""], + ]))); + } +} diff --git a/crates/buzz-core/src/kind.rs b/crates/buzz-core/src/kind.rs index 9f66666467f..1a221868125 100644 --- a/crates/buzz-core/src/kind.rs +++ b/crates/buzz-core/src/kind.rs @@ -505,6 +505,18 @@ pub const KIND_WORKFLOW_DEF: u32 = 30620; /// `hidden_at` per viewer; this is the only Nostr-visible projection of it. pub const KIND_DM_VISIBILITY: u32 = 30622; +/// NIP-IG: integration record (parameterized replaceable, d=dedup key). +/// +/// Published by an integration publisher such as `buzz-ingest` for every +/// external occurrence (a Linear comment, a GitHub check run, a Slack message) +/// and every external thread it mirrors. The `d` tag is the stable dedup key, +/// so a redelivered provider webhook resolves to the same record instead of a +/// second message. The `message` tag names the kind:9 event that renders the +/// occurrence in the channel. The record content holds a bounded base64 +/// write-ahead plan; ordinary message searches select kind 9. Requires an `h` tag. See +/// `docs/nips/NIP-IG.md`. +pub const KIND_INTEGRATION_RECORD: u32 = 30623; + /// Lower bound of the NIP-33 parameterized replaceable range (30000–39999). pub const PARAM_REPLACEABLE_KIND_MIN: u32 = 30000; /// Upper bound of the NIP-33 parameterized replaceable range (30000–39999). @@ -842,6 +854,7 @@ pub const ALL_KINDS: &[u32] = &[ KIND_GIT_STATUS_CLOSED, KIND_GIT_STATUS_DRAFT, KIND_PROJECT, + KIND_INTEGRATION_RECORD, ]; /// Returns `true` if `kind` is in the ephemeral range (20000–29999). @@ -947,6 +960,7 @@ const _: () = assert!(is_parameterized_replaceable(KIND_WORKFLOW_DEF)); // 30620 const _: () = assert!(is_parameterized_replaceable(KIND_EVENT_REMINDER)); // 30300 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_DM_VISIBILITY)); // 30622 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_PROJECT)); // 30621 ∈ 30000–39999 +const _: () = assert!(is_parameterized_replaceable(KIND_INTEGRATION_RECORD)); // 30623 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_THREAD_SUMMARY)); // 39005 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_WINDOW_BOUNDS)); // 39006 ∈ 30000–39999 diff --git a/crates/buzz-core/src/lib.rs b/crates/buzz-core/src/lib.rs index 99b4aa68032..43e8f7fcf6b 100644 --- a/crates/buzz-core/src/lib.rs +++ b/crates/buzz-core/src/lib.rs @@ -20,6 +20,8 @@ pub mod event; pub mod filter; /// Git permission types — ref patterns, protection rules, policy evaluation. pub mod git_perms; +/// NIP-IG integration event tags (`ext-*`) shared by publishers and consumers. +pub mod integration; /// Shared invite-link contract constants. pub mod invite; /// Buzz kind number registry — custom event type constants. diff --git a/crates/buzz-ingest/Cargo.toml b/crates/buzz-ingest/Cargo.toml new file mode 100644 index 00000000000..3b94174fcb0 --- /dev/null +++ b/crates/buzz-ingest/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "buzz-ingest" +description = "Verified provider webhooks published as relay-backed Buzz threads" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow = { workspace = true } +axum = { workspace = true } +base64 = { workspace = true } +buzz-core = { path = "../buzz-core" } +chrono = { workspace = true } +hex = { workspace = true } +hmac = { workspace = true } +nostr = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +url = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +tower = { workspace = true } diff --git a/crates/buzz-ingest/README.md b/crates/buzz-ingest/README.md new file mode 100644 index 00000000000..789c12a383d --- /dev/null +++ b/crates/buzz-ingest/README.md @@ -0,0 +1,105 @@ +# buzz-ingest + +`buzz-ingest` verifies GitHub, Linear, and public Slack webhooks, then publishes +ordinary thread messages to Buzz. It runs beside the relay, with its own member +key. The relay remains the durable store. Slack DMs and private channels are excluded. + +| Source | Supported activity | +| --- | --- | +| GitHub | Pull requests, PR comments/reviews/review comments, and check runs. Checks target at most four linked PRs, or a commit thread. | +| Linear | Issues, labels added through issue updates, and comments. | +| Slack | Public-channel messages and message edits; signed URL verification. | + +GitHub actions derive from authenticated body fields. The unsigned event header +must match the payload shape. Unsupported or unmapped activity returns an explicit +ignored response. Mapped publication returns 200 only after the relay accepts its +plans, anchor, and reply; incomplete writes return 503 for provider retry. + +See [NIP-IG](../../docs/nips/NIP-IG.md) for the event and idempotency contract. +Single-replica Recreate deployment is required. No PVC or local journal is needed. + +## Configuration + +Set `BUZZ_INGEST_CONFIG` to this JSON file: + +```json +{ + "relay_url": "https://buzz.voltage.cloud", + "github_repos": {"voltagecloud/buzz": ""}, + "linear_teams": {"": ""}, + "linear_projects": {"": ""}, + "linear_webhooks": {"": ""}, + "linear_labels": {"