From c17923e4ca9d3f1f6ec4ec82f8a9c6639a1aa927 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Thu, 5 Nov 2026 13:07:40 -0600 Subject: [PATCH 1/2] fix(huddle-bridge): key "playing" and the cut on the voice's speech GPT-Live streams audio deltas all session, silence between answers included. The bridge treated any PCM as the voice playing, so in the 5 Nov live test the first sound on Russell's mic cut a silent stream about 200 ms in. A cut released only after 300 ms with no PCM, which never came, so every answer for the next 69 s was dropped. Output now keeps each frame's level. Only frames at speech level (Barge::speech_dbov, -45 dBov) count toward until(), which drives barge-in and Input's half-duplex window. A cut releases after release_ms with no speech, so streamed silence ends it. Silence still goes to the room, where Opus DTX makes it tiny. A real answer that a person talks over is still cut, and the rest of it is dropped while it keeps speaking. frames.rs and the schema description no longer say Talos sends nothing between responses. Co-Authored-By: Claude Opus 5.5 --- .../buzz-huddle-bridge/frames-v1/schema.json | 2 +- crates/buzz-huddle-bridge/src/frames.rs | 8 +- crates/buzz-huddle-bridge/src/output.rs | 151 +++++++++++++++--- 3 files changed, 131 insertions(+), 30 deletions(-) diff --git a/crates/buzz-huddle-bridge/frames-v1/schema.json b/crates/buzz-huddle-bridge/frames-v1/schema.json index 154bdf63319..c090ad6482d 100644 --- a/crates/buzz-huddle-bridge/frames-v1/schema.json +++ b/crates/buzz-huddle-bridge/frames-v1/schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "buzz-huddle-bridge/frames-v1", - "description": "Text frames between a PC voice bridge and Talos /df/voice. Audio is binary PCM16 LE mono 24 kHz. Talos sends a response's PCM paced to real time and none between responses: the PC queues at most 320 ms and drops the rest, and after a barge-in drops PCM until none has come for 300 ms or a newer floor token arrives. $defs/to_talos are PC to Talos and $defs/to_pc Talos to PC; each side accepts only the other side's frames, with no extra fields. t_ms, start_ms, end_ms: session audio ms, the ms of PCM the PC has sent on this socket, live and replayed alike. from_ms, to_ms: unix ms on the sender's clock; the PC widens each secret window by a skew margin on both sides and starts dropping on receipt, so skew only widens it; replay windows are best effort. Between replay and replay_end the PC sends only replayed PCM, no live audio. expires_at, until: unix seconds. span: per-session speaker span id. token: floor generation, never lower for a huddle. Talos refuses an open whose agent is not the socket's NIP-98 signer, and a resume whose lease is not keyed to that pubkey and huddle. speaker, onset and roster grant no authority: turns, waking and labels only.", + "description": "Text frames between a PC voice bridge and Talos /df/voice. Audio is binary PCM16 LE mono 24 kHz. Talos sends the voice's PCM paced to real time; GPT-Live streams it all session, silence between responses included, so the PC counts only PCM at speech level as the voice speaking. The PC queues at most 320 ms and drops the rest, and after a barge-in drops PCM until none at speech level has come for 300 ms or a newer floor token arrives. $defs/to_talos are PC to Talos and $defs/to_pc Talos to PC; each side accepts only the other side's frames, with no extra fields. t_ms, start_ms, end_ms: session audio ms, the ms of PCM the PC has sent on this socket, live and replayed alike. from_ms, to_ms: unix ms on the sender's clock; the PC widens each secret window by a skew margin on both sides and starts dropping on receipt, so skew only widens it; replay windows are best effort. Between replay and replay_end the PC sends only replayed PCM, no live audio. expires_at, until: unix seconds. span: per-session speaker span id. token: floor generation, never lower for a huddle. Talos refuses an open whose agent is not the socket's NIP-98 signer, and a resume whose lease is not keyed to that pubkey and huddle. speaker, onset and roster grant no authority: turns, waking and labels only.", "oneOf": [{"$ref": "#/$defs/to_talos"}, {"$ref": "#/$defs/to_pc"}], "$defs": { "hex": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, diff --git a/crates/buzz-huddle-bridge/src/frames.rs b/crates/buzz-huddle-bridge/src/frames.rs index 85350730b30..21edef9cbe7 100644 --- a/crates/buzz-huddle-bridge/src/frames.rs +++ b/crates/buzz-huddle-bridge/src/frames.rs @@ -2,9 +2,11 @@ //! //! JSON text frames carry `"v": 1` and a `type`; audio is binary PCM16 LE, mono, 24 kHz. //! -//! Talos sends a response's PCM paced to real time, and none between responses. The PC queues -//! at most 320 ms and drops the rest; after a barge-in it drops PCM until none has come for -//! 300 ms, or a newer `floor` token arrives. A huddle's `floor` token never goes down. +//! Talos sends the voice's PCM paced to real time. GPT-Live streams it all session, silence +//! between responses included, so the PC counts only PCM at speech level as the voice speaking. +//! The PC queues at most 320 ms and drops the rest; after a barge-in it drops PCM until none at +//! speech level has come for 300 ms, or a newer `floor` token arrives. A huddle's `floor` token +//! never goes down. //! [`ToTalos`] and [`ToPc`] are the two directions: each side parses only the other side's //! frames, and unknown fields are refused. `frames-v1/schema.json` and one fixture per frame in //! `frames-v1/fixtures/` are the contract Talos copies. diff --git a/crates/buzz-huddle-bridge/src/output.rs b/crates/buzz-huddle-bridge/src/output.rs index a99f5fbe440..b23a5fe53d9 100644 --- a/crates/buzz-huddle-bridge/src/output.rs +++ b/crates/buzz-huddle-bridge/src/output.rs @@ -3,8 +3,12 @@ //! 48 kHz and cut into 20 ms frames, at most [`QUEUE_FRAMES`] queued, and one frame is encoded //! per 20 ms tick. It plays only while this voice holds the floor (the speak token), and a //! human who talks over it cuts it (barge-in). The cut takes the place of the desktop's epoch: -//! the queue goes, and the rest of that response is dropped as it arrives, until Talos has sent -//! nothing for `release_ms` or gives the floor again with a newer token. +//! the queue goes, and the rest of that response is dropped as it arrives, until the voice has +//! sent no speech for `release_ms` or Talos gives the floor again with a newer token. +//! +//! GPT-Live streams PCM all session, silence between answers included, so only frames at speech +//! level ([`Barge::speech_dbov`]) count as the voice playing, for barge-in, the half-duplex +//! window and a cut's release. Silence still goes to the room, where Opus DTX makes it tiny. //! //! The caller owns the socket and the clock, as for [`crate::input::Input`]: it calls //! [`Output::hear`] every 10 ms and [`Output::tick`] every 20 ms. @@ -23,8 +27,11 @@ const TICK_MS: u64 = 10; /// Barge-in knobs; probe P measures the real values. #[derive(Clone, Debug)] pub struct Barge { - /// Decoded level (dBov) at or above which a human's 10 ms counts as energy. Keep it equal to - /// [`crate::input::Tuning::speech_dbov`], so a cut and Input's onset agree. + /// Decoded level (dBov) at or above which a human's 10 ms counts as energy, and a 20 ms frame + /// of the voice as speech. Keep it equal to [`crate::input::Tuning::speech_dbov`], so a cut + /// and Input's onset agree. GPT-Live's silence sits far below it; a human's room noise can sit + /// on it (5 Nov: 10 of 26 speech starts read -44 or -45), which only matters while the voice + /// speaks. pub speech_dbov: i8, /// Energy that cuts the voice from the owner's isolated desktop. pub isolated_ms: u64, @@ -39,8 +46,9 @@ pub struct Barge { pub echo_margin_db: i8, /// How long a frame's echo may still come back, so how long its level counts. pub echo_ms: u64, - /// After a cut, how long Talos must send nothing (the cut response is over) before the - /// voice may play again. + /// After a cut, how long the voice must send no speech (the cut response is over) before it + /// may play again. A pause inside the cut answer shorter than this keeps it cut; the next + /// answer starts 0.5 to 0.7 s after a person stops (5 Nov), so it isn't swallowed. pub release_ms: u64, } @@ -62,7 +70,8 @@ pub struct Output { me: String, barge: Barge, encoder: AudioEncoder, - queue: VecDeque>, + /// Each frame with its level. + queue: VecDeque<(Vec, i8)>, partial: Vec, prev: f32, /// The partial frame waited one tick for more PCM and goes out at the next. @@ -73,11 +82,12 @@ pub struct Output { floor: bool, /// Unix ms and level of each frame sent within `echo_ms`. sent: VecDeque<(u64, i8)>, - /// When the last frame sent ends. + /// When the last frame sent ends, and the last one with speech. last_end_ms: u64, + spoke_end_ms: u64, /// Each peer's ms of energy while the voice plays, and the ms since its last energy. runs: HashMap, - /// Ms since Talos last sent PCM, while cut; `None` when not cut. + /// Ms since the voice last sent speech, while cut; `None` when not cut. cut: Option, } @@ -97,6 +107,7 @@ impl Output { floor: false, sent: VecDeque::new(), last_end_ms: 0, + spoke_end_ms: 0, runs: HashMap::new(), cut: None, }) @@ -109,8 +120,8 @@ impl Output { if token < self.token { return; } - if token > self.token { - self.cut = None; + if token > self.token && self.cut.take().is_some() { + self.flush(); } self.token = token; self.floor = holder == Some(self.me.as_str()); @@ -125,10 +136,6 @@ impl Output { if !self.floor { return; } - if let Some(quiet) = &mut self.cut { - *quiet = 0; - return; - } self.tail_due = false; for s in pcm.as_chunks::<2>().0 { let s = f32::from(i16::from_le_bytes(*s)) / 32_768.0; @@ -137,8 +144,14 @@ impl Output { self.prev = s; if self.partial.len() == FRAME_SAMPLES { let frame = std::mem::take(&mut self.partial); - if self.queue.len() < QUEUE_FRAMES { - self.queue.push_back(frame); + let level = audio_level_dbov(&frame); + if let Some(quiet) = &mut self.cut { + // Only speech holds a cut, not the silence a live model streams after it. + if level >= self.barge.speech_dbov { + *quiet = 0; + } + } else if self.queue.len() < QUEUE_FRAMES { + self.queue.push_back((frame, level)); } else if !std::mem::replace(&mut self.overflow, true) { tracing::warn!( "huddle bridge: voice PCM ahead of real time, dropped past 320 ms" @@ -152,28 +165,49 @@ impl Output { /// waits one tick for the PCM that may finish it, so a chunk just in time pads no silence /// into speech, then goes out padded by the encoder. pub fn tick(&mut self, now_ms: u64) -> Option> { - let frame = if let Some(frame) = self.queue.pop_front() { - frame - } else if self.partial.is_empty() || !std::mem::replace(&mut self.tail_due, true) { + let (frame, level) = if let Some(queued) = self.queue.pop_front() { + queued + } else if self.cut.is_some() + || self.partial.is_empty() + || !std::mem::replace(&mut self.tail_due, true) + { + // A cut keeps resampling to hear when the voice stops, but sends none of it. self.overflow = false; return None; } else { - std::mem::take(&mut self.partial) + let frame = std::mem::take(&mut self.partial); + let level = audio_level_dbov(&frame); + (frame, level) }; let payload = self .encoder .encode(&frame) .inspect_err(|e| tracing::warn!(error = %e, "huddle bridge: voice encode")) .ok()?; - self.sent.push_back((now_ms, audio_level_dbov(&frame))); + self.sent.push_back((now_ms, level)); self.last_end_ms = now_ms + FRAME_MS; + if level >= self.barge.speech_dbov { + self.spoke_end_ms = self.last_end_ms; + } Some(payload) } - /// When the voice's output ends, queued audio included, for the half-duplex window. + /// When the voice's speech ends, queued speech included, for the half-duplex window and + /// barge-in. Silence after it, sent or queued, doesn't count. pub fn until(&self) -> u64 { - let frames = self.queue.len() + usize::from(!self.partial.is_empty()); - self.last_end_ms + FRAME_MS * frames as u64 + let speech = self.barge.speech_dbov; + let tail = self.cut.is_none() + && !self.partial.is_empty() + && audio_level_dbov(&self.partial) >= speech; + let queued = self.queue.iter().rposition(|(_, level)| *level >= speech); + let frames = if tail { + Some(self.queue.len() + 1) + } else { + queued.map(|i| i + 1) + }; + frames.map_or(self.spoke_end_ms, |n| { + self.last_end_ms + FRAME_MS * n as u64 + }) } /// Barge-in, every 10 ms, with each admitted human's level. While the voice plays, a human @@ -198,6 +232,8 @@ impl Output { self.cut = (quiet < b.release_ms).then_some(quiet); if self.cut.is_none() { tracing::info!("huddle bridge: the cut response is over"); + // Under a frame of the cut response, resampled but never sent. + self.flush(); } return false; } @@ -316,7 +352,7 @@ mod tests { o.on_pcm(&pcm(7, 0.3)); o.on_pcm(&pcm(13, 0.3)); assert_eq!((o.queue.len(), o.partial.len()), (1, 0)); - assert!(o.queue[0].len() == FRAME_SAMPLES); + assert!(o.queue[0].0.len() == FRAME_SAMPLES); o.on_pcm(&pcm(1_000, 0.3)); assert_eq!(o.queue.len(), QUEUE_FRAMES, "320 ms, the rest dropped"); let mut sent = 0; @@ -411,6 +447,69 @@ mod tests { assert!(o.cut.is_none()); } + /// GPT-Live's stream: 20 ms chunks all session, silence (about -57 dBov) except speech at + /// about -23 dBov while `speaking(t)`. + fn live(speaking: impl Fn(u64) -> bool) -> impl Fn(u64) -> Option> { + move |t| (t % FRAME_MS == 0).then(|| pcm(20, if speaking(t) { 0.1 } else { 0.002 })) + } + + /// The 5 Nov field failure: a person's first words cut GPT-Live's silence 200 ms in, the + /// silence that kept coming held the cut, and no answer ever reached the room. + #[test] + fn a_live_models_silence_is_not_the_voice_playing() { + for human in [level(1, true, -40), level(2, false, -40)] { + let mut o = output(); + let mut now = T0; + let quiet = move |_| level(human.index, human.isolated, -127); + play(&mut o, &mut now, 500, live(|_| false), quiet); + assert!(o.until() <= T0, "silence holds the echo window open"); + let (_, cut) = play(&mut o, &mut now, 1_500, live(|_| false), |_| human); + assert_eq!(cut, None, "{human:?}: the question cut the silence"); + let (sent, cut) = play(&mut o, &mut now, 1_000, live(|_| true), quiet); + assert_eq!((sent, cut), (50, None), "{human:?}: the whole answer"); + } + } + + #[test] + fn talking_over_a_live_answer_still_cuts_it_and_drops_the_rest() { + let mut o = output(); + let mut now = T0; + let owner = |_| level(1, true, -35); + let quiet = |_| level(1, true, -127); + assert_eq!( + play(&mut o, &mut now, 1_000, live(|_| false), owner).1, + None + ); + assert_eq!( + play(&mut o, &mut now, 400, live(|_| true), quiet), + (20, None) + ); + // 200 ms of energy, the first counted at once since the answer is already playing. + assert_eq!( + play(&mut o, &mut now, 400, live(|_| true), owner).1, + Some(190) + ); + assert_eq!(play(&mut o, &mut now, 2_000, live(|_| true), quiet).0, 0); + } + + #[test] + fn after_a_cut_the_streamed_silence_ends_it_and_the_next_answer_plays() { + let mut o = output(); + let mut now = T0; + let owner = |_| level(1, true, -35); + let quiet = |_| level(1, true, -127); + assert_eq!( + play(&mut o, &mut now, 400, live(|_| true), owner).1, + Some(200) + ); + // A pause in speech shorter than `release_ms` holds the cut; a longer one ends it. + play(&mut o, &mut now, 200, live(|_| false), quiet); + assert!(o.cut.is_some()); + play(&mut o, &mut now, 200, live(|_| false), quiet); + assert!(o.cut.is_none(), "the silence held the cut"); + assert_eq!(play(&mut o, &mut now, 1_000, live(|_| true), quiet).0, 50); + } + /// Input and Output together in link.rs's step order, with a decoded talker: `loud(t)` says /// whether the talker's mic carries speech `t` ms in, and `voice(t)` whether Talos sends. /// How far in the cut came, and the first frame sent after it. From 547cf8deb37d027c056d785ab3218712ec7c2c65 Mon Sep 17 00:00:00 2001 From: QuicksilverSlick Date: Thu, 5 Nov 2026 13:45:16 -0600 Subject: [PATCH 2/2] fix(huddle-bridge): review of the silent-stream fix Barge-in now counts the voice as playing through a pause shorter than release_ms (300 ms), so a talker's run carries across the voice's phrases. Keyed on speech alone, a voice that paused over ~100 ms reset the run and could not be cut on speakers. The voice's speech line is its own knob, Barge::voice_dbov (-45, as before), apart from the people's line, so either can move alone. A stream that never stops never runs dry, so a burst's backlog lasted the session. Silence after silence is now dropped once two frames are queued, so a backlog drains through the voice's silence; a lone quiet frame inside speech is kept. The overflow warning re-arms once the backlog is down to a frame. Barge-in on the isolated desktop takes 320 ms, as on speakers, and the desktop's echo tail is 600 ms: probe step 6 heard the voice come back 361-508 ms after it played, and with "playing" keyed on speech the echo window now closes at every pause, not only at an answer's end. New tests cover phrases with pauses on both routes, a soft voice between the two lines, unaligned deltas, a cut answer's scrap under a frame (after the release and after a newer token), the backlog, and the desktop's speaker tail. Co-Authored-By: Claude Opus 5.5 --- crates/buzz-huddle-bridge/src/input.rs | 24 ++-- crates/buzz-huddle-bridge/src/output.rs | 150 ++++++++++++++++---- crates/buzz-huddle-bridge/tests/loopback.rs | 21 +-- 3 files changed, 145 insertions(+), 50 deletions(-) diff --git a/crates/buzz-huddle-bridge/src/input.rs b/crates/buzz-huddle-bridge/src/input.rs index 16e7ee7ebef..eb4a5295fcc 100644 --- a/crates/buzz-huddle-bridge/src/input.rs +++ b/crates/buzz-huddle-bridge/src/input.rs @@ -55,10 +55,11 @@ pub struct Tuning { pub onset_ms: u64, /// Silence before the forwarded human releases the floor. pub hangover_ms: u64, - /// Echo tail after the voice's output, for the owner's desktop. ponytail: both tails are - /// guesses until probe P measures them from `output_until`, the desktop's and the phone's. + /// Echo tail after the voice's speech, for the owner's desktop on speakers, at its end and at + /// every pause in it. Probe step 6 (5 Nov) heard the voice come back 361-508 ms after it + /// played, which leaked past 300 ms. pub desktop_tail_ms: u64, - /// Echo tail for every other peer. + /// Echo tail for every other peer. ponytail: a guess; the phone sent back none of 5 clips. pub remote_tail_ms: u64, /// Widening on both sides of each `secret` window, for clock skew with Talos. pub secret_skew_ms: u64, @@ -73,7 +74,7 @@ impl Default for Tuning { speech_dbov: -45, onset_ms: 100, hangover_ms: 300, - desktop_tail_ms: 300, + desktop_tail_ms: 600, remote_tail_ms: 600, secret_skew_ms: 1_000, floor_cap_ms: 30_000, @@ -929,13 +930,15 @@ mod tests { #[test] fn half_duplex_unless_the_desktop_here_is_isolated() { let mut now = T0; - for (here, echo) in [(false, true), (true, false)] { + // The phone, the desktop on speakers, the desktop on headphones. + for (here, isolated, echo) in [ + (false, true, true), + (true, false, true), + (true, true, false), + ] { let mut input = input(); input.set_awake(true); - input.set_route(Route { - here, - isolated: true, - }); + input.set_route(Route { here, isolated }); input.set_output_until(now + 1_000); let mut owner = [talker(1, 0.3, None)]; let start = now; @@ -950,7 +953,8 @@ mod tests { assert_eq!(voiced(&during), !echo, "here: {here}"); let after = run(&mut input, &mut now, 1_200, &mut owner); assert!(voiced(&after) && input.who_said(0, u64::MAX).all(|s| s.echo == echo)); - // Echo lasts to start + 1.6 s (1 s of output, 600 ms tail) and never enters the ring. + // Echo lasts to start + 1.6 s (1 s of output, a 600 ms tail on either speakers) and + // never enters the ring. let kept = if echo { 20 } else { 180 }; assert_eq!(ring(&input, start, now, now), kept); } diff --git a/crates/buzz-huddle-bridge/src/output.rs b/crates/buzz-huddle-bridge/src/output.rs index b23a5fe53d9..418eb227ce2 100644 --- a/crates/buzz-huddle-bridge/src/output.rs +++ b/crates/buzz-huddle-bridge/src/output.rs @@ -6,9 +6,10 @@ //! the queue goes, and the rest of that response is dropped as it arrives, until the voice has //! sent no speech for `release_ms` or Talos gives the floor again with a newer token. //! -//! GPT-Live streams PCM all session, silence between answers included, so only frames at speech -//! level ([`Barge::speech_dbov`]) count as the voice playing, for barge-in, the half-duplex -//! window and a cut's release. Silence still goes to the room, where Opus DTX makes it tiny. +//! GPT-Live streams PCM all session, silence between answers included, so only frames at the +//! voice's speech level ([`Barge::voice_dbov`]) count as the voice playing, for barge-in, the +//! half-duplex window and a cut's release. Silence still goes to the room, where Opus DTX makes it +//! tiny, except that a backlog drains through it: a stream that never stops never runs dry. //! //! The caller owns the socket and the clock, as for [`crate::input::Input`]: it calls //! [`Output::hear`] every 10 ms and [`Output::tick`] every 20 ms. @@ -19,21 +20,27 @@ use buzz_audio_client::wire::audio_level_dbov; use std::collections::{HashMap, VecDeque}; /// 16 frames, 320 ms: the most a barge-in ever has to cut. Talos paces its PCM to real time -/// (frames v1); a burst past this is dropped, with one warning per response. +/// (frames v1); a burst past this is dropped, with one warning until the backlog clears. pub const QUEUE_FRAMES: usize = 16; +/// With this many frames queued, silence after silence is dropped, so a backlog drains. +const BACKLOG_FRAMES: usize = 2; const FRAME_MS: u64 = 20; const TICK_MS: u64 = 10; /// Barge-in knobs; probe P measures the real values. #[derive(Clone, Debug)] pub struct Barge { - /// Decoded level (dBov) at or above which a human's 10 ms counts as energy, and a 20 ms frame - /// of the voice as speech. Keep it equal to [`crate::input::Tuning::speech_dbov`], so a cut - /// and Input's onset agree. GPT-Live's silence sits far below it; a human's room noise can sit - /// on it (5 Nov: 10 of 26 speech starts read -44 or -45), which only matters while the voice - /// speaks. + /// Decoded level (dBov) at or above which a human's 10 ms counts as energy. Keep it equal to + /// [`crate::input::Tuning::speech_dbov`], so a cut and Input's onset agree. A human's room + /// noise can sit on it (5 Nov: 19 of 59 speech starts read -44 or -45), which only matters + /// while the voice speaks. pub speech_dbov: i8, - /// Energy that cuts the voice from the owner's isolated desktop. + /// Level (dBov) at or above which a 20 ms frame of the voice is speech, kept apart from the + /// people's line so either can move alone. ponytail: GPT-Live's silence level isn't measured + /// (the tests assume about -57), and a voice softer than this line can't be cut. + pub voice_dbov: i8, + /// Energy that cuts the voice from the owner's isolated desktop. As long as on speakers, so a + /// cough or a chair doesn't cut a real answer (the 1:1 runs); GPT-Live hears a real one. pub isolated_ms: u64, /// Energy above the echo threshold that cuts it from any other peer. pub sustained_ms: u64, @@ -46,9 +53,11 @@ pub struct Barge { pub echo_margin_db: i8, /// How long a frame's echo may still come back, so how long its level counts. pub echo_ms: u64, - /// After a cut, how long the voice must send no speech (the cut response is over) before it - /// may play again. A pause inside the cut answer shorter than this keeps it cut; the next - /// answer starts 0.5 to 0.7 s after a person stops (5 Nov), so it isn't swallowed. + /// A pause in the voice's speech shorter than this is inside one answer: barge-in still + /// counts the voice as playing, and a cut holds. After a cut, the voice must send no speech + /// this long before it may play again. ponytail: a cut answer that pauses this long plays on + /// after the pause, but the next answer starts 0.5 to 0.7 s after a person stops (5 Nov), so + /// a longer wait would swallow it. Upgrade: a frames v1 response boundary from Talos. pub release_ms: u64, } @@ -56,7 +65,8 @@ impl Default for Barge { fn default() -> Self { Self { speech_dbov: -45, - isolated_ms: 200, + voice_dbov: -45, + isolated_ms: 320, sustained_ms: 320, gap_ms: 80, echo_margin_db: 6, @@ -76,7 +86,7 @@ pub struct Output { prev: f32, /// The partial frame waited one tick for more PCM and goes out at the next. tail_due: bool, - /// A frame was dropped past the cap since the queue last ran empty. + /// A frame was dropped past the cap since the backlog last cleared. overflow: bool, token: u64, floor: bool, @@ -145,11 +155,17 @@ impl Output { if self.partial.len() == FRAME_SAMPLES { let frame = std::mem::take(&mut self.partial); let level = audio_level_dbov(&frame); + let voice = self.barge.voice_dbov; + let silent_after_silence = level < voice + && self.queue.len() >= BACKLOG_FRAMES + && self.queue.back().is_some_and(|(_, l)| *l < voice); if let Some(quiet) = &mut self.cut { // Only speech holds a cut, not the silence a live model streams after it. - if level >= self.barge.speech_dbov { + if level >= voice { *quiet = 0; } + } else if silent_after_silence { + // A backlog drains here; a lone quiet frame inside speech is kept. } else if self.queue.len() < QUEUE_FRAMES { self.queue.push_back((frame, level)); } else if !std::mem::replace(&mut self.overflow, true) { @@ -179,6 +195,8 @@ impl Output { let level = audio_level_dbov(&frame); (frame, level) }; + // The backlog is down to a frame: the next overflow warns again. + self.overflow &= self.queue.len() > 1; let payload = self .encoder .encode(&frame) @@ -186,7 +204,7 @@ impl Output { .ok()?; self.sent.push_back((now_ms, level)); self.last_end_ms = now_ms + FRAME_MS; - if level >= self.barge.speech_dbov { + if level >= self.barge.voice_dbov { self.spoke_end_ms = self.last_end_ms; } Some(payload) @@ -195,7 +213,7 @@ impl Output { /// When the voice's speech ends, queued speech included, for the half-duplex window and /// barge-in. Silence after it, sent or queued, doesn't count. pub fn until(&self) -> u64 { - let speech = self.barge.speech_dbov; + let speech = self.barge.voice_dbov; let tail = self.cut.is_none() && !self.partial.is_empty() && audio_level_dbov(&self.partial) >= speech; @@ -239,7 +257,8 @@ impl Output { } let recent = self.sent.iter().map(|(_, l)| *l).max().unwrap_or(-127); let echo = recent.saturating_add(b.echo_margin_db).max(b.speech_dbov); - let playing = now_ms < self.until(); + // Through a pause inside an answer, so a run carries across the voice's phrases. + let playing = now_ms < self.until() + b.release_ms; let mut barged = None; let runs = levels .into_iter() @@ -375,10 +394,11 @@ mod tests { // 1 s in 50 ms chunks: 50 frames, no padded tail between chunks. let quiet = |_| level(2, false, -127); assert_eq!(play(&mut o, &mut now, 1_200, talos(50, 1_000), quiet).0, 50); - // The queue holds under a frame now and then; a talker on speakers still cuts at 320 ms. + // The queue holds under a frame now and then; a talker on speakers still cuts after + // 320 ms, the first 10 counted at once as the voice spoke 200 ms ago, inside a pause. let loud = |_| level(2, false, -10); let (_, cut) = play(&mut o, &mut now, 1_000, talos(50, 1_000), loud); - assert_eq!(cut, Some(320)); + assert_eq!(cut, Some(310)); } #[test] @@ -397,14 +417,14 @@ mod tests { } #[test] - fn barge_in_takes_200_ms_isolated_or_320_ms_over_the_echo_and_outlasts_dips() { + fn barge_in_takes_320_ms_isolated_or_over_the_echo_and_outlasts_dips() { // Speech with a 20 ms dip every 200 ms; the voice plays from the first tick. let dipping = |index, isolated, dbov| { move |t: u64| level(index, isolated, if t % 200 >= 180 { -60 } else { dbov }) }; // On headphones the voice's level doesn't matter; on speakers it plays at about // -23 dBov, so its echo threshold is about -17 dBov. - for (human, cut) in [(dipping(1, true, -40), 220), (dipping(2, false, -10), 340)] { + for (human, cut) in [(dipping(1, true, -40), 340), (dipping(2, false, -10), 340)] { let mut o = output(); let mut now = T0; let (_, at) = play(&mut o, &mut now, 1_000, talos(20, 1_000), human); @@ -427,7 +447,7 @@ mod tests { let mut now = T0; let owner = |_| level(1, true, -30); let voice = || talos(20, u64::MAX); - assert_eq!(play(&mut o, &mut now, 400, voice(), owner).1, Some(200)); + assert_eq!(play(&mut o, &mut now, 400, voice(), owner).1, Some(320)); // The response goes on, the room is quiet or noisy: none of it plays. let noise = |_| level(2, false, -44); assert_eq!(play(&mut o, &mut now, 3_000, voice(), noise).0, 0); @@ -447,10 +467,15 @@ mod tests { assert!(o.cut.is_none()); } - /// GPT-Live's stream: 20 ms chunks all session, silence (about -57 dBov) except speech at - /// about -23 dBov while `speaking(t)`. + /// A stream all session in `chunk` ms deltas at `amp(t)`: 0.1 is about -23 dBov, 0.015 + /// about -40 and 0.002 (silence) about -57. + fn stream(chunk: u64, amp: impl Fn(u64) -> f32) -> impl Fn(u64) -> Option> { + move |t| (t % chunk == 0).then(|| pcm(chunk as usize, amp(t))) + } + + /// GPT-Live's stream: 20 ms chunks all session, silence except speech while `speaking(t)`. fn live(speaking: impl Fn(u64) -> bool) -> impl Fn(u64) -> Option> { - move |t| (t % FRAME_MS == 0).then(|| pcm(20, if speaking(t) { 0.1 } else { 0.002 })) + stream(FRAME_MS, move |t| if speaking(t) { 0.1 } else { 0.002 }) } /// The 5 Nov field failure: a person's first words cut GPT-Live's silence 200 ms in, the @@ -484,10 +509,10 @@ mod tests { play(&mut o, &mut now, 400, live(|_| true), quiet), (20, None) ); - // 200 ms of energy, the first counted at once since the answer is already playing. + // 320 ms of energy, the first counted at once since the answer is already playing. assert_eq!( play(&mut o, &mut now, 400, live(|_| true), owner).1, - Some(190) + Some(310) ); assert_eq!(play(&mut o, &mut now, 2_000, live(|_| true), quiet).0, 0); } @@ -500,7 +525,7 @@ mod tests { let quiet = |_| level(1, true, -127); assert_eq!( play(&mut o, &mut now, 400, live(|_| true), owner).1, - Some(200) + Some(320) ); // A pause in speech shorter than `release_ms` holds the cut; a longer one ends it. play(&mut o, &mut now, 200, live(|_| false), quiet); @@ -510,6 +535,71 @@ mod tests { assert_eq!(play(&mut o, &mut now, 1_000, live(|_| true), quiet).0, 50); } + #[test] + fn a_talker_cuts_an_answer_that_pauses_between_phrases() { + // Phrases of 250 ms with 150 ms pauses: the run carries across them, on either route. + for human in [level(1, true, -35), level(2, false, -10)] { + let mut o = output(); + let mut now = T0; + let phrases = live(|t| t % 400 < 250); + let (_, cut) = play(&mut o, &mut now, 2_000, phrases, move |_| human); + assert_eq!(cut, Some(320), "{human:?}"); + } + } + + /// Soft speech between the two lines, unaligned deltas, and a cut answer's scrap under a frame. + #[test] + fn a_soft_answer_plays_and_cuts_and_a_cut_leaves_no_scrap() { + let mut o = output(); + let mut now = T0; + let owner = |_| level(1, true, -35); + let quiet = |_| level(1, true, -127); + // 30 ms deltas of silence leave a scrap between them, which isn't speech either. + let silence = stream(30, |_| 0.002); + assert_eq!(play(&mut o, &mut now, 1_000, silence, owner).1, None); + assert!(o.until() <= now, "silence holds the echo window open"); + let soft = || stream(FRAME_MS, |_| 0.015); + assert_eq!(play(&mut o, &mut now, 400, soft(), quiet), (20, None)); + // Cut on the same tick as a loud answer; a scrap that comes after holds nothing open. + assert_eq!(play(&mut o, &mut now, 320, soft(), owner).1, Some(310)); + o.on_pcm(&pcm(10, 0.015)); + assert!( + o.until() <= now, + "the cut answer holds the echo window open" + ); + // Talos stops: the scrap never plays, during the cut, after it, or after a newer token. + assert_eq!(play(&mut o, &mut now, 600, |_| None, quiet).0, 0); + assert!(o.cut.is_none()); + assert!(play(&mut o, &mut now, 400, soft(), owner).1.is_some()); + o.on_pcm(&pcm(10, 0.015)); + o.set_floor(Some(ME), 2); + assert_eq!(play(&mut o, &mut now, 100, |_| None, quiet).0, 0); + } + + /// A stream that never stops never runs dry, so a burst's backlog drains through silence, + /// and the next overflow warns again. + #[test] + fn a_backlog_drains_through_the_streamed_silence() { + let mut o = output(); + let mut now = T0; + o.on_pcm(&pcm(100, 0.002)); + o.on_pcm(&pcm(400, 0.1)); + assert!(o.overflow && o.queue.len() == QUEUE_FRAMES); + let quiet = |_| level(1, true, -127); + assert_eq!(play(&mut o, &mut now, 1_000, live(|_| false), quiet).0, 50); + assert!( + o.queue.len() < BACKLOG_FRAMES && !o.overflow, + "{}", + o.queue.len() + ); + // A quiet frame alone inside speech still plays. + let before = o.queue.len(); + o.on_pcm(&pcm(60, 0.1)); + o.on_pcm(&pcm(20, 0.002)); + o.on_pcm(&pcm(20, 0.1)); + assert_eq!(o.queue.len(), before + 5); + } + /// Input and Output together in link.rs's step order, with a decoded talker: `loud(t)` says /// whether the talker's mic carries speech `t` ms in, and `voice(t)` whether Talos sends. /// How far in the cut came, and the first frame sent after it. diff --git a/crates/buzz-huddle-bridge/tests/loopback.rs b/crates/buzz-huddle-bridge/tests/loopback.rs index 30aaae40e3b..da95e141fe5 100644 --- a/crates/buzz-huddle-bridge/tests/loopback.rs +++ b/crates/buzz-huddle-bridge/tests/loopback.rs @@ -413,8 +413,8 @@ const DESKTOP: Route = Route { /// How long the voice's audio takes from the relay back into the owner's mic on `route`: never on /// headphones, 40 ms from the desktop, 200 ms from the owner's phone. A tail covers this plus the -/// bridge's own NetEQ delay, here at its 200 ms cap, so echo leaks from 65 ms on the desktop (its -/// 300 ms tail) and 380 ms on the phone (600 ms). Probe P measures the real path. +/// bridge's own NetEQ delay, here at its 200 ms cap, so a 600 ms tail lets echo leak from about +/// 380 ms. Probe step 6 (5 Nov) heard the desktop's echo come back 361-508 ms after it played. fn bleed(route: Route) -> Option { (!route.isolated).then(|| ms(if route.here { 40 } else { 200 })) } @@ -549,16 +549,17 @@ async fn the_voice_hears_the_owner_and_says_it_back_paced_without_hearing_its_ow } } -/// The owner talking over the voice cuts it: after 200 ms of energy on the isolated desktop, or -/// 320 ms over the echo on speakers. The rest of that response never plays, and the owner is heard: -/// at once on headphones, else once the echo tail after the cut (desktop 300 ms, phone 600) passes. +/// The owner talking over the voice cuts it: after 320 ms of energy on the isolated desktop, or +/// over the echo on speakers. The rest of that response never plays, and the owner is heard: at +/// once on headphones, else once the 600 ms echo tail after the cut passes. #[tokio::test(start_paused = true)] -async fn talking_over_the_voice_cuts_it_after_200_ms_on_headphones_or_320_ms_on_speakers() { +async fn talking_over_the_voice_cuts_it_after_320_ms_on_headphones_or_speakers() { let headphones = Route { here: true, isolated: true, }; - for (route, need) in [(headphones, 200), (DESKTOP, 320), (Route::default(), 320)] { + let need = 320; + for route in [headphones, DESKTOP, Route::default()] { let (relay, talos) = (&Relay::with(&[(1, OWNER, 0.0)]), &Talos::default()); relay.bleed.set(bleed(route)); talos.repeat.set(4); @@ -580,11 +581,11 @@ async fn talking_over_the_voice_cuts_it_after_200_ms_on_headphones_or_320_ms_on_ assert!(lag >= ms(need), "{route:?}: cut {lag:?} after the barge"); let onset = first(talos, true, OWNER, barged).expect("the owner is heard"); if route.isolated { - // Heard at once: 100 ms to the onset, 200 ms to the cut. + // Heard at once: 100 ms to the onset, 320 ms to the cut. let after = cut - onset; - assert!((ms(80)..ms(130)).contains(&after), "cut {after:?} after"); + assert!((ms(200)..ms(250)).contains(&after), "cut {after:?} after"); } else { - let (after, tail) = (onset - cut, ms(if route.here { 300 } else { 600 })); + let (after, tail) = (onset - cut, ms(600)); let heard = (tail..tail + ms(40)).contains(&after); assert!(heard, "{route:?}: heard {after:?} after the cut"); assert!(lag < ms(need + 300), "cut {lag:?} after the barge");