feat(cli): the api provider speaks the HeyGen audio analysis contract

docs/packages/cli.mdx

@@ -566,7 +566,10 @@ answers:
 | describe | `api`, `apple-sound` |
 
 - `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and
-  `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat,
+  `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono
+  WAV. Channels are averaged unless cancellation removes over 90% of the loudest
+  channel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on each
+  request; the answer may come wrapped in `{ "data": … }`. Every beat,
   downbeat or section boundary it answers moves onto the nearest beat signal
   processing measured, when one is within a quarter beat; the rest are listed in
   `snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went.

packages/cli/src/audio/analysis/providers/api.test.ts

@@ -11,6 +11,7 @@ import { analyzeSignal } from "../analyze.js";
 import { apiProvider, type AudioApiRequest, type AudioApiResponse } from "./api.js";
 
 const RATE = 22_050;
+const UPLOAD_RATE = 16_000;
 
 /** Clicks every half second from 0.25 s: a grid the signal tracker finds. */
 function clicks(seconds: number): AudioSignal {
@@ -27,7 +28,9 @@ function clicks(seconds: number): AudioSignal {
 
 interface Received {
   method: string;
+  apiKey: string | undefined;
   authorization: string | undefined;
+  idempotencyKey: string | undefined;
   request: AudioApiRequest;
   audio: { type: string; bytes: Uint8Array };
 }
@@ -38,6 +41,7 @@ let reply: (request: AudioApiRequest) => { status: number; body: unknown } = ()
   status: 500,
   body: {},
 });
+let enveloped = true;
 
 async function readForm(req: IncomingMessage) {
   const chunks: Buffer[] = [];
@@ -57,9 +61,16 @@ async function readForm(req: IncomingMessage) {
 beforeAll(async () => {
   server = createServer(async (req, res) => {
     const form = await readForm(req);
-    received.push({ method: req.method!, authorization: req.headers.authorization, ...form });
+    received.push({
+      method: req.method!,
+      apiKey: req.headers["x-api-key"] as string | undefined,
+      authorization: req.headers.authorization,
+      idempotencyKey: req.headers["idempotency-key"] as string | undefined,
+      ...form,
+    });
     const { status, body } = reply(form.request);
-    res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body));
+    const sent = status === 200 && enveloped ? { data: body } : body;
+    res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(sent));
   });
   await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));
 });
@@ -68,6 +79,7 @@ afterAll(() => new Promise<void>((done) => server.close(() => done())));
 
 afterEach(() => {
   received = [];
+  enveloped = true;
   delete process.env.HYPERFRAMES_AUDIO_API_URL;
   delete process.env.HYPERFRAMES_AUDIO_API_KEY;
 });
@@ -92,6 +104,34 @@ describe("apiProvider", () => {
     expect(apiProvider.available!()).toEqual({ ok: true });
   });
 
+  it("refuses redirects without sending the API key to another origin", async () => {
+    const forwarded: string[] = [];
+    const destination = createServer((req, res) => {
+      forwarded.push(req.headers["x-api-key"]?.toString() ?? "");
+      res.writeHead(200).end("{}");
+    });
+    await new Promise<void>((done) => destination.listen(0, "127.0.0.1", done));
+    const address = destination.address();
+    if (!address || typeof address === "string") throw new Error("No destination port");
+    const redirect = createServer((_req, res) => {
+      res.writeHead(307, { location: `http://127.0.0.1:${address.port}/` }).end();
+    });
+    await new Promise<void>((done) => redirect.listen(0, "127.0.0.1", done));
+    try {
+      const source = redirect.address();
+      if (!source || typeof source === "string") throw new Error("No redirect port");
+      process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${source.port}/`;
+      process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key";
+      await expect(apiProvider.analyze("beats", clicks(1), {})).rejects.toThrow();
+      expect(forwarded).toEqual([]);
+    } finally {
+      await Promise.all([
+        new Promise<void>((done) => redirect.close(() => done())),
+        new Promise<void>((done) => destination.close(() => done())),
+      ]);
+    }
+  });
+
   it("posts the audio and the capabilities, and snaps each answered beat onto the signal's own", async () => {
     configure();
     const signal = clicks(12);
@@ -103,6 +143,7 @@ describe("apiProvider", () => {
       body: {
         version: 1,
         source: "stand-in",
+        time_resolution_seconds: 1 / 32,
         beats: { times: answered },
         downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 },
       } satisfies AudioApiResponse,
@@ -114,15 +155,18 @@ describe("apiProvider", () => {
 
     expect(received).toHaveLength(1);
     const [call] = received;
-    expect(call).toMatchObject({ method: "POST", authorization: "Bearer test-key" });
+    expect(call).toMatchObject({ method: "POST", apiKey: "test-key", authorization: undefined });
+    expect(call!.idempotencyKey).toMatch(
+      /^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/,
+    );
     expect(call!.request).toEqual({
       version: 1,
       capabilities: ["beats", "downbeats"],
-      audio: { sampleRate: RATE, channels: 1, duration: 12 },
+      audio: { sampleRate: UPLOAD_RATE, channels: 1, duration: 12 },
     });
     expect(call!.audio.type).toBe("audio/wav");
     expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");
-    expect(call!.audio.bytes.length).toBe(44 + 12 * RATE * 2);
+    expect(call!.audio.bytes.length).toBe(44 + 12 * UPLOAD_RATE * 2);
 
     expect(analysis.provenance).toEqual({ beats: "api", downbeats: "api", bpm: "api" });
     const beats = analysis.values.beats as { times: number[]; snap: object; source: string };
@@ -201,6 +245,7 @@ describe("apiProvider", () => {
       signalProvider,
     ]);
     expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);
+    expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2);
     expect(analysis.values.sections).toMatchObject({
       spans: [
         { start: 0, end: beat, label: "intro" },
@@ -235,10 +280,96 @@ describe("apiProvider", () => {
     expect(garbled.warnings[0]).toContain("api failed (the service answered beats in a shape");
   });
 
-  it("never sends the key anywhere but the authorization header", async () => {
+  it("uploads the channels averaged to one at 16 kHz, with nothing above 8 kHz folded back in", async () => {
+    configure();
+    const rate = 44_100;
+    const tone = (hz: number) =>
+      Float32Array.from(
+        { length: 2 * rate },
+        (_, k) => 0.3 * Math.sin((2 * Math.PI * hz * k) / rate),
+      );
+    const [low, side, high] = [tone(440), tone(2_000), tone(9_000)];
+    const left = low.map((v, k) => v + side[k]! + high[k]!);
+    const right = low.map((v, k) => v - side[k]! + high[k]!);
+    reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } });
+    await analyzeSignal(
+      { sampleRate: rate, channels: [left, right] },
+      { capabilities: ["describe"] },
+      [apiProvider],
+    );
+
+    const wav = Buffer.from(received[0]!.audio.bytes);
+    expect([wav.readUInt16LE(22), wav.readUInt32LE(24), wav.readUInt16LE(34)]).toEqual([
+      1,
+      UPLOAD_RATE,
+      16,
+    ]);
+    const samples = Array.from(
+      { length: (wav.length - 44) / 2 },
+      (_, k) => wav.readInt16LE(44 + 2 * k) / 32767,
+    );
+    const level = (hz: number) => {
+      let re = 0;
+      let im = 0;
+      samples.forEach((v, k) => {
+        re += v * Math.cos((2 * Math.PI * hz * k) / UPLOAD_RATE);
+        im += v * Math.sin((2 * Math.PI * hz * k) / UPLOAD_RATE);
+      });
+      return (2 * Math.hypot(re, im)) / samples.length;
+    };
+    expect(level(440)).toBeCloseTo(0.3, 2);
+    expect(level(2_000)).toBeLessThan(0.005);
+    expect(level(7_000)).toBeLessThan(0.005);
+  });
+
+  it.each([
+    { phase: Math.PI, quiet: 1, expected: 0.7 },
+    { phase: Math.PI * 0.9, quiet: 1, expected: 0.7 },
+    { phase: Math.PI, quiet: 0.8, expected: 0.7 },
+    { phase: Math.PI / 2, quiet: 1, expected: 0.7 / Math.sqrt(2) },
+    { phase: 0, quiet: 1, expected: 0.7 },
+  ])(
+    "preserves stereo energy for phase $phase and channel gain $quiet",
+    async ({ phase, quiet, expected }) => {
+      configure();
+      const rate = 44_100;
+      const left = Float32Array.from(
+        { length: rate },
+        (_, k) => 0.7 * quiet * Math.sin((2 * Math.PI * 440 * k) / rate),
+      );
+      const right = Float32Array.from(
+        { length: rate },
+        (_, k) => 0.7 * Math.sin((2 * Math.PI * 440 * k) / rate + phase),
+      );
+      reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } });
+      await analyzeSignal(
+        { sampleRate: rate, channels: [left, right] },
+        { capabilities: ["describe"] },
+        [apiProvider],
+      );
+      const wav = Buffer.from(received[0]!.audio.bytes);
+      expect(wav.readUInt32LE(24)).toBe(UPLOAD_RATE);
+      let energy = 0;
+      for (let k = 0; k < UPLOAD_RATE; k++) energy += (wav.readInt16LE(44 + 2 * k) / 32767) ** 2;
+      expect(Math.sqrt((2 * energy) / UPLOAD_RATE)).toBeCloseTo(expected, 2);
+    },
+  );
+
+  it("reads an answer sent without the data envelope too", async () => {
+    configure();
+    enveloped = false;
+    reply = () => ({ status: 200, body: { version: 1, describe: { text: "bare", tags: [] } } });
+    const analysis = await analyzeSignal(clicks(2), { capabilities: ["describe"] }, [apiProvider]);
+    expect(analysis.values.describe).toEqual({ text: "bare", tags: [] });
+  });
+
+  it("never sends the key anywhere but the x-api-key header", async () => {
     configure();
     reply = () => ({ status: 401, body: { error: { message: "bad key" } } });
     const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]);
     expect(JSON.stringify(analysis)).not.toContain("test-key");
+    const [call] = received;
+    expect(call).toMatchObject({ apiKey: "test-key", authorization: undefined });
+    expect(JSON.stringify(call!.request)).not.toContain("test-key");
   });
 });

packages/cli/src/audio/analysis/providers/api.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import {
   medianInterval,
   rhythm,
@@ -12,14 +13,14 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.
 /**
  * The audio analysis service contract.
  *
- * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with
- * `Authorization: Bearer <HYPERFRAMES_AUDIO_API_KEY>` and a `multipart/form-data` body of two parts:
+ * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with `x-api-key: <HYPERFRAMES_AUDIO_API_KEY>`,
+ * a new `Idempotency-Key` per request, and a `multipart/form-data` body of two parts:
  * - `request`: `application/json`, an {@link AudioApiRequest};
- * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, channels as decoded.
+ * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, mono at 16 kHz.
  *
- * Response: `200` with `application/json`, an {@link AudioApiResponse}. Every time is in seconds from the start of the
- * uploaded audio. A capability the service cannot answer is left out of the response. Any other status is a failure;
- * its body may be `{ "error": { "message": string } }`.
+ * Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read
+ * too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left
+ * out of the response. Any other status is a failure; its body may be `{ "error": { "message": string } }`.
  */
 export interface AudioApiRequest {
   version: 1;
@@ -33,6 +34,8 @@ export interface AudioApiResponse {
   version: 1;
   /** What answered, as the service wants it reported. */
   source?: string;
+  /** The finest step the service's times resolve, in seconds. */
+  time_resolution_seconds?: number;
   beats?: { times: number[] };
   downbeats?: { times: number[]; meter?: number | null };
   sections?: { spans: { start: number; end: number; label?: string }[] };
@@ -43,6 +46,68 @@ const URL_ENV = "HYPERFRAMES_AUDIO_API_URL";
 const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY";
 const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS";
 const DEFAULT_TIMEOUT_MS = 120_000;
+const UPLOAD_RATE = 16_000;
+const SINC_ZEROS = 16;
+const ROLLOFF = 0.9;
+const SINC_STEPS = 512;
+const HANN_SINC = Float32Array.from({ length: SINC_ZEROS * SINC_STEPS + 2 }, (_, k) => {
+  const x = k / SINC_STEPS;
+  if (x >= SINC_ZEROS) return 0;
+  const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
+  return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / SINC_ZEROS));
+});
+
+/** Windowed-sinc resampling with its cutoff just under the lower Nyquist, so nothing above it folds back in. */
+function resample(input: Float32Array, from: number, to: number): Float32Array {
+  if (from === to) return input;
+  const step = from / to;
+  const scale = Math.min(1, to / from) * ROLLOFF;
+  const reach = SINC_ZEROS / scale;
+  const output = new Float32Array(Math.round(input.length / step));
+  for (let i = 0; i < output.length; i++) {
+    const center = i * step;
+    const last = Math.min(input.length - 1, Math.floor(center + reach));
+    let sum = 0;
+    let weights = 0;
+    for (let n = Math.max(0, Math.ceil(center - reach)); n <= last; n++) {
+      const at = Math.abs(n - center) * scale * SINC_STEPS;
+      const k = Math.floor(at);
+      const weight = HANN_SINC[k]! + (at - k) * (HANN_SINC[k + 1]! - HANN_SINC[k]!);
+      sum += input[n]! * weight;
+      weights += weight;
+    }
+    output[i] = weights ? sum / weights : 0;
+  }
+  return output;
+}
+
+/** Preserve a channel when destructive interference removes over 90% of its energy. */
+function monoInput(channels: readonly Float32Array[]): Float32Array {
+  const mono = new Float32Array(channels[0]?.length ?? 0);
+  let loudest: Float32Array = mono;
+  let loudestEnergy = 0;
+  for (const channel of channels) {
+    let energy = 0;
+    for (let i = 0; i < mono.length; i++) {
+      const value = channel[i]!;
+      mono[i]! += value / channels.length;
+      energy += value * value;
+    }
+    if (energy > loudestEnergy) {
+      loudest = channel;
+      loudestEnergy = energy;
+    }
+  }
+  const monoEnergy = mono.reduce((energy, value) => energy + value * value, 0);
+  return monoEnergy < loudestEnergy * 0.1 ? loudest : mono;
+}
+
+function uploaded(signal: AudioSignal): AudioSignal {
+  return {
+    sampleRate: UPLOAD_RATE,
+    channels: [resample(monoInput(signal.channels), signal.sampleRate, UPLOAD_RATE)],
+  };
+}
 
 /** Beats and downbeats come from one request; the others each from their own. */
 const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] =>
@@ -90,16 +155,17 @@ async function failure(response: Response): Promise<Error> {
   return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);
 }
 
-async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {
+async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) {
   const { url, key, timeoutMs } = config();
   if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);
+  const signal = uploaded(analyzed);
   const request: AudioApiRequest = {
     version: 1,
     capabilities,
     audio: {
       sampleRate: signal.sampleRate,
-      channels: signal.channels.length,
-      duration: (signal.channels[0]?.length ?? 0) / signal.sampleRate,
+      channels: 1,
+      duration: signal.channels[0]!.length / signal.sampleRate,
     },
   };
   const form = new FormData();
@@ -111,12 +177,14 @@ async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {
   );
   const response = await fetch(url, {
     method: "POST",
-    headers: { authorization: `Bearer ${key}` },
+    redirect: "error",
+    headers: { "x-api-key": key, "idempotency-key": randomUUID() },
     body: form,
     signal: AbortSignal.timeout(timeoutMs),
   });
   if (!response.ok) throw await failure(response);
-  return (await response.json()) as AudioApiResponse;
+  const body = (await response.json()) as { data?: AudioApiResponse } | null;
+  return (body?.data ?? body ?? {}) as AudioApiResponse;
 }
 
 const requests = new Map<string, (signal: AudioSignal) => Promise<AudioApiResponse>>();