diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 77290194e7..42845571cc 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -566,7 +566,10 @@ answers: | describe | `api`, `apple-sound` | - `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and - `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat, + `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded as 16 kHz mono + WAV. Channels are averaged unless cancellation removes over 90% of the loudest + channel’s energy, in which case that channel is used. Requests carry the key in an `x-api-key` header and a new `Idempotency-Key` on each + request; the answer may come wrapped in `{ "data": … }`. Every beat, downbeat or section boundary it answers moves onto the nearest beat signal processing measured, when one is within a quarter beat; the rest are listed in `snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went. diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts index 8488f0bb76..2fb5d919c2 100644 --- a/packages/cli/src/audio/analysis/providers/api.test.ts +++ b/packages/cli/src/audio/analysis/providers/api.test.ts @@ -11,6 +11,7 @@ import { analyzeSignal } from "../analyze.js"; import { apiProvider, type AudioApiRequest, type AudioApiResponse } from "./api.js"; const RATE = 22_050; +const UPLOAD_RATE = 16_000; /** Clicks every half second from 0.25 s: a grid the signal tracker finds. */ function clicks(seconds: number): AudioSignal { @@ -27,7 +28,9 @@ function clicks(seconds: number): AudioSignal { interface Received { method: string; + apiKey: string | undefined; authorization: string | undefined; + idempotencyKey: string | undefined; request: AudioApiRequest; audio: { type: string; bytes: Uint8Array }; } @@ -38,6 +41,7 @@ let reply: (request: AudioApiRequest) => { status: number; body: unknown } = () status: 500, body: {}, }); +let enveloped = true; async function readForm(req: IncomingMessage) { const chunks: Buffer[] = []; @@ -57,9 +61,16 @@ async function readForm(req: IncomingMessage) { beforeAll(async () => { server = createServer(async (req, res) => { const form = await readForm(req); - received.push({ method: req.method!, authorization: req.headers.authorization, ...form }); + received.push({ + method: req.method!, + apiKey: req.headers["x-api-key"] as string | undefined, + authorization: req.headers.authorization, + idempotencyKey: req.headers["idempotency-key"] as string | undefined, + ...form, + }); const { status, body } = reply(form.request); - res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body)); + const sent = status === 200 && enveloped ? { data: body } : body; + res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(sent)); }); await new Promise((done) => server.listen(0, "127.0.0.1", done)); }); @@ -68,6 +79,7 @@ afterAll(() => new Promise((done) => server.close(() => done()))); afterEach(() => { received = []; + enveloped = true; delete process.env.HYPERFRAMES_AUDIO_API_URL; delete process.env.HYPERFRAMES_AUDIO_API_KEY; }); @@ -92,6 +104,34 @@ describe("apiProvider", () => { expect(apiProvider.available!()).toEqual({ ok: true }); }); + it("refuses redirects without sending the API key to another origin", async () => { + const forwarded: string[] = []; + const destination = createServer((req, res) => { + forwarded.push(req.headers["x-api-key"]?.toString() ?? ""); + res.writeHead(200).end("{}"); + }); + await new Promise((done) => destination.listen(0, "127.0.0.1", done)); + const address = destination.address(); + if (!address || typeof address === "string") throw new Error("No destination port"); + const redirect = createServer((_req, res) => { + res.writeHead(307, { location: `http://127.0.0.1:${address.port}/` }).end(); + }); + await new Promise((done) => redirect.listen(0, "127.0.0.1", done)); + try { + const source = redirect.address(); + if (!source || typeof source === "string") throw new Error("No redirect port"); + process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${source.port}/`; + process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key"; + await expect(apiProvider.analyze("beats", clicks(1), {})).rejects.toThrow(); + expect(forwarded).toEqual([]); + } finally { + await Promise.all([ + new Promise((done) => redirect.close(() => done())), + new Promise((done) => destination.close(() => done())), + ]); + } + }); + it("posts the audio and the capabilities, and snaps each answered beat onto the signal's own", async () => { configure(); const signal = clicks(12); @@ -103,6 +143,7 @@ describe("apiProvider", () => { body: { version: 1, source: "stand-in", + time_resolution_seconds: 1 / 32, beats: { times: answered }, downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 }, } satisfies AudioApiResponse, @@ -114,15 +155,18 @@ describe("apiProvider", () => { expect(received).toHaveLength(1); const [call] = received; - expect(call).toMatchObject({ method: "POST", authorization: "Bearer test-key" }); + expect(call).toMatchObject({ method: "POST", apiKey: "test-key", authorization: undefined }); + expect(call!.idempotencyKey).toMatch( + /^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/, + ); expect(call!.request).toEqual({ version: 1, capabilities: ["beats", "downbeats"], - audio: { sampleRate: RATE, channels: 1, duration: 12 }, + audio: { sampleRate: UPLOAD_RATE, channels: 1, duration: 12 }, }); expect(call!.audio.type).toBe("audio/wav"); expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF"); - expect(call!.audio.bytes.length).toBe(44 + 12 * RATE * 2); + expect(call!.audio.bytes.length).toBe(44 + 12 * UPLOAD_RATE * 2); expect(analysis.provenance).toEqual({ beats: "api", downbeats: "api", bpm: "api" }); const beats = analysis.values.beats as { times: number[]; snap: object; source: string }; @@ -201,6 +245,7 @@ describe("apiProvider", () => { signalProvider, ]); expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]); + expect(new Set(received.map((r) => r.idempotencyKey)).size).toBe(2); expect(analysis.values.sections).toMatchObject({ spans: [ { start: 0, end: beat, label: "intro" }, @@ -235,10 +280,96 @@ describe("apiProvider", () => { expect(garbled.warnings[0]).toContain("api failed (the service answered beats in a shape"); }); - it("never sends the key anywhere but the authorization header", async () => { + it("uploads the channels averaged to one at 16 kHz, with nothing above 8 kHz folded back in", async () => { + configure(); + const rate = 44_100; + const tone = (hz: number) => + Float32Array.from( + { length: 2 * rate }, + (_, k) => 0.3 * Math.sin((2 * Math.PI * hz * k) / rate), + ); + const [low, side, high] = [tone(440), tone(2_000), tone(9_000)]; + const left = low.map((v, k) => v + side[k]! + high[k]!); + const right = low.map((v, k) => v - side[k]! + high[k]!); + reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } }); + await analyzeSignal( + { sampleRate: rate, channels: [left, right] }, + { capabilities: ["describe"] }, + [apiProvider], + ); + + const wav = Buffer.from(received[0]!.audio.bytes); + expect([wav.readUInt16LE(22), wav.readUInt32LE(24), wav.readUInt16LE(34)]).toEqual([ + 1, + UPLOAD_RATE, + 16, + ]); + const samples = Array.from( + { length: (wav.length - 44) / 2 }, + (_, k) => wav.readInt16LE(44 + 2 * k) / 32767, + ); + const level = (hz: number) => { + let re = 0; + let im = 0; + samples.forEach((v, k) => { + re += v * Math.cos((2 * Math.PI * hz * k) / UPLOAD_RATE); + im += v * Math.sin((2 * Math.PI * hz * k) / UPLOAD_RATE); + }); + return (2 * Math.hypot(re, im)) / samples.length; + }; + expect(level(440)).toBeCloseTo(0.3, 2); + expect(level(2_000)).toBeLessThan(0.005); + expect(level(7_000)).toBeLessThan(0.005); + }); + + it.each([ + { phase: Math.PI, quiet: 1, expected: 0.7 }, + { phase: Math.PI * 0.9, quiet: 1, expected: 0.7 }, + { phase: Math.PI, quiet: 0.8, expected: 0.7 }, + { phase: Math.PI / 2, quiet: 1, expected: 0.7 / Math.sqrt(2) }, + { phase: 0, quiet: 1, expected: 0.7 }, + ])( + "preserves stereo energy for phase $phase and channel gain $quiet", + async ({ phase, quiet, expected }) => { + configure(); + const rate = 44_100; + const left = Float32Array.from( + { length: rate }, + (_, k) => 0.7 * quiet * Math.sin((2 * Math.PI * 440 * k) / rate), + ); + const right = Float32Array.from( + { length: rate }, + (_, k) => 0.7 * Math.sin((2 * Math.PI * 440 * k) / rate + phase), + ); + reply = () => ({ status: 200, body: { version: 1, describe: { tags: [] } } }); + await analyzeSignal( + { sampleRate: rate, channels: [left, right] }, + { capabilities: ["describe"] }, + [apiProvider], + ); + const wav = Buffer.from(received[0]!.audio.bytes); + expect(wav.readUInt32LE(24)).toBe(UPLOAD_RATE); + let energy = 0; + for (let k = 0; k < UPLOAD_RATE; k++) energy += (wav.readInt16LE(44 + 2 * k) / 32767) ** 2; + expect(Math.sqrt((2 * energy) / UPLOAD_RATE)).toBeCloseTo(expected, 2); + }, + ); + + it("reads an answer sent without the data envelope too", async () => { + configure(); + enveloped = false; + reply = () => ({ status: 200, body: { version: 1, describe: { text: "bare", tags: [] } } }); + const analysis = await analyzeSignal(clicks(2), { capabilities: ["describe"] }, [apiProvider]); + expect(analysis.values.describe).toEqual({ text: "bare", tags: [] }); + }); + + it("never sends the key anywhere but the x-api-key header", async () => { configure(); reply = () => ({ status: 401, body: { error: { message: "bad key" } } }); const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]); expect(JSON.stringify(analysis)).not.toContain("test-key"); + const [call] = received; + expect(call).toMatchObject({ apiKey: "test-key", authorization: undefined }); + expect(JSON.stringify(call!.request)).not.toContain("test-key"); }); }); diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts index 5659f19206..8db80a854c 100644 --- a/packages/cli/src/audio/analysis/providers/api.ts +++ b/packages/cli/src/audio/analysis/providers/api.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { medianInterval, rhythm, @@ -12,14 +13,14 @@ import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared. /** * The audio analysis service contract. * - * Request: `POST `, exactly as configured, with - * `Authorization: Bearer ` and a `multipart/form-data` body of two parts: + * Request: `POST `, exactly as configured, with `x-api-key: `, + * a new `Idempotency-Key` per request, and a `multipart/form-data` body of two parts: * - `request`: `application/json`, an {@link AudioApiRequest}; - * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, channels as decoded. + * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, mono at 16 kHz. * - * Response: `200` with `application/json`, an {@link AudioApiResponse}. Every time is in seconds from the start of the - * uploaded audio. A capability the service cannot answer is left out of the response. Any other status is a failure; - * its body may be `{ "error": { "message": string } }`. + * Response: `200` with `application/json`, `{ "data": AudioApiResponse }`; a bare {@link AudioApiResponse} is read + * too. Every time is in seconds from the start of the uploaded audio. A capability the service cannot answer is left + * out of the response. Any other status is a failure; its body may be `{ "error": { "message": string } }`. */ export interface AudioApiRequest { version: 1; @@ -33,6 +34,8 @@ export interface AudioApiResponse { version: 1; /** What answered, as the service wants it reported. */ source?: string; + /** The finest step the service's times resolve, in seconds. */ + time_resolution_seconds?: number; beats?: { times: number[] }; downbeats?: { times: number[]; meter?: number | null }; sections?: { spans: { start: number; end: number; label?: string }[] }; @@ -43,6 +46,68 @@ const URL_ENV = "HYPERFRAMES_AUDIO_API_URL"; const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY"; const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS"; const DEFAULT_TIMEOUT_MS = 120_000; +const UPLOAD_RATE = 16_000; +const SINC_ZEROS = 16; +const ROLLOFF = 0.9; +const SINC_STEPS = 512; +const HANN_SINC = Float32Array.from({ length: SINC_ZEROS * SINC_STEPS + 2 }, (_, k) => { + const x = k / SINC_STEPS; + if (x >= SINC_ZEROS) return 0; + const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x); + return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / SINC_ZEROS)); +}); + +/** Windowed-sinc resampling with its cutoff just under the lower Nyquist, so nothing above it folds back in. */ +function resample(input: Float32Array, from: number, to: number): Float32Array { + if (from === to) return input; + const step = from / to; + const scale = Math.min(1, to / from) * ROLLOFF; + const reach = SINC_ZEROS / scale; + const output = new Float32Array(Math.round(input.length / step)); + for (let i = 0; i < output.length; i++) { + const center = i * step; + const last = Math.min(input.length - 1, Math.floor(center + reach)); + let sum = 0; + let weights = 0; + for (let n = Math.max(0, Math.ceil(center - reach)); n <= last; n++) { + const at = Math.abs(n - center) * scale * SINC_STEPS; + const k = Math.floor(at); + const weight = HANN_SINC[k]! + (at - k) * (HANN_SINC[k + 1]! - HANN_SINC[k]!); + sum += input[n]! * weight; + weights += weight; + } + output[i] = weights ? sum / weights : 0; + } + return output; +} + +/** Preserve a channel when destructive interference removes over 90% of its energy. */ +function monoInput(channels: readonly Float32Array[]): Float32Array { + const mono = new Float32Array(channels[0]?.length ?? 0); + let loudest: Float32Array = mono; + let loudestEnergy = 0; + for (const channel of channels) { + let energy = 0; + for (let i = 0; i < mono.length; i++) { + const value = channel[i]!; + mono[i]! += value / channels.length; + energy += value * value; + } + if (energy > loudestEnergy) { + loudest = channel; + loudestEnergy = energy; + } + } + const monoEnergy = mono.reduce((energy, value) => energy + value * value, 0); + return monoEnergy < loudestEnergy * 0.1 ? loudest : mono; +} + +function uploaded(signal: AudioSignal): AudioSignal { + return { + sampleRate: UPLOAD_RATE, + channels: [resample(monoInput(signal.channels), signal.sampleRate, UPLOAD_RATE)], + }; +} /** Beats and downbeats come from one request; the others each from their own. */ const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] => @@ -90,16 +155,17 @@ async function failure(response: Response): Promise { return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`); } -async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) { +async function ask(analyzed: AudioSignal, capabilities: AudioApiCapability[]) { const { url, key, timeoutMs } = config(); if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`); + const signal = uploaded(analyzed); const request: AudioApiRequest = { version: 1, capabilities, audio: { sampleRate: signal.sampleRate, - channels: signal.channels.length, - duration: (signal.channels[0]?.length ?? 0) / signal.sampleRate, + channels: 1, + duration: signal.channels[0]!.length / signal.sampleRate, }, }; const form = new FormData(); @@ -111,12 +177,14 @@ async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) { ); const response = await fetch(url, { method: "POST", - headers: { authorization: `Bearer ${key}` }, + redirect: "error", + headers: { "x-api-key": key, "idempotency-key": randomUUID() }, body: form, signal: AbortSignal.timeout(timeoutMs), }); if (!response.ok) throw await failure(response); - return (await response.json()) as AudioApiResponse; + const body = (await response.json()) as { data?: AudioApiResponse } | null; + return (body?.data ?? body ?? {}) as AudioApiResponse; } const requests = new Map Promise>();