From 018bd98011e7eb2855814c72f59226e360bd28cc Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 1 Nov 2026 13:14:24 +0100 Subject: [PATCH 1/5] feat(android): stream transcription of live pcm audio doStream pipes 16 kHz mono 16-bit PCM into ML Kit's speech recognition through a ParcelFileDescriptor, and sends partial text as deltas and final results as segments timed from their word timings. Part of #363 Co-Authored-By: Claude Opus 5.5 --- harness/android/transcription.harness.ts | 54 +++++- .../callstack/ai/android/AndroidAIModule.kt | 82 +++++++++ .../com/callstack/ai/android/Transcription.kt | 34 ++++ packages/android/src/NativeAndroidAI.ts | 23 +++ packages/android/src/__tests__/ai-sdk.test.ts | 88 ++++++++++ packages/android/src/ai-sdk.ts | 158 ++++++++++++++++++ .../docs/docs/android/transcription.md | 34 +++- 7 files changed, 469 insertions(+), 4 deletions(-) diff --git a/harness/android/transcription.harness.ts b/harness/android/transcription.harness.ts index 035b2d56..d38d46df 100644 --- a/harness/android/transcription.harness.ts +++ b/harness/android/transcription.harness.ts @@ -1,11 +1,26 @@ import { android } from '@react-native-ai/android' -import { experimental_transcribe as transcribe } from 'ai' +import { + experimental_streamTranscribe as streamTranscribe, + experimental_transcribe as transcribe, +} from 'ai' import { describe, expect, it } from 'react-native-harness' -import { base64ToBytes, speech } from '../fixtures' +import { base64ToBytes, speech, speechPcm8k } from '../fixtures' const model = android.transcriptionModel({ language: 'en-US', mode: 'basic' }) +// The 8 kHz fixture at the 16 kHz ML Kit streams, by linear interpolation +function to16k(pcm8k: Uint8Array) { + const input = new Int16Array(pcm8k.buffer, pcm8k.byteOffset, pcm8k.length / 2) + const output = new Int16Array(input.length * 2) + input.forEach((sample, index) => { + const next = input[index + 1] ?? sample + output[index * 2] = sample + output[index * 2 + 1] = (sample + next) >> 1 + }) + return new Uint8Array(output.buffer) +} + describe('android transcription', () => { it('reports recognizer availability', async () => { const info = await model.getModelInfo() @@ -33,4 +48,39 @@ describe('android transcription', () => { expect(text).toMatch(/quick brown fox/i) expect(text).toMatch(/paris/i) }) + + it('streams PCM with deltas and timed segments', async () => { + const pcm = to16k(base64ToBytes(speechPcm8k)) + // 100 ms chunks, as a microphone sends them + const audio = new ReadableStream({ + start(controller) { + for (let offset = 0; offset < pcm.length; offset += 3200) { + controller.enqueue(pcm.subarray(offset, offset + 3200)) + } + controller.close() + }, + }) + + const result = streamTranscribe({ + model, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + audio, + }) + let deltas = '' + const segments: { + text: string + startSecond?: number + endSecond?: number + }[] = [] + for await (const part of result.fullStream) { + if (part.type === 'transcript-delta') deltas += part.delta + if (part.type === 'transcript-final') segments.push(part) + } + console.log('[android] streamed segments:', JSON.stringify(segments)) + + expect(await result.text).toMatch(/quick brown fox/i) + expect(deltas).toMatch(/quick brown fox/i) + expect(segments.length).toBeGreaterThan(0) + expect(segments[0]!.endSecond).toBeGreaterThan(segments[0]!.startSecond!) + }) }) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt index 089c6ff9..37f31a70 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -1,5 +1,7 @@ package com.callstack.ai.android +import android.os.ParcelFileDescriptor +import android.util.Base64 import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -8,6 +10,8 @@ import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.ReadableType import com.facebook.react.bridge.WritableMap import com.google.mlkit.genai.common.FunctionCall +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers @@ -16,6 +20,7 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import java.io.OutputStream import java.util.concurrent.ConcurrentHashMap class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISpec(reactContext) { @@ -41,6 +46,11 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp // Generations and transcriptions, by id, so they can be cancelled private val activeRuns = ConcurrentHashMap() + // Streamed transcriptions: the pipe audio is written to, and the text once the stream ends + private class TranscriptionStream(val audio: OutputStream, val text: CompletableDeferred) + + private val transcriptionStreams = ConcurrentHashMap() + private fun Promise.rejectWithCode(error: Exception) { reject(AndroidAIErrors.codeFor(error), error.message ?: "Unknown ML Kit error", error) } @@ -159,6 +169,7 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp override fun cancel(runId: String) { activeRuns.remove(runId)?.cancel() + transcriptionStreams.remove(runId)?.audio?.close() } override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) = @@ -172,6 +183,77 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp promise.resolve(transcription.transcribe(audio, TranscriptionConfig.from(options))) } + override fun startTranscriptionStream(streamId: String, options: ReadableMap, promise: Promise) { + if (transcriptionStreams.containsKey(streamId)) { + promise.reject(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId") + return + } + val (read, write) = ParcelFileDescriptor.createPipe() + val stream = TranscriptionStream(ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) + transcriptionStreams[streamId] = stream + + var segmentIndex = 0 + // A failure is reported by the next append, or by finish, which removes the stream + launchRun(streamId, { error -> + stream.text.completeExceptionally(error) + stream.audio.close() + }) { + val text = transcription.transcribeStream(read, TranscriptionConfig.from(options)) { response -> + emitOnTranscriptionEvent( + Arguments.createMap().apply { + putString("streamId", streamId) + putInt("segmentIndex", segmentIndex) + when (response) { + is SpeechRecognizerResponse.PartialTextResponse -> { + putString("type", "partial") + putString("text", response.text) + } + is SpeechRecognizerResponse.FinalTextResponse -> { + putString("type", "final") + putString("text", response.text) + putArray( + "words", + Arguments.createArray().apply { + response.wordTimings.forEach { word -> + pushMap( + Arguments.createMap().apply { + putString("text", word.text) + putDouble("startMs", word.timestampMs.toDouble()) + }, + ) + } + }, + ) + segmentIndex++ + } + else -> {} + } + }, + ) + } + stream.text.complete(text) + } + promise.resolve(null) + } + + override fun appendTranscriptionAudio(streamId: String, pcmBase64: String, promise: Promise) = + launch(promise) { + val stream = transcriptionStreams[streamId] + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") + // Recognition ended early, so rethrow why + if (stream.text.isCompleted) stream.text.await() + stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT)) + } + + override fun finishTranscriptionStream(streamId: String, promise: Promise) = + launch(promise) { + val stream = transcriptionStreams.remove(streamId) + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") + // Closing the pipe ends the audio, so recognition finishes what it has heard + stream.audio.close() + stream.text.await() + } + override fun getEmbeddingModelInfo(promise: Promise) = launch(promise) { val info = embeddings.getModelInfo() diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt index 9f5ba38c..f0e75264 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt @@ -1,6 +1,7 @@ package com.callstack.ai.android import android.content.Context +import android.os.ParcelFileDescriptor import android.util.Base64 import com.facebook.react.bridge.ReadableMap import com.google.mlkit.genai.common.FeatureStatus @@ -10,6 +11,7 @@ import com.google.mlkit.genai.speechrecognition.SpeechRecognizer import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.recognitionOptions import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -103,4 +105,36 @@ class Transcription( file.delete() } } + + /** + * Transcribes 16 kHz mono 16-bit PCM read from [audio] until it is closed, reporting each + * partial and final result with its word timings. Returns the whole text. + */ + suspend fun transcribeStream( + audio: ParcelFileDescriptor, + config: TranscriptionConfig, + onResult: (response: SpeechRecognizerResponse) -> Unit, + ): String { + prepare(config) + AudioSource.fromPfd(audio, AudioSource.Mode.STREAMING).use { source -> + val text = StringBuilder() + val request = speechRecognizerRequest { + audioSource = source + recognitionOptions { enableWordTiming = true } + } + client(config).startRecognition(request).collect { response -> + when (response) { + is SpeechRecognizerResponse.PartialTextResponse -> onResult(response) + is SpeechRecognizerResponse.FinalTextResponse -> { + if (text.isNotEmpty() && response.text.isNotEmpty()) text.append(' ') + text.append(response.text) + onResult(response) + } + is SpeechRecognizerResponse.ErrorResponse -> throw response.e + else -> {} + } + } + return text.toString() + } + } } diff --git a/packages/android/src/NativeAndroidAI.ts b/packages/android/src/NativeAndroidAI.ts index 083bea12..86eced17 100644 --- a/packages/android/src/NativeAndroidAI.ts +++ b/packages/android/src/NativeAndroidAI.ts @@ -105,6 +105,16 @@ export interface NativeTranscriptionOptions { mode?: string } +export interface TranscriptionEvent { + streamId: string + /** `partial` text of the segment so far, or its `final` text */ + type: string + segmentIndex: number + text: string + /** Final results only: each word and when it starts */ + words?: { text: string; startMs: number }[] +} + export interface NativeEmbeddingModelInfo { availability: string dimension?: number @@ -148,6 +158,18 @@ export interface Spec extends TurboModule { options: NativeTranscriptionOptions ): Promise + /** + * Starts transcribing 16 kHz mono 16-bit PCM appended to the stream, with + * results as `onTranscriptionEvent`. Cancel it with `cancel(streamId)` + */ + startTranscriptionStream( + streamId: string, + options: NativeTranscriptionOptions + ): Promise + appendTranscriptionAudio(streamId: string, pcmBase64: string): Promise + /** Ends the audio and resolves with the whole text */ + finishTranscriptionStream(streamId: string): Promise + getEmbeddingModelInfo(): Promise prepareEmbedding(): Promise embed(values: string[], taskType?: string): Promise @@ -171,6 +193,7 @@ export interface Spec extends TurboModule { onStreamEvent: CodegenTypes.EventEmitter onDownloadProgress: CodegenTypes.EventEmitter + onTranscriptionEvent: CodegenTypes.EventEmitter } const NativeAndroidAI: Spec | null = diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts index c93cf2a1..cf6808d5 100644 --- a/packages/android/src/__tests__/ai-sdk.test.ts +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -20,6 +20,7 @@ function createEmitter() { const onStreamEvent = createEmitter() const onDownloadProgress = createEmitter() +const onTranscriptionEvent = createEmitter() let generateTextResult: Record = { text: 'ok', @@ -50,6 +51,15 @@ const getTranscriptionModelInfo = mock( availability: 'downloadable', }) ) +const startTranscriptionStream = mock( + async (_streamId: string, _options: unknown) => {} +) +const appendTranscriptionAudio = mock( + async (_streamId: string, _pcmBase64: string) => {} +) +const finishTranscriptionStream = mock( + async (_streamId: string) => 'hello world' +) const prepareTask = mock(async (_task: string, _options: unknown) => {}) const getTaskModelInfo = mock( async ( @@ -85,6 +95,10 @@ mock.module('react-native', () => ({ prepareTranscription, prepareEmbedding, embed, + startTranscriptionStream, + appendTranscriptionAudio, + finishTranscriptionStream, + onTranscriptionEvent, prepareTask, getTaskModelInfo, runTask, @@ -732,3 +746,77 @@ describe('task models', () => { }) }) }) + +describe('streaming transcription', () => { + const model = createAndroidProvider().transcriptionModel({ + language: 'en-US', + }) + + const audioStream = (chunks: Uint8Array[]) => + new ReadableStream({ + start(controller) { + chunks.forEach((chunk) => controller.enqueue(chunk)) + controller.close() + }, + }) + + test('sends deltas, final segments with timings and the whole text', async () => { + appendTranscriptionAudio.mockImplementationOnce(async (streamId) => { + const event = (type: string, text: string, words?: unknown) => + onTranscriptionEvent.emit({ + streamId, + type, + segmentIndex: 0, + text, + words, + }) + event('partial', 'hel') + event('partial', 'hello') + event('final', 'hello world', [ + { text: 'hello', startMs: 100 }, + { text: 'world', startMs: 600 }, + ]) + }) + await model.prepare() + + const { stream } = await model.doStream({ + audio: audioStream([new Uint8Array([1, 2, 3]), new Uint8Array([4])]), + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const parts: any[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + + // The odd byte waits for the next chunk + expect(appendTranscriptionAudio.mock.calls.map((call) => call[1])).toEqual([ + 'AQI=', + 'AwQ=', + ]) + expect( + parts + .filter((part) => part.type === 'transcript-delta') + .map((part) => part.delta) + ).toEqual(['hel', 'lo', ' world']) + expect(parts.find((part) => part.type === 'transcript-final')).toEqual({ + type: 'transcript-final', + id: 'segment-0', + text: 'hello world', + startSecond: 0.1, + endSecond: 0.6, + }) + expect(parts.at(-1)).toMatchObject({ type: 'finish', text: 'hello world' }) + }) + + test('rejects audio ML Kit cannot stream', async () => { + await expect( + model.doStream({ + audio: audioStream([]), + inputAudioFormat: { type: 'audio/pcm', rate: 48000 }, + } as any) + ).rejects.toThrow('16 kHz') + }) +}) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index 7630eecd..b7fd0e89 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -2,6 +2,9 @@ import type { EmbeddingModelV4, EmbeddingModelV4CallOptions, EmbeddingModelV4Result, + Experimental_TranscriptionModelV4StreamOptions as TranscriptionModelV4StreamOptions, + Experimental_TranscriptionModelV4StreamPart as TranscriptionModelV4StreamPart, + Experimental_TranscriptionModelV4StreamResult as TranscriptionModelV4StreamResult, LanguageModelV4, LanguageModelV4CallOptions, LanguageModelV4Content, @@ -15,6 +18,7 @@ import type { TranscriptionModelV4, TranscriptionModelV4CallOptions, } from '@ai-sdk/provider' +import { UnsupportedFunctionalityError } from '@ai-sdk/provider' import { generateId } from '@ai-sdk/provider-utils' import { Platform } from 'react-native' @@ -979,6 +983,160 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { abortSignal?.removeEventListener('abort', onAbort) } } + + /** + * Transcribes live 16 kHz mono 16-bit PCM. A segment's text grows until a + * pause ends it, and its final text comes with word timings + */ + async doStream( + options: TranscriptionModelV4StreamOptions + ): Promise { + const { inputAudioFormat, abortSignal } = options + if ( + inputAudioFormat.type !== 'audio/pcm' || + (inputAudioFormat.rate ?? 16000) !== 16000 + ) { + throw new UnsupportedFunctionalityError({ + functionality: `streaming transcription of ${inputAudioFormat.type}`, + message: + 'ML Kit streams 16 kHz mono 16-bit PCM audio (audio/pcm, rate 16000).', + }) + } + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + if (!this.prepared) { + console.warn(notPreparedWarning) + await this.prepare() + } + + const native = getNativeAndroidAI() + const streamId = generateId() + const modelId = this.modelId + const transcriptionOptions = this.options + const audio = options.audio.getReader() + let listener: { remove(): void } | undefined + const cleanup = () => { + listener?.remove() + abortSignal?.removeEventListener('abort', onAbort) + } + function onAbort() { + native.cancel(streamId) + audio.cancel().catch(() => {}) + } + + const stream = new ReadableStream({ + async start(controller) { + controller.enqueue({ type: 'stream-start', warnings: [] }) + controller.enqueue({ + type: 'response-metadata', + timestamp: new Date(), + modelId, + }) + + const segments: { + text: string + startSecond: number + endSecond: number + }[] = [] + // Text sent so far of each segment, to send what the next result adds + const sentText = new Map() + listener = native.onTranscriptionEvent((event) => { + if (event.streamId !== streamId) { + return + } + const id = `segment-${event.segmentIndex}` + const sent = sentText.get(event.segmentIndex) ?? '' + const text = event.text.trim() + if (text.startsWith(sent) && text.length > sent.length) { + const separator = sent === '' && event.segmentIndex > 0 ? ' ' : '' + controller.enqueue({ + type: 'transcript-delta', + id, + delta: separator + text.slice(sent.length), + }) + sentText.set(event.segmentIndex, text) + } + if (event.type === 'final' && text) { + // ML Kit times when each word starts, so a segment ends as its last word starts + const words = event.words ?? [] + const segment = { + text, + startSecond: (words[0]?.startMs ?? 0) / 1000, + endSecond: (words.at(-1)?.startMs ?? 0) / 1000, + } + segments.push(segment) + controller.enqueue({ type: 'transcript-final', id, ...segment }) + } + }) + abortSignal?.addEventListener('abort', onAbort) + + try { + await native.startTranscriptionStream(streamId, transcriptionOptions) + // A sample can be split across chunks + let carry: number | undefined + while (true) { + const { done, value } = await audio.read() + if (done || abortSignal?.aborted) { + break + } + let chunk = + typeof value === 'string' + ? Uint8Array.from(atob(value), (char) => char.charCodeAt(0)) + : value + if (carry !== undefined) { + const joined = new Uint8Array(chunk.length + 1) + joined[0] = carry + joined.set(chunk, 1) + chunk = joined + carry = undefined + } + if (chunk.length % 2 === 1) { + carry = chunk[chunk.length - 1] + chunk = chunk.subarray(0, chunk.length - 1) + } + if (chunk.length > 0) { + await native.appendTranscriptionAudio( + streamId, + uint8ArrayToBase64(chunk) + ) + } + } + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + const text = await native.finishTranscriptionStream(streamId) + controller.enqueue({ + type: 'finish', + text, + segments, + language: transcriptionOptions.locale?.split(/[-_]/)[0], + durationInSeconds: undefined, + }) + controller.close() + } catch (error) { + native.cancel(streamId) + // Stops a live source, such as a microphone, from capturing on + audio.cancel(error).catch(() => {}) + controller.error( + abortSignal?.aborted + ? getAbortReason(abortSignal) + : toAndroidAIError(error) + ) + } finally { + cleanup() + audio.releaseLock() + } + }, + cancel() { + onAbort() + cleanup() + }, + }) + + return { stream, response: { timestamp: new Date(), modelId } } + } } class AndroidEmbeddingModel implements EmbeddingModelV4 { diff --git a/website/src/content/docs/docs/android/transcription.md b/website/src/content/docs/docs/android/transcription.md index 84094890..5c8e6187 100644 --- a/website/src/content/docs/docs/android/transcription.md +++ b/website/src/content/docs/docs/android/transcription.md @@ -18,8 +18,38 @@ const { text } = await transcribe({ ``` ML Kit decodes the file on the device. It returns the text only: word timings -come with streamed audio, which this model doesn't take yet, so `segments` is -empty. +come with streamed audio, so `segments` is empty. + +## Streaming + +Stream live audio, such as microphone chunks, to +`experimental_streamTranscribe` as 16 kHz mono 16-bit PCM: + +```typescript +import { experimental_streamTranscribe as streamTranscribe } from 'ai' + +const result = streamTranscribe({ + model, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + audio: microphoneChunks, // ReadableStream +}) + +for await (const part of result.fullStream) { + if (part.type === 'transcript-delta') { + process(part.delta) + } + if (part.type === 'transcript-final') { + console.log(part.text, part.startSecond, part.endSecond) + } +} +``` + +- Deltas append to the current segment. When the speaker pauses, the segment + closes and arrives as a `transcript-final` part. +- ML Kit times when each word starts, so a segment's `endSecond` is when its + last word starts. +- Other sample rates are rejected: resample to 16 kHz first. +- Abort the call, or cancel the stream, to stop. ## Options From 5521dba6b1596762422a89f88f64b6bf3be961be Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 1 Nov 2026 13:48:09 +0100 Subject: [PATCH 2/5] fix(android): clean up streamed transcriptions and end cancelled streams Close both ends of a stream's pipe when recognition ends, fails or the module is invalidated, and drop audio appended after recognition ended on its own. Cancelling the output stream now ends the audio loop without a result. Finals without word timings carry no times, and finish reports the language the recognizer used. Co-Authored-By: Claude Opus 5.5 --- harness/android/transcription.harness.ts | 22 ++-- .../callstack/ai/android/AndroidAIModule.kt | 51 ++++++-- .../com/callstack/ai/android/Transcription.kt | 4 + packages/android/src/NativeAndroidAI.ts | 10 +- packages/android/src/__tests__/ai-sdk.test.ts | 66 +++++++++- packages/android/src/ai-sdk.ts | 119 ++++++++++-------- 6 files changed, 197 insertions(+), 75 deletions(-) diff --git a/harness/android/transcription.harness.ts b/harness/android/transcription.harness.ts index d38d46df..58c7a96c 100644 --- a/harness/android/transcription.harness.ts +++ b/harness/android/transcription.harness.ts @@ -9,15 +9,19 @@ import { base64ToBytes, speech, speechPcm8k } from '../fixtures' const model = android.transcriptionModel({ language: 'en-US', mode: 'basic' }) -// The 8 kHz fixture at the 16 kHz ML Kit streams, by linear interpolation +// The 8 kHz fixture at the 16 kHz ML Kit streams, by linear interpolation of +// its little-endian samples function to16k(pcm8k: Uint8Array) { - const input = new Int16Array(pcm8k.buffer, pcm8k.byteOffset, pcm8k.length / 2) - const output = new Int16Array(input.length * 2) - input.forEach((sample, index) => { - const next = input[index + 1] ?? sample + const view = new DataView(pcm8k.buffer, pcm8k.byteOffset, pcm8k.byteLength) + const sampleCount = Math.floor(pcm8k.byteLength / 2) + const output = new Int16Array(sampleCount * 2) + for (let index = 0; index < sampleCount; index++) { + const sample = view.getInt16(index * 2, true) + const next = + index + 1 < sampleCount ? view.getInt16((index + 1) * 2, true) : sample output[index * 2] = sample output[index * 2 + 1] = (sample + next) >> 1 - }) + } return new Uint8Array(output.buffer) } @@ -81,6 +85,10 @@ describe('android transcription', () => { expect(await result.text).toMatch(/quick brown fox/i) expect(deltas).toMatch(/quick brown fox/i) expect(segments.length).toBeGreaterThan(0) - expect(segments[0]!.endSecond).toBeGreaterThan(segments[0]!.startSecond!) + // A one-word segment ends as it starts + const [first] = segments + if (first!.startSecond !== undefined) { + expect(first!.endSecond).toBeGreaterThanOrEqual(first!.startSecond) + } }) }) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt index 37f31a70..8165a0a1 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -46,8 +46,19 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp // Generations and transcriptions, by id, so they can be cancelled private val activeRuns = ConcurrentHashMap() - // Streamed transcriptions: the pipe audio is written to, and the text once the stream ends - private class TranscriptionStream(val audio: OutputStream, val text: CompletableDeferred) + // Streamed transcriptions: both ends of the pipe audio is written to, and the result once the + // stream ends. Both ends close when recognition ends, so a stream JS never finishes holds no + // file descriptors + private class TranscriptionStream( + val read: ParcelFileDescriptor, + val audio: OutputStream, + val result: CompletableDeferred, + ) { + fun close() { + runCatching { audio.close() } + runCatching { read.close() } + } + } private val transcriptionStreams = ConcurrentHashMap() @@ -169,7 +180,7 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp override fun cancel(runId: String) { activeRuns.remove(runId)?.cancel() - transcriptionStreams.remove(runId)?.audio?.close() + transcriptionStreams.remove(runId)?.close() } override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) = @@ -189,16 +200,18 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp return } val (read, write) = ParcelFileDescriptor.createPipe() - val stream = TranscriptionStream(ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) + val stream = TranscriptionStream(read, ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) transcriptionStreams[streamId] = stream + // The collector calls back one response at a time, so the count needs no lock var segmentIndex = 0 // A failure is reported by the next append, or by finish, which removes the stream launchRun(streamId, { error -> - stream.text.completeExceptionally(error) - stream.audio.close() + stream.result.completeExceptionally(error) + stream.close() }) { - val text = transcription.transcribeStream(read, TranscriptionConfig.from(options)) { response -> + val config = TranscriptionConfig.from(options) + val text = transcription.transcribeStream(read, config) { response -> emitOnTranscriptionEvent( Arguments.createMap().apply { putString("streamId", streamId) @@ -231,7 +244,13 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp }, ) } - stream.text.complete(text) + stream.close() + stream.result.complete( + Arguments.createMap().apply { + putString("text", text) + putString("language", transcription.languageOf(config)) + }, + ) } promise.resolve(null) } @@ -240,8 +259,11 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp launch(promise) { val stream = transcriptionStreams[streamId] ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") - // Recognition ended early, so rethrow why - if (stream.text.isCompleted) stream.text.await() + // Recognition ended early: rethrow why, or drop audio it can no longer hear + if (stream.result.isCompleted) { + stream.result.await() + return@launch null + } stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT)) } @@ -250,8 +272,8 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp val stream = transcriptionStreams.remove(streamId) ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") // Closing the pipe ends the audio, so recognition finishes what it has heard - stream.audio.close() - stream.text.await() + runCatching { stream.audio.close() } + stream.result.await() } override fun getEmbeddingModelInfo(promise: Promise) = @@ -299,6 +321,11 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp } override fun invalidate() { + transcriptionStreams.values.forEach { stream -> + stream.close() + stream.result.cancel() + } + transcriptionStreams.clear() activeRuns.values.forEach { it.cancel() } activeRuns.clear() scope.cancel() diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt index f0e75264..fdf879f6 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt @@ -57,6 +57,10 @@ class Transcription( "Not a BCP 47 language tag: $tag. Use a tag such as en-US", ) + /** The language the recognizer transcribes, as a BCP 47 tag. */ + fun languageOf(config: TranscriptionConfig): String = + (config.locale?.let(::locale) ?: Locale.getDefault()).toLanguageTag() + suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo = ModelInfo(availability = availabilityOf(client(config).checkStatus())) diff --git a/packages/android/src/NativeAndroidAI.ts b/packages/android/src/NativeAndroidAI.ts index 86eced17..e37ff021 100644 --- a/packages/android/src/NativeAndroidAI.ts +++ b/packages/android/src/NativeAndroidAI.ts @@ -115,6 +115,12 @@ export interface TranscriptionEvent { words?: { text: string; startMs: number }[] } +export interface NativeStreamTranscriptionResult { + text: string + /** BCP 47 locale the recognizer transcribed, such as en-US */ + language: string +} + export interface NativeEmbeddingModelInfo { availability: string dimension?: number @@ -168,7 +174,9 @@ export interface Spec extends TurboModule { ): Promise appendTranscriptionAudio(streamId: string, pcmBase64: string): Promise /** Ends the audio and resolves with the whole text */ - finishTranscriptionStream(streamId: string): Promise + finishTranscriptionStream( + streamId: string + ): Promise getEmbeddingModelInfo(): Promise prepareEmbedding(): Promise diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts index cf6808d5..b2664c81 100644 --- a/packages/android/src/__tests__/ai-sdk.test.ts +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -57,9 +57,10 @@ const startTranscriptionStream = mock( const appendTranscriptionAudio = mock( async (_streamId: string, _pcmBase64: string) => {} ) -const finishTranscriptionStream = mock( - async (_streamId: string) => 'hello world' -) +const finishTranscriptionStream = mock(async (_streamId: string) => ({ + text: 'hello world', + language: 'en-US', +})) const prepareTask = mock(async (_task: string, _options: unknown) => {}) const getTaskModelInfo = mock( async ( @@ -808,7 +809,11 @@ describe('streaming transcription', () => { startSecond: 0.1, endSecond: 0.6, }) - expect(parts.at(-1)).toMatchObject({ type: 'finish', text: 'hello world' }) + expect(parts.at(-1)).toMatchObject({ + type: 'finish', + text: 'hello world', + language: 'en', + }) }) test('rejects audio ML Kit cannot stream', async () => { @@ -819,4 +824,57 @@ describe('streaming transcription', () => { } as any) ).rejects.toThrow('16 kHz') }) + + test('sends a final without word timings without times', async () => { + appendTranscriptionAudio.mockImplementationOnce(async (streamId) => { + onTranscriptionEvent.emit({ + streamId, + type: 'final', + segmentIndex: 0, + text: 'hi', + words: [], + }) + }) + + const { stream } = await model.doStream({ + audio: audioStream([new Uint8Array([1, 2])]), + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const parts: any[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + + expect(parts.find((part) => part.type === 'transcript-final')).toEqual({ + type: 'transcript-final', + id: 'segment-0', + text: 'hi', + }) + expect(parts.at(-1)).toMatchObject({ type: 'finish', segments: [] }) + }) + + test('ends a cancelled live stream without finishing it', async () => { + finishTranscriptionStream.mockClear() + // A microphone stream never closes on its own + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2])) + }, + }) + + const { stream } = await model.doStream({ + audio, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const reader = stream.getReader() + await reader.read() + await reader.cancel() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(cancel).toHaveBeenCalled() + expect(finishTranscriptionStream).not.toHaveBeenCalled() + }) }) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index b7fd0e89..bc841e9b 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -1016,6 +1016,8 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { const transcriptionOptions = this.options const audio = options.audio.getReader() let listener: { remove(): void } | undefined + // Set when the consumer cancels the stream, which ends it without a result + let cancelled = false const cleanup = () => { listener?.remove() abortSignal?.removeEventListener('abort', onAbort) @@ -1027,57 +1029,64 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { const stream = new ReadableStream({ async start(controller) { - controller.enqueue({ type: 'stream-start', warnings: [] }) - controller.enqueue({ - type: 'response-metadata', - timestamp: new Date(), - modelId, - }) + try { + controller.enqueue({ type: 'stream-start', warnings: [] }) + controller.enqueue({ + type: 'response-metadata', + timestamp: new Date(), + modelId, + }) - const segments: { - text: string - startSecond: number - endSecond: number - }[] = [] - // Text sent so far of each segment, to send what the next result adds - const sentText = new Map() - listener = native.onTranscriptionEvent((event) => { - if (event.streamId !== streamId) { - return - } - const id = `segment-${event.segmentIndex}` - const sent = sentText.get(event.segmentIndex) ?? '' - const text = event.text.trim() - if (text.startsWith(sent) && text.length > sent.length) { - const separator = sent === '' && event.segmentIndex > 0 ? ' ' : '' - controller.enqueue({ - type: 'transcript-delta', - id, - delta: separator + text.slice(sent.length), - }) - sentText.set(event.segmentIndex, text) - } - if (event.type === 'final' && text) { - // ML Kit times when each word starts, so a segment ends as its last word starts - const words = event.words ?? [] - const segment = { - text, - startSecond: (words[0]?.startMs ?? 0) / 1000, - endSecond: (words.at(-1)?.startMs ?? 0) / 1000, + const segments: { + text: string + startSecond: number + endSecond: number + }[] = [] + // Text sent so far of each segment, to send what the next result adds. + // Deltas only append, so a final text that rewrites the partial comes + // in its transcript-final part only + const sentText = new Map() + listener = native.onTranscriptionEvent((event) => { + if (event.streamId !== streamId) { + return } - segments.push(segment) - controller.enqueue({ type: 'transcript-final', id, ...segment }) - } - }) - abortSignal?.addEventListener('abort', onAbort) + const id = `segment-${event.segmentIndex}` + const sent = sentText.get(event.segmentIndex) ?? '' + const text = event.text.trim() + if (text.startsWith(sent) && text.length > sent.length) { + const separator = sent === '' && event.segmentIndex > 0 ? ' ' : '' + controller.enqueue({ + type: 'transcript-delta', + id, + delta: separator + text.slice(sent.length), + }) + sentText.set(event.segmentIndex, text) + } + if (event.type === 'final' && text) { + // ML Kit times when each word starts, so a segment ends as its + // last word starts. Without word timings it has no times + const words = event.words ?? [] + if (words.length === 0) { + controller.enqueue({ type: 'transcript-final', id, text }) + return + } + const segment = { + text, + startSecond: words[0]!.startMs / 1000, + endSecond: words.at(-1)!.startMs / 1000, + } + segments.push(segment) + controller.enqueue({ type: 'transcript-final', id, ...segment }) + } + }) + abortSignal?.addEventListener('abort', onAbort, { once: true }) - try { await native.startTranscriptionStream(streamId, transcriptionOptions) // A sample can be split across chunks let carry: number | undefined while (true) { const { done, value } = await audio.read() - if (done || abortSignal?.aborted) { + if (done || cancelled || abortSignal?.aborted) { break } let chunk = @@ -1102,16 +1111,19 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { ) } } + if (cancelled) { + return + } if (abortSignal?.aborted) { throw getAbortReason(abortSignal) } - const text = await native.finishTranscriptionStream(streamId) + const result = await native.finishTranscriptionStream(streamId) controller.enqueue({ type: 'finish', - text, + text: result.text, segments, - language: transcriptionOptions.locale?.split(/[-_]/)[0], + language: result.language.split(/[-_]/)[0], durationInSeconds: undefined, }) controller.close() @@ -1119,17 +1131,22 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { native.cancel(streamId) // Stops a live source, such as a microphone, from capturing on audio.cancel(error).catch(() => {}) - controller.error( - abortSignal?.aborted - ? getAbortReason(abortSignal) - : toAndroidAIError(error) - ) + // A cancelled stream takes no more parts + if (!cancelled) { + controller.error( + abortSignal?.aborted + ? getAbortReason(abortSignal) + : toAndroidAIError(error) + ) + } } finally { cleanup() audio.releaseLock() } }, + // Ends the audio too, so a live source such as a microphone stops cancel() { + cancelled = true onAbort() cleanup() }, From 5a9f79e54134dbd1ed3cded8d9e5d629dd0a0c2e Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 00:38:26 +0100 Subject: [PATCH 3/5] fix(android): settle a failed stream start and leave the read end to ml kit on cancel Co-Authored-By: Claude Opus 5.5 --- .../callstack/ai/android/AndroidAIModule.kt | 113 +++++++++--------- .../com/callstack/ai/android/Transcription.kt | 2 +- .../docs/docs/android/transcription.md | 2 +- 3 files changed, 59 insertions(+), 58 deletions(-) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt index 8165a0a1..8f83e111 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -180,7 +180,8 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp override fun cancel(runId: String) { activeRuns.remove(runId)?.cancel() - transcriptionStreams.remove(runId)?.close() + // Only the write end: ML Kit may still be reading the other, which the run closes as it ends + transcriptionStreams.remove(runId)?.let { runCatching { it.audio.close() } } } override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) = @@ -194,66 +195,66 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp promise.resolve(transcription.transcribe(audio, TranscriptionConfig.from(options))) } - override fun startTranscriptionStream(streamId: String, options: ReadableMap, promise: Promise) { - if (transcriptionStreams.containsKey(streamId)) { - promise.reject(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId") - return - } - val (read, write) = ParcelFileDescriptor.createPipe() - val stream = TranscriptionStream(read, ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) - transcriptionStreams[streamId] = stream - - // The collector calls back one response at a time, so the count needs no lock - var segmentIndex = 0 - // A failure is reported by the next append, or by finish, which removes the stream - launchRun(streamId, { error -> - stream.result.completeExceptionally(error) - stream.close() - }) { - val config = TranscriptionConfig.from(options) - val text = transcription.transcribeStream(read, config) { response -> - emitOnTranscriptionEvent( - Arguments.createMap().apply { - putString("streamId", streamId) - putInt("segmentIndex", segmentIndex) - when (response) { - is SpeechRecognizerResponse.PartialTextResponse -> { - putString("type", "partial") - putString("text", response.text) - } - is SpeechRecognizerResponse.FinalTextResponse -> { - putString("type", "final") - putString("text", response.text) - putArray( - "words", - Arguments.createArray().apply { - response.wordTimings.forEach { word -> - pushMap( - Arguments.createMap().apply { - putString("text", word.text) - putDouble("startMs", word.timestampMs.toDouble()) - }, - ) - } - }, - ) - segmentIndex++ + override fun startTranscriptionStream(streamId: String, options: ReadableMap, promise: Promise) = + launch(promise) { + if (transcriptionStreams.containsKey(streamId)) { + throw AndroidAIErrors.CodedException(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId") + } + val (read, write) = ParcelFileDescriptor.createPipe() + val stream = TranscriptionStream(read, ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) + transcriptionStreams[streamId] = stream + + // The collector calls back one response at a time, so the count needs no lock + var segmentIndex = 0 + // A failure is reported by the next append, or by finish, which removes the stream + launchRun(streamId, { error -> + stream.result.completeExceptionally(error) + stream.close() + }) { + val config = TranscriptionConfig.from(options) + val text = transcription.transcribeStream(read, config) { response -> + emitOnTranscriptionEvent( + Arguments.createMap().apply { + putString("streamId", streamId) + putInt("segmentIndex", segmentIndex) + when (response) { + is SpeechRecognizerResponse.PartialTextResponse -> { + putString("type", "partial") + putString("text", response.text) + } + is SpeechRecognizerResponse.FinalTextResponse -> { + putString("type", "final") + putString("text", response.text) + putArray( + "words", + Arguments.createArray().apply { + response.wordTimings.forEach { word -> + pushMap( + Arguments.createMap().apply { + putString("text", word.text) + putDouble("startMs", word.timestampMs.toDouble()) + }, + ) + } + }, + ) + segmentIndex++ + } + else -> {} } - else -> {} - } + }, + ) + } + stream.close() + stream.result.complete( + Arguments.createMap().apply { + putString("text", text) + putString("language", transcription.languageOf(config)) }, ) } - stream.close() - stream.result.complete( - Arguments.createMap().apply { - putString("text", text) - putString("language", transcription.languageOf(config)) - }, - ) + null } - promise.resolve(null) - } override fun appendTranscriptionAudio(streamId: String, pcmBase64: String, promise: Promise) = launch(promise) { diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt index fdf879f6..f15815c7 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt @@ -10,8 +10,8 @@ import com.google.mlkit.genai.speechrecognition.SpeechRecognition import com.google.mlkit.genai.speechrecognition.SpeechRecognizer import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse -import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions import com.google.mlkit.genai.speechrecognition.recognitionOptions +import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock diff --git a/website/src/content/docs/docs/android/transcription.md b/website/src/content/docs/docs/android/transcription.md index 5c8e6187..c3c30602 100644 --- a/website/src/content/docs/docs/android/transcription.md +++ b/website/src/content/docs/docs/android/transcription.md @@ -36,7 +36,7 @@ const result = streamTranscribe({ for await (const part of result.fullStream) { if (part.type === 'transcript-delta') { - process(part.delta) + setTranscript((transcript) => transcript + part.delta) } if (part.type === 'transcript-final') { console.log(part.text, part.startSecond, part.endSecond) From 9fe80d8983800c79c7b9df0bcd1a38ea342c4dc2 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 00:52:35 +0100 Subject: [PATCH 4/5] fix(android): cancel a stream native starts after the cancel and tolerate a closed pipe Co-Authored-By: Claude Opus 5.5 --- .../callstack/ai/android/AndroidAIModule.kt | 10 +++++++- packages/android/src/__tests__/ai-sdk.test.ts | 25 +++++++++++++++++++ packages/android/src/ai-sdk.ts | 3 +++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt index 8f83e111..508e9b3d 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -20,6 +20,7 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import java.io.IOException import java.io.OutputStream import java.util.concurrent.ConcurrentHashMap @@ -265,7 +266,14 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp stream.result.await() return@launch null } - stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT)) + try { + stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT)) + } catch (error: IOException) { + // Recognition ended, and closed the pipe, while this append was writing + if (!stream.result.isCompleted) throw error + stream.result.await() + } + null } override fun finishTranscriptionStream(streamId: String, promise: Promise) = diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts index b2664c81..e2652608 100644 --- a/packages/android/src/__tests__/ai-sdk.test.ts +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -877,4 +877,29 @@ describe('streaming transcription', () => { expect(cancel).toHaveBeenCalled() expect(finishTranscriptionStream).not.toHaveBeenCalled() }) + + test('cancels a stream that native starts after the cancel', async () => { + let started!: () => void + startTranscriptionStream.mockImplementationOnce( + () => new Promise((resolve) => (started = resolve)) + ) + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2])) + }, + }) + + const { stream } = await model.doStream({ + audio, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const reader = stream.getReader() + await reader.read() + await reader.cancel() + cancel.mockClear() + started() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(cancel).toHaveBeenCalled() + }) }) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index bc841e9b..53943e60 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -1112,6 +1112,9 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { } } if (cancelled) { + // The cancel can land before native registers the stream, which + // then keeps listening + native.cancel(streamId) return } if (abortSignal?.aborted) { From a5875206ea77be8b4b0ebf0731f28b047c858a75 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 01:07:00 +0100 Subject: [PATCH 5/5] fix(android): name the sample rate when streaming audio is rejected Co-Authored-By: Claude Opus 5.5 --- packages/android/src/ai-sdk.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index 53943e60..127fab9e 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -997,7 +997,7 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { (inputAudioFormat.rate ?? 16000) !== 16000 ) { throw new UnsupportedFunctionalityError({ - functionality: `streaming transcription of ${inputAudioFormat.type}`, + functionality: `streaming transcription of ${inputAudioFormat.type}${inputAudioFormat.rate ? ` at ${inputAudioFormat.rate} Hz` : ''}`, message: 'ML Kit streams 16 kHz mono 16-bit PCM audio (audio/pcm, rate 16000).', })