diff --git a/harness/android/transcription.harness.ts b/harness/android/transcription.harness.ts index 035b2d56..58c7a96c 100644 --- a/harness/android/transcription.harness.ts +++ b/harness/android/transcription.harness.ts @@ -1,11 +1,30 @@ import { android } from '@react-native-ai/android' -import { experimental_transcribe as transcribe } from 'ai' +import { + experimental_streamTranscribe as streamTranscribe, + experimental_transcribe as transcribe, +} from 'ai' import { describe, expect, it } from 'react-native-harness' -import { base64ToBytes, speech } from '../fixtures' +import { base64ToBytes, speech, speechPcm8k } from '../fixtures' const model = android.transcriptionModel({ language: 'en-US', mode: 'basic' }) +// The 8 kHz fixture at the 16 kHz ML Kit streams, by linear interpolation of +// its little-endian samples +function to16k(pcm8k: Uint8Array) { + const view = new DataView(pcm8k.buffer, pcm8k.byteOffset, pcm8k.byteLength) + const sampleCount = Math.floor(pcm8k.byteLength / 2) + const output = new Int16Array(sampleCount * 2) + for (let index = 0; index < sampleCount; index++) { + const sample = view.getInt16(index * 2, true) + const next = + index + 1 < sampleCount ? view.getInt16((index + 1) * 2, true) : sample + output[index * 2] = sample + output[index * 2 + 1] = (sample + next) >> 1 + } + return new Uint8Array(output.buffer) +} + describe('android transcription', () => { it('reports recognizer availability', async () => { const info = await model.getModelInfo() @@ -33,4 +52,43 @@ describe('android transcription', () => { expect(text).toMatch(/quick brown fox/i) expect(text).toMatch(/paris/i) }) + + it('streams PCM with deltas and timed segments', async () => { + const pcm = to16k(base64ToBytes(speechPcm8k)) + // 100 ms chunks, as a microphone sends them + const audio = new ReadableStream({ + start(controller) { + for (let offset = 0; offset < pcm.length; offset += 3200) { + controller.enqueue(pcm.subarray(offset, offset + 3200)) + } + controller.close() + }, + }) + + const result = streamTranscribe({ + model, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + audio, + }) + let deltas = '' + const segments: { + text: string + startSecond?: number + endSecond?: number + }[] = [] + for await (const part of result.fullStream) { + if (part.type === 'transcript-delta') deltas += part.delta + if (part.type === 'transcript-final') segments.push(part) + } + console.log('[android] streamed segments:', JSON.stringify(segments)) + + expect(await result.text).toMatch(/quick brown fox/i) + expect(deltas).toMatch(/quick brown fox/i) + expect(segments.length).toBeGreaterThan(0) + // A one-word segment ends as it starts + const [first] = segments + if (first!.startSecond !== undefined) { + expect(first!.endSecond).toBeGreaterThanOrEqual(first!.startSecond) + } + }) }) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt index 089c6ff9..508e9b3d 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -1,5 +1,7 @@ package com.callstack.ai.android +import android.os.ParcelFileDescriptor +import android.util.Base64 import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -8,6 +10,8 @@ import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.ReadableType import com.facebook.react.bridge.WritableMap import com.google.mlkit.genai.common.FunctionCall +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers @@ -16,6 +20,8 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import java.io.IOException +import java.io.OutputStream import java.util.concurrent.ConcurrentHashMap class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISpec(reactContext) { @@ -41,6 +47,22 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp // Generations and transcriptions, by id, so they can be cancelled private val activeRuns = ConcurrentHashMap() + // Streamed transcriptions: both ends of the pipe audio is written to, and the result once the + // stream ends. Both ends close when recognition ends, so a stream JS never finishes holds no + // file descriptors + private class TranscriptionStream( + val read: ParcelFileDescriptor, + val audio: OutputStream, + val result: CompletableDeferred, + ) { + fun close() { + runCatching { audio.close() } + runCatching { read.close() } + } + } + + private val transcriptionStreams = ConcurrentHashMap() + private fun Promise.rejectWithCode(error: Exception) { reject(AndroidAIErrors.codeFor(error), error.message ?: "Unknown ML Kit error", error) } @@ -159,6 +181,8 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp override fun cancel(runId: String) { activeRuns.remove(runId)?.cancel() + // Only the write end: ML Kit may still be reading the other, which the run closes as it ends + transcriptionStreams.remove(runId)?.let { runCatching { it.audio.close() } } } override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) = @@ -172,6 +196,95 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp promise.resolve(transcription.transcribe(audio, TranscriptionConfig.from(options))) } + override fun startTranscriptionStream(streamId: String, options: ReadableMap, promise: Promise) = + launch(promise) { + if (transcriptionStreams.containsKey(streamId)) { + throw AndroidAIErrors.CodedException(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId") + } + val (read, write) = ParcelFileDescriptor.createPipe() + val stream = TranscriptionStream(read, ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) + transcriptionStreams[streamId] = stream + + // The collector calls back one response at a time, so the count needs no lock + var segmentIndex = 0 + // A failure is reported by the next append, or by finish, which removes the stream + launchRun(streamId, { error -> + stream.result.completeExceptionally(error) + stream.close() + }) { + val config = TranscriptionConfig.from(options) + val text = transcription.transcribeStream(read, config) { response -> + emitOnTranscriptionEvent( + Arguments.createMap().apply { + putString("streamId", streamId) + putInt("segmentIndex", segmentIndex) + when (response) { + is SpeechRecognizerResponse.PartialTextResponse -> { + putString("type", "partial") + putString("text", response.text) + } + is SpeechRecognizerResponse.FinalTextResponse -> { + putString("type", "final") + putString("text", response.text) + putArray( + "words", + Arguments.createArray().apply { + response.wordTimings.forEach { word -> + pushMap( + Arguments.createMap().apply { + putString("text", word.text) + putDouble("startMs", word.timestampMs.toDouble()) + }, + ) + } + }, + ) + segmentIndex++ + } + else -> {} + } + }, + ) + } + stream.close() + stream.result.complete( + Arguments.createMap().apply { + putString("text", text) + putString("language", transcription.languageOf(config)) + }, + ) + } + null + } + + override fun appendTranscriptionAudio(streamId: String, pcmBase64: String, promise: Promise) = + launch(promise) { + val stream = transcriptionStreams[streamId] + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") + // Recognition ended early: rethrow why, or drop audio it can no longer hear + if (stream.result.isCompleted) { + stream.result.await() + return@launch null + } + try { + stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT)) + } catch (error: IOException) { + // Recognition ended, and closed the pipe, while this append was writing + if (!stream.result.isCompleted) throw error + stream.result.await() + } + null + } + + override fun finishTranscriptionStream(streamId: String, promise: Promise) = + launch(promise) { + val stream = transcriptionStreams.remove(streamId) + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") + // Closing the pipe ends the audio, so recognition finishes what it has heard + runCatching { stream.audio.close() } + stream.result.await() + } + override fun getEmbeddingModelInfo(promise: Promise) = launch(promise) { val info = embeddings.getModelInfo() @@ -217,6 +330,11 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp } override fun invalidate() { + transcriptionStreams.values.forEach { stream -> + stream.close() + stream.result.cancel() + } + transcriptionStreams.clear() activeRuns.values.forEach { it.cancel() } activeRuns.clear() scope.cancel() diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt index 9f5ba38c..f15815c7 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt @@ -1,6 +1,7 @@ package com.callstack.ai.android import android.content.Context +import android.os.ParcelFileDescriptor import android.util.Base64 import com.facebook.react.bridge.ReadableMap import com.google.mlkit.genai.common.FeatureStatus @@ -9,6 +10,7 @@ import com.google.mlkit.genai.speechrecognition.SpeechRecognition import com.google.mlkit.genai.speechrecognition.SpeechRecognizer import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import com.google.mlkit.genai.speechrecognition.recognitionOptions import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest import kotlinx.coroutines.sync.Mutex @@ -55,6 +57,10 @@ class Transcription( "Not a BCP 47 language tag: $tag. Use a tag such as en-US", ) + /** The language the recognizer transcribes, as a BCP 47 tag. */ + fun languageOf(config: TranscriptionConfig): String = + (config.locale?.let(::locale) ?: Locale.getDefault()).toLanguageTag() + suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo = ModelInfo(availability = availabilityOf(client(config).checkStatus())) @@ -103,4 +109,36 @@ class Transcription( file.delete() } } + + /** + * Transcribes 16 kHz mono 16-bit PCM read from [audio] until it is closed, reporting each + * partial and final result with its word timings. Returns the whole text. + */ + suspend fun transcribeStream( + audio: ParcelFileDescriptor, + config: TranscriptionConfig, + onResult: (response: SpeechRecognizerResponse) -> Unit, + ): String { + prepare(config) + AudioSource.fromPfd(audio, AudioSource.Mode.STREAMING).use { source -> + val text = StringBuilder() + val request = speechRecognizerRequest { + audioSource = source + recognitionOptions { enableWordTiming = true } + } + client(config).startRecognition(request).collect { response -> + when (response) { + is SpeechRecognizerResponse.PartialTextResponse -> onResult(response) + is SpeechRecognizerResponse.FinalTextResponse -> { + if (text.isNotEmpty() && response.text.isNotEmpty()) text.append(' ') + text.append(response.text) + onResult(response) + } + is SpeechRecognizerResponse.ErrorResponse -> throw response.e + else -> {} + } + } + return text.toString() + } + } } diff --git a/packages/android/src/NativeAndroidAI.ts b/packages/android/src/NativeAndroidAI.ts index 083bea12..e37ff021 100644 --- a/packages/android/src/NativeAndroidAI.ts +++ b/packages/android/src/NativeAndroidAI.ts @@ -105,6 +105,22 @@ export interface NativeTranscriptionOptions { mode?: string } +export interface TranscriptionEvent { + streamId: string + /** `partial` text of the segment so far, or its `final` text */ + type: string + segmentIndex: number + text: string + /** Final results only: each word and when it starts */ + words?: { text: string; startMs: number }[] +} + +export interface NativeStreamTranscriptionResult { + text: string + /** BCP 47 locale the recognizer transcribed, such as en-US */ + language: string +} + export interface NativeEmbeddingModelInfo { availability: string dimension?: number @@ -148,6 +164,20 @@ export interface Spec extends TurboModule { options: NativeTranscriptionOptions ): Promise + /** + * Starts transcribing 16 kHz mono 16-bit PCM appended to the stream, with + * results as `onTranscriptionEvent`. Cancel it with `cancel(streamId)` + */ + startTranscriptionStream( + streamId: string, + options: NativeTranscriptionOptions + ): Promise + appendTranscriptionAudio(streamId: string, pcmBase64: string): Promise + /** Ends the audio and resolves with the whole text */ + finishTranscriptionStream( + streamId: string + ): Promise + getEmbeddingModelInfo(): Promise prepareEmbedding(): Promise embed(values: string[], taskType?: string): Promise @@ -171,6 +201,7 @@ export interface Spec extends TurboModule { onStreamEvent: CodegenTypes.EventEmitter onDownloadProgress: CodegenTypes.EventEmitter + onTranscriptionEvent: CodegenTypes.EventEmitter } const NativeAndroidAI: Spec | null = diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts index c93cf2a1..e2652608 100644 --- a/packages/android/src/__tests__/ai-sdk.test.ts +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -20,6 +20,7 @@ function createEmitter() { const onStreamEvent = createEmitter() const onDownloadProgress = createEmitter() +const onTranscriptionEvent = createEmitter() let generateTextResult: Record = { text: 'ok', @@ -50,6 +51,16 @@ const getTranscriptionModelInfo = mock( availability: 'downloadable', }) ) +const startTranscriptionStream = mock( + async (_streamId: string, _options: unknown) => {} +) +const appendTranscriptionAudio = mock( + async (_streamId: string, _pcmBase64: string) => {} +) +const finishTranscriptionStream = mock(async (_streamId: string) => ({ + text: 'hello world', + language: 'en-US', +})) const prepareTask = mock(async (_task: string, _options: unknown) => {}) const getTaskModelInfo = mock( async ( @@ -85,6 +96,10 @@ mock.module('react-native', () => ({ prepareTranscription, prepareEmbedding, embed, + startTranscriptionStream, + appendTranscriptionAudio, + finishTranscriptionStream, + onTranscriptionEvent, prepareTask, getTaskModelInfo, runTask, @@ -732,3 +747,159 @@ describe('task models', () => { }) }) }) + +describe('streaming transcription', () => { + const model = createAndroidProvider().transcriptionModel({ + language: 'en-US', + }) + + const audioStream = (chunks: Uint8Array[]) => + new ReadableStream({ + start(controller) { + chunks.forEach((chunk) => controller.enqueue(chunk)) + controller.close() + }, + }) + + test('sends deltas, final segments with timings and the whole text', async () => { + appendTranscriptionAudio.mockImplementationOnce(async (streamId) => { + const event = (type: string, text: string, words?: unknown) => + onTranscriptionEvent.emit({ + streamId, + type, + segmentIndex: 0, + text, + words, + }) + event('partial', 'hel') + event('partial', 'hello') + event('final', 'hello world', [ + { text: 'hello', startMs: 100 }, + { text: 'world', startMs: 600 }, + ]) + }) + await model.prepare() + + const { stream } = await model.doStream({ + audio: audioStream([new Uint8Array([1, 2, 3]), new Uint8Array([4])]), + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const parts: any[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + + // The odd byte waits for the next chunk + expect(appendTranscriptionAudio.mock.calls.map((call) => call[1])).toEqual([ + 'AQI=', + 'AwQ=', + ]) + expect( + parts + .filter((part) => part.type === 'transcript-delta') + .map((part) => part.delta) + ).toEqual(['hel', 'lo', ' world']) + expect(parts.find((part) => part.type === 'transcript-final')).toEqual({ + type: 'transcript-final', + id: 'segment-0', + text: 'hello world', + startSecond: 0.1, + endSecond: 0.6, + }) + expect(parts.at(-1)).toMatchObject({ + type: 'finish', + text: 'hello world', + language: 'en', + }) + }) + + test('rejects audio ML Kit cannot stream', async () => { + await expect( + model.doStream({ + audio: audioStream([]), + inputAudioFormat: { type: 'audio/pcm', rate: 48000 }, + } as any) + ).rejects.toThrow('16 kHz') + }) + + test('sends a final without word timings without times', async () => { + appendTranscriptionAudio.mockImplementationOnce(async (streamId) => { + onTranscriptionEvent.emit({ + streamId, + type: 'final', + segmentIndex: 0, + text: 'hi', + words: [], + }) + }) + + const { stream } = await model.doStream({ + audio: audioStream([new Uint8Array([1, 2])]), + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const parts: any[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + + expect(parts.find((part) => part.type === 'transcript-final')).toEqual({ + type: 'transcript-final', + id: 'segment-0', + text: 'hi', + }) + expect(parts.at(-1)).toMatchObject({ type: 'finish', segments: [] }) + }) + + test('ends a cancelled live stream without finishing it', async () => { + finishTranscriptionStream.mockClear() + // A microphone stream never closes on its own + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2])) + }, + }) + + const { stream } = await model.doStream({ + audio, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const reader = stream.getReader() + await reader.read() + await reader.cancel() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(cancel).toHaveBeenCalled() + expect(finishTranscriptionStream).not.toHaveBeenCalled() + }) + + test('cancels a stream that native starts after the cancel', async () => { + let started!: () => void + startTranscriptionStream.mockImplementationOnce( + () => new Promise((resolve) => (started = resolve)) + ) + const audio = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2])) + }, + }) + + const { stream } = await model.doStream({ + audio, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + } as any) + const reader = stream.getReader() + await reader.read() + await reader.cancel() + cancel.mockClear() + started() + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(cancel).toHaveBeenCalled() + }) +}) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index 7630eecd..127fab9e 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -2,6 +2,9 @@ import type { EmbeddingModelV4, EmbeddingModelV4CallOptions, EmbeddingModelV4Result, + Experimental_TranscriptionModelV4StreamOptions as TranscriptionModelV4StreamOptions, + Experimental_TranscriptionModelV4StreamPart as TranscriptionModelV4StreamPart, + Experimental_TranscriptionModelV4StreamResult as TranscriptionModelV4StreamResult, LanguageModelV4, LanguageModelV4CallOptions, LanguageModelV4Content, @@ -15,6 +18,7 @@ import type { TranscriptionModelV4, TranscriptionModelV4CallOptions, } from '@ai-sdk/provider' +import { UnsupportedFunctionalityError } from '@ai-sdk/provider' import { generateId } from '@ai-sdk/provider-utils' import { Platform } from 'react-native' @@ -979,6 +983,180 @@ class AndroidTranscriptionModel implements TranscriptionModelV4 { abortSignal?.removeEventListener('abort', onAbort) } } + + /** + * Transcribes live 16 kHz mono 16-bit PCM. A segment's text grows until a + * pause ends it, and its final text comes with word timings + */ + async doStream( + options: TranscriptionModelV4StreamOptions + ): Promise { + const { inputAudioFormat, abortSignal } = options + if ( + inputAudioFormat.type !== 'audio/pcm' || + (inputAudioFormat.rate ?? 16000) !== 16000 + ) { + throw new UnsupportedFunctionalityError({ + functionality: `streaming transcription of ${inputAudioFormat.type}${inputAudioFormat.rate ? ` at ${inputAudioFormat.rate} Hz` : ''}`, + message: + 'ML Kit streams 16 kHz mono 16-bit PCM audio (audio/pcm, rate 16000).', + }) + } + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + if (!this.prepared) { + console.warn(notPreparedWarning) + await this.prepare() + } + + const native = getNativeAndroidAI() + const streamId = generateId() + const modelId = this.modelId + const transcriptionOptions = this.options + const audio = options.audio.getReader() + let listener: { remove(): void } | undefined + // Set when the consumer cancels the stream, which ends it without a result + let cancelled = false + const cleanup = () => { + listener?.remove() + abortSignal?.removeEventListener('abort', onAbort) + } + function onAbort() { + native.cancel(streamId) + audio.cancel().catch(() => {}) + } + + const stream = new ReadableStream({ + async start(controller) { + try { + controller.enqueue({ type: 'stream-start', warnings: [] }) + controller.enqueue({ + type: 'response-metadata', + timestamp: new Date(), + modelId, + }) + + const segments: { + text: string + startSecond: number + endSecond: number + }[] = [] + // Text sent so far of each segment, to send what the next result adds. + // Deltas only append, so a final text that rewrites the partial comes + // in its transcript-final part only + const sentText = new Map() + listener = native.onTranscriptionEvent((event) => { + if (event.streamId !== streamId) { + return + } + const id = `segment-${event.segmentIndex}` + const sent = sentText.get(event.segmentIndex) ?? '' + const text = event.text.trim() + if (text.startsWith(sent) && text.length > sent.length) { + const separator = sent === '' && event.segmentIndex > 0 ? ' ' : '' + controller.enqueue({ + type: 'transcript-delta', + id, + delta: separator + text.slice(sent.length), + }) + sentText.set(event.segmentIndex, text) + } + if (event.type === 'final' && text) { + // ML Kit times when each word starts, so a segment ends as its + // last word starts. Without word timings it has no times + const words = event.words ?? [] + if (words.length === 0) { + controller.enqueue({ type: 'transcript-final', id, text }) + return + } + const segment = { + text, + startSecond: words[0]!.startMs / 1000, + endSecond: words.at(-1)!.startMs / 1000, + } + segments.push(segment) + controller.enqueue({ type: 'transcript-final', id, ...segment }) + } + }) + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + await native.startTranscriptionStream(streamId, transcriptionOptions) + // A sample can be split across chunks + let carry: number | undefined + while (true) { + const { done, value } = await audio.read() + if (done || cancelled || abortSignal?.aborted) { + break + } + let chunk = + typeof value === 'string' + ? Uint8Array.from(atob(value), (char) => char.charCodeAt(0)) + : value + if (carry !== undefined) { + const joined = new Uint8Array(chunk.length + 1) + joined[0] = carry + joined.set(chunk, 1) + chunk = joined + carry = undefined + } + if (chunk.length % 2 === 1) { + carry = chunk[chunk.length - 1] + chunk = chunk.subarray(0, chunk.length - 1) + } + if (chunk.length > 0) { + await native.appendTranscriptionAudio( + streamId, + uint8ArrayToBase64(chunk) + ) + } + } + if (cancelled) { + // The cancel can land before native registers the stream, which + // then keeps listening + native.cancel(streamId) + return + } + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + const result = await native.finishTranscriptionStream(streamId) + controller.enqueue({ + type: 'finish', + text: result.text, + segments, + language: result.language.split(/[-_]/)[0], + durationInSeconds: undefined, + }) + controller.close() + } catch (error) { + native.cancel(streamId) + // Stops a live source, such as a microphone, from capturing on + audio.cancel(error).catch(() => {}) + // A cancelled stream takes no more parts + if (!cancelled) { + controller.error( + abortSignal?.aborted + ? getAbortReason(abortSignal) + : toAndroidAIError(error) + ) + } + } finally { + cleanup() + audio.releaseLock() + } + }, + // Ends the audio too, so a live source such as a microphone stops + cancel() { + cancelled = true + onAbort() + cleanup() + }, + }) + + return { stream, response: { timestamp: new Date(), modelId } } + } } class AndroidEmbeddingModel implements EmbeddingModelV4 { diff --git a/website/src/content/docs/docs/android/transcription.md b/website/src/content/docs/docs/android/transcription.md index 84094890..c3c30602 100644 --- a/website/src/content/docs/docs/android/transcription.md +++ b/website/src/content/docs/docs/android/transcription.md @@ -18,8 +18,38 @@ const { text } = await transcribe({ ``` ML Kit decodes the file on the device. It returns the text only: word timings -come with streamed audio, which this model doesn't take yet, so `segments` is -empty. +come with streamed audio, so `segments` is empty. + +## Streaming + +Stream live audio, such as microphone chunks, to +`experimental_streamTranscribe` as 16 kHz mono 16-bit PCM: + +```typescript +import { experimental_streamTranscribe as streamTranscribe } from 'ai' + +const result = streamTranscribe({ + model, + inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, + audio: microphoneChunks, // ReadableStream +}) + +for await (const part of result.fullStream) { + if (part.type === 'transcript-delta') { + setTranscript((transcript) => transcript + part.delta) + } + if (part.type === 'transcript-final') { + console.log(part.text, part.startSecond, part.endSecond) + } +} +``` + +- Deltas append to the current segment. When the speaker pauses, the segment + closes and arrives as a `transcript-final` part. +- ML Kit times when each word starts, so a segment's `endSecond` is when its + last word starts. +- Other sample rates are rejected: resample to 16 kHz first. +- Abort the call, or cancel the stream, to stop. ## Options