Repository navigation
feat(android): stream transcription of live pcm audio #375
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/android-mlkit-task-apis
Are you sure you want to change the base?
Changes from all commits
018bd98
5521dba
5a9f79e
9fe80d8
a587520
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,30 @@ | ||
| import { android } from '@react-native-ai/android' | ||
| import { experimental_transcribe as transcribe } from 'ai' | ||
| import { | ||
| experimental_streamTranscribe as streamTranscribe, | ||
| experimental_transcribe as transcribe, | ||
| } from 'ai' | ||
| import { describe, expect, it } from 'react-native-harness' | ||
|
|
||
| import { base64ToBytes, speech } from '../fixtures' | ||
| import { base64ToBytes, speech, speechPcm8k } from '../fixtures' | ||
|
|
||
| const model = android.transcriptionModel({ language: 'en-US', mode: 'basic' }) | ||
|
|
||
| // The 8 kHz fixture at the 16 kHz ML Kit streams, by linear interpolation of | ||
| // its little-endian samples | ||
| function to16k(pcm8k: Uint8Array) { | ||
| const view = new DataView(pcm8k.buffer, pcm8k.byteOffset, pcm8k.byteLength) | ||
| const sampleCount = Math.floor(pcm8k.byteLength / 2) | ||
| const output = new Int16Array(sampleCount * 2) | ||
| for (let index = 0; index < sampleCount; index++) { | ||
| const sample = view.getInt16(index * 2, true) | ||
| const next = | ||
| index + 1 < sampleCount ? view.getInt16((index + 1) * 2, true) : sample | ||
| output[index * 2] = sample | ||
| output[index * 2 + 1] = (sample + next) >> 1 | ||
| } | ||
| return new Uint8Array(output.buffer) | ||
| } | ||
|
|
||
| describe('android transcription', () => { | ||
| it('reports recognizer availability', async () => { | ||
| const info = await model.getModelInfo() | ||
|
|
@@ -33,4 +52,43 @@ describe('android transcription', () => { | |
| expect(text).toMatch(/quick brown fox/i) | ||
| expect(text).toMatch(/paris/i) | ||
| }) | ||
|
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · Harness stream test asserts a phrase survives the delta chain and logs JSON into the run output In harness/android/transcription.harness.ts the new case asserts Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. It's a device test against one known clip, and the log is for reading the device run. The timing check depends on whether the device returns word timings. 🤖 Reply from Claude Code |
||
| it('streams PCM with deltas and timed segments', async () => { | ||
| const pcm = to16k(base64ToBytes(speechPcm8k)) | ||
| // 100 ms chunks, as a microphone sends them | ||
| const audio = new ReadableStream<Uint8Array>({ | ||
| start(controller) { | ||
| for (let offset = 0; offset < pcm.length; offset += 3200) { | ||
| controller.enqueue(pcm.subarray(offset, offset + 3200)) | ||
| } | ||
| controller.close() | ||
| }, | ||
| }) | ||
|
|
||
| const result = streamTranscribe({ | ||
| model, | ||
| inputAudioFormat: { type: 'audio/pcm', rate: 16000 }, | ||
| audio, | ||
| }) | ||
| let deltas = '' | ||
| const segments: { | ||
| text: string | ||
| startSecond?: number | ||
| endSecond?: number | ||
| }[] = [] | ||
| for await (const part of result.fullStream) { | ||
| if (part.type === 'transcript-delta') deltas += part.delta | ||
| if (part.type === 'transcript-final') segments.push(part) | ||
| } | ||
| console.log('[android] streamed segments:', JSON.stringify(segments)) | ||
|
|
||
| expect(await result.text).toMatch(/quick brown fox/i) | ||
| expect(deltas).toMatch(/quick brown fox/i) | ||
| expect(segments.length).toBeGreaterThan(0) | ||
| // A one-word segment ends as it starts | ||
| const [first] = segments | ||
| if (first!.startSecond !== undefined) { | ||
| expect(first!.endSecond).toBeGreaterThanOrEqual(first!.startSecond) | ||
| } | ||
| }) | ||
| }) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,7 @@ | ||
| package com.callstack.ai.android | ||
|
|
||
| import android.os.ParcelFileDescriptor | ||
| import android.util.Base64 | ||
| import com.facebook.react.bridge.Arguments | ||
| import com.facebook.react.bridge.Promise | ||
| import com.facebook.react.bridge.ReactApplicationContext | ||
|
|
@@ -8,6 +10,8 @@ import com.facebook.react.bridge.ReadableMap | |
| import com.facebook.react.bridge.ReadableType | ||
| import com.facebook.react.bridge.WritableMap | ||
| import com.google.mlkit.genai.common.FunctionCall | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse | ||
| import kotlinx.coroutines.CompletableDeferred | ||
| import kotlinx.coroutines.CoroutineScope | ||
| import kotlinx.coroutines.CoroutineStart | ||
| import kotlinx.coroutines.Dispatchers | ||
|
|
@@ -16,6 +20,8 @@ import kotlinx.coroutines.SupervisorJob | |
| import kotlinx.coroutines.cancel | ||
| import kotlinx.coroutines.isActive | ||
| import kotlinx.coroutines.launch | ||
| import java.io.IOException | ||
| import java.io.OutputStream | ||
| import java.util.concurrent.ConcurrentHashMap | ||
|
|
||
| class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISpec(reactContext) { | ||
|
|
@@ -41,6 +47,22 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp | |
| // Generations and transcriptions, by id, so they can be cancelled | ||
| private val activeRuns = ConcurrentHashMap<String, Job>() | ||
|
|
||
| // Streamed transcriptions: both ends of the pipe audio is written to, and the result once the | ||
|
grabbou marked this conversation as resolved.
Resolved
|
||
| // stream ends. Both ends close when recognition ends, so a stream JS never finishes holds no | ||
| // file descriptors | ||
| private class TranscriptionStream( | ||
| val read: ParcelFileDescriptor, | ||
| val audio: OutputStream, | ||
| val result: CompletableDeferred<WritableMap>, | ||
| ) { | ||
| fun close() { | ||
| runCatching { audio.close() } | ||
| runCatching { read.close() } | ||
| } | ||
| } | ||
|
|
||
| private val transcriptionStreams = ConcurrentHashMap<String, TranscriptionStream>() | ||
|
|
||
| private fun Promise.rejectWithCode(error: Exception) { | ||
| reject(AndroidAIErrors.codeFor(error), error.message ?: "Unknown ML Kit error", error) | ||
| } | ||
|
|
@@ -159,6 +181,8 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp | |
|
|
||
| override fun cancel(runId: String) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟡 Minor · Cancelled/errored stream never calls native finish, so the JS
Suggested change Either resolve the abandoned run's result (e.g. emit a final event carrying the text, or expose the deferred to a cancel promise) so a late cancellation does not discard a completed transcription, and remove the Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's intended: a cancel means no result, so the text is dropped. The leftover entry was answered in the earlier thread. 🤖 Reply from Claude Code There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit ·
Suggested change Either document that a runId must be unique across streaming text, task streaming and transcription streams, or key each registry with a kind prefix so the duplicate-id check covers both. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. Run ids are unique per call across text, task and transcription streams, since they all come from 🤖 Reply from Claude Code |
||
| activeRuns.remove(runId)?.cancel() | ||
| // Only the write end: ML Kit may still be reading the other, which the run closes as it ends | ||
| transcriptionStreams.remove(runId)?.let { runCatching { it.audio.close() } } | ||
| } | ||
|
|
||
| override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) = | ||
|
|
@@ -172,6 +196,95 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp | |
| promise.resolve(transcription.transcribe(audio, TranscriptionConfig.from(options))) | ||
| } | ||
|
|
||
| override fun startTranscriptionStream(streamId: String, options: ReadableMap, promise: Promise) = | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved from an earlier thread. 🔵 Nit · check-then-act on
Suggested change Create the pipe and insert it with DetailsAlso raised hereAlso raised at this location by senior-developer: startTranscriptionStream's duplicate-id check is a non-atomic containsKey + put — persona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. Ids come from 🤖 Reply from Claude Code There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · Stream entry stays in transcriptionStreams when recognition fails and JS never finishes or cancels
Suggested change Remove the entry from transcriptionStreams when the run ends, in launchRun's completion handler (or in the onError path), so an abandoned stream does not stay in the map until invalidate(). Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as the earlier thread: the entry stays so the next call can report why recognition ended, and 🤖 Reply from Claude Code |
||
| launch(promise) { | ||
| if (transcriptionStreams.containsKey(streamId)) { | ||
| throw AndroidAIErrors.CodedException(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId") | ||
| } | ||
| val (read, write) = ParcelFileDescriptor.createPipe() | ||
| val stream = TranscriptionStream(read, ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred()) | ||
| transcriptionStreams[streamId] = stream | ||
|
|
||
| // The collector calls back one response at a time, so the count needs no lock | ||
| var segmentIndex = 0 | ||
| // A failure is reported by the next append, or by finish, which removes the stream | ||
| launchRun(streamId, { error -> | ||
| stream.result.completeExceptionally(error) | ||
| stream.close() | ||
| }) { | ||
| val config = TranscriptionConfig.from(options) | ||
| val text = transcription.transcribeStream(read, config) { response -> | ||
| emitOnTranscriptionEvent( | ||
| Arguments.createMap().apply { | ||
| putString("streamId", streamId) | ||
| putInt("segmentIndex", segmentIndex) | ||
| when (response) { | ||
| is SpeechRecognizerResponse.PartialTextResponse -> { | ||
| putString("type", "partial") | ||
| putString("text", response.text) | ||
| } | ||
| is SpeechRecognizerResponse.FinalTextResponse -> { | ||
| putString("type", "final") | ||
| putString("text", response.text) | ||
| putArray( | ||
| "words", | ||
| Arguments.createArray().apply { | ||
| response.wordTimings.forEach { word -> | ||
| pushMap( | ||
| Arguments.createMap().apply { | ||
| putString("text", word.text) | ||
| putDouble("startMs", word.timestampMs.toDouble()) | ||
| }, | ||
| ) | ||
| } | ||
| }, | ||
| ) | ||
| segmentIndex++ | ||
| } | ||
| else -> {} | ||
| } | ||
| }, | ||
| ) | ||
| } | ||
| stream.close() | ||
| stream.result.complete( | ||
| Arguments.createMap().apply { | ||
| putString("text", text) | ||
| putString("language", transcription.languageOf(config)) | ||
| }, | ||
| ) | ||
| } | ||
| null | ||
| } | ||
|
|
||
| override fun appendTranscriptionAudio(streamId: String, pcmBase64: String, promise: Promise) = | ||
|
grabbou marked this conversation as resolved.
Resolved
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟡 Minor · finishTranscriptionStream can await a deferred that nothing ever completes (hung promise, stream never removed)
Suggested change Guarantee the deferred always settles: complete it in the launchRun block's own DetailsAlso raised hereAlso raised at this location by senior-developer: Aborted stream leaks the native pipe fds and the running recognizer job (JS never cancels after abort) — In Also raised at this location by jvm: persona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. A failed run completes 🤖 Reply from Claude Code There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · Unknown-stream error reuses INVALID_MESSAGE while start uses GENERATION_ERROR
Suggested change Use one error code for unknown/duplicate stream ids across start/append/finish instead of mixing GENERATION_ERROR and INVALID_MESSAGE. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. An unknown id on append or finish is a bad argument from the caller, while a duplicate id on start is a failure to start. 🤖 Reply from Claude Code
grabbou marked this conversation as resolved.
Resolved
|
||
| launch(promise) { | ||
| val stream = transcriptionStreams[streamId] | ||
| ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") | ||
| // Recognition ended early: rethrow why, or drop audio it can no longer hear | ||
| if (stream.result.isCompleted) { | ||
| stream.result.await() | ||
| return@launch null | ||
| } | ||
| try { | ||
| stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT)) | ||
| } catch (error: IOException) { | ||
| // Recognition ended, and closed the pipe, while this append was writing | ||
| if (!stream.result.isCompleted) throw error | ||
| stream.result.await() | ||
| } | ||
| null | ||
| } | ||
|
|
||
| override fun finishTranscriptionStream(streamId: String, promise: Promise) = | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟠 Major · Audio still in the kernel pipe is discarded when finish closes the write end right after a native promise resolves
Suggested change Do not treat a resolved append as acknowledgement that ML Kit consumed the audio. Either keep the write end open until the recognizer signals it reached EOF (or a short idle window after the last append), or have the JS loop wait for recognition to catch up before calling finishTranscriptionStream, so bytes buffered in the pipe are not dropped by the close. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Closing a pipe's write end doesn't discard what's still buffered. The reader drains the remaining bytes and only then sees EOF, which is POSIX pipe behavior. So finish closes the write end without losing the tail. 🤖 Reply from Claude Code |
||
| launch(promise) { | ||
| val stream = transcriptionStreams.remove(streamId) | ||
| ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId") | ||
| // Closing the pipe ends the audio, so recognition finishes what it has heard | ||
| runCatching { stream.audio.close() } | ||
| stream.result.await() | ||
| } | ||
|
|
||
| override fun getEmbeddingModelInfo(promise: Promise) = | ||
| launch(promise) { | ||
| val info = embeddings.getModelInfo() | ||
|
grabbou marked this conversation as resolved.
Resolved
|
||
|
|
@@ -217,6 +330,11 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp | |
| } | ||
|
|
||
| override fun invalidate() { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟠 Major · invalidate() deadlocks every pending appendTranscriptionAudio: the deferred is cancelled but append awaits it without a timeout
Suggested change Complete the result exceptionally instead of cancelling it in invalidate() (e.g. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. 🤖 Reply from Claude Code |
||
| transcriptionStreams.values.forEach { stream -> | ||
| stream.close() | ||
| stream.result.cancel() | ||
| } | ||
| transcriptionStreams.clear() | ||
| activeRuns.values.forEach { it.cancel() } | ||
| activeRuns.clear() | ||
| scope.cancel() | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,7 @@ | ||
| package com.callstack.ai.android | ||
|
|
||
| import android.content.Context | ||
| import android.os.ParcelFileDescriptor | ||
| import android.util.Base64 | ||
| import com.facebook.react.bridge.ReadableMap | ||
| import com.google.mlkit.genai.common.FeatureStatus | ||
|
|
@@ -9,6 +10,7 @@ import com.google.mlkit.genai.speechrecognition.SpeechRecognition | |
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizer | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse | ||
| import com.google.mlkit.genai.speechrecognition.recognitionOptions | ||
|
grabbou marked this conversation as resolved.
Resolved
|
||
| import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions | ||
| import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest | ||
| import kotlinx.coroutines.sync.Mutex | ||
|
|
@@ -55,6 +57,10 @@ class Transcription( | |
| "Not a BCP 47 language tag: $tag. Use a tag such as en-US", | ||
| ) | ||
|
|
||
| /** The language the recognizer transcribes, as a BCP 47 tag. */ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟡 Minor · finish.language now comes from the device locale when the model has no language, unlike doGenerate
Suggested change Pick one source for finish/doGenerate language (the configured locale, falling back to the device tag in both paths) and note it on the Android transcription docs page. DetailsAlso raised hereAlso raised at this location by jvm: New native stream result carries a language the JS side derives by splitting, and the sample rate the caller declares is never sent to native — persona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as the earlier thread: streaming reports the locale ML Kit recognized with. File transcription reports only the locale you asked for, because native doesn't return one there. 🤖 Reply from Claude Code There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟡 Minor · Language reported for a stream comes from the config, not the device default it claims to fall back to
Suggested change Either omit DetailsAlso raised hereAlso raised at this location by maintainability: languageOf reads config.locale, a field never referenced anywhere else in the file — persona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The recognizer is built with that same locale: Transcription.kt:43 sets 🤖 Reply from Claude Code |
||
| fun languageOf(config: TranscriptionConfig): String = | ||
| (config.locale?.let(::locale) ?: Locale.getDefault()).toLanguageTag() | ||
|
|
||
| suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo = | ||
| ModelInfo(availability = availabilityOf(client(config).checkStatus())) | ||
|
|
||
|
|
@@ -103,4 +109,36 @@ class Transcription( | |
| file.delete() | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Transcribes 16 kHz mono 16-bit PCM read from [audio] until it is closed, reporting each | ||
| * partial and final result with its word timings. Returns the whole text. | ||
| */ | ||
| suspend fun transcribeStream( | ||
| audio: ParcelFileDescriptor, | ||
| config: TranscriptionConfig, | ||
| onResult: (response: SpeechRecognizerResponse) -> Unit, | ||
| ): String { | ||
| prepare(config) | ||
| AudioSource.fromPfd(audio, AudioSource.Mode.STREAMING).use { source -> | ||
| val text = StringBuilder() | ||
| val request = speechRecognizerRequest { | ||
| audioSource = source | ||
| recognitionOptions { enableWordTiming = true } | ||
| } | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · An ErrorResponse after the first segment is swallowed: doStream resolves with empty text Transcription.kt:135 throws on SpeechRecognizerResponse.ErrorResponse, which ends transcribeStream and never reaches Suggested change Either complete the deferred with the text collected so far (plus the error as a separate signal) or push the error into the same result the JS side awaits, so a mid-stream ML Kit error after a final segment does not leave the caller with an empty text and no finish part. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's intended: an ML Kit error ends the stream with that error, as the other providers do. The text heard so far has already arrived as delta and final parts. 🤖 Reply from Claude Code |
||
| client(config).startRecognition(request).collect { response -> | ||
| when (response) { | ||
| is SpeechRecognizerResponse.PartialTextResponse -> onResult(response) | ||
| is SpeechRecognizerResponse.FinalTextResponse -> { | ||
| if (text.isNotEmpty() && response.text.isNotEmpty()) text.append(' ') | ||
| text.append(response.text) | ||
| onResult(response) | ||
| } | ||
| is SpeechRecognizerResponse.ErrorResponse -> throw response.e | ||
| else -> {} | ||
| } | ||
| } | ||
| return text.toString() | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -105,6 +105,22 @@ export interface NativeTranscriptionOptions { | |||||||||||||||||||||
| mode?: string | ||||||||||||||||||||||
| } | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| export interface TranscriptionEvent { | ||||||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔵 Nit · TranscriptionEvent.type is a bare string while the sibling StreamEvent types are literal unions
Suggested change
Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this: 🤖 Addressed by Claude Code |
||||||||||||||||||||||
| streamId: string | ||||||||||||||||||||||
| /** `partial` text of the segment so far, or its `final` text */ | ||||||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · TranscriptionEvent.type is a plain string, not the 'partial' | 'final' union it documents The doc comment two lines above enumerates the values, and ai-sdk.ts compares
Suggested change
Suggested change Type Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as the earlier thread: the event type stays a codegen-friendly string, and the listener compares it against the two values. 🤖 Reply from Claude Code |
||||||||||||||||||||||
| type: string | ||||||||||||||||||||||
| segmentIndex: number | ||||||||||||||||||||||
| text: string | ||||||||||||||||||||||
| /** Final results only: each word and when it starts */ | ||||||||||||||||||||||
| words?: { text: string; startMs: number }[] | ||||||||||||||||||||||
| } | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| export interface NativeStreamTranscriptionResult { | ||||||||||||||||||||||
| text: string | ||||||||||||||||||||||
| /** BCP 47 locale the recognizer transcribed, such as en-US */ | ||||||||||||||||||||||
| language: string | ||||||||||||||||||||||
| } | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| export interface NativeEmbeddingModelInfo { | ||||||||||||||||||||||
| availability: string | ||||||||||||||||||||||
| dimension?: number | ||||||||||||||||||||||
|
|
@@ -148,6 +164,20 @@ export interface Spec extends TurboModule { | |||||||||||||||||||||
| options: NativeTranscriptionOptions | ||||||||||||||||||||||
| ): Promise<string> | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| /** | ||||||||||||||||||||||
| * Starts transcribing 16 kHz mono 16-bit PCM appended to the stream, with | ||||||||||||||||||||||
| * results as `onTranscriptionEvent`. Cancel it with `cancel(streamId)` | ||||||||||||||||||||||
| */ | ||||||||||||||||||||||
| startTranscriptionStream( | ||||||||||||||||||||||
| streamId: string, | ||||||||||||||||||||||
| options: NativeTranscriptionOptions | ||||||||||||||||||||||
| ): Promise<void> | ||||||||||||||||||||||
| appendTranscriptionAudio(streamId: string, pcmBase64: string): Promise<void> | ||||||||||||||||||||||
| /** Ends the audio and resolves with the whole text */ | ||||||||||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · finishTranscriptionStream is the only method in the spec wrapped across lines for no length reason The neighbouring one-argument methods (
Suggested change
Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Prettier formats that line. It's over the 80-column limit on one line. 🤖 Reply from Claude Code |
||||||||||||||||||||||
| finishTranscriptionStream( | ||||||||||||||||||||||
| streamId: string | ||||||||||||||||||||||
| ): Promise<NativeStreamTranscriptionResult> | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| getEmbeddingModelInfo(): Promise<NativeEmbeddingModelInfo> | ||||||||||||||||||||||
| prepareEmbedding(): Promise<void> | ||||||||||||||||||||||
| embed(values: string[], taskType?: string): Promise<number[][]> | ||||||||||||||||||||||
|
|
@@ -171,6 +201,7 @@ export interface Spec extends TurboModule { | |||||||||||||||||||||
|
|
||||||||||||||||||||||
| onStreamEvent: CodegenTypes.EventEmitter<StreamEvent> | ||||||||||||||||||||||
| onDownloadProgress: CodegenTypes.EventEmitter<DownloadProgressEvent> | ||||||||||||||||||||||
| onTranscriptionEvent: CodegenTypes.EventEmitter<TranscriptionEvent> | ||||||||||||||||||||||
| } | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
| const NativeAndroidAI: Spec | null = | ||||||||||||||||||||||
|
|
||||||||||||||||||||||
Uh oh!
There was an error while loading. Please reload this page.