Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 60 additions & 2 deletions harness/android/transcription.harness.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,30 @@
import { android } from '@react-native-ai/android'
import { experimental_transcribe as transcribe } from 'ai'
import {
experimental_streamTranscribe as streamTranscribe,
experimental_transcribe as transcribe,
} from 'ai'
import { describe, expect, it } from 'react-native-harness'

import { base64ToBytes, speech } from '../fixtures'
import { base64ToBytes, speech, speechPcm8k } from '../fixtures'

const model = android.transcriptionModel({ language: 'en-US', mode: 'basic' })

// The 8 kHz fixture at the 16 kHz ML Kit streams, by linear interpolation of
// its little-endian samples
function to16k(pcm8k: Uint8Array) {
Comment thread
grabbou marked this conversation as resolved.
Resolved
const view = new DataView(pcm8k.buffer, pcm8k.byteOffset, pcm8k.byteLength)
const sampleCount = Math.floor(pcm8k.byteLength / 2)
const output = new Int16Array(sampleCount * 2)
for (let index = 0; index < sampleCount; index++) {
const sample = view.getInt16(index * 2, true)
const next =
index + 1 < sampleCount ? view.getInt16((index + 1) * 2, true) : sample
output[index * 2] = sample
output[index * 2 + 1] = (sample + next) >> 1
}
return new Uint8Array(output.buffer)
}

describe('android transcription', () => {
it('reports recognizer availability', async () => {
const info = await model.getModelInfo()
Expand Down Expand Up @@ -33,4 +52,43 @@ describe('android transcription', () => {
expect(text).toMatch(/quick brown fox/i)
expect(text).toMatch(/paris/i)
})

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · Harness stream test asserts a phrase survives the delta chain and logs JSON into the run output

In harness/android/transcription.harness.ts the new case asserts expect(deltas).toMatch(/quick brown fox/i), which only holds if that phrase arrives inside one segment and every partial was a strict extension of the previous one — the JS side deliberately drops a delta when text.startsWith(sent) fails (ai-sdk.ts:1056), so a recognizer that rewrites a partial makes this device test fail on a correct implementation. The timing check is also behind if (first!.startSecond !== undefined), so it passes without asserting anything when word timings are absent. The console.log('[android] streamed segments:', JSON.stringify(segments)) line dumps raw native output on every run; assert on the shape instead (or gate the log).

Details

persona: jvm

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. It's a device test against one known clip, and the log is for reading the device run. The timing check depends on whether the device returns word timings.

🤖 Reply from Claude Code

it('streams PCM with deltas and timed segments', async () => {
const pcm = to16k(base64ToBytes(speechPcm8k))
// 100 ms chunks, as a microphone sends them
const audio = new ReadableStream<Uint8Array>({
start(controller) {
for (let offset = 0; offset < pcm.length; offset += 3200) {
controller.enqueue(pcm.subarray(offset, offset + 3200))
}
controller.close()
},
})

const result = streamTranscribe({
model,
inputAudioFormat: { type: 'audio/pcm', rate: 16000 },
audio,
})
let deltas = ''
const segments: {
text: string
startSecond?: number
endSecond?: number
}[] = []
for await (const part of result.fullStream) {
if (part.type === 'transcript-delta') deltas += part.delta
if (part.type === 'transcript-final') segments.push(part)
}
console.log('[android] streamed segments:', JSON.stringify(segments))

expect(await result.text).toMatch(/quick brown fox/i)
expect(deltas).toMatch(/quick brown fox/i)
expect(segments.length).toBeGreaterThan(0)
// A one-word segment ends as it starts
const [first] = segments
if (first!.startSecond !== undefined) {
expect(first!.endSecond).toBeGreaterThanOrEqual(first!.startSecond)
}
})
})
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.callstack.ai.android

import android.os.ParcelFileDescriptor
import android.util.Base64
import com.facebook.react.bridge.Arguments
import com.facebook.react.bridge.Promise
import com.facebook.react.bridge.ReactApplicationContext
Expand All @@ -8,6 +10,8 @@ import com.facebook.react.bridge.ReadableMap
import com.facebook.react.bridge.ReadableType
import com.facebook.react.bridge.WritableMap
import com.google.mlkit.genai.common.FunctionCall
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.CoroutineStart
import kotlinx.coroutines.Dispatchers
Expand All @@ -16,6 +20,8 @@ import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import java.io.IOException
import java.io.OutputStream
import java.util.concurrent.ConcurrentHashMap

class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISpec(reactContext) {
Expand All @@ -41,6 +47,22 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp
// Generations and transcriptions, by id, so they can be cancelled
private val activeRuns = ConcurrentHashMap<String, Job>()

// Streamed transcriptions: both ends of the pipe audio is written to, and the result once the
Comment thread
grabbou marked this conversation as resolved.
Resolved
// stream ends. Both ends close when recognition ends, so a stream JS never finishes holds no
// file descriptors
private class TranscriptionStream(
val read: ParcelFileDescriptor,
val audio: OutputStream,
val result: CompletableDeferred<WritableMap>,
) {
fun close() {
runCatching { audio.close() }
runCatching { read.close() }
}
}

private val transcriptionStreams = ConcurrentHashMap<String, TranscriptionStream>()

private fun Promise.rejectWithCode(error: Exception) {
reject(AndroidAIErrors.codeFor(error), error.message ?: "Unknown ML Kit error", error)
}
Expand Down Expand Up @@ -159,6 +181,8 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp

override fun cancel(runId: String) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟡 Minor · Cancelled/errored stream never calls native finish, so the JS finish part can be dropped and the native stream entry leaks

AndroidTranscriptionModel.doStream never calls native.finishTranscriptionStream when the consumer cancels or the call aborts: onAbort()/the catch path only call native.cancel(streamId). On the native side cancel() (AndroidAIModule.kt:181-185) removes the entry from transcriptionStreams and closes only the write end of the pipe, so ML Kit sees EOF, the collector completes normally, and the run then calls stream.result.complete(...) (AndroidAIModule.kt:248-254) with the full text — but nothing on the JS side is awaiting that deferred any more, so the text of a transcription that finished successfully after a cancel is silently thrown away (the finishTranscriptionStream mock is asserted NOT to be called in the new test, which locks the behavior in). Conversely, when a run ends on its own (recognition error or natural end) the map entry is never removed by that path: only cancel and finishTranscriptionStream remove it, so a JS consumer that abandons the stream without either call keeps a TranscriptionStream (and its CompletableDeferred + WritableMap) in transcriptionStreams for the lifetime of the module even though close() already released the fds. The comment on lines 49-51 claims the fds are held only until recognition ends, which is true, but the map entry is not cleaned up.

Suggested change

Either resolve the abandoned run's result (e.g. emit a final event carrying the text, or expose the deferred to a cancel promise) so a late cancellation does not discard a completed transcription, and remove the transcriptionStreams entry from the run's own finally when recognition ends so a consumer that neither finishes nor cancels does not keep the entry alive.

Details

persona: jvm

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's intended: a cancel means no result, so the text is dropped. The leftover entry was answered in the earlier thread. doStream always ends with finish or cancel, and the pipe is closed by then.

🤖 Reply from Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · cancel(runId) silently becomes a two-registry namespace, so a shared id cancels both a text stream and a transcription stream

cancel now clears both registries for the same id: activeRuns.remove(runId)?.cancel() and transcriptionStreams.remove(runId)?.let { ... } (AndroidAIModule.kt:181-185). Ids come from generateId() in JS today, so it works, but startTranscriptionStream only checks transcriptionStreams for duplicates (line 200) and streamText only checks activeRuns (line 142): an id used by a live text stream is accepted as a transcription stream id, and one cancel(id) then tears down both unrelated runs. The single-cancel-namespace contract is now implicit across four methods rather than stated anywhere.

Suggested change

Either document that a runId must be unique across streaming text, task streaming and transcription streams, or key each registry with a kind prefix so the duplicate-id check covers both.

Details

persona: jvm

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. Run ids are unique per call across text, task and transcription streams, since they all come from generateId().

🤖 Reply from Claude Code

activeRuns.remove(runId)?.cancel()
// Only the write end: ML Kit may still be reading the other, which the run closes as it ends
transcriptionStreams.remove(runId)?.let { runCatching { it.audio.close() } }
}

override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) =
Expand All @@ -172,6 +196,95 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp
promise.resolve(transcription.transcribe(audio, TranscriptionConfig.from(options)))
}

override fun startTranscriptionStream(streamId: String, options: ReadableMap, promise: Promise) =

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved from an earlier thread.

🔵 Nit · check-then-act on transcriptionStreams in startTranscriptionStream can leak a pipe pair

startTranscriptionStream guards with if (transcriptionStreams.containsKey(streamId)) (line 200) and only later does transcriptionStreams[streamId] = stream (line 205). These are two separate ConcurrentHashMap operations, and the module's scope is Dispatchers.IO (line 30), so two calls with the same id can run concurrently — and they can, because cancel(runId) only removes from transcriptionStreams, it does not stop a launch(promise) body already past the contains check, and startTranscriptionStream is not tracked in activeRuns before its job is registered. The loser's ParcelFileDescriptor.createPipe() pair (line 203) is then overwritten and never closed: the write end is an AutoCloseOutputStream nobody holds a reference to and the read fd leaks until the process dies. launchRun's own activeRuns[runId] = job (line 97) silently replaces an in-flight job the same way. Use putIfAbsent for the stream so the second caller gets the documented GENERATION_ERROR and closes its own fds.

Suggested change

Create the pipe and insert it with transcriptionStreams.putIfAbsent(streamId, stream); if a stream is already mapped, close the freshly created pipe pair and throw the duplicate-id error, so the guard and the insert are one atomic step.

Details

Also raised here

Also raised at this location by senior-developer: startTranscriptionStream's duplicate-id check is a non-atomic containsKey + put — if (transcriptionStreams.containsKey(streamId)) throw ... (line 200) followed by transcriptionStreams[streamId] = stream (line 205) is check-then-act on a ConcurrentHashMap, and the check runs on the launch(promise) coroutine rather than the caller thread, so two startTranscriptionStream calls for the same id can both pass the check and the second replaces the first entry. The first TranscriptionStream's pipe ends are then unreachable from the map: cancel() and finishTranscriptionStream can no longer find them, so its read fd is only released when ML Kit's own run happens to end, and its result can never be awaited. It works today only because generateId() in ai-sdk.ts:1014 makes an id collision unlikely — the same guard in streamText (line 142) at least runs synchronously on the caller thread. Use putIfAbsent and reject when it returns non-null, and create the pipe only after the slot is claimed.

persona: jvm, senior-developer, maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. Ids come from generateId() per call, so two starts never share one.

🤖 Reply from Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · Stream entry stays in transcriptionStreams when recognition fails and JS never finishes or cancels

startTranscriptionStream inserts the stream into transcriptionStreams (line 206) and the failure path only completes the deferred exceptionally and closes the fds (lines 211-214) — the map entry is removed exclusively by finishTranscriptionStream (line 281) or cancel (line 185). If the recognizer fails right away (e.g. the PERMISSION_DENIED the PR description reports for every recognition call on the Pixel 9 Pro) and the consumer reacts to the error thrown by the first appendTranscriptionAudio by dropping the stream without calling finish or cancel, the entry — holding the streamId, the config and the deferred — is retained until invalidate(). The class comment claims "a stream JS never finishes holds no file descriptors": true for the fds, but not for the map, which grows one entry per abandoned stream.

Suggested change

Remove the entry from transcriptionStreams when the run ends, in launchRun's completion handler (or in the onError path), so an abandoned stream does not stay in the map until invalidate().

Details

persona: senior-developer

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the earlier thread: the entry stays so the next call can report why recognition ended, and doStream always ends with finish or cancel.

🤖 Reply from Claude Code

launch(promise) {
if (transcriptionStreams.containsKey(streamId)) {
throw AndroidAIErrors.CodedException(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId")
}
val (read, write) = ParcelFileDescriptor.createPipe()
val stream = TranscriptionStream(read, ParcelFileDescriptor.AutoCloseOutputStream(write), CompletableDeferred())
transcriptionStreams[streamId] = stream

// The collector calls back one response at a time, so the count needs no lock
var segmentIndex = 0
// A failure is reported by the next append, or by finish, which removes the stream
launchRun(streamId, { error ->
stream.result.completeExceptionally(error)
stream.close()
}) {
val config = TranscriptionConfig.from(options)
val text = transcription.transcribeStream(read, config) { response ->
emitOnTranscriptionEvent(
Arguments.createMap().apply {
putString("streamId", streamId)
putInt("segmentIndex", segmentIndex)
when (response) {
is SpeechRecognizerResponse.PartialTextResponse -> {
putString("type", "partial")
putString("text", response.text)
}
is SpeechRecognizerResponse.FinalTextResponse -> {
putString("type", "final")
putString("text", response.text)
putArray(
"words",
Arguments.createArray().apply {
response.wordTimings.forEach { word ->
pushMap(
Arguments.createMap().apply {
putString("text", word.text)
putDouble("startMs", word.timestampMs.toDouble())
},
)
}
},
)
segmentIndex++
}
else -> {}
}
},
)
}
stream.close()
stream.result.complete(
Arguments.createMap().apply {
putString("text", text)
putString("language", transcription.languageOf(config))
},
)
}
null
}

override fun appendTranscriptionAudio(streamId: String, pcmBase64: String, promise: Promise) =
Comment thread
grabbou marked this conversation as resolved.
Resolved

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟡 Minor · finishTranscriptionStream can await a deferred that nothing ever completes (hung promise, stream never removed)

finishTranscriptionStream removes the stream from transcriptionStreams and then suspends on stream.result.await() (AndroidAIModule.kt:270-277). The deferred is completed only by the launchRun body (success or onError). If the run job is cancelled instead (e.g. cancel(streamId) from a competing abort path on the same id, or the map entry removed by cancel between finish's remove and ... more concretely: after finish removes the entry, a subsequent cancel(streamId) is a no-op, so cancellation cannot be the trigger after finish — but before finish, cancel(streamId) removes the entry, and then a late finishTranscriptionStream(streamId) rejects 'Unknown transcription stream' while the recognizer keeps reading from the still-open write end until the pipe EOFs. In the other ordering, if the module's invalidate() races a pending finish, scope.cancel() kills the collector coroutine that owns the await, and the TurboModule promise is never resolved or rejected on that path (launch's promise machinery is itself in the cancelled scope), leaving the JS await native.finishTranscriptionStream(...) pending forever. The design comment 'A failure is reported by the next append, or by finish' does not cover the case where the failure is a cancellation of the collector: the job's finally only does activeRuns.remove(runId) (AndroidAIModule.kt:93-95), so a collector that dies without hitting either the success complete or the onError callback leaves the CompletableDeferred un-completed and finish hanging — which in JS means doStream never reaches finish/error and the stream is neither closed nor errored.

Suggested change

Guarantee the deferred always settles: complete it in the launchRun block's own finally (e.g. completeExceptionally with a cancellation/invalidated error if still pending) rather than relying on success/onError only, and resolve/reject finish instead of hanging when the run is gone.

Details

Also raised here

Also raised at this location by senior-developer: Aborted stream leaks the native pipe fds and the running recognizer job (JS never cancels after abort) — In doStream, when the audio source ends because the abort signal fired, the loop breaks at if (done || cancelled || abortSignal?.aborted), then if (abortSignal?.aborted) throw getAbortReason(abortSignal) throws before native.finishTranscriptionStream(streamId) is called. The catch block only calls native.cancel(streamId) in the error path; here the abort check short-circuits to a throw whose catch... does run native.cancel — but only if the throw happens inside the try. It does, so cancel is called; the leak is in the sibling case: when done is true and abortSignal.aborted becomes true between the loop break and the abort check... — concretely, the real leak is different and simpler: after break on done, if abortSignal?.aborted is true, native.cancel(streamId) runs (good). But when the consumer merely stops reading a cancelled stream, the native side is fine too. The actual leak: the throw of getAbortReason is wrapped by toAndroidAIError? …

Also raised at this location by jvm: finish is awaited forever if the consumer cancels (or aborts) while await native.finishTranscriptionStream(...) is pending — In AndroidTranscriptionModel.doStream, the audio loop breaks when done || cancelled || abortSignal?.aborted and then reaches const result = await native.finishTranscriptionStream(streamId) (packages/android/src/ai-sdk.ts:1121) only when cancelled was false at the last check. The cancelled flag and abortSignal.aborted are only re-checked before that await, never after. …

persona: senior-developer, jvm

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. A failed run completes result exceptionally, which finish rethrows. A cancel cancels the run job, whose onError completes it. Either way the await returns.

🤖 Reply from Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · Unknown-stream error reuses INVALID_MESSAGE while start uses GENERATION_ERROR

appendTranscriptionAudio reuses AndroidAIErrors.INVALID_MESSAGE ("Invalid message") for what is a bad streamId, while startTranscriptionStream one screen above rejects the same class of problem ("Stream ID already in use") with GENERATION_ERROR. The unknown-stream error is the one a JS caller is most likely to see in practice — recognition dying early makes every later append take this path — so a generic "invalid message" code makes it hard to tell a dead stream from wrong arguments, and the three sibling stream methods don't share a code. Use one code (GENERATION_ERROR or a new INVALID_STREAM) in both append and finish.

Suggested change

Use one error code for unknown/duplicate stream ids across start/append/finish instead of mixing GENERATION_ERROR and INVALID_MESSAGE.

Details

persona: maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. An unknown id on append or finish is a bad argument from the caller, while a duplicate id on start is a failure to start.

🤖 Reply from Claude Code

Comment thread
grabbou marked this conversation as resolved.
Resolved
launch(promise) {
val stream = transcriptionStreams[streamId]
?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId")
// Recognition ended early: rethrow why, or drop audio it can no longer hear
if (stream.result.isCompleted) {
stream.result.await()
return@launch null
}
try {
stream.audio.write(Base64.decode(pcmBase64, Base64.DEFAULT))
} catch (error: IOException) {
// Recognition ended, and closed the pipe, while this append was writing
if (!stream.result.isCompleted) throw error
stream.result.await()
}
null
}

override fun finishTranscriptionStream(streamId: String, promise: Promise) =

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟠 Major · Audio still in the kernel pipe is discarded when finish closes the write end right after a native promise resolves

appendTranscriptionAudio resolves as soon as OutputStream.write() returns, but AutoCloseOutputStream.write only copies bytes into the pipe buffer — no read-side acknowledgement. In JS, the read loop awaits the append and, when the audio source is at EOF, immediately calls finishTranscriptionStream, which does runCatching { stream.audio.close() } (AndroidAIModule.kt:284). If ML Kit has not drained the pipe yet (the normal case at 100 ms/3200-byte chunks, where JS can be far ahead of real-time consumption), closing the write end discards the bytes still sitting in the pipe buffer, so the last word(s) spoken are never recognised. JS then reports result.text (from finishTranscriptionStream) and segments/deltas that are missing that tail, while expect(result.text).toMatch(/quick brown fox/i) in the harness only passes by luck. Nothing separates await audio.read() -> done from the close: there is no flush, and OutputStream.flush() on a pipe is a no-op anyway, so the only fix is to stop writing only after the reader has consumed, e.g. keep the write end open until recognition reports it has seen EOF / a small idle window, or have JS delay closing after the last append.

Suggested change

Do not treat a resolved append as acknowledgement that ML Kit consumed the audio. Either keep the write end open until the recognizer signals it reached EOF (or a short idle window after the last append), or have the JS loop wait for recognition to catch up before calling finishTranscriptionStream, so bytes buffered in the pipe are not dropped by the close.

Details

persona: senior-developer

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Closing a pipe's write end doesn't discard what's still buffered. The reader drains the remaining bytes and only then sees EOF, which is POSIX pipe behavior. So finish closes the write end without losing the tail.

🤖 Reply from Claude Code

launch(promise) {
val stream = transcriptionStreams.remove(streamId)
?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown transcription stream: $streamId")
// Closing the pipe ends the audio, so recognition finishes what it has heard
runCatching { stream.audio.close() }
stream.result.await()
}

override fun getEmbeddingModelInfo(promise: Promise) =
launch(promise) {
val info = embeddings.getModelInfo()
Comment thread
grabbou marked this conversation as resolved.
Resolved
Expand Down Expand Up @@ -217,6 +330,11 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp
}

override fun invalidate() {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟠 Major · invalidate() deadlocks every pending appendTranscriptionAudio: the deferred is cancelled but append awaits it without a timeout

invalidate() puts each stream into the state append treats as 'recognition ended early' — it closes the pipes and calls stream.result.cancel() (AndroidAIModule.kt:323-328), which completes the CompletableDeferred in the cancelled state. Then any appendTranscriptionAudio already dispatched (or dispatched before transcriptionStreams.clear() takes effect, and any that was suspended at stream.result.await() when invalidation happens) takes the if (stream.result.isCompleted) stream.result.await() branch at AndroidAIModule.kt:263-266. isCompleted is true for a cancelled deferred, but await() on a cancelled deferred never returns: in kotlinx.coroutines a cancelled job's await() awaits its completion forever, unlike an exceptionally-completed one. So that coroutine suspends permanently (and its promise is never settled) for as long as the scope survives — which it does, because scope.cancel() only stops new work and never interrupts a coroutine already suspended on another job's completion. The JS side then hangs on await native.appendTranscriptionAudio(...) in the doStream audio loop, and since the ReadableStream's start() never resolves, the consumer's streamTranscribe promise never settles either. Note finishTranscriptionStream (line 276) has the same await shape, but there launch already rejects when !scope.isActive, so append is the reachable path.

Suggested change

Complete the result exceptionally instead of cancelling it in invalidate() (e.g. stream.result.completeExceptionally(invalidated())), and/or make append check isCancelled and throw the invalidation error rather than awaiting a cancelled deferred. Same guard applies to finishTranscriptionStream's await if a stream's deferred can ever be cancelled.

Details

persona: senior-developer

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. await() on a cancelled Deferred throws CancellationException right away; it doesn't suspend. launch catches that and rejects the promise, and scope.cancel() cancels any append still suspended, which rejects the same way.

🤖 Reply from Claude Code

transcriptionStreams.values.forEach { stream ->
stream.close()
stream.result.cancel()
}
transcriptionStreams.clear()
activeRuns.values.forEach { it.cancel() }
activeRuns.clear()
scope.cancel()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.callstack.ai.android

import android.content.Context
import android.os.ParcelFileDescriptor
import android.util.Base64
import com.facebook.react.bridge.ReadableMap
import com.google.mlkit.genai.common.FeatureStatus
Expand All @@ -9,6 +10,7 @@ import com.google.mlkit.genai.speechrecognition.SpeechRecognition
import com.google.mlkit.genai.speechrecognition.SpeechRecognizer
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions
import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
import com.google.mlkit.genai.speechrecognition.recognitionOptions
Comment thread
grabbou marked this conversation as resolved.
Resolved
import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions
import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest
import kotlinx.coroutines.sync.Mutex
Expand Down Expand Up @@ -55,6 +57,10 @@ class Transcription(
"Not a BCP 47 language tag: $tag. Use a tag such as en-US",
)

/** The language the recognizer transcribes, as a BCP 47 tag. */

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟡 Minor · finish.language now comes from the device locale when the model has no language, unlike doGenerate

doStream reports result.language.split(/[-_]/)[0] (ai-sdk.ts:1126) where the native value comes from Transcription.languageOf (Transcription.kt:60-62) — config.locale ?: Locale.getDefault(). doGenerate for the same model reports this.options.locale?.split(/[-_]/)[0] (ai-sdk.ts:972), i.e. undefined. So android.transcriptionModel({}) (no language) yields finish.language === 'en' (or whatever the device says) through experimental_streamTranscribe but no language through experimental_transcribe, and the language reported was never chosen by the caller. The two paths should agree, and the docs should say which one is authoritative; the streaming section of website/src/content/docs/docs/android/transcription.md documents neither.

Suggested change

Pick one source for finish/doGenerate language (the configured locale, falling back to the device tag in both paths) and note it on the Android transcription docs page.

Details

Also raised here

Also raised at this location by jvm: New native stream result carries a language the JS side derives by splitting, and the sample rate the caller declares is never sent to native — appendTranscriptionAudio/startTranscriptionStream never pass the declared sample rate to native (NativeAndroidAI.ts:171-179, AndroidAIModule.kt:198), so the 16 kHz contract lives only in the JS guard in doStream plus the Transcription.kt KDoc ("16 kHz mono 16-bit PCM"): nothing on the native side checks or uses the rate, so a caller that lies about the rate gets silently wrong audio. Symmetrically, the language now comes back from native via languageOf(config) (AndroidAIModule.kt:252, Transcription.kt:60-62), which resolves to Locale.getDefault() when the model has no explicit locale, and JS immediately reduces it with result.language.split(/[-_]/)[0] (ai-sdk.ts:1126) — the full BCP 47 tag the new NativeStreamTranscriptionResult documents is discarded on arrival, unlike doGenerate, which reports this.options.locale's prefix. …

persona: senior-developer, jvm

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the earlier thread: streaming reports the locale ML Kit recognized with. File transcription reports only the locale you asked for, because native doesn't return one there.

🤖 Reply from Claude Code

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟡 Minor · Language reported for a stream comes from the config, not the device default it claims to fall back to

languageOf returns Locale.getDefault().toLanguageTag() when no locale was configured (Transcription.kt:61-62). finishTranscriptionStream puts that in language, and JS reduces it to the ISO-639-1 prefix (result.language.split(/[-_]/)[0], ai-sdk.ts:1129), so the finish part reports whatever the device's default locale happens to be — potentially a language the recognizer did not transcribe — with no warning, unlike doGenerate which simply reports this.options.locale (ai-sdk.ts:972). Callers that branch on result.language get a value that silently varies per device for the same model config. Reporting the default is only correct when ML Kit really used it; nothing here verifies that, and ML Kit can pick a different model.

Suggested change

Either omit language when no locale was configured (as doGenerate effectively does), or document that the reported language falls back to the device default and is not what the recognizer confirmed; do not present a device-dependent tag as the transcription language.

Details

Also raised here

Also raised at this location by maintainability: languageOf reads config.locale, a field never referenced anywhere else in the file — languageOf (and so the language field of NativeStreamTranscriptionResult) resolves to Locale.getDefault() whenever config.locale is null. Every other language decision in Transcription goes through the recognizer client built by client(config), whose locale is not read back here — and the recognizer may auto-detect. TranscriptionConfig is parsed from the JS options, so config.locale looks like a field no caller sets; if so the whole ?.let(::locale) branch is dead and the reported language is just the device locale, not what was transcribed. Either read the language off the recognizer/response, or drop the branch and say in the doc comment that this is the configured (or device) locale, not a detected one.

persona: senior-developer, maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The recognizer is built with that same locale: Transcription.kt:43 sets config.locale ?: Locale.getDefault(), and languageOf returns the same value. So language is the locale ML Kit transcribed with, not a guess.

🤖 Reply from Claude Code

fun languageOf(config: TranscriptionConfig): String =
(config.locale?.let(::locale) ?: Locale.getDefault()).toLanguageTag()

suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo =
ModelInfo(availability = availabilityOf(client(config).checkStatus()))

Expand Down Expand Up @@ -103,4 +109,36 @@ class Transcription(
file.delete()
}
}

/**
* Transcribes 16 kHz mono 16-bit PCM read from [audio] until it is closed, reporting each
* partial and final result with its word timings. Returns the whole text.
*/
suspend fun transcribeStream(
audio: ParcelFileDescriptor,
config: TranscriptionConfig,
onResult: (response: SpeechRecognizerResponse) -> Unit,
): String {
prepare(config)
AudioSource.fromPfd(audio, AudioSource.Mode.STREAMING).use { source ->
val text = StringBuilder()
val request = speechRecognizerRequest {
audioSource = source
recognitionOptions { enableWordTiming = true }
}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · An ErrorResponse after the first segment is swallowed: doStream resolves with empty text

Transcription.kt:135 throws on SpeechRecognizerResponse.ErrorResponse, which ends transcribeStream and never reaches stream.result.complete(...) in AndroidAIModule.kt:253-258. If that error arrives after one or more FinalTextResponses — i.e. after doStream has already emitted transcript-delta/transcript-final parts — startTranscriptionStream has already resolved, so the JS side surfaces it from appendTranscriptionAudio/finishTranscriptionStream, and ai-sdk.ts routes it to the catch: the stream is errored and no finish part is ever enqueued, so the ai caller awaiting result.text gets '' instead of the partial text or an error. Carry the accumulated text through the completion so a late error still resolves with what was heard.

Suggested change

Either complete the deferred with the text collected so far (plus the error as a separate signal) or push the error into the same result the JS side awaits, so a mid-stream ML Kit error after a final segment does not leave the caller with an empty text and no finish part.

Details

persona: maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's intended: an ML Kit error ends the stream with that error, as the other providers do. The text heard so far has already arrived as delta and final parts.

🤖 Reply from Claude Code

client(config).startRecognition(request).collect { response ->
when (response) {
is SpeechRecognizerResponse.PartialTextResponse -> onResult(response)
is SpeechRecognizerResponse.FinalTextResponse -> {
if (text.isNotEmpty() && response.text.isNotEmpty()) text.append(' ')
text.append(response.text)
onResult(response)
}
is SpeechRecognizerResponse.ErrorResponse -> throw response.e
else -> {}
}
}
return text.toString()
}
}
}
31 changes: 31 additions & 0 deletions packages/android/src/NativeAndroidAI.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,22 @@ export interface NativeTranscriptionOptions {
mode?: string
}

export interface TranscriptionEvent {

@apex-sentinel-staging apex-sentinel-staging Bot Nov 1, 2026 •

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 Nit · TranscriptionEvent.type is a bare string while the sibling StreamEvent types are literal unions

type: string lets any caller pass or compare an arbitrary string, and the only documentation of the legal values is a prose comment. The event types emitted by the native side are exactly 'partial' | 'final', and ai-sdk.ts:1050 compares against the 'final' literal. Narrowing to a union makes that comparison checked and keeps the JS side honest if a third response type is ever emitted natively instead of swallowed by the else -> {} branch in AndroidAIModule.kt.

Suggested change
export interface TranscriptionEvent {
export interface TranscriptionEvent {
streamId: string
/** `partial` text of the segment so far, or its `final` text */
type: 'partial' | 'final'
segmentIndex: number
text: string
/** Final results only: each word and when it starts */
words?: { text: string; startMs: number }[]
}
Details

persona: maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this: StreamEvent.type and DownloadProgressEvent.phase in this file are plain strings too, and I'm keeping event payloads plain for codegen.

🤖 Addressed by Claude Code

streamId: string
/** `partial` text of the segment so far, or its `final` text */

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · TranscriptionEvent.type is a plain string, not the 'partial' | 'final' union it documents

The doc comment two lines above enumerates the values, and ai-sdk.ts compares event.type === 'final', so typing it as string gives the new TS-side consumer none of the checking the native when(response) branches already have. availability/status being string elsewhere is a different case: those are mapped from an enum whose values cross the docs, not branched on in this package.

Suggested change
/** `partial` text of the segment so far, or its `final` text */
/** `partial` text of the segment so far, or its `final` text */
type: "partial" | "final"

Suggested change

Type type as "partial" | "final" (or a shared exported union) so the emitter and the doStream listener agree at compile time.

Details

persona: maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the earlier thread: the event type stays a codegen-friendly string, and the listener compares it against the two values.

🤖 Reply from Claude Code

type: string
segmentIndex: number
text: string
/** Final results only: each word and when it starts */
words?: { text: string; startMs: number }[]
}

export interface NativeStreamTranscriptionResult {
text: string
/** BCP 47 locale the recognizer transcribed, such as en-US */
language: string
}

export interface NativeEmbeddingModelInfo {
availability: string
dimension?: number
Expand Down Expand Up @@ -148,6 +164,20 @@ export interface Spec extends TurboModule {
options: NativeTranscriptionOptions
): Promise<string>

/**
* Starts transcribing 16 kHz mono 16-bit PCM appended to the stream, with
* results as `onTranscriptionEvent`. Cancel it with `cancel(streamId)`
*/
startTranscriptionStream(
streamId: string,
options: NativeTranscriptionOptions
): Promise<void>
appendTranscriptionAudio(streamId: string, pcmBase64: string): Promise<void>
/** Ends the audio and resolves with the whole text */

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · finishTranscriptionStream is the only method in the spec wrapped across lines for no length reason

The neighbouring one-argument methods (prepareTranscription(): Promise<void>, getEmbeddingModelInfo(), and appendTranscriptionAudio(streamId: string, pcmBase64: string) two lines above) sit on a single line; finishTranscriptionStream(streamId: string) fits well inside the width and its multi-line form reads as if it had more parameters. Put it on one line.

Suggested change
/** Ends the audio and resolves with the whole text */
appendTranscriptionAudio(streamId: string, pcmBase64: string): Promise<void>
/** Ends the audio and resolves with the whole text */
finishTranscriptionStream(streamId: string): Promise<NativeStreamTranscriptionResult>
Details

persona: maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prettier formats that line. It's over the 80-column limit on one line.

🤖 Reply from Claude Code

finishTranscriptionStream(
streamId: string
): Promise<NativeStreamTranscriptionResult>

getEmbeddingModelInfo(): Promise<NativeEmbeddingModelInfo>
prepareEmbedding(): Promise<void>
embed(values: string[], taskType?: string): Promise<number[][]>
Expand All @@ -171,6 +201,7 @@ export interface Spec extends TurboModule {

onStreamEvent: CodegenTypes.EventEmitter<StreamEvent>
onDownloadProgress: CodegenTypes.EventEmitter<DownloadProgressEvent>
onTranscriptionEvent: CodegenTypes.EventEmitter<TranscriptionEvent>
}

const NativeAndroidAI: Spec | null =
Expand Down
Loading