From 2dab75401d1489fa5f538e3681d68e9293b14998 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 25 Oct 2026 04:19:59 +0400 Subject: [PATCH 1/4] feat(adk): add on-device transcription model with ml kit speech recognition Co-Authored-By: Claude Opus 5.5 --- packages/adk/android/build.gradle | 1 + .../com/callstack/ai/adk/AdkTranscriber.kt | 354 ++++++++++++++++++ .../callstack/ai/adk/NativeAdkEngineModule.kt | 68 +++- packages/adk/src/NativeAdkEngine.ts | 24 +- packages/adk/src/__tests__/ai-sdk.test.ts | 113 ++++++ packages/adk/src/ai-sdk.ts | 135 ++++++- packages/adk/src/index.ts | 2 + 7 files changed, 684 insertions(+), 13 deletions(-) create mode 100644 packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt diff --git a/packages/adk/android/build.gradle b/packages/adk/android/build.gradle index 0f5c53f7..0000ab86 100644 --- a/packages/adk/android/build.gradle +++ b/packages/adk/android/build.gradle @@ -96,6 +96,7 @@ dependencies { implementation "com.google.adk:google-adk-kotlin-core:1.4.0" implementation "com.google.adk:google-adk-kotlin-mlkit-android:1.4.0-beta" implementation "com.google.mlkit:genai-prompt:1.0.0-beta4" + implementation "com.google.mlkit:genai-speech-recognition:1.0.0-alpha1" } react { diff --git a/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt b/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt new file mode 100644 index 00000000..91cb6b85 --- /dev/null +++ b/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt @@ -0,0 +1,354 @@ +package com.callstack.ai.adk + +import android.content.Context +import android.media.AudioFormat +import android.media.MediaCodec +import android.media.MediaExtractor +import android.media.MediaFormat +import android.os.ParcelFileDescriptor +import com.facebook.react.bridge.ReadableMap +import com.google.mlkit.genai.common.DownloadStatus +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.GenAiException +import com.google.mlkit.genai.common.audio.AudioSource +import com.google.mlkit.genai.speechrecognition.SpeechRecognition +import com.google.mlkit.genai.speechrecognition.SpeechRecognizer +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.takeWhile +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import java.io.File +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.Locale + +data class TranscriptionConfig(val language: String?, val mode: String?) { + companion object { + fun from(map: ReadableMap?): TranscriptionConfig = + TranscriptionConfig( + language = map?.takeIf { it.hasKey("language") }?.getString("language"), + mode = map?.takeIf { it.hasKey("mode") }?.getString("mode"), + ) + } +} + +data class TranscriptionResult(val text: String, val durationInSeconds: Double) + +/** On-device speech to text with ML Kit GenAI speech recognition. */ +class AdkTranscriber( + private val context: Context, + private val onDownloadProgress: ( + phase: String, + bytesDownloaded: Long?, + bytesToDownload: Long?, + ) -> Unit, +) { + private fun client(config: TranscriptionConfig): SpeechRecognizer = + SpeechRecognition.getClient( + speechRecognizerOptions { + locale = config.language?.let { Locale.forLanguageTag(it) } ?: Locale.getDefault() + preferredMode = + if (config.mode == "advanced") { + SpeechRecognizerOptions.Mode.MODE_ADVANCED + } else { + SpeechRecognizerOptions.Mode.MODE_BASIC + } + }, + ) + + suspend fun getAvailability(config: TranscriptionConfig): String = + client(config).use { recognizer -> + val status = + try { + recognizer.checkStatus() + } catch (_: Exception) { + return "unknown" + } + when (status) { + FeatureStatus.UNAVAILABLE -> "unavailable" + FeatureStatus.DOWNLOADABLE -> "downloadable" + FeatureStatus.DOWNLOADING -> "downloading" + FeatureStatus.AVAILABLE -> "available" + else -> "unknown" + } + } + + /** Downloads the speech recognition model for the locale when needed. */ + suspend fun prepare(config: TranscriptionConfig) { + client(config).use { prepare(it) } + } + + private suspend fun prepare(recognizer: SpeechRecognizer) { + val status = recognizer.checkStatus() + when (status) { + FeatureStatus.AVAILABLE -> return + FeatureStatus.UNAVAILABLE -> + throw AdkErrors.AdkException( + AdkErrors.MODEL_UNAVAILABLE, + "Speech recognition is not supported for this locale and mode on this device", + ) + } + + try { + recognizer.download().collect { downloadStatus -> + when (downloadStatus) { + is DownloadStatus.DownloadStarted -> + onDownloadProgress("started", 0, downloadStatus.bytesToDownload) + is DownloadStatus.DownloadProgress -> + onDownloadProgress("progress", downloadStatus.totalBytesDownloaded, null) + is DownloadStatus.DownloadCompleted -> onDownloadProgress("completed", null, null) + is DownloadStatus.DownloadFailed -> + throw AdkErrors.AdkException( + AdkErrors.DOWNLOAD_FAILED, + "Speech recognition model download failed: ${downloadStatus.e.message}", + downloadStatus.e, + ) + } + } + } catch (error: GenAiException) { + if (status == FeatureStatus.DOWNLOADING) { + throw AdkErrors.AdkException( + AdkErrors.MODEL_DOWNLOADING, + "The speech recognition model is still downloading", + error, + ) + } + throw error + } + } + + suspend fun transcribe(audio: ByteArray, config: TranscriptionConfig): TranscriptionResult = + withContext(Dispatchers.IO) { + val pcm = decodeToPcm(audio) + + client(config).use { recognizer -> + prepare(recognizer) + + val (readSide, writeSide) = ParcelFileDescriptor.createPipe() + val request = speechRecognizerRequest { + audioSource = AudioSource.fromPfd(readSide, AudioSource.Mode.STREAMING, PCM_FORMAT) + } + val finalTexts = mutableListOf() + var partialText = "" + + coroutineScope { + val writer = launch { writePcm(writeSide, pcm) } + // Recognition only ends on its own when ML Kit detects the end of the input + val stopper = launch { + writer.join() + delay(STOP_GRACE_MS) + recognizer.stopRecognition() + } + + try { + recognizer.startRecognition(request) + .takeWhile { it !is SpeechRecognizerResponse.CompletedResponse } + .collect { response -> + when (response) { + is SpeechRecognizerResponse.FinalTextResponse -> { + finalTexts += response.text + partialText = "" + } + is SpeechRecognizerResponse.PartialTextResponse -> partialText = response.text + is SpeechRecognizerResponse.ErrorResponse -> throw response.e + else -> Unit + } + } + } finally { + writer.cancel() + stopper.cancel() + // Unblocks a writer stuck on a full pipe + runCatching { readSide.close() } + } + } + + TranscriptionResult( + text = (finalTexts + partialText).map { it.trim() }.filter { it.isNotEmpty() } + .joinToString(" "), + durationInSeconds = pcm.size / 2.0 / SAMPLE_RATE, + ) + } + } + + /** + * ML Kit reads raw PCM from the descriptor at real-time pace, so the audio is written in + * chunks spaced by their duration rather than all at once. + */ + private suspend fun writePcm(writeSide: ParcelFileDescriptor, pcm: ByteArray) { + ParcelFileDescriptor.AutoCloseOutputStream(writeSide).use { output -> + val startedAt = System.nanoTime() + var offset = 0 + var chunk = 0 + while (offset < pcm.size) { + val waitMs = (startedAt + chunk * CHUNK_MS * 1_000_000L - System.nanoTime()) / 1_000_000L + if (waitMs > 0) delay(waitMs) + val length = minOf(CHUNK_BYTES, pcm.size - offset) + try { + output.write(pcm, offset, length) + } catch (_: IOException) { + // Recognition finished and closed the read side + return + } + offset += length + chunk++ + } + } + } + + /** Decodes any audio container Android supports into 16 kHz mono 16-bit PCM. */ + private fun decodeToPcm(audio: ByteArray): ByteArray { + val file = File.createTempFile("adk-transcription", null, context.cacheDir) + val extractor = MediaExtractor() + var codec: MediaCodec? = null + try { + file.writeBytes(audio) + try { + extractor.setDataSource(file.path) + } catch (error: IOException) { + throw AdkErrors.AdkException( + AdkErrors.INVALID_MESSAGE, + "Could not read the audio. Use a format Android can decode, such as WAV, MP3, AAC or Ogg", + error, + ) + } + + val track = (0 until extractor.trackCount).firstOrNull { + extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true + } ?: throw AdkErrors.AdkException(AdkErrors.INVALID_MESSAGE, "The audio has no audio track") + extractor.selectTrack(track) + val format = extractor.getTrackFormat(track) + + val decoder = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!) + codec = decoder + decoder.configure(format, null, null, 0) + decoder.start() + + val samples = FloatSamples() + var sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE) + val info = MediaCodec.BufferInfo() + var inputDone = false + + while (true) { + if (!inputDone) { + val inputIndex = decoder.dequeueInputBuffer(CODEC_TIMEOUT_US) + if (inputIndex >= 0) { + val size = extractor.readSampleData(decoder.getInputBuffer(inputIndex)!!, 0) + if (size < 0) { + decoder.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + } else { + decoder.queueInputBuffer(inputIndex, 0, size, extractor.sampleTime, 0) + extractor.advance() + } + } + } + + val outputIndex = decoder.dequeueOutputBuffer(info, CODEC_TIMEOUT_US) + if (outputIndex < 0) continue + + val outputFormat = decoder.getOutputFormat(outputIndex) + sampleRate = outputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) + val buffer = decoder.getOutputBuffer(outputIndex)!! + buffer.position(info.offset) + buffer.limit(info.offset + info.size) + samples.appendMono( + buffer.order(ByteOrder.LITTLE_ENDIAN), + channels = outputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT), + encoding = + if (outputFormat.containsKey(MediaFormat.KEY_PCM_ENCODING)) { + outputFormat.getInteger(MediaFormat.KEY_PCM_ENCODING) + } else { + AudioFormat.ENCODING_PCM_16BIT + }, + ) + decoder.releaseOutputBuffer(outputIndex, false) + + if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) break + } + + return samples.toPcm16(sampleRate) + } finally { + codec?.let { + runCatching { it.stop() } + it.release() + } + extractor.release() + file.delete() + } + } + + /** Growable buffer of mono samples in [-1, 1]. */ + private class FloatSamples { + private var data = FloatArray(SAMPLE_RATE) + private var size = 0 + + fun appendMono(buffer: ByteBuffer, channels: Int, encoding: Int) { + when (encoding) { + AudioFormat.ENCODING_PCM_16BIT -> { + val shorts = buffer.asShortBuffer() + while (shorts.remaining() >= channels) { + var sum = 0f + repeat(channels) { sum += shorts.get() / 32768f } + add(sum / channels) + } + } + AudioFormat.ENCODING_PCM_FLOAT -> { + val floats = buffer.asFloatBuffer() + while (floats.remaining() >= channels) { + var sum = 0f + repeat(channels) { sum += floats.get() } + add(sum / channels) + } + } + else -> + throw AdkErrors.AdkException( + AdkErrors.INVALID_MESSAGE, + "Unsupported PCM encoding $encoding. Use 16-bit or float audio", + ) + } + } + + private fun add(sample: Float) { + if (size == data.size) data = data.copyOf(data.size * 2) + data[size++] = sample + } + + /** Linearly resamples to [SAMPLE_RATE] and converts to little-endian 16-bit PCM. */ + fun toPcm16(sourceRate: Int): ByteArray { + val outputSize = (size.toLong() * SAMPLE_RATE / sourceRate).toInt() + val output = ByteBuffer.allocate(outputSize * 2).order(ByteOrder.LITTLE_ENDIAN) + val step = sourceRate.toDouble() / SAMPLE_RATE + for (index in 0 until outputSize) { + val position = index * step + val left = position.toInt().coerceAtMost(size - 1) + val right = (left + 1).coerceAtMost(size - 1) + val fraction = (position - left).toFloat() + val sample = data[left] + (data[right] - data[left]) * fraction + output.putShort((sample.coerceIn(-1f, 1f) * Short.MAX_VALUE).toInt().toShort()) + } + return output.array() + } + } + + companion object { + private const val SAMPLE_RATE = 16_000 + private const val CHUNK_MS = 100 + private const val CHUNK_BYTES = SAMPLE_RATE * 2 * CHUNK_MS / 1000 + private const val STOP_GRACE_MS = 1_500L + private const val CODEC_TIMEOUT_US = 10_000L + + private val PCM_FORMAT: AudioFormat = + AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(SAMPLE_RATE) + .setChannelMask(AudioFormat.CHANNEL_IN_MONO) + .build() + } +} diff --git a/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt b/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt index 0348a547..72a40060 100644 --- a/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt +++ b/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt @@ -1,5 +1,6 @@ package com.callstack.ai.adk +import android.util.Base64 import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -37,7 +38,20 @@ class NativeAdkEngineModule(reactContext: ReactApplicationContext) : }, ) - // Streams and generateText runs, by id, so both can be cancelled + private val transcriber = AdkTranscriber( + context = reactContext, + onDownloadProgress = { phase, bytesDownloaded, bytesToDownload -> + emitOnTranscriptionDownloadProgress( + Arguments.createMap().apply { + putString("phase", phase) + bytesDownloaded?.let { putDouble("bytesDownloaded", it.toDouble()) } + bytesToDownload?.let { putDouble("bytesToDownload", it.toDouble()) } + }, + ) + }, + ) + + // Streams, generateText runs and transcriptions, by id, so all can be cancelled private val activeRuns = ConcurrentHashMap() private fun Promise.rejectWithCode(error: Exception) { @@ -189,6 +203,58 @@ class NativeAdkEngineModule(reactContext: ReactApplicationContext) : promise.resolve(null) } + override fun getTranscriptionAvailability(config: ReadableMap?, promise: Promise) { + scope.launch { + try { + promise.resolve(transcriber.getAvailability(TranscriptionConfig.from(config))) + } catch (error: Exception) { + promise.rejectWithCode(error) + } + } + } + + override fun prepareTranscription(config: ReadableMap?, promise: Promise) { + scope.launch { + try { + transcriber.prepare(TranscriptionConfig.from(config)) + promise.resolve(null) + } catch (error: Exception) { + promise.rejectWithCode(error) + } + } + } + + override fun transcribe( + requestId: String, + audio: String, + config: ReadableMap?, + promise: Promise, + ) { + val job = scope.launch(start = CoroutineStart.LAZY) { + try { + val bytes = + try { + Base64.decode(audio, Base64.DEFAULT) + } catch (error: IllegalArgumentException) { + throw AdkErrors.AdkException(AdkErrors.INVALID_MESSAGE, "Audio is not valid base64", error) + } + val result = transcriber.transcribe(bytes, TranscriptionConfig.from(config)) + promise.resolve( + Arguments.createMap().apply { + putString("text", result.text) + putDouble("durationInSeconds", result.durationInSeconds) + }, + ) + } catch (error: Exception) { + promise.rejectWithCode(error) + } finally { + activeRuns.remove(requestId) + } + } + activeRuns[requestId] = job + job.start() + } + private fun emitDelta(streamId: String, delta: String, kind: String) { emitOnStreamUpdate( Arguments.createMap().apply { diff --git a/packages/adk/src/NativeAdkEngine.ts b/packages/adk/src/NativeAdkEngine.ts index 62249194..79008f31 100644 --- a/packages/adk/src/NativeAdkEngine.ts +++ b/packages/adk/src/NativeAdkEngine.ts @@ -88,6 +88,18 @@ export interface NanoDownloadProgressEvent { bytesToDownload?: number } +export interface AdkTranscriptionConfig { + /** BCP 47 locale such as `en-US`. Defaults to the device locale */ + language?: string + /** `basic` (default) or `advanced`, which uses a GenAI model on supported devices */ + mode?: 'basic' | 'advanced' +} + +export interface AdkTranscriptionResult { + text: string + durationInSeconds: number +} + export interface Spec extends TurboModule { getModelInfo(nano?: AdkNanoModelConfig): Promise prepareNano(nano?: AdkNanoModelConfig): Promise @@ -107,13 +119,23 @@ export interface Spec extends TurboModule { options?: AdkGenerationOptions ): Promise - /** Cancels a `streamText` or `generateText` run by its id */ + getTranscriptionAvailability(config?: AdkTranscriptionConfig): Promise + prepareTranscription(config?: AdkTranscriptionConfig): Promise + /** Transcribes base64-encoded audio in any format Android can decode */ + transcribe( + requestId: string, + audio: string, + config?: AdkTranscriptionConfig + ): Promise + + /** Cancels a stream, a `generateText` run or a transcription by its id */ cancelStream(streamId: string): Promise onStreamUpdate: CodegenTypes.EventEmitter onStreamComplete: CodegenTypes.EventEmitter onStreamError: CodegenTypes.EventEmitter onNanoDownloadProgress: CodegenTypes.EventEmitter + onTranscriptionDownloadProgress: CodegenTypes.EventEmitter } const NativeAdkEngine: Spec | null = diff --git a/packages/adk/src/__tests__/ai-sdk.test.ts b/packages/adk/src/__tests__/ai-sdk.test.ts index 761031c6..eaf09d83 100644 --- a/packages/adk/src/__tests__/ai-sdk.test.ts +++ b/packages/adk/src/__tests__/ai-sdk.test.ts @@ -22,6 +22,7 @@ const onStreamUpdate = createEmitter() const onStreamComplete = createEmitter() const onStreamError = createEmitter() const onNanoDownloadProgress = createEmitter() +const onTranscriptionDownloadProgress = createEmitter() let generateTextResult: Record = { content: 'ok', @@ -50,6 +51,17 @@ const getModelInfo = mock(async (_nano?: unknown) => ({ availability: 'downloadable', })) +const transcribe = mock( + async (_requestId: string, _audio: string, _config?: unknown) => ({ + text: 'Hello world', + durationInSeconds: 2.5, + }) +) +const prepareTranscription = mock(async (_config?: unknown) => {}) +const getTranscriptionAvailability = mock( + async (_config?: unknown) => 'downloadable' +) + mock.module('react-native', () => ({ Platform: { OS: 'android' }, TurboModuleRegistry: { @@ -63,6 +75,10 @@ mock.module('react-native', () => ({ onStreamComplete, onStreamError, onNanoDownloadProgress, + transcribe, + prepareTranscription, + getTranscriptionAvailability, + onTranscriptionDownloadProgress, }), }, })) @@ -487,3 +503,100 @@ describe('Gemini Nano preparation', () => { }) }) }) + +describe('transcription', () => { + test('transcribes binary audio as base64', async () => { + const model = createAdkProvider().transcriptionModel({ + language: 'en-US', + mode: 'advanced', + }) + + const result = await model.doGenerate({ + audio: new Uint8Array([1, 2, 3]), + mediaType: 'audio/wav', + }) + + const [, audio, config] = transcribe.mock.calls.at(-1)! + expect(audio).toBe('AQID') + expect(config).toEqual({ language: 'en-US', mode: 'advanced' }) + expect(result).toMatchObject({ + text: 'Hello world', + segments: [], + language: 'en-US', + durationInSeconds: 2.5, + warnings: [], + response: { modelId: 'genai-speech-recognition' }, + }) + }) + + test('strips data URL prefixes from base64 audio', async () => { + await createAdkProvider().transcriptionModel().doGenerate({ + audio: 'data:audio/wav;base64,AQID', + mediaType: 'audio/wav', + }) + + expect(transcribe.mock.calls.at(-1)![1]).toBe('AQID') + }) + + test('cancels the native request on abort', async () => { + const controller = new AbortController() + transcribe.mockImplementationOnce(async (requestId) => { + controller.abort() + expect(cancelStream.mock.calls.at(-1)![0]).toBe(requestId) + throw new Error('cancelled') + }) + + await expect( + createAdkProvider().transcriptionModel().doGenerate({ + audio: 'AQID', + mediaType: 'audio/wav', + abortSignal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + + test('maps native errors to AdkErrorCodes', async () => { + transcribe.mockImplementationOnce(async () => { + throw Object.assign(new Error('Not supported'), { + code: 'MODEL_UNAVAILABLE', + }) + }) + + await expect( + createAdkProvider() + .transcriptionModel() + .doGenerate({ audio: 'AQID', mediaType: 'audio/wav' }) + ).rejects.toMatchObject({ code: 'MODEL_UNAVAILABLE' }) + }) + + test('reports availability', async () => { + const model = createAdkProvider().transcriptionModel({ language: 'fr-FR' }) + + expect(await model.getAvailability()).toBe('downloadable') + expect(await model.isAvailable()).toBe(true) + expect(getTranscriptionAvailability.mock.calls.at(-1)![0]).toEqual({ + language: 'fr-FR', + }) + + getTranscriptionAvailability.mockImplementationOnce(async () => 'weird') + expect(await model.getAvailability()).toBe('unknown') + }) + + test('reports download progress while preparing', async () => { + const progress: unknown[] = [] + prepareTranscription.mockImplementationOnce(async () => { + onTranscriptionDownloadProgress.emit({ + phase: 'progress', + bytesDownloaded: 50, + }) + }) + + await createAdkProvider() + .transcriptionModel() + .prepare({ onProgress: (event) => progress.push(event) }) + + expect(progress).toEqual([ + { phase: 'progress', bytesDownloaded: 50, bytesToDownload: undefined }, + ]) + }) +}) diff --git a/packages/adk/src/ai-sdk.ts b/packages/adk/src/ai-sdk.ts index 49c5a928..7c4c0985 100644 --- a/packages/adk/src/ai-sdk.ts +++ b/packages/adk/src/ai-sdk.ts @@ -8,9 +8,12 @@ import type { LanguageModelV4StreamPart, LanguageModelV4Usage, SharedV4Warning, + TranscriptionModelV4, + TranscriptionModelV4CallOptions, + TranscriptionModelV4Result, } from '@ai-sdk/provider' import { generateId, resolveFullMediaType } from '@ai-sdk/provider-utils' -import { Platform } from 'react-native' +import { type CodegenTypes, Platform } from 'react-native' import { AdkErrorCodes, createAdkError, toAdkError } from './errors' import { @@ -19,6 +22,7 @@ import { type AdkMessage, type AdkMessagePart, type AdkNanoModelConfig, + type AdkTranscriptionConfig, type AdkUsageMetadata, getNativeAdkEngine, type NanoDownloadProgressEvent, @@ -75,6 +79,16 @@ export interface AdkLanguageModel extends LanguageModelV4 { countTokens: (text: string) => Promise } +export type AdkTranscriptionOptions = AdkTranscriptionConfig + +export interface AdkTranscriptionModel extends TranscriptionModelV4 { + /** Whether speech recognition for this locale and mode is downloaded or downloadable */ + isAvailable: () => Promise + getAvailability: () => Promise + /** Downloads the speech recognition model, reporting download progress */ + prepare: (options?: AdkPrepareOptions) => Promise +} + export interface AdkProvider { (options?: AdkLanguageModelOptions): AdkLanguageModel readonly specificationVersion: 'v4' @@ -85,6 +99,10 @@ export interface AdkProvider { getModelInfo: (options?: { nano?: AdkNanoModelConfig }) => Promise + /** On-device speech to text with ML Kit GenAI speech recognition */ + transcriptionModel: ( + options?: AdkTranscriptionOptions + ) => AdkTranscriptionModel } const toModelAvailability = (value: string): AdkModelAvailability => { @@ -100,18 +118,17 @@ const toModelAvailability = (value: string): AdkModelAvailability => { } const listenForDownloadProgress = ( + subscribe: CodegenTypes.EventEmitter, onProgress: AdkPrepareOptions['onProgress'] ) => onProgress - ? getNativeAdkEngine().onNanoDownloadProgress( - (event: NanoDownloadProgressEvent) => { - onProgress({ - phase: event.phase as AdkDownloadProgress['phase'], - bytesDownloaded: event.bytesDownloaded, - bytesToDownload: event.bytesToDownload, - }) - } - ) + ? subscribe((event: NanoDownloadProgressEvent) => { + onProgress({ + phase: event.phase as AdkDownloadProgress['phase'], + bytesDownloaded: event.bytesDownloaded, + bytesToDownload: event.bytesToDownload, + }) + }) : null async function getModelInfo( @@ -150,6 +167,8 @@ export function createAdkProvider( } provider.getModelInfo = (options: { nano?: AdkNanoModelConfig } = {}) => getModelInfo(options.nano ?? defaults.nano) + provider.transcriptionModel = (options?: AdkTranscriptionOptions) => + new AdkSpeechTranscriptionModel(options) return provider } @@ -318,7 +337,10 @@ class AdkChatLanguageModel implements AdkLanguageModel { } async prepare(options: AdkPrepareOptions = {}): Promise { - const listener = listenForDownloadProgress(options.onProgress) + const listener = listenForDownloadProgress( + getNativeAdkEngine().onNanoDownloadProgress, + options.onProgress + ) try { await this.ensurePrepared() } finally { @@ -684,3 +706,94 @@ class AdkChatLanguageModel implements AdkLanguageModel { } } } + +class AdkSpeechTranscriptionModel implements AdkTranscriptionModel { + readonly specificationVersion = 'v4' + readonly provider = 'adk' + readonly modelId = 'genai-speech-recognition' + + private readonly config: AdkTranscriptionConfig + + constructor(options: AdkTranscriptionOptions = {}) { + this.config = options + } + + async getAvailability(): Promise { + if (Platform.OS !== 'android') { + return 'unavailable' + } + const availability = await getNativeAdkEngine() + .getTranscriptionAvailability(this.config) + .catch((error: unknown) => { + throw toAdkError(error) + }) + return toModelAvailability(availability) + } + + async isAvailable(): Promise { + const availability = await this.getAvailability() + return availability === 'available' || availability === 'downloadable' + } + + async prepare(options: AdkPrepareOptions = {}): Promise { + const native = getNativeAdkEngine() + const listener = listenForDownloadProgress( + native.onTranscriptionDownloadProgress, + options.onProgress + ) + try { + await native.prepareTranscription(this.config) + } catch (error) { + throw toAdkError(error) + } finally { + listener?.remove() + } + } + + async doGenerate( + options: TranscriptionModelV4CallOptions + ): Promise { + const native = getNativeAdkEngine() + const requestId = generateId() + const abortSignal = options.abortSignal + + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + const onAbort = () => { + void native + .cancelStream(requestId) + .catch((error) => console.warn('[adk] cancel failed:', error)) + } + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + const audio = + typeof options.audio === 'string' + ? options.audio.replace(/^data:[^;]+;base64,/, '') + : uint8ArrayToBase64(options.audio) + + try { + const result = await native.transcribe(requestId, audio, this.config) + return { + text: result.text, + // ML Kit speech recognition does not report timestamps + segments: [], + language: this.config.language, + durationInSeconds: result.durationInSeconds, + warnings: [], + response: { + timestamp: new Date(), + modelId: this.modelId, + }, + } + } catch (error) { + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + throw toAdkError(error) + } finally { + abortSignal?.removeEventListener('abort', onAbort) + } + } +} diff --git a/packages/adk/src/index.ts b/packages/adk/src/index.ts index 3f71d399..c081a220 100644 --- a/packages/adk/src/index.ts +++ b/packages/adk/src/index.ts @@ -7,6 +7,8 @@ export type { AdkPrepareOptions, AdkProvider, AdkProviderOptions, + AdkTranscriptionModel, + AdkTranscriptionOptions, } from './ai-sdk' export { adk, createAdkProvider } from './ai-sdk' export type { AdkError, AdkErrorCode } from './errors' From 0fa270c5cb685ac1974db3f9683fe40bcebe8c55 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Thu, 29 Oct 2026 22:14:46 +0100 Subject: [PATCH 2/4] refactor(adk): let ml kit decode the audio to transcribe ML Kit speech recognition 1.0.0-alpha1 now fails with "Api access revoked". alpha2 reads whole files with AudioSource.fromFile and decodes WAV, MP3, FLAC, M4A and Ogg itself, so drop the provider's own decoding, resampling and real-time streaming. ML Kit reports no duration for files, so durationInSeconds is undefined. alpha2 is built with Kotlin 2.4, which apps now need. Co-Authored-By: Claude Opus 5.5 --- packages/adk/android/build.gradle | 2 +- .../com/callstack/ai/adk/AdkTranscriber.kt | 240 ++---------------- .../callstack/ai/adk/NativeAdkEngineModule.kt | 1 - packages/adk/src/NativeAdkEngine.ts | 1 - packages/adk/src/__tests__/ai-sdk.test.ts | 3 +- packages/adk/src/ai-sdk.ts | 2 +- 6 files changed, 19 insertions(+), 230 deletions(-) diff --git a/packages/adk/android/build.gradle b/packages/adk/android/build.gradle index 0000ab86..b675b10c 100644 --- a/packages/adk/android/build.gradle +++ b/packages/adk/android/build.gradle @@ -96,7 +96,7 @@ dependencies { implementation "com.google.adk:google-adk-kotlin-core:1.4.0" implementation "com.google.adk:google-adk-kotlin-mlkit-android:1.4.0-beta" implementation "com.google.mlkit:genai-prompt:1.0.0-beta4" - implementation "com.google.mlkit:genai-speech-recognition:1.0.0-alpha1" + implementation "com.google.mlkit:genai-speech-recognition:1.0.0-alpha2" } react { diff --git a/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt b/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt index 91cb6b85..dda6c2f3 100644 --- a/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt +++ b/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt @@ -1,11 +1,6 @@ package com.callstack.ai.adk import android.content.Context -import android.media.AudioFormat -import android.media.MediaCodec -import android.media.MediaExtractor -import android.media.MediaFormat -import android.os.ParcelFileDescriptor import com.facebook.react.bridge.ReadableMap import com.google.mlkit.genai.common.DownloadStatus import com.google.mlkit.genai.common.FeatureStatus @@ -18,15 +13,9 @@ import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay import kotlinx.coroutines.flow.takeWhile -import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.io.File -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.ByteOrder import java.util.Locale data class TranscriptionConfig(val language: String?, val mode: String?) { @@ -39,7 +28,7 @@ data class TranscriptionConfig(val language: String?, val mode: String?) { } } -data class TranscriptionResult(val text: String, val durationInSeconds: Double) +data class TranscriptionResult(val text: String) /** On-device speech to text with ML Kit GenAI speech recognition. */ class AdkTranscriber( @@ -124,231 +113,34 @@ class AdkTranscriber( } } + /** Transcribes a whole audio file. ML Kit decodes formats such as WAV, MP3, M4A and Ogg. */ suspend fun transcribe(audio: ByteArray, config: TranscriptionConfig): TranscriptionResult = withContext(Dispatchers.IO) { - val pcm = decodeToPcm(audio) - - client(config).use { recognizer -> - prepare(recognizer) - - val (readSide, writeSide) = ParcelFileDescriptor.createPipe() - val request = speechRecognizerRequest { - audioSource = AudioSource.fromPfd(readSide, AudioSource.Mode.STREAMING, PCM_FORMAT) - } - val finalTexts = mutableListOf() - var partialText = "" - - coroutineScope { - val writer = launch { writePcm(writeSide, pcm) } - // Recognition only ends on its own when ML Kit detects the end of the input - val stopper = launch { - writer.join() - delay(STOP_GRACE_MS) - recognizer.stopRecognition() - } + val file = File.createTempFile("adk-transcription", null, context.cacheDir) + try { + file.writeBytes(audio) + client(config).use { recognizer -> + prepare(recognizer) - try { + AudioSource.fromFile(file).use { source -> + val request = speechRecognizerRequest { audioSource = source } + val texts = mutableListOf() recognizer.startRecognition(request) .takeWhile { it !is SpeechRecognizerResponse.CompletedResponse } .collect { response -> when (response) { - is SpeechRecognizerResponse.FinalTextResponse -> { - finalTexts += response.text - partialText = "" - } - is SpeechRecognizerResponse.PartialTextResponse -> partialText = response.text + is SpeechRecognizerResponse.FinalTextResponse -> texts += response.text is SpeechRecognizerResponse.ErrorResponse -> throw response.e else -> Unit } } - } finally { - writer.cancel() - stopper.cancel() - // Unblocks a writer stuck on a full pipe - runCatching { readSide.close() } - } - } - - TranscriptionResult( - text = (finalTexts + partialText).map { it.trim() }.filter { it.isNotEmpty() } - .joinToString(" "), - durationInSeconds = pcm.size / 2.0 / SAMPLE_RATE, - ) - } - } - - /** - * ML Kit reads raw PCM from the descriptor at real-time pace, so the audio is written in - * chunks spaced by their duration rather than all at once. - */ - private suspend fun writePcm(writeSide: ParcelFileDescriptor, pcm: ByteArray) { - ParcelFileDescriptor.AutoCloseOutputStream(writeSide).use { output -> - val startedAt = System.nanoTime() - var offset = 0 - var chunk = 0 - while (offset < pcm.size) { - val waitMs = (startedAt + chunk * CHUNK_MS * 1_000_000L - System.nanoTime()) / 1_000_000L - if (waitMs > 0) delay(waitMs) - val length = minOf(CHUNK_BYTES, pcm.size - offset) - try { - output.write(pcm, offset, length) - } catch (_: IOException) { - // Recognition finished and closed the read side - return - } - offset += length - chunk++ - } - } - } - - /** Decodes any audio container Android supports into 16 kHz mono 16-bit PCM. */ - private fun decodeToPcm(audio: ByteArray): ByteArray { - val file = File.createTempFile("adk-transcription", null, context.cacheDir) - val extractor = MediaExtractor() - var codec: MediaCodec? = null - try { - file.writeBytes(audio) - try { - extractor.setDataSource(file.path) - } catch (error: IOException) { - throw AdkErrors.AdkException( - AdkErrors.INVALID_MESSAGE, - "Could not read the audio. Use a format Android can decode, such as WAV, MP3, AAC or Ogg", - error, - ) - } - - val track = (0 until extractor.trackCount).firstOrNull { - extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true - } ?: throw AdkErrors.AdkException(AdkErrors.INVALID_MESSAGE, "The audio has no audio track") - extractor.selectTrack(track) - val format = extractor.getTrackFormat(track) - - val decoder = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)!!) - codec = decoder - decoder.configure(format, null, null, 0) - decoder.start() - - val samples = FloatSamples() - var sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE) - val info = MediaCodec.BufferInfo() - var inputDone = false - - while (true) { - if (!inputDone) { - val inputIndex = decoder.dequeueInputBuffer(CODEC_TIMEOUT_US) - if (inputIndex >= 0) { - val size = extractor.readSampleData(decoder.getInputBuffer(inputIndex)!!, 0) - if (size < 0) { - decoder.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) - inputDone = true - } else { - decoder.queueInputBuffer(inputIndex, 0, size, extractor.sampleTime, 0) - extractor.advance() - } - } - } - - val outputIndex = decoder.dequeueOutputBuffer(info, CODEC_TIMEOUT_US) - if (outputIndex < 0) continue - - val outputFormat = decoder.getOutputFormat(outputIndex) - sampleRate = outputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) - val buffer = decoder.getOutputBuffer(outputIndex)!! - buffer.position(info.offset) - buffer.limit(info.offset + info.size) - samples.appendMono( - buffer.order(ByteOrder.LITTLE_ENDIAN), - channels = outputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT), - encoding = - if (outputFormat.containsKey(MediaFormat.KEY_PCM_ENCODING)) { - outputFormat.getInteger(MediaFormat.KEY_PCM_ENCODING) - } else { - AudioFormat.ENCODING_PCM_16BIT - }, - ) - decoder.releaseOutputBuffer(outputIndex, false) - - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) break - } - - return samples.toPcm16(sampleRate) - } finally { - codec?.let { - runCatching { it.stop() } - it.release() - } - extractor.release() - file.delete() - } - } - - /** Growable buffer of mono samples in [-1, 1]. */ - private class FloatSamples { - private var data = FloatArray(SAMPLE_RATE) - private var size = 0 - - fun appendMono(buffer: ByteBuffer, channels: Int, encoding: Int) { - when (encoding) { - AudioFormat.ENCODING_PCM_16BIT -> { - val shorts = buffer.asShortBuffer() - while (shorts.remaining() >= channels) { - var sum = 0f - repeat(channels) { sum += shorts.get() / 32768f } - add(sum / channels) - } - } - AudioFormat.ENCODING_PCM_FLOAT -> { - val floats = buffer.asFloatBuffer() - while (floats.remaining() >= channels) { - var sum = 0f - repeat(channels) { sum += floats.get() } - add(sum / channels) + TranscriptionResult( + text = texts.map { it.trim() }.filter { it.isNotEmpty() }.joinToString(" "), + ) } } - else -> - throw AdkErrors.AdkException( - AdkErrors.INVALID_MESSAGE, - "Unsupported PCM encoding $encoding. Use 16-bit or float audio", - ) + } finally { + file.delete() } } - - private fun add(sample: Float) { - if (size == data.size) data = data.copyOf(data.size * 2) - data[size++] = sample - } - - /** Linearly resamples to [SAMPLE_RATE] and converts to little-endian 16-bit PCM. */ - fun toPcm16(sourceRate: Int): ByteArray { - val outputSize = (size.toLong() * SAMPLE_RATE / sourceRate).toInt() - val output = ByteBuffer.allocate(outputSize * 2).order(ByteOrder.LITTLE_ENDIAN) - val step = sourceRate.toDouble() / SAMPLE_RATE - for (index in 0 until outputSize) { - val position = index * step - val left = position.toInt().coerceAtMost(size - 1) - val right = (left + 1).coerceAtMost(size - 1) - val fraction = (position - left).toFloat() - val sample = data[left] + (data[right] - data[left]) * fraction - output.putShort((sample.coerceIn(-1f, 1f) * Short.MAX_VALUE).toInt().toShort()) - } - return output.array() - } - } - - companion object { - private const val SAMPLE_RATE = 16_000 - private const val CHUNK_MS = 100 - private const val CHUNK_BYTES = SAMPLE_RATE * 2 * CHUNK_MS / 1000 - private const val STOP_GRACE_MS = 1_500L - private const val CODEC_TIMEOUT_US = 10_000L - - private val PCM_FORMAT: AudioFormat = - AudioFormat.Builder() - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .setSampleRate(SAMPLE_RATE) - .setChannelMask(AudioFormat.CHANNEL_IN_MONO) - .build() - } } diff --git a/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt b/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt index 72a40060..98001074 100644 --- a/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt +++ b/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt @@ -242,7 +242,6 @@ class NativeAdkEngineModule(reactContext: ReactApplicationContext) : promise.resolve( Arguments.createMap().apply { putString("text", result.text) - putDouble("durationInSeconds", result.durationInSeconds) }, ) } catch (error: Exception) { diff --git a/packages/adk/src/NativeAdkEngine.ts b/packages/adk/src/NativeAdkEngine.ts index 79008f31..73caae5c 100644 --- a/packages/adk/src/NativeAdkEngine.ts +++ b/packages/adk/src/NativeAdkEngine.ts @@ -97,7 +97,6 @@ export interface AdkTranscriptionConfig { export interface AdkTranscriptionResult { text: string - durationInSeconds: number } export interface Spec extends TurboModule { diff --git a/packages/adk/src/__tests__/ai-sdk.test.ts b/packages/adk/src/__tests__/ai-sdk.test.ts index eaf09d83..7b11fe8d 100644 --- a/packages/adk/src/__tests__/ai-sdk.test.ts +++ b/packages/adk/src/__tests__/ai-sdk.test.ts @@ -54,7 +54,6 @@ const getModelInfo = mock(async (_nano?: unknown) => ({ const transcribe = mock( async (_requestId: string, _audio: string, _config?: unknown) => ({ text: 'Hello world', - durationInSeconds: 2.5, }) ) const prepareTranscription = mock(async (_config?: unknown) => {}) @@ -523,7 +522,7 @@ describe('transcription', () => { text: 'Hello world', segments: [], language: 'en-US', - durationInSeconds: 2.5, + durationInSeconds: undefined, warnings: [], response: { modelId: 'genai-speech-recognition' }, }) diff --git a/packages/adk/src/ai-sdk.ts b/packages/adk/src/ai-sdk.ts index 7c4c0985..3473ccde 100644 --- a/packages/adk/src/ai-sdk.ts +++ b/packages/adk/src/ai-sdk.ts @@ -780,7 +780,7 @@ class AdkSpeechTranscriptionModel implements AdkTranscriptionModel { // ML Kit speech recognition does not report timestamps segments: [], language: this.config.language, - durationInSeconds: result.durationInSeconds, + durationInSeconds: undefined, warnings: [], response: { timestamp: new Date(), From 513433c8796a41aafaa4bd2d5dac59305b89d644 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Thu, 29 Oct 2026 22:14:46 +0100 Subject: [PATCH 3/4] docs(adk): document the transcription model Co-Authored-By: Claude Opus 5.5 --- packages/adk/README.md | 1 + website/astro.config.mjs | 1 + .../content/docs/docs/adk/getting-started.mdx | 1 + .../content/docs/docs/adk/transcription.md | 109 ++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 website/src/content/docs/docs/adk/transcription.md diff --git a/packages/adk/README.md b/packages/adk/README.md index 72f9eec8..d343e8c2 100644 --- a/packages/adk/README.md +++ b/packages/adk/README.md @@ -25,6 +25,7 @@ const { text } = await generateText({ - ✅ On-device Gemini Nano through ML Kit GenAI, with model selection - ✅ Availability, download progress and token counting +- ✅ On-device transcription with ML Kit GenAI speech recognition - ✅ Streaming - ✅ Image prompts - ✅ Token usage diff --git a/website/astro.config.mjs b/website/astro.config.mjs index cef56585..33ef2792 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -98,6 +98,7 @@ export default defineConfig({ ['getting-started', 'Getting started'], ['generating', 'Generating'], ['models', 'Models and availability'], + ['transcription', 'Transcription'], ]), group('Core AI', 'core-ai', [ ['getting-started', 'Getting started'], diff --git a/website/src/content/docs/docs/adk/getting-started.mdx b/website/src/content/docs/docs/adk/getting-started.mdx index ffc2b016..9b8c233f 100644 --- a/website/src/content/docs/docs/adk/getting-started.mdx +++ b/website/src/content/docs/docs/adk/getting-started.mdx @@ -90,3 +90,4 @@ if (await adk.isAvailable()) { - [Generating](/docs/adk/generating): streaming, image input, token usage and errors - [Models & Availability](/docs/adk/models): checking for Gemini Nano, preparing it and provider options +- [Transcription](/docs/adk/transcription): on-device speech to text diff --git a/website/src/content/docs/docs/adk/transcription.md b/website/src/content/docs/docs/adk/transcription.md new file mode 100644 index 00000000..e370eb60 --- /dev/null +++ b/website/src/content/docs/docs/adk/transcription.md @@ -0,0 +1,109 @@ +--- +title: "Transcription" +--- + +Convert speech to text on the device with ML Kit's +[GenAI Speech Recognition API](https://developers.google.com/ml-kit/genai/speech-recognition/android). +`adk.transcriptionModel()` returns an AI SDK transcription model, and the audio +never leaves the device. + +:::caution +ML Kit GenAI speech recognition is in alpha, and Google may change it in ways +that break this provider. +::: + +## Requirements + +- A physical Android device with a locked bootloader. Emulators aren't supported. +- Android 12 (API 31) or newer for `basic` mode, and a device with Gemini Nano, + such as a Pixel 10, for `advanced` mode. +- Kotlin 2.4 in your app, which ML Kit's speech recognition is built with. With + Expo, set it with `expo-build-properties`: + +```json +{ + "expo": { + "plugins": [ + ["expo-build-properties", { "android": { "kotlinVersion": "2.4.0" } }] + ] + } +} +``` + +In a bare React Native app, set `android.kotlinVersion=2.4.0` in +`android/gradle.properties`. + +## Usage + +```typescript +import { adk } from '@react-native-ai/adk' +import { experimental_transcribe as transcribe } from 'ai' + +const { text } = await transcribe({ + model: adk.transcriptionModel({ language: 'en-US' }), + audio, +}) +``` + +`audio` can be an `ArrayBuffer`, a `Uint8Array` or a base64 string, in a format +ML Kit decodes: WAV, MP3, FLAC, M4A or Ogg. + +## Options + +```typescript +const model = adk.transcriptionModel({ + language: 'fr-FR', + mode: 'advanced', +}) +``` + +| Option | Type | Description | +| ---------- | ----------------------- | ---------------------------------------------------------------------------------------- | +| `language` | `string` | BCP 47 locale, such as `en-US` (default: the device locale) | +| `mode` | `'basic' \| 'advanced'` | `basic` (default) uses the traditional on-device model, `advanced` the Gemini Nano model | + +See the +[ML Kit documentation](https://developers.google.com/ml-kit/genai/speech-recognition/android#supported-languages) +for the locales each mode supports. + +## Availability + +Check whether the locale and mode can be used on the device: + +```typescript +const model = adk.transcriptionModel({ language: 'en-US' }) + +const availability = await model.getAvailability() +// 'available' | 'downloadable' | 'downloading' | 'unavailable' | 'unknown' + +const canTranscribe = await model.isAvailable() // available or downloadable +``` + +## Preparing + +ML Kit may need to download the speech recognition model first. The first +transcription downloads it, or call `prepare()` ahead of time to show progress: + +```typescript +await model.prepare({ + onProgress: ({ phase, bytesDownloaded, bytesToDownload }) => { + // phase is 'started', 'progress' or 'completed' + }, +}) +``` + +## Errors + +| `code` | Meaning | +| ------------------- | --------------------------------------------------------- | +| `MODEL_UNAVAILABLE` | The locale or mode is not supported on this device | +| `MODEL_DOWNLOADING` | Android is still downloading the speech recognition model | +| `DOWNLOAD_FAILED` | Downloading the speech recognition model failed | +| `RATE_LIMITED` | AICore is busy, or the app's quota ran out | + +## Limitations + +- ML Kit reports no timestamps for whole files, so `segments` is empty and + `durationInSeconds` is undefined. +- Only whole files are transcribed. Live microphone transcription isn't exposed + yet. From fe9c9a2f9d917c1c75c1383f745f77e998271617 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Thu, 29 Oct 2026 22:25:40 +0100 Subject: [PATCH 4/4] chore(adk): transcribe speech on device The example app builds with Kotlin 2.4, which ML Kit speech recognition needs. Co-Authored-By: Claude Opus 5.5 --- apps/example/app.json | 1 + harness/adk/transcription.harness.ts | 33 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 harness/adk/transcription.harness.ts diff --git a/apps/example/app.json b/apps/example/app.json index 739a492b..3db21629 100644 --- a/apps/example/app.json +++ b/apps/example/app.json @@ -36,6 +36,7 @@ { "android": { "minSdkVersion": 26, + "kotlinVersion": "2.4.0", "packagingOptions": { "exclude": ["META-INF/INDEX.LIST", "META-INF/DEPENDENCIES"] } diff --git a/harness/adk/transcription.harness.ts b/harness/adk/transcription.harness.ts new file mode 100644 index 00000000..8f03b839 --- /dev/null +++ b/harness/adk/transcription.harness.ts @@ -0,0 +1,33 @@ +import { adk } from '@react-native-ai/adk' +import { experimental_transcribe as transcribe } from 'ai' +import { describe, expect, it } from 'react-native-harness' + +import { speech } from '../fixtures' + +// ML Kit decides per device which locales and modes it supports +describe('adk transcription', () => { + const model = adk.transcriptionModel({ language: 'en-US' }) + + it('reports speech recognition availability', async () => { + expect([ + 'available', + 'downloadable', + 'downloading', + 'unavailable', + ]).toContain(await model.getAvailability()) + }) + + it('transcribes speech where ML Kit supports it', async () => { + if (!(await model.isAvailable())) { + await expect(model.prepare()).rejects.toMatchObject({ + code: 'MODEL_UNAVAILABLE', + }) + return + } + + await model.prepare() + const { text } = await transcribe({ model, audio: speech }) + + expect(text).toMatch(/quick brown fox/i) + }) +})