diff --git a/apps/example/app.json b/apps/example/app.json index 739a492b..3db21629 100644 --- a/apps/example/app.json +++ b/apps/example/app.json @@ -36,6 +36,7 @@ { "android": { "minSdkVersion": 26, + "kotlinVersion": "2.4.0", "packagingOptions": { "exclude": ["META-INF/INDEX.LIST", "META-INF/DEPENDENCIES"] } diff --git a/harness/adk/transcription.harness.ts b/harness/adk/transcription.harness.ts new file mode 100644 index 00000000..8f03b839 --- /dev/null +++ b/harness/adk/transcription.harness.ts @@ -0,0 +1,33 @@ +import { adk } from '@react-native-ai/adk' +import { experimental_transcribe as transcribe } from 'ai' +import { describe, expect, it } from 'react-native-harness' + +import { speech } from '../fixtures' + +// ML Kit decides per device which locales and modes it supports +describe('adk transcription', () => { + const model = adk.transcriptionModel({ language: 'en-US' }) + + it('reports speech recognition availability', async () => { + expect([ + 'available', + 'downloadable', + 'downloading', + 'unavailable', + ]).toContain(await model.getAvailability()) + }) + + it('transcribes speech where ML Kit supports it', async () => { + if (!(await model.isAvailable())) { + await expect(model.prepare()).rejects.toMatchObject({ + code: 'MODEL_UNAVAILABLE', + }) + return + } + + await model.prepare() + const { text } = await transcribe({ model, audio: speech }) + + expect(text).toMatch(/quick brown fox/i) + }) +}) diff --git a/packages/adk/README.md b/packages/adk/README.md index 72f9eec8..d343e8c2 100644 --- a/packages/adk/README.md +++ b/packages/adk/README.md @@ -25,6 +25,7 @@ const { text } = await generateText({ - ✅ On-device Gemini Nano through ML Kit GenAI, with model selection - ✅ Availability, download progress and token counting +- ✅ On-device transcription with ML Kit GenAI speech recognition - ✅ Streaming - ✅ Image prompts - ✅ Token usage diff --git a/packages/adk/android/build.gradle b/packages/adk/android/build.gradle index 0f5c53f7..b675b10c 100644 --- a/packages/adk/android/build.gradle +++ b/packages/adk/android/build.gradle @@ -96,6 +96,7 @@ dependencies { implementation "com.google.adk:google-adk-kotlin-core:1.4.0" implementation "com.google.adk:google-adk-kotlin-mlkit-android:1.4.0-beta" implementation "com.google.mlkit:genai-prompt:1.0.0-beta4" + implementation "com.google.mlkit:genai-speech-recognition:1.0.0-alpha2" } react { diff --git a/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt b/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt new file mode 100644 index 00000000..dda6c2f3 --- /dev/null +++ b/packages/adk/android/src/main/java/com/callstack/ai/adk/AdkTranscriber.kt @@ -0,0 +1,146 @@ +package com.callstack.ai.adk + +import android.content.Context +import com.facebook.react.bridge.ReadableMap +import com.google.mlkit.genai.common.DownloadStatus +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.GenAiException +import com.google.mlkit.genai.common.audio.AudioSource +import com.google.mlkit.genai.speechrecognition.SpeechRecognition +import com.google.mlkit.genai.speechrecognition.SpeechRecognizer +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.takeWhile +import kotlinx.coroutines.withContext +import java.io.File +import java.util.Locale + +data class TranscriptionConfig(val language: String?, val mode: String?) { + companion object { + fun from(map: ReadableMap?): TranscriptionConfig = + TranscriptionConfig( + language = map?.takeIf { it.hasKey("language") }?.getString("language"), + mode = map?.takeIf { it.hasKey("mode") }?.getString("mode"), + ) + } +} + +data class TranscriptionResult(val text: String) + +/** On-device speech to text with ML Kit GenAI speech recognition. */ +class AdkTranscriber( + private val context: Context, + private val onDownloadProgress: ( + phase: String, + bytesDownloaded: Long?, + bytesToDownload: Long?, + ) -> Unit, +) { + private fun client(config: TranscriptionConfig): SpeechRecognizer = + SpeechRecognition.getClient( + speechRecognizerOptions { + locale = config.language?.let { Locale.forLanguageTag(it) } ?: Locale.getDefault() + preferredMode = + if (config.mode == "advanced") { + SpeechRecognizerOptions.Mode.MODE_ADVANCED + } else { + SpeechRecognizerOptions.Mode.MODE_BASIC + } + }, + ) + + suspend fun getAvailability(config: TranscriptionConfig): String = + client(config).use { recognizer -> + val status = + try { + recognizer.checkStatus() + } catch (_: Exception) { + return "unknown" + } + when (status) { + FeatureStatus.UNAVAILABLE -> "unavailable" + FeatureStatus.DOWNLOADABLE -> "downloadable" + FeatureStatus.DOWNLOADING -> "downloading" + FeatureStatus.AVAILABLE -> "available" + else -> "unknown" + } + } + + /** Downloads the speech recognition model for the locale when needed. */ + suspend fun prepare(config: TranscriptionConfig) { + client(config).use { prepare(it) } + } + + private suspend fun prepare(recognizer: SpeechRecognizer) { + val status = recognizer.checkStatus() + when (status) { + FeatureStatus.AVAILABLE -> return + FeatureStatus.UNAVAILABLE -> + throw AdkErrors.AdkException( + AdkErrors.MODEL_UNAVAILABLE, + "Speech recognition is not supported for this locale and mode on this device", + ) + } + + try { + recognizer.download().collect { downloadStatus -> + when (downloadStatus) { + is DownloadStatus.DownloadStarted -> + onDownloadProgress("started", 0, downloadStatus.bytesToDownload) + is DownloadStatus.DownloadProgress -> + onDownloadProgress("progress", downloadStatus.totalBytesDownloaded, null) + is DownloadStatus.DownloadCompleted -> onDownloadProgress("completed", null, null) + is DownloadStatus.DownloadFailed -> + throw AdkErrors.AdkException( + AdkErrors.DOWNLOAD_FAILED, + "Speech recognition model download failed: ${downloadStatus.e.message}", + downloadStatus.e, + ) + } + } + } catch (error: GenAiException) { + if (status == FeatureStatus.DOWNLOADING) { + throw AdkErrors.AdkException( + AdkErrors.MODEL_DOWNLOADING, + "The speech recognition model is still downloading", + error, + ) + } + throw error + } + } + + /** Transcribes a whole audio file. ML Kit decodes formats such as WAV, MP3, M4A and Ogg. */ + suspend fun transcribe(audio: ByteArray, config: TranscriptionConfig): TranscriptionResult = + withContext(Dispatchers.IO) { + val file = File.createTempFile("adk-transcription", null, context.cacheDir) + try { + file.writeBytes(audio) + client(config).use { recognizer -> + prepare(recognizer) + + AudioSource.fromFile(file).use { source -> + val request = speechRecognizerRequest { audioSource = source } + val texts = mutableListOf() + recognizer.startRecognition(request) + .takeWhile { it !is SpeechRecognizerResponse.CompletedResponse } + .collect { response -> + when (response) { + is SpeechRecognizerResponse.FinalTextResponse -> texts += response.text + is SpeechRecognizerResponse.ErrorResponse -> throw response.e + else -> Unit + } + } + TranscriptionResult( + text = texts.map { it.trim() }.filter { it.isNotEmpty() }.joinToString(" "), + ) + } + } + } finally { + file.delete() + } + } +} diff --git a/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt b/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt index 0348a547..98001074 100644 --- a/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt +++ b/packages/adk/android/src/main/java/com/callstack/ai/adk/NativeAdkEngineModule.kt @@ -1,5 +1,6 @@ package com.callstack.ai.adk +import android.util.Base64 import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -37,7 +38,20 @@ class NativeAdkEngineModule(reactContext: ReactApplicationContext) : }, ) - // Streams and generateText runs, by id, so both can be cancelled + private val transcriber = AdkTranscriber( + context = reactContext, + onDownloadProgress = { phase, bytesDownloaded, bytesToDownload -> + emitOnTranscriptionDownloadProgress( + Arguments.createMap().apply { + putString("phase", phase) + bytesDownloaded?.let { putDouble("bytesDownloaded", it.toDouble()) } + bytesToDownload?.let { putDouble("bytesToDownload", it.toDouble()) } + }, + ) + }, + ) + + // Streams, generateText runs and transcriptions, by id, so all can be cancelled private val activeRuns = ConcurrentHashMap() private fun Promise.rejectWithCode(error: Exception) { @@ -189,6 +203,57 @@ class NativeAdkEngineModule(reactContext: ReactApplicationContext) : promise.resolve(null) } + override fun getTranscriptionAvailability(config: ReadableMap?, promise: Promise) { + scope.launch { + try { + promise.resolve(transcriber.getAvailability(TranscriptionConfig.from(config))) + } catch (error: Exception) { + promise.rejectWithCode(error) + } + } + } + + override fun prepareTranscription(config: ReadableMap?, promise: Promise) { + scope.launch { + try { + transcriber.prepare(TranscriptionConfig.from(config)) + promise.resolve(null) + } catch (error: Exception) { + promise.rejectWithCode(error) + } + } + } + + override fun transcribe( + requestId: String, + audio: String, + config: ReadableMap?, + promise: Promise, + ) { + val job = scope.launch(start = CoroutineStart.LAZY) { + try { + val bytes = + try { + Base64.decode(audio, Base64.DEFAULT) + } catch (error: IllegalArgumentException) { + throw AdkErrors.AdkException(AdkErrors.INVALID_MESSAGE, "Audio is not valid base64", error) + } + val result = transcriber.transcribe(bytes, TranscriptionConfig.from(config)) + promise.resolve( + Arguments.createMap().apply { + putString("text", result.text) + }, + ) + } catch (error: Exception) { + promise.rejectWithCode(error) + } finally { + activeRuns.remove(requestId) + } + } + activeRuns[requestId] = job + job.start() + } + private fun emitDelta(streamId: String, delta: String, kind: String) { emitOnStreamUpdate( Arguments.createMap().apply { diff --git a/packages/adk/src/NativeAdkEngine.ts b/packages/adk/src/NativeAdkEngine.ts index 62249194..73caae5c 100644 --- a/packages/adk/src/NativeAdkEngine.ts +++ b/packages/adk/src/NativeAdkEngine.ts @@ -88,6 +88,17 @@ export interface NanoDownloadProgressEvent { bytesToDownload?: number } +export interface AdkTranscriptionConfig { + /** BCP 47 locale such as `en-US`. Defaults to the device locale */ + language?: string + /** `basic` (default) or `advanced`, which uses a GenAI model on supported devices */ + mode?: 'basic' | 'advanced' +} + +export interface AdkTranscriptionResult { + text: string +} + export interface Spec extends TurboModule { getModelInfo(nano?: AdkNanoModelConfig): Promise prepareNano(nano?: AdkNanoModelConfig): Promise @@ -107,13 +118,23 @@ export interface Spec extends TurboModule { options?: AdkGenerationOptions ): Promise - /** Cancels a `streamText` or `generateText` run by its id */ + getTranscriptionAvailability(config?: AdkTranscriptionConfig): Promise + prepareTranscription(config?: AdkTranscriptionConfig): Promise + /** Transcribes base64-encoded audio in any format Android can decode */ + transcribe( + requestId: string, + audio: string, + config?: AdkTranscriptionConfig + ): Promise + + /** Cancels a stream, a `generateText` run or a transcription by its id */ cancelStream(streamId: string): Promise onStreamUpdate: CodegenTypes.EventEmitter onStreamComplete: CodegenTypes.EventEmitter onStreamError: CodegenTypes.EventEmitter onNanoDownloadProgress: CodegenTypes.EventEmitter + onTranscriptionDownloadProgress: CodegenTypes.EventEmitter } const NativeAdkEngine: Spec | null = diff --git a/packages/adk/src/__tests__/ai-sdk.test.ts b/packages/adk/src/__tests__/ai-sdk.test.ts index 761031c6..7b11fe8d 100644 --- a/packages/adk/src/__tests__/ai-sdk.test.ts +++ b/packages/adk/src/__tests__/ai-sdk.test.ts @@ -22,6 +22,7 @@ const onStreamUpdate = createEmitter() const onStreamComplete = createEmitter() const onStreamError = createEmitter() const onNanoDownloadProgress = createEmitter() +const onTranscriptionDownloadProgress = createEmitter() let generateTextResult: Record = { content: 'ok', @@ -50,6 +51,16 @@ const getModelInfo = mock(async (_nano?: unknown) => ({ availability: 'downloadable', })) +const transcribe = mock( + async (_requestId: string, _audio: string, _config?: unknown) => ({ + text: 'Hello world', + }) +) +const prepareTranscription = mock(async (_config?: unknown) => {}) +const getTranscriptionAvailability = mock( + async (_config?: unknown) => 'downloadable' +) + mock.module('react-native', () => ({ Platform: { OS: 'android' }, TurboModuleRegistry: { @@ -63,6 +74,10 @@ mock.module('react-native', () => ({ onStreamComplete, onStreamError, onNanoDownloadProgress, + transcribe, + prepareTranscription, + getTranscriptionAvailability, + onTranscriptionDownloadProgress, }), }, })) @@ -487,3 +502,100 @@ describe('Gemini Nano preparation', () => { }) }) }) + +describe('transcription', () => { + test('transcribes binary audio as base64', async () => { + const model = createAdkProvider().transcriptionModel({ + language: 'en-US', + mode: 'advanced', + }) + + const result = await model.doGenerate({ + audio: new Uint8Array([1, 2, 3]), + mediaType: 'audio/wav', + }) + + const [, audio, config] = transcribe.mock.calls.at(-1)! + expect(audio).toBe('AQID') + expect(config).toEqual({ language: 'en-US', mode: 'advanced' }) + expect(result).toMatchObject({ + text: 'Hello world', + segments: [], + language: 'en-US', + durationInSeconds: undefined, + warnings: [], + response: { modelId: 'genai-speech-recognition' }, + }) + }) + + test('strips data URL prefixes from base64 audio', async () => { + await createAdkProvider().transcriptionModel().doGenerate({ + audio: 'data:audio/wav;base64,AQID', + mediaType: 'audio/wav', + }) + + expect(transcribe.mock.calls.at(-1)![1]).toBe('AQID') + }) + + test('cancels the native request on abort', async () => { + const controller = new AbortController() + transcribe.mockImplementationOnce(async (requestId) => { + controller.abort() + expect(cancelStream.mock.calls.at(-1)![0]).toBe(requestId) + throw new Error('cancelled') + }) + + await expect( + createAdkProvider().transcriptionModel().doGenerate({ + audio: 'AQID', + mediaType: 'audio/wav', + abortSignal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + + test('maps native errors to AdkErrorCodes', async () => { + transcribe.mockImplementationOnce(async () => { + throw Object.assign(new Error('Not supported'), { + code: 'MODEL_UNAVAILABLE', + }) + }) + + await expect( + createAdkProvider() + .transcriptionModel() + .doGenerate({ audio: 'AQID', mediaType: 'audio/wav' }) + ).rejects.toMatchObject({ code: 'MODEL_UNAVAILABLE' }) + }) + + test('reports availability', async () => { + const model = createAdkProvider().transcriptionModel({ language: 'fr-FR' }) + + expect(await model.getAvailability()).toBe('downloadable') + expect(await model.isAvailable()).toBe(true) + expect(getTranscriptionAvailability.mock.calls.at(-1)![0]).toEqual({ + language: 'fr-FR', + }) + + getTranscriptionAvailability.mockImplementationOnce(async () => 'weird') + expect(await model.getAvailability()).toBe('unknown') + }) + + test('reports download progress while preparing', async () => { + const progress: unknown[] = [] + prepareTranscription.mockImplementationOnce(async () => { + onTranscriptionDownloadProgress.emit({ + phase: 'progress', + bytesDownloaded: 50, + }) + }) + + await createAdkProvider() + .transcriptionModel() + .prepare({ onProgress: (event) => progress.push(event) }) + + expect(progress).toEqual([ + { phase: 'progress', bytesDownloaded: 50, bytesToDownload: undefined }, + ]) + }) +}) diff --git a/packages/adk/src/ai-sdk.ts b/packages/adk/src/ai-sdk.ts index 49c5a928..3473ccde 100644 --- a/packages/adk/src/ai-sdk.ts +++ b/packages/adk/src/ai-sdk.ts @@ -8,9 +8,12 @@ import type { LanguageModelV4StreamPart, LanguageModelV4Usage, SharedV4Warning, + TranscriptionModelV4, + TranscriptionModelV4CallOptions, + TranscriptionModelV4Result, } from '@ai-sdk/provider' import { generateId, resolveFullMediaType } from '@ai-sdk/provider-utils' -import { Platform } from 'react-native' +import { type CodegenTypes, Platform } from 'react-native' import { AdkErrorCodes, createAdkError, toAdkError } from './errors' import { @@ -19,6 +22,7 @@ import { type AdkMessage, type AdkMessagePart, type AdkNanoModelConfig, + type AdkTranscriptionConfig, type AdkUsageMetadata, getNativeAdkEngine, type NanoDownloadProgressEvent, @@ -75,6 +79,16 @@ export interface AdkLanguageModel extends LanguageModelV4 { countTokens: (text: string) => Promise } +export type AdkTranscriptionOptions = AdkTranscriptionConfig + +export interface AdkTranscriptionModel extends TranscriptionModelV4 { + /** Whether speech recognition for this locale and mode is downloaded or downloadable */ + isAvailable: () => Promise + getAvailability: () => Promise + /** Downloads the speech recognition model, reporting download progress */ + prepare: (options?: AdkPrepareOptions) => Promise +} + export interface AdkProvider { (options?: AdkLanguageModelOptions): AdkLanguageModel readonly specificationVersion: 'v4' @@ -85,6 +99,10 @@ export interface AdkProvider { getModelInfo: (options?: { nano?: AdkNanoModelConfig }) => Promise + /** On-device speech to text with ML Kit GenAI speech recognition */ + transcriptionModel: ( + options?: AdkTranscriptionOptions + ) => AdkTranscriptionModel } const toModelAvailability = (value: string): AdkModelAvailability => { @@ -100,18 +118,17 @@ const toModelAvailability = (value: string): AdkModelAvailability => { } const listenForDownloadProgress = ( + subscribe: CodegenTypes.EventEmitter, onProgress: AdkPrepareOptions['onProgress'] ) => onProgress - ? getNativeAdkEngine().onNanoDownloadProgress( - (event: NanoDownloadProgressEvent) => { - onProgress({ - phase: event.phase as AdkDownloadProgress['phase'], - bytesDownloaded: event.bytesDownloaded, - bytesToDownload: event.bytesToDownload, - }) - } - ) + ? subscribe((event: NanoDownloadProgressEvent) => { + onProgress({ + phase: event.phase as AdkDownloadProgress['phase'], + bytesDownloaded: event.bytesDownloaded, + bytesToDownload: event.bytesToDownload, + }) + }) : null async function getModelInfo( @@ -150,6 +167,8 @@ export function createAdkProvider( } provider.getModelInfo = (options: { nano?: AdkNanoModelConfig } = {}) => getModelInfo(options.nano ?? defaults.nano) + provider.transcriptionModel = (options?: AdkTranscriptionOptions) => + new AdkSpeechTranscriptionModel(options) return provider } @@ -318,7 +337,10 @@ class AdkChatLanguageModel implements AdkLanguageModel { } async prepare(options: AdkPrepareOptions = {}): Promise { - const listener = listenForDownloadProgress(options.onProgress) + const listener = listenForDownloadProgress( + getNativeAdkEngine().onNanoDownloadProgress, + options.onProgress + ) try { await this.ensurePrepared() } finally { @@ -684,3 +706,94 @@ class AdkChatLanguageModel implements AdkLanguageModel { } } } + +class AdkSpeechTranscriptionModel implements AdkTranscriptionModel { + readonly specificationVersion = 'v4' + readonly provider = 'adk' + readonly modelId = 'genai-speech-recognition' + + private readonly config: AdkTranscriptionConfig + + constructor(options: AdkTranscriptionOptions = {}) { + this.config = options + } + + async getAvailability(): Promise { + if (Platform.OS !== 'android') { + return 'unavailable' + } + const availability = await getNativeAdkEngine() + .getTranscriptionAvailability(this.config) + .catch((error: unknown) => { + throw toAdkError(error) + }) + return toModelAvailability(availability) + } + + async isAvailable(): Promise { + const availability = await this.getAvailability() + return availability === 'available' || availability === 'downloadable' + } + + async prepare(options: AdkPrepareOptions = {}): Promise { + const native = getNativeAdkEngine() + const listener = listenForDownloadProgress( + native.onTranscriptionDownloadProgress, + options.onProgress + ) + try { + await native.prepareTranscription(this.config) + } catch (error) { + throw toAdkError(error) + } finally { + listener?.remove() + } + } + + async doGenerate( + options: TranscriptionModelV4CallOptions + ): Promise { + const native = getNativeAdkEngine() + const requestId = generateId() + const abortSignal = options.abortSignal + + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + const onAbort = () => { + void native + .cancelStream(requestId) + .catch((error) => console.warn('[adk] cancel failed:', error)) + } + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + const audio = + typeof options.audio === 'string' + ? options.audio.replace(/^data:[^;]+;base64,/, '') + : uint8ArrayToBase64(options.audio) + + try { + const result = await native.transcribe(requestId, audio, this.config) + return { + text: result.text, + // ML Kit speech recognition does not report timestamps + segments: [], + language: this.config.language, + durationInSeconds: undefined, + warnings: [], + response: { + timestamp: new Date(), + modelId: this.modelId, + }, + } + } catch (error) { + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + throw toAdkError(error) + } finally { + abortSignal?.removeEventListener('abort', onAbort) + } + } +} diff --git a/packages/adk/src/index.ts b/packages/adk/src/index.ts index 3f71d399..c081a220 100644 --- a/packages/adk/src/index.ts +++ b/packages/adk/src/index.ts @@ -7,6 +7,8 @@ export type { AdkPrepareOptions, AdkProvider, AdkProviderOptions, + AdkTranscriptionModel, + AdkTranscriptionOptions, } from './ai-sdk' export { adk, createAdkProvider } from './ai-sdk' export type { AdkError, AdkErrorCode } from './errors' diff --git a/website/astro.config.mjs b/website/astro.config.mjs index cef56585..33ef2792 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -98,6 +98,7 @@ export default defineConfig({ ['getting-started', 'Getting started'], ['generating', 'Generating'], ['models', 'Models and availability'], + ['transcription', 'Transcription'], ]), group('Core AI', 'core-ai', [ ['getting-started', 'Getting started'], diff --git a/website/src/content/docs/docs/adk/getting-started.mdx b/website/src/content/docs/docs/adk/getting-started.mdx index ffc2b016..9b8c233f 100644 --- a/website/src/content/docs/docs/adk/getting-started.mdx +++ b/website/src/content/docs/docs/adk/getting-started.mdx @@ -90,3 +90,4 @@ if (await adk.isAvailable()) { - [Generating](/docs/adk/generating): streaming, image input, token usage and errors - [Models & Availability](/docs/adk/models): checking for Gemini Nano, preparing it and provider options +- [Transcription](/docs/adk/transcription): on-device speech to text diff --git a/website/src/content/docs/docs/adk/transcription.md b/website/src/content/docs/docs/adk/transcription.md new file mode 100644 index 00000000..e370eb60 --- /dev/null +++ b/website/src/content/docs/docs/adk/transcription.md @@ -0,0 +1,109 @@ +--- +title: "Transcription" +--- + +Convert speech to text on the device with ML Kit's +[GenAI Speech Recognition API](https://developers.google.com/ml-kit/genai/speech-recognition/android). +`adk.transcriptionModel()` returns an AI SDK transcription model, and the audio +never leaves the device. + +:::caution +ML Kit GenAI speech recognition is in alpha, and Google may change it in ways +that break this provider. +::: + +## Requirements + +- A physical Android device with a locked bootloader. Emulators aren't supported. +- Android 12 (API 31) or newer for `basic` mode, and a device with Gemini Nano, + such as a Pixel 10, for `advanced` mode. +- Kotlin 2.4 in your app, which ML Kit's speech recognition is built with. With + Expo, set it with `expo-build-properties`: + +```json +{ + "expo": { + "plugins": [ + ["expo-build-properties", { "android": { "kotlinVersion": "2.4.0" } }] + ] + } +} +``` + +In a bare React Native app, set `android.kotlinVersion=2.4.0` in +`android/gradle.properties`. + +## Usage + +```typescript +import { adk } from '@react-native-ai/adk' +import { experimental_transcribe as transcribe } from 'ai' + +const { text } = await transcribe({ + model: adk.transcriptionModel({ language: 'en-US' }), + audio, +}) +``` + +`audio` can be an `ArrayBuffer`, a `Uint8Array` or a base64 string, in a format +ML Kit decodes: WAV, MP3, FLAC, M4A or Ogg. + +## Options + +```typescript +const model = adk.transcriptionModel({ + language: 'fr-FR', + mode: 'advanced', +}) +``` + +| Option | Type | Description | +| ---------- | ----------------------- | ---------------------------------------------------------------------------------------- | +| `language` | `string` | BCP 47 locale, such as `en-US` (default: the device locale) | +| `mode` | `'basic' \| 'advanced'` | `basic` (default) uses the traditional on-device model, `advanced` the Gemini Nano model | + +See the +[ML Kit documentation](https://developers.google.com/ml-kit/genai/speech-recognition/android#supported-languages) +for the locales each mode supports. + +## Availability + +Check whether the locale and mode can be used on the device: + +```typescript +const model = adk.transcriptionModel({ language: 'en-US' }) + +const availability = await model.getAvailability() +// 'available' | 'downloadable' | 'downloading' | 'unavailable' | 'unknown' + +const canTranscribe = await model.isAvailable() // available or downloadable +``` + +## Preparing + +ML Kit may need to download the speech recognition model first. The first +transcription downloads it, or call `prepare()` ahead of time to show progress: + +```typescript +await model.prepare({ + onProgress: ({ phase, bytesDownloaded, bytesToDownload }) => { + // phase is 'started', 'progress' or 'completed' + }, +}) +``` + +## Errors + +| `code` | Meaning | +| ------------------- | --------------------------------------------------------- | +| `MODEL_UNAVAILABLE` | The locale or mode is not supported on this device | +| `MODEL_DOWNLOADING` | Android is still downloading the speech recognition model | +| `DOWNLOAD_FAILED` | Downloading the speech recognition model failed | +| `RATE_LIMITED` | AICore is busy, or the app's quota ran out | + +## Limitations + +- ML Kit reports no timestamps for whole files, so `segments` is empty and + `durationInSeconds` is undefined. +- Only whole files are transcribed. Live microphone transcription isn't exposed + yet.