Repository navigation
feat(adk): add on-device transcription model #271
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
2dab754
0fa270c
513433c
fe9c9a2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| import { adk } from '@react-native-ai/adk' | ||
| import { experimental_transcribe as transcribe } from 'ai' | ||
| import { describe, expect, it } from 'react-native-harness' | ||
|
|
||
| import { speech } from '../fixtures' | ||
|
|
||
| // ML Kit decides per device which locales and modes it supports | ||
| describe('adk transcription', () => { | ||
| const model = adk.transcriptionModel({ language: 'en-US' }) | ||
|
|
||
| it('reports speech recognition availability', async () => { | ||
| expect([ | ||
| 'available', | ||
| 'downloadable', | ||
| 'downloading', | ||
| 'unavailable', | ||
| ]).toContain(await model.getAvailability()) | ||
| }) | ||
|
|
||
| it('transcribes speech where ML Kit supports it', async () => { | ||
| if (!(await model.isAvailable())) { | ||
| await expect(model.prepare()).rejects.toMatchObject({ | ||
| code: 'MODEL_UNAVAILABLE', | ||
| }) | ||
| return | ||
| } | ||
|
|
||
| await model.prepare() | ||
| const { text } = await transcribe({ model, audio: speech }) | ||
|
|
||
| expect(text).toMatch(/quick brown fox/i) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| package com.callstack.ai.adk | ||
|
|
||
| import android.content.Context | ||
| import com.facebook.react.bridge.ReadableMap | ||
| import com.google.mlkit.genai.common.DownloadStatus | ||
| import com.google.mlkit.genai.common.FeatureStatus | ||
| import com.google.mlkit.genai.common.GenAiException | ||
| import com.google.mlkit.genai.common.audio.AudioSource | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognition | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizer | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions | ||
| import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse | ||
| import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions | ||
| import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest | ||
| import kotlinx.coroutines.Dispatchers | ||
| import kotlinx.coroutines.flow.takeWhile | ||
| import kotlinx.coroutines.withContext | ||
| import java.io.File | ||
| import java.util.Locale | ||
|
|
||
| data class TranscriptionConfig(val language: String?, val mode: String?) { | ||
| companion object { | ||
| fun from(map: ReadableMap?): TranscriptionConfig = | ||
| TranscriptionConfig( | ||
| language = map?.takeIf { it.hasKey("language") }?.getString("language"), | ||
| mode = map?.takeIf { it.hasKey("mode") }?.getString("mode"), | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| data class TranscriptionResult(val text: String) | ||
|
|
||
| /** On-device speech to text with ML Kit GenAI speech recognition. */ | ||
| class AdkTranscriber( | ||
| private val context: Context, | ||
| private val onDownloadProgress: ( | ||
| phase: String, | ||
| bytesDownloaded: Long?, | ||
| bytesToDownload: Long?, | ||
| ) -> Unit, | ||
| ) { | ||
| private fun client(config: TranscriptionConfig): SpeechRecognizer = | ||
| SpeechRecognition.getClient( | ||
| speechRecognizerOptions { | ||
| locale = config.language?.let { Locale.forLanguageTag(it) } ?: Locale.getDefault() | ||
| preferredMode = | ||
| if (config.mode == "advanced") { | ||
| SpeechRecognizerOptions.Mode.MODE_ADVANCED | ||
| } else { | ||
| SpeechRecognizerOptions.Mode.MODE_BASIC | ||
| } | ||
| }, | ||
| ) | ||
|
|
||
| suspend fun getAvailability(config: TranscriptionConfig): String = | ||
| client(config).use { recognizer -> | ||
| val status = | ||
| try { | ||
| recognizer.checkStatus() | ||
| } catch (_: Exception) { | ||
| return "unknown" | ||
| } | ||
|
Comment on lines
+60
to
+62
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do not swallow
Rethrow cancellation before the fallback. 🛠️ Proposed fix
Based on learnings: flag broad catches in suspend functions that swallow
|
||
| when (status) { | ||
| FeatureStatus.UNAVAILABLE -> "unavailable" | ||
| FeatureStatus.DOWNLOADABLE -> "downloadable" | ||
| FeatureStatus.DOWNLOADING -> "downloading" | ||
| FeatureStatus.AVAILABLE -> "available" | ||
| else -> "unknown" | ||
| } | ||
| } | ||
|
|
||
| /** Downloads the speech recognition model for the locale when needed. */ | ||
| suspend fun prepare(config: TranscriptionConfig) { | ||
| client(config).use { prepare(it) } | ||
| } | ||
|
|
||
| private suspend fun prepare(recognizer: SpeechRecognizer) { | ||
| val status = recognizer.checkStatus() | ||
| when (status) { | ||
| FeatureStatus.AVAILABLE -> return | ||
| FeatureStatus.UNAVAILABLE -> | ||
| throw AdkErrors.AdkException( | ||
| AdkErrors.MODEL_UNAVAILABLE, | ||
| "Speech recognition is not supported for this locale and mode on this device", | ||
| ) | ||
| } | ||
|
|
||
| try { | ||
| recognizer.download().collect { downloadStatus -> | ||
| when (downloadStatus) { | ||
| is DownloadStatus.DownloadStarted -> | ||
| onDownloadProgress("started", 0, downloadStatus.bytesToDownload) | ||
| is DownloadStatus.DownloadProgress -> | ||
| onDownloadProgress("progress", downloadStatus.totalBytesDownloaded, null) | ||
| is DownloadStatus.DownloadCompleted -> onDownloadProgress("completed", null, null) | ||
| is DownloadStatus.DownloadFailed -> | ||
| throw AdkErrors.AdkException( | ||
| AdkErrors.DOWNLOAD_FAILED, | ||
| "Speech recognition model download failed: ${downloadStatus.e.message}", | ||
| downloadStatus.e, | ||
| ) | ||
| } | ||
| } | ||
| } catch (error: GenAiException) { | ||
| if (status == FeatureStatus.DOWNLOADING) { | ||
| throw AdkErrors.AdkException( | ||
| AdkErrors.MODEL_DOWNLOADING, | ||
| "The speech recognition model is still downloading", | ||
| error, | ||
| ) | ||
| } | ||
| throw error | ||
| } | ||
| } | ||
|
|
||
| /** Transcribes a whole audio file. ML Kit decodes formats such as WAV, MP3, M4A and Ogg. */ | ||
| suspend fun transcribe(audio: ByteArray, config: TranscriptionConfig): TranscriptionResult = | ||
| withContext(Dispatchers.IO) { | ||
| val file = File.createTempFile("adk-transcription", null, context.cacheDir) | ||
| try { | ||
| file.writeBytes(audio) | ||
| client(config).use { recognizer -> | ||
| prepare(recognizer) | ||
|
|
||
| AudioSource.fromFile(file).use { source -> | ||
| val request = speechRecognizerRequest { audioSource = source } | ||
| val texts = mutableListOf<String>() | ||
| recognizer.startRecognition(request) | ||
| .takeWhile { it !is SpeechRecognizerResponse.CompletedResponse } | ||
| .collect { response -> | ||
| when (response) { | ||
| is SpeechRecognizerResponse.FinalTextResponse -> texts += response.text | ||
| is SpeechRecognizerResponse.ErrorResponse -> throw response.e | ||
| else -> Unit | ||
| } | ||
| } | ||
| TranscriptionResult( | ||
| text = texts.map { it.trim() }.filter { it.isNotEmpty() }.joinToString(" "), | ||
| ) | ||
| } | ||
| } | ||
| } finally { | ||
| file.delete() | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The new alpha dependency forces Kotlin 2.4.0 on every consumer of
@react-native-ai/adk.genai-speech-recognition:1.0.0-alpha2is declared asimplementationon the whole library module. The PR objectives state that alpha2 is built with Kotlin 2.4. Every app that links@react-native-ai/adkmust therefore setkotlinVersion: "2.4.0", or the Kotlin compile of the app will probably fail on newer class metadata. This applies even to apps that only useadk(...)text generation and never calltranscriptionModel. An upgrade of the package can break existing consumers.Reduce the blast radius with one of these options:
AdkTranscription_enabled. KeepAdkTranscriberout of the default source set, or load it lazily.app.json.Also confirm that the library's own
kotlin_version(in the unchanged lines of this file) can compile against alpha2 metadata.