From cbc0bbf364a54e205c29c05a85f9eff1d934e835 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 1 Nov 2026 13:11:02 +0100 Subject: [PATCH 1/4] feat(android): summarize, proofread, rewrite and describe images Wrap ML Kit's task APIs as models on the provider: summarizer, proofreader, rewriter and imageDescriber, with getModelInfo, prepare, streamed text and abort. Context caching and speech translation are left out: caching fails on the test device and translation is alpha. Part of #366 Co-Authored-By: Claude Opus 5.5 --- harness/android/tasks.harness.ts | 96 +++++++ packages/android/android/build.gradle | 4 + .../callstack/ai/android/AndroidAIModule.kt | 19 ++ .../com/callstack/ai/android/Downloads.kt | 28 ++ .../java/com/callstack/ai/android/Tasks.kt | 268 ++++++++++++++++++ packages/android/src/NativeAndroidAI.ts | 34 ++- packages/android/src/__tests__/ai-sdk.test.ts | 97 +++++++ packages/android/src/ai-sdk.ts | 190 +++++++++++++ packages/android/src/index.ts | 9 + website/astro.config.mjs | 1 + .../src/content/docs/docs/android/tasks.md | 46 +++ 11 files changed, 790 insertions(+), 2 deletions(-) create mode 100644 harness/android/tasks.harness.ts create mode 100644 packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt create mode 100644 website/src/content/docs/docs/android/tasks.md diff --git a/harness/android/tasks.harness.ts b/harness/android/tasks.harness.ts new file mode 100644 index 00000000..3e63ceba --- /dev/null +++ b/harness/android/tasks.harness.ts @@ -0,0 +1,96 @@ +import { android } from '@react-native-ai/android' +import { describe, expect, it } from 'react-native-harness' + +import { cats } from '../fixtures' +import { cooldown } from './helpers' + +const article = + 'The city council approved a plan on Tuesday to turn the old rail yard into a park. ' + + 'The project will add twelve acres of green space, a playground and a community garden. ' + + 'Construction starts next spring and is expected to take two years. Funding comes from a ' + + 'state grant and a local bond measure that voters passed last fall. Residents who spoke at ' + + 'the meeting mostly supported the plan, though some asked for more parking near the entrance.' + +describe('android task apis', () => { + it('reports each task', async () => { + for (const task of [ + android.summarizer(), + android.proofreader(), + android.rewriter(), + android.imageDescriber(), + ]) { + const info = await task.getModelInfo() + console.log('[android] task:', task.constructor.name, info.availability) + expect(info.availability).not.toBe('unknown') + } + }) + + it('summarizes an article into bullets', async () => { + const summarizer = android.summarizer({ outputType: 'twoBullets' }) + await summarizer.prepare() + + const chunks: string[] = [] + const summary = await summarizer.summarize(article, { + onText: (text) => chunks.push(text), + }) + console.log('[android] summary:', JSON.stringify(summary), chunks.length) + + expect(summary.toLowerCase()).toContain('park') + // Each chunk is new text, not the text so far + expect(chunks.length).toBeGreaterThan(1) + expect(chunks.join('').trim()).toBe(summary.trim()) + await cooldown() + }) + + it('proofreads a sentence', async () => { + const proofreader = android.proofreader() + await proofreader.prepare() + + const suggestions = await proofreader.proofread( + 'The praject is compleet but needs too be reviewd' + ) + console.log('[android] proofread:', JSON.stringify(suggestions)) + + expect(suggestions[0]).toContain('project') + await cooldown() + }) + + it('rewrites a sentence in a style', async () => { + const rewriter = android.rewriter({ style: 'professional' }) + await rewriter.prepare() + + const suggestions = await rewriter.rewrite( + 'hey, cant make it today, lets do tmrw' + ) + console.log('[android] rewrite:', JSON.stringify(suggestions)) + + expect(suggestions.length).toBeGreaterThan(0) + expect(suggestions[0]!.length).toBeGreaterThan(0) + await cooldown() + }) + + it('describes an image', async () => { + const describer = android.imageDescriber() + await describer.prepare() + + const description = await describer.describe(cats) + console.log('[android] description:', JSON.stringify(description)) + + expect(description.toLowerCase()).toMatch(/cat|kitten/) + await cooldown() + }) + + it('stops when aborted', async () => { + const summarizer = android.summarizer() + await summarizer.prepare() + const controller = new AbortController() + + const result = summarizer.summarize(article, { + abortSignal: controller.signal, + }) + controller.abort() + + await expect(result).rejects.toMatchObject({ name: 'AbortError' }) + await cooldown() + }) +}) diff --git a/packages/android/android/build.gradle b/packages/android/android/build.gradle index 9519c8aa..944bf85b 100644 --- a/packages/android/android/build.gradle +++ b/packages/android/android/build.gradle @@ -98,6 +98,10 @@ dependencies { implementation "com.google.mlkit:genai-schema:1.0.0-alpha2" implementation "com.google.mlkit:genai-speech-recognition:1.0.0-alpha2" implementation "com.google.mlkit:genai-embedding:1.0.0-alpha1" + implementation "com.google.mlkit:genai-summarization:1.0.0-beta1" + implementation "com.google.mlkit:genai-proofreading:1.0.0-beta1" + implementation "com.google.mlkit:genai-rewriting:1.0.0-beta1" + implementation "com.google.mlkit:genai-image-description:1.0.0-beta1" } react { diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt index 27d130fc..089c6ff9 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -36,6 +36,7 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp private val nano = GeminiNano(reactContext, downloads) private val transcription = Transcription(reactContext, downloads) private val embeddings = Embeddings(downloads) + private val tasks = Tasks(reactContext, downloads) // Generations and transcriptions, by id, so they can be cancelled private val activeRuns = ConcurrentHashMap() @@ -197,6 +198,24 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp } } + override fun getTaskModelInfo(task: String, options: ReadableMap, promise: Promise) = + launch(promise) { tasks.getModelInfo(TaskConfig.from(task, options)).toWritableMap() } + + override fun prepareTask(task: String, options: ReadableMap, promise: Promise) = + launch(promise) { tasks.prepare(TaskConfig.from(task, options)) } + + override fun runTask(runId: String, task: String, input: String, options: ReadableMap, promise: Promise) = + launchRun(runId, { promise.rejectWithCode(it) }) { + val onText = + if (options.isSet("stream") && options.getBoolean("stream")) { + { text: String -> emitStreamEvent(runId, "text") { putString("delta", text) } } + } else { + null + } + val results = tasks.run(TaskConfig.from(task, options), input, onText) + promise.resolve(Arguments.fromList(results)) + } + override fun invalidate() { activeRuns.values.forEach { it.cancel() } activeRuns.clear() diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt index 18936b56..10eb10ba 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt @@ -1,5 +1,7 @@ package com.callstack.ai.android +import com.google.common.util.concurrent.ListenableFuture +import com.google.mlkit.genai.common.DownloadCallback import com.google.mlkit.genai.common.DownloadStatus import com.google.mlkit.genai.common.FeatureStatus import com.google.mlkit.genai.common.GenAiException @@ -37,4 +39,30 @@ class Downloads( throw error } } + + /** Like [collect], for ML Kit's task APIs, which report downloads to a [DownloadCallback]. */ + suspend fun collect( + model: String, + displayName: String, + status: Int, + download: (DownloadCallback) -> ListenableFuture, + ) { + val callback = object : DownloadCallback { + override fun onDownloadStarted(bytesToDownload: Long) = onProgress(model, "started", 0, bytesToDownload) + + override fun onDownloadProgress(totalBytesDownloaded: Long) = + onProgress(model, "progress", totalBytesDownloaded, null) + + override fun onDownloadCompleted() = onProgress(model, "completed", null, null) + } + try { + download(callback).await() + } catch (error: GenAiException) { + throw AndroidAIErrors.CodedException( + if (status == FeatureStatus.DOWNLOADING) AndroidAIErrors.MODEL_DOWNLOADING else AndroidAIErrors.DOWNLOAD_FAILED, + "$displayName download failed: ${error.message}", + error, + ) + } + } } diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt new file mode 100644 index 00000000..fa3521ff --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt @@ -0,0 +1,268 @@ +package com.callstack.ai.android + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.net.Uri +import android.util.Base64 +import com.facebook.react.bridge.ReadableMap +import com.google.common.util.concurrent.ListenableFuture +import com.google.mlkit.genai.common.DownloadCallback +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.StreamingCallback +import com.google.mlkit.genai.imagedescription.ImageDescriberOptions +import com.google.mlkit.genai.imagedescription.ImageDescription +import com.google.mlkit.genai.imagedescription.ImageDescriptionRequest +import com.google.mlkit.genai.proofreading.ProofreaderOptions +import com.google.mlkit.genai.proofreading.Proofreading +import com.google.mlkit.genai.proofreading.ProofreadingRequest +import com.google.mlkit.genai.rewriting.RewriterOptions +import com.google.mlkit.genai.rewriting.Rewriting +import com.google.mlkit.genai.rewriting.RewritingRequest +import com.google.mlkit.genai.summarization.Summarization +import com.google.mlkit.genai.summarization.SummarizationRequest +import com.google.mlkit.genai.summarization.SummarizerOptions +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutionException +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +/** A task and its options, which pick the ML Kit client and its LoRA adapter. */ +data class TaskConfig( + val task: String, + val inputType: String?, + val outputType: String?, + val language: String?, + val autoTruncate: Boolean, +) { + companion object { + fun from(task: String, map: ReadableMap?): TaskConfig = + TaskConfig( + task = task, + inputType = map?.optString("inputType"), + outputType = map?.optString("outputType"), + language = map?.optString("language"), + autoTruncate = map?.takeIf { it.isSet("autoTruncate") }?.getBoolean("autoTruncate") ?: false, + ) + } +} + +/** The calls the ML Kit task clients share, which have no common interface. */ +private class TaskClient( + val checkFeatureStatus: () -> ListenableFuture, + val downloadFeature: (DownloadCallback) -> ListenableFuture, + val prepareInferenceEngine: () -> ListenableFuture, + /** Runs the task on its input and returns its results, best first. */ + val run: suspend (input: String, callback: StreamingCallback?) -> List, +) + +/** + * Summarization, proofreading, rewriting and image description, ML Kit's task APIs. Each runs + * Gemini Nano with a LoRA adapter for the task, downloaded on first use. + */ +class Tasks( + private val context: Context, + private val downloads: Downloads, +) { + private val clients = ConcurrentHashMap() + private val prepared = ConcurrentHashMap.newKeySet() + private val prepareMutexes = ConcurrentHashMap() + + private fun client(config: TaskConfig): TaskClient = clients.computeIfAbsent(config, ::createClient) + + private fun createClient(config: TaskConfig): TaskClient = + when (config.task) { + "summarization" -> { + val client = Summarization.getClient( + SummarizerOptions.builder(context) + .setInputType( + option(config.inputType, "inputType", SummarizerOptions.InputType.ARTICLE) { + when (it) { + "article" -> SummarizerOptions.InputType.ARTICLE + "conversation" -> SummarizerOptions.InputType.CONVERSATION + else -> null + } + }, + ) + .setOutputType( + option(config.outputType, "outputType", SummarizerOptions.OutputType.ONE_BULLET) { + when (it) { + "oneBullet" -> SummarizerOptions.OutputType.ONE_BULLET + "twoBullets" -> SummarizerOptions.OutputType.TWO_BULLETS + "threeBullets" -> SummarizerOptions.OutputType.THREE_BULLETS + else -> null + } + }, + ) + .setLanguage( + option(config.language, "language", SummarizerOptions.Language.ENGLISH) { + when (it) { + "en" -> SummarizerOptions.Language.ENGLISH + "ja" -> SummarizerOptions.Language.JAPANESE + "ko" -> SummarizerOptions.Language.KOREAN + else -> null + } + }, + ) + .setLongInputAutoTruncationEnabled(config.autoTruncate) + .build(), + ) + TaskClient(client::checkFeatureStatus, client::downloadFeature, client::prepareInferenceEngine) { input, callback -> + val request = SummarizationRequest.builder(input).build() + val result = if (callback == null) client.runInference(request) else client.runInference(request, callback) + listOf(result.await().summary) + } + } + "proofreading" -> { + val client = Proofreading.getClient( + ProofreaderOptions.builder(context) + .setInputType( + option(config.inputType, "inputType", ProofreaderOptions.InputType.KEYBOARD) { + when (it) { + "keyboard" -> ProofreaderOptions.InputType.KEYBOARD + "voice" -> ProofreaderOptions.InputType.VOICE + else -> null + } + }, + ) + .setLanguage(option(config.language, "language", ProofreaderOptions.Language.ENGLISH, ::proofreaderLanguage)) + .build(), + ) + TaskClient(client::checkFeatureStatus, client::downloadFeature, client::prepareInferenceEngine) { input, callback -> + val request = ProofreadingRequest.builder(input).build() + val result = if (callback == null) client.runInference(request) else client.runInference(request, callback) + result.await().results.map { it.text } + } + } + "rewriting" -> { + val client = Rewriting.getClient( + RewriterOptions.builder(context) + .setOutputType( + option(config.outputType, "outputType", RewriterOptions.OutputType.REPHRASE) { + when (it) { + "elaborate" -> RewriterOptions.OutputType.ELABORATE + "emojify" -> RewriterOptions.OutputType.EMOJIFY + "shorten" -> RewriterOptions.OutputType.SHORTEN + "friendly" -> RewriterOptions.OutputType.FRIENDLY + "professional" -> RewriterOptions.OutputType.PROFESSIONAL + "rephrase" -> RewriterOptions.OutputType.REPHRASE + else -> null + } + }, + ) + .setLanguage(option(config.language, "language", RewriterOptions.Language.ENGLISH, ::rewriterLanguage)) + .build(), + ) + TaskClient(client::checkFeatureStatus, client::downloadFeature, client::prepareInferenceEngine) { input, callback -> + val request = RewritingRequest.builder(input).build() + val result = if (callback == null) client.runInference(request) else client.runInference(request, callback) + result.await().results.map { it.text } + } + } + "imageDescription" -> { + val client = ImageDescription.getClient(ImageDescriberOptions.builder(context).build()) + TaskClient(client::checkFeatureStatus, client::downloadFeature, client::prepareInferenceEngine) { input, callback -> + val request = ImageDescriptionRequest.builder(bitmap(input)).build() + val result = if (callback == null) client.runInference(request) else client.runInference(request, callback) + listOf(result.await().description) + } + } + else -> throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown task: ${config.task}") + } + + // Proofreading and rewriting take the same languages, as constants of their own options + private fun proofreaderLanguage(code: String): Int? = + when (code) { + "en" -> ProofreaderOptions.Language.ENGLISH + "ja" -> ProofreaderOptions.Language.JAPANESE + "de" -> ProofreaderOptions.Language.GERMAN + "fr" -> ProofreaderOptions.Language.FRENCH + "it" -> ProofreaderOptions.Language.ITALIAN + "es" -> ProofreaderOptions.Language.SPANISH + "ko" -> ProofreaderOptions.Language.KOREAN + else -> null + } + + private fun rewriterLanguage(code: String): Int? = + when (code) { + "en" -> RewriterOptions.Language.ENGLISH + "ja" -> RewriterOptions.Language.JAPANESE + "de" -> RewriterOptions.Language.GERMAN + "fr" -> RewriterOptions.Language.FRENCH + "it" -> RewriterOptions.Language.ITALIAN + "es" -> RewriterOptions.Language.SPANISH + "ko" -> RewriterOptions.Language.KOREAN + else -> null + } + + private fun option(value: String?, name: String, default: Int, parse: (String) -> Int?): Int = + if (value == null) { + default + } else { + parse(value) ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unsupported $name: $value") + } + + // A file://, content:// URI or base64 image + private fun bitmap(image: String): Bitmap { + val bytes = + if (image.startsWith("file:") || image.startsWith("content:")) { + context.contentResolver.openInputStream(Uri.parse(image))?.use { it.readBytes() } + } else { + Base64.decode(image, Base64.DEFAULT) + } + return bytes?.let { BitmapFactory.decodeByteArray(it, 0, it.size) } + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Cannot read image: ${image.take(64)}") + } + + suspend fun getModelInfo(config: TaskConfig): ModelInfo = + ModelInfo(availability = availabilityOf(client(config).checkFeatureStatus().await())) + + /** Downloads the task's model when needed, reporting progress, then loads it. */ + suspend fun prepare(config: TaskConfig) { + if (config in prepared) return + prepareMutexes.computeIfAbsent(config) { Mutex() }.withLock { + if (config in prepared) return + + val client = client(config) + when (val status = client.checkFeatureStatus().await()) { + FeatureStatus.UNAVAILABLE -> + throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_UNAVAILABLE, + "${config.task} is not available on this device", + ) + FeatureStatus.DOWNLOADABLE, FeatureStatus.DOWNLOADING -> + downloads.collect(config.task, "The ${config.task} model", status, client.downloadFeature) + } + client.prepareInferenceEngine().await() + prepared.add(config) + } + } + + /** Runs the task, streaming its text to [onText] when given, and returns its results. */ + suspend fun run(config: TaskConfig, input: String, onText: ((String) -> Unit)?): List { + prepare(config) + val callback = onText?.let { StreamingCallback { text -> it(text) } } + return client(config).run(input, callback) + } +} + +/** Waits for an ML Kit future, cancelling it with the coroutine. */ +suspend fun ListenableFuture.await(): T = + suspendCancellableCoroutine { continuation -> + addListener( + { + try { + continuation.resume(get()) + } catch (error: ExecutionException) { + continuation.resumeWithException(error.cause ?: error) + } catch (error: Exception) { + continuation.resumeWithException(error) + } + }, + Runnable::run, + ) + continuation.invokeOnCancellation { cancel(true) } + } diff --git a/packages/android/src/NativeAndroidAI.ts b/packages/android/src/NativeAndroidAI.ts index 9fa31d36..083bea12 100644 --- a/packages/android/src/NativeAndroidAI.ts +++ b/packages/android/src/NativeAndroidAI.ts @@ -90,7 +90,7 @@ export interface StreamEvent { } export interface DownloadProgressEvent { - /** `language`, `transcription` or `embedding` */ + /** `language`, `transcription`, `embedding` or a task name */ model: string /** `started`, `progress` or `completed` */ phase: string @@ -111,6 +111,19 @@ export interface NativeEmbeddingModelInfo { modelSignature?: string } +export interface NativeTaskOptions { + /** Summarization: `article` or `conversation`. Proofreading: `keyboard` or `voice` */ + inputType?: string + /** Summarization: `oneBullet`, `twoBullets` or `threeBullets`. Rewriting: the style */ + outputType?: string + /** ISO 639-1 language code, such as en */ + language?: string + /** Summarization: truncates input past the model's token limit */ + autoTruncate?: boolean + /** Sends the text as it is generated as `text` stream events of the run */ + stream?: boolean +} + export interface Spec extends TurboModule { getModelInfo(model?: NanoModelConfig): Promise prepare(model?: NanoModelConfig): Promise @@ -121,7 +134,7 @@ export interface Spec extends TurboModule { request: NativeGenerateRequest ): Promise streamText(streamId: string, request: NativeGenerateRequest): Promise - /** Cancels a `generateText`, `streamText` or `transcribe` run by its id */ + /** Cancels a `generateText`, `streamText`, `transcribe` or `runTask` run by its id */ cancel(runId: string): void getTranscriptionModelInfo( @@ -139,6 +152,23 @@ export interface Spec extends TurboModule { prepareEmbedding(): Promise embed(values: string[], taskType?: string): Promise + /** `summarization`, `proofreading`, `rewriting` or `imageDescription` */ + getTaskModelInfo( + task: string, + options: NativeTaskOptions + ): Promise + prepareTask(task: string, options: NativeTaskOptions): Promise + /** + * Runs a task on text, or for image description on a file://, content:// or + * base64 image. Resolves with its results, best first + */ + runTask( + runId: string, + task: string, + input: string, + options: NativeTaskOptions + ): Promise + onStreamEvent: CodegenTypes.EventEmitter onDownloadProgress: CodegenTypes.EventEmitter } diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts index c78aaad8..944eb5fd 100644 --- a/packages/android/src/__tests__/ai-sdk.test.ts +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -50,6 +50,23 @@ const getTranscriptionModelInfo = mock( availability: 'downloadable', }) ) +const prepareTask = mock(async (_task: string, _options: unknown) => {}) +const getTaskModelInfo = mock( + async ( + _task: string, + _options: unknown + ): Promise> => ({ + availability: 'downloadable', + }) +) +const runTask = mock( + async ( + _runId: string, + _task: string, + _input: string, + _options: Record + ): Promise => ['first', 'second'] +) const embed = mock(async (values: string[], _taskType?: string) => values.map(() => [0.1, 0.2]) ) @@ -68,6 +85,9 @@ mock.module('react-native', () => ({ prepareTranscription, prepareEmbedding, embed, + prepareTask, + getTaskModelInfo, + runTask, onStreamEvent, onDownloadProgress, }), @@ -627,3 +647,80 @@ describe('other models', () => { ]) }) }) + +describe('task models', () => { + const provider = createAndroidProvider() + + test('summarizes with the summarizer options and returns the summary', async () => { + const summarizer = provider.summarizer({ outputType: 'threeBullets' }) + await summarizer.prepare() + + expect(await summarizer.summarize('text')).toBe('first') + expect(prepareTask).toHaveBeenLastCalledWith('summarization', { + outputType: 'threeBullets', + }) + expect(runTask.mock.calls.at(-1)!.slice(1)).toEqual([ + 'summarization', + 'text', + { outputType: 'threeBullets', stream: false }, + ]) + }) + + test('returns every proofreading and rewriting suggestion', async () => { + const rewriter = provider.rewriter({ style: 'shorten', language: 'de' }) + await rewriter.prepare() + + expect(await rewriter.rewrite('text')).toEqual(['first', 'second']) + expect(runTask.mock.calls.at(-1)![3]).toEqual({ + outputType: 'shorten', + language: 'de', + stream: false, + }) + + const proofreader = provider.proofreader() + await proofreader.prepare() + expect(await proofreader.proofread('text')).toEqual(['first', 'second']) + }) + + test('streams text of its own run', async () => { + runTask.mockImplementationOnce(async (runId) => { + onStreamEvent.emit({ streamId: 'other', type: 'text', delta: 'x' }) + onStreamEvent.emit({ streamId: runId, type: 'text', delta: 'A cat' }) + return ['A cat'] + }) + const describer = provider.imageDescriber() + await describer.prepare() + const onText = mock(() => {}) + + expect( + await describer.describe(new Uint8Array([1, 2, 3]), { onText }) + ).toBe('A cat') + expect(onText.mock.calls).toEqual([['A cat']]) + expect(runTask.mock.calls.at(-1)!.slice(1)).toEqual([ + 'imageDescription', + 'AQID', + { stream: true }, + ]) + }) + + test('cancels the native run on abort', async () => { + const controller = new AbortController() + runTask.mockImplementationOnce(async (runId) => { + controller.abort() + expect(cancel).toHaveBeenCalledWith(runId) + throw new Error('cancelled') + }) + const summarizer = provider.summarizer() + await summarizer.prepare() + + await expect( + summarizer.summarize('text', { abortSignal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + + test('reports task availability', async () => { + expect(await provider.summarizer().getModelInfo()).toEqual({ + availability: 'downloadable', + }) + }) +}) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index 0241a620..8f181fee 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -33,6 +33,7 @@ import { type NativeMessage, type NativeMessagePart, type NativeModelInfo, + type NativeTaskOptions, type NativeTranscriptionOptions, } from './NativeAndroidAI' @@ -139,6 +140,46 @@ export interface AndroidEmbeddingModelInfo { modelSignature?: string } +/** Languages proofreading and rewriting take, as ISO 639-1 codes */ +export type AndroidTaskLanguage = 'en' | 'ja' | 'de' | 'fr' | 'it' | 'es' | 'ko' + +export interface AndroidSummarizerOptions { + /** `article` (default) or `conversation`, as `: ` lines */ + inputType?: 'article' | 'conversation' + /** How many bullets the summary has, `oneBullet` by default */ + outputType?: 'oneBullet' | 'twoBullets' | 'threeBullets' + /** `en` (default), `ja` or `ko` */ + language?: 'en' | 'ja' | 'ko' + /** Truncates input past the model's limit of about 4000 tokens, instead of failing */ + autoTruncate?: boolean +} + +export interface AndroidProofreaderOptions { + /** `keyboard` (default) or `voice`, for text transcribed from speech */ + inputType?: 'keyboard' | 'voice' + /** `en` by default */ + language?: AndroidTaskLanguage +} + +export interface AndroidRewriterOptions { + /** `rephrase` by default */ + style?: + | 'elaborate' + | 'emojify' + | 'shorten' + | 'friendly' + | 'professional' + | 'rephrase' + /** `en` by default */ + language?: AndroidTaskLanguage +} + +export interface AndroidTaskCallOptions { + /** Called with the text as it is generated */ + onText?: (text: string) => void + abortSignal?: AbortSignal +} + const toModelAvailability = (value: string): AndroidModelAvailability => { switch (value) { case 'available': @@ -230,6 +271,17 @@ export function createAndroidProvider( new AndroidTranscriptionModel(options) provider.embeddingModel = (options: AndroidEmbeddingOptions = {}) => new AndroidEmbeddingModel(options) + /** Summarizes articles or conversations into one to three bullets */ + provider.summarizer = (options: AndroidSummarizerOptions = {}) => + new AndroidSummarizer(options) + /** Fixes grammar and spelling, with suggestions best first */ + provider.proofreader = (options: AndroidProofreaderOptions = {}) => + new AndroidProofreader(options) + /** Rewrites text in a style, with suggestions best first */ + provider.rewriter = (options: AndroidRewriterOptions = {}) => + new AndroidRewriter(options) + /** Describes images in a short English sentence */ + provider.imageDescriber = () => new AndroidImageDescriber() return provider } @@ -991,3 +1043,141 @@ class AndroidEmbeddingModel implements EmbeddingModelV4 { return { embeddings, warnings: [] } } } + +/** + * One of ML Kit's task APIs, which run Gemini Nano with an adapter for the + * task. Each downloads its adapter on first use + */ +class AndroidTaskModel { + private prepared = false + + constructor( + private readonly task: string, + private readonly options: NativeTaskOptions + ) {} + + /** The task's availability on this device */ + async getModelInfo(): Promise<{ availability: AndroidModelAvailability }> { + if (Platform.OS !== 'android') { + return { availability: 'unavailable' } + } + try { + const info = await getNativeAndroidAI().getTaskModelInfo( + this.task, + this.options + ) + return { availability: toModelAvailability(info.availability) } + } catch (error) { + throw toAndroidAIError(error) + } + } + + /** Downloads the task's model when needed and loads it, reporting download progress */ + async prepare(options: AndroidPrepareOptions = {}): Promise { + await withProgress(this.task, options, () => + getNativeAndroidAI().prepareTask(this.task, this.options) + ) + this.prepared = true + } + + protected async run( + input: string, + { onText, abortSignal }: AndroidTaskCallOptions + ): Promise { + const native = getNativeAndroidAI() + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + if (!this.prepared) { + console.warn(notPreparedWarning) + await this.prepare() + } + + const runId = generateId() + const listener = onText + ? native.onStreamEvent((event) => { + if (event.streamId === runId && event.type === 'text') { + onText(event.delta ?? '') + } + }) + : null + const onAbort = () => native.cancel(runId) + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + try { + return await native.runTask(runId, this.task, input, { + ...this.options, + stream: Boolean(onText), + }) + } catch (error) { + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + throw toAndroidAIError(error) + } finally { + listener?.remove() + abortSignal?.removeEventListener('abort', onAbort) + } + } +} + +export class AndroidSummarizer extends AndroidTaskModel { + constructor(options: AndroidSummarizerOptions = {}) { + super('summarization', options) + } + + async summarize( + text: string, + options: AndroidTaskCallOptions = {} + ): Promise { + const [summary = ''] = await this.run(text, options) + return summary + } +} + +export class AndroidProofreader extends AndroidTaskModel { + constructor(options: AndroidProofreaderOptions = {}) { + super('proofreading', options) + } + + /** Corrected versions of the text, best first. Takes up to about 256 tokens */ + proofread( + text: string, + options: AndroidTaskCallOptions = {} + ): Promise { + return this.run(text, options) + } +} + +export class AndroidRewriter extends AndroidTaskModel { + constructor({ style, language }: AndroidRewriterOptions = {}) { + super('rewriting', { outputType: style, language }) + } + + /** Rewritten versions of the text, best first. Takes up to about 256 tokens */ + rewrite( + text: string, + options: AndroidTaskCallOptions = {} + ): Promise { + return this.run(text, options) + } +} + +export class AndroidImageDescriber extends AndroidTaskModel { + constructor() { + super('imageDescription', {}) + } + + /** Describes an image, given as bytes, base64, or a file:// or content:// URI */ + async describe( + image: Uint8Array | string, + options: AndroidTaskCallOptions = {} + ): Promise { + const input = + typeof image === 'string' + ? image.replace(/^data:[^;]+;base64,/, '') + : uint8ArrayToBase64(image) + const [description = ''] = await this.run(input, options) + return description + } +} diff --git a/packages/android/src/index.ts b/packages/android/src/index.ts index 171cb69d..8c846a74 100644 --- a/packages/android/src/index.ts +++ b/packages/android/src/index.ts @@ -3,6 +3,7 @@ export type { AndroidEmbeddingModelInfo, AndroidEmbeddingOptions, AndroidEmbeddingTaskType, + AndroidImageDescriber, AndroidLanguageModel, AndroidLanguageModelOptions, AndroidModelAvailability, @@ -10,8 +11,16 @@ export type { AndroidModelInfo, AndroidNanoModel, AndroidPrepareOptions, + AndroidProofreader, + AndroidProofreaderOptions, AndroidProvider, AndroidProviderOptions, + AndroidRewriter, + AndroidRewriterOptions, + AndroidSummarizer, + AndroidSummarizerOptions, + AndroidTaskCallOptions, + AndroidTaskLanguage, AndroidTranscriptionOptions, } from './ai-sdk' export { android, createAndroidProvider } from './ai-sdk' diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 7f661155..564ce282 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -102,6 +102,7 @@ export default defineConfig({ ['errors', 'Error handling'], ['embeddings', 'Embeddings'], ['transcription', 'Transcription'], + ['tasks', 'Summarize and rewrite'], ]), group('Core AI', 'core-ai', [ ['getting-started', 'Getting started'], diff --git a/website/src/content/docs/docs/android/tasks.md b/website/src/content/docs/docs/android/tasks.md new file mode 100644 index 00000000..464058cd --- /dev/null +++ b/website/src/content/docs/docs/android/tasks.md @@ -0,0 +1,46 @@ +--- +title: "Summarize, Proofread, Rewrite and Describe" +--- + +ML Kit's [task APIs](https://developers.google.com/ml-kit/genai) run Gemini +Nano with an adapter trained for one job. They're beta APIs with no AI SDK +model type, so each is a model of its own on the provider: + +```typescript +import { android } from '@react-native-ai/android' + +const summarizer = android.summarizer({ outputType: 'threeBullets' }) +await summarizer.prepare() + +const summary = await summarizer.summarize(article) +``` + +- `android.summarizer(options)` summarizes an article, or a conversation of `: ` lines, into one to three bullets. Options: `inputType` (`article` or `conversation`), `outputType` (`oneBullet`, `twoBullets` or `threeBullets`), `language` (`en`, `ja` or `ko`) and `autoTruncate`, which cuts input past about 4000 tokens instead of failing. +- `android.proofreader(options).proofread(text)` fixes grammar and spelling. `inputType: 'voice'` is for text transcribed from speech. +- `android.rewriter({ style }).rewrite(text)` rewrites in a `style`: `elaborate`, `emojify`, `shorten`, `friendly`, `professional` or `rephrase` (the default). +- Proofreading and rewriting take up to about 256 tokens, and return their suggestions best first. They take `language`: `en`, `ja`, `de`, `fr`, `it`, `es` or `ko`. +- `android.imageDescriber().describe(image)` describes an image in a short English sentence. The image is a `Uint8Array`, base64, or a `file://` or `content://` URI. + +## Streaming and Aborting + +Every call takes `onText`, called with the text as it is generated, and an +`abortSignal`: + +```typescript +await summarizer.summarize(article, { + onText: (text) => setSummary((summary) => summary + text), + abortSignal: controller.signal, +}) +``` + +## Availability and Download + +```typescript +const { availability } = await summarizer.getModelInfo() + +await summarizer.prepare({ onProgress: (progress) => console.log(progress) }) +``` + +Each task downloads its adapter on first use, which is quick once Gemini Nano +is on the device. A call prepares the task on its own when you don't, and logs +a warning. From 0b29448f437882764faf2dc01af3498a889373a5 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 00:23:15 +0100 Subject: [PATCH 2/4] fix(android): honor aborts during task prepare and export the task classes Co-Authored-By: Claude Opus 5.5 --- .../com/callstack/ai/android/Downloads.kt | 10 +++++- .../java/com/callstack/ai/android/Tasks.kt | 35 +++++-------------- packages/android/src/ai-sdk.ts | 4 +++ packages/android/src/index.ts | 13 ++++--- website/astro.config.mjs | 2 +- .../src/content/docs/docs/android/tasks.md | 2 +- 6 files changed, 32 insertions(+), 34 deletions(-) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt index 10eb10ba..bb9e869f 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt @@ -6,6 +6,7 @@ import com.google.mlkit.genai.common.DownloadStatus import com.google.mlkit.genai.common.FeatureStatus import com.google.mlkit.genai.common.GenAiException import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.guava.await /** Collects ML Kit model downloads and reports their progress. */ class Downloads( @@ -58,8 +59,15 @@ class Downloads( try { download(callback).await() } catch (error: GenAiException) { + if (status == FeatureStatus.DOWNLOADING) { + throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_DOWNLOADING, + "$displayName is still downloading", + error, + ) + } throw AndroidAIErrors.CodedException( - if (status == FeatureStatus.DOWNLOADING) AndroidAIErrors.MODEL_DOWNLOADING else AndroidAIErrors.DOWNLOAD_FAILED, + AndroidAIErrors.DOWNLOAD_FAILED, "$displayName download failed: ${error.message}", error, ) diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt index fa3521ff..6bd4a7fc 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt @@ -22,13 +22,10 @@ import com.google.mlkit.genai.rewriting.RewritingRequest import com.google.mlkit.genai.summarization.Summarization import com.google.mlkit.genai.summarization.SummarizationRequest import com.google.mlkit.genai.summarization.SummarizerOptions -import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.guava.await import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ExecutionException -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException /** A task and its options, which pick the ML Kit client and its LoRA adapter. */ data class TaskConfig( @@ -208,10 +205,14 @@ class Tasks( // A file://, content:// URI or base64 image private fun bitmap(image: String): Bitmap { val bytes = - if (image.startsWith("file:") || image.startsWith("content:")) { - context.contentResolver.openInputStream(Uri.parse(image))?.use { it.readBytes() } - } else { - Base64.decode(image, Base64.DEFAULT) + try { + if (image.startsWith("file:") || image.startsWith("content:")) { + context.contentResolver.openInputStream(Uri.parse(image))?.use { it.readBytes() } + } else { + Base64.decode(image, Base64.DEFAULT) + } + } catch (error: Exception) { + null } return bytes?.let { BitmapFactory.decodeByteArray(it, 0, it.size) } ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Cannot read image: ${image.take(64)}") @@ -248,21 +249,3 @@ class Tasks( return client(config).run(input, callback) } } - -/** Waits for an ML Kit future, cancelling it with the coroutine. */ -suspend fun ListenableFuture.await(): T = - suspendCancellableCoroutine { continuation -> - addListener( - { - try { - continuation.resume(get()) - } catch (error: ExecutionException) { - continuation.resumeWithException(error.cause ?: error) - } catch (error: Exception) { - continuation.resumeWithException(error) - } - }, - Runnable::run, - ) - continuation.invokeOnCancellation { cancel(true) } - } diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index 8f181fee..89753366 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -1091,6 +1091,10 @@ class AndroidTaskModel { if (!this.prepared) { console.warn(notPreparedWarning) await this.prepare() + // Preparing can take minutes, before there is a native run to cancel + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } } const runId = generateId() diff --git a/packages/android/src/index.ts b/packages/android/src/index.ts index 8c846a74..76944597 100644 --- a/packages/android/src/index.ts +++ b/packages/android/src/index.ts @@ -3,7 +3,6 @@ export type { AndroidEmbeddingModelInfo, AndroidEmbeddingOptions, AndroidEmbeddingTaskType, - AndroidImageDescriber, AndroidLanguageModel, AndroidLanguageModelOptions, AndroidModelAvailability, @@ -11,19 +10,23 @@ export type { AndroidModelInfo, AndroidNanoModel, AndroidPrepareOptions, - AndroidProofreader, AndroidProofreaderOptions, AndroidProvider, AndroidProviderOptions, - AndroidRewriter, AndroidRewriterOptions, - AndroidSummarizer, AndroidSummarizerOptions, AndroidTaskCallOptions, AndroidTaskLanguage, AndroidTranscriptionOptions, } from './ai-sdk' -export { android, createAndroidProvider } from './ai-sdk' +export { + android, + AndroidImageDescriber, + AndroidProofreader, + AndroidRewriter, + AndroidSummarizer, + createAndroidProvider, +} from './ai-sdk' export type { AndroidAIError, AndroidAIErrorCode } from './errors' export { AndroidAIErrorCodes } from './errors' export { default as AndroidAI, getNativeAndroidAI } from './NativeAndroidAI' diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 564ce282..4bb15751 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -102,7 +102,7 @@ export default defineConfig({ ['errors', 'Error handling'], ['embeddings', 'Embeddings'], ['transcription', 'Transcription'], - ['tasks', 'Summarize and rewrite'], + ['tasks', 'Summarize, proofread, rewrite and describe'], ]), group('Core AI', 'core-ai', [ ['getting-started', 'Getting started'], diff --git a/website/src/content/docs/docs/android/tasks.md b/website/src/content/docs/docs/android/tasks.md index 464058cd..184a59c7 100644 --- a/website/src/content/docs/docs/android/tasks.md +++ b/website/src/content/docs/docs/android/tasks.md @@ -18,7 +18,7 @@ const summary = await summarizer.summarize(article) - `android.summarizer(options)` summarizes an article, or a conversation of `: ` lines, into one to three bullets. Options: `inputType` (`article` or `conversation`), `outputType` (`oneBullet`, `twoBullets` or `threeBullets`), `language` (`en`, `ja` or `ko`) and `autoTruncate`, which cuts input past about 4000 tokens instead of failing. - `android.proofreader(options).proofread(text)` fixes grammar and spelling. `inputType: 'voice'` is for text transcribed from speech. - `android.rewriter({ style }).rewrite(text)` rewrites in a `style`: `elaborate`, `emojify`, `shorten`, `friendly`, `professional` or `rephrase` (the default). -- Proofreading and rewriting take up to about 256 tokens, and return their suggestions best first. They take `language`: `en`, `ja`, `de`, `fr`, `it`, `es` or `ko`. +- `proofread` and `rewrite` take up to about 256 tokens, and return an array of suggestions, best first. Both take `language`: `en`, `ja`, `de`, `fr`, `it`, `es` or `ko`. - `android.imageDescriber().describe(image)` describes an image in a short English sentence. The image is a `Uint8Array`, base64, or a `file://` or `content://` URI. ## Streaming and Aborting From 23aec3538a1fcf5ec14b4ef4775a3f2b1b55a7b8 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 00:40:27 +0100 Subject: [PATCH 3/4] fix(android): declare coroutines-guava and keep task download and image errors Co-Authored-By: Claude Opus 5.5 --- packages/android/android/build.gradle | 2 ++ .../src/main/java/com/callstack/ai/android/Downloads.kt | 7 ++----- .../src/main/java/com/callstack/ai/android/Tasks.kt | 8 +++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/android/android/build.gradle b/packages/android/android/build.gradle index 944bf85b..bd4cc71d 100644 --- a/packages/android/android/build.gradle +++ b/packages/android/android/build.gradle @@ -93,6 +93,8 @@ dependencies { implementation "com.facebook.react:react-native:+" implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.11.0" + // ListenableFuture.await() for the ML Kit task clients + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.11.0" // genai-prompt beta5 and speech recognition alpha2 are built with Kotlin 2.4 implementation "com.google.mlkit:genai-prompt:1.0.0-beta5" implementation "com.google.mlkit:genai-schema:1.0.0-alpha2" diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt index bb9e869f..44e49af8 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt @@ -66,11 +66,8 @@ class Downloads( error, ) } - throw AndroidAIErrors.CodedException( - AndroidAIErrors.DOWNLOAD_FAILED, - "$displayName download failed: ${error.message}", - error, - ) + // Its error code, such as not enough disk space, picks the error JS gets + throw error } } } diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt index 6bd4a7fc..72d74e7b 100644 --- a/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Tasks.kt @@ -204,6 +204,9 @@ class Tasks( // A file://, content:// URI or base64 image private fun bitmap(image: String): Bitmap { + val cannotRead = { cause: Exception? -> + AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Cannot read image: ${image.take(64)}", cause) + } val bytes = try { if (image.startsWith("file:") || image.startsWith("content:")) { @@ -212,10 +215,9 @@ class Tasks( Base64.decode(image, Base64.DEFAULT) } } catch (error: Exception) { - null + throw cannotRead(error) } - return bytes?.let { BitmapFactory.decodeByteArray(it, 0, it.size) } - ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Cannot read image: ${image.take(64)}") + return bytes?.let { BitmapFactory.decodeByteArray(it, 0, it.size) } ?: throw cannotRead(null) } suspend fun getModelInfo(config: TaskConfig): ModelInfo = From 1a4954e3757113807319c099ad7dfb0425c80bb5 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 00:54:27 +0100 Subject: [PATCH 4/4] fix(android): strip data uris with parameters from images to describe Co-Authored-By: Claude Opus 5.5 --- packages/android/src/__tests__/ai-sdk.test.ts | 8 ++++++++ packages/android/src/ai-sdk.ts | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts index 944eb5fd..c93cf2a1 100644 --- a/packages/android/src/__tests__/ai-sdk.test.ts +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -703,6 +703,14 @@ describe('task models', () => { ]) }) + test('strips a data URI prefix from a base64 image', async () => { + const describer = provider.imageDescriber() + await describer.prepare() + + await describer.describe('data:image/png;charset=utf-8;base64,AQID') + expect(runTask.mock.calls.at(-1)![2]).toBe('AQID') + }) + test('cancels the native run on abort', async () => { const controller = new AbortController() runTask.mockImplementationOnce(async (runId) => { diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts index 89753366..7630eecd 100644 --- a/packages/android/src/ai-sdk.ts +++ b/packages/android/src/ai-sdk.ts @@ -1172,14 +1172,14 @@ export class AndroidImageDescriber extends AndroidTaskModel { super('imageDescription', {}) } - /** Describes an image, given as bytes, base64, or a file:// or content:// URI */ + /** Describes an image, given as bytes, base64 or a base64 data URI, or a file:// or content:// URI */ async describe( image: Uint8Array | string, options: AndroidTaskCallOptions = {} ): Promise { const input = typeof image === 'string' - ? image.replace(/^data:[^;]+;base64,/, '') + ? image.replace(/^data:[^,]*;base64,/, '') : uint8ArrayToBase64(image) const [description = ''] = await this.run(input, options) return description