diff --git a/apps/example/app.json b/apps/example/app.json index 0e2fa15c..1a27faf3 100644 --- a/apps/example/app.json +++ b/apps/example/app.json @@ -35,7 +35,8 @@ "expo-build-properties", { "android": { - "minSdkVersion": 26 + "minSdkVersion": 26, + "kotlinVersion": "2.4.0" } } ], diff --git a/apps/example/jest.harness.config.mjs b/apps/example/jest.harness.config.mjs index f688529e..4b1dd7a8 100644 --- a/apps/example/jest.harness.config.mjs +++ b/apps/example/jest.harness.config.mjs @@ -1,13 +1,20 @@ +const ignored = [ + // Tests that need a physical device are left out on simulators + ...(process.env.HARNESS_SIMULATOR === 'true' + ? ['\\.device\\.harness\\.ts$'] + : []), + // Each platform leaves out the providers that only run on the other one + ...(process.env.HARNESS_PLATFORM === 'android' + ? ['/harness/apple/', '/harness/core-ai/', '/harness/json-ui/apple\\.'] + : ['/harness/android/']), +] + // The tests live in the harness folder at the monorepo root export default { preset: 'react-native-harness', roots: ['/../../harness'], testMatch: ['**/*.harness.ts'], - // Tests that need a physical device are left out on simulators - testPathIgnorePatterns: - process.env.HARNESS_SIMULATOR === 'true' - ? ['\\.device\\.harness\\.ts$'] - : [], + testPathIgnorePatterns: ignored, setupFiles: ['/../../harness/setup.ts'], // The test files are few, so a crawl is fast and does not depend on Watchman watchman: false, diff --git a/apps/example/package.json b/apps/example/package.json index be6616a3..274568f7 100644 --- a/apps/example/package.json +++ b/apps/example/package.json @@ -8,7 +8,8 @@ "ios": "expo run:ios", "typecheck": "tsc --noEmit", "test:harness:simulator": "scripts/test-harness.sh simulator", - "test:harness:device": "scripts/test-harness.sh device" + "test:harness:device": "scripts/test-harness.sh device", + "test:harness:android": "scripts/test-harness.sh android" }, "dependencies": { "@ai-sdk/provider": "^4.0.33", @@ -16,6 +17,7 @@ "@expo/ui": "~58.0.23", "@expo/vector-icons": "^15.0.3", "@legendapp/list": "^3.7.0", + "@react-native-ai/android": "workspace:*", "@react-native-ai/apple": "workspace:*", "@react-native-ai/core-ai": "workspace:*", "@react-native-ai/json-ui": "workspace:*", @@ -55,6 +57,7 @@ }, "devDependencies": { "@babel/core": "^7.20.0", + "@react-native-harness/platform-android": "1.6.0", "@react-native-harness/platform-apple": "1.6.0", "@types/react": "~19.3.0", "@types/ungap__structured-clone": "^1.2.0", diff --git a/apps/example/rn-harness.config.mjs b/apps/example/rn-harness.config.mjs index 7c838321..60e7be2a 100644 --- a/apps/example/rn-harness.config.mjs +++ b/apps/example/rn-harness.config.mjs @@ -1,3 +1,7 @@ +import { + androidPlatform, + physicalAndroidDevice, +} from '@react-native-harness/platform-android' import { applePhysicalDevice, applePlatform, @@ -8,9 +12,14 @@ const deviceName = process.env.HARNESS_IOS_DEVICE // Builds signed with entitlements the default App ID lacks use another one const bundleId = process.env.HARNESS_IOS_BUNDLE_ID ?? 'com.callstack.ai.example' +const androidDevice = process.env.HARNESS_ANDROID_DEVICE +const androidManufacturer = process.env.HARNESS_ANDROID_MANUFACTURER ?? 'Google' + // Tests run in a debug build of the app. The ios runner uses the physical // device named with HARNESS_IOS_DEVICE (as `xcrun devicectl list devices` -// shows it), and ios-simulator a simulator +// shows it), and ios-simulator a simulator. The android runner uses the +// physical device whose model is HARNESS_ANDROID_DEVICE (as +// `adb shell getprop ro.product.model` shows it, such as Pixel 9 Pro) export default { entryPoint: './index.js', appRegistryComponentName: 'main', @@ -29,6 +38,15 @@ export default { device: appleSimulator('iPhone 18 Pro', '27.0'), bundleId: 'com.callstack.ai.example', }), + ...(androidDevice + ? [ + androidPlatform({ + name: 'android', + device: physicalAndroidDevice(androidManufacturer, androidDevice), + bundleId: 'com.callstack.ai.example', + }), + ] + : []), ], defaultRunner: deviceName ? 'ios' : 'ios-simulator', // On-device models load and generate slowly, but a test that hangs should diff --git a/apps/example/scripts/test-harness.sh b/apps/example/scripts/test-harness.sh index 9789eb03..267f5195 100755 --- a/apps/example/scripts/test-harness.sh +++ b/apps/example/scripts/test-harness.sh @@ -5,6 +5,7 @@ # # bun run test:harness:simulator [test path] # HARNESS_IOS_DEVICE="My iPhone" bun run test:harness:device [test path] +# HARNESS_ANDROID_DEVICE="Pixel 9 Pro" bun run test:harness:android [test path] set -euo pipefail @@ -35,8 +36,32 @@ case "$TARGET" in npx expo run:ios --device "$HARNESS_IOS_DEVICE" --no-bundler npx react-native-harness --harnessRunner ios "$@" ;; + android) + if [[ -z "${HARNESS_ANDROID_DEVICE:-}" ]]; then + echo 'Set HARNESS_ANDROID_DEVICE to the device model, as `adb shell getprop ro.product.model` shows it' >&2 + exit 1 + fi + # Harness runs the tests on the first connected device of that model and + # ignores ANDROID_SERIAL, so exactly one may be connected, and the build is + # installed on it + matching=() + for serial in $(adb devices | awk 'NR > 1 && $2 == "device" { print $1 }'); do + model="$(adb -s "$serial" shell getprop ro.product.model | tr -d '\r')" + if [[ "$model" == "$HARNESS_ANDROID_DEVICE" ]]; then + matching+=("$serial") + fi + done + if [[ "${#matching[@]}" != 1 ]]; then + echo "Connect exactly one $HARNESS_ANDROID_DEVICE, found ${#matching[@]}" >&2 + exit 1 + fi + export ANDROID_SERIAL="${matching[0]}" + npx expo prebuild --clean --platform android + npx expo run:android --no-bundler --device "$ANDROID_SERIAL" + HARNESS_PLATFORM=android npx react-native-harness --harnessRunner android "$@" + ;; *) - echo "Usage: $0 simulator|device [test path]" >&2 + echo "Usage: $0 simulator|device|android [test path]" >&2 exit 1 ;; esac diff --git a/apps/example/tsconfig.json b/apps/example/tsconfig.json index d1f25d04..c3b870d7 100644 --- a/apps/example/tsconfig.json +++ b/apps/example/tsconfig.json @@ -8,6 +8,9 @@ "" ], "paths": { + "@react-native-ai/android": [ + "../../packages/android/src" + ], "@react-native-ai/apple": [ "../../packages/apple-llm/src" ], diff --git a/bun.lock b/bun.lock index 77a4f4a5..7d977b76 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ "@expo/ui": "~58.0.23", "@expo/vector-icons": "^15.0.3", "@legendapp/list": "^3.7.0", + "@react-native-ai/android": "workspace:*", "@react-native-ai/apple": "workspace:*", "@react-native-ai/core-ai": "workspace:*", "@react-native-ai/json-ui": "workspace:*", @@ -78,6 +79,7 @@ }, "devDependencies": { "@babel/core": "^7.20.0", + "@react-native-harness/platform-android": "1.6.0", "@react-native-harness/platform-apple": "1.6.0", "@types/react": "~19.3.0", "@types/ungap__structured-clone": "^1.2.0", @@ -98,6 +100,20 @@ "react-native": ">=0.76.0", }, }, + "packages/android": { + "name": "@react-native-ai/android", + "version": "0.12.0", + "dependencies": { + "@ai-sdk/provider": "^4.0.33", + "@ai-sdk/provider-utils": "^5.0.67", + }, + "devDependencies": { + "react-native": "0.88.0", + }, + "peerDependencies": { + "react-native": ">=0.76.0", + }, + }, "packages/apple-llm": { "name": "@react-native-ai/apple", "version": "0.12.0", @@ -940,6 +956,8 @@ "@react-native-ai/adk": ["@react-native-ai/adk@workspace:packages/adk"], + "@react-native-ai/android": ["@react-native-ai/android@workspace:packages/android"], + "@react-native-ai/apple": ["@react-native-ai/apple@workspace:packages/apple-llm"], "@react-native-ai/core-ai": ["@react-native-ai/core-ai@workspace:packages/core-ai"], @@ -972,6 +990,8 @@ "@react-native-harness/metro": ["@react-native-harness/metro@1.6.0", "", { "dependencies": { "tslib": "^2.3.0" }, "peerDependencies": { "metro": "*" } }, "sha512-FbV/eAMQZkzPv3kZF/vlHM5AFY898vyRxLzSQiwjLeWiX0pa56wH/H/3FAjOfxlo3MSxeeUxaSGYNO6ZCt5wOA=="], + "@react-native-harness/platform-android": ["@react-native-harness/platform-android@1.6.0", "", { "dependencies": { "@react-native-harness/config": "1.6.0", "@react-native-harness/platforms": "1.6.0", "@react-native-harness/tools": "1.6.0", "tslib": "^2.3.0", "vite": "^7.2.2", "zod": "^3.25.67" } }, "sha512-Wfv3GrJLAMdHtlHd5pGgIZlmc2Aj1NSphAprKLL5GAhFxQJld4mYw5QdruwT5Kn92Vj31BAeKuKAqBq1vU7BPg=="], + "@react-native-harness/platform-apple": ["@react-native-harness/platform-apple@1.6.0", "", { "dependencies": { "@react-native-harness/config": "1.6.0", "@react-native-harness/platforms": "1.6.0", "@react-native-harness/tools": "1.6.0", "tslib": "^2.3.0", "yargs": "^17.7.2", "zod": "^3.25.67" } }, "sha512-2LanhJpnke56A6A2tMakBV58PyaXvOIZzlLI7eJFGSTCFdkCvN8nzE2H0MpzW6pqpfzlSfpw3iIxhydFJhRcLg=="], "@react-native-harness/platforms": ["@react-native-harness/platforms@1.6.0", "", { "dependencies": { "@react-native-harness/tools": "1.6.0", "tslib": "^2.3.0" } }, "sha512-k6LJUBam9UCURgIOicPIg4PY+ms+u8g1AFiIGYIpS3rNVCCk07qH9EupdjVGX0BfueYcqDVghjrZ6h+PVNriUg=="], @@ -3398,6 +3418,8 @@ "@react-native-harness/jest/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], + "@react-native-harness/platform-android/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@react-native-harness/platform-apple/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], "@react-native-harness/platform-apple/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], diff --git a/harness/android/availability.harness.ts b/harness/android/availability.harness.ts new file mode 100644 index 00000000..75aba9b4 --- /dev/null +++ b/harness/android/availability.harness.ts @@ -0,0 +1,40 @@ +import { android } from '@react-native-ai/android' +import { describe, expect, it } from 'react-native-harness' + +import { nano4 } from './helpers' + +// Gemini Nano runs only on physical devices with AICore, such as a Pixel 9 +describe('android availability', () => { + it('reports Gemini Nano details', async () => { + const info = await android.getModelInfo() + console.log('[android] Gemini Nano:', JSON.stringify(info)) + + expect(['available', 'downloadable', 'downloading']).toContain( + info.availability + ) + }) + + // On a fresh device this downloads Gemini Nano, which takes minutes + it('prepares Gemini Nano with download progress', async () => { + await android().prepare({ onProgress: () => {} }) + + const info = await android.getModelInfo() + expect(info.availability).toBe('available') + expect(info.contextSize).toBeGreaterThan(0) + expect(info.features).toBeDefined() + expect(android.isAvailable()).toBe(true) + }) + + it('reports Gemini Nano 4 preview details', async () => { + const info = await nano4.getModelInfo() + console.log('[android] Gemini Nano preview fast:', JSON.stringify(info)) + + expect(info.availability).not.toBe('unknown') + }) + + it('counts tokens with the Gemini Nano tokenizer', async () => { + expect( + await android().countTokens('How many tokens is this?') + ).toBeGreaterThan(0) + }) +}) diff --git a/harness/android/embedding.harness.ts b/harness/android/embedding.harness.ts new file mode 100644 index 00000000..7688a0c7 --- /dev/null +++ b/harness/android/embedding.harness.ts @@ -0,0 +1,32 @@ +import { android } from '@react-native-ai/android' +import { cosineSimilarity, embedMany } from 'ai' +import { describe, expect, it } from 'react-native-harness' + +const model = android.embeddingModel({ taskType: 'semanticSimilarity' }) + +describe('android embeddings', () => { + it('reports the embedding model', async () => { + const info = await model.getModelInfo() + console.log('[android] embedding model:', JSON.stringify(info)) + + expect(info.availability).not.toBe('unknown') + }) + + it('embeds similar sentences closer together', async () => { + await model.prepare({ onProgress: () => {} }) + + const { embeddings } = await embedMany({ + model, + values: [ + 'The cat sleeps on the sofa.', + 'A kitten naps on the couch.', + 'Stock markets fell sharply today.', + ], + }) + + expect(embeddings[0]!.length).toBeGreaterThan(0) + expect(cosineSimilarity(embeddings[0]!, embeddings[1]!)).toBeGreaterThan( + cosineSimilarity(embeddings[0]!, embeddings[2]!) + ) + }) +}) diff --git a/harness/android/generate.harness.ts b/harness/android/generate.harness.ts new file mode 100644 index 00000000..e6e101a8 --- /dev/null +++ b/harness/android/generate.harness.ts @@ -0,0 +1,250 @@ +import { android } from '@react-native-ai/android' +import { generateText, streamText } from 'ai' +import { File, Paths } from 'expo-file-system' +import { + beforeAll, + beforeEach, + describe, + expect, + it, +} from 'react-native-harness' + +import { base64ToBytes, redSquare } from '../fixtures' +import { cooldown, prompt, readAll } from './helpers' + +describe('android gemini nano', () => { + beforeAll(async () => { + await android().prepare() + }) + beforeEach(cooldown) + + it('answers a prompt', async () => { + const result = await android().doGenerate({ + prompt: prompt('Reply with exactly the word BANANA and nothing else.'), + }) + + expect(result.content.at(-1)).toMatchObject({ + type: 'text', + text: expect.stringMatching(/banana/i), + }) + expect(result.finishReason.unified).toBe('stop') + }) + + it('follows the system message', async () => { + const { text } = await generateText({ + model: android(), + system: 'Whatever the user says, reply with only the word PINEAPPLE.', + prompt: 'How are you?', + }) + + expect(text).toMatch(/pineapple/i) + }) + + it('remembers earlier turns', async () => { + const { text } = await generateText({ + model: android(), + messages: [ + { role: 'user', content: 'My favourite colour is teal. Just say OK.' }, + { role: 'assistant', content: 'OK' }, + { + role: 'user', + content: 'What is my favourite colour? Answer with one word.', + }, + ], + }) + + expect(text).toMatch(/teal/i) + }) + + it('streams text in several deltas', async () => { + const result = streamText({ + model: android(), + prompt: 'Count from 1 to 10, separated by spaces.', + }) + + let deltas = 0 + for await (const delta of result.textStream) { + if (delta) deltas++ + } + expect(await result.text).toMatch(/1.*2.*3.*4.*5/s) + expect(deltas).toBeGreaterThan(1) + }) + + it('describes an image', async () => { + const { text } = await generateText({ + model: android(), + messages: [ + { + role: 'user', + content: [ + { + type: 'file', + mediaType: 'image/png', + data: base64ToBytes(redSquare), + }, + { + type: 'text', + text: 'What colour is this image? Answer with one word.', + }, + ], + }, + ], + }) + + expect(text).toMatch(/red/i) + }) + + it('reads a local file image', async () => { + const file = new File(Paths.cache, 'harness-red.png') + file.write(base64ToBytes(redSquare)) + + const { text } = await generateText({ + model: android(), + messages: [ + { + role: 'user', + content: [ + { type: 'image', image: new URL(file.uri) }, + { + type: 'text', + text: 'What colour is this image? Answer with one word.', + }, + ], + }, + ], + }) + + expect(text).toMatch(/red/i) + }) + + it('reads a data URL image', async () => { + const { text } = await generateText({ + model: android(), + messages: [ + { + role: 'user', + content: [ + { type: 'image', image: `data:image/png;base64,${redSquare}` }, + { + type: 'text', + text: 'What colour is this image? Answer with one word.', + }, + ], + }, + ], + }) + + expect(text).toMatch(/red/i) + }) + + it('remembers earlier turns after a system message', async () => { + const { text } = await generateText({ + model: android(), + system: 'You are a helpful assistant. Answer briefly.', + messages: [ + { role: 'user', content: 'My dog is called Biscuit. Just say OK.' }, + { role: 'assistant', content: 'OK' }, + { + role: 'user', + content: 'What is my dog called? Answer with one word.', + }, + ], + }) + + expect(text).toMatch(/biscuit/i) + // A merged prompt has no role labels, which Nano should not echo + expect(text).not.toMatch(/^(user|model|assistant):/im) + }) + + it('stops at the output token limit', async () => { + const result = await android().doGenerate({ + prompt: prompt('Write a long story about a dragon.'), + maxOutputTokens: 8, + }) + + expect(result.finishReason.unified).toBe('length') + }) + + // Gemini Nano before v4, such as nano-v2 on a Pixel 9 Pro, has neither + it('streams reasoning in thinking mode, or warns without it', async () => { + const info = await android.getModelInfo() + + const { stream } = await android().doStream({ + prompt: prompt('Is 17 a prime number? Answer yes or no.'), + reasoning: 'high', + }) + const parts = await readAll(stream) + const types = parts.map((part) => part.type) + + expect(types).toContain('text-delta') + if (info.features?.includes('thinking')) { + expect(types).toContain('reasoning-delta') + } else { + expect(parts[0]).toMatchObject({ + type: 'stream-start', + warnings: [expect.objectContaining({ feature: 'reasoning' })], + }) + } + }) + + it('warns that tools are unsupported without function calling', async () => { + const info = await android.getModelInfo() + if (info.features?.includes('functionCalling')) { + return + } + + const { warnings } = await android().doGenerate({ + prompt: prompt('What is the weather in Paris?'), + tools: [ + { + type: 'function', + name: 'getWeather', + inputSchema: { type: 'object', properties: {} }, + }, + ], + }) + + expect(warnings).toContainEqual( + expect.objectContaining({ type: 'unsupported', feature: 'tools' }) + ) + }) + + it('warns that structured output is unsupported', async () => { + const { warnings } = await android().doGenerate({ + prompt: prompt('Describe a cat in one sentence.'), + responseFormat: { + type: 'json', + schema: { type: 'object', properties: { name: { type: 'string' } } }, + }, + topP: 0.5, + }) + + expect(warnings).toContainEqual( + expect.objectContaining({ + type: 'unsupported', + feature: 'responseFormat', + }) + ) + expect(warnings).toContainEqual( + expect.objectContaining({ type: 'unsupported', feature: 'topP' }) + ) + }) + + it('cancels a stream', async () => { + const controller = new AbortController() + const result = streamText({ + model: android(), + prompt: 'Write a long story about a dragon.', + abortSignal: controller.signal, + }) + + // The AI SDK ends the stream when it is aborted. Deltas already queued + // before the abort lands can still arrive + let deltas = 0 + for await (const delta of result.textStream) { + if (delta && ++deltas === 2) controller.abort() + } + expect(deltas).toBeGreaterThanOrEqual(2) + expect(deltas).toBeLessThan(10) + }) +}) diff --git a/harness/android/helpers.ts b/harness/android/helpers.ts new file mode 100644 index 00000000..fd99feb9 --- /dev/null +++ b/harness/android/helpers.ts @@ -0,0 +1,31 @@ +import type { + LanguageModelV4CallOptions, + LanguageModelV4StreamPart, +} from '@ai-sdk/provider' +import { createAndroidProvider } from '@react-native-ai/android' + +export function prompt(text: string): LanguageModelV4CallOptions['prompt'] { + return [{ role: 'user', content: [{ type: 'text', text }] }] +} + +export async function readAll( + stream: ReadableStream +) { + const parts: LanguageModelV4StreamPart[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) return parts + parts.push(value) + } +} + +// AICore rejects requests an app sends in quick succession with a quota error +export const cooldown = () => + new Promise((resolve) => setTimeout(resolve, 3000)) + +// Function calling needs Gemini Nano 4, which ML Kit serves as the preview +// release with the fast preference +export const nano4 = createAndroidProvider({ + model: { releaseStage: 'preview', preference: 'fast' }, +}) diff --git a/harness/android/tools.harness.ts b/harness/android/tools.harness.ts new file mode 100644 index 00000000..af364aae --- /dev/null +++ b/harness/android/tools.harness.ts @@ -0,0 +1,121 @@ +import { generateText, isStepCount, jsonSchema, streamText, tool } from 'ai' +import { + beforeAll, + beforeEach, + describe, + expect, + it, +} from 'react-native-harness' +import { z } from 'zod' + +import { cooldown, nano4 } from './helpers' + +function createTools() { + const calls: { name: string; input: unknown }[] = [] + const tools = { + getWeather: tool({ + description: 'Gets the current weather in a city.', + inputSchema: z.object({ + city: z.string().describe('The city name, such as London'), + }), + execute: async (input) => { + calls.push({ name: 'getWeather', input }) + return { city: input.city, condition: 'sunny', temperatureC: 24 } + }, + }), + getCurrentTime: tool({ + description: 'Gets the current local time.', + inputSchema: jsonSchema>({ + type: 'object', + properties: {}, + }), + execute: async (input) => { + calls.push({ name: 'getCurrentTime', input }) + return { time: '10:42' } + }, + }), + } + return { calls, tools } +} + +// Function calling needs Gemini Nano 4 through the AICore developer preview +describe('android tools', () => { + beforeAll(async () => { + await nano4().prepare() + }) + beforeEach(cooldown) + + it('calls a tool with the arguments the model chose', async () => { + const { calls, tools } = createTools() + + const result = await generateText({ + model: nano4(), + tools, + prompt: 'What is the weather in Paris? Use your tool.', + }) + + expect(calls).toEqual([ + { name: 'getWeather', input: { city: expect.stringMatching(/paris/i) } }, + ]) + expect(result.finishReason).toBe('tool-calls') + }) + + it('calls a tool without arguments', async () => { + const { calls, tools } = createTools() + + await generateText({ + model: nano4(), + tools, + prompt: 'What time is it right now? Use your tool.', + }) + + expect(calls).toEqual([{ name: 'getCurrentTime', input: {} }]) + }) + + it('answers with the tool result over several steps', async () => { + const { calls, tools } = createTools() + + const result = await generateText({ + model: nano4(), + tools, + stopWhen: isStepCount(3), + prompt: + 'What is the weather in Paris? Use your tool, then tell me the temperature.', + }) + + expect(calls.map((call) => call.name)).toContain('getWeather') + expect(result.steps.length).toBeGreaterThan(1) + expect(result.text).toMatch(/24/) + }) + + it('streams a tool call', async () => { + const { calls, tools } = createTools() + + const result = streamText({ + model: nano4(), + tools, + prompt: 'What is the weather in Tokyo? Use your tool.', + }) + + const types: string[] = [] + for await (const part of result.fullStream) { + types.push(part.type) + } + expect(types).toContain('tool-call') + expect(types).toContain('tool-result') + expect(calls[0]?.name).toBe('getWeather') + }) + + it('calls no tools for none tool choice', async () => { + const { calls, tools } = createTools() + + await generateText({ + model: nano4(), + tools, + toolChoice: 'none', + prompt: 'What is the weather in Paris?', + }) + + expect(calls).toEqual([]) + }) +}) diff --git a/harness/android/transcription.harness.ts b/harness/android/transcription.harness.ts new file mode 100644 index 00000000..035b2d56 --- /dev/null +++ b/harness/android/transcription.harness.ts @@ -0,0 +1,36 @@ +import { android } from '@react-native-ai/android' +import { experimental_transcribe as transcribe } from 'ai' +import { describe, expect, it } from 'react-native-harness' + +import { base64ToBytes, speech } from '../fixtures' + +const model = android.transcriptionModel({ language: 'en-US', mode: 'basic' }) + +describe('android transcription', () => { + it('reports recognizer availability', async () => { + const info = await model.getModelInfo() + console.log('[android] speech recognition basic:', JSON.stringify(info)) + console.log( + '[android] speech recognition advanced:', + JSON.stringify( + await android + .transcriptionModel({ language: 'en-US', mode: 'advanced' }) + .getModelInfo() + ) + ) + + expect(info.availability).not.toBe('unknown') + }) + + it('transcribes an M4A file', async () => { + await model.prepare({ onProgress: () => {} }) + + const { text } = await transcribe({ + model, + audio: base64ToBytes(speech), + }) + + expect(text).toMatch(/quick brown fox/i) + expect(text).toMatch(/paris/i) + }) +}) diff --git a/packages/android/README.md b/packages/android/README.md new file mode 100644 index 00000000..b7103756 --- /dev/null +++ b/packages/android/README.md @@ -0,0 +1,42 @@ +# Android Provider for Vercel AI SDK + +A Vercel AI SDK provider that runs Gemini Nano on device in React Native apps on Android, through Google's [ML Kit GenAI](https://developers.google.com/ml-kit/genai) libraries. + +**Requirements:** + +- Android only, with `minSdkVersion` 26+ and Kotlin 2.4+ set in your app +- A device that supports Gemini Nano through AICore (see [ML Kit GenAI device support](https://developers.google.com/ml-kit/genai#feature-device)) +- Vercel AI SDK v7 +- React Native New Architecture + +See [Getting Started](https://react-native-ai.com/docs/android/getting-started) for the Expo and bare React Native setup. + +```ts +import { android } from '@react-native-ai/android' +import { generateText } from 'ai' + +const { text } = await generateText({ + model: android(), + prompt: 'Summarize on-device AI in one sentence.', +}) +``` + +## Features + +- ✅ Gemini Nano text generation and streaming (`genai-prompt`) +- ✅ System instructions, conversations, image prompts and thinking mode +- ✅ Tool calling with AI SDK tools, on Gemini Nano 4 (`genai-prompt` function calling) +- ✅ Availability, model selection, download progress and token counting +- ✅ Transcription of audio files (`genai-speech-recognition`) +- ✅ Embeddings (`genai-embedding`) +- ✅ Error codes + +ML Kit only generates structured output for Kotlin classes compiled at build time, so JSON output returns an `unsupported` warning. + +## Documentation + +For complete installation instructions and API documentation, visit our [documentation site](https://react-native-ai.com/docs/android/getting-started). + +## License + +MIT diff --git a/packages/android/android/build.gradle b/packages/android/android/build.gradle new file mode 100644 index 00000000..9519c8aa --- /dev/null +++ b/packages/android/android/build.gradle @@ -0,0 +1,113 @@ +import com.android.Version + +buildscript { + def kotlin_version = rootProject.ext.has("kotlinVersion") ? rootProject.ext.get("kotlinVersion") : project.properties["AndroidAI_kotlinVersion"] + + repositories { + google() + mavenCentral() + } + + dependencies { + classpath "com.android.tools.build:gradle:7.2.1" + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + } +} + +apply plugin: "com.android.library" +apply plugin: "kotlin-android" +apply plugin: "com.facebook.react" + +def getExtOrDefault(name) { + return rootProject.ext.has(name) ? rootProject.ext.get(name) : project.properties["AndroidAI_" + name] +} + +def getExtOrIntegerDefault(name) { + return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["AndroidAI_" + name]).toInteger() +} + +static def supportsNamespace() { + def parsed = Version.ANDROID_GRADLE_PLUGIN_VERSION.tokenize('.') + def major = parsed[0].toInteger() + def minor = parsed[1].toInteger() + + return (major == 7 && minor >= 3) || major >= 8 +} + +android { + if (supportsNamespace()) { + namespace "com.callstack.ai.android" + + sourceSets { + main { + manifest.srcFile "src/main/AndroidManifest.xml" + } + } + } + + compileSdkVersion getExtOrIntegerDefault("compileSdkVersion") + + defaultConfig { + minSdkVersion getExtOrIntegerDefault("minSdkVersion") + targetSdkVersion getExtOrIntegerDefault("targetSdkVersion") + } + + buildFeatures { + buildConfig true + } + + buildTypes { + release { + minifyEnabled false + } + } + + lintOptions { + disable "GradleCompatible" + } + + compileOptions { + sourceCompatibility JavaVersion.VERSION_17 + targetCompatibility JavaVersion.VERSION_17 + } + + sourceSets { + main { + java.srcDirs += [ + "src", + "generated/java", + "generated/jni" + ] + } + } +} + +repositories { + mavenCentral() + google() +} + +def kotlin_version = getExtOrDefault("kotlinVersion") + +dependencies { + implementation "com.facebook.react:react-native:+" + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.11.0" + // genai-prompt beta5 and speech recognition alpha2 are built with Kotlin 2.4 + implementation "com.google.mlkit:genai-prompt:1.0.0-beta5" + implementation "com.google.mlkit:genai-schema:1.0.0-alpha2" + implementation "com.google.mlkit:genai-speech-recognition:1.0.0-alpha2" + implementation "com.google.mlkit:genai-embedding:1.0.0-alpha1" +} + +react { + jsRootDir = file("../src/") + libraryName = "AndroidAI" + codegenJavaPackageName = "com.callstack.ai.android" +} + +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach { + kotlinOptions { + jvmTarget = "17" + } +} diff --git a/packages/android/android/gradle.properties b/packages/android/android/gradle.properties new file mode 100644 index 00000000..0024d3b6 --- /dev/null +++ b/packages/android/android/gradle.properties @@ -0,0 +1,4 @@ +AndroidAI_kotlinVersion=2.4.0 +AndroidAI_minSdkVersion=26 +AndroidAI_targetSdkVersion=36 +AndroidAI_compileSdkVersion=36 diff --git a/packages/android/android/src/main/AndroidManifest.xml b/packages/android/android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..d5ee5eee --- /dev/null +++ b/packages/android/android/src/main/AndroidManifest.xml @@ -0,0 +1,7 @@ + + + + + + diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIErrors.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIErrors.kt new file mode 100644 index 00000000..0e6199b5 --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIErrors.kt @@ -0,0 +1,54 @@ +package com.callstack.ai.android + +import com.google.mlkit.genai.common.GenAiException +import com.google.mlkit.genai.common.GenAiException.ErrorCode +import org.json.JSONException + +/** Error codes shared with `AndroidAIErrorCodes` in `src/errors.ts`. */ +object AndroidAIErrors { + const val MODEL_UNAVAILABLE = "MODEL_UNAVAILABLE" + const val MODEL_DOWNLOADING = "MODEL_DOWNLOADING" + const val DOWNLOAD_FAILED = "DOWNLOAD_FAILED" + const val FEATURE_NOT_SUPPORTED = "FEATURE_NOT_SUPPORTED" + const val GENERATION_ERROR = "GENERATION_ERROR" + const val INVALID_MESSAGE = "INVALID_MESSAGE" + const val INVALID_SCHEMA = "INVALID_SCHEMA" + const val CONTEXT_WINDOW_EXCEEDED = "CONTEXT_WINDOW_EXCEEDED" + const val RATE_LIMITED = "RATE_LIMITED" + const val BACKGROUND_USE_BLOCKED = "BACKGROUND_USE_BLOCKED" + const val MAX_TURNS_EXCEEDED = "MAX_TURNS_EXCEEDED" + + /** Exception that already knows its error code. */ + class CodedException(val code: String, message: String, cause: Throwable? = null) : + Exception(message, cause) + + fun codeFor(error: Throwable): String { + var current: Throwable? = error + while (current != null) { + when (current) { + is CodedException -> return current.code + is GenAiException -> return codeFor(current) + is JSONException, is IllegalArgumentException -> return INVALID_MESSAGE + } + current = current.cause + } + return GENERATION_ERROR + } + + private fun codeFor(error: GenAiException): String = + when (error.errorCode) { + ErrorCode.NOT_AVAILABLE, + ErrorCode.AICORE_INCOMPATIBLE, + ErrorCode.NEEDS_SYSTEM_UPDATE, + ErrorCode.NOT_ENOUGH_DISK_SPACE -> MODEL_UNAVAILABLE + ErrorCode.NOT_SUPPORTED -> FEATURE_NOT_SUPPORTED + ErrorCode.BUSY, + ErrorCode.PER_APP_BATTERY_USE_QUOTA_EXCEEDED -> RATE_LIMITED + ErrorCode.BACKGROUND_USE_BLOCKED -> BACKGROUND_USE_BLOCKED + ErrorCode.REQUEST_TOO_LARGE -> CONTEXT_WINDOW_EXCEEDED + ErrorCode.MAX_TURNS_EXCEEDED -> MAX_TURNS_EXCEEDED + ErrorCode.INVALID_INPUT_IMAGE, + ErrorCode.REQUEST_TOO_SMALL -> INVALID_MESSAGE + else -> GENERATION_ERROR + } +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt new file mode 100644 index 00000000..8de499dd --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIModule.kt @@ -0,0 +1,215 @@ +package com.callstack.ai.android + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import com.facebook.react.bridge.WritableMap +import com.google.mlkit.genai.common.FunctionCall +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentHashMap + +class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISpec(reactContext) { + + override fun getName(): String = NAME + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val downloads = Downloads { model, phase, bytesDownloaded, bytesToDownload -> + emitOnDownloadProgress( + Arguments.createMap().apply { + putString("model", model) + putString("phase", phase) + bytesDownloaded?.let { putDouble("bytesDownloaded", it.toDouble()) } + bytesToDownload?.let { putDouble("bytesToDownload", it.toDouble()) } + }, + ) + } + private val nano = GeminiNano(reactContext, downloads) + private val transcription = Transcription(reactContext, downloads) + private val embeddings = Embeddings(downloads) + + // Generations and transcriptions, by id, so they can be cancelled + private val activeRuns = ConcurrentHashMap() + + private fun Promise.rejectWithCode(error: Exception) { + reject(AndroidAIErrors.codeFor(error), error.message ?: "Unknown ML Kit error", error) + } + + private fun invalidated() = IllegalStateException("The AndroidAI module was invalidated") + + /** Resolves [promise] with what [block] returns, or rejects it with an error code. */ + private fun launch(promise: Promise, block: suspend () -> Any?) { + // A cancelled scope never runs the block, which would leave the promise pending + if (!scope.isActive) return promise.rejectWithCode(invalidated()) + scope.launch { + try { + promise.resolve(block().takeUnless { it is Unit }) + } catch (error: Exception) { + promise.rejectWithCode(error) + } + } + } + + /** Like [launch], tracked by [runId] so [cancel] can stop it. */ + private fun launchRun(runId: String, onError: (Exception) -> Unit, block: suspend () -> Unit) { + if (!scope.isActive) return onError(invalidated()) + // Started after it is tracked, so a run that ends at once is still removed + val job = scope.launch(start = CoroutineStart.LAZY) { + try { + block() + } catch (error: Exception) { + onError(error) + } finally { + activeRuns.remove(runId) + } + } + activeRuns[runId] = job + job.start() + } + + private fun ModelInfo.toWritableMap(): WritableMap = + Arguments.createMap().apply { + putString("availability", availability) + baseModelName?.let { putString("baseModelName", it) } + contextSize?.let { putInt("contextSize", it) } + features?.let { putArray("features", Arguments.fromList(it)) } + } + + private fun FunctionCall.toWritableMap(): WritableMap = + Arguments.createMap().apply { + putString("id", id) + putString("name", name) + putString("arguments", FunctionJson.argumentsToJson(arguments).toString()) + } + + override fun isAvailable(model: ReadableMap?): Boolean = nano.isAvailable(NanoConfig.from(model)) + + override fun getModelInfo(model: ReadableMap?, promise: Promise) = + launch(promise) { nano.getModelInfo(NanoConfig.from(model)).toWritableMap() } + + override fun prepare(model: ReadableMap?, promise: Promise) = + launch(promise) { nano.prepare(NanoConfig.from(model)) } + + override fun countTokens(text: String, model: ReadableMap?, promise: Promise) = + launch(promise) { nano.countTokens(text, NanoConfig.from(model)).toDouble() } + + override fun generateText(runId: String, request: ReadableMap, promise: Promise) = + launchRun(runId, { promise.rejectWithCode(it) }) { + val result = nano.generate(request) + promise.resolve( + Arguments.createMap().apply { + putString("text", result.text) + result.reasoning?.let { putString("reasoning", it) } + putArray( + "functionCalls", + Arguments.createArray().apply { result.functionCalls.forEach { pushMap(it.toWritableMap()) } }, + ) + result.finishReason?.let { putString("finishReason", it) } + }, + ) + } + + override fun streamText(streamId: String, request: ReadableMap, promise: Promise) { + if (activeRuns.containsKey(streamId)) { + promise.reject(AndroidAIErrors.GENERATION_ERROR, "Stream ID already in use: $streamId") + return + } + + val listener = object : StreamListener { + override fun onText(delta: String) = emitStreamEvent(streamId, "text") { putString("delta", delta) } + + override fun onReasoning(delta: String) = emitStreamEvent(streamId, "reasoning") { putString("delta", delta) } + + override fun onFunctionCall(call: FunctionCall) = + emitStreamEvent(streamId, "functionCall") { putMap("functionCall", call.toWritableMap()) } + } + + launchRun( + streamId, + onError = { error -> + emitStreamEvent(streamId, "error") { + putString("error", error.message ?: "Unknown ML Kit stream error") + putString("code", AndroidAIErrors.codeFor(error)) + } + }, + ) { + val result = nano.stream(request, listener) + emitStreamEvent(streamId, "finish") { result.finishReason?.let { putString("finishReason", it) } } + } + promise.resolve(null) + } + + private fun emitStreamEvent(streamId: String, type: String, fill: WritableMap.() -> Unit) { + emitOnStreamEvent( + Arguments.createMap().apply { + putString("streamId", streamId) + putString("type", type) + fill() + }, + ) + } + + override fun cancel(runId: String) { + activeRuns.remove(runId)?.cancel() + } + + override fun isTranscriptionAvailable(options: ReadableMap): Boolean = + transcription.isAvailable(TranscriptionConfig.from(options)) + + override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) = + launch(promise) { transcription.getModelInfo(TranscriptionConfig.from(options)).toWritableMap() } + + override fun prepareTranscription(options: ReadableMap, promise: Promise) = + launch(promise) { transcription.prepare(TranscriptionConfig.from(options)) } + + override fun transcribe(runId: String, audio: String, options: ReadableMap, promise: Promise) = + launchRun(runId, { promise.rejectWithCode(it) }) { + promise.resolve(transcription.transcribe(audio, TranscriptionConfig.from(options))) + } + + override fun getEmbeddingModelInfo(promise: Promise) = + launch(promise) { + val info = embeddings.getModelInfo() + Arguments.createMap().apply { + putString("availability", info.availability) + info.dimension?.let { putInt("dimension", it) } + info.modelSignature?.let { putString("modelSignature", it) } + } + } + + override fun prepareEmbedding(promise: Promise) = launch(promise) { embeddings.prepare(); null } + + override fun embed(values: ReadableArray, taskType: String?, promise: Promise) = + launch(promise) { + val strings = List(values.size()) { index -> + values.takeIf { it.getType(index) == ReadableType.String }?.getString(index) + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Every value to embed must be a string") + } + val vectors = embeddings.embed(strings, taskType) + Arguments.createArray().apply { + vectors.forEach { vector -> + pushArray(Arguments.createArray().apply { vector.forEach { pushDouble(it.toDouble()) } }) + } + } + } + + override fun invalidate() { + activeRuns.values.forEach { it.cancel() } + activeRuns.clear() + scope.cancel() + super.invalidate() + } + + companion object { + const val NAME = "AndroidAI" + } +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIPackage.kt b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIPackage.kt new file mode 100644 index 00000000..b2de91d2 --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/AndroidAIPackage.kt @@ -0,0 +1,33 @@ +package com.callstack.ai.android + +import com.facebook.react.BaseReactPackage +import com.facebook.react.bridge.NativeModule +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.module.model.ReactModuleInfo +import com.facebook.react.module.model.ReactModuleInfoProvider +import java.util.HashMap + +class AndroidAIPackage : BaseReactPackage() { + override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? = + if (name == AndroidAIModule.NAME) { + AndroidAIModule(reactContext) + } else { + null + } + + override fun getReactModuleInfoProvider(): ReactModuleInfoProvider = + ReactModuleInfoProvider { + val moduleInfos: MutableMap = HashMap() + moduleInfos[AndroidAIModule.NAME] = ReactModuleInfo( + AndroidAIModule.NAME, + AndroidAIModule.NAME, + canOverrideExistingModule = false, + needsEagerInit = false, + hasConstants = false, + isCxxModule = false, + isTurboModule = true + ) + + moduleInfos + } +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt new file mode 100644 index 00000000..18936b56 --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Downloads.kt @@ -0,0 +1,40 @@ +package com.callstack.ai.android + +import com.google.mlkit.genai.common.DownloadStatus +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.GenAiException +import kotlinx.coroutines.flow.Flow + +/** Collects ML Kit model downloads and reports their progress. */ +class Downloads( + private val onProgress: (model: String, phase: String, bytesDownloaded: Long?, bytesToDownload: Long?) -> Unit, +) { + suspend fun collect(model: String, displayName: String, status: Int, download: Flow) { + try { + download.collect { downloadStatus -> + when (downloadStatus) { + is DownloadStatus.DownloadStarted -> + onProgress(model, "started", 0, downloadStatus.bytesToDownload) + is DownloadStatus.DownloadProgress -> + onProgress(model, "progress", downloadStatus.totalBytesDownloaded, null) + is DownloadStatus.DownloadCompleted -> onProgress(model, "completed", null, null) + is DownloadStatus.DownloadFailed -> + throw AndroidAIErrors.CodedException( + AndroidAIErrors.DOWNLOAD_FAILED, + "$displayName download failed: ${downloadStatus.e.message}", + downloadStatus.e, + ) + } + } + } catch (error: GenAiException) { + if (status == FeatureStatus.DOWNLOADING) { + throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_DOWNLOADING, + "$displayName is still downloading", + error, + ) + } + throw error + } + } +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Embeddings.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Embeddings.kt new file mode 100644 index 00000000..e24bd2b8 --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Embeddings.kt @@ -0,0 +1,95 @@ +package com.callstack.ai.android + +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.embedding.Content +import com.google.mlkit.genai.embedding.EmbeddingGeneration +import com.google.mlkit.genai.embedding.EmbeddingModel +import com.google.mlkit.genai.embedding.EmbeddingOptions +import com.google.mlkit.genai.embedding.EmbeddingRequest +import com.google.mlkit.genai.embedding.TaskConfig +import com.google.mlkit.genai.embedding.TextPart +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +data class EmbeddingModelInfo( + val availability: String, + val dimension: Int? = null, + val modelSignature: String? = null, +) + +/** Embeds text with ML Kit's on-device embedding model. */ +class Embeddings(private val downloads: Downloads) { + private val mutex = Mutex() + private val prepareMutex = Mutex() + private var model: EmbeddingModel? = null + private var prepared = false + + /** The device's default embedding model, or null when it has none. */ + private suspend fun model(): EmbeddingModel? = + mutex.withLock { + model ?: EmbeddingGeneration.getEmbeddingModels(EmbeddingOptions()).firstOrNull()?.also { model = it } + } + + suspend fun getModelInfo(): EmbeddingModelInfo { + val model = model() ?: return EmbeddingModelInfo(availability = "unavailable") + val status = model.checkStatus() + if (status != FeatureStatus.AVAILABLE) { + return EmbeddingModelInfo(availability = availabilityOf(status)) + } + return EmbeddingModelInfo( + availability = "available", + dimension = runCatching { model.getMetadata()?.dimension }.getOrNull(), + modelSignature = runCatching { model.getModelSignature() }.getOrNull(), + ) + } + + /** Downloads the embedding model when needed, reporting progress, then warms it up. */ + suspend fun prepare(): EmbeddingModel { + val model = model() + ?: throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_UNAVAILABLE, + "This device has no on-device embedding model", + ) + prepareMutex.withLock { + if (prepared) return model + + when (val status = model.checkStatus()) { + FeatureStatus.UNAVAILABLE -> + throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_UNAVAILABLE, + "The embedding model is not available on this device", + ) + FeatureStatus.DOWNLOADABLE, FeatureStatus.DOWNLOADING -> + downloads.collect("embedding", "The embedding model", status, model.download()) + } + model.warmup() + prepared = true + return model + } + } + + suspend fun embed(values: List, taskType: String?): List { + val model = prepare() + val taskConfig = taskConfig(taskType) + return values.map { value -> + val content = Content(listOf(TextPart(value))) + val request = taskConfig?.let { EmbeddingRequest(content, it) } ?: EmbeddingRequest(content) + model.generateEmbedding(request).embedding.vector + } + } + + private fun taskConfig(taskType: String?): TaskConfig? = + when (taskType) { + null -> null + "retrievalQuery" -> TaskConfig.RetrievalQuery() + "retrievalDocument" -> TaskConfig.RetrievalDocument() + "semanticSimilarity" -> TaskConfig.SemanticSimilarity() + "classification" -> TaskConfig.Classification() + "clustering" -> TaskConfig.Clustering() + "questionAnswering" -> TaskConfig.QuestionAnswering() + "factVerification" -> TaskConfig.FactVerification() + "codeRetrieval" -> TaskConfig.CodeRetrieval() + else -> + throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown embedding task type: $taskType") + } +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/FunctionJson.kt b/packages/android/android/src/main/java/com/callstack/ai/android/FunctionJson.kt new file mode 100644 index 00000000..f1af9589 --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/FunctionJson.kt @@ -0,0 +1,125 @@ +package com.callstack.ai.android + +import com.google.mlkit.genai.common.FunctionArgValue +import com.google.mlkit.genai.schema.tools.FunctionDeclaration +import com.google.mlkit.genai.schema.tools.FunctionSchema +import com.google.mlkit.genai.schema.tools.FunctionType +import org.json.JSONArray +import org.json.JSONObject + +/** + * Converts between JSON and ML Kit's function calling types: JSON Schema to [FunctionSchema], + * and JSON values to and from [FunctionArgValue]. + */ +object FunctionJson { + + fun declaration(name: String, description: String, parameters: String): FunctionDeclaration = + try { + FunctionDeclaration(name, description, schema(JSONObject(parameters))) + } catch (error: Exception) { + throw AndroidAIErrors.CodedException( + AndroidAIErrors.INVALID_SCHEMA, + "Cannot convert the parameters of $name: ${error.message}", + error, + ) + } + + /** Maps the JSON Schema keywords that [FunctionSchema] has. Others are left out. */ + fun schema(json: JSONObject): FunctionSchema { + // A type such as ["string", "null"] is a nullable string + val types = when (val type = json.opt("type")) { + is JSONArray -> List(type.length()) { type.getString(it) } + is String -> listOf(type) + else -> emptyList() + } + val type = types.firstOrNull { it != "null" } ?: types.firstOrNull() + + return FunctionSchema( + type = functionType(type), + description = json.optStringOrNull("description"), + properties = json.optJSONObject("properties")?.let { properties -> + properties.keys().asSequence().associateWith { schema(properties.getJSONObject(it)) } + }, + required = json.optJSONArray("required")?.strings(), + enum = json.optJSONArray("enum")?.let { values -> List(values.length()) { values.get(it).toString() } }, + format = json.optStringOrNull("format"), + nullable = if (types.size > 1 && "null" in types) true else null, + items = json.optJSONObject("items")?.let(::schema), + minItems = json.optLongOrNull("minItems"), + maxItems = json.optLongOrNull("maxItems"), + minimum = json.optDoubleOrNull("minimum"), + maximum = json.optDoubleOrNull("maximum"), + minLength = json.optLongOrNull("minLength"), + maxLength = json.optLongOrNull("maxLength"), + pattern = json.optStringOrNull("pattern"), + anyOf = json.optJSONArray("anyOf")?.let { schemas -> List(schemas.length()) { schema(schemas.getJSONObject(it)) } }, + title = json.optStringOrNull("title"), + minProperties = json.optLongOrNull("minProperties"), + maxProperties = json.optLongOrNull("maxProperties"), + ) + } + + private fun functionType(type: String?): Int = + when (type) { + "string" -> FunctionType.STRING + "number" -> FunctionType.NUMBER + "integer" -> FunctionType.INTEGER + "boolean" -> FunctionType.BOOLEAN + "array" -> FunctionType.ARRAY + "object" -> FunctionType.OBJECT + "null" -> FunctionType.NULL + else -> FunctionType.UNSPECIFIED + } + + fun argumentsToJson(arguments: Map): JSONObject = + JSONObject().apply { arguments.forEach { (key, value) -> put(key, toJson(value)) } } + + private fun toJson(value: FunctionArgValue): Any = + when (value) { + is FunctionArgValue.StringValue -> value.value + is FunctionArgValue.IntValue -> value.value + is FunctionArgValue.LongValue -> value.value + is FunctionArgValue.DoubleValue -> value.value + is FunctionArgValue.BoolValue -> value.value + is FunctionArgValue.ListValue -> JSONArray().apply { value.values.forEach { put(toJson(it)) } } + is FunctionArgValue.MapValue -> argumentsToJson(value.map) + } + + fun argumentsFromJson(json: JSONObject): Map = + json.keys().asSequence().mapNotNull { key -> fromJson(json.get(key))?.let { key to it } }.toMap() + + private fun fromJson(value: Any?): FunctionArgValue? = + when (value) { + is String -> FunctionArgValue.StringValue(value) + is Boolean -> FunctionArgValue.BoolValue(value) + is Int -> FunctionArgValue.IntValue(value) + is Long -> FunctionArgValue.LongValue(value) + is Number -> FunctionArgValue.DoubleValue(value.toDouble()) + is JSONArray -> FunctionArgValue.ListValue(List(value.length()) { value.get(it) }.mapNotNull(::fromJson)) + is JSONObject -> FunctionArgValue.MapValue(argumentsFromJson(value)) + else -> null + } + + /** A function response for [com.google.mlkit.genai.prompt.FunctionResponsePart]. */ + fun toMap(json: JSONObject): Map = + json.keys().asSequence().associateWith { toPlain(json.get(it)) } + + private fun toPlain(value: Any?): Any? = + when (value) { + is JSONObject -> toMap(value) + is JSONArray -> List(value.length()) { toPlain(value.get(it)) } + JSONObject.NULL -> null + else -> value + } + + private fun JSONArray.strings(): List = List(length()) { getString(it) } + + private fun JSONObject.optStringOrNull(key: String): String? = + if (has(key) && !isNull(key)) getString(key) else null + + private fun JSONObject.optLongOrNull(key: String): Long? = + if (has(key) && !isNull(key)) getLong(key) else null + + private fun JSONObject.optDoubleOrNull(key: String): Double? = + if (has(key) && !isNull(key)) getDouble(key) else null +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/GeminiNano.kt b/packages/android/android/src/main/java/com/callstack/ai/android/GeminiNano.kt new file mode 100644 index 00000000..08424c10 --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/GeminiNano.kt @@ -0,0 +1,351 @@ +package com.callstack.ai.android + +import android.content.Context +import android.content.pm.PackageManager +import android.net.Uri +import android.util.Base64 +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.FunctionCall +import com.google.mlkit.genai.common.StreamingCallback +import com.google.mlkit.genai.prompt.Candidate +import com.google.mlkit.genai.prompt.Content +import com.google.mlkit.genai.prompt.Feature +import com.google.mlkit.genai.prompt.FunctionCallPart +import com.google.mlkit.genai.prompt.FunctionResponsePart +import com.google.mlkit.genai.prompt.GenerateContentRequest +import com.google.mlkit.genai.prompt.GenerateContentResponse +import com.google.mlkit.genai.prompt.Generation +import com.google.mlkit.genai.prompt.GenerativeModel +import com.google.mlkit.genai.prompt.ImagePart +import com.google.mlkit.genai.prompt.ModelPreference +import com.google.mlkit.genai.prompt.ModelReleaseStage +import com.google.mlkit.genai.prompt.SystemInstruction +import com.google.mlkit.genai.prompt.TextPart +import com.google.mlkit.genai.prompt.generateContentRequest +import com.google.mlkit.genai.prompt.generationConfig +import com.google.mlkit.genai.prompt.modelConfig +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONObject +import java.util.concurrent.ConcurrentHashMap + +private const val AICORE_PACKAGE = "com.google.android.aicore" + +fun isAICoreInstalled(context: Context): Boolean = + try { + context.packageManager.getPackageInfo(AICORE_PACKAGE, 0) + true + } catch (_: PackageManager.NameNotFoundException) { + false + } + +/** Gemini Nano model selection, see ML Kit `ModelConfig`. */ +data class NanoConfig( + val releaseStage: String? = null, + val preference: String? = null, +) { + companion object { + fun from(map: ReadableMap?): NanoConfig = + NanoConfig( + releaseStage = map?.optString("releaseStage"), + preference = map?.optString("preference"), + ) + } +} + +data class ModelInfo( + val availability: String, + val baseModelName: String? = null, + val contextSize: Int? = null, + val features: List? = null, +) + +data class GenerateResult( + val text: String, + val reasoning: String?, + val functionCalls: List, + val finishReason: String?, +) + +/** Receives a streamed response as it is generated. */ +interface StreamListener { + fun onText(delta: String) + fun onReasoning(delta: String) + fun onFunctionCall(call: FunctionCall) +} + +/** Whether [key] holds a value, since the bridge can send a key with an explicit null. */ +fun ReadableMap.isSet(key: String): Boolean = hasKey(key) && !isNull(key) + +fun ReadableMap.optString(key: String): String? = if (isSet(key)) getString(key) else null + +fun availabilityOf(status: Int): String = + when (status) { + FeatureStatus.UNAVAILABLE -> "unavailable" + FeatureStatus.DOWNLOADABLE -> "downloadable" + FeatureStatus.DOWNLOADING -> "downloading" + FeatureStatus.AVAILABLE -> "available" + else -> "unknown" + } + +/** Runs Gemini Nano through ML Kit's Prompt API. */ +class GeminiNano( + private val context: Context, + private val downloads: Downloads, +) { + private val clients = ConcurrentHashMap() + private val prepared = ConcurrentHashMap.newKeySet() + // One lock per model selection, so one download does not hold up another selection + private val prepareMutexes = ConcurrentHashMap() + // The last status seen per model selection, for the synchronous check + private val statuses = ConcurrentHashMap() + private val systemPromptSupport = ConcurrentHashMap() + + private fun client(config: NanoConfig): GenerativeModel = + clients.computeIfAbsent(config) { + if (config.releaseStage == null && config.preference == null) { + Generation.getClient() + } else { + Generation.getClient( + generationConfig { + modelConfig = modelConfig { + config.releaseStage?.let { + releaseStage = if (it == "preview") ModelReleaseStage.PREVIEW else ModelReleaseStage.STABLE + } + config.preference?.let { + preference = if (it == "fast") ModelPreference.FAST else ModelPreference.FULL + } + } + }, + ) + } + } + + /** + * Whether Gemini Nano is available, downloadable or downloading, answered without async work: from the last + * status this process saw, or else from whether AICore is installed. + */ + fun isAvailable(config: NanoConfig): Boolean { + statuses[config]?.let { + return it == FeatureStatus.AVAILABLE || it == FeatureStatus.DOWNLOADABLE || it == FeatureStatus.DOWNLOADING + } + return isAICoreInstalled(context) + } + + suspend fun getModelInfo(config: NanoConfig): ModelInfo { + val model = client(config) + val status = + try { + model.checkStatus() + } catch (_: Exception) { + return ModelInfo(availability = "unknown") + } + statuses[config] = status + if (status != FeatureStatus.AVAILABLE) { + return ModelInfo(availability = availabilityOf(status)) + } + + return ModelInfo( + availability = "available", + baseModelName = runCatching { model.getBaseModelName() }.getOrNull(), + contextSize = runCatching { model.getTokenLimit() }.getOrNull(), + features = runCatching { model.getAvailableFeatures().mapNotNull(::featureName) }.getOrNull(), + ) + } + + private fun featureName(feature: Int): String? = + when (feature) { + Feature.SYSTEM_PROMPT -> "systemPrompt" + Feature.THINKING_MODE -> "thinking" + Feature.PREFIX_CACHING -> "prefixCaching" + Feature.STRUCTURED_OUTPUT -> "structuredOutput" + Feature.FUNCTION_CALLING -> "functionCalling" + else -> null + } + + /** Downloads Gemini Nano when needed, reporting progress, then warms it up. */ + suspend fun prepare(config: NanoConfig) { + if (config in prepared) return + prepareMutexes.computeIfAbsent(config) { Mutex() }.withLock { + if (config in prepared) return + + val model = client(config) + val status = model.checkStatus() + statuses[config] = status + when (status) { + FeatureStatus.UNAVAILABLE -> + throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_UNAVAILABLE, + "Gemini Nano is not supported on this device or the device has not fetched the latest configuration to support it", + ) + FeatureStatus.DOWNLOADABLE, FeatureStatus.DOWNLOADING -> + downloads.collect("language", "Gemini Nano", status, model.download()) + } + model.warmup() + prepared.add(config) + statuses[config] = FeatureStatus.AVAILABLE + } + } + + private suspend fun preparedClient(config: NanoConfig): GenerativeModel { + prepare(config) + return client(config) + } + + suspend fun countTokens(text: String, config: NanoConfig): Int = + preparedClient(config).countTokens(generateContentRequest(TextPart(text)) {}).totalTokens + + suspend fun generate(request: ReadableMap): GenerateResult { + val config = NanoConfig.from(request.getMap("model")) + val model = preparedClient(config) + val response = model.generateContent(buildRequest(request, model, config)) + return result(response, response.candidates.firstOrNull()?.functionCalls.orEmpty()) + } + + suspend fun stream(request: ReadableMap, listener: StreamListener): GenerateResult { + val config = NanoConfig.from(request.getMap("model")) + val streamedCalls = mutableListOf() + val callback = object : StreamingCallback { + override fun onNewText(text: String) = listener.onText(text) + + override fun onNewThought(thought: String) = listener.onReasoning(thought) + + override fun onNewFunctionCall(functionCall: FunctionCall) { + streamedCalls.add(functionCall) + listener.onFunctionCall(functionCall) + } + } + val model = preparedClient(config) + val response = model.generateContent(buildRequest(request, model, config), callback) + // Calls arrive through the callback or with the final response + val finalCalls = response.candidates.firstOrNull()?.functionCalls.orEmpty() + .filter { call -> streamedCalls.none { it.id == call.id && it.name == call.name } } + finalCalls.forEach(listener::onFunctionCall) + return result(response, streamedCalls + finalCalls) + } + + private fun result(response: GenerateContentResponse, functionCalls: List): GenerateResult { + val candidate = response.candidates.firstOrNull() + return GenerateResult( + text = candidate?.text.orEmpty(), + reasoning = response.thoughtProcess.joinToString("") { it.text.orEmpty() }.takeIf { it.isNotEmpty() }, + functionCalls = functionCalls, + finishReason = candidate?.finishReason?.let(::finishReasonName), + ) + } + + private fun finishReasonName(reason: Int): String = + when (reason) { + Candidate.FinishReason.STOP -> "STOP" + Candidate.FinishReason.MAX_TOKENS -> "MAX_TOKENS" + else -> "OTHER" + } + + private suspend fun supportsSystemPrompt(model: GenerativeModel, config: NanoConfig): Boolean = + systemPromptSupport[config] + // A failed check is not cached, so a transient AICore error doesn't stick for the session + ?: runCatching { model.isSystemPromptAvailable() } + .onSuccess { systemPromptSupport[config] = it } + .getOrDefault(false) + + private suspend fun buildRequest( + request: ReadableMap, + model: GenerativeModel, + config: NanoConfig, + ): GenerateContentRequest { + var contents = request.getArray("messages")?.let(::contents).orEmpty() + if (contents.isEmpty()) { + throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "The prompt has no messages") + } + var systemInstruction = request.optString("systemInstruction") + + // Gemini Nano without system prompt support, such as nano-v2, reads only the first text part + // of a request, so it would answer the system instruction or the first turn and never see the + // rest. Like ADK's NanoSingleTextPrompt, the system instruction and every turn's text become one + // part, after the images. Turns are labeled by role when there are several, so the model can + // tell its own earlier answers apart from the user's. Remove once ML Kit handles this (issue #364). + val parts = contents.flatMap { it.parts } + if ((parts.size > 1 || systemInstruction != null) && !supportsSystemPrompt(model, config)) { + val turns = contents.mapNotNull { content -> + content.parts.filterIsInstance().joinToString("\n\n") { it.textString } + .takeIf { it.isNotEmpty() } + ?.let { text -> + val label = if (content.role == Content.Role.MODEL) "Model" else "User" + if (contents.size > 1) "$label: $text" else text + } + } + val text = (listOfNotNull(systemInstruction) + turns).joinToString("\n\n") + val images = parts.filterIsInstance() + contents = listOf( + Content.builder() + .role(Content.Role.USER) + .apply { (images + TextPart(text)).forEach(::addPart) } + .build(), + ) + systemInstruction = null + } + + val tools = request.getArray("tools")?.let { tools -> + List(tools.size()) { index -> + val tool = tools.getMap(index)!! + FunctionJson.declaration( + tool.getString("name")!!, + tool.getString("description").orEmpty(), + tool.getString("parameters")!!, + ) + } + } + + return GenerateContentRequest.Builder(contents) + .apply { + systemInstruction?.let { this.systemInstruction = SystemInstruction(it) } + if (request.isSet("temperature")) temperature = request.getDouble("temperature").toFloat() + if (request.isSet("topK")) topK = request.getInt("topK") + if (request.isSet("seed")) seed = request.getInt("seed") + if (request.isSet("maxOutputTokens")) maxOutputTokens = request.getInt("maxOutputTokens") + if (request.isSet("enableThinking")) enableThinking = request.getBoolean("enableThinking") + if (!tools.isNullOrEmpty()) setTools(tools) + } + .build() + } + + private fun contents(messages: ReadableArray): List = + List(messages.size()) { index -> + val message = messages.getMap(index)!! + val parts = message.getArray("parts")!! + Content.builder() + .role(if (message.getString("role") == "model") Content.Role.MODEL else Content.Role.USER) + .apply { for (part in 0 until parts.size()) addPart(part(parts.getMap(part)!!)) } + .build() + } + + // ML Kit's ImagePart(Uri) refuses files in the app's own data directory, such as its cache, so + // the app reads the URI it was given, as ADK did + private fun readUri(uri: String): ByteArray = + context.contentResolver.openInputStream(Uri.parse(uri))?.use { it.readBytes() } + ?: throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Cannot read image: $uri") + + private fun part(part: ReadableMap) = + when (val type = part.getString("type")) { + "text" -> TextPart(part.getString("text").orEmpty()) + "image" -> + ImagePart(part.optString("uri")?.let(::readUri) ?: Base64.decode(part.getString("data"), Base64.DEFAULT)) + "functionCall" -> + FunctionCallPart( + FunctionCall( + part.getString("name")!!, + FunctionJson.argumentsFromJson(JSONObject(part.optString("json") ?: "{}")), + part.optString("id").orEmpty(), + ), + ) + "functionResponse" -> + FunctionResponsePart( + part.optString("id").orEmpty(), + part.getString("name")!!, + FunctionJson.toMap(JSONObject(part.optString("json") ?: "{}")), + ) + else -> throw AndroidAIErrors.CodedException(AndroidAIErrors.INVALID_MESSAGE, "Unknown message part: $type") + } +} diff --git a/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt new file mode 100644 index 00000000..230c6f4e --- /dev/null +++ b/packages/android/android/src/main/java/com/callstack/ai/android/Transcription.kt @@ -0,0 +1,127 @@ +package com.callstack.ai.android + +import android.content.Context +import android.os.Build +import android.util.Base64 +import com.facebook.react.bridge.ReadableMap +import com.google.mlkit.genai.common.FeatureStatus +import com.google.mlkit.genai.common.audio.AudioSource +import com.google.mlkit.genai.speechrecognition.SpeechRecognition +import com.google.mlkit.genai.speechrecognition.SpeechRecognizer +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse +import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions +import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.io.File +import java.util.Locale +import java.util.concurrent.ConcurrentHashMap + +data class TranscriptionConfig(val locale: String?, val mode: String?) { + companion object { + fun from(map: ReadableMap?): TranscriptionConfig = + TranscriptionConfig(map?.optString("locale"), map?.optString("mode")) + } +} + +/** Transcribes audio files with ML Kit's speech recognition. */ +class Transcription( + private val context: Context, + private val downloads: Downloads, +) { + private val clients = ConcurrentHashMap() + private val prepared = ConcurrentHashMap.newKeySet() + // The last status seen per language and mode, for the synchronous check + private val statuses = ConcurrentHashMap() + // One lock per language and mode, so one download does not hold up another + private val prepareMutexes = ConcurrentHashMap() + + private fun client(config: TranscriptionConfig): SpeechRecognizer = + clients.computeIfAbsent(config) { + SpeechRecognition.getClient( + speechRecognizerOptions { + locale = config.locale?.let(::locale) ?: Locale.getDefault() + when (config.mode) { + "basic" -> preferredMode = SpeechRecognizerOptions.Mode.MODE_BASIC + "advanced" -> preferredMode = SpeechRecognizerOptions.Mode.MODE_ADVANCED + } + }, + ) + } + + // forLanguageTag returns an undefined locale for a tag it can't parse, such as en_US + private fun locale(tag: String): Locale = + Locale.forLanguageTag(tag).takeIf { it.language.isNotEmpty() } + ?: throw AndroidAIErrors.CodedException( + AndroidAIErrors.INVALID_MESSAGE, + "Not a BCP 47 language tag: $tag. Use a tag such as en-US", + ) + + /** + * Whether the recognizer is available, downloadable or downloading, answered without async work: from the last + * status this process saw, or else from what the mode needs (Android 12 for basic, AICore for advanced). + */ + fun isAvailable(config: TranscriptionConfig): Boolean { + statuses[config]?.let { + return it == FeatureStatus.AVAILABLE || it == FeatureStatus.DOWNLOADABLE || it == FeatureStatus.DOWNLOADING + } + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && + (config.mode != "advanced" || isAICoreInstalled(context)) + } + + suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo { + val status = client(config).checkStatus() + statuses[config] = status + return ModelInfo(availability = availabilityOf(status)) + } + + /** Downloads the recognizer's model when needed, reporting progress, then warms it up. */ + suspend fun prepare(config: TranscriptionConfig) { + if (config in prepared) return + prepareMutexes.computeIfAbsent(config) { Mutex() }.withLock { + if (config in prepared) return + + val recognizer = client(config) + val status = recognizer.checkStatus() + statuses[config] = status + when (status) { + FeatureStatus.UNAVAILABLE -> + throw AndroidAIErrors.CodedException( + AndroidAIErrors.MODEL_UNAVAILABLE, + "Speech recognition is not available for ${config.locale ?: Locale.getDefault().toLanguageTag()} on this device", + ) + FeatureStatus.DOWNLOADABLE, FeatureStatus.DOWNLOADING -> + downloads.collect("transcription", "The speech recognition model", status, recognizer.download()) + } + recognizer.warmup() + prepared.add(config) + statuses[config] = FeatureStatus.AVAILABLE + } + } + + /** Transcribes a whole audio file, which ML Kit decodes (MP3, WAV, FLAC, M4A or OGG). */ + suspend fun transcribe(audio: String, config: TranscriptionConfig): String { + prepare(config) + val file = File.createTempFile("transcription", null, context.cacheDir) + try { + file.writeBytes(Base64.decode(audio, Base64.DEFAULT)) + AudioSource.fromFile(file).use { source -> + val text = StringBuilder() + client(config).startRecognition(speechRecognizerRequest { audioSource = source }).collect { response -> + when (response) { + is SpeechRecognizerResponse.FinalTextResponse -> { + if (text.isNotEmpty() && response.text.isNotEmpty()) text.append(' ') + text.append(response.text) + } + is SpeechRecognizerResponse.ErrorResponse -> throw response.e + else -> {} + } + } + return text.toString() + } + } finally { + file.delete() + } + } +} diff --git a/packages/android/babel.config.js b/packages/android/babel.config.js new file mode 100644 index 00000000..3e0218e6 --- /dev/null +++ b/packages/android/babel.config.js @@ -0,0 +1,3 @@ +module.exports = { + presets: ['module:@react-native/babel-preset'], +} diff --git a/packages/android/package.json b/packages/android/package.json new file mode 100644 index 00000000..8848555e --- /dev/null +++ b/packages/android/package.json @@ -0,0 +1,84 @@ +{ + "name": "@react-native-ai/android", + "version": "0.12.0", + "description": "On-device Gemini Nano provider for Vercel AI SDK, built on ML Kit GenAI", + "main": "lib/commonjs/index", + "module": "lib/module/index", + "types": "lib/typescript/index.d.ts", + "react-native": "src/index", + "source": "src/index", + "files": [ + "src", + "lib", + "android", + "!android/build", + "!android/gradle", + "!android/gradlew", + "!android/gradlew.bat", + "!android/local.properties", + "!**/__tests__", + "!**/__fixtures__", + "!**/__mocks__", + "!**/.*" + ], + "scripts": { + "clean": "del-cli lib", + "typecheck": "tsc --noEmit", + "prepare": "bob build" + }, + "keywords": [ + "react-native", + "android", + "gemini-nano", + "ml-kit", + "llm", + "ai", + "sdk", + "vercel" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/callstackincubator/ai.git" + }, + "author": "Mike Grabowski ", + "license": "MIT", + "bugs": { + "url": "https://github.com/callstackincubator/ai/issues" + }, + "homepage": "https://github.com/callstackincubator/ai#readme", + "publishConfig": { + "registry": "https://registry.npmjs.org/" + }, + "dependencies": { + "@ai-sdk/provider": "^4.0.33", + "@ai-sdk/provider-utils": "^5.0.67" + }, + "peerDependencies": { + "react-native": ">=0.76.0" + }, + "devDependencies": { + "react-native": "0.88.0" + }, + "react-native-builder-bob": { + "source": "src", + "output": "lib", + "targets": [ + "commonjs", + "module", + [ + "typescript", + { + "project": "tsconfig.build.json" + } + ] + ] + }, + "codegenConfig": { + "name": "NativeAndroidAI", + "type": "modules", + "jsSrcsDir": "src", + "android": { + "javaPackageName": "com.callstack.ai.android" + } + } +} diff --git a/packages/android/src/NativeAndroidAI.ts b/packages/android/src/NativeAndroidAI.ts new file mode 100644 index 00000000..6d0c4d7a --- /dev/null +++ b/packages/android/src/NativeAndroidAI.ts @@ -0,0 +1,165 @@ +import type { CodegenTypes, TurboModule } from 'react-native' +import { Platform, TurboModuleRegistry } from 'react-native' + +import { AndroidAIErrorCodes, createAndroidAIError } from './errors' + +/** Gemini Nano model selection, see ML Kit `ModelConfig` */ +export interface NanoModelConfig { + /** `stable` (default) or `preview`, which needs the AICore developer preview */ + releaseStage?: string + /** `full` (default) favors quality, `fast` favors latency */ + preference?: string +} + +/** + * One part of a Gemini Nano message. `type` is `text`, `image`, + * `functionCall` or `functionResponse` + */ +export interface NativeMessagePart { + type: string + text?: string + /** Base64 image bytes */ + data?: string + /** file:// or content:// image URI */ + uri?: string + /** Function call id */ + id?: string + /** Function name */ + name?: string + /** Function call arguments or function response, as a JSON object */ + json?: string +} + +export interface NativeMessage { + /** `user` or `model` */ + role: string + parts: NativeMessagePart[] +} + +export interface NativeFunctionDeclaration { + name: string + description: string + /** JSON Schema of the parameters */ + parameters: string +} + +export interface NativeGenerateRequest { + messages: NativeMessage[] + systemInstruction?: string + tools?: NativeFunctionDeclaration[] + temperature?: number + topK?: number + seed?: number + maxOutputTokens?: number + enableThinking?: boolean + model?: NanoModelConfig +} + +export interface NativeFunctionCall { + id: string + name: string + /** Arguments as a JSON object */ + arguments: string +} + +export interface NativeGenerateResponse { + text: string + reasoning?: string + functionCalls: NativeFunctionCall[] + /** `STOP`, `MAX_TOKENS` or `OTHER` */ + finishReason?: string +} + +export interface NativeModelInfo { + availability: string + baseModelName?: string + contextSize?: number + /** ML Kit features: systemPrompt, thinking, prefixCaching, structuredOutput, functionCalling */ + features?: string[] +} + +export interface StreamEvent { + streamId: string + /** `text`, `reasoning`, `functionCall`, `finish` or `error` */ + type: string + delta?: string + functionCall?: NativeFunctionCall + finishReason?: string + error?: string + code?: string +} + +export interface DownloadProgressEvent { + /** `language`, `transcription` or `embedding` */ + model: string + /** `started`, `progress` or `completed` */ + phase: string + bytesDownloaded?: number + bytesToDownload?: number +} + +export interface NativeTranscriptionOptions { + /** BCP 47 locale, such as en-US */ + locale?: string + /** `basic` or `advanced` */ + mode?: string +} + +export interface NativeEmbeddingModelInfo { + availability: string + dimension?: number + modelSignature?: string +} + +export interface Spec extends TurboModule { + /** Quick synchronous check, see `AndroidProvider.isAvailable` */ + isAvailable(model?: NanoModelConfig): boolean + getModelInfo(model?: NanoModelConfig): Promise + prepare(model?: NanoModelConfig): Promise + countTokens(text: string, model?: NanoModelConfig): Promise + + generateText( + runId: string, + request: NativeGenerateRequest + ): Promise + streamText(streamId: string, request: NativeGenerateRequest): Promise + /** Cancels a `generateText`, `streamText` or `transcribe` run by its id */ + cancel(runId: string): void + + /** Quick synchronous check, see `AndroidTranscriptionModel.isAvailable` */ + isTranscriptionAvailable(options: NativeTranscriptionOptions): boolean + getTranscriptionModelInfo( + options: NativeTranscriptionOptions + ): Promise + prepareTranscription(options: NativeTranscriptionOptions): Promise + /** Transcribes a whole audio file (MP3, WAV, FLAC, M4A or OGG) */ + transcribe( + runId: string, + audio: string, + options: NativeTranscriptionOptions + ): Promise + + getEmbeddingModelInfo(): Promise + prepareEmbedding(): Promise + embed(values: string[], taskType?: string): Promise + + onStreamEvent: CodegenTypes.EventEmitter + onDownloadProgress: CodegenTypes.EventEmitter +} + +const NativeAndroidAI: Spec | null = + Platform.OS === 'android' + ? TurboModuleRegistry.getEnforcing('AndroidAI') + : null + +export default NativeAndroidAI + +export function getNativeAndroidAI(): Spec { + if (!NativeAndroidAI) { + throw createAndroidAIError( + '@react-native-ai/android is only available on Android', + AndroidAIErrorCodes.UnsupportedPlatform + ) + } + return NativeAndroidAI +} diff --git a/packages/android/src/__tests__/ai-sdk.test.ts b/packages/android/src/__tests__/ai-sdk.test.ts new file mode 100644 index 00000000..a1433e1c --- /dev/null +++ b/packages/android/src/__tests__/ai-sdk.test.ts @@ -0,0 +1,652 @@ +import type { + LanguageModelV4CallOptions, + LanguageModelV4StreamPart, +} from '@ai-sdk/provider' +import { beforeEach, describe, expect, mock, spyOn, test } from 'bun:test' + +type Listener = (event: any) => void + +function createEmitter() { + const listeners = new Set() + const subscribe = (listener: Listener) => { + listeners.add(listener) + return { remove: () => listeners.delete(listener) } + } + subscribe.emit = (event: unknown) => { + for (const listener of listeners) listener(event) + } + return subscribe +} + +const onStreamEvent = createEmitter() +const onDownloadProgress = createEmitter() + +let generateTextResult: Record = { + text: 'ok', + functionCalls: [], + finishReason: 'STOP', +} + +const generateText = mock( + async (_runId: string, _request: any): Promise> => + generateTextResult +) +const streamText = mock(async (_streamId: string, _request: any) => {}) +const prepare = mock(async (_model?: unknown) => {}) +const cancel = mock((_id: string) => {}) +const isAvailable = mock((_model?: unknown) => true) +const isTranscriptionAvailable = mock((_options?: unknown) => true) +const prepareTranscription = mock(async (_options?: unknown) => {}) +const prepareEmbedding = mock(async () => {}) +const getModelInfo = mock( + async (_model?: unknown): Promise> => ({ + availability: 'available', + features: ['functionCalling', 'thinking'], + }) +) +const transcribe = mock( + async (_runId: string, _audio: string, _options: unknown) => 'hello world' +) +const getTranscriptionModelInfo = mock( + async (_options?: unknown): Promise> => ({ + availability: 'downloadable', + }) +) +const embed = mock(async (values: string[], _taskType?: string) => + values.map(() => [0.1, 0.2]) +) + +mock.module('react-native', () => ({ + Platform: { OS: 'android' }, + TurboModuleRegistry: { + getEnforcing: () => ({ + generateText, + streamText, + prepare, + cancel, + isAvailable, + getModelInfo, + transcribe, + isTranscriptionAvailable, + getTranscriptionModelInfo, + prepareTranscription, + prepareEmbedding, + embed, + onStreamEvent, + onDownloadProgress, + }), + }, +})) + +const { createAndroidProvider } = await import('../ai-sdk') + +const nano = createAndroidProvider()() + +const pngBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47]) + +function createCallOptions( + overrides: Partial = {} +): LanguageModelV4CallOptions { + return { + prompt: [{ role: 'user', content: [{ type: 'text', text: 'hello' }] }], + ...overrides, + } +} + +const lastRequest = () => generateText.mock.calls.at(-1)![1] + +async function readAll(stream: ReadableStream) { + const parts: LanguageModelV4StreamPart[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) return parts + parts.push(value) + } +} + +// Starts a stream, then plays native events once the native call is made +async function runStream( + options: LanguageModelV4CallOptions, + events: Record[] +) { + streamText.mockImplementationOnce(async (streamId: string) => { + queueMicrotask(() => { + for (const event of events) onStreamEvent.emit({ streamId, ...event }) + }) + }) + const { stream } = await nano.doStream(options) + return readAll(stream) +} + +beforeEach(() => { + generateText.mockClear() + streamText.mockClear() + prepare.mockClear() + cancel.mockClear() + getModelInfo.mockClear() + generateTextResult = { text: 'ok', functionCalls: [], finishReason: 'STOP' } +}) + +describe('provider', () => { + test('creates Gemini Nano models', () => { + expect(nano.modelId).toBe('gemini-nano') + expect(nano.provider).toBe('android') + }) + + test('reports availability for the model selection', async () => { + getModelInfo.mockImplementationOnce(async () => ({ + availability: 'available', + baseModelName: 'nano-v4', + contextSize: 4000, + features: ['functionCalling', 'someFutureFeature'], + })) + const provider = createAndroidProvider({ + model: { releaseStage: 'preview', preference: 'fast' }, + }) + + expect(await provider.getModelInfo()).toEqual({ + availability: 'available', + baseModelName: 'nano-v4', + contextSize: 4000, + features: ['functionCalling'], + }) + expect(getModelInfo.mock.calls.at(-1)![0]).toEqual({ + releaseStage: 'preview', + preference: 'fast', + }) + }) + + test('checks availability synchronously', () => { + expect(createAndroidProvider().isAvailable()).toBe(true) + isAvailable.mockImplementationOnce(() => false) + expect(nano.isAvailable()).toBe(false) + }) + + test('reports download progress while preparing', async () => { + const model = createAndroidProvider()() + prepare.mockImplementationOnce(async () => { + onDownloadProgress.emit({ model: 'embedding', phase: 'started' }) + onDownloadProgress.emit({ + model: 'language', + phase: 'started', + bytesToDownload: 10, + }) + }) + const progress: unknown[] = [] + + await model.prepare({ onProgress: (event) => progress.push(event) }) + + expect(progress).toEqual([ + { phase: 'started', bytesDownloaded: undefined, bytesToDownload: 10 }, + ]) + }) +}) + +describe('doGenerate', () => { + test('sends history, system prompt and settings', async () => { + await nano.doGenerate( + createCallOptions({ + prompt: [ + { role: 'system', content: 'Be brief.' }, + { + role: 'user', + content: [ + { type: 'text', text: 'What is this?' }, + { + type: 'file', + mediaType: 'image/png', + data: { type: 'data', data: pngBytes }, + }, + ], + }, + { + role: 'assistant', + content: [ + { type: 'reasoning', text: 'hmm' }, + { type: 'text', text: 'A logo.' }, + ], + }, + { role: 'user', content: [{ type: 'text', text: 'Thanks' }] }, + ], + temperature: 0.2, + topK: 3, + seed: 7, + maxOutputTokens: 100, + }) + ) + + expect(lastRequest()).toEqual({ + messages: [ + { + role: 'user', + parts: [ + { type: 'text', text: 'What is this?' }, + { type: 'image', data: 'iVBORw==' }, + ], + }, + { role: 'model', parts: [{ type: 'text', text: 'A logo.' }] }, + { role: 'user', parts: [{ type: 'text', text: 'Thanks' }] }, + ], + systemInstruction: 'Be brief.', + tools: undefined, + temperature: 0.2, + topK: 3, + seed: 7, + maxOutputTokens: 100, + enableThinking: undefined, + model: undefined, + }) + }) + + test('returns text and reasoning', async () => { + generateTextResult = { + text: 'Hi', + reasoning: 'Thinking', + functionCalls: [], + finishReason: 'MAX_TOKENS', + } + + const result = await nano.doGenerate(createCallOptions()) + + expect(result.content).toEqual([ + { type: 'reasoning', text: 'Thinking' }, + { type: 'text', text: 'Hi' }, + ]) + expect(result.finishReason).toEqual({ + unified: 'length', + raw: 'MAX_TOKENS', + }) + }) + + test('declares tools and returns their calls', async () => { + generateTextResult = { + text: '', + functionCalls: [ + { id: 'call-1', name: 'getWeather', arguments: '{"city":"Paris"}' }, + ], + finishReason: 'STOP', + } + const inputSchema = { + type: 'object' as const, + properties: { city: { type: 'string' } }, + required: ['city'], + } + + const result = await nano.doGenerate( + createCallOptions({ + tools: [ + { + type: 'function', + name: 'getWeather', + description: 'Gets the weather', + inputSchema, + }, + ], + }) + ) + + expect(lastRequest().tools).toEqual([ + { + name: 'getWeather', + description: 'Gets the weather', + parameters: JSON.stringify(inputSchema), + }, + ]) + expect(result.content).toEqual([ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: '{"city":"Paris"}', + }, + ]) + expect(result.finishReason.unified).toBe('tool-calls') + }) + + test('sends tool calls and results as function parts', async () => { + const { warnings } = await nano.doGenerate( + createCallOptions({ + prompt: [ + { role: 'user', content: [{ type: 'text', text: 'Weather?' }] }, + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: { city: 'Paris' }, + }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'getWeather', + output: { type: 'text', value: 'Sunny' }, + }, + ], + }, + ], + }) + ) + + expect(lastRequest().messages.slice(1)).toEqual([ + { + role: 'model', + parts: [ + { + type: 'functionCall', + id: 'call-1', + name: 'getWeather', + json: '{"city":"Paris"}', + }, + ], + }, + { + role: 'user', + parts: [ + { + type: 'functionResponse', + id: 'call-1', + name: 'getWeather', + json: '{"result":"Sunny"}', + }, + ], + }, + ]) + // The mocked model has no systemPrompt feature, so the history is merged + expect(warnings).toContainEqual( + expect.objectContaining({ + type: 'unsupported', + feature: 'tool call history', + }) + ) + }) + + test('sends tool call input given as a JSON string as is', async () => { + await nano.doGenerate( + createCallOptions({ + prompt: [ + { role: 'user', content: [{ type: 'text', text: 'Weather?' }] }, + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: '{"city":"Paris"}', + }, + ], + }, + ], + }) + ) + + expect(lastRequest().messages[1].parts[0].json).toBe('{"city":"Paris"}') + }) + + test('sends denied tool calls as errors', async () => { + await nano.doGenerate( + createCallOptions({ + prompt: [ + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'call-1', + toolName: 'pay', + output: { type: 'execution-denied', reason: 'No' }, + }, + ], + }, + ], + }) + ) + + expect(lastRequest().messages[0].parts[0].json).toBe('{"error":"No"}') + }) + + test('warns and leaves out tools and thinking the model lacks', async () => { + getModelInfo.mockImplementationOnce(async () => ({ + availability: 'available', + features: [], + })) + const model = createAndroidProvider()() + + const { warnings } = await model.doGenerate( + createCallOptions({ + tools: [ + { type: 'function', name: 'a', inputSchema: { type: 'object' } }, + ], + reasoning: 'high', + }) + ) + + expect(lastRequest().tools).toBeUndefined() + expect(lastRequest().enableThinking).toBeUndefined() + expect(warnings.map((warning) => (warning as any).feature)).toEqual([ + 'tools', + 'reasoning', + ]) + }) + + test('leaves tools out with toolChoice none', async () => { + await nano.doGenerate( + createCallOptions({ + tools: [ + { + type: 'function', + name: 'getWeather', + inputSchema: { type: 'object' }, + }, + ], + toolChoice: { type: 'none' }, + }) + ) + + expect(lastRequest().tools).toBeUndefined() + }) + + test('warns about unsupported settings', async () => { + const { warnings } = await nano.doGenerate( + createCallOptions({ + topP: 0.9, + stopSequences: ['x'], + responseFormat: { type: 'json' }, + tools: [ + { type: 'function', name: 'a', inputSchema: { type: 'object' } }, + ], + toolChoice: { type: 'required' }, + }) + ) + + expect(warnings.map((warning) => (warning as any).feature)).toEqual([ + 'topP', + 'stopSequences', + 'responseFormat', + 'toolChoice', + ]) + }) + + test('maps reasoning to thinking mode', async () => { + await nano.doGenerate(createCallOptions({ reasoning: 'high' })) + expect(lastRequest().enableThinking).toBe(true) + + await nano.doGenerate(createCallOptions({ reasoning: 'none' })) + expect(lastRequest().enableThinking).toBe(false) + + await nano.doGenerate( + createCallOptions({ + providerOptions: { android: { enableThinking: true } }, + }) + ) + expect(lastRequest().enableThinking).toBe(true) + }) + + test('rejects non-image files', async () => { + await expect( + nano.doGenerate( + createCallOptions({ + prompt: [ + { + role: 'user', + content: [ + { + type: 'file', + mediaType: 'audio/wav', + data: { type: 'data', data: pngBytes }, + }, + ], + }, + ], + }) + ) + ).rejects.toMatchObject({ code: 'INVALID_MESSAGE' }) + }) + + test('keeps native error codes', async () => { + generateText.mockImplementationOnce(async () => { + throw Object.assign(new Error('Quota'), { code: 'RATE_LIMITED' }) + }) + + await expect(nano.doGenerate(createCallOptions())).rejects.toMatchObject({ + code: 'RATE_LIMITED', + message: 'Quota', + }) + }) + + test('cancels the native run when aborted', async () => { + const controller = new AbortController() + generateText.mockImplementationOnce(async () => { + controller.abort() + throw new Error('cancelled') + }) + + await expect( + nano.doGenerate(createCallOptions({ abortSignal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(cancel).toHaveBeenCalledTimes(1) + }) + test('stops when aborted while preparing', async () => { + const controller = new AbortController() + prepare.mockImplementationOnce(async () => controller.abort()) + generateText.mockClear() + + await expect( + createAndroidProvider()().doGenerate( + createCallOptions({ abortSignal: controller.signal }) + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(generateText).not.toHaveBeenCalled() + }) +}) + +describe('doStream', () => { + test('streams text, reasoning and tool calls', async () => { + const parts = await runStream(createCallOptions(), [ + { type: 'reasoning', delta: 'Hmm' }, + { type: 'text', delta: 'Hel' }, + { type: 'text', delta: 'lo' }, + { + type: 'functionCall', + functionCall: { id: 'c1', name: 'getTime', arguments: '{}' }, + }, + { type: 'finish', finishReason: 'STOP' }, + ]) + + expect(parts.map((part) => part.type)).toEqual([ + 'stream-start', + 'reasoning-start', + 'reasoning-delta', + 'reasoning-end', + 'text-start', + 'text-delta', + 'text-delta', + 'tool-call', + 'text-end', + 'finish', + ]) + expect(parts.at(-1)).toMatchObject({ + finishReason: { unified: 'tool-calls' }, + }) + }) + + test('reports stream errors with their code', async () => { + const parts = await runStream(createCallOptions(), [ + { type: 'error', error: 'Busy', code: 'RATE_LIMITED' }, + ]) + + expect(parts.at(-1)).toMatchObject({ + type: 'error', + error: { code: 'RATE_LIMITED', message: 'Busy' }, + }) + }) +}) + +describe('other models', () => { + test('transcribes audio', async () => { + const model = createAndroidProvider().transcriptionModel({ + language: 'en-US', + mode: 'basic', + }) + + const result = await model.doGenerate({ + audio: pngBytes, + mediaType: 'audio/wav', + }) + + expect(transcribe.mock.calls.at(-1)!.slice(1)).toEqual([ + 'iVBORw==', + { locale: 'en-US', mode: 'basic' }, + ]) + expect(result.text).toBe('hello world') + expect(result.language).toBe('en') + }) + + test('checks transcription availability synchronously', () => { + const model = createAndroidProvider().transcriptionModel({ + language: 'en-US', + }) + expect(model.isAvailable()).toBe(true) + isTranscriptionAvailable.mockImplementationOnce(() => false) + expect(model.isAvailable()).toBe(false) + expect(isTranscriptionAvailable.mock.calls.at(-1)![0]).toEqual({ + locale: 'en-US', + mode: undefined, + }) + }) + + test('warns and prepares when a model is used before prepare', async () => { + const warn = spyOn(console, 'warn').mockImplementation(() => {}) + const model = createAndroidProvider().embeddingModel() + + await model.doEmbed({ values: ['a'] }) + await model.doEmbed({ values: ['b'] }) + + expect(prepareEmbedding).toHaveBeenCalledTimes(1) + expect(warn).toHaveBeenCalledTimes(1) + expect(warn.mock.calls[0][0]).toContain('Model not prepared') + warn.mockRestore() + }) + + test('embeds values with the task type', async () => { + const model = createAndroidProvider().embeddingModel({ + taskType: 'semanticSimilarity', + }) + + const result = await model.doEmbed({ values: ['a', 'b'] }) + + expect(embed.mock.calls.at(-1)).toEqual([['a', 'b'], 'semanticSimilarity']) + expect(result.embeddings).toEqual([ + [0.1, 0.2], + [0.1, 0.2], + ]) + }) +}) diff --git a/packages/android/src/ai-sdk.ts b/packages/android/src/ai-sdk.ts new file mode 100644 index 00000000..fdc6d2d4 --- /dev/null +++ b/packages/android/src/ai-sdk.ts @@ -0,0 +1,1022 @@ +import type { + EmbeddingModelV4, + EmbeddingModelV4CallOptions, + EmbeddingModelV4Result, + LanguageModelV4, + LanguageModelV4CallOptions, + LanguageModelV4Content, + LanguageModelV4FilePart, + LanguageModelV4FinishReason, + LanguageModelV4Prompt, + LanguageModelV4StreamPart, + LanguageModelV4ToolResultOutput, + LanguageModelV4Usage, + SharedV4Warning, + TranscriptionModelV4, + TranscriptionModelV4CallOptions, +} from '@ai-sdk/provider' +import { generateId } from '@ai-sdk/provider-utils' +import { Platform } from 'react-native' + +import { + AndroidAIErrorCodes, + createAndroidAIError, + toAndroidAIError, +} from './errors' +import { + type DownloadProgressEvent, + getNativeAndroidAI, + type NanoModelConfig, + type NativeFunctionCall, + type NativeFunctionDeclaration, + type NativeGenerateRequest, + type NativeMessage, + type NativeMessagePart, + type NativeModelInfo, + type NativeTranscriptionOptions, +} from './NativeAndroidAI' + +const MODEL_ID = 'gemini-nano' + +export interface AndroidNanoModel { + /** `stable` (default) or `preview`, which needs the AICore developer preview */ + releaseStage?: 'stable' | 'preview' + /** + * `full` (default) favors quality, `fast` favors latency. Function calling + * needs Gemini Nano 4, so `releaseStage: 'preview'` with `fast` + */ + preference?: 'full' | 'fast' +} + +export interface AndroidLanguageModelOptions { + /** Gemini Nano model selection, see ML Kit `ModelConfig` */ + model?: AndroidNanoModel +} + +/** Per-call options passed as `providerOptions.android` */ +export interface AndroidProviderOptions { + /** + * Turns Gemini Nano's thinking mode on or off when `reasoning` is not a + * level, that is when it is unset or `provider-default` + */ + enableThinking?: boolean +} + +export type AndroidModelAvailability = + | 'available' + | 'downloadable' + | 'downloading' + | 'unavailable' + | 'unknown' + +const modelFeatures = [ + 'systemPrompt', + 'thinking', + 'prefixCaching', + 'structuredOutput', + 'functionCalling', +] as const + +export type AndroidModelFeature = (typeof modelFeatures)[number] + +export interface AndroidModelInfo { + availability: AndroidModelAvailability + /** Name of the Gemini Nano base model, when ML Kit reports it */ + baseModelName?: string + /** Gemini Nano input token limit, when ML Kit reports it */ + contextSize?: number + /** Features this device's Gemini Nano supports, when it is available */ + features?: AndroidModelFeature[] +} + +export interface AndroidDownloadProgress { + phase: 'started' | 'progress' | 'completed' + bytesDownloaded?: number + bytesToDownload?: number +} + +export interface AndroidPrepareOptions { + onProgress?: (progress: AndroidDownloadProgress) => void +} + +export interface AndroidLanguageModel extends LanguageModelV4 { + /** Quick synchronous check, see `AndroidProvider.isAvailable` */ + isAvailable: () => boolean + /** Gemini Nano's availability and features for this model selection */ + getModelInfo: () => Promise + /** Downloads Gemini Nano when needed and loads it, reporting download progress */ + prepare: (options?: AndroidPrepareOptions) => Promise + /** Counts tokens with the Gemini Nano tokenizer */ + countTokens: (text: string) => Promise +} + +export interface AndroidTranscriptionOptions { + /** BCP 47 locale, such as en-US. Defaults to the device locale */ + language?: string + /** + * `basic` uses the traditional on-device recognizer, `advanced` Gemini Nano. + * ML Kit picks one when it is not set + */ + mode?: 'basic' | 'advanced' +} + +export type AndroidEmbeddingTaskType = + | 'retrievalQuery' + | 'retrievalDocument' + | 'semanticSimilarity' + | 'classification' + | 'clustering' + | 'questionAnswering' + | 'factVerification' + | 'codeRetrieval' + +export interface AndroidEmbeddingOptions { + /** What the embeddings are for, see ML Kit `TaskConfig` */ + taskType?: AndroidEmbeddingTaskType +} + +export interface AndroidEmbeddingModelInfo { + availability: AndroidModelAvailability + dimension?: number + modelSignature?: string +} + +const toModelAvailability = (value: string): AndroidModelAvailability => { + switch (value) { + case 'available': + case 'downloadable': + case 'downloading': + case 'unavailable': + return value + default: + return 'unknown' + } +} + +const notPreparedWarning = + '[android] Model not prepared. Call prepare() ahead of time to optimize performance.' + +const isModelFeature = (value: string): value is AndroidModelFeature => + (modelFeatures as readonly string[]).includes(value) + +const toModelInfo = (info: NativeModelInfo): AndroidModelInfo => ({ + availability: toModelAvailability(info.availability), + baseModelName: info.baseModelName, + contextSize: info.contextSize, + features: info.features?.filter(isModelFeature), +}) + +const listenForDownloadProgress = ( + model: DownloadProgressEvent['model'], + onProgress: AndroidPrepareOptions['onProgress'] +) => + onProgress + ? getNativeAndroidAI().onDownloadProgress((event) => { + if (event.model !== model) { + return + } + onProgress({ + phase: event.phase as AndroidDownloadProgress['phase'], + bytesDownloaded: event.bytesDownloaded, + bytesToDownload: event.bytesToDownload, + }) + }) + : null + +async function withProgress( + model: DownloadProgressEvent['model'], + options: AndroidPrepareOptions, + run: () => Promise +): Promise { + const listener = listenForDownloadProgress(model, options.onProgress) + try { + return await run() + } catch (error) { + throw toAndroidAIError(error) + } finally { + listener?.remove() + } +} + +function isAvailable(model: NanoModelConfig | undefined) { + return Platform.OS === 'android' && getNativeAndroidAI().isAvailable(model) +} + +async function getModelInfo( + model: NanoModelConfig | undefined +): Promise { + if (Platform.OS !== 'android') { + return { availability: 'unavailable' } + } + + try { + return toModelInfo(await getNativeAndroidAI().getModelInfo(model)) + } catch (error) { + throw toAndroidAIError(error) + } +} + +export function createAndroidProvider( + defaults: AndroidLanguageModelOptions = {} +) { + const createLanguageModel = ( + options: AndroidLanguageModelOptions = {} + ): AndroidLanguageModel => + new AndroidChatLanguageModel({ ...defaults, ...options }) + + const provider = function (options?: AndroidLanguageModelOptions) { + return createLanguageModel(options) + } + provider.specificationVersion = 'v4' as const + provider.languageModel = createLanguageModel + /** + * Quick synchronous check whether Gemini Nano can be used. It reflects the + * last status `getModelInfo()` or `prepare()` saw, or else whether AICore is + * installed. Use `getModelInfo()` for the exact availability + */ + provider.isAvailable = () => isAvailable(defaults.model) + /** Gemini Nano's availability, for the provider's model selection or `model` */ + provider.getModelInfo = (options: { model?: AndroidNanoModel } = {}) => + getModelInfo(options.model ?? defaults.model) + provider.transcriptionModel = (options: AndroidTranscriptionOptions = {}) => + new AndroidTranscriptionModel(options) + provider.embeddingModel = (options: AndroidEmbeddingOptions = {}) => + new AndroidEmbeddingModel(options) + return provider +} + +export type AndroidProvider = ReturnType + +export const android = createAndroidProvider() + +const emptyUsage = (): LanguageModelV4Usage => ({ + // ML Kit reports no token usage for a request + inputTokens: { + total: undefined, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { total: undefined, text: undefined, reasoning: undefined }, +}) + +const convertFinishReason = ( + finishReason: string | undefined, + hasToolCalls: boolean +): LanguageModelV4FinishReason => { + if (hasToolCalls) { + return { unified: 'tool-calls', raw: finishReason } + } + switch (finishReason) { + case undefined: + case 'STOP': + return { unified: 'stop', raw: finishReason } + case 'MAX_TOKENS': + return { unified: 'length', raw: finishReason } + default: + return { unified: 'other', raw: finishReason } + } +} + +const uint8ArrayToBase64 = (bytes: Uint8Array): string => { + // One String.fromCharCode per byte stalls the JS thread on large images and audio + const chunkSize = 0x8000 + let binary = '' + for (let start = 0; start < bytes.length; start += chunkSize) { + binary += String.fromCharCode(...bytes.subarray(start, start + chunkSize)) + } + return btoa(binary) +} + +const localUrlProtocols = new Set(['file:', 'content:']) + +const toImagePart = (part: LanguageModelV4FilePart): NativeMessagePart => { + if (!part.mediaType.startsWith('image')) { + throw createAndroidAIError( + `Gemini Nano only takes image files. Got ${part.mediaType}`, + AndroidAIErrorCodes.InvalidMessage + ) + } + + switch (part.data.type) { + case 'data': + return { + type: 'image', + data: + typeof part.data.data === 'string' + ? part.data.data.replace(/^data:[^;]+;base64,/, '') + : uint8ArrayToBase64(part.data.data), + } + case 'url': + // The AI SDK downloads other URLs because they are not in `supportedUrls` + if (!localUrlProtocols.has(part.data.url.protocol)) { + throw createAndroidAIError( + `Gemini Nano only reads file:// and content:// URLs. Got ${part.data.url.protocol}`, + AndroidAIErrorCodes.InvalidMessage + ) + } + return { type: 'image', uri: part.data.url.toString() } + default: + throw createAndroidAIError( + 'Gemini Nano does not take text files or provider file references.', + AndroidAIErrorCodes.InvalidMessage + ) + } +} + +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value) + +// ML Kit takes a function response as a JSON object +const toFunctionResponse = ( + output: LanguageModelV4ToolResultOutput +): Record => { + switch (output.type) { + case 'json': + return isPlainObject(output.value) + ? output.value + : { result: output.value } + case 'error-text': + case 'error-json': + return { error: output.value } + case 'execution-denied': + return { error: output.reason ?? 'Execution denied' } + default: + return { result: output.value } + } +} + +function prepareMessages( + prompt: LanguageModelV4Prompt, + warnings: SharedV4Warning[] +) { + const systemPrompts: string[] = [] + const messages: NativeMessage[] = [] + + // ML Kit takes turns that alternate between the user and the model, so tool + // results join the user turn around them + const addParts = (role: 'user' | 'model', parts: NativeMessagePart[]) => { + if (parts.length === 0) { + return + } + const last = messages.at(-1) + if (last?.role === role) { + last.parts.push(...parts) + } else { + messages.push({ role, parts }) + } + } + + for (const message of prompt) { + switch (message.role) { + case 'system': + systemPrompts.push(message.content) + break + case 'user': + addParts( + 'user', + message.content.map((part) => + part.type === 'text' + ? { type: 'text', text: part.text } + : toImagePart(part) + ) + ) + break + case 'assistant': + addParts( + 'model', + message.content.flatMap((part): NativeMessagePart[] => { + switch (part.type) { + case 'text': + return part.text ? [{ type: 'text', text: part.text }] : [] + case 'reasoning': + return [] + case 'tool-call': + return [ + { + type: 'functionCall', + id: part.toolCallId, + name: part.toolName, + json: + typeof part.input === 'string' + ? part.input + : JSON.stringify(part.input ?? {}), + }, + ] + default: + warnings.push({ + type: 'unsupported', + feature: `assistant ${part.type} parts`, + details: 'Gemini Nano takes text and function call history.', + }) + return [] + } + }) + ) + break + case 'tool': + addParts( + 'user', + message.content.flatMap((part): NativeMessagePart[] => + part.type === 'tool-result' + ? [ + { + type: 'functionResponse', + id: part.toolCallId, + name: part.toolName, + json: JSON.stringify(toFunctionResponse(part.output)), + }, + ] + : [] + ) + ) + break + } + } + + return { + messages, + systemInstruction: + systemPrompts.length > 0 ? systemPrompts.join('\n\n') : undefined, + } +} + +function prepareTools( + options: LanguageModelV4CallOptions, + features: AndroidModelFeature[], + warnings: SharedV4Warning[] +): NativeFunctionDeclaration[] | undefined { + if (!options.tools?.length || options.toolChoice?.type === 'none') { + return undefined + } + + if ( + options.toolChoice?.type === 'required' || + options.toolChoice?.type === 'tool' + ) { + warnings.push({ + type: 'unsupported', + feature: 'toolChoice', + details: 'Gemini Nano decides itself whether to call a function.', + }) + } + + // Without it, ML Kit fails the whole request + if (!features.includes('functionCalling')) { + warnings.push({ + type: 'unsupported', + feature: 'tools', + details: + "This device's Gemini Nano has no function calling. It needs Gemini Nano 4, the preview release with the fast preference.", + }) + return undefined + } + + return options.tools.flatMap((tool) => { + if (tool.type !== 'function') { + warnings.push({ + type: 'unsupported', + feature: `provider tool ${tool.name}`, + details: 'Gemini Nano only takes function tools.', + }) + return [] + } + return [ + { + name: tool.name, + description: tool.description ?? '', + parameters: JSON.stringify(tool.inputSchema), + }, + ] + }) +} + +function prepareThinking( + options: LanguageModelV4CallOptions, + androidOptions: AndroidProviderOptions, + features: AndroidModelFeature[], + warnings: SharedV4Warning[] +) { + let enableThinking: boolean | undefined + switch (options.reasoning) { + case undefined: + case 'provider-default': + enableThinking = androidOptions.enableThinking + break + case 'none': + enableThinking = false + break + default: + // Gemini Nano's thinking mode is on or off + enableThinking = true + } + + if (enableThinking && !features.includes('thinking')) { + warnings.push({ + type: 'unsupported', + feature: 'reasoning', + details: "This device's Gemini Nano has no thinking mode.", + }) + return undefined + } + return enableThinking +} + +function prepareRequest( + options: LanguageModelV4CallOptions, + model: NanoModelConfig | undefined, + features: AndroidModelFeature[] +) { + const warnings: SharedV4Warning[] = [] + const androidOptions = (options.providerOptions?.android ?? + {}) as AndroidProviderOptions + + // ML Kit's GenerateContentRequest has no setting for these, and structured + // output only takes Kotlin classes generated at build time + const unsupported = { + topP: options.topP, + stopSequences: options.stopSequences, + presencePenalty: options.presencePenalty, + frequencyPenalty: options.frequencyPenalty, + responseFormat: + options.responseFormat?.type === 'json' ? 'json' : undefined, + } + for (const [feature, value] of Object.entries(unsupported)) { + if (value !== undefined) { + warnings.push({ + type: 'unsupported', + feature, + details: 'Gemini Nano does not support this setting.', + }) + } + } + + const { messages, systemInstruction } = prepareMessages( + options.prompt, + warnings + ) + // Without system prompt support the native side merges the conversation + // into one text part, which leaves function parts out + if ( + !features.includes('systemPrompt') && + messages.some((message) => + message.parts.some( + (part) => + part.type === 'functionCall' || part.type === 'functionResponse' + ) + ) + ) { + warnings.push({ + type: 'unsupported', + feature: 'tool call history', + details: + 'This Gemini Nano model gets the conversation as one text prompt, so earlier tool calls and results are left out.', + }) + } + + const request: NativeGenerateRequest = { + messages, + systemInstruction, + tools: prepareTools(options, features, warnings), + temperature: options.temperature, + topK: options.topK, + seed: options.seed, + maxOutputTokens: options.maxOutputTokens, + enableThinking: prepareThinking( + options, + androidOptions, + features, + warnings + ), + model, + } + + return { request, warnings } +} + +const toToolCall = (call: NativeFunctionCall) => ({ + type: 'tool-call' as const, + toolCallId: call.id || generateId(), + toolName: call.name, + input: call.arguments, +}) + +const getAbortReason = (signal: AbortSignal) => { + if (signal.reason !== undefined) { + return signal.reason + } + const error = new Error('The operation was aborted.') + error.name = 'AbortError' + return error +} + +class AndroidChatLanguageModel implements AndroidLanguageModel { + readonly specificationVersion = 'v4' + // expo-file-system and Android's Uri.fromFile write file:/path, with one slash + readonly supportedUrls = { + 'image/*': [/^file:/, /^content:/], + } + + readonly provider = 'android' + readonly modelId = MODEL_ID + + private readonly model: NanoModelConfig | undefined + // Resolves to the prepared model's features + private preparePromise: Promise | null = null + + constructor(options: AndroidLanguageModelOptions) { + this.model = options.model + } + + isAvailable(): boolean { + return isAvailable(this.model) + } + + getModelInfo(): Promise { + return getModelInfo(this.model) + } + + prepare(options: AndroidPrepareOptions = {}): Promise { + return withProgress('language', options, async () => { + await this.ensurePrepared() + }) + } + + async countTokens(text: string): Promise { + await this.ensurePrepared() + try { + return await getNativeAndroidAI().countTokens(text, this.model) + } catch (error) { + throw toAndroidAIError(error) + } + } + + private ensurePrepared(): Promise { + if (!this.preparePromise) { + const native = getNativeAndroidAI() + this.preparePromise = native + .prepare(this.model) + .then(() => native.getModelInfo(this.model)) + .then((info) => toModelInfo(info).features ?? []) + .catch((error: unknown) => { + this.preparePromise = null + throw toAndroidAIError(error) + }) + } + return this.preparePromise + } + + async doGenerate(options: LanguageModelV4CallOptions) { + const native = getNativeAndroidAI() + const runId = generateId() + const abortSignal = options.abortSignal + + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + const onAbort = () => native.cancel(runId) + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + try { + if (!this.preparePromise) { + console.warn(notPreparedWarning) + } + const features = await this.ensurePrepared() + // Preparing can take minutes, before there is a native run to cancel + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + const { request, warnings } = prepareRequest( + options, + this.model, + features + ) + const response = await native.generateText(runId, request) + + const content: LanguageModelV4Content[] = [] + if (response.reasoning) { + content.push({ type: 'reasoning', text: response.reasoning }) + } + if (response.text) { + content.push({ type: 'text', text: response.text }) + } + content.push(...response.functionCalls.map(toToolCall)) + + return { + content, + finishReason: convertFinishReason( + response.finishReason, + response.functionCalls.length > 0 + ), + usage: emptyUsage(), + warnings, + request: { body: request }, + } + } catch (error) { + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + throw toAndroidAIError(error) + } finally { + abortSignal?.removeEventListener('abort', onAbort) + } + } + + async doStream(options: LanguageModelV4CallOptions) { + const native = getNativeAndroidAI() + const streamId = generateId() + const abortSignal = options.abortSignal + + if (typeof ReadableStream === 'undefined') { + throw new Error( + 'ReadableStream is not available in this environment. Please load a polyfill, such as web-streams-polyfill.' + ) + } + + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + if (!this.preparePromise) { + console.warn(notPreparedWarning) + } + const features = await this.ensurePrepared() + // Preparing can take minutes, before there is a native stream to cancel + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + const { request, warnings } = prepareRequest(options, this.model, features) + + let listener: { remove(): void } | undefined + let onAbort = () => {} + + const cleanup = () => { + listener?.remove() + listener = undefined + abortSignal?.removeEventListener('abort', onAbort) + } + + const stream = new ReadableStream({ + start: async (controller) => { + const textId = `${streamId}-text` + let textOpen = false + let reasoningId: string | undefined + let reasoningCount = 0 + let hasToolCalls = false + + const closeReasoning = () => { + if (reasoningId) { + controller.enqueue({ type: 'reasoning-end', id: reasoningId }) + reasoningId = undefined + } + } + + controller.enqueue({ type: 'stream-start', warnings }) + + onAbort = () => { + cleanup() + native.cancel(streamId) + controller.error(abortSignal && getAbortReason(abortSignal)) + } + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + listener = native.onStreamEvent((event) => { + if (event.streamId !== streamId) { + return + } + + switch (event.type) { + case 'reasoning': + if (!event.delta) return + if (!reasoningId) { + reasoningId = `${streamId}-reasoning-${reasoningCount++}` + controller.enqueue({ type: 'reasoning-start', id: reasoningId }) + } + controller.enqueue({ + type: 'reasoning-delta', + id: reasoningId, + delta: event.delta, + }) + return + case 'text': + if (!event.delta) return + closeReasoning() + if (!textOpen) { + controller.enqueue({ type: 'text-start', id: textId }) + textOpen = true + } + controller.enqueue({ + type: 'text-delta', + id: textId, + delta: event.delta, + }) + return + case 'functionCall': + if (!event.functionCall) return + closeReasoning() + hasToolCalls = true + controller.enqueue(toToolCall(event.functionCall)) + return + case 'finish': + closeReasoning() + if (textOpen) { + controller.enqueue({ type: 'text-end', id: textId }) + } + controller.enqueue({ + type: 'finish', + finishReason: convertFinishReason( + event.finishReason, + hasToolCalls + ), + usage: emptyUsage(), + }) + cleanup() + controller.close() + return + case 'error': + controller.enqueue({ + type: 'error', + error: toAndroidAIError(event.error, event.code), + }) + cleanup() + controller.close() + return + } + }) + + try { + await native.streamText(streamId, request) + } catch (error) { + cleanup() + controller.error(toAndroidAIError(error)) + } + }, + cancel: () => { + cleanup() + native.cancel(streamId) + }, + }) + + return { stream, request: { body: request } } + } +} + +const toAudioBase64 = (audio: TranscriptionModelV4CallOptions['audio']) => + typeof audio === 'string' + ? audio.replace(/^data:[^;]+;base64,/, '') + : uint8ArrayToBase64(audio) + +class AndroidTranscriptionModel implements TranscriptionModelV4 { + readonly specificationVersion = 'v4' + readonly provider = 'android' + readonly modelId = 'speech-recognition' + + private readonly options: NativeTranscriptionOptions + private prepared = false + + constructor(options: AndroidTranscriptionOptions) { + this.options = { locale: options.language, mode: options.mode } + } + + /** Recognizer availability for the model's language and mode */ + async getModelInfo(): Promise { + if (Platform.OS !== 'android') { + return { availability: 'unavailable' } + } + try { + return toModelInfo( + await getNativeAndroidAI().getTranscriptionModelInfo(this.options) + ) + } catch (error) { + throw toAndroidAIError(error) + } + } + + /** + * Whether the recognizer for this language and mode is available, + * downloadable or downloading, answered synchronously from the last status + * `getModelInfo()` or `prepare()` saw. Before either runs, it checks what the + * mode needs: Android 12 for basic, AICore for advanced + */ + isAvailable(): boolean { + return ( + Platform.OS === 'android' && + getNativeAndroidAI().isTranscriptionAvailable(this.options) + ) + } + + /** Downloads the recognizer's model when needed, reporting download progress */ + async prepare(options: AndroidPrepareOptions = {}): Promise { + await withProgress('transcription', options, () => + getNativeAndroidAI().prepareTranscription(this.options) + ) + this.prepared = true + } + + async doGenerate(options: TranscriptionModelV4CallOptions) { + const native = getNativeAndroidAI() + const runId = generateId() + const abortSignal = options.abortSignal + + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + + if (!this.prepared) { + console.warn(notPreparedWarning) + await this.prepare() + } + + const onAbort = () => native.cancel(runId) + abortSignal?.addEventListener('abort', onAbort, { once: true }) + + try { + const text = await native.transcribe( + runId, + toAudioBase64(options.audio), + this.options + ) + + return { + text, + // Word timings only come with streamed audio, not a whole file + segments: [], + language: this.options.locale?.split(/[-_]/)[0], + durationInSeconds: undefined, + warnings: [], + response: { timestamp: new Date(), modelId: this.modelId }, + } + } catch (error) { + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + throw toAndroidAIError(error) + } finally { + abortSignal?.removeEventListener('abort', onAbort) + } + } +} + +class AndroidEmbeddingModel implements EmbeddingModelV4 { + readonly specificationVersion = 'v4' + readonly provider = 'android' + readonly modelId = 'embedding' + readonly maxEmbeddingsPerCall = Infinity + readonly supportsParallelCalls = false + + private readonly taskType: AndroidEmbeddingTaskType | undefined + private prepared = false + + constructor(options: AndroidEmbeddingOptions) { + this.taskType = options.taskType + } + + /** Embedding model availability, dimension and signature */ + async getModelInfo(): Promise { + if (Platform.OS !== 'android') { + return { availability: 'unavailable' } + } + try { + const info = await getNativeAndroidAI().getEmbeddingModelInfo() + return { ...info, availability: toModelAvailability(info.availability) } + } catch (error) { + throw toAndroidAIError(error) + } + } + + /** Downloads the embedding model when needed, reporting download progress */ + async prepare(options: AndroidPrepareOptions = {}): Promise { + await withProgress('embedding', options, () => + getNativeAndroidAI().prepareEmbedding() + ) + this.prepared = true + } + + async doEmbed( + options: EmbeddingModelV4CallOptions + ): Promise { + const abortSignal = options.abortSignal + // ML Kit cannot stop an embedding once it starts, so abort is checked around the call + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + if (!this.prepared) { + console.warn(notPreparedWarning) + await this.prepare() + } + let embeddings: number[][] + try { + embeddings = await getNativeAndroidAI().embed( + options.values, + this.taskType + ) + } catch (error) { + throw toAndroidAIError(error) + } + if (abortSignal?.aborted) { + throw getAbortReason(abortSignal) + } + return { embeddings, warnings: [] } + } +} diff --git a/packages/android/src/errors.ts b/packages/android/src/errors.ts new file mode 100644 index 00000000..f99f0692 --- /dev/null +++ b/packages/android/src/errors.ts @@ -0,0 +1,63 @@ +export type AndroidAIError = Error & { + code: AndroidAIErrorCode +} + +export const AndroidAIErrorCodes = { + UnsupportedPlatform: 'UNSUPPORTED_PLATFORM', + ModelUnavailable: 'MODEL_UNAVAILABLE', + ModelDownloading: 'MODEL_DOWNLOADING', + DownloadFailed: 'DOWNLOAD_FAILED', + FeatureNotSupported: 'FEATURE_NOT_SUPPORTED', + GenerationError: 'GENERATION_ERROR', + InvalidMessage: 'INVALID_MESSAGE', + InvalidSchema: 'INVALID_SCHEMA', + ContextWindowExceeded: 'CONTEXT_WINDOW_EXCEEDED', + RateLimited: 'RATE_LIMITED', + BackgroundUseBlocked: 'BACKGROUND_USE_BLOCKED', + MaxTurnsExceeded: 'MAX_TURNS_EXCEEDED', +} as const + +export type AndroidAIErrorCode = + (typeof AndroidAIErrorCodes)[keyof typeof AndroidAIErrorCodes] + +const androidAIErrorCodes = new Set(Object.values(AndroidAIErrorCodes)) + +export function isAndroidAIErrorCode( + value: unknown +): value is AndroidAIErrorCode { + return typeof value === 'string' && androidAIErrorCodes.has(value) +} + +export function createAndroidAIError( + message: string, + code: AndroidAIErrorCode, + cause?: unknown +): AndroidAIError { + const error = new Error(message, { cause }) as AndroidAIError + error.code = code + return error +} + +/** + * Wraps a native rejection or stream error so it carries an + * `AndroidAIErrorCode`. Errors without a known code become `GENERATION_ERROR` + */ +export function toAndroidAIError( + error: unknown, + code?: string +): AndroidAIError { + const nativeCode = + code ?? + (typeof error === 'object' && error !== null && 'code' in error + ? String((error as { code: unknown }).code) + : undefined) + const message = error instanceof Error ? error.message : String(error) + + return createAndroidAIError( + message, + isAndroidAIErrorCode(nativeCode) + ? nativeCode + : AndroidAIErrorCodes.GenerationError, + error + ) +} diff --git a/packages/android/src/index.ts b/packages/android/src/index.ts new file mode 100644 index 00000000..171cb69d --- /dev/null +++ b/packages/android/src/index.ts @@ -0,0 +1,20 @@ +export type { + AndroidDownloadProgress, + AndroidEmbeddingModelInfo, + AndroidEmbeddingOptions, + AndroidEmbeddingTaskType, + AndroidLanguageModel, + AndroidLanguageModelOptions, + AndroidModelAvailability, + AndroidModelFeature, + AndroidModelInfo, + AndroidNanoModel, + AndroidPrepareOptions, + AndroidProvider, + AndroidProviderOptions, + AndroidTranscriptionOptions, +} from './ai-sdk' +export { android, createAndroidProvider } from './ai-sdk' +export type { AndroidAIError, AndroidAIErrorCode } from './errors' +export { AndroidAIErrorCodes } from './errors' +export { default as AndroidAI, getNativeAndroidAI } from './NativeAndroidAI' diff --git a/packages/android/tsconfig.build.json b/packages/android/tsconfig.build.json new file mode 100644 index 00000000..acbe349f --- /dev/null +++ b/packages/android/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig", + "include": ["src/**/*"], + "exclude": ["**/__tests__/**"] +} diff --git a/packages/android/tsconfig.json b/packages/android/tsconfig.json new file mode 100644 index 00000000..54f5777b --- /dev/null +++ b/packages/android/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"], + "exclude": ["src/**/__tests__/**"] +} diff --git a/website/astro.config.mjs b/website/astro.config.mjs index cef56585..3b546c0d 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -94,6 +94,15 @@ export default defineConfig({ ['ios-versions', 'iOS versions'], ['running-on-simulator', 'Running on Simulator'], ]), + group('Android', 'android', [ + ['getting-started', 'Getting started'], + ['generating', 'Generating'], + ['tools', 'Tools'], + ['models', 'Models and availability'], + ['errors', 'Error handling'], + ['embeddings', 'Embeddings'], + ['transcription', 'Transcription'], + ]), group('ADK', 'adk', [ ['getting-started', 'Getting started'], ['generating', 'Generating'], diff --git a/website/src/content/docs/docs/android/embeddings.md b/website/src/content/docs/docs/android/embeddings.md new file mode 100644 index 00000000..7333e3dd --- /dev/null +++ b/website/src/content/docs/docs/android/embeddings.md @@ -0,0 +1,37 @@ +--- +title: "Embeddings" +--- + +Embed text on device with ML Kit's +[embedding model](https://developers.google.com/ml-kit/genai). It's an alpha +API, and not every device has a model. + +```typescript +import { android } from '@react-native-ai/android' +import { cosineSimilarity, embedMany } from 'ai' + +const model = android.embeddingModel({ taskType: 'semanticSimilarity' }) + +const { embeddings } = await embedMany({ + model, + values: ['The cat sleeps on the sofa.', 'A kitten naps on the couch.'], +}) + +console.log(cosineSimilarity(embeddings[0], embeddings[1])) +``` + +`taskType` tells the model what the embeddings are for: `retrievalQuery`, +`retrievalDocument`, `semanticSimilarity`, `classification`, `clustering`, +`questionAnswering`, `factVerification` or `codeRetrieval`. + +## Availability and Download + +```typescript +const info = await model.getModelInfo() +// { availability: 'available', dimension: 768, modelSignature: '...' } + +await model.prepare({ onProgress: (progress) => console.log(progress) }) +``` + +`availability` is `unavailable` when the device has no embedding model. +Embedding prepares the model on its own when you don't, and logs a warning. diff --git a/website/src/content/docs/docs/android/errors.md b/website/src/content/docs/docs/android/errors.md new file mode 100644 index 00000000..7d62cb14 --- /dev/null +++ b/website/src/content/docs/docs/android/errors.md @@ -0,0 +1,71 @@ +--- +title: "Error Handling" +--- + +Errors from ML Kit carry a stable `code` from `AndroidAIErrorCodes` you can +branch on. + +## Catching Errors + +```typescript +import { + AndroidAIErrorCodes, + type AndroidAIError, + android, +} from '@react-native-ai/android' +import { generateText, streamText } from 'ai' + +try { + await generateText({ + model: android(), + messages, + }) +} catch (error) { + const { code } = error as Partial + + if (code === AndroidAIErrorCodes.RateLimited) { + // Wait a few seconds and retry. + } + + throw error +} +``` + +With `streamText`, errors arrive as `error` parts in `stream`: + +```typescript +const result = streamText({ + model: android(), + messages, +}) + +for await (const part of result.stream) { + if (part.type === 'error') { + const { code } = part.error as Partial + + if (code === AndroidAIErrorCodes.ModelUnavailable) { + // Show fallback UI. + } + } +} +``` + +AICore rejects requests an app sends in quick succession, so space out +automated requests by a few seconds. + +## Error Codes + +| Code | Meaning | +| ------------------------- | ------------------------------------------------------------------- | +| `MODEL_UNAVAILABLE` | The device can't run the model, or needs an AICore or system update | +| `MODEL_DOWNLOADING` | The model is still downloading | +| `DOWNLOAD_FAILED` | Downloading the model failed | +| `FEATURE_NOT_SUPPORTED` | ML Kit reports the request needs a feature the model doesn't have | +| `RATE_LIMITED` | AICore is busy or the app used up its quota | +| `BACKGROUND_USE_BLOCKED` | AICore only serves apps in the foreground | +| `CONTEXT_WINDOW_EXCEEDED` | The request is longer than the context window | +| `INVALID_MESSAGE` | The prompt has something Gemini Nano can't take, such as audio | +| `INVALID_SCHEMA` | A tool's JSON Schema can't be converted | +| `MAX_TURNS_EXCEEDED` | ML Kit's limit on turns was reached | +| `UNSUPPORTED_PLATFORM` | The provider was called on a platform other than Android | +| `GENERATION_ERROR` | Any other failure | diff --git a/website/src/content/docs/docs/android/generating.md b/website/src/content/docs/docs/android/generating.md new file mode 100644 index 00000000..d64ea217 --- /dev/null +++ b/website/src/content/docs/docs/android/generating.md @@ -0,0 +1,142 @@ +--- +title: "Generating" +--- + +Generate text with Gemini Nano: pass `android()` to the AI SDK's `generateText` +or `streamText`. + +## Text Generation + +```typescript +import { android } from '@react-native-ai/android' +import { generateText } from 'ai' + +const { text } = await generateText({ + model: android(), + system: 'You are a helpful travel assistant.', + prompt: 'Suggest three things to do in Kraków.', +}) +``` + +System messages go to ML Kit as its system instruction. + +## Options + +`temperature`, `topK`, `seed` and `maxOutputTokens` go to ML Kit. `topP`, +`stopSequences`, `presencePenalty` and `frequencyPenalty` return an +`unsupported` warning. + +## Streaming + +```typescript +import { android } from '@react-native-ai/android' +import { streamText } from 'ai' + +const result = streamText({ + model: android(), + prompt: 'Write a short poem about the sea.', +}) + +for await (const delta of result.textStream) { + console.log(delta) +} +``` + +Pass an `abortSignal` to stop a generation. The native request is cancelled too. + +## Structured Output + +Not supported: `Output.object` and `responseFormat: { type: 'json' }` return an +`unsupported` warning, and the model answers in text. ML Kit only generates +structured output for Kotlin classes compiled with its KSP processor, not for a +JSON Schema at runtime. + +## Conversations + +Pass the conversation as `messages`. The provider sends the whole history to +Gemini Nano on every call, as user and model turns, so it keeps no session of +its own. Keep the history within the [context window](#context-window). + +:::note +Gemini Nano versions without system prompt support (the `systemPrompt` +feature), such as `nano-v2` on a Pixel 9 Pro, read only the first text part of +a request. On those, the provider merges the system instruction and the text +of every turn into one prompt, with the images first and each turn labeled by +role, so the model sees the whole conversation. Models with system prompt support get the instruction and +the turns separately. +::: + +```typescript +const { text } = await generateText({ + model: android(), + messages: [ + { role: 'user', content: 'My name is Ada.' }, + { role: 'assistant', content: 'Nice to meet you, Ada!' }, + { role: 'user', content: 'What is my name?' }, + ], +}) +``` + +## Image Prompts + +Gemini Nano takes up to 10 images per request. Pass bytes, base64, or a local +`file://` or `content://` URL: + +```typescript +const { text } = await generateText({ + model: android(), + messages: [ + { + role: 'user', + content: [ + { + type: 'file', + mediaType: 'image/jpeg', + data: new URL('file:///path/to/photo.jpg'), + }, + { type: 'text', text: 'What is in this photo?' }, + ], + }, + ], +}) +``` + +Other file types, such as audio, are rejected with `INVALID_MESSAGE`. + +## Reasoning + +Where the device's Gemini Nano supports thinking (`features` includes +`'thinking'`), any `reasoning` level other than `'none'` turns it on, and the +thoughts come back as reasoning parts. Gemini Nano's thinking is on or off, +with no budget. Without thinking support, the request runs without it and +returns an `unsupported` warning for `reasoning`. + +```typescript +const result = streamText({ + model: android(), + reasoning: 'high', + prompt: 'Is 17 a prime number?', +}) +``` + +`providerOptions.android.enableThinking` turns it on or off when `reasoning` is +not set. + +## Context Window + +The context size depends on the device's Gemini Nano, so read it at runtime: + +```typescript +const { contextSize } = await android.getModelInfo() +``` + +To keep long chats inside it, wrap the model with +`wrapLanguageModelWithHistory`, which keeps a rolling window and summarizes +older messages. See [Conversation History](/docs/history). + +ML Kit reports no token usage, so `usage` is empty. Count a prompt's tokens +with `countTokens`: + +```typescript +const tokens = await android().countTokens('How long is this prompt?') +``` diff --git a/website/src/content/docs/docs/android/getting-started.mdx b/website/src/content/docs/docs/android/getting-started.mdx new file mode 100644 index 00000000..7d4eea03 --- /dev/null +++ b/website/src/content/docs/docs/android/getting-started.mdx @@ -0,0 +1,95 @@ +--- +title: "Getting Started" +--- + +import PackageManagerTabs from '../../../../components/starlight/PackageManagerTabs.astro' + +Run Gemini Nano on Android, on the device, with the Vercel AI SDK. The provider +wraps Google's [ML Kit GenAI](https://developers.google.com/ml-kit/genai) libraries +directly. Android serves the models through AICore, so there's no model to +bundle and no API key. + +| Feature | Model | ML Kit API | +| ---------------------------------------------- | ------------------------------ | -------------------------- | +| [Text generation](/docs/android/generating) | `android()` | `genai-prompt` | +| [Tool calling](/docs/android/tools) | `android()`, on Gemini Nano 4 | `genai-prompt` | +| [Embeddings](/docs/android/embeddings) | `android.embeddingModel()` | `genai-embedding` | +| [Transcription](/docs/android/transcription) | `android.transcriptionModel()` | `genai-speech-recognition` | + +## Installation + + + +The provider works with AI SDK v7 and needs its [polyfills](/docs/polyfills). +Examples with schemas, such as the [tools](/docs/android/tools) examples, also +use `zod`. + +## Requirements + +- Android with `minSdkVersion` 26 or higher +- Kotlin 2.4 or newer, which the current ML Kit GenAI libraries are built with +- React Native 0.76 or newer, with the New Architecture +- A physical device with AICore support and a locked bootloader. See the + [ML Kit GenAI documentation](https://developers.google.com/ml-kit/genai#feature-device) + for the supported devices. Emulators and device clouds such as Firebase Test + Lab have no Gemini Nano. + +## Native Setup + +### Expo + +Install the config plugin: + +```bash +npx expo install expo-build-properties +``` + +Set `minSdkVersion` and the Kotlin version with `expo-build-properties`: + +```json +{ + "expo": { + "plugins": [ + [ + "expo-build-properties", + { + "android": { + "minSdkVersion": 26, + "kotlinVersion": "2.4.0" + } + } + ] + ] + } +} +``` + +Then regenerate the native project: + +```bash +npx expo prebuild --clean +``` + +### Bare React Native + +Set `minSdkVersion = 26` and `kotlinVersion = "2.4.0"` in the `ext` block of +`android/build.gradle`. + +## Basic Usage + +Pass `android()` to the AI SDK. Check that Gemini Nano can run on the device first: + +```typescript +import { android } from '@react-native-ai/android' +import { generateText } from 'ai' + +if (android.isAvailable()) { + const { text } = await generateText({ + model: android(), + prompt: 'Summarize on-device AI in one sentence.', + }) +} +``` + +[Models & Availability](/docs/android/models) covers the availability checks +and downloading Gemini Nano. diff --git a/website/src/content/docs/docs/android/models.md b/website/src/content/docs/docs/android/models.md new file mode 100644 index 00000000..0b02e75e --- /dev/null +++ b/website/src/content/docs/docs/android/models.md @@ -0,0 +1,76 @@ +--- +title: "Models & Availability" +--- + +Gemini Nano comes with AICore, Android's system service for on-device models. +AICore downloads it on first use and updates it on its own. + +## Checking Availability + +`isAvailable()` answers synchronously, from the last status `getModelInfo()` or +`prepare()` saw, or else from whether AICore is installed: + +```typescript +import { android } from '@react-native-ai/android' + +if (android.isAvailable()) { + // Offer on-device features +} +``` + +`getModelInfo()` checks with AICore: + +```typescript +const info = await android.getModelInfo() +// { +// availability: 'available', +// baseModelName: 'nano-v2', +// contextSize: 8192, +// features: ['systemPrompt', 'thinking', 'structuredOutput', 'functionCalling'], +// } +``` + +| `availability` | Meaning | +| --- | --- | +| `available` | Ready to use | +| `downloadable` | Supported, and downloads with `prepare()` | +| `downloading` | Downloading now | +| `unavailable` | Not supported on this device | +| `unknown` | AICore didn't answer | + +`baseModelName`, `contextSize` and `features` are only set once the model is +available. + +## Preparing + +`prepare()` downloads Gemini Nano when needed and loads it into memory, so the +first request answers sooner. Generating prepares on its own when you don't, and logs a warning. + +```typescript +await android().prepare({ + onProgress: ({ phase, bytesDownloaded, bytesToDownload }) => { + console.log(phase, bytesDownloaded, bytesToDownload) + }, +}) +``` + +## Choosing a Model + +ML Kit's `ModelConfig` picks between Gemini Nano releases: + +```typescript +import { createAndroidProvider } from '@react-native-ai/android' + +const preview = createAndroidProvider({ + model: { releaseStage: 'preview', preference: 'fast' }, +}) +``` + +- `releaseStage`: `stable` (default) or `preview`, the newest Gemini Nano for + devices in the AICore developer preview +- `preference`: `full` (default) favors quality, `fast` favors latency + +[Tool calling](/docs/android/tools) needs Gemini Nano 4: the preview release +with `fast`, or the default model on devices that ship it, such as the Galaxy Z +Flip 8 and Fold 8. +`getModelInfo` and `isAvailable` take the same selection. diff --git a/website/src/content/docs/docs/android/tools.md b/website/src/content/docs/docs/android/tools.md new file mode 100644 index 00000000..fc51f8c0 --- /dev/null +++ b/website/src/content/docs/docs/android/tools.md @@ -0,0 +1,78 @@ +--- +title: "Tools" +--- + +AI SDK tools run through ML Kit's +[function calling](https://developers.google.com/ml-kit/genai/prompt/android/function-calling). +Gemini Nano picks a function and its arguments, the AI SDK runs your tool's +`execute` in JavaScript, and sends the result back for the next step. + +ML Kit's agent library (`genai-agent`) only runs tools declared as Android App +Functions at build time, so AI SDK tools go through the Prompt API's function +calling instead. + +## Requirements + +Function calling needs Gemini Nano 4. On the Samsung Galaxy Z Flip 8 and Fold 8, +the default model is Gemini Nano 4. Elsewhere, ML Kit serves it as the preview +release with the fast preference: + +- Select the model with `releaseStage: 'preview'` and `preference: 'fast'`. +- The device's Google account must be in the + [AICore developer preview](https://developers.google.com/ml-kit/genai/prompt/android/function-calling), + and the AICore app must list "Gemini Nano 4 Fast". + +Check `features` in [`getModelInfo()`](/docs/android/models) for +`'functionCalling'`. Without it, ML Kit fails any request that declares +functions, so the provider sends the request without the tools and returns an +`unsupported` warning for `tools`. Gemini Nano then answers in text. + +## Calling Tools + +```typescript +import { createAndroidProvider } from '@react-native-ai/android' +import { generateText, isStepCount, tool } from 'ai' +import { z } from 'zod' + +const nano = createAndroidProvider({ + model: { releaseStage: 'preview', preference: 'fast' }, +}) + +const { text } = await generateText({ + model: nano(), + stopWhen: isStepCount(3), + tools: { + getWeather: tool({ + description: 'Gets the current weather in a city.', + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }) => ({ city, condition: 'sunny', temperatureC: 24 }), + }), + }, + prompt: 'What is the weather in Paris?', +}) +``` + +Without `stopWhen`, `generateText` stops after the tool call, with +`finishReason: 'tool-calls'`. + +Tool calls stream as whole `tool-call` parts, not argument by argument, since +ML Kit returns each call complete. + +## Schemas + +Tool input schemas are converted to ML Kit's `FunctionSchema`: `type` (with +`null` for nullable), `description`, `properties`, `required`, `enum`, +`format`, `items`, `anyOf`, `title` and the length, item, property and number +limits. Other keywords, such as `$ref` and `oneOf`, are left out. + +## Tool Choice + +`toolChoice: 'none'` sends no tools. `'auto'` is the default. `'required'` and a +specific tool return an `unsupported` warning, since Gemini Nano decides itself +whether to call a function. + +## Tool Results + +Results go back to Gemini Nano as function responses, which are JSON objects: +an object result is sent as is, other values as `{ result }`, and errors and +denied approvals as `{ error }`. diff --git a/website/src/content/docs/docs/android/transcription.md b/website/src/content/docs/docs/android/transcription.md new file mode 100644 index 00000000..f2021309 --- /dev/null +++ b/website/src/content/docs/docs/android/transcription.md @@ -0,0 +1,49 @@ +--- +title: "Transcription" +--- + +Convert speech to text on device with ML Kit's +[speech recognition](https://developers.google.com/ml-kit/genai/speech-recognition/android). + +```typescript +import { android } from '@react-native-ai/android' +import { experimental_transcribe as transcribe } from 'ai' + +const model = android.transcriptionModel({ language: 'en-US' }) + +const { text } = await transcribe({ + model, + audio, // Uint8Array or base64 of an MP3, WAV, FLAC, M4A or OGG file +}) +``` + +ML Kit decodes the file on the device. It returns the text only: word timings +come with streamed audio, which this model doesn't take yet, so `segments` is +empty. + +## Options + +- `language`: BCP 47 locale, such as `en-US`. Defaults to the device locale. +- `mode`: `basic` uses the traditional on-device recognizer (Android 12 or + newer, about 15 locales). `advanced` uses Gemini Nano, with more locales and + better quality, on newer devices such as the Pixel 10. ML Kit picks one when + you don't set it. + +## Availability and Download + +`isAvailable()` answers synchronously, from the last status `getModelInfo()` +or `prepare()` saw. Before either has run, it checks what the mode needs: +Android 12 for `basic`, AICore for `advanced`. + +```typescript +if (model.isAvailable()) { + // available, downloadable or downloading +} + +const info = await model.getModelInfo() +// { availability: 'available' | 'downloadable' | 'downloading' | 'unavailable' } + +await model.prepare({ onProgress: (progress) => console.log(progress) }) +``` + +Transcribing prepares the model on its own when you don't, and logs a warning. diff --git a/website/src/content/docs/docs/index.md b/website/src/content/docs/docs/index.md index fe3a3679..f4de5d69 100644 --- a/website/src/content/docs/docs/index.md +++ b/website/src/content/docs/docs/index.md @@ -28,6 +28,7 @@ const result = streamText({ | ---------------------------------------------- | --------------- | ------------------------------------------------------ | ------------------------------------------ | | [Apple](/docs/apple/getting-started) | iOS | Built in: Apple Intelligence, speech and embeddings | Native, provided by the platform | | [Core AI](/docs/core-ai/getting-started) | iOS 27 | Yours: language, image, speech and vision models | Native, provided by the platform | +| [Android](/docs/android/getting-started) | Android | Built in: Gemini Nano, speech recognition, embeddings | Native, provided by the platform | | [ADK](/docs/adk/getting-started) | Android | Built in: Gemini Nano, or cloud Gemini | Native, provided by the platform | | [Llama](/docs/llama/getting-started) | iOS and Android | Yours: GGUF models from Hugging Face | Third-party binding, llama.rn |