From 6994a5a909eb1f6268c9ebf83de03b084481b658 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Thu, 29 Oct 2026 22:24:54 +0100 Subject: [PATCH 1/3] chore(adk): run adk harness tests on android Add the android runner to the example app's harness, with the ADK provider and its packaging options, and test Gemini Nano on a physical device. HARNESS_ANDROID_DEVICE="Pixel 9 Pro" bun run test:harness:android /harness/adk/ Co-Authored-By: Claude Opus 5.5 --- apps/example/app.json | 5 +- apps/example/jest.harness.config.mjs | 17 ++-- apps/example/package.json | 5 +- apps/example/rn-harness.config.mjs | 20 ++++- apps/example/scripts/test-harness.sh | 14 +++- apps/example/tsconfig.json | 3 + bun.lock | 6 ++ harness/adk/availability.harness.ts | 29 +++++++ harness/adk/generate.harness.ts | 111 +++++++++++++++++++++++++++ harness/adk/helpers.ts | 24 ++++++ 10 files changed, 225 insertions(+), 9 deletions(-) create mode 100644 harness/adk/availability.harness.ts create mode 100644 harness/adk/generate.harness.ts create mode 100644 harness/adk/helpers.ts diff --git a/apps/example/app.json b/apps/example/app.json index 0e2fa15c..739a492b 100644 --- a/apps/example/app.json +++ b/apps/example/app.json @@ -35,7 +35,10 @@ "expo-build-properties", { "android": { - "minSdkVersion": 26 + "minSdkVersion": 26, + "packagingOptions": { + "exclude": ["META-INF/INDEX.LIST", "META-INF/DEPENDENCIES"] + } } } ], diff --git a/apps/example/jest.harness.config.mjs b/apps/example/jest.harness.config.mjs index f688529e..3d407ae0 100644 --- a/apps/example/jest.harness.config.mjs +++ b/apps/example/jest.harness.config.mjs @@ -1,13 +1,20 @@ +const ignored = [ + // Tests that need a physical device are left out on simulators + ...(process.env.HARNESS_SIMULATOR === 'true' + ? ['\\.device\\.harness\\.ts$'] + : []), + // Each platform leaves out the providers that only run on the other one + ...(process.env.HARNESS_PLATFORM === 'android' + ? ['/harness/apple/', '/harness/core-ai/', '/harness/json-ui/apple\\.'] + : ['/harness/adk/']), +] + // The tests live in the harness folder at the monorepo root export default { preset: 'react-native-harness', roots: ['/../../harness'], testMatch: ['**/*.harness.ts'], - // Tests that need a physical device are left out on simulators - testPathIgnorePatterns: - process.env.HARNESS_SIMULATOR === 'true' - ? ['\\.device\\.harness\\.ts$'] - : [], + testPathIgnorePatterns: ignored, setupFiles: ['/../../harness/setup.ts'], // The test files are few, so a crawl is fast and does not depend on Watchman watchman: false, diff --git a/apps/example/package.json b/apps/example/package.json index be6616a3..44a7621c 100644 --- a/apps/example/package.json +++ b/apps/example/package.json @@ -8,7 +8,8 @@ "ios": "expo run:ios", "typecheck": "tsc --noEmit", "test:harness:simulator": "scripts/test-harness.sh simulator", - "test:harness:device": "scripts/test-harness.sh device" + "test:harness:device": "scripts/test-harness.sh device", + "test:harness:android": "scripts/test-harness.sh android" }, "dependencies": { "@ai-sdk/provider": "^4.0.33", @@ -16,6 +17,7 @@ "@expo/ui": "~58.0.23", "@expo/vector-icons": "^15.0.3", "@legendapp/list": "^3.7.0", + "@react-native-ai/adk": "workspace:*", "@react-native-ai/apple": "workspace:*", "@react-native-ai/core-ai": "workspace:*", "@react-native-ai/json-ui": "workspace:*", @@ -55,6 +57,7 @@ }, "devDependencies": { "@babel/core": "^7.20.0", + "@react-native-harness/platform-android": "1.6.0", "@react-native-harness/platform-apple": "1.6.0", "@types/react": "~19.3.0", "@types/ungap__structured-clone": "^1.2.0", diff --git a/apps/example/rn-harness.config.mjs b/apps/example/rn-harness.config.mjs index 7c838321..60e7be2a 100644 --- a/apps/example/rn-harness.config.mjs +++ b/apps/example/rn-harness.config.mjs @@ -1,3 +1,7 @@ +import { + androidPlatform, + physicalAndroidDevice, +} from '@react-native-harness/platform-android' import { applePhysicalDevice, applePlatform, @@ -8,9 +12,14 @@ const deviceName = process.env.HARNESS_IOS_DEVICE // Builds signed with entitlements the default App ID lacks use another one const bundleId = process.env.HARNESS_IOS_BUNDLE_ID ?? 'com.callstack.ai.example' +const androidDevice = process.env.HARNESS_ANDROID_DEVICE +const androidManufacturer = process.env.HARNESS_ANDROID_MANUFACTURER ?? 'Google' + // Tests run in a debug build of the app. The ios runner uses the physical // device named with HARNESS_IOS_DEVICE (as `xcrun devicectl list devices` -// shows it), and ios-simulator a simulator +// shows it), and ios-simulator a simulator. The android runner uses the +// physical device whose model is HARNESS_ANDROID_DEVICE (as +// `adb shell getprop ro.product.model` shows it, such as Pixel 9 Pro) export default { entryPoint: './index.js', appRegistryComponentName: 'main', @@ -29,6 +38,15 @@ export default { device: appleSimulator('iPhone 18 Pro', '27.0'), bundleId: 'com.callstack.ai.example', }), + ...(androidDevice + ? [ + androidPlatform({ + name: 'android', + device: physicalAndroidDevice(androidManufacturer, androidDevice), + bundleId: 'com.callstack.ai.example', + }), + ] + : []), ], defaultRunner: deviceName ? 'ios' : 'ios-simulator', // On-device models load and generate slowly, but a test that hangs should diff --git a/apps/example/scripts/test-harness.sh b/apps/example/scripts/test-harness.sh index 9789eb03..c2555567 100755 --- a/apps/example/scripts/test-harness.sh +++ b/apps/example/scripts/test-harness.sh @@ -5,6 +5,7 @@ # # bun run test:harness:simulator [test path] # HARNESS_IOS_DEVICE="My iPhone" bun run test:harness:device [test path] +# HARNESS_ANDROID_DEVICE="Pixel 9 Pro" bun run test:harness:android [test path] set -euo pipefail @@ -35,8 +36,19 @@ case "$TARGET" in npx expo run:ios --device "$HARNESS_IOS_DEVICE" --no-bundler npx react-native-harness --harnessRunner ios "$@" ;; + android) + if [[ -z "${HARNESS_ANDROID_DEVICE:-}" ]]; then + echo 'Set HARNESS_ANDROID_DEVICE to the device model, as `adb shell getprop ro.product.model` shows it' >&2 + exit 1 + fi + # Installs on the connected device, or the one ANDROID_SERIAL names when + # more than one is connected + npx expo prebuild --clean --platform android + npx expo run:android --no-bundler ${ANDROID_SERIAL:+--device "$ANDROID_SERIAL"} + HARNESS_PLATFORM=android npx react-native-harness --harnessRunner android "$@" + ;; *) - echo "Usage: $0 simulator|device [test path]" >&2 + echo "Usage: $0 simulator|device|android [test path]" >&2 exit 1 ;; esac diff --git a/apps/example/tsconfig.json b/apps/example/tsconfig.json index d1f25d04..64bf36d4 100644 --- a/apps/example/tsconfig.json +++ b/apps/example/tsconfig.json @@ -8,6 +8,9 @@ "" ], "paths": { + "@react-native-ai/adk": [ + "../../packages/adk/src" + ], "@react-native-ai/apple": [ "../../packages/apple-llm/src" ], diff --git a/bun.lock b/bun.lock index 77a4f4a5..ef91d68f 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ "@expo/ui": "~58.0.23", "@expo/vector-icons": "^15.0.3", "@legendapp/list": "^3.7.0", + "@react-native-ai/adk": "workspace:*", "@react-native-ai/apple": "workspace:*", "@react-native-ai/core-ai": "workspace:*", "@react-native-ai/json-ui": "workspace:*", @@ -78,6 +79,7 @@ }, "devDependencies": { "@babel/core": "^7.20.0", + "@react-native-harness/platform-android": "1.6.0", "@react-native-harness/platform-apple": "1.6.0", "@types/react": "~19.3.0", "@types/ungap__structured-clone": "^1.2.0", @@ -972,6 +974,8 @@ "@react-native-harness/metro": ["@react-native-harness/metro@1.6.0", "", { "dependencies": { "tslib": "^2.3.0" }, "peerDependencies": { "metro": "*" } }, "sha512-FbV/eAMQZkzPv3kZF/vlHM5AFY898vyRxLzSQiwjLeWiX0pa56wH/H/3FAjOfxlo3MSxeeUxaSGYNO6ZCt5wOA=="], + "@react-native-harness/platform-android": ["@react-native-harness/platform-android@1.6.0", "", { "dependencies": { "@react-native-harness/config": "1.6.0", "@react-native-harness/platforms": "1.6.0", "@react-native-harness/tools": "1.6.0", "tslib": "^2.3.0", "vite": "^7.2.2", "zod": "^3.25.67" } }, "sha512-Wfv3GrJLAMdHtlHd5pGgIZlmc2Aj1NSphAprKLL5GAhFxQJld4mYw5QdruwT5Kn92Vj31BAeKuKAqBq1vU7BPg=="], + "@react-native-harness/platform-apple": ["@react-native-harness/platform-apple@1.6.0", "", { "dependencies": { "@react-native-harness/config": "1.6.0", "@react-native-harness/platforms": "1.6.0", "@react-native-harness/tools": "1.6.0", "tslib": "^2.3.0", "yargs": "^17.7.2", "zod": "^3.25.67" } }, "sha512-2LanhJpnke56A6A2tMakBV58PyaXvOIZzlLI7eJFGSTCFdkCvN8nzE2H0MpzW6pqpfzlSfpw3iIxhydFJhRcLg=="], "@react-native-harness/platforms": ["@react-native-harness/platforms@1.6.0", "", { "dependencies": { "@react-native-harness/tools": "1.6.0", "tslib": "^2.3.0" } }, "sha512-k6LJUBam9UCURgIOicPIg4PY+ms+u8g1AFiIGYIpS3rNVCCk07qH9EupdjVGX0BfueYcqDVghjrZ6h+PVNriUg=="], @@ -3398,6 +3402,8 @@ "@react-native-harness/jest/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], + "@react-native-harness/platform-android/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + "@react-native-harness/platform-apple/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], "@react-native-harness/platform-apple/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], diff --git a/harness/adk/availability.harness.ts b/harness/adk/availability.harness.ts new file mode 100644 index 00000000..b8028d37 --- /dev/null +++ b/harness/adk/availability.harness.ts @@ -0,0 +1,29 @@ +import { adk } from '@react-native-ai/adk' +import { describe, expect, it } from 'react-native-harness' + +// Gemini Nano runs only on physical devices with AICore, such as a Pixel 9 +describe('adk availability', () => { + it('reports Gemini Nano details', async () => { + const info = await adk.getModelInfo() + + expect(['available', 'downloadable', 'downloading']).toContain( + info.availability + ) + }) + + // On a fresh device this downloads Gemini Nano, which takes minutes + it('prepares Gemini Nano with download progress', async () => { + await adk().prepare({ onProgress: () => {} }) + + const info = await adk.getModelInfo() + expect(info.availability).toBe('available') + expect(info.contextSize).toBeGreaterThan(0) + expect(await adk.isAvailable()).toBe(true) + }) + + it('counts tokens with the Gemini Nano tokenizer', async () => { + expect(await adk().countTokens('How many tokens is this?')).toBeGreaterThan( + 0 + ) + }) +}) diff --git a/harness/adk/generate.harness.ts b/harness/adk/generate.harness.ts new file mode 100644 index 00000000..40b8d0f2 --- /dev/null +++ b/harness/adk/generate.harness.ts @@ -0,0 +1,111 @@ +import { adk, createAdkProvider } from '@react-native-ai/adk' +import { generateText, jsonSchema, streamText, tool } from 'ai' +import { + beforeAll, + beforeEach, + describe, + expect, + it, +} from 'react-native-harness' + +import { cooldown, prompt } from './helpers' + +describe('adk gemini nano', () => { + beforeAll(async () => { + await adk().prepare() + }) + beforeEach(cooldown) + + it('answers a prompt', async () => { + const result = await adk().doGenerate({ + prompt: prompt('Reply with exactly the word BANANA and nothing else.'), + }) + + expect(result.content.at(-1)).toMatchObject({ + type: 'text', + text: expect.stringMatching(/banana/i), + }) + expect(result.finishReason.unified).toBe('stop') + }) + + it('follows the system message', async () => { + const { text } = await generateText({ + model: adk(), + system: 'Whatever the user says, reply with only the word PINEAPPLE.', + prompt: 'How are you?', + }) + + expect(text).toMatch(/pineapple/i) + }) + + it('follows the provider instruction', async () => { + const model = createAdkProvider({ + instruction: 'Whatever the user says, reply with only the word MANGO.', + })() + + const { text } = await generateText({ model, prompt: 'How are you?' }) + + expect(text).toMatch(/mango/i) + }) + + it('remembers earlier turns', async () => { + const { text } = await generateText({ + model: adk(), + messages: [ + { role: 'user', content: 'My favourite colour is teal. Just say OK.' }, + { role: 'assistant', content: 'OK' }, + { + role: 'user', + content: 'What is my favourite colour? Answer with one word.', + }, + ], + }) + + expect(text).toMatch(/teal/i) + }) + + it('streams text', async () => { + const result = streamText({ + model: adk(), + prompt: 'Count from 1 to 5, separated by spaces.', + }) + + let deltas = 0 + for await (const delta of result.textStream) { + if (delta) deltas++ + } + expect(await result.text).toMatch(/1.*2.*3.*4.*5/s) + expect(deltas).toBeGreaterThan(0) + }) + + it('warns that tools are unsupported', async () => { + const { warnings } = await generateText({ + model: adk(), + prompt: 'What is the weather in Paris?', + tools: { + getWeather: tool({ + inputSchema: jsonSchema<{ city: string }>({ type: 'object' }), + execute: async () => 'Sunny', + }), + }, + }) + + expect(warnings).toContainEqual( + expect.objectContaining({ type: 'unsupported', feature: 'tools' }) + ) + }) + + it('warns that JSON output is unsupported', async () => { + const { warnings } = await adk().doGenerate({ + prompt: prompt('Describe a cat.'), + responseFormat: { type: 'json' }, + }) + + expect(warnings).toContainEqual( + expect.objectContaining({ + type: 'unsupported', + feature: 'responseFormat', + }) + ) + }) +}) diff --git a/harness/adk/helpers.ts b/harness/adk/helpers.ts new file mode 100644 index 00000000..41ca4547 --- /dev/null +++ b/harness/adk/helpers.ts @@ -0,0 +1,24 @@ +import type { + LanguageModelV4CallOptions, + LanguageModelV4StreamPart, +} from '@ai-sdk/provider' + +export function prompt(text: string): LanguageModelV4CallOptions['prompt'] { + return [{ role: 'user', content: [{ type: 'text', text }] }] +} + +export async function readAll( + stream: ReadableStream +) { + const parts: LanguageModelV4StreamPart[] = [] + const reader = stream.getReader() + while (true) { + const { done, value } = await reader.read() + if (done) return parts + parts.push(value) + } +} + +// AICore rejects requests an app sends in quick succession with a quota error +export const cooldown = () => + new Promise((resolve) => setTimeout(resolve, 3000)) From 9ea1cadc1ff82c68117711f1b69ae7c5bf77b802 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Thu, 29 Oct 2026 23:15:58 +0100 Subject: [PATCH 2/3] fix(example): install and run the android harness on the same device Resolve the device once: the one ANDROID_SERIAL names, or the only connected one, and check it is the model Harness runs on. Co-Authored-By: Claude Opus 5.5 --- apps/example/scripts/test-harness.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/example/scripts/test-harness.sh b/apps/example/scripts/test-harness.sh index c2555567..9c5212ef 100755 --- a/apps/example/scripts/test-harness.sh +++ b/apps/example/scripts/test-harness.sh @@ -41,10 +41,24 @@ case "$TARGET" in echo 'Set HARNESS_ANDROID_DEVICE to the device model, as `adb shell getprop ro.product.model` shows it' >&2 exit 1 fi - # Installs on the connected device, or the one ANDROID_SERIAL names when - # more than one is connected + # Harness runs the tests on the device of that model, so the build is + # installed on that same device: the one ANDROID_SERIAL names, or the only + # connected one + if [[ -z "${ANDROID_SERIAL:-}" ]]; then + serials="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1 }')" + if [[ "$(grep -c . <<< "$serials")" != 1 ]]; then + echo 'Connect one device, or set ANDROID_SERIAL to the one to test on' >&2 + exit 1 + fi + export ANDROID_SERIAL="$serials" + fi + model="$(adb -s "$ANDROID_SERIAL" shell getprop ro.product.model | tr -d '\r')" + if [[ "$model" != "$HARNESS_ANDROID_DEVICE" ]]; then + echo "$ANDROID_SERIAL is a $model, not a $HARNESS_ANDROID_DEVICE" >&2 + exit 1 + fi npx expo prebuild --clean --platform android - npx expo run:android --no-bundler ${ANDROID_SERIAL:+--device "$ANDROID_SERIAL"} + npx expo run:android --no-bundler --device "$ANDROID_SERIAL" HARNESS_PLATFORM=android npx react-native-harness --harnessRunner android "$@" ;; *) From d841ee9e9be03e511f6349fac067717f3a982a87 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Thu, 29 Oct 2026 23:20:16 +0100 Subject: [PATCH 3/3] fix(example): require exactly one android device of the harness model Harness runs on the first connected device of the model and ignores ANDROID_SERIAL, so two such devices could still split the install and the test run. Co-Authored-By: Claude Opus 5.5 --- apps/example/scripts/test-harness.sh | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/apps/example/scripts/test-harness.sh b/apps/example/scripts/test-harness.sh index 9c5212ef..267f5195 100755 --- a/apps/example/scripts/test-harness.sh +++ b/apps/example/scripts/test-harness.sh @@ -41,22 +41,21 @@ case "$TARGET" in echo 'Set HARNESS_ANDROID_DEVICE to the device model, as `adb shell getprop ro.product.model` shows it' >&2 exit 1 fi - # Harness runs the tests on the device of that model, so the build is - # installed on that same device: the one ANDROID_SERIAL names, or the only - # connected one - if [[ -z "${ANDROID_SERIAL:-}" ]]; then - serials="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1 }')" - if [[ "$(grep -c . <<< "$serials")" != 1 ]]; then - echo 'Connect one device, or set ANDROID_SERIAL to the one to test on' >&2 - exit 1 + # Harness runs the tests on the first connected device of that model and + # ignores ANDROID_SERIAL, so exactly one may be connected, and the build is + # installed on it + matching=() + for serial in $(adb devices | awk 'NR > 1 && $2 == "device" { print $1 }'); do + model="$(adb -s "$serial" shell getprop ro.product.model | tr -d '\r')" + if [[ "$model" == "$HARNESS_ANDROID_DEVICE" ]]; then + matching+=("$serial") fi - export ANDROID_SERIAL="$serials" - fi - model="$(adb -s "$ANDROID_SERIAL" shell getprop ro.product.model | tr -d '\r')" - if [[ "$model" != "$HARNESS_ANDROID_DEVICE" ]]; then - echo "$ANDROID_SERIAL is a $model, not a $HARNESS_ANDROID_DEVICE" >&2 + done + if [[ "${#matching[@]}" != 1 ]]; then + echo "Connect exactly one $HARNESS_ANDROID_DEVICE, found ${#matching[@]}" >&2 exit 1 fi + export ANDROID_SERIAL="${matching[0]}" npx expo prebuild --clean --platform android npx expo run:android --no-bundler --device "$ANDROID_SERIAL" HARNESS_PLATFORM=android npx react-native-harness --harnessRunner android "$@"