Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions apps/example/src/adapters/apple.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,14 @@ export function createAppleLanguageSetupAdapter(): SetupAdapter<LanguageModelV4>
// Image prompts need iOS 27
imageInput:
Platform.OS === 'ios' && parseInt(String(Platform.Version), 10) >= 27,
isAvailable() {
return Platform.OS === 'ios' && apple.isAvailable() ? 'yes' : 'no'
async isAvailable() {
if (Platform.OS !== 'ios') return 'no'
try {
const { availability } = await apple.getModelInfo()
return availability === 'available' ? 'yes' : 'no'
} catch {
return 'no'
}
},
async download() {},
async delete() {},
Expand Down
2 changes: 1 addition & 1 deletion apps/example/src/adapters/llama.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export function createLlamaLanguageSetupAdapter({
sizeBytes,
builtIn: false,
imageInput: Boolean(projectorId),
isAvailable(): Availability {
async isAvailable(): Promise<Availability> {
return files.every(isDownloaded) ? 'yes' : 'availableForDownload'
},
async download(onProgress) {
Expand Down
2 changes: 1 addition & 1 deletion apps/example/src/adapters/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export interface SetupAdapter<TModel> {
// Whether the model accepts images, omitted means text only
imageInput?: boolean
// Check if model is ready, unavailable, or downloadable
isAvailable: () => Availability
isAvailable: () => Promise<Availability>
// Download the model with progress callback
download: (onProgress: (percentage: number) => void) => Promise<void>
// Remove the downloaded model from storage
Expand Down
7 changes: 4 additions & 3 deletions apps/example/src/components/FeatureRequirements.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import type { Capability, Runtime } from '../adapters/types'
import { isSimulator } from '../simulator'
import type { FeatureId } from '../store/features'
import {
isTranscriptionAvailable,
useEmbeddingsStatus,
useEnabledFeatures,
useTranscriptionAvailable,
} from '../store/features'
import {
chatModels,
Expand Down Expand Up @@ -128,6 +128,7 @@ function EmbeddingsRow() {
// What a feature needs, with a way to get each missing piece
export function FeatureRequirements({ feature }: { feature: FeatureId }) {
const { isEnabled } = useEnabledFeatures()
const transcriptionAvailable = useTranscriptionAvailable()
switch (feature) {
case 'chat':
return (
Expand All @@ -145,11 +146,11 @@ export function FeatureRequirements({ feature }: { feature: FeatureId }) {
subtitle={
isSimulator
? 'Simulator preview: hears a scripted question'
: isTranscriptionAvailable()
: transcriptionAvailable
? 'Transcribes on device as you speak'
: 'Not available for your language'
}
state={isTranscriptionAvailable() ? 'ready' : 'unavailable'}
state={transcriptionAvailable ? 'ready' : 'unavailable'}
/>
<RequirementRow
symbol="speaker.wave.2"
Expand Down
22 changes: 17 additions & 5 deletions apps/example/src/store/features.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { apple, AppleEmbeddings, AppleUtils } from '@react-native-ai/apple'
import type { SFSymbol } from 'expo-symbols'
import { useAtom, useAtomValue, useSetAtom } from 'jotai'
import { atomWithRefresh, loadable } from 'jotai/utils'
import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai'
import { atomWithRefresh, loadable, unwrap } from 'jotai/utils'
import { Platform } from 'react-native'

import { isSimulator } from '../simulator'
Expand Down Expand Up @@ -88,9 +88,21 @@ function deviceLanguage() {
return Platform.OS === 'ios' ? AppleUtils.getCurrentLocale() : 'en-US'
}

export function isTranscriptionAvailable() {
if (isSimulator) return true
return Platform.OS === 'ios' && apple.transcriptionModel().isAvailable()
const transcriptionAvailableAtom = unwrap(
atom(async () => {
if (isSimulator) return true
if (Platform.OS !== 'ios') return false
try {
return (await apple.transcriptionModel().getModelInfo()).isAvailable
} catch {
return false
}
}),
() => false
)

export function useTranscriptionAvailable() {
return useAtomValue(transcriptionAvailableAtom)
}

// Apple's embedding assets ship separately and download on first prepare
Expand Down
19 changes: 15 additions & 4 deletions apps/example/src/store/models.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { LanguageModelV4 } from '@ai-sdk/provider'
import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai'
import { atomWithRefresh } from 'jotai/utils'
import { atomWithRefresh, unwrap } from 'jotai/utils'

import { createAppleLanguageSetupAdapter } from '../adapters/apple'
import { createLlamaLanguageSetupAdapter } from '../adapters/llama'
Expand Down Expand Up @@ -44,12 +44,23 @@ export function formatSize(bytes: number) {
}

const availabilityAtom = atomWithRefresh(
() =>
async () =>
new Map<string, Availability>(
chatModels.map((adapter) => [adapter.modelId, adapter.isAvailable()])
await Promise.all(
chatModels.map(
async (adapter) =>
[adapter.modelId, await adapter.isAvailable()] as const
)
)
)
)

// Keeps the last result while a refresh is in flight
const availabilityValueAtom = unwrap(
availabilityAtom,
(prev) => prev ?? new Map<string, Availability>()
)

const downloadProgressAtom = atom<Record<string, number>>({})

const selectedModelIdAtom = atomWithPersistence<string | null>(
Expand All @@ -63,7 +74,7 @@ let modelTransition = Promise.resolve()
let activeModel: SetupAdapter<LanguageModelV4> | undefined

export function useModels() {
const availability = useAtomValue(availabilityAtom)
const availability = useAtomValue(availabilityValueAtom)
const refreshAvailability = useSetAtom(availabilityAtom)
const setDownloadProgress = useSetAtom(downloadProgressAtom)
const [selectedModelId, setSelectedModelId] = useAtom(selectedModelIdAtom)
Expand Down
5 changes: 3 additions & 2 deletions apps/example/src/store/readiness.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
import type { FeatureId } from './features'
import { isTranscriptionAvailable, useEmbeddingsStatus } from './features'
import { useEmbeddingsStatus, useTranscriptionAvailable } from './features'
import { useModels } from './models'

// Whether each feature has what it needs on this device right now
export function useFeatureReadiness(): Record<FeatureId, boolean> {
const { readyModels } = useModels()
const { status: embeddingsStatus } = useEmbeddingsStatus()
const transcriptionAvailable = useTranscriptionAvailable()
const hasChatModel = readyModels.length > 0

return {
chat: hasChatModel,
// Voice answers with the chat model
voice: hasChatModel && isTranscriptionAvailable(),
voice: hasChatModel && transcriptionAvailable,
search: embeddingsStatus === 'ready',
}
}
1 change: 0 additions & 1 deletion harness/android/availability.harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ describe('android availability', () => {
expect(info.availability).toBe('available')
expect(info.contextSize).toBeGreaterThan(0)
expect(info.features).toBeDefined()
expect(android.isAvailable()).toBe(true)
})

it('reports Gemini Nano 4 preview details', async () => {
Expand Down
4 changes: 0 additions & 4 deletions harness/apple/availability.harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@ import { apple, AppleUtils } from '@react-native-ai/apple'
import { describe, expect, it } from 'react-native-harness'

describe('apple availability', () => {
it('is available', () => {
expect(apple.isAvailable()).toBe(true)
})

it('describes the on-device model', async () => {
const info = await apple.getModelInfo()

Expand Down
6 changes: 0 additions & 6 deletions harness/apple/transcription.device.harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,6 @@ beforeAll(async () => {
})

describe('apple transcription', () => {
it('is available for english', () => {
expect(apple.transcriptionModel({ language: 'en-US' }).isAvailable()).toBe(
true
)
})

it('describes the locale a language resolves to', async () => {
const model = apple.transcriptionModel({ language: 'en' })
await model.prepare()
Expand Down
4 changes: 0 additions & 4 deletions packages/android/android/src/main/AndroidManifest.xml
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.callstack.ai.android">
<!-- Lets isAvailable() check whether AICore, which runs Gemini Nano, is installed -->
<queries>
<package android:name="com.google.android.aicore" />
</queries>
</manifest>
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp
putString("arguments", FunctionJson.argumentsToJson(arguments).toString())
}

override fun isAvailable(model: ReadableMap?): Boolean = nano.isAvailable(NanoConfig.from(model))

override fun getModelInfo(model: ReadableMap?, promise: Promise) =
launch(promise) { nano.getModelInfo(NanoConfig.from(model)).toWritableMap() }

Expand Down Expand Up @@ -162,9 +160,6 @@ class AndroidAIModule(reactContext: ReactApplicationContext) : NativeAndroidAISp
activeRuns.remove(runId)?.cancel()
}

override fun isTranscriptionAvailable(options: ReadableMap): Boolean =
transcription.isAvailable(TranscriptionConfig.from(options))

override fun getTranscriptionModelInfo(options: ReadableMap, promise: Promise) =
launch(promise) { transcription.getModelInfo(TranscriptionConfig.from(options)).toWritableMap() }

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.callstack.ai.android

import android.content.Context
import android.content.pm.PackageManager
import android.net.Uri
import android.util.Base64
import com.facebook.react.bridge.ReadableArray
Expand Down Expand Up @@ -31,16 +30,6 @@ import kotlinx.coroutines.sync.withLock
import org.json.JSONObject
import java.util.concurrent.ConcurrentHashMap

private const val AICORE_PACKAGE = "com.google.android.aicore"

fun isAICoreInstalled(context: Context): Boolean =
try {
context.packageManager.getPackageInfo(AICORE_PACKAGE, 0)
true
} catch (_: PackageManager.NameNotFoundException) {
false
}

/** Gemini Nano model selection, see ML Kit `ModelConfig`. */
data class NanoConfig(
val releaseStage: String? = null,
Expand Down Expand Up @@ -99,8 +88,6 @@ class GeminiNano(
private val prepared = ConcurrentHashMap.newKeySet<NanoConfig>()
// One lock per model selection, so one download does not hold up another selection
private val prepareMutexes = ConcurrentHashMap<NanoConfig, Mutex>()
// The last status seen per model selection, for the synchronous check
private val statuses = ConcurrentHashMap<NanoConfig, Int>()
private val systemPromptSupport = ConcurrentHashMap<NanoConfig, Boolean>()

private fun client(config: NanoConfig): GenerativeModel =
Expand All @@ -123,17 +110,6 @@ class GeminiNano(
}
}

/**
* Whether Gemini Nano is available, downloadable or downloading, answered without async work: from the last
* status this process saw, or else from whether AICore is installed.
*/
fun isAvailable(config: NanoConfig): Boolean {
statuses[config]?.let {
return it == FeatureStatus.AVAILABLE || it == FeatureStatus.DOWNLOADABLE || it == FeatureStatus.DOWNLOADING
}
return isAICoreInstalled(context)
}

suspend fun getModelInfo(config: NanoConfig): ModelInfo {
val model = client(config)
val status =
Expand All @@ -142,7 +118,6 @@ class GeminiNano(
} catch (_: Exception) {
return ModelInfo(availability = "unknown")
}
statuses[config] = status
if (status != FeatureStatus.AVAILABLE) {
return ModelInfo(availability = availabilityOf(status))
}
Expand Down Expand Up @@ -172,9 +147,7 @@ class GeminiNano(
if (config in prepared) return

val model = client(config)
val status = model.checkStatus()
statuses[config] = status
when (status) {
when (val status = model.checkStatus()) {
FeatureStatus.UNAVAILABLE ->
throw AndroidAIErrors.CodedException(
AndroidAIErrors.MODEL_UNAVAILABLE,
Expand All @@ -185,7 +158,6 @@ class GeminiNano(
}
model.warmup()
prepared.add(config)
statuses[config] = FeatureStatus.AVAILABLE
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.callstack.ai.android

import android.content.Context
import android.os.Build
import android.util.Base64
import com.facebook.react.bridge.ReadableMap
import com.google.mlkit.genai.common.FeatureStatus
Expand Down Expand Up @@ -32,8 +31,6 @@ class Transcription(
) {
private val clients = ConcurrentHashMap<TranscriptionConfig, SpeechRecognizer>()
private val prepared = ConcurrentHashMap.newKeySet<TranscriptionConfig>()
// The last status seen per language and mode, for the synchronous check
private val statuses = ConcurrentHashMap<TranscriptionConfig, Int>()
// One lock per language and mode, so one download does not hold up another
private val prepareMutexes = ConcurrentHashMap<TranscriptionConfig, Mutex>()

Expand All @@ -58,23 +55,8 @@ class Transcription(
"Not a BCP 47 language tag: $tag. Use a tag such as en-US",
)

/**
* Whether the recognizer is available, downloadable or downloading, answered without async work: from the last
* status this process saw, or else from what the mode needs (Android 12 for basic, AICore for advanced).
*/
fun isAvailable(config: TranscriptionConfig): Boolean {
statuses[config]?.let {
return it == FeatureStatus.AVAILABLE || it == FeatureStatus.DOWNLOADABLE || it == FeatureStatus.DOWNLOADING
}
return Build.VERSION.SDK_INT >= Build.VERSION_CODES.S &&
(config.mode != "advanced" || isAICoreInstalled(context))
}

suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo {
val status = client(config).checkStatus()
statuses[config] = status
return ModelInfo(availability = availabilityOf(status))
}
suspend fun getModelInfo(config: TranscriptionConfig): ModelInfo =
ModelInfo(availability = availabilityOf(client(config).checkStatus()))

/** Downloads the recognizer's model when needed, reporting progress, then warms it up. */
suspend fun prepare(config: TranscriptionConfig) {
Expand All @@ -83,9 +65,7 @@ class Transcription(
if (config in prepared) return

val recognizer = client(config)
val status = recognizer.checkStatus()
statuses[config] = status
when (status) {
when (val status = recognizer.checkStatus()) {
FeatureStatus.UNAVAILABLE ->
throw AndroidAIErrors.CodedException(
AndroidAIErrors.MODEL_UNAVAILABLE,
Expand All @@ -96,7 +76,6 @@ class Transcription(
}
recognizer.warmup()
prepared.add(config)
statuses[config] = FeatureStatus.AVAILABLE
}
}

Expand Down
4 changes: 0 additions & 4 deletions packages/android/src/NativeAndroidAI.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,6 @@ export interface NativeEmbeddingModelInfo {
}

export interface Spec extends TurboModule {
/** Quick synchronous check, see `AndroidProvider.isAvailable` */
isAvailable(model?: NanoModelConfig): boolean
getModelInfo(model?: NanoModelConfig): Promise<NativeModelInfo>
prepare(model?: NanoModelConfig): Promise<void>
countTokens(text: string, model?: NanoModelConfig): Promise<number>
Expand All @@ -126,8 +124,6 @@ export interface Spec extends TurboModule {
/** Cancels a `generateText`, `streamText` or `transcribe` run by its id */
cancel(runId: string): void

/** Quick synchronous check, see `AndroidTranscriptionModel.isAvailable` */
isTranscriptionAvailable(options: NativeTranscriptionOptions): boolean
getTranscriptionModelInfo(
options: NativeTranscriptionOptions
): Promise<NativeModelInfo>
Expand Down
Loading
Loading