Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ keep the panel mounted without receiving telemetry.
| [Android](#android) | ✅ Yes | Android | [ML Kit GenAI](https://developers.google.com/ml-kit/genai) | On-device Gemini Nano, transcription and embeddings |
| [Core AI](#core-ai) | ❌ No | iOS | [Core AI](https://github.com/apple/coreai-models) | Text, image, video, speech, embedding and vision models |
| [Llama](#llama) | ❌ No | iOS, Android | [llama.rn](https://github.com/mybigday/llama.rn) | Run GGUF models via llama.rn |
| [LiteRT-LM](#litert-lm) | ❌ No | iOS, Android | [react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm) | Run `.litertlm` models via react-native-litert-lm |

---

Expand Down Expand Up @@ -275,6 +276,40 @@ Any GGUF model from HuggingFace can be used. Use the format `owner/repo/filename

📚 **[View full Llama documentation →](https://react-native-ai.dev/docs/llama/getting-started)**

---

### LiteRT-LM

Run `.litertlm` models on-device using [LiteRT-LM](https://ai.google.dev/edge/litert-lm) through [react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm). **Requires download** - models are downloaded from HuggingFace.

#### Installation

```bash
npm install @react-native-ai/litert react-native-litert-lm react-native-nitro-modules
```

#### Usage

```typescript
import { litert } from '@react-native-ai/litert'
import { generateText } from 'ai'

const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm')

// Download once and load into memory
await model.prepare()

const { text } = await generateText({
model,
prompt: 'Write a haiku about coding.',
})

// Cleanup when done
await model.unload()
```

📚 **[View full LiteRT-LM documentation →](https://react-native-ai.dev/docs/litert/getting-started)**

## JSON UI

`@react-native-ai/json-ui` lets a tool-calling model build React Native UI. The model calls tools that add, update, and delete nodes in a JSON spec, and `GenerativeUIView` renders it. It's designed for small on-device models with limited context.
Expand Down
25 changes: 25 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

38 changes: 38 additions & 0 deletions packages/litert/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# LiteRT-LM Provider for Vercel AI SDK

A Vercel AI SDK provider for [LiteRT-LM](https://ai.google.dev/edge/litert-lm), Google's runtime for LLMs on the device. It runs `.litertlm` models on iOS and Android through [react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm).

**Requirements:**
- react-native-litert-lm >= 0.8.2
- Vercel AI SDK v7

```ts
import { litert } from '@react-native-ai/litert'
import { generateText } from 'ai'

const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm')
await model.prepare()

const { text } = await generateText({
model,
prompt: 'Write a haiku about coding.',
})
```

## Features

- ✅ Text generation with `.litertlm` models from Hugging Face, a URL or a file
- ✅ Streaming
- ✅ Tool calling
- ✅ Structured output (`Output.object`), enforced by the engine
- ✅ Multi-turn conversations that continue in the engine, so the history isn't sent again
- ✅ Model download with progress
- ✅ GPU and CPU backends. NPU with a bundle compiled for the phone's chip, not yet verified on a device

## Documentation

For the complete guide, see the [LiteRT-LM documentation](https://react-native-ai.dev/docs/litert/getting-started).

## License

MIT
73 changes: 73 additions & 0 deletions packages/litert/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"name": "@react-native-ai/litert",
"version": "0.12.0",
"description": "LiteRT-LM provider for Vercel AI SDK",
"main": "lib/commonjs/index",
"module": "lib/module/index",
"types": "lib/typescript/index.d.ts",
"react-native": "src/index",
"source": "src/index",
"files": [
"src",
"lib",
"!**/__tests__",
"!**/__fixtures__",
"!**/__mocks__",
"!**/.*"
],
"license": "MIT",
"homepage": "https://github.com/callstackincubator/ai#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/callstackincubator/ai.git"
},
"bugs": {
"url": "https://github.com/callstackincubator/ai/issues"
},
"publishConfig": {
"registry": "https://registry.npmjs.org/"
},
"scripts": {
"clean": "del-cli lib",
"typecheck": "tsc --noEmit",
"test": "bun test",
"prepare": "bob build"
},
"dependencies": {
"@ai-sdk/provider": "^4.0.33"
},
"peerDependencies": {
"react-native": ">=0.76.0",
"react-native-litert-lm": ">=0.8.2"
},
"devDependencies": {
"ai": "^7.0.0",
"react-native": "0.88.0",
"react-native-litert-lm": "0.8.2",
"zod": "^4.0.0"
},
"react-native-builder-bob": {
"source": "src",
"output": "lib",
"targets": [
"commonjs",
"module",
[
"typescript",
{
"project": "tsconfig.build.json"
}
]
]
},
"keywords": [
"react-native",
"litert",
"litert-lm",
"llm",
"on-device",
"ai",
"sdk",
"vercel"
]
}
136 changes: 136 additions & 0 deletions packages/litert/src/__tests__/fakeBinding.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import type { LiteRTLMInstance } from 'react-native-litert-lm'

import type { LiteRTBinding } from '../runtime'

function cancelledError(): Error {
return Object.assign(new Error('The message was cancelled.'), {
code: 'INFERENCE_CANCELLED',
})
}

export interface ScriptedTurn {
/** Tokens the engine streams, in order. */
tokens: string[]
/** Rejects after the tokens, the way the engine reports a failed stream. */
failWith?: Error
/** Leaves the message generating until it is cancelled. */
hangAfterTokens?: boolean
}

/** A scripted stand-in for the phone's LiteRT-LM binding that records what it was asked. */
export class FakeBinding {
turns: ScriptedTurn[] = []
loads: { path: string; config: Record<string, any>; options: any }[] = []
resets: { historyJson?: string; systemPrompt?: string }[] = []
executes: { parts: any[]; options: any }[] = []
loadError: Error | undefined
closes = 0
unloads = 0
instancesCreated = 0
manifest: any = null
platform: 'android' | 'ios' = 'android'
manifestCalls: any[] = []
keptFiles: string[] = []
/** False once the system has taken the engine's memory back. */
ready = true
/** Holds a load until it is called, like a long download. */
holdLoad: Promise<void> | undefined

private cancel: (() => void) | undefined

/** The system takes the engine's memory back: a message in progress ends as cancelled. */
freeMemory(): void {
this.ready = false
this.cancel?.()
}

binding: LiteRTBinding = {
createLLM: () => {
this.instancesCreated += 1
return this.instance()
},
getDeviceInfo: (() => ({
platform: this.platform,
})) as LiteRTBinding['getDeviceInfo'],
listKeptFiles: () => this.keptFiles,
resolveFromManifest: (async (repo: string, options: unknown) => {
this.manifestCalls.push({ repo, options })
return this.manifest
}) as LiteRTBinding['resolveFromManifest'],
toolResultPart: (name, response) => ({
type: 'toolResult',
name,
responseJson: JSON.stringify(response) ?? 'null',
}),
}

private instance(): LiteRTLMInstance {
const self = this
return {
async loadModel(path: string, config: Record<string, any>, options: any) {
self.loads.push({ path, config, options })
self.ready = true
options?.onProgress?.(0.5)
if (self.holdLoad !== undefined) await self.holdLoad
if (self.loadError !== undefined) {
const error = self.loadError
self.loadError = undefined
throw error
}
},
resetConversation(historyJson?: string, systemPrompt?: string) {
self.resets.push({ historyJson, systemPrompt })
self.cancel?.()
},
execute(
parts: any[],
onToken: (token: string, done: boolean) => void,
options: any
) {
self.executes.push({ parts, options })
const turn = self.turns.shift()
if (turn === undefined)
return Promise.reject(new Error('No scripted turn left.'))
return new Promise<string>((resolve, reject) => {
self.cancel = () => {
self.cancel = undefined
onToken('', true)
reject(cancelledError())
}
void (async () => {
for (const token of turn.tokens) {
await Promise.resolve()
onToken(token, false)
}
if (turn.hangAfterTokens) return
self.cancel = undefined
if (turn.failWith !== undefined) {
onToken(`Error: ${turn.failWith.message}`, true)
reject(turn.failWith)
return
}
onToken('', true)
resolve(turn.tokens.join(''))
})()
})
},
isReady: () => self.ready,
getStats: () => ({
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
timeToFirstToken: 120,
totalTime: 500,
tokensPerSecond: 42,
}),
countTokens: () => -1,
getContextTokenCount: () => -1,
async unload() {
self.unloads += 1
},
close() {
self.closes += 1
},
} as unknown as LiteRTLMInstance
}
}
Loading