diff --git a/__tests__/harness/nativeBoundary.ts b/__tests__/harness/nativeBoundary.ts
index fe03c10f0..50ff2bbe1 100644
--- a/__tests__/harness/nativeBoundary.ts
+++ b/__tests__/harness/nativeBoundary.ts
@@ -273,6 +273,8 @@ function makeLiteRTFake(handle: FakeEmitterHandle): LiteRTFake {
}),
stopGeneration: jest.fn().mockResolvedValue(undefined),
unloadModel: jest.fn().mockResolvedValue(undefined),
+ // Not a Google Tensor phone unless a test says otherwise (e.g. a Pixel 10: supported, generation 5).
+ getTpuSupport: jest.fn().mockResolvedValue({ supported: false, tensorGeneration: null, reason: 'not_tensor' }),
getMemoryInfo: jest.fn().mockResolvedValue({
totalRamMb: 12000,
usedRamMb: 4000,
diff --git a/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx b/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx
index 1510dc4de..1da2c2a01 100644
--- a/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx
+++ b/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx
@@ -117,6 +117,7 @@ jest.mock('../../../src/services/hardware', () => ({
getTotalMemoryGB: () => mockGetTotalMemoryGB(),
getModelRecommendation: jest.fn(() => ({ maxParameters: 14, recommendedQuantization: 'Q4_K_M', recommendedModels: [], warning: undefined })),
getImageModelRecommendation: jest.fn(() => Promise.resolve({ recommendedBackend: 'mnn', maxModelSizeMB: 2048, canRunSD: true, canRunQNN: false })),
+ getTensorTpuGeneration: jest.fn(() => Promise.resolve(null)), // not a Pixel Tensor TPU
},
}));
diff --git a/__tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx b/__tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx
new file mode 100644
index 000000000..a104f7af3
--- /dev/null
+++ b/__tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx
@@ -0,0 +1,43 @@
+/**
+ * Pixel 10 (Tensor G5): its TPU runs only LiteRT builds compiled for it, and its GPU has no working
+ * LiteRT path, so onboarding offers the Tensor G5 build there — and nowhere else, since no other phone
+ * can run it.
+ *
+ * Real AdvancedSetupScreen + real hardwareService + curated registry; fakes ONLY the native boundary
+ * (the LiteRT module's TPU probe answers like a Pixel 10, or like any other phone).
+ */
+import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary';
+
+jest.mock('@react-navigation/native', () => ({
+ useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {}, replace: () => {} }),
+ useRoute: () => ({ params: {} }),
+ useFocusEffect: () => {}, useIsFocused: () => true,
+}));
+
+async function renderOnboarding(tpu: { supported: boolean; tensorGeneration: number | null; reason: string | null }) {
+ const boundary = installNativeBoundary({ ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB } });
+ boundary.litert.module.getTpuSupport.mockResolvedValue(tpu);
+ const React = require('react');
+ const rtl = requireRTL();
+ const { hardwareService } = require('../../../src/services/hardware');
+ const { AdvancedSetupScreen } = require('../../../src/screens/ModelDownloadScreen');
+ await hardwareService.getDeviceInfo();
+ const nav: any = { navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {}, replace: () => {} };
+ const view = rtl.render(React.createElement(AdvancedSetupScreen, { navigation: nav }));
+ await rtl.waitFor(() => { expect(view.getByTestId('onboarding-litert-model-1')).toBeTruthy(); }, { timeout: 10000 });
+ return { rtl, view };
+}
+
+describe('Tensor TPU LiteRT build in onboarding', () => {
+ it('offers the Tensor G5 build on a Pixel 10 whose TPU LiteRT can reach', async () => {
+ const { rtl, view } = await renderOnboarding({ supported: true, tensorGeneration: 5, reason: null });
+ const card = await rtl.waitFor(() => view.getByTestId('onboarding-litert-model-2'), { timeout: 10000 });
+ expect(rtl.within(card).getByText('Gemma 4 E2B (Tensor TPU)')).toBeTruthy();
+ }, 60000);
+
+ it('does not offer it on a phone without a Tensor G5 TPU', async () => {
+ const { view } = await renderOnboarding({ supported: false, tensorGeneration: null, reason: 'not_tensor' });
+ expect(view.queryByText('Gemma 4 E2B (Tensor TPU)')).toBeNull();
+ expect(view.queryByTestId('onboarding-litert-model-2')).toBeNull();
+ }, 60000);
+});
diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx
index 84a19af5b..e0ddfc465 100644
--- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx
+++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx
@@ -10,6 +10,113 @@ it('uses GPU at the native boundary when an older build saved NPU', async () =>
]);
});
+const TENSOR_G5_FILE = '/models/gemma-4-E2B-it_Google_Tensor_G5.litertlm';
+const PIXEL_10_TPU = { supported: true, tensorGeneration: 5, reason: null };
+
+it('loads a Tensor G5 build on the TPU of a phone with a Tensor G5 TPU, whatever the saved backend', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ const { liteRTService } = require('../../../src/services/litert');
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu');
+ expect(boundary.litert.calls.loadModel).toEqual([[TENSOR_G5_FILE, 'npu', false, false, 4096]]);
+});
+
+it('keeps a portable LiteRT file off the NPU even on a phone with a Tensor TPU', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ const { liteRTService } = require('../../../src/services/litert');
+ await liteRTService.loadModel('/models/gemma-4-E2B-it.litertlm', 'npu');
+ expect(boundary.litert.calls.loadModel).toEqual([['/models/gemma-4-E2B-it.litertlm', 'gpu', false, false, 4096]]);
+});
+
+it('does not send a Tensor G5 build to the NPU when the phone has no usable Tensor G5 TPU', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue({ supported: false, tensorGeneration: 5, reason: 'dispatch_lib_missing' });
+ const { liteRTService } = require('../../../src/services/litert');
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'cpu');
+ expect(boundary.litert.calls.loadModel).toEqual([[TENSOR_G5_FILE, 'cpu', false, false, 4096]]);
+});
+
+it('adopts the media the native engine came up with when a TPU load drops to text-only', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ boundary.litert.module.loadModel.mockResolvedValueOnce({ backend: 'npu', maxNumTokens: 4096, vision: false, audio: false });
+ const { liteRTService } = require('../../../src/services/litert');
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true, supportsAudio: true });
+ expect(liteRTService.getActiveBackend()).toBe('npu');
+ expect(liteRTService.loadedTextOnly(TENSOR_G5_FILE)).toBe(true);
+ expect(liteRTService.supportsAudio()).toBe(false);
+});
+
+it('refuses an image for a vision model whose TPU engine came up text-only, before any generation path', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ boundary.litert.module.loadModel.mockResolvedValueOnce({ backend: 'npu', maxNumTokens: 4096, vision: false, audio: false });
+ const { liteRTService } = require('../../../src/services/litert');
+ const { localModelAcceptsImages, activeLocalTextCapabilities } = require('../../../src/services/engines');
+ const model = createDownloadedModel({ id: 'g5', engine: 'litert', filePath: TENSOR_G5_FILE, fileName: 'gemma-4-E2B-it_Google_Tensor_G5.litertlm', liteRTVision: true });
+ expect(localModelAcceptsImages(model)).toBe(true); // not loaded yet: the model's own flag decides
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true });
+ expect(localModelAcceptsImages(model)).toBe(false);
+ expect(activeLocalTextCapabilities(model).vision).toBe(false);
+ await liteRTService.unloadModel();
+ expect(localModelAcceptsImages(model)).toBe(true);
+});
+
+it('refuses an image at the engine when the TPU load came up text-only, on the first turn after a lazy load', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ boundary.litert.module.loadModel.mockResolvedValueOnce({ backend: 'npu', maxNumTokens: 4096, vision: false, audio: false });
+ const { liteRTService } = require('../../../src/services/litert');
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true });
+ // The tool loop's entry point (generateRaw) and the plain path (sendMessage) both refuse.
+ await expect(liteRTService.generateRaw('what is this?', { imageUris: ['file:///pic.png'] }))
+ .rejects.toThrow(/Images are not available for this model on this device/);
+ const onError = jest.fn();
+ await liteRTService.sendMessage('what is this?', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError }, { imageUris: ['file:///pic.png'] });
+ expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringMatching(/Images are not available/) }));
+ expect(boundary.litert.calls.sendMessageWithImages).toEqual([]);
+ // Audio on an engine that came up without it would reach the model as empty text: refused too.
+ await expect(liteRTService.generateRaw('', { audioUris: ['file:///note.wav'] }))
+ .rejects.toThrow(/Audio input is not available for this model on this device/);
+ expect(boundary.litert.calls.sendMessageWithMedia).toEqual([]);
+});
+
+it('reports a spent TPU context as context-full, so the chat compacts and retries instead of failing', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ const { liteRTService } = require('../../../src/services/litert');
+ const { contextCompactionService } = require('../../../src/services/contextCompaction');
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu');
+ // LiteRT-LM's TPU executor refuses a prefill that would pass the context compiled into the model.
+ boundary.litert.scriptError('Prefill length (128) plus current step (3990) exceeds max sequence length (4096).');
+ const onError = jest.fn();
+ await liteRTService.sendMessage('hi', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError });
+ await new Promise(resolve => setTimeout(resolve, 0));
+ expect(onError).toHaveBeenCalledTimes(1);
+ expect(contextCompactionService.isContextFullError(onError.mock.calls[0][0])).toBe(true);
+});
+
+it('rebuilds the native conversation from the compacted history when the chat retries a full context', async () => {
+ const boundary = installNativeBoundary();
+ boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU);
+ const { liteRTService } = require('../../../src/services/litert');
+ await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu');
+ const history = [
+ { role: 'user', content: 'first question' }, { role: 'assistant', content: 'first answer' },
+ { role: 'user', content: 'second question' }, { role: 'assistant', content: 'second answer' },
+ ];
+ await liteRTService.prepareConversation('chat-1', 'sys', { history });
+ boundary.litert.scriptError('Status Code: 3. Message: Prefill length (128) plus current step (3990) exceeds max sequence length (4096).');
+ await liteRTService.sendMessage('third question', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError: jest.fn() });
+ await new Promise(resolve => setTimeout(resolve, 0));
+ // The native conversation still holds the refused turn, so the retry must not reuse it.
+ const compacted = [{ role: 'user', content: 'Summary: two questions answered.' }, { role: 'assistant', content: 'Noted.' }];
+ await liteRTService.prepareConversation('chat-1', 'sys', { history: compacted });
+ const historyJson = boundary.litert.calls.resetConversation.at(-1)?.[5];
+ expect(historyJson).toBe(JSON.stringify(compacted));
+});
+
it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => {
installNativeBoundary();
const React = require('react');
diff --git a/__tests__/rntl/navigation/AppNavigator.test.tsx b/__tests__/rntl/navigation/AppNavigator.test.tsx
index 1d87fd43a..b3942edef 100644
--- a/__tests__/rntl/navigation/AppNavigator.test.tsx
+++ b/__tests__/rntl/navigation/AppNavigator.test.tsx
@@ -95,6 +95,7 @@ jest.mock('../../../src/services/hardware', () => ({
})),
formatBytes: jest.fn((bytes: number) => `${(bytes / 1024 / 1024 / 1024).toFixed(1)} GB`),
formatModelSize: jest.fn(() => '4.0 GB'),
+ getTensorTpuGeneration: jest.fn(() => Promise.resolve(null)), // not a Pixel Tensor TPU
},
}));
diff --git a/__tests__/unit/hooks/useChatModelActions.test.ts b/__tests__/unit/hooks/useChatModelActions.test.ts
index eb08852bd..862559ea0 100644
--- a/__tests__/unit/hooks/useChatModelActions.test.ts
+++ b/__tests__/unit/hooks/useChatModelActions.test.ts
@@ -44,7 +44,7 @@ jest.mock('../../../src/services/llm', () => ({
// LiteRT boundary mocked as a dumb flag reader so engines.isModelReady/deriveEngineCapabilities
// (which read this DIRECT module, not the barrel) resolve deterministically for the litert path.
jest.mock('../../../src/services/litert', () => ({
- liteRTService: { isModelLoaded: jest.fn(() => false) },
+ liteRTService: { isModelLoaded: jest.fn(() => false), loadedTextOnly: jest.fn(() => false) },
}));
// Get mock references after hoisting
diff --git a/__tests__/unit/hooks/useChatModelStateSync.test.ts b/__tests__/unit/hooks/useChatModelStateSync.test.ts
index 028387032..667d3a654 100644
--- a/__tests__/unit/hooks/useChatModelStateSync.test.ts
+++ b/__tests__/unit/hooks/useChatModelStateSync.test.ts
@@ -18,7 +18,7 @@ jest.mock('../../../src/services/llm', () => ({
},
}));
jest.mock('../../../src/services/litert', () => ({
- liteRTService: { isModelLoaded: jest.fn(() => false) },
+ liteRTService: { isModelLoaded: jest.fn(() => false), loadedTextOnly: jest.fn(() => false) },
}));
const { llmService } = require('../../../src/services/llm');
diff --git a/__tests__/unit/services/activeModelService.loaders.branches.test.ts b/__tests__/unit/services/activeModelService.loaders.branches.test.ts
index 5d896fdb2..4844a30bb 100644
--- a/__tests__/unit/services/activeModelService.loaders.branches.test.ts
+++ b/__tests__/unit/services/activeModelService.loaders.branches.test.ts
@@ -190,6 +190,20 @@ describe('doLoadLiteRTModel branches', () => {
expect(ToastAndroid.showWithGravity).toHaveBeenCalled();
});
+ it('caps the context at a curated build\'s own limit (the Tensor TPU build is compiled for 4096 tokens)', async () => {
+ mockedLiteRT.loadModel.mockResolvedValue(undefined);
+ mockedLiteRT.getActiveBackend.mockReturnValue('npu');
+ const fileName = 'gemma-4-E2B-it_Google_Tensor_G5.litertlm';
+ const ctx = liteCtx({
+ model: { id: `offgrid/litert-recommended/${fileName}`, engine: 'litert', fileName, filePath: `/models/${fileName}`, fileSize: 3113545589 },
+ store: { settings: { ...makeStore().settings, liteRTBackend: 'gpu', liteRTMaxTokens: 8192 } },
+ });
+
+ await doLoadTextModel(ctx);
+
+ expect(mockedLiteRT.loadModel).toHaveBeenCalledWith(`/models/${fileName}`, 'gpu', expect.objectContaining({ maxNumTokens: 4096 }));
+ });
+
it('runs warmup when active backend is gpu/npu', async () => {
mockedLiteRT.loadModel.mockResolvedValue(undefined);
mockedLiteRT.getActiveBackend.mockReturnValue('npu');
diff --git a/__tests__/unit/services/contextCompaction.test.ts b/__tests__/unit/services/contextCompaction.test.ts
index 39273b3a3..747618864 100644
--- a/__tests__/unit/services/contextCompaction.test.ts
+++ b/__tests__/unit/services/contextCompaction.test.ts
@@ -73,6 +73,11 @@ describe('isContextFullError', () => {
['context window exceeded', true],
['context length exceeded', true],
['context is full', true],
+ // LiteRT-LM's own limit errors, as the native module forwards them
+ ['Max number of tokens reached, context window out of bounds', true],
+ ['Input token ids are too long. Exceeding the maximum number of tokens allowed: 4100 >= 4096', true],
+ ['Reached maximum number of tokens.', true],
+ ['Generation failed: Prefill length (128) plus current step (3990) exceeds max sequence length (4096).', true],
])('"%s" → %s', (msg, expected) => {
const input = typeof msg === 'string' ? new Error(msg) : msg;
expect(contextCompactionService.isContextFullError(input)).toBe(expected);
diff --git a/__tests__/unit/services/curatedLiteRTContextLimit.test.ts b/__tests__/unit/services/curatedLiteRTContextLimit.test.ts
index 80f09a415..49621ba3f 100644
--- a/__tests__/unit/services/curatedLiteRTContextLimit.test.ts
+++ b/__tests__/unit/services/curatedLiteRTContextLimit.test.ts
@@ -6,11 +6,16 @@ it.each(CURATED_LITERT_ENTRIES)('matches the pinned artifact for $displayName wi
engine: 'litert', id: `${LITERT_PARENT_ID}/${entry.fileName}`,
fileName: entry.fileName, fileSize: entry.sizeBytes,
});
- expect(getCuratedLiteRTContextLimit(model)).toBe(32000);
+ expect(getCuratedLiteRTContextLimit(model)).toBe(entry.maxContextTokens);
expect(getCuratedLiteRTContextLimit({ ...model, id: 'imported' })).toBeNull();
expect(getCuratedLiteRTContextLimit({ ...model, fileSize: 1 })).toBeNull();
const origin = { repoId: entry.hfRepoId, revision: entry.commitHash, path: entry.fileName };
- expect(getCuratedLiteRTContextLimit({ ...model, id: 'transferred', origin })).toBe(32000);
+ expect(getCuratedLiteRTContextLimit({ ...model, id: 'transferred', origin })).toBe(entry.maxContextTokens);
expect(getCuratedLiteRTContextLimit({ ...model, origin: { ...origin, revision: 'main' } })).toBeNull();
expect(getCuratedLiteRTContextLimit({ ...model, engine: 'llama' })).toBeNull();
});
+
+it('caps the Tensor TPU build at its compiled 4096-token KV cache', () => {
+ const entry = CURATED_LITERT_ENTRIES.find(e => e.fileName === 'gemma-4-E2B-it_Google_Tensor_G5.litertlm')!;
+ expect(entry.maxContextTokens).toBe(4096);
+});
diff --git a/__tests__/unit/services/curatedLiteRTTensorTpu.test.ts b/__tests__/unit/services/curatedLiteRTTensorTpu.test.ts
new file mode 100644
index 000000000..e0cbadc68
--- /dev/null
+++ b/__tests__/unit/services/curatedLiteRTTensorTpu.test.ts
@@ -0,0 +1,36 @@
+/**
+ * A LiteRT file compiled for a Google Tensor TPU runs only on that generation's TPU, so the
+ * curated list shows it only there (Models tab + onboarding share liteRTFileRunsOnDevice).
+ * The target generation comes from Google's file naming, read by liteRTTensorTarget.
+ */
+import { CURATED_LITERT_ENTRIES, liteRTFileRunsOnDevice } from '../../../src/services/curatedLiteRTRegistry';
+import { liteRTTensorTarget } from '../../../src/utils/modelHelpers';
+
+const tensorEntry = CURATED_LITERT_ENTRIES.find(e => liteRTTensorTarget(e.fileName) !== null);
+
+describe('Tensor TPU LiteRT builds', () => {
+ it('reads the Tensor generation from Google’s file naming', () => {
+ expect(liteRTTensorTarget('gemma-4-E2B-it_Google_Tensor_G5.litertlm')).toBe(5);
+ expect(liteRTTensorTarget('Gemma3-1B-IT_google_tensor_g4.litertlm')).toBe(4);
+ expect(liteRTTensorTarget('gemma-4-E2B-it.litertlm')).toBeNull();
+ expect(liteRTTensorTarget('gemma-4-E2B-it_Google_Tensor_G5.gguf')).toBeNull();
+ });
+
+ it('curates a Tensor G5 build (fixture ground-truth)', () => {
+ expect(tensorEntry?.fileName).toBe('gemma-4-E2B-it_Google_Tensor_G5.litertlm');
+ });
+
+ it('lists the Tensor G5 build only on a phone whose TPU is Tensor G5', () => {
+ const name = tensorEntry!.fileName;
+ expect(liteRTFileRunsOnDevice(name, 5)).toBe(true);
+ expect(liteRTFileRunsOnDevice(name, 4)).toBe(false);
+ expect(liteRTFileRunsOnDevice(name, null)).toBe(false);
+ });
+
+ it('lists portable builds on every phone', () => {
+ for (const entry of CURATED_LITERT_ENTRIES.filter(e => e !== tensorEntry)) {
+ expect(liteRTFileRunsOnDevice(entry.fileName, null)).toBe(true);
+ expect(liteRTFileRunsOnDevice(entry.fileName, 5)).toBe(true);
+ }
+ });
+});
diff --git a/__tests__/unit/services/generationToolLoop.branches.test.ts b/__tests__/unit/services/generationToolLoop.branches.test.ts
index e8385c8da..0a69d617c 100644
--- a/__tests__/unit/services/generationToolLoop.branches.test.ts
+++ b/__tests__/unit/services/generationToolLoop.branches.test.ts
@@ -46,6 +46,7 @@ jest.mock('../../../src/stores', () => ({
addMessage: mockAddMessage,
setStreamingMessage: mockSetStreamingMessage,
setIsThinking: mockSetIsThinking,
+ conversations: [],
}),
},
useRemoteServerStore: { getState: () => ({ activeServerId: null }) },
@@ -400,6 +401,21 @@ describe('runToolLoop — LiteRT loop branches', () => {
expect(ctx.onFinalResponse).toHaveBeenCalledWith('recovered answer');
});
+ it('leaves a full context (also Status Code 3) to compaction instead of dropping the tools', async () => {
+ mockedLiteRT.generateRaw.mockRejectedValue(new Error(
+ 'Status Code: 3. Message: Prefill length (128) plus current step (3990) exceeds max sequence length (4096).',
+ ));
+
+ const ctx = createContext({
+ messages: [makeMessage({ role: 'user', content: 'hi' })],
+ });
+ await expect(runToolLoop(ctx)).rejects.toThrow(/exceeds max sequence length/);
+ const toolless = mockedLiteRT.prepareConversation.mock.calls.filter(
+ (c: any[]) => Array.isArray(c[2]?.tools) && c[2].tools.length === 0,
+ );
+ expect(toolless).toEqual([]);
+ });
+
it('rethrows a non-parse error without retrying (line 427 negative branch)', async () => {
mockedLiteRT.generateRaw.mockRejectedValue(new Error('out of memory'));
diff --git a/android/app/build.gradle b/android/app/build.gradle
index 88875fc44..d2f8f478f 100644
--- a/android/app/build.gradle
+++ b/android/app/build.gradle
@@ -146,6 +146,12 @@ android {
// stores .so files inside the APK zip — File.exists() returns false for them and
// exec() fails with EACCES (error=13) because you can't fork-exec from a zip entry.
useLegacyPackaging = true
+ // Google Tensor TPU runtime (see the dependency below). Off Grid loads only models
+ // compiled ahead of time for the TPU, which need the dispatch library alone, not the
+ // runtime's on-device compiler plugin. pickFirst guards against a second copy of the
+ // dispatch library arriving with a future litertlm-android.
+ excludes.add('**/libLiteRtCompilerPlugin_google_tensor.so')
+ pickFirsts.add('**/libLiteRtDispatch_GoogleTensor.so')
}
}
aaptOptions {
@@ -179,6 +185,14 @@ dependencies {
// LiteRT-LM on-device LLM inference (pinned — do not use latest.release)
implementation("com.google.ai.edge.litertlm:litertlm-android:0.17.1")
+ // Google Tensor TPU dispatch library (libLiteRtDispatch_GoogleTensor.so). LiteRT-LM's
+ // Backend.NPU loads it from nativeLibraryDir on a Pixel; litertlm-android does not ship it
+ // (Google's Pixel 10 TPU sample bundles it the same way). Its version must match the LiteRT
+ // runtime inside litertlm-android: 0.17.1 embeds a pre-release LiteRT 2.3.0 with the same
+ // dispatch API version (0.1.0); its runtime function table lacks only append_metric, which the
+ // Google Tensor dispatch library never calls. 2.3.0 is the only published version.
+ // Not transitive: only this .so, never a second LiteRT runtime beside litertlm's own.
+ implementation("com.google.ai.edge.litert:litert-npu-runtime-google-tensor:2.3.0") { transitive = false }
// Download layer — Room + WorkManager + OkHttp
def room_version = "2.8.2"
diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml
index 04fef1ba4..34f791d1f 100644
--- a/android/app/src/main/AndroidManifest.xml
+++ b/android/app/src/main/AndroidManifest.xml
@@ -76,6 +76,14 @@
+
+
+
+
+
+
{
val skipGpu = shouldSkipGpu()
return when (requested) {
- is Backend.NPU -> listOfNotNull(
- Backend.NPU(nativeLibraryDir = reactContext.applicationInfo.nativeLibraryDir),
- if (skipGpu) null else Backend.GPU(),
- Backend.CPU(),
- )
+ // NPU is requested only for a file compiled for this phone's TPU, whose graphs are
+ // TPU bytecode (DISPATCH_OP): no GPU or CPU tier can run it, so fail with the TPU's
+ // own error instead of masking it with a doomed fallback.
+ is Backend.NPU -> listOf(Backend.NPU(nativeLibraryDir = reactContext.applicationInfo.nativeLibraryDir))
is Backend.GPU -> if (skipGpu) listOf(Backend.CPU()) else listOf(Backend.GPU(), Backend.CPU())
else -> listOf(Backend.CPU())
}
@@ -191,8 +201,27 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
audioBackend = if (audioEnabled) Backend.CPU() else null,
)
eng = Engine(cfg)
- withTimeout(initTimeoutMs(backend, configuredMaxTokens)) { eng.initialize() }
+ // A broken TPU runtime aborts the process (SIGABRT) rather than throwing. Mark the
+ // attempt so the next launch reports the TPU as unusable instead of crashing again.
+ if (backend is Backend.NPU) markTpuLoadInFlight(true)
+ try {
+ withTimeout(initTimeoutMs(backend, configuredMaxTokens)) {
+ eng.initialize()
+ // LiteRT-LM starts the vision and audio encoders with the first conversation, not
+ // in initialize(). Open one now, within the same time budget, so an encoder that
+ // can't run on the TPU fails this attempt, and the text-only retry below, instead
+ // of every chat turn.
+ if (backend is Backend.NPU && (visionEnabled || audioEnabled)) {
+ eng.createConversation().close()
+ }
+ }
+ } finally {
+ if (backend is Backend.NPU) markTpuLoadInFlight(false)
+ }
+ if (backend is Backend.NPU) recordTpuLoadSucceeded()
engine = eng
+ supportsVision = visionEnabled
+ supportsAudio = audioEnabled
true
} catch (e: Exception) {
Log.w(TAG, "initializeWithFallback — $name failed: ${e.message}")
@@ -207,31 +236,42 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
private suspend fun initializeWithFallback(modelPath: String, requested: Backend, visionEnabled: Boolean, audioEnabled: Boolean): Backend {
val chain = buildBackendChain(requested)
+ for (backend in chain) {
+ if (tryTier(modelPath, backend, visionEnabled, audioEnabled)) return backend
+ if (backend != chain.last()) {
+ Log.i(TAG, "initializeWithFallback — ${backendName(backend)} exhausted retries, falling back to next tier")
+ }
+ }
+ throw IllegalStateException("All backends failed")
+ }
- // GPU/NPU failures can be transient (e.g. VRAM not yet released after a model switch).
- // Retry up to 2 extra times with backoff before giving up on a non-CPU backend.
+ /** Every attempt one backend tier gets before the chain moves to the next. */
+ private suspend fun tryTier(modelPath: String, backend: Backend, visionEnabled: Boolean, audioEnabled: Boolean): Boolean {
+ val name = backendName(backend)
+ // GPU failures can be transient (e.g. VRAM not yet released after a model switch).
+ // Retry up to 2 extra times with backoff. CPU and NPU failures are not, and each NPU
+ // attempt re-maps a multi-GB model.
val gpuRetries = 2
val gpuRetryDelayMs = 600L
-
- for (backend in chain) {
- val name = backendName(backend)
- val maxAttempts = if (backend is Backend.CPU) 1 else gpuRetries + 1
-
- for (attempt in 1..maxAttempts) {
- if (attempt > 1) {
- Log.i(TAG, "initializeWithFallback — $name retry $attempt/$maxAttempts after ${gpuRetryDelayMs}ms")
- delay(gpuRetryDelayMs)
- } else {
- Log.i(TAG, "initializeWithFallback — trying $name vision=$visionEnabled audio=$audioEnabled")
- }
- if (tryInitBackend(modelPath, backend, name, visionEnabled, audioEnabled)) return backend
+ val maxAttempts = if (backend is Backend.GPU) gpuRetries + 1 else 1
+
+ for (attempt in 1..maxAttempts) {
+ if (attempt > 1) {
+ Log.i(TAG, "initializeWithFallback — $name retry $attempt/$maxAttempts after ${gpuRetryDelayMs}ms")
+ delay(gpuRetryDelayMs)
+ } else {
+ Log.i(TAG, "initializeWithFallback — trying $name vision=$visionEnabled audio=$audioEnabled")
}
+ if (tryInitBackend(modelPath, backend, name, visionEnabled, audioEnabled)) return true
+ }
- if (backend != chain.last()) {
- Log.i(TAG, "initializeWithFallback — $name exhausted retries, falling back to next tier")
- }
+ // Google's Tensor TPU reference app keeps a model on the TPU text-only when its vision
+ // or audio executor can't start there, rather than dropping to a far slower tier.
+ if (backend is Backend.NPU && (visionEnabled || audioEnabled)) {
+ Log.i(TAG, "initializeWithFallback — $name retrying text-only")
+ return tryInitBackend(modelPath, backend, name, visionEnabled = false, audioEnabled = false)
}
- throw IllegalStateException("All backends failed")
+ return false
}
// -------------------------------------------------------------------------
@@ -613,11 +653,79 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) :
else -> "cpu"
}
- /** Vision delegate matched to the main backend tier (CPU main -> CPU vision).
- * Also honors the Pixel 10 GPU gate: never hand the GPU vision delegate to a
- * device where GPU init crashes, even on the NPU/GPU main path. */
- private fun visionBackendFor(mainBackend: Backend): Backend =
- if (mainBackend is Backend.CPU || shouldSkipGpu()) Backend.CPU() else Backend.GPU()
+ /** Vision delegate matched to the main backend tier (CPU main -> CPU vision, NPU main ->
+ * NPU vision, as in Google's Tensor TPU reference app). Also honors the Pixel 10 GPU
+ * gate: never hand the GPU vision delegate to a device where GPU init crashes. */
+ private fun visionBackendFor(mainBackend: Backend): Backend = when {
+ mainBackend is Backend.NPU -> Backend.NPU(nativeLibraryDir = reactContext.applicationInfo.nativeLibraryDir)
+ mainBackend is Backend.CPU || shouldSkipGpu() -> Backend.CPU()
+ else -> Backend.GPU()
+ }
+
+ // -------------------------------------------------------------------------
+ // getTpuSupport — can this phone run a LiteRT model on its Google Tensor TPU?
+ // -------------------------------------------------------------------------
+
+ @ReactMethod
+ fun getTpuSupport(promise: Promise) {
+ val safe = SafePromise(promise, TAG)
+ val sdk = Build.VERSION.SDK_INT
+ val socModel = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MODEL else ""
+ val socManufacturer = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MANUFACTURER else ""
+ val generation = TensorTpu.generation(socModel, socManufacturer)
+ val dispatchLibPresent = File(reactContext.applicationInfo.nativeLibraryDir, TensorTpu.DISPATCH_LIB).exists()
+ // Only probe the vendor library on a phone that passed the cheap checks.
+ val eligible = generation != null && sdk >= TensorTpu.MIN_SDK && dispatchLibPresent
+ val reason = TensorTpu.unsupportedReason(
+ generation, sdk, Build.ID ?: "", dispatchLibPresent,
+ vendorLibLoads = !eligible || vendorLibLoads(),
+ previousInitCrashed = tpuLoadCrashedBefore(),
+ )
+ Log.i(TAG, "getTpuSupport — soc=$socModel/$socManufacturer build=${Build.ID} tensorGeneration=$generation sdk=$sdk dispatchLib=$dispatchLibPresent reason=$reason")
+ safe.resolve(Arguments.createMap().apply {
+ putBoolean("supported", reason == null)
+ if (generation != null) putInt("tensorGeneration", generation) else putNull("tensorGeneration")
+ if (reason != null) putString("reason", reason) else putNull("reason")
+ })
+ }
+
+ private fun vendorLibLoads(): Boolean = try {
+ System.loadLibrary(TensorTpu.VENDOR_LIB)
+ true
+ } catch (e: Throwable) {
+ Log.w(TAG, "getTpuSupport — lib${TensorTpu.VENDOR_LIB}.so does not load: ${e.message}")
+ false
+ }
+
+ // Strikes are keyed to this install: an app update gets a fresh start.
+ private val tpuPrefs by lazy { reactContext.getSharedPreferences("litert_tpu", Context.MODE_PRIVATE) }
+ private fun installStamp(): Long = try {
+ reactContext.packageManager.getPackageInfo(reactContext.packageName, 0).lastUpdateTime
+ } catch (e: Exception) { 0L }
+
+ private fun markTpuLoadInFlight(inFlight: Boolean) {
+ // commit(), not apply(): the marker must be on disk before a native abort can kill us.
+ val edit = tpuPrefs.edit()
+ if (inFlight) edit.putBoolean(TPU_LOAD_IN_FLIGHT, true) else edit.remove(TPU_LOAD_IN_FLIGHT)
+ edit.commit()
+ }
+
+ private fun recordTpuLoadSucceeded() { tpuPrefs.edit().remove(TPU_LOAD_STRIKES).commit() }
+
+ /**
+ * A marker still set at launch means a TPU load never returned: the process died in it. Two
+ * such deaths in a row (one may just be the user closing the app mid-load) turn the TPU off
+ * for this install, so a crashing TPU runtime cannot crash the app on every load.
+ */
+ private fun tpuLoadCrashedBefore(): Boolean {
+ val stamp = installStamp()
+ var strikes = if (tpuPrefs.getLong(TPU_STRIKES_STAMP, stamp) == stamp) tpuPrefs.getInt(TPU_LOAD_STRIKES, 0) else 0
+ if (tpuPrefs.getBoolean(TPU_LOAD_IN_FLIGHT, false)) {
+ strikes++
+ tpuPrefs.edit().remove(TPU_LOAD_IN_FLIGHT).putInt(TPU_LOAD_STRIKES, strikes).putLong(TPU_STRIKES_STAMP, stamp).commit()
+ }
+ return strikes >= TPU_MAX_STRIKES
+ }
/** Current free system RAM in MB. */
private fun availableRamMb(): Long {
diff --git a/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt b/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt
new file mode 100644
index 000000000..dc96ff27c
--- /dev/null
+++ b/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt
@@ -0,0 +1,46 @@
+package ai.offgridmobile.litert
+
+/**
+ * Can this phone run a LiteRT model on its Google Tensor TPU? Pure (no Android deps) so it is
+ * unit-testable; [LiteRTModule.getTpuSupport] feeds it the Build properties and runtime probes.
+ */
+internal object TensorTpu {
+ /** LiteRT's Google Tensor dispatch library, shipped in the app's native lib dir. */
+ const val DISPATCH_LIB = "libLiteRtDispatch_GoogleTensor.so"
+ /** The Pixel vendor library the dispatch library opens (System.loadLibrary name). */
+ const val VENDOR_LIB = "edgetpu_litert"
+ /** LiteRT supports the Tensor TPU from Android 16 (API 36). */
+ const val MIN_SDK = 36
+ /** LiteRT's own NPU check rejects this Android 16 build family: it shipped without NPU support. */
+ private const val UNSUPPORTED_BUILD_PREFIX = "BP2A"
+
+ private val SOC_MODEL = Regex("""^tensor\s*g(\d+)$""")
+
+ /**
+ * Tensor generation (5 for a Pixel 10's "Tensor G5"), or null when this is not a Google Tensor
+ * SoC. Every Tensor G3+ Pixel reports SOC_MODEL "Tensor G" with SOC_MANUFACTURER "Google"
+ * (factory images); device codenames are not used — other vendors reuse them.
+ */
+ fun generation(socModel: String, socManufacturer: String): Int? {
+ if (!socManufacturer.equals("Google", ignoreCase = true)) return null
+ return SOC_MODEL.find(socModel.trim().lowercase())?.groupValues?.get(1)?.toIntOrNull()
+ }
+
+ /** Why the TPU can't be used here, or null when it can. */
+ fun unsupportedReason(
+ generation: Int?,
+ sdkInt: Int,
+ buildId: String,
+ dispatchLibPresent: Boolean,
+ vendorLibLoads: Boolean,
+ previousInitCrashed: Boolean,
+ ): String? = when {
+ generation == null -> "not_tensor"
+ sdkInt < MIN_SDK -> "android_too_old"
+ buildId.startsWith(UNSUPPORTED_BUILD_PREFIX) -> "android_build_unsupported"
+ !dispatchLibPresent -> "dispatch_lib_missing"
+ !vendorLibLoads -> "vendor_lib_unavailable"
+ previousInitCrashed -> "previous_tpu_load_crashed"
+ else -> null
+ }
+}
diff --git a/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt b/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt
new file mode 100644
index 000000000..e51272059
--- /dev/null
+++ b/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt
@@ -0,0 +1,59 @@
+package ai.offgridmobile.litert
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNull
+import org.junit.Test
+
+/**
+ * Tests for TensorTpu — whether a phone can run a LiteRT model on its Google Tensor TPU.
+ * SOC_MODEL / SOC_MANUFACTURER values are the ones Google's factory images report (vendor build.prop).
+ */
+@Suppress("kotlin:S100") // Backtick test names are idiomatic Kotlin
+class TensorTpuTest {
+
+ @Test
+ fun `reads the generation from a Pixel's SOC_MODEL`() {
+ assertEquals(5, TensorTpu.generation("Tensor G5", "Google")) // Pixel 10 family
+ assertEquals(4, TensorTpu.generation("Tensor G4", "Google")) // Pixel 9 family, Pixel 10a
+ assertEquals(3, TensorTpu.generation("Tensor G3", "Google")) // Pixel 8 family
+ assertEquals(6, TensorTpu.generation("Tensor G6", "Google")) // Pixel 11 family
+ }
+
+ @Test
+ fun `is not a Tensor TPU when the chip carries no generation`() {
+ assertNull(TensorTpu.generation("Tensor", "Google")) // Pixel 6
+ assertNull(TensorTpu.generation("GS201", "Google")) // Pixel 7
+ }
+
+ @Test
+ fun `is not a Tensor TPU on other vendors, even with a look-alike SOC_MODEL`() {
+ assertNull(TensorTpu.generation("SM8750", "QTI"))
+ assertNull(TensorTpu.generation("MT6989", "Mediatek"))
+ assertNull(TensorTpu.generation("Tensor G5", "Other"))
+ assertNull(TensorTpu.generation("", ""))
+ }
+
+ @Test
+ fun `allows the TPU on a Pixel 10 on Android 16 with a working runtime`() {
+ assertNull(reason())
+ }
+
+ @Test
+ fun `explains why the TPU cannot be used`() {
+ assertEquals("not_tensor", reason(generation = null))
+ assertEquals("android_too_old", reason(sdkInt = 35))
+ assertEquals("android_build_unsupported", reason(buildId = "BP2A.250605.031"))
+ assertEquals("dispatch_lib_missing", reason(dispatchLibPresent = false))
+ assertEquals("vendor_lib_unavailable", reason(vendorLibLoads = false))
+ assertEquals("previous_tpu_load_crashed", reason(previousInitCrashed = true))
+ }
+
+ private fun reason(
+ generation: Int? = 5,
+ sdkInt: Int = 36,
+ buildId: String = "CP3A.261005.005",
+ dispatchLibPresent: Boolean = true,
+ vendorLibLoads: Boolean = true,
+ previousInitCrashed: Boolean = false,
+ ) = TensorTpu.unsupportedReason(generation, sdkInt, buildId, dispatchLibPresent, vendorLibLoads, previousInitCrashed)
+}
diff --git a/src/screens/ModelsScreen/TextModelsTab.tsx b/src/screens/ModelsScreen/TextModelsTab.tsx
index 57bf41018..019a9e324 100644
--- a/src/screens/ModelsScreen/TextModelsTab.tsx
+++ b/src/screens/ModelsScreen/TextModelsTab.tsx
@@ -1,4 +1,4 @@
-import React, { useEffect } from 'react';
+import React, { useEffect, useState } from 'react';
import { View, Text, FlatList, TextInput, RefreshControl, TouchableOpacity, Platform } from 'react-native';
import { LoadingDots } from '../../components/LoadingDots';
import DeviceInfo from 'react-native-device-info';
@@ -23,18 +23,32 @@ import { TextFiltersSection } from './TextFiltersSection';
import { FilterState, SortOption } from './types';
import { SORT_OPTIONS } from './constants';
import { formatNumber, getTextModelCompatibility } from './utils';
-import { buildCuratedLiteRTFiles, curatedLiteRTDownloadWarning, getCuratedLiteRTEntry, LITERT_PARENT_ID } from '../../services/curatedLiteRTRegistry';
+import { buildCuratedLiteRTFiles, curatedLiteRTDownloadWarning, getCuratedLiteRTEntry, liteRTFileRunsOnDevice, LITERT_PARENT_ID } from '../../services/curatedLiteRTRegistry';
import { LITERT_FILE_META, LITERT_RECOMMENDED_MODEL, LITERT_PARENT_RECOMMENDED } from './litertRecommended';
import { modelManager } from '../../services';
+import { hardwareService } from '../../services/hardware';
import { modelDownloadService } from '../../services/modelDownloadService';
import { uniformDownloadId } from '../../services/modelDownloadService/uniformId';
import { fetchModelFiles } from '../../services/modelCatalogFiles';
import { predictGgufCapabilities } from '../../utils/ggufCapabilities';
+import { liteRTTensorTarget } from '../../utils/modelHelpers';
function hasNonSortFilters(fs: FilterState): boolean {
return fs.orgs.length > 0 || fs.type !== 'all' || fs.source !== 'all' || fs.size !== 'all' || fs.quant !== 'all';
}
+/** This phone's Google Tensor TPU generation, or null (not a Pixel, or before the probe answers).
+ * Gates the TPU-only LiteRT build in the curated list. */
+function useTensorTpuGeneration(): number | null {
+ const [generation, setGeneration] = useState(null);
+ useEffect(() => {
+ let live = true;
+ hardwareService.getTensorTpuGeneration().then(g => { if (live) setGeneration(g); }).catch(() => undefined);
+ return () => { live = false; };
+ }, []);
+ return generation;
+}
+
function getEmptyText(hasSearched: boolean, hasActiveFilters: boolean): string {
if (!hasSearched) return 'No recommended models available.';
if (hasActiveFilters) return 'No models match your filters. Try adjusting or clearing them.';
@@ -104,6 +118,9 @@ const ModelDetailView: React.FC = ({
}) => {
const { colors } = useTheme();
const styles = useThemedStyles(createStyles);
+ const tpuGeneration = useTensorTpuGeneration();
+ // A build for THIS phone's TPU is listed (a Pixel 10a's Tensor G4 has none yet).
+ const tpuBuildListed = tpuGeneration !== null && modelFiles.some(f => liteRTTensorTarget(f.name) === tpuGeneration);
// Pre-set the next pending (Download Manager icon) so it fires regardless of
// how the user dismisses step 9 (button or backdrop tap).
@@ -257,7 +274,9 @@ const ModelDetailView: React.FC = ({
{selectedModel.id === LITERT_PARENT_ID && Platform.OS === 'android' && DeviceInfo.getModel().toLowerCase().includes('pixel 10') && (
- {'GPU acceleration is not yet supported on Pixel 10. Models will run on CPU. Support coming soon.'}
+ {!tpuBuildListed
+ ? 'GPU acceleration is not yet supported on Pixel 10. Models will run on CPU. Support coming soon.'
+ : "Pixel 10 can't run LiteRT models on its GPU. The Tensor TPU build runs on the TPU; the other files run on the CPU."}
)}
Available Files
@@ -272,7 +291,7 @@ const ModelDetailView: React.FC = ({
) : (
f.size > 0 && !fileExceedsBudget(f.size, ramGB) && (filterState.quant === 'all' || f.name.includes(filterState.quant)))
+ .filter(f => f.size > 0 && !fileExceedsBudget(f.size, ramGB) && liteRTFileRunsOnDevice(f.name, tpuGeneration) && (filterState.quant === 'all' || f.name.includes(filterState.quant)))
.sort((a, b) => {
if (selectedModel.id === LITERT_PARENT_ID) return a.size - b.size; // curated: small-first
// Tier: Q4_K_M (CPU default, lowest size) → GPU/NPU Q4_0/Q8_0 → rest (CPU
@@ -368,6 +387,7 @@ export const TextModelsTab: React.FC = (props) => {
const { colors } = useTheme();
const styles = useThemedStyles(createStyles);
+ const tpuGeneration = useTensorTpuGeneration();
const downloadRecommendedFile = async (item: ModelInfo) => {
const files = await fetchModelFiles([item]);
@@ -395,7 +415,7 @@ export const TextModelsTab: React.FC = (props) => {
};
const onboardingLiteRTCards = onboarding && Platform.OS === 'android'
- ? buildCuratedLiteRTFiles().map((file, index) => {
+ ? buildCuratedLiteRTFiles().filter(file => liteRTFileRunsOnDevice(file.name, tpuGeneration)).map((file, index) => {
const entry = getCuratedLiteRTEntry(file.name);
const model = { ...LITERT_RECOMMENDED_MODEL, name: entry?.displayName ?? file.name };
const proceedDownload = () => { handleDownload(model, file); };
diff --git a/src/services/activeModelService/loaders.ts b/src/services/activeModelService/loaders.ts
index 294c7f724..2e5317955 100644
--- a/src/services/activeModelService/loaders.ts
+++ b/src/services/activeModelService/loaders.ts
@@ -8,6 +8,7 @@ import { useAppStore } from '../../stores';
import { DownloadedModel, LlamaDownloadedModel, ONNXImageModel, INFERENCE_BACKENDS } from '../../types';
import { llmService } from '../llm';
import { liteRTService } from '../litert';
+import { getCuratedLiteRTContextLimit } from '../curatedLiteRTRegistry';
import { unloadAllTextEngines } from '../engines';
import { localDreamGeneratorService as onnxImageGeneratorService } from '../localDreamGenerator';
import { modelManager } from '../modelManager';
@@ -106,7 +107,9 @@ async function doLoadLiteRTModel(ctx: TextLoadContext): Promise {
const preferredBackend = ctx.store.settings.liteRTBackend;
- const maxTokens = ctx.store.settings.liteRTMaxTokens ?? 4096;
+ // A curated build's own limit wins over a larger setting: native would clamp silently (the
+ // Tensor TPU build's KV cache is fixed at 4096) while JS kept the larger budget for compaction.
+ const maxTokens = Math.min(ctx.store.settings.liteRTMaxTokens ?? 4096, getCuratedLiteRTContextLimit(liteRTModel) ?? Infinity);
const contextScalar = Math.max(1, maxTokens / 4096);
const baseTimeoutMs = 90_000;
const timeoutMs = Math.min(Math.ceil(baseTimeoutMs * contextScalar), 180_000);
diff --git a/src/services/contextCompaction.ts b/src/services/contextCompaction.ts
index 3dba50c86..69d6f7a34 100644
--- a/src/services/contextCompaction.ts
+++ b/src/services/contextCompaction.ts
@@ -28,6 +28,12 @@ const CONTEXT_FULL_PATTERNS = [
'not enough context space',
'context window exceeded',
'context length exceeded',
+ // LiteRT-LM (any backend)
+ 'context window out of bounds',
+ 'exceeding the maximum number of tokens',
+ // LiteRT-LM on the Tensor TPU, whose context is fixed when the model is compiled
+ 'reached maximum number of tokens',
+ 'exceeds max sequence length',
];
/** Fraction of context allocated to the summary */
diff --git a/src/services/curatedLiteRTRegistry.ts b/src/services/curatedLiteRTRegistry.ts
index 835c96e2a..753420d7f 100644
--- a/src/services/curatedLiteRTRegistry.ts
+++ b/src/services/curatedLiteRTRegistry.ts
@@ -1,4 +1,5 @@
import { DownloadedModel, ModelFile } from '../types';
+import { liteRTTensorTarget } from '../utils/modelHelpers';
import { fileExceedsBudget } from './memoryBudget';
// Synthetic parent id for the curated LiteRT models. Used both as the model id
@@ -57,8 +58,34 @@ export const CURATED_LITERT_ENTRIES: readonly CuratedLiteRTEntry[] = [
"The model you have selected may exceed your device's memory and might not run reliably. For the best experience, try a smaller model.",
},
},
+ {
+ // Gemma 4 E2B compiled for the Tensor G5 TPU (Pixel 10) — the model Google's Pixel 10 TPU
+ // sample runs: https://github.com/google-ai-edge/litert-samples/tree/main/samples/litert/google/sample_app_tpu
+ // Listed only on a Tensor G5 phone (liteRTFileRunsOnDevice) and loaded on the TPU
+ // (litert.ts resolveLiteRTBackend).
+ fileName: 'gemma-4-E2B-it_Google_Tensor_G5.litertlm',
+ hfRepoId: 'litert-community/gemma-4-E2B-it-litert-lm',
+ commitHash: 'b3ca0d2f076785a8f4b2219ddbd2bdb99954eae1',
+ sizeBytes: 3113545589,
+ displayName: 'Gemma 4 E2B (Tensor TPU)',
+ highlight: 'Runs on the Pixel 10 Tensor TPU',
+ liteRTVision: true,
+ liteRTAudio: false,
+ // Compiled for the TPU with a fixed 4096-token KV cache (every prefill/decode signature).
+ maxContextTokens: 4096,
+ },
];
+/**
+ * Whether a LiteRT file can run on this phone: a build compiled for a Google Tensor TPU only on
+ * that generation's TPU (`tpuGeneration` from hardwareService.getTensorTpuGeneration), any
+ * portable build everywhere. Both the Models tab and onboarding filter the curated list with it.
+ */
+export function liteRTFileRunsOnDevice(fileName: string, tpuGeneration: number | null): boolean {
+ const target = liteRTTensorTarget(fileName);
+ return target === null || target === tpuGeneration;
+}
+
const CURATED_LITERT_INDEX: Map = new Map(
CURATED_LITERT_ENTRIES.map(e => [e.fileName, e]),
);
diff --git a/src/services/engines.ts b/src/services/engines.ts
index 618192828..d9f8bd527 100644
--- a/src/services/engines.ts
+++ b/src/services/engines.ts
@@ -144,10 +144,16 @@ export function isModelReady(model: { engine?: string; filePath?: string } | nul
* loads (used to gate image sends). llama needs a present projector (mmProjPath); LiteRT carries vision in
* the bundle (the liteRTVision flag). A missing/absent projector means the native completion would throw
* "Multimodal support not enabled" — so the send must be blocked here, not sent and crashed (device 2026-07-14).
+ * A LiteRT model whose engine came up text-only (a TPU load that dropped its vision executor) would have
+ * native discard the image silently, so it is refused here too — once, for every generation path.
*/
export function localModelAcceptsImages(model: DownloadedModel | null | undefined): boolean {
if (!model) return false;
- return isLiteRTModel(model) ? !!model.liteRTVision : !!model.mmProjPath;
+ return isLiteRTModel(model) ? liteRTAcceptsImages(model) : !!model.mmProjPath;
+}
+
+function liteRTAcceptsImages(model: { liteRTVision?: boolean; filePath: string }): boolean {
+ return !!model.liteRTVision && !liteRTService.loadedTextOnly(model.filePath);
}
/**
@@ -181,7 +187,7 @@ export function activeTextCapabilities(i: {
isRemote: i.isRemote,
remoteCaps: i.remoteCaps,
engine: i.model?.engine,
- liteRTVision: litert ? litert.liteRTVision : undefined,
+ liteRTVision: litert ? liteRTAcceptsImages(litert) : undefined,
liteRTAudio: litert ? litert.liteRTAudio : undefined,
liteRTLoaded: liteRTService.isModelLoaded(),
llama: {
diff --git a/src/services/generationToolLoop.ts b/src/services/generationToolLoop.ts
index 25eba489c..ae9c86ac8 100644
--- a/src/services/generationToolLoop.ts
+++ b/src/services/generationToolLoop.ts
@@ -884,6 +884,8 @@ async function callLiteRTForLoop(
// The litertlm native FC parser hard-fails (Status Code 3) when a small model emits
// a malformed tool call. Rather than surface a raw "Generation Error", retry once
// WITHOUT tools so the user still gets a text answer instead of a crashed turn.
+ // A full context is also Status Code 3; the caller compacts for that instead.
+ if (contextCompactionService.isContextFullError(msg)) throw e;
if (!/parse (tool|FC) calls|Status Code: 3/i.test(msg)) throw e;
logger.warn(
`[ToolLoop] LiteRT tool-call parse failed; retrying without tools: ${msg.slice(
diff --git a/src/services/hardware.ts b/src/services/hardware.ts
index efa6d5779..794a51842 100644
--- a/src/services/hardware.ts
+++ b/src/services/hardware.ts
@@ -6,6 +6,7 @@ import { readPerformanceCoreCount } from './cpuTopologyReader';
// Access NativeModules.LocalDreamModule dynamically (not destructured)
// so it can be mocked in tests after module import.
const getLocalDreamModule = () => NativeModules.LocalDreamModule;
+const getLiteRTModule = () => NativeModules.LiteRTModule;
import {
DeviceInfo as DeviceInfoType,
ModelRecommendation,
@@ -30,6 +31,7 @@ class HardwareService {
private cachedSoCInfo: SoCInfo | null = null;
private cachedImageRecommendation: ImageModelRecommendation | null = null;
private cachedOpenCLCapability: { supported: boolean; reason?: string } | null = null;
+ private cachedTensorTpuGeneration: Promise | null = null;
async getDeviceInfo(): Promise {
if (this.cachedDeviceInfo) {
return this.cachedDeviceInfo;
@@ -479,6 +481,28 @@ class HardwareService {
return { hasNpu: HTP_ENABLED && soc.hasNPU, hasGpu: opencl.supported };
}
+ /**
+ * The Google Tensor generation whose TPU can run LiteRT models on this phone (5 on a Pixel 10),
+ * or null. The native probe checks the SoC, Android 16+, and that LiteRT's Tensor dispatch
+ * library shipped in the APK. Separate from getAccelerationCapability: llama.rn has no TPU path.
+ */
+ getTensorTpuGeneration(): Promise {
+ this.cachedTensorTpuGeneration ??= this.probeTensorTpu();
+ return this.cachedTensorTpuGeneration;
+ }
+
+ private async probeTensorTpu(): Promise {
+ const mod = getLiteRTModule();
+ if (Platform.OS !== 'android' || !mod?.getTpuSupport) return null;
+ try {
+ const res: { supported: boolean; tensorGeneration: number | null; reason: string | null } = await mod.getTpuSupport();
+ logger.log(`[WIRE-TPU] ${JSON.stringify(res)}`); // [WIRE] real Tensor TPU probe (gates TPU routing + the Tensor model card)
+ return res.supported ? res.tensorGeneration : null;
+ } catch {
+ return null;
+ }
+ }
+
async getOpenCLCapability(): Promise<{ supported: boolean; reason?: string }> {
if (this.cachedOpenCLCapability) return this.cachedOpenCLCapability;
if (Platform.OS !== 'android') return { supported: false, reason: 'not_android' };
diff --git a/src/services/litert.ts b/src/services/litert.ts
index 4bb6a21ec..24f21710e 100644
--- a/src/services/litert.ts
+++ b/src/services/litert.ts
@@ -12,7 +12,9 @@
import { NativeModules, NativeEventEmitter, EmitterSubscription } from 'react-native';
import logger from '../utils/logger';
+import { liteRTTensorTarget } from '../utils/modelHelpers';
import { summarizeSession, runCompaction } from './liteRTCompaction';
+import { hardwareService } from './hardware';
const TAG = '[LiteRTService]';
@@ -27,6 +29,22 @@ const EVENT_TOOL_CALL = 'litert_tool_call';
type LiteRTBackend = 'cpu' | 'gpu' | 'npu';
+/**
+ * The backend a load asks native for. A file compiled for this phone's Google Tensor TPU
+ * (…_Google_Tensor_G5.litertlm on a Pixel 10) runs on the TPU: that build exists only for it,
+ * and Pixel 10 has no working LiteRT GPU path. Any other NPU request stays out of this release —
+ * a saved NPU selection loads on GPU rather than attempting an unsupported native load.
+ */
+async function resolveLiteRTBackend(modelPath: string, requested: LiteRTBackend): Promise {
+ const target = liteRTTensorTarget(modelPath.split('/').pop() ?? modelPath);
+ if (target !== null) {
+ const tpu = await hardwareService.getTensorTpuGeneration();
+ if (tpu === target) return 'npu';
+ logger.log(TAG, `loadModel — file targets Tensor G${target} TPU, device TPU=${tpu ?? 'none'}; using ${requested}`);
+ }
+ return requested === 'npu' ? 'gpu' : requested;
+}
+
interface GenerateRawHandlers {
onToken?: (token: string) => void;
onToolCall?: (name: string, args: Record) => Promise;
@@ -61,6 +79,8 @@ interface LiteRTGenerationCallbacks {
class LiteRTService {
private loaded = false;
private modelSupportsAudio = false;
+ private modelSupportsVision = false;
+ private loadedModelPath: string | null = null;
private activeBackend: LiteRTBackend | null = null;
private readonly emitter: NativeEventEmitter | null = null;
private subscriptions: EmitterSubscription[] = [];
@@ -100,9 +120,7 @@ class LiteRTService {
async loadModel(modelPath: string, preferredBackend: LiteRTBackend, opts: { supportsVision?: boolean; supportsAudio?: boolean; maxNumTokens?: number } = {}): Promise {
if (!this.isAvailable()) throw new Error('LiteRT is not available on this platform');
- // NPU was exposed in a test build, but is not part of this release.
- // Keep saved selections usable without attempting an unsupported native load.
- if (preferredBackend === 'npu') preferredBackend = 'gpu';
+ preferredBackend = await resolveLiteRTBackend(modelPath, preferredBackend);
const { supportsVision = false, supportsAudio = false, maxNumTokens = 4096 } = opts;
this.configuredMaxTokens = maxNumTokens;
logger.log(TAG, `loadModel — path=${modelPath} backend=${preferredBackend} supportsVision=${supportsVision} supportsAudio=${supportsAudio} maxNumTokens=${maxNumTokens}`);
@@ -112,7 +130,7 @@ class LiteRTService {
// budget after the native RAM clamp (may be < requested). Adopt it so compaction
// thresholds + the context-usage bar aren't stale. Tolerate a bare string from an
// older native build (backward-compatible).
- const res: string | { backend: string; maxNumTokens?: number } =
+ const res: string | { backend: string; maxNumTokens?: number; vision?: boolean; audio?: boolean } =
await LiteRTModule.loadModel(modelPath, preferredBackend, supportsVision, supportsAudio, maxNumTokens);
logger.log(`[WIRE-LITERT-LOAD] ${JSON.stringify({ requested: preferredBackend, supportsVision, supportsAudio, maxNumTokens, res })}`); // [WIRE]
const actualBackend = typeof res === 'string' ? res : res.backend;
@@ -124,12 +142,17 @@ class LiteRTService {
}
this.activeBackend = actualBackend as LiteRTBackend;
this.loaded = true;
- this.modelSupportsAudio = supportsAudio;
+ // Native reports what the engine came up with: a TPU load can drop to text-only.
+ this.modelSupportsVision = typeof res === 'object' && typeof res.vision === 'boolean' ? res.vision : supportsVision;
+ this.modelSupportsAudio = typeof res === 'object' && typeof res.audio === 'boolean' ? res.audio : supportsAudio;
+ this.loadedModelPath = modelPath;
logger.log(TAG, `loadModel — loaded on ${this.activeBackend}`);
} catch (e) {
this.loaded = false;
this.activeBackend = null;
this.modelSupportsAudio = false;
+ this.modelSupportsVision = false;
+ this.loadedModelPath = null;
logger.log(TAG, `loadModel — failed: ${String(e)}`);
throw e;
}
@@ -140,6 +163,12 @@ class LiteRTService {
return this.loaded && this.modelSupportsAudio;
}
+ /** True when `modelPath` is the loaded model and its engine came up without vision: a TPU load
+ * whose vision executor could not start dropped to text-only, so native would discard images. */
+ loadedTextOnly(modelPath: string): boolean {
+ return this.loaded && this.loadedModelPath === modelPath && !this.modelSupportsVision;
+ }
+
// ---------------------------------------------------------------------------
// resetConversation — cheap: closes + recreates Conversation, Engine stays
// ---------------------------------------------------------------------------
@@ -297,6 +326,19 @@ class LiteRTService {
media?: { imageUris?: string[]; audioUris?: string[] },
): Promise {
if (!this.isAvailable() || !this.loaded) { callbacks.onError(new Error('No LiteRT model loaded')); return; }
+ // An engine without vision or audio (a TPU load that dropped to text-only) would have native
+ // discard the media without a word — an audio-only turn would even reach the model as empty
+ // text. Every generation path sends through here, after any lazy load, so refuse it once, here.
+ if (media?.imageUris?.some(Boolean) && !this.modelSupportsVision) {
+ logger.log(TAG, 'sendMessage — refused images: the loaded engine has no vision');
+ callbacks.onError(new Error('Images are not available for this model on this device. Remove the image to continue.'));
+ return;
+ }
+ if (media?.audioUris?.some(Boolean) && !this.modelSupportsAudio) {
+ logger.log(TAG, 'sendMessage — refused audio: the loaded engine has no audio input');
+ callbacks.onError(new Error('Audio input is not available for this model on this device.'));
+ return;
+ }
// Reset accumulators
this.currentContent = '';
@@ -377,6 +419,10 @@ class LiteRTService {
this.emitter!.addListener(EVENT_ERROR, (message: string) => {
logger.log(TAG, `sendMessage — error: ${message}`);
this.clearSubscriptions();
+ // The native conversation keeps the failed turn (and, on the TPU, the prefill that ran
+ // before the limit), so the next prepareConversation must rebuild it — e.g. the
+ // compaction retry after a full context.
+ this.invalidateConversation();
this.currentToolCallHandler = null;
callbacks.onError(new Error(message));
@@ -416,6 +462,7 @@ class LiteRTService {
}
} catch (e) {
this.clearSubscriptions();
+ this.invalidateConversation();
const err = e instanceof Error ? e : new Error(String(e));
logger.log(TAG, `sendMessage — native error: ${err.message}`);
callbacks.onError(err);
@@ -515,6 +562,8 @@ class LiteRTService {
} finally {
this.loaded = false;
this.modelSupportsAudio = false;
+ this.modelSupportsVision = false;
+ this.loadedModelPath = null;
this.activeBackend = null;
}
}
diff --git a/src/utils/modelHelpers.ts b/src/utils/modelHelpers.ts
index 62ac9d5c6..c5ce293aa 100644
--- a/src/utils/modelHelpers.ts
+++ b/src/utils/modelHelpers.ts
@@ -13,3 +13,13 @@ export const getMmProjFileSize = (m?: DownloadedModel): number =>
*/
export const isLiteRTFileName = (fileName: string): boolean =>
fileName.toLowerCase().endsWith('.litertlm');
+
+/**
+ * The Google Tensor generation a LiteRT file was compiled for, read from Google's naming
+ * (`gemma-4-E2B-it_Google_Tensor_G5.litertlm` → 5), or null for a portable CPU/GPU build. Such a
+ * file runs only on that generation's TPU.
+ */
+export const liteRTTensorTarget = (fileName: string): number | null => {
+ const match = /_google_tensor_g(\d+)\.litertlm$/i.exec(fileName);
+ return match ? Number(match[1]) : null;
+};