From 30af1073708a13d9ea0f27b58354f4ab4aabcb1c Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 1 Nov 2026 13:06:35 +0100 Subject: [PATCH 1/3] feat(core-ai): report progress for image and video generation Image and video models get withProgress(onProgress), since the AI SDK has no progress API and providerOptions only takes JSON. Native emits onGenerationProgress per request id. Image generation now takes a request id too, so an abort signal stops it after the current step. Closes #344 Co-Authored-By: Claude Opus 5.5 --- .../core-ai/availability.device.harness.ts | 2 +- .../image-generation.device.harness.ts | 33 +++++ .../video-generation.device.harness.ts | 19 +++ packages/core-ai/ios/CoreAI.mm | 23 +++- packages/core-ai/ios/CoreAIImpl.swift | 72 ++++++---- .../core-ai/ios/CoreAIVideoGeneration.swift | 25 +++- packages/core-ai/src/NativeCoreAI.ts | 11 ++ packages/core-ai/src/__tests__/ai-sdk.test.ts | 86 +++++++++++- packages/core-ai/src/ai-sdk.ts | 127 ++++++++++++------ packages/core-ai/src/index.ts | 1 + packages/core-ai/src/types.ts | 13 ++ .../docs/docs/core-ai/images-and-video.md | 20 +++ 12 files changed, 356 insertions(+), 76 deletions(-) diff --git a/harness/core-ai/availability.device.harness.ts b/harness/core-ai/availability.device.harness.ts index 85f9cf4a..fae2d01d 100644 --- a/harness/core-ai/availability.device.harness.ts +++ b/harness/core-ai/availability.device.harness.ts @@ -21,7 +21,7 @@ describe('core-ai availability', () => { it('rejects a model that is not on disk', async () => { await expect( - CoreAI.generateImage({ source: '/missing' }, 'a cat', {}) + CoreAI.generateImage('missing-model', { source: '/missing' }, 'a cat', {}) ).rejects.toMatchObject({ code: expect.any(String) }) }) diff --git a/harness/core-ai/image-generation.device.harness.ts b/harness/core-ai/image-generation.device.harness.ts index d2b46bcc..de13e2ea 100644 --- a/harness/core-ai/image-generation.device.harness.ts +++ b/harness/core-ai/image-generation.device.harness.ts @@ -1,3 +1,4 @@ +import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai' import { generateImage } from 'ai' import { describe, expect, it } from 'react-native-harness' @@ -80,4 +81,36 @@ describe('core-ai image generation', () => { expect.objectContaining({ type: 'unsupported', feature: 'size' }), ]) }) + + it('reports each denoising step in order', async () => { + const progress: CoreAIGenerationProgress[] = [] + await generateImage({ + model: model.withProgress((update) => progress.push(update)), + prompt, + seed: 1, + providerOptions: { 'core-ai': { stepCount: 2 } }, + }) + + expect(progress).toEqual([ + { phase: 'denoising', step: 1, totalSteps: 2 }, + { phase: 'denoising', step: 2, totalSteps: 2 }, + ]) + }) + + it('stops after the current step when aborted', async () => { + const controller = new AbortController() + const progress: CoreAIGenerationProgress[] = [] + const result = generateImage({ + model: model.withProgress((update) => { + progress.push(update) + controller.abort() + }), + prompt, + abortSignal: controller.signal, + providerOptions: { 'core-ai': { stepCount: 4 } }, + }) + + await expect(result).rejects.toThrow() + expect(progress).toHaveLength(1) + }) }) diff --git a/harness/core-ai/video-generation.device.harness.ts b/harness/core-ai/video-generation.device.harness.ts index 6fc81dc3..3348b077 100644 --- a/harness/core-ai/video-generation.device.harness.ts +++ b/harness/core-ai/video-generation.device.harness.ts @@ -1,3 +1,4 @@ +import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai' import { experimental_generateVideo as generateVideo } from 'ai' import { describe, expect, it } from 'react-native-harness' @@ -43,6 +44,24 @@ describe('core-ai video generation', () => { ) }) + it('reports each phase and denoising step in order', async () => { + const progress: CoreAIGenerationProgress[] = [] + await generateVideo({ + model: model.withProgress((update) => progress.push(update)), + prompt: 'a red ball rolling across a wooden table', + seed: 1, + providerOptions: { 'core-ai': { stepCount: 2, cfgCutoff: 1 } }, + }) + + expect(progress.map(({ phase, step }) => `${phase} ${step}`)).toEqual([ + 'encoding 0', + 'denoising 0', + 'denoising 1', + 'decoding 2', + 'assembling 2', + ]) + }) + it('stops when aborted', async () => { const controller = new AbortController() const result = generate({}, controller.signal) diff --git a/packages/core-ai/ios/CoreAI.mm b/packages/core-ai/ios/CoreAI.mm index 6f0f3e6c..cde7e812 100644 --- a/packages/core-ai/ios/CoreAI.mm +++ b/packages/core-ai/ios/CoreAI.mm @@ -237,15 +237,32 @@ - (void)generateVideo:(nonnull NSString *)requestId options:(nonnull NSDictionary *)options resolve:(nonnull RCTPromiseResolveBlock)resolve reject:(nonnull RCTPromiseRejectBlock)reject { - [_coreAI generateVideo:requestId config:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject]; + [_coreAI generateVideo:requestId + config:CoreAIConfigToDictionary(config) + prompt:prompt + options:options + onProgress:^(NSDictionary *progress) { + [self emitOnGenerationProgress:progress]; + } + resolve:resolve + reject:reject]; } -- (void)generateImage:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config +- (void)generateImage:(nonnull NSString *)requestId + config:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config prompt:(nonnull NSString *)prompt options:(nonnull NSDictionary *)options resolve:(nonnull RCTPromiseResolveBlock)resolve reject:(nonnull RCTPromiseRejectBlock)reject { - [_coreAI generateImage:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject]; + [_coreAI generateImage:requestId + config:CoreAIConfigToDictionary(config) + prompt:prompt + options:options + onProgress:^(NSDictionary *progress) { + [self emitOnGenerationProgress:progress]; + } + resolve:resolve + reject:reject]; } - (void)runTask:(nonnull NSString *)task diff --git a/packages/core-ai/ios/CoreAIImpl.swift b/packages/core-ai/ios/CoreAIImpl.swift index 292bfa67..c2879439 100644 --- a/packages/core-ai/ios/CoreAIImpl.swift +++ b/packages/core-ai/ios/CoreAIImpl.swift @@ -649,6 +649,7 @@ public class CoreAIImpl: NSObject { config: [String: Any], prompt: String, options: [String: Any]?, + onProgress: @escaping ([String: Any]) -> Void, resolve: @escaping (Any?) -> Void, reject: @escaping (String, String, Error?) -> Void ) { @@ -659,7 +660,9 @@ public class CoreAIImpl: NSObject { startTask(requestId, onError: { _, code, message in reject(code, message, nil) }) { - resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:])) + resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:]) { + onProgress($0.merging(["requestId": requestId]) { current, _ in current }) + }) } } catch { reject(errorCode(error), error.localizedDescription, error) @@ -672,11 +675,15 @@ public class CoreAIImpl: NSObject { #endif } + /// Generates images, reporting each denoising step. cancelStream(requestId) + /// stops it after the current step @objc public func generateImage( - _ config: [String: Any], + _ requestId: String, + config: [String: Any], prompt: String, options: [String: Any]?, + onProgress: @escaping ([String: Any]) -> Void, resolve: @escaping (Any?) -> Void, reject: @escaping (String, String, Error?) -> Void ) { @@ -684,34 +691,41 @@ public class CoreAIImpl: NSObject { if #available(iOS 27, macOS 27, *) { do { let sourceURL = try resolveSourceURL(config) - Task { - do { - let startingImage = try Self.startingImage(from: options) - let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) - // Exports without a VAE encoder generate from the prompt alone, - // which the metadata reports so JS can warn - let editsImage = startingImage != nil && pipeline.supportsImageToImage - let configuration = try self.makePipelineConfiguration( - prompt: prompt, - pipeline: pipeline, - options: options, - startingImage: editsImage ? startingImage : nil - ) - let result = try await pipeline.generateImages(configuration: configuration) { _ in true } - let images = try result.images.map(Self.pngBase64) - var metadata: [String: Any] = [ - "count": images.count, - "imageFormat": "image/png", - "width": result.images.first?.width ?? 0, - "height": result.images.first?.height ?? 0 - ] - if startingImage != nil { - metadata["imageToImage"] = editsImage - } - resolve(["images": images, "metadata": metadata]) - } catch { - reject(errorCode(error), error.localizedDescription, error) + startTask(requestId, onError: { _, code, message in + reject(code, message, nil) + }) { + let startingImage = try Self.startingImage(from: options) + let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) + // Exports without a VAE encoder generate from the prompt alone, + // which the metadata reports so JS can warn + let editsImage = startingImage != nil && pipeline.supportsImageToImage + let configuration = try self.makePipelineConfiguration( + prompt: prompt, + pipeline: pipeline, + options: options, + startingImage: editsImage ? startingImage : nil + ) + let result = try await pipeline.generateImages(configuration: configuration) { progress in + onProgress([ + "requestId": requestId, + "phase": "denoising", + "step": progress.step, + "totalSteps": progress.totalSteps + ]) + return !Task.isCancelled + } + try Task.checkCancellation() + let images = try result.images.map(Self.pngBase64) + var metadata: [String: Any] = [ + "count": images.count, + "imageFormat": "image/png", + "width": result.images.first?.width ?? 0, + "height": result.images.first?.height ?? 0 + ] + if startingImage != nil { + metadata["imageToImage"] = editsImage } + resolve(["images": images, "metadata": metadata]) } } catch { reject(errorCode(error), error.localizedDescription, error) diff --git a/packages/core-ai/ios/CoreAIVideoGeneration.swift b/packages/core-ai/ios/CoreAIVideoGeneration.swift index fd6e6bda..29bcab04 100644 --- a/packages/core-ai/ios/CoreAIVideoGeneration.swift +++ b/packages/core-ai/ios/CoreAIVideoGeneration.swift @@ -15,11 +15,21 @@ enum VideoGeneration { // only while it runs nonisolated(unsafe) private static var loaded: (key: String, load: Task<(WanPipeline, ExportedShape), Error>)? - static func generate(prompt: String, sourceURL: URL, options: [String: Any]) async throws -> [String: Any] { + static func generate( + prompt: String, + sourceURL: URL, + options: [String: Any], + onProgress: @escaping @Sendable ([String: Any]) -> Void + ) async throws -> [String: Any] { let (pipeline, shape) = try await self.pipeline(at: sourceURL, options: options) let configuration = try makeConfiguration(prompt: prompt, pipeline: pipeline, shape: shape, options: options) - let result = try await pipeline.generateVideo(configuration: configuration) { _ in - !Task.isCancelled + let result = try await pipeline.generateVideo(configuration: configuration) { progress in + onProgress([ + "phase": phaseName(progress.phase), + "step": progress.step, + "totalSteps": progress.totalSteps + ]) + return !Task.isCancelled } try Task.checkCancellation() @@ -41,6 +51,15 @@ enum VideoGeneration { ] } + private static func phaseName(_ phase: VideoProgress.Phase) -> String { + switch phase { + case .encoding: return "encoding" + case .denoising: return "denoising" + case .decoding: return "decoding" + case .assembling: return "assembling" + } + } + // Concurrent calls for the same bundle share one load private static func pipeline(at url: URL, options: [String: Any]) async throws -> (WanPipeline, ExportedShape) { let lazy = options["lazyModelLoading"] as? Bool ?? true diff --git a/packages/core-ai/src/NativeCoreAI.ts b/packages/core-ai/src/NativeCoreAI.ts index 9b0a38f7..5bd79300 100644 --- a/packages/core-ai/src/NativeCoreAI.ts +++ b/packages/core-ai/src/NativeCoreAI.ts @@ -60,6 +60,13 @@ export type CoreAITranscriptionUpdateEvent = { endSecond: number } +export type CoreAIGenerationProgressEvent = { + requestId: string + phase: string + step: number + totalSteps: number +} + export interface Spec extends TurboModule { getCapabilities(): Promise @@ -138,7 +145,9 @@ export interface Spec extends TurboModule { prompt: string, options: CodegenTypes.UnsafeObject ): Promise + // Cancel with cancelStream(requestId) generateImage( + requestId: string, config: NativeCoreAIModelConfig, prompt: string, options: CodegenTypes.UnsafeObject @@ -161,6 +170,8 @@ export interface Spec extends TurboModule { onStreamComplete: CodegenTypes.EventEmitter onStreamError: CodegenTypes.EventEmitter onTranscriptionUpdate: CodegenTypes.EventEmitter + // Progress of generateImage and generateVideo requests + onGenerationProgress: CodegenTypes.EventEmitter } export default TurboModuleRegistry.getEnforcing('NativeCoreAI') diff --git a/packages/core-ai/src/__tests__/ai-sdk.test.ts b/packages/core-ai/src/__tests__/ai-sdk.test.ts index 308deaf0..9b6a02d3 100644 --- a/packages/core-ai/src/__tests__/ai-sdk.test.ts +++ b/packages/core-ai/src/__tests__/ai-sdk.test.ts @@ -70,6 +70,7 @@ const generateVideo = mock( ) const generateImage = mock( async ( + _requestId: string, _config: Record, _prompt: string, options: Record @@ -108,6 +109,7 @@ const onStreamUpdate = createEmitter() const onStreamComplete = createEmitter() const onStreamError = createEmitter() const onTranscriptionUpdate = createEmitter() +const onGenerationProgress = createEmitter() const transcriptionResult = { text: 'hello world', @@ -151,6 +153,7 @@ mock.module('react-native', () => ({ appendTranscriptionAudio, finishTranscriptionStream, onTranscriptionUpdate, + onGenerationProgress, }), }, })) @@ -1238,7 +1241,7 @@ describe('core ai image editing', () => { ...overrides, }) - const lastOptions = () => generateImage.mock.calls.at(-1)![2] + const lastOptions = () => generateImage.mock.calls.at(-1)![3] test('passes the first image as base64', async () => { const { warnings } = await imageModel().doGenerate( @@ -1314,3 +1317,84 @@ describe('core ai image editing', () => { ]) }) }) + +describe('core ai generation progress', () => { + const imageModel = () => coreAI.imageModel({ id: 'flux', source: '/flux' }) + const videoModel = () => coreAI.videoModel({ id: 'wan', source: '/wan' }) + + test('reports progress of its own image request', async () => { + generateImage.mockImplementationOnce(async (requestId) => { + onGenerationProgress.emit({ + requestId: 'other', + phase: 'denoising', + step: 1, + totalSteps: 2, + }) + onGenerationProgress.emit({ + requestId, + phase: 'denoising', + step: 1, + totalSteps: 2, + }) + return { images: ['AAAA'] } + }) + const onProgress = mock(() => {}) + + await imageModel().withProgress(onProgress).doGenerate({ + prompt: 'a cat', + n: 1, + size: undefined, + aspectRatio: undefined, + seed: undefined, + files: undefined, + mask: undefined, + providerOptions: {}, + }) + + expect(onProgress.mock.calls).toEqual([ + [{ phase: 'denoising', step: 1, totalSteps: 2 }], + ]) + // The callback stays in JS + expect(generateImage.mock.calls.at(-1)![3]).not.toHaveProperty('onProgress') + }) + + test('stops reporting video progress once the call finishes', async () => { + let requestId = '' + generateVideo.mockImplementationOnce(async (id) => { + requestId = id + onGenerationProgress.emit({ + requestId, + phase: 'encoding', + step: 0, + totalSteps: 2, + }) + return { video: 'AAAA', mediaType: 'video/mp4' } + }) + const onProgress = mock(() => {}) + + await videoModel().generate('a cat', { onProgress }) + onGenerationProgress.emit({ + requestId, + phase: 'denoising', + step: 1, + totalSteps: 2, + }) + + expect(onProgress.mock.calls).toEqual([ + [{ phase: 'encoding', step: 0, totalSteps: 2 }], + ]) + }) + + test('cancels the native image request on abort', async () => { + const controller = new AbortController() + generateImage.mockImplementationOnce(async (requestId) => { + controller.abort() + expect(cancelStream).toHaveBeenCalledWith(requestId) + throw new Error('cancelled') + }) + + await expect( + imageModel().generate('a cat', {}, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index d6fdf9b3..6a670ec9 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -38,6 +38,7 @@ import type { CoreAIEmbeddingResult, CoreAIGenerationOptions, CoreAIGenerationPart, + CoreAIGenerationProgress, CoreAIGenerationResult, CoreAIImageGenerationOptions, CoreAIImageGenerationResult, @@ -434,32 +435,51 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { readonly modelId: string readonly maxImagesPerCall = 1 readonly config: CoreAIModelConfig + private readonly onProgress?: (progress: CoreAIGenerationProgress) => void - constructor(config: CoreAIModelConfig) { + constructor( + config: CoreAIModelConfig, + onProgress?: (progress: CoreAIGenerationProgress) => void + ) { this.config = { ...config, task: 'diffusion' } this.modelId = getModelId(config) + this.onProgress = onProgress + } + + // The same model, reporting the progress of the calls it is passed to. + // providerOptions only takes JSON, so the AI SDK can't carry a callback + withProgress(onProgress: (progress: CoreAIGenerationProgress) => void) { + return new CoreAIImageGenerationModel(this.config, onProgress) } + // The abort signal stops generation after the current denoising step async generate( prompt: string, - options: CoreAIImageGenerationOptions = {} + { onProgress, ...options }: CoreAIImageGenerationOptions = {}, + abortSignal?: AbortSignal ): Promise { - return NativeCoreAI.generateImage( - this.config, - prompt, - options - ) as Promise + return runGeneration( + (requestId) => + NativeCoreAI.generateImage(requestId, this.config, prompt, options), + onProgress, + abortSignal + ) } async doGenerate(options: ImageModelV4CallOptions) { - const result = await this.generate(options.prompt ?? '', { - n: options.n, - size: options.size, - aspectRatio: options.aspectRatio, - seed: options.seed, - ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), - ...(options.providerOptions?.['core-ai'] ?? {}), - }) + const result = await this.generate( + options.prompt ?? '', + { + n: options.n, + size: options.size, + aspectRatio: options.aspectRatio, + seed: options.seed, + ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), + onProgress: this.onProgress, + ...(options.providerOptions?.['core-ai'] ?? {}), + }, + options.abortSignal + ) return { images: result.images, @@ -484,6 +504,40 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { } } +// Runs a native image or video request, forwards its progress events, and +// cancels it when the signal aborts +async function runGeneration( + run: (requestId: string) => Promise, + onProgress: ((progress: CoreAIGenerationProgress) => void) | undefined, + abortSignal: AbortSignal | undefined +): Promise { + // React Native's AbortSignal has no throwIfAborted() + if (abortSignal?.aborted) { + throw toAbortError(abortSignal) + } + const requestId = generateId() + const onAbort = () => NativeCoreAI.cancelStream(requestId) + abortSignal?.addEventListener('abort', onAbort) + const subscription = + onProgress && + NativeCoreAI.onGenerationProgress(({ requestId: id, ...progress }) => { + if (id === requestId) { + onProgress(progress as CoreAIGenerationProgress) + } + }) + try { + return (await run(requestId)) as Result + } catch (error) { + if (abortSignal?.aborted) { + throw toAbortError(abortSignal) + } + throw error + } finally { + subscription?.remove() + abortSignal?.removeEventListener('abort', onAbort) + } +} + // Wan 2.1 sizes for the aspect ratios it is trained on const VIDEO_SIZES: Record = { '16:9': { width: 832, height: 480 }, @@ -497,41 +551,35 @@ export class CoreAIVideoModel implements VideoModelV4 { readonly modelId: string readonly maxVideosPerCall = 1 readonly config: CoreAIModelConfig + private readonly onProgress?: (progress: CoreAIGenerationProgress) => void - constructor(config: CoreAIModelConfig) { + constructor( + config: CoreAIModelConfig, + onProgress?: (progress: CoreAIGenerationProgress) => void + ) { this.config = { ...config, task: 'video-diffusion' } this.modelId = getModelId(config) + this.onProgress = onProgress + } + + // The same model, reporting the progress of the calls it is passed to + withProgress(onProgress: (progress: CoreAIGenerationProgress) => void) { + return new CoreAIVideoModel(this.config, onProgress) } // Generates an MP4 from the prompt. Takes minutes on device, and the abort // signal stops it after the current denoising step async generate( prompt: string, - options: CoreAIVideoGenerationOptions = {}, + { onProgress, ...options }: CoreAIVideoGenerationOptions = {}, abortSignal?: AbortSignal ): Promise { - // React Native's AbortSignal has no throwIfAborted() - if (abortSignal?.aborted) { - throw toAbortError(abortSignal) - } - const requestId = generateId() - const onAbort = () => NativeCoreAI.cancelStream(requestId) - abortSignal?.addEventListener('abort', onAbort) - try { - return (await NativeCoreAI.generateVideo( - requestId, - this.config, - prompt, - options - )) as CoreAIVideoGenerationResult - } catch (error) { - if (abortSignal?.aborted) { - throw toAbortError(abortSignal) - } - throw error - } finally { - abortSignal?.removeEventListener('abort', onAbort) - } + return runGeneration( + (requestId) => + NativeCoreAI.generateVideo(requestId, this.config, prompt, options), + onProgress, + abortSignal + ) } async doGenerate( @@ -575,6 +623,7 @@ export class CoreAIVideoModel implements VideoModelV4 { duration: options.duration, fps: options.fps, seed: options.seed, + onProgress: this.onProgress, ...(options.providerOptions?.['core-ai'] ?? {}), }, options.abortSignal diff --git a/packages/core-ai/src/index.ts b/packages/core-ai/src/index.ts index 90f615e2..e8c95eef 100644 --- a/packages/core-ai/src/index.ts +++ b/packages/core-ai/src/index.ts @@ -18,6 +18,7 @@ export type { CoreAIEmbeddingResult, CoreAIGenerationOptions, CoreAIGenerationPart, + CoreAIGenerationProgress, CoreAIImageGenerationOptions, CoreAIImageGenerationResult, CoreAIImageInput, diff --git a/packages/core-ai/src/types.ts b/packages/core-ai/src/types.ts index c0cc700d..024e2670 100644 --- a/packages/core-ai/src/types.ts +++ b/packages/core-ai/src/types.ts @@ -184,6 +184,15 @@ export interface CoreAITranscriptionResult { metadata?: CodegenTypes.UnsafeObject } +export interface CoreAIGenerationProgress { + // Images report each denoising step once it is done, from step 1. Videos + // report encoding the prompt, each denoising step as it starts, from step + // 0, then decoding and assembling the frames + phase: 'encoding' | 'denoising' | 'decoding' | 'assembling' + step: number + totalSteps: number +} + export interface CoreAIImageGenerationOptions { n?: number size?: `${number}x${number}` @@ -203,6 +212,8 @@ export interface CoreAIImageGenerationOptions { // How many tokens of the input image FLUX.2 attends to: more follows the // image more closely and runs slower. Defaults to the finest grid the export has referenceGrid?: 'full' | 'half' | 'quarter' + // Called after each denoising step + onProgress?: (progress: CoreAIGenerationProgress) => void } export interface CoreAIImageGenerationResult { @@ -369,6 +380,8 @@ export interface CoreAIVideoGenerationOptions { vaeTileSize?: number // Loads each component only while it runs, true by default lazyModelLoading?: boolean + // Called as each phase and denoising step starts + onProgress?: (progress: CoreAIGenerationProgress) => void } export interface CoreAIVideoGenerationResult { diff --git a/website/src/content/docs/docs/core-ai/images-and-video.md b/website/src/content/docs/docs/core-ai/images-and-video.md index 05a14d90..7d5588ed 100644 --- a/website/src/content/docs/docs/core-ai/images-and-video.md +++ b/website/src/content/docs/docs/core-ai/images-and-video.md @@ -52,6 +52,26 @@ const { image } = await generateImage({ - One image is edited per call, and there's no inpainting: extra images and a `mask` are left out with a warning. - `coreAI.imageModel(config).generate(prompt, { imageUri })` (or `imageBase64`) edits outside the AI SDK. +## Progress + +Generation takes seconds per image and minutes per video. The AI SDK has no progress API, and `providerOptions` only takes JSON, so pass the model through `withProgress`. The callback gets `{ phase, step, totalSteps }`: + +```typescript +const { video } = await generateVideo({ + model: coreAI + .videoModel({ id: 'wan2.1-t2v-1.3b', source: wanDirectory }) + .withProgress(({ phase, step, totalSteps }) => { + setStatus(`${phase} ${step}/${totalSteps}`) + }), + prompt: 'a red ball rolling across a wooden table', +}) +``` + +- Images report `denoising` after each step, from step 1. +- Videos report `encoding` as the prompt is encoded, `denoising` as each step starts (from step 0), then `decoding` and `assembling` the frames. +- `withProgress` returns a copy of the model, so make one per call. The `generate(prompt, { onProgress })` methods of `coreAI.imageModel(...)` and `coreAI.videoModel(...)` take the callback directly. +- An `abortSignal` stops image and video generation after the current step. + ## Video Generation Pass `coreAI.videoModel(config)` to `experimental_generateVideo` to get a short MP4 clip. From 4f748cf32396f77371c61570ca9286a9ccb4e64c Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 1 Nov 2026 13:39:51 +0100 Subject: [PATCH 2/3] fix(core-ai): stop reporting progress once generation is cancelled Check for cancellation before reporting a step, so a cancelled call doesn't report work it skips. withProgress now wins over provider options, and both native progress callbacks are Sendable. The image abort harness test allows for a step that is already running. Co-Authored-By: Claude Opus 5.5 --- harness/core-ai/image-generation.device.harness.ts | 3 ++- packages/core-ai/ios/CoreAIImpl.swift | 8 +++++--- packages/core-ai/ios/CoreAIVideoGeneration.swift | 4 +++- packages/core-ai/src/ai-sdk.ts | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/harness/core-ai/image-generation.device.harness.ts b/harness/core-ai/image-generation.device.harness.ts index de13e2ea..5a200d11 100644 --- a/harness/core-ai/image-generation.device.harness.ts +++ b/harness/core-ai/image-generation.device.harness.ts @@ -111,6 +111,7 @@ describe('core-ai image generation', () => { }) await expect(result).rejects.toThrow() - expect(progress).toHaveLength(1) + // A step already running when the cancel lands still finishes + expect(progress.length).toBeLessThan(4) }) }) diff --git a/packages/core-ai/ios/CoreAIImpl.swift b/packages/core-ai/ios/CoreAIImpl.swift index c2879439..d103bfe3 100644 --- a/packages/core-ai/ios/CoreAIImpl.swift +++ b/packages/core-ai/ios/CoreAIImpl.swift @@ -649,7 +649,7 @@ public class CoreAIImpl: NSObject { config: [String: Any], prompt: String, options: [String: Any]?, - onProgress: @escaping ([String: Any]) -> Void, + onProgress: @escaping @Sendable ([String: Any]) -> Void, resolve: @escaping (Any?) -> Void, reject: @escaping (String, String, Error?) -> Void ) { @@ -683,7 +683,7 @@ public class CoreAIImpl: NSObject { config: [String: Any], prompt: String, options: [String: Any]?, - onProgress: @escaping ([String: Any]) -> Void, + onProgress: @escaping @Sendable ([String: Any]) -> Void, resolve: @escaping (Any?) -> Void, reject: @escaping (String, String, Error?) -> Void ) { @@ -706,13 +706,15 @@ public class CoreAIImpl: NSObject { startingImage: editsImage ? startingImage : nil ) let result = try await pipeline.generateImages(configuration: configuration) { progress in + // Steps after a cancel aren't reported, since generation stops + guard !Task.isCancelled else { return false } onProgress([ "requestId": requestId, "phase": "denoising", "step": progress.step, "totalSteps": progress.totalSteps ]) - return !Task.isCancelled + return true } try Task.checkCancellation() let images = try result.images.map(Self.pngBase64) diff --git a/packages/core-ai/ios/CoreAIVideoGeneration.swift b/packages/core-ai/ios/CoreAIVideoGeneration.swift index 29bcab04..7889fcc7 100644 --- a/packages/core-ai/ios/CoreAIVideoGeneration.swift +++ b/packages/core-ai/ios/CoreAIVideoGeneration.swift @@ -24,12 +24,14 @@ enum VideoGeneration { let (pipeline, shape) = try await self.pipeline(at: sourceURL, options: options) let configuration = try makeConfiguration(prompt: prompt, pipeline: pipeline, shape: shape, options: options) let result = try await pipeline.generateVideo(configuration: configuration) { progress in + // Phases after a cancel aren't reported, since generation stops + guard !Task.isCancelled else { return false } onProgress([ "phase": phaseName(progress.phase), "step": progress.step, "totalSteps": progress.totalSteps ]) - return !Task.isCancelled + return true } try Task.checkCancellation() diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index 6a670ec9..f4fad7b1 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -475,8 +475,8 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { aspectRatio: options.aspectRatio, seed: options.seed, ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), - onProgress: this.onProgress, ...(options.providerOptions?.['core-ai'] ?? {}), + onProgress: this.onProgress, }, options.abortSignal ) @@ -623,8 +623,8 @@ export class CoreAIVideoModel implements VideoModelV4 { duration: options.duration, fps: options.fps, seed: options.seed, - onProgress: this.onProgress, ...(options.providerOptions?.['core-ai'] ?? {}), + onProgress: this.onProgress, }, options.abortSignal ) From fa8cfe1f5e757e6db64ec875ab313ea18736ee22 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Mon, 2 Nov 2026 00:08:56 +0100 Subject: [PATCH 3/3] refactor(core-ai): take onProgress in providerOptions instead of withProgress Co-Authored-By: Claude Opus 5.5 --- .../image-generation.device.harness.ts | 26 +++++++++++++----- .../video-generation.device.harness.ts | 12 +++++++-- packages/core-ai/src/__tests__/ai-sdk.test.ts | 4 +-- packages/core-ai/src/ai-sdk.ts | 27 ++----------------- .../docs/docs/core-ai/images-and-video.md | 19 ++++++++----- 5 files changed, 45 insertions(+), 43 deletions(-) diff --git a/harness/core-ai/image-generation.device.harness.ts b/harness/core-ai/image-generation.device.harness.ts index 5a200d11..a17fbaa6 100644 --- a/harness/core-ai/image-generation.device.harness.ts +++ b/harness/core-ai/image-generation.device.harness.ts @@ -85,10 +85,17 @@ describe('core-ai image generation', () => { it('reports each denoising step in order', async () => { const progress: CoreAIGenerationProgress[] = [] await generateImage({ - model: model.withProgress((update) => progress.push(update)), + model, prompt, seed: 1, - providerOptions: { 'core-ai': { stepCount: 2 } }, + providerOptions: { + 'core-ai': { + stepCount: 2, + // providerOptions is typed as JSON, so a callback needs a cast + onProgress: ((update: CoreAIGenerationProgress) => + progress.push(update)) as any, + }, + }, }) expect(progress).toEqual([ @@ -101,13 +108,18 @@ describe('core-ai image generation', () => { const controller = new AbortController() const progress: CoreAIGenerationProgress[] = [] const result = generateImage({ - model: model.withProgress((update) => { - progress.push(update) - controller.abort() - }), + model, prompt, abortSignal: controller.signal, - providerOptions: { 'core-ai': { stepCount: 4 } }, + providerOptions: { + 'core-ai': { + stepCount: 4, + onProgress: ((update: CoreAIGenerationProgress) => { + progress.push(update) + controller.abort() + }) as any, + }, + }, }) await expect(result).rejects.toThrow() diff --git a/harness/core-ai/video-generation.device.harness.ts b/harness/core-ai/video-generation.device.harness.ts index 3348b077..a2f0a458 100644 --- a/harness/core-ai/video-generation.device.harness.ts +++ b/harness/core-ai/video-generation.device.harness.ts @@ -47,10 +47,18 @@ describe('core-ai video generation', () => { it('reports each phase and denoising step in order', async () => { const progress: CoreAIGenerationProgress[] = [] await generateVideo({ - model: model.withProgress((update) => progress.push(update)), + model, prompt: 'a red ball rolling across a wooden table', seed: 1, - providerOptions: { 'core-ai': { stepCount: 2, cfgCutoff: 1 } }, + providerOptions: { + 'core-ai': { + stepCount: 2, + cfgCutoff: 1, + // providerOptions is typed as JSON, so a callback needs a cast + onProgress: ((update: CoreAIGenerationProgress) => + progress.push(update)) as any, + }, + }, }) expect(progress.map(({ phase, step }) => `${phase} ${step}`)).toEqual([ diff --git a/packages/core-ai/src/__tests__/ai-sdk.test.ts b/packages/core-ai/src/__tests__/ai-sdk.test.ts index 9b6a02d3..664a3207 100644 --- a/packages/core-ai/src/__tests__/ai-sdk.test.ts +++ b/packages/core-ai/src/__tests__/ai-sdk.test.ts @@ -1340,7 +1340,7 @@ describe('core ai generation progress', () => { }) const onProgress = mock(() => {}) - await imageModel().withProgress(onProgress).doGenerate({ + await imageModel().doGenerate({ prompt: 'a cat', n: 1, size: undefined, @@ -1348,7 +1348,7 @@ describe('core ai generation progress', () => { seed: undefined, files: undefined, mask: undefined, - providerOptions: {}, + providerOptions: { 'core-ai': { onProgress: onProgress as any } }, }) expect(onProgress.mock.calls).toEqual([ diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index f4fad7b1..db2805f0 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -435,21 +435,10 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { readonly modelId: string readonly maxImagesPerCall = 1 readonly config: CoreAIModelConfig - private readonly onProgress?: (progress: CoreAIGenerationProgress) => void - constructor( - config: CoreAIModelConfig, - onProgress?: (progress: CoreAIGenerationProgress) => void - ) { + constructor(config: CoreAIModelConfig) { this.config = { ...config, task: 'diffusion' } this.modelId = getModelId(config) - this.onProgress = onProgress - } - - // The same model, reporting the progress of the calls it is passed to. - // providerOptions only takes JSON, so the AI SDK can't carry a callback - withProgress(onProgress: (progress: CoreAIGenerationProgress) => void) { - return new CoreAIImageGenerationModel(this.config, onProgress) } // The abort signal stops generation after the current denoising step @@ -476,7 +465,6 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { seed: options.seed, ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), ...(options.providerOptions?.['core-ai'] ?? {}), - onProgress: this.onProgress, }, options.abortSignal ) @@ -551,20 +539,10 @@ export class CoreAIVideoModel implements VideoModelV4 { readonly modelId: string readonly maxVideosPerCall = 1 readonly config: CoreAIModelConfig - private readonly onProgress?: (progress: CoreAIGenerationProgress) => void - constructor( - config: CoreAIModelConfig, - onProgress?: (progress: CoreAIGenerationProgress) => void - ) { + constructor(config: CoreAIModelConfig) { this.config = { ...config, task: 'video-diffusion' } this.modelId = getModelId(config) - this.onProgress = onProgress - } - - // The same model, reporting the progress of the calls it is passed to - withProgress(onProgress: (progress: CoreAIGenerationProgress) => void) { - return new CoreAIVideoModel(this.config, onProgress) } // Generates an MP4 from the prompt. Takes minutes on device, and the abort @@ -624,7 +602,6 @@ export class CoreAIVideoModel implements VideoModelV4 { fps: options.fps, seed: options.seed, ...(options.providerOptions?.['core-ai'] ?? {}), - onProgress: this.onProgress, }, options.abortSignal ) diff --git a/website/src/content/docs/docs/core-ai/images-and-video.md b/website/src/content/docs/docs/core-ai/images-and-video.md index 7d5588ed..686072be 100644 --- a/website/src/content/docs/docs/core-ai/images-and-video.md +++ b/website/src/content/docs/docs/core-ai/images-and-video.md @@ -54,22 +54,27 @@ const { image } = await generateImage({ ## Progress -Generation takes seconds per image and minutes per video. The AI SDK has no progress API, and `providerOptions` only takes JSON, so pass the model through `withProgress`. The callback gets `{ phase, step, totalSteps }`: +Generation takes seconds per image and minutes per video. Pass `onProgress` in `providerOptions['core-ai']` to follow it. The callback gets `{ phase, step, totalSteps }`: ```typescript +import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai' + +const onProgress = ({ phase, step, totalSteps }: CoreAIGenerationProgress) => + setStatus(`${phase} ${step}/${totalSteps}`) + const { video } = await generateVideo({ - model: coreAI - .videoModel({ id: 'wan2.1-t2v-1.3b', source: wanDirectory }) - .withProgress(({ phase, step, totalSteps }) => { - setStatus(`${phase} ${step}/${totalSteps}`) - }), + model: coreAI.videoModel({ id: 'wan2.1-t2v-1.3b', source: wanDirectory }), prompt: 'a red ball rolling across a wooden table', + providerOptions: { + // providerOptions is typed as JSON, so a callback needs a cast + 'core-ai': { onProgress: onProgress as any }, + }, }) ``` - Images report `denoising` after each step, from step 1. - Videos report `encoding` as the prompt is encoded, `denoising` as each step starts (from step 0), then `decoding` and `assembling` the frames. -- `withProgress` returns a copy of the model, so make one per call. The `generate(prompt, { onProgress })` methods of `coreAI.imageModel(...)` and `coreAI.videoModel(...)` take the callback directly. +- The `generate(prompt, { onProgress })` methods of `coreAI.imageModel(...)` and `coreAI.videoModel(...)` take the callback without a cast. - An `abortSignal` stops image and video generation after the current step. ## Video Generation