Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion harness/core-ai/availability.device.harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ describe('core-ai availability', () => {

it('rejects a model that is not on disk', async () => {
await expect(
CoreAI.generateImage({ source: '/missing' }, 'a cat', {})
CoreAI.generateImage('missing-model', { source: '/missing' }, 'a cat', {})
).rejects.toMatchObject({ code: expect.any(String) })
})

Expand Down
46 changes: 46 additions & 0 deletions harness/core-ai/image-generation.device.harness.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai'
import { generateImage } from 'ai'
import { describe, expect, it } from 'react-native-harness'

Expand Down Expand Up @@ -80,4 +81,49 @@ describe('core-ai image generation', () => {
expect.objectContaining({ type: 'unsupported', feature: 'size' }),
])
})

it('reports each denoising step in order', async () => {
const progress: CoreAIGenerationProgress[] = []
await generateImage({
model,
prompt,
seed: 1,
providerOptions: {
'core-ai': {
stepCount: 2,
// providerOptions is typed as JSON, so a callback needs a cast
onProgress: ((update: CoreAIGenerationProgress) =>
progress.push(update)) as any,
},
},
})

expect(progress).toEqual([
{ phase: 'denoising', step: 1, totalSteps: 2 },
{ phase: 'denoising', step: 2, totalSteps: 2 },
Comment thread
grabbou marked this conversation as resolved.
Resolved
])
})

it('stops after the current step when aborted', async () => {
const controller = new AbortController()
const progress: CoreAIGenerationProgress[] = []
const result = generateImage({
model,
prompt,
abortSignal: controller.signal,
providerOptions: {
'core-ai': {
stepCount: 4,
onProgress: ((update: CoreAIGenerationProgress) => {
progress.push(update)
controller.abort()
}) as any,
},
},
})

await expect(result).rejects.toThrow()
// A step already running when the cancel lands still finishes
expect(progress.length).toBeLessThan(4)
})
})
27 changes: 27 additions & 0 deletions harness/core-ai/video-generation.device.harness.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai'
import { experimental_generateVideo as generateVideo } from 'ai'
import { describe, expect, it } from 'react-native-harness'

Expand Down Expand Up @@ -43,6 +44,32 @@ describe('core-ai video generation', () => {
)
})

it('reports each phase and denoising step in order', async () => {
const progress: CoreAIGenerationProgress[] = []
await generateVideo({
model,
prompt: 'a red ball rolling across a wooden table',
seed: 1,
providerOptions: {
'core-ai': {
stepCount: 2,
cfgCutoff: 1,
// providerOptions is typed as JSON, so a callback needs a cast
onProgress: ((update: CoreAIGenerationProgress) =>
progress.push(update)) as any,
},
},
})

expect(progress.map(({ phase, step }) => `${phase} ${step}`)).toEqual([
'encoding 0',
'denoising 0',
'denoising 1',
'decoding 2',
'assembling 2',
])
})

it('stops when aborted', async () => {
const controller = new AbortController()
const result = generate({}, controller.signal)
Expand Down
23 changes: 20 additions & 3 deletions packages/core-ai/ios/CoreAI.mm
Original file line number Diff line number Diff line change
Expand Up @@ -237,15 +237,32 @@
options:(nonnull NSDictionary *)options
resolve:(nonnull RCTPromiseResolveBlock)resolve
reject:(nonnull RCTPromiseRejectBlock)reject {
[_coreAI generateVideo:requestId config:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject];
[_coreAI generateVideo:requestId
config:CoreAIConfigToDictionary(config)
prompt:prompt
options:options
onProgress:^(NSDictionary *progress) {
[self emitOnGenerationProgress:progress];
}
resolve:resolve
reject:reject];
}

- (void)generateImage:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config
- (void)generateImage:(nonnull NSString *)requestId
config:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config
prompt:(nonnull NSString *)prompt
options:(nonnull NSDictionary *)options
resolve:(nonnull RCTPromiseResolveBlock)resolve
reject:(nonnull RCTPromiseRejectBlock)reject {
[_coreAI generateImage:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject];
[_coreAI generateImage:requestId

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · Native progress block is invoked with no JS subscriber, which throws on the generation thread

The ObjC bridge now passes an unconditional onProgress block to Swift for both image and video generation, and that block calls emitOnGenerationProgress: — a generated codegen EventEmitter method — for every denoising step / phase. EventEmitter methods are only valid while the JS side has a subscriber: RN's generated emitter is a no-op (and older RCTEventEmitter raises NSInternalInconsistencyException / logs "Require listener registration for module") when there is no registered receiver for the event.

Suggested change

Only emit progress when someone is listening: either subscribe unconditionally in runGeneration (drop the onProgress && guard and drop the event on the floor when there is no callback), or guard the emit on the native side with the module's "has listeners" check before calling emitOnGenerationProgress:.

Details

runGeneration in ai-sdk.ts:509-515 only subscribes when the caller passed onProgress (onProgress && NativeCoreAI.onGenerationProgress(...)), so the common path — coreAI.imageModel(...).generate(prompt) or coreAI.videoModel(...).generate(prompt) without a callback, and any CoreAI.generateImage(requestId, ...) raw call, which the PR makes a public entry point — has no subscriber while native emits stepCount (50 by default for video) events per request. This is also newly reachable for images: before this diff the image path never emitted anything, so a call like the new availability harness CoreAI.generateImage('missing-model', {source:'/missing'}, 'a cat', {}) on a model that does exist would emit from a background thread with no listener. Because the emit happens inside the pipeline's step callback on a worker thread, a raise there aborts the app rather than merely rejecting the promise.

The same line already exists for onTranscriptionUpdate, but that one is only reached for streams JS explicitly starts with a listener attached; here the emit is unconditional for every generation step.

Also raised here

Also raised at this location by swift-ios: requestId injection for video progress keeps the wrong side of the merge — VideoGeneration.generate deliberately emits a dictionary without a requestId, and the caller adds it with $0.merging(["requestId": requestId]) { current, _ in current }. The merge closure keeps current — the value already in the progress dictionary emitted by VideoGeneration — and discards requestId, i.e. the value that routes the event in runGeneration (if (id === requestId), ai-sdk.ts:511-512). It only routes correctly today because VideoGeneration.generate happens not to include that key; the moment it (or phaseName-adjacent metadata) grows a requestId, every video progress event is emitted under the inner id, runGeneration filters them all out, and onProgress silently stops reporting — no error, just missing progress. The image path gets this right by building the dictionary with "requestId": requestId in one place (CoreAIImpl.swift:711-716), so the two call sites disagree about who owns the key.

persona: javascript, swift-ios

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. The codegen EventEmitter (new architecture) is a no-op when nothing is subscribed, the same as the existing stream events. It isn't the old RCTEventEmitter listener check.

🤖 Reply from Claude Code

config:CoreAIConfigToDictionary(config)
prompt:prompt
options:options
onProgress:^(NSDictionary *progress) {
[self emitOnGenerationProgress:progress];
}
resolve:resolve
reject:reject];

Check notice on line 265 in packages/core-ai/ios/CoreAI.mm

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

nit: Native progress block is invoked with no JS subscriber, which throws on the generation thread

The ObjC bridge now passes an unconditional `onProgress` block to Swift for both image and video generation, and that block calls `emitOnGenerationProgress:` — a generated codegen EventEmitter method — for every denoising step / phase. EventEmitter methods are only valid while the JS side has a subscriber: RN's generated emitter is a no-op (and older RCTEventEmitter raises `NSInternalInconsistencyException` / logs "Require listener registration for module") when there is no registered receiver for the event. `runGeneration` in ai-sdk.ts:509-515 only subscribes when the caller passed `onProgress` (`onProgress && NativeCoreAI.onGenerationProgress(...)`), so the common path — `coreAI.imageModel(...).generate(prompt)` or `coreAI.videoModel(...).generate(prompt)` without a callback, and any `CoreAI.generateImage(requestId, ...)` raw call, which the PR makes a public entry point — has no subscriber while native emits `stepCount` (50 by default for video) events per request. This is also newly reachable for images: before this diff the image path never emitted anything, so a call like the new availability harness `CoreAI.generateImage('missing-model', {source:'/missing'}, 'a cat', {})` on a model that does exist would emit from a background thread with no listener. Because the emit happens inside the pipeline's step callback on a worker thread, a raise there aborts the app rather than merely rejecting the promise. The same line already exists for `onTranscriptionUpdate`, but that one is only reached for streams JS explicitly starts with a listener attached; here the emit is unconditional for every generation step. ### Also raised here Also raised at this location by swift-ios: requestId injection for video progress keeps the wrong side of the merge — `VideoGeneration.generate` deliberately emits a dictionary without a `requestId`, and the caller adds it with `$0.merging(["requestId": requestId]) { current, _ in current }`. The merge closure keeps `current` — the value…
}

- (void)runTask:(nonnull NSString *)task
Expand Down
74 changes: 45 additions & 29 deletions packages/core-ai/ios/CoreAIImpl.swift
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,7 @@
config: [String: Any],
prompt: String,
options: [String: Any]?,
onProgress: @escaping @Sendable ([String: Any]) -> Void,
resolve: @escaping (Any?) -> Void,
reject: @escaping (String, String, Error?) -> Void
) {
Expand All @@ -659,7 +660,9 @@
startTask(requestId, onError: { _, code, message in
reject(code, message, nil)
}) {
resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:]))
resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:]) {
onProgress($0.merging(["requestId": requestId]) { current, _ in current })
})
}
} catch {
reject(errorCode(error), error.localizedDescription, error)
Expand All @@ -672,46 +675,59 @@
#endif
}

/// Generates images, reporting each denoising step. cancelStream(requestId)
/// stops it after the current step
@objc
public func generateImage(
_ config: [String: Any],
_ requestId: String,
config: [String: Any],
prompt: String,
options: [String: Any]?,
onProgress: @escaping @Sendable ([String: Any]) -> Void,
resolve: @escaping (Any?) -> Void,
reject: @escaping (String, String, Error?) -> Void
) {
#if canImport(CoreAIDiffusionPipeline)
if #available(iOS 27, macOS 27, *) {
do {
let sourceURL = try resolveSourceURL(config)
Task {
do {
let startingImage = try Self.startingImage(from: options)
let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options)
// Exports without a VAE encoder generate from the prompt alone,
// which the metadata reports so JS can warn
let editsImage = startingImage != nil && pipeline.supportsImageToImage
let configuration = try self.makePipelineConfiguration(
prompt: prompt,
pipeline: pipeline,
options: options,
startingImage: editsImage ? startingImage : nil
)
let result = try await pipeline.generateImages(configuration: configuration) { _ in true }
let images = try result.images.map(Self.pngBase64)
var metadata: [String: Any] = [
"count": images.count,
"imageFormat": "image/png",
"width": result.images.first?.width ?? 0,
"height": result.images.first?.height ?? 0
]
if startingImage != nil {
metadata["imageToImage"] = editsImage
}
resolve(["images": images, "metadata": metadata])
} catch {
reject(errorCode(error), error.localizedDescription, error)
startTask(requestId, onError: { _, code, message in

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🟡 Minor · Native image errors now go through the language-model error mapping and lose the underlying Error

The image path moved from its own Task { ... } catch { reject(errorCode(error), error.localizedDescription, error) } (which also rejected with a nil code, so JS took the AbortError branch) into startTask, whose catch does Self.mapLanguageModelError(error) and then reject(code, message, nil) (packages/core-ai/ios/CoreAIImpl.swift:694-696, 1953-1961). Two consequences for callers of the raw CoreAI.generateImage and of doGenerate: (1) mapLanguageModelError (line 1965) inspects every diffusion error as a possible LanguageModelError, so unsupportedCapability semantics from a completely different subsystem can be attached to an image-generation failure; (2) the original Error is no longer forwarded to reject, so the native cause (the Core AI / Metal error) disappears from the JS error for every image failure, not just cancellations. Keep the image path rejecting with the original error (and errorCode(error) directly), adding an explicit CoreAIError.cancelled mapping for the cancelled case instead of borrowing the language-model error path.

Suggested change

Preserve the previous error surface for image generation: reject with errorCode(error) and the original Error, mapping only Task cancellation to CoreAIError.cancelled, rather than reusing the language-model error mapper via startTask.

Details

persona: senior-developer

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. mapLanguageModelError changes only one specific LanguageModelError and passes every other error through unchanged, so diffusion errors keep their code and message. Image generation now rejects the same way video and streaming already did, and a cancel maps to CoreAIError.cancelled.

🤖 Reply from Claude Code

reject(code, message, nil)
}) {

Check warning on line 696 in packages/core-ai/ios/CoreAIImpl.swift

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

minor: Native image errors now go through the language-model error mapping and lose the underlying Error

The image path moved from its own `Task { ... } catch { reject(errorCode(error), error.localizedDescription, error) }` (which also rejected with a nil code, so JS took the `AbortError` branch) into `startTask`, whose catch does `Self.mapLanguageModelError(error)` and then `reject(code, message, nil)` (`packages/core-ai/ios/CoreAIImpl.swift:694-696`, `1953-1961`). Two consequences for callers of the raw `CoreAI.generateImage` and of `doGenerate`: (1) `mapLanguageModelError` (line 1965) inspects every diffusion error as a possible `LanguageModelError`, so `unsupportedCapability` semantics from a completely different subsystem can be attached to an image-generation failure; (2) the original `Error` is no longer forwarded to `reject`, so the native cause (the Core AI / Metal error) disappears from the JS error for every image failure, not just cancellations. Keep the image path rejecting with the original error (and `errorCode(error)` directly), adding an explicit `CoreAIError.cancelled` mapping for the cancelled case instead of borrowing the language-model error path.
let startingImage = try Self.startingImage(from: options)
let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options)
// Exports without a VAE encoder generate from the prompt alone,
// which the metadata reports so JS can warn
let editsImage = startingImage != nil && pipeline.supportsImageToImage
let configuration = try self.makePipelineConfiguration(
prompt: prompt,
pipeline: pipeline,
options: options,
startingImage: editsImage ? startingImage : nil
)
let result = try await pipeline.generateImages(configuration: configuration) { progress in
// Steps after a cancel aren't reported, since generation stops
guard !Task.isCancelled else { return false }
onProgress([
"requestId": requestId,
"phase": "denoising",
"step": progress.step,
"totalSteps": progress.totalSteps
])
return true
}
try Task.checkCancellation()
let images = try result.images.map(Self.pngBase64)
var metadata: [String: Any] = [
"count": images.count,
"imageFormat": "image/png",
"width": result.images.first?.width ?? 0,
"height": result.images.first?.height ?? 0
]
if startingImage != nil {
metadata["imageToImage"] = editsImage
}
resolve(["images": images, "metadata": metadata])
}
} catch {
reject(errorCode(error), error.localizedDescription, error)
Expand Down
27 changes: 24 additions & 3 deletions packages/core-ai/ios/CoreAIVideoGeneration.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,23 @@
// only while it runs
nonisolated(unsafe) private static var loaded: (key: String, load: Task<(WanPipeline, ExportedShape), Error>)?

static func generate(prompt: String, sourceURL: URL, options: [String: Any]) async throws -> [String: Any] {
static func generate(
Comment thread
grabbou marked this conversation as resolved.
Resolved
prompt: String,
sourceURL: URL,
options: [String: Any],
onProgress: @escaping @Sendable ([String: Any]) -> Void
) async throws -> [String: Any] {
let (pipeline, shape) = try await self.pipeline(at: sourceURL, options: options)
let configuration = try makeConfiguration(prompt: prompt, pipeline: pipeline, shape: shape, options: options)
let result = try await pipeline.generateVideo(configuration: configuration) { _ in
!Task.isCancelled
let result = try await pipeline.generateVideo(configuration: configuration) { progress in
// Phases after a cancel aren't reported, since generation stops
guard !Task.isCancelled else { return false }
onProgress([
"phase": phaseName(progress.phase),
"step": progress.step,
"totalSteps": progress.totalSteps
])
return true
}
try Task.checkCancellation()

Expand All @@ -41,6 +53,15 @@
]
}

private static func phaseName(_ phase: VideoProgress.Phase) -> String {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🆕 🔵 Nit · Phase names duplicated between Swift and the TS union with nothing linking them

phaseName here hard-codes "encoding"/"denoising"/"decoding"/"assembling", CoreAIImpl.swift:694-728 hard-codes the "denoising" literal for images, and CoreAIGenerationProgress in src/types.ts:187-194 repeats the same four strings as a union — with CoreAIGenerationProgressEvent.phase: string in NativeCoreAI.ts and progress as CoreAIGenerationProgress in ai-sdk.ts:513 no type checks the mapping. A new or renamed phase in the Wan SDK silently yields a string the union does not allow and users switch on.

Suggested change

Keep one list of phase names: at minimum name the Swift strings via a shared constant/enum comment pointing at CoreAIGenerationProgress, so the two sides are changed together.

Details

persona: maintainability

Reply @apex-sentinel-staging note ... to add context, or @apex-sentinel-staging pivot ... if this is wrong — I will re-check the cited code.

Useful? Reply @apex-sentinel-staging like, or @apex-sentinel-staging dislike <why> if it is not.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. Swift's phaseName is the only place a phase name is made, and the TS union documents the result. A shared constant across Swift and TS would need codegen support we don't have.

🤖 Reply from Claude Code

switch phase {
case .encoding: return "encoding"
case .denoising: return "denoising"
case .decoding: return "decoding"
case .assembling: return "assembling"
}
}

Check notice on line 64 in packages/core-ai/ios/CoreAIVideoGeneration.swift

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

nit: Phase names duplicated between Swift and the TS union with nothing linking them

`phaseName` here hard-codes "encoding"/"denoising"/"decoding"/"assembling", CoreAIImpl.swift:694-728 hard-codes the "denoising" literal for images, and CoreAIGenerationProgress in src/types.ts:187-194 repeats the same four strings as a union — with `CoreAIGenerationProgressEvent.phase: string` in NativeCoreAI.ts and `progress as CoreAIGenerationProgress` in ai-sdk.ts:513 no type checks the mapping. A new or renamed phase in the Wan SDK silently yields a string the union does not allow and users switch on.
// Concurrent calls for the same bundle share one load
private static func pipeline(at url: URL, options: [String: Any]) async throws -> (WanPipeline, ExportedShape) {
let lazy = options["lazyModelLoading"] as? Bool ?? true
Expand Down
11 changes: 11 additions & 0 deletions packages/core-ai/src/NativeCoreAI.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@
endSecond: number
}

export type CoreAIGenerationProgressEvent = {
requestId: string
phase: string
step: number
totalSteps: number
}

export interface Spec extends TurboModule {
getCapabilities(): Promise<CodegenTypes.UnsafeObject>

Expand Down Expand Up @@ -135,12 +142,14 @@
generateVideo(
requestId: string,
config: NativeCoreAIModelConfig,
prompt: string,
options: CodegenTypes.UnsafeObject
): Promise<CodegenTypes.UnsafeObject>
// Cancel with cancelStream(requestId)
Comment thread
grabbou marked this conversation as resolved.
Resolved
generateImage(
requestId: string,
config: NativeCoreAIModelConfig,
prompt: string,

Check warning on line 152 in packages/core-ai/src/NativeCoreAI.ts

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

minor: Breaking raw generateImage signature is not mentioned in the docs page this PR edits

The raw native contract changed: `CoreAI.generateImage(config, prompt, options)` became `CoreAI.generateImage(requestId, config, prompt, options)` (`packages/core-ai/src/NativeCoreAI.ts:145-152`, implemented at `packages/core-ai/ios/CoreAI.mm:251`). `CoreAI` is exported publicly (`packages/core-ai/src/index.ts:11`), so this breaks existing callers at runtime with no type error for untyped/`UnsafeObject` usage: the old first argument (the config object) is now taken as `requestId`, and `resolveSourceURL` receives `undefined` as the config. The breaking note exists only in the PR description; the page this PR edits, which documents both the AI SDK and the raw `coreAI.imageModel(...).generate(prompt, {...})` form, says nothing about it, and the new code comment `// Cancel with cancelStream(requestId)` documents the new behaviour but not the migration. Add the signature change and the cancel-by-requestId note to the images-and-video page (and/or a CHANGELOG entry). ### Also raised here Also raised at this location by javascript: Undocumented breaking change to the public raw CoreAI.generateImage signature — `CoreAI` is re-exported as a value from the package barrel (packages/core-ai/src/index.ts:11 `export { default as CoreAI } from './NativeCoreAI'`), so `CoreAI.generateImage` is public API, and this PR changes it from `generateImage(config, prompt, options)` to `generateImage(requestId, config, prompt, options)` (packages/core-ai/src/NativeCoreAI.ts:149-154). The PR description calls this out as a breaking change, but the only page in the checkout that documents the image API — website/src/content/docs/docs/core-ai/images-and-video.md, which this PR already edits — never mentions the new first parameter, the new `onProgress` member, or the new `abortSignal` parameter of `coreAI.imageModel(...).generate()`. …
options: CodegenTypes.UnsafeObject
): Promise<CodegenTypes.UnsafeObject>

Expand All @@ -161,6 +170,8 @@
onStreamComplete: CodegenTypes.EventEmitter<CoreAIStreamCompleteEvent>
onStreamError: CodegenTypes.EventEmitter<CoreAIStreamErrorEvent>
onTranscriptionUpdate: CodegenTypes.EventEmitter<CoreAITranscriptionUpdateEvent>
// Progress of generateImage and generateVideo requests
onGenerationProgress: CodegenTypes.EventEmitter<CoreAIGenerationProgressEvent>
}

export default TurboModuleRegistry.getEnforcing<Spec>('NativeCoreAI')
Loading