From 3c7c9460232b1d6f106159b6098d9c9bab759cd6 Mon Sep 17 00:00:00 2001 From: Mike Grabowski Date: Sun, 1 Nov 2026 12:57:41 +0100 Subject: [PATCH] feat(core-ai): edit images with generateImage Pass the first input image to the FLUX.2 pipeline as its starting image, with a referenceGrid option. Warn about extra images, masks and exports without image-to-image components instead of silently ignoring them. Closes #343 Co-Authored-By: Claude Opus 5.5 --- packages/core-ai/ios/CoreAIImpl.swift | 80 ++++++++++--- packages/core-ai/src/__tests__/ai-sdk.test.ts | 111 ++++++++++++++++++ packages/core-ai/src/ai-sdk.ts | 57 ++++++++- packages/core-ai/src/types.ts | 10 ++ .../docs/docs/core-ai/images-and-video.md | 19 +++ 5 files changed, 261 insertions(+), 16 deletions(-) diff --git a/packages/core-ai/ios/CoreAIImpl.swift b/packages/core-ai/ios/CoreAIImpl.swift index f6d91d6e..292bfa67 100644 --- a/packages/core-ai/ios/CoreAIImpl.swift +++ b/packages/core-ai/ios/CoreAIImpl.swift @@ -686,19 +686,29 @@ public class CoreAIImpl: NSObject { let sourceURL = try resolveSourceURL(config) Task { do { + let startingImage = try Self.startingImage(from: options) let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) - let configuration = try self.makePipelineConfiguration(prompt: prompt, pipeline: pipeline, options: options) + // Exports without a VAE encoder generate from the prompt alone, + // which the metadata reports so JS can warn + let editsImage = startingImage != nil && pipeline.supportsImageToImage + let configuration = try self.makePipelineConfiguration( + prompt: prompt, + pipeline: pipeline, + options: options, + startingImage: editsImage ? startingImage : nil + ) let result = try await pipeline.generateImages(configuration: configuration) { _ in true } let images = try result.images.map(Self.pngBase64) - resolve([ - "images": images, - "metadata": [ - "count": images.count, - "imageFormat": "image/png", - "width": result.images.first?.width ?? 0, - "height": result.images.first?.height ?? 0 - ] - ]) + var metadata: [String: Any] = [ + "count": images.count, + "imageFormat": "image/png", + "width": result.images.first?.width ?? 0, + "height": result.images.first?.height ?? 0 + ] + if startingImage != nil { + metadata["imageToImage"] = editsImage + } + resolve(["images": images, "metadata": metadata]) } catch { reject(errorCode(error), error.localizedDescription, error) } @@ -1279,7 +1289,8 @@ extension CoreAIImpl { private func makePipelineConfiguration( prompt: String, pipeline: FlowTransformerPipeline, - options: [String: Any]? + options: [String: Any]?, + startingImage: CGImage? = nil ) throws -> PipelineConfiguration { let defaults = pipeline.config let scheduler = SchedulerType(rawValue: options?["schedulerType"] as? String ?? "") ?? .discreteFlow @@ -1295,7 +1306,9 @@ extension CoreAIImpl { stepCount: stepCount, guidanceScale: guidanceScale, schedulerType: scheduler, + startingImage: startingImage, strength: Float(options?["strength"] as? Double ?? 1.0), + referenceGrid: try Self.referenceGrid(from: options?["referenceGrid"], pipeline: pipeline), encoderScaleFactor: defaults.encoderScaleFactor ?? DiffusionDefaults.Image.scaleFactor, decoderScaleFactor: defaults.decoderScaleFactor ?? DiffusionDefaults.Image.scaleFactor, decoderShiftFactor: defaults.decoderShiftFactor ?? DiffusionDefaults.Image.decoderShiftFactor, @@ -1303,6 +1316,37 @@ extension CoreAIImpl { lazyModelLoading: options?["lazyModelLoading"] as? Bool ?? true ) } + + /// The requested reference grid, or the finest one the export has. iOS + /// exports carry the half grid only + private static func referenceGrid( + from option: Any?, + pipeline: FlowTransformerPipeline + ) throws -> ReferenceGrid { + if let name = option as? String { + guard let grid = ReferenceGrid(rawValue: name) else { + throw CoreAIError.invalidInput("referenceGrid must be full, half or quarter, got \(name).") + } + return grid + } + return [.full, .half, .quarter].first { pipeline.img2imgRoutes[$0] != nil } ?? .full + } + + /// The image to edit, decoded with its EXIF orientation applied. The + /// pipeline resizes it to the model's size + private static func startingImage(from options: [String: Any]?) throws -> CGImage? { + if let base64 = options?["imageBase64"] as? String { + guard let data = Data(base64Encoded: base64), + let source = CGImageSourceCreateWithData(data as CFData, nil) else { + throw CoreAIError.invalidInput("Could not decode imageBase64.") + } + return try decodeCGImage(source, description: "imageBase64") + } + if let imageUri = options?["imageUri"] as? String { + return try loadCGImage(from: ["imageUri": imageUri]) + } + return nil + } } #endif @@ -1804,11 +1848,17 @@ extension CoreAIImpl { url = URL(fileURLWithPath: imageUri) } - guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), - let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any], + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil) else { + throw CoreAIError.invalidInput("Could not load image at \(url.path).") + } + return try decodeCGImage(source, description: "image at \(url.path)") + } + + static func decodeCGImage(_ source: CGImageSource, description: String) throws -> CGImage { + guard let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any], let width = properties[kCGImagePropertyPixelWidth] as? Int, let height = properties[kCGImagePropertyPixelHeight] as? Int else { - throw CoreAIError.invalidInput("Could not load image at \(url.path).") + throw CoreAIError.invalidInput("Could not load \(description).") } // Decode with the EXIF orientation applied, so pixel coordinates match // the image as it is displayed @@ -1818,7 +1868,7 @@ extension CoreAIImpl { kCGImageSourceThumbnailMaxPixelSize: max(width, height) ] guard let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else { - throw CoreAIError.invalidInput("Could not load image at \(url.path).") + throw CoreAIError.invalidInput("Could not load \(description).") } return image } diff --git a/packages/core-ai/src/__tests__/ai-sdk.test.ts b/packages/core-ai/src/__tests__/ai-sdk.test.ts index 85fe92d1..308deaf0 100644 --- a/packages/core-ai/src/__tests__/ai-sdk.test.ts +++ b/packages/core-ai/src/__tests__/ai-sdk.test.ts @@ -68,6 +68,24 @@ const generateVideo = mock( seed: 1, }) ) +const generateImage = mock( + async ( + _config: Record, + _prompt: string, + options: Record + ): Promise> => ({ + images: ['AAAA'], + metadata: { + count: 1, + imageFormat: 'image/png', + width: 512, + height: 512, + ...(options.imageUri || options.imageBase64 + ? { imageToImage: true } + : {}), + }, + }) +) const embed = mock( async ( _config: Record, @@ -122,6 +140,7 @@ mock.module('react-native', () => ({ streamText, cancelStream, generateVideo, + generateImage, embed, runTask, createLanguageSession, @@ -1203,3 +1222,95 @@ describe('core ai video generation', () => { ) }) }) + +describe('core ai image editing', () => { + const imageModel = () => coreAI.imageModel({ id: 'flux', source: '/flux' }) + + const callOptions = (overrides: object = {}) => ({ + prompt: 'make it blue', + n: 1, + size: undefined, + aspectRatio: undefined, + seed: undefined, + files: undefined, + mask: undefined, + providerOptions: {}, + ...overrides, + }) + + const lastOptions = () => generateImage.mock.calls.at(-1)![2] + + test('passes the first image as base64', async () => { + const { warnings } = await imageModel().doGenerate( + callOptions({ + files: [ + { + type: 'file', + mediaType: 'image/png', + data: new Uint8Array([1, 2, 3]), + }, + ], + providerOptions: { 'core-ai': { referenceGrid: 'half' } }, + }) + ) + + expect(lastOptions()).toMatchObject({ + imageBase64: 'AQID', + referenceGrid: 'half', + }) + expect(warnings).toEqual([]) + }) + + test('passes data urls as base64 and local urls as uris', async () => { + await imageModel().doGenerate( + callOptions({ + files: [{ type: 'url', url: 'data:image/png;base64,AQID' }], + }) + ) + expect(lastOptions()).toMatchObject({ imageBase64: 'AQID' }) + + await imageModel().doGenerate( + callOptions({ files: [{ type: 'url', url: 'file:///tmp/a.png' }] }) + ) + expect(lastOptions()).toMatchObject({ imageUri: 'file:///tmp/a.png' }) + }) + + test('warns about extra images and masks', async () => { + const file = { type: 'file', mediaType: 'image/png', data: 'AQID' } + const { warnings } = await imageModel().doGenerate( + callOptions({ files: [file, file], mask: file }) + ) + + expect(warnings).toEqual([ + expect.objectContaining({ type: 'unsupported', feature: 'files' }), + expect.objectContaining({ type: 'unsupported', feature: 'mask' }), + ]) + }) + + test('warns when the export cannot edit images', async () => { + generateImage.mockImplementationOnce(async () => ({ + images: ['AAAA'], + metadata: { + count: 1, + imageFormat: 'image/png', + width: 512, + height: 512, + imageToImage: false, + }, + })) + + const { warnings } = await imageModel().doGenerate( + callOptions({ + files: [{ type: 'file', mediaType: 'image/png', data: 'AQID' }], + }) + ) + + expect(warnings).toEqual([ + expect.objectContaining({ + type: 'unsupported', + feature: 'files', + details: expect.stringContaining('no image-to-image components'), + }), + ]) + }) +}) diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index bf914dfe..d6fdf9b3 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -10,6 +10,7 @@ import type { Experimental_VideoModelV4Result as VideoModelV4Result, ImageModelV4, ImageModelV4CallOptions, + ImageModelV4File, LanguageModelV4, LanguageModelV4CallOptions, LanguageModelV4Content, @@ -456,12 +457,16 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { size: options.size, aspectRatio: options.aspectRatio, seed: options.seed, + ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), ...(options.providerOptions?.['core-ai'] ?? {}), }) return { images: result.images, - warnings: getImageWarnings(options, result.metadata), + warnings: [ + ...getImageInputWarnings(options, result.metadata), + ...getImageWarnings(options, result.metadata), + ], response: { timestamp: new Date(), modelId: this.modelId, @@ -781,6 +786,56 @@ export class CoreAITranscriptionModel implements TranscriptionModelV4 { } } +// The image to edit, as base64 data or a local file. Remote images are +// downloaded first, since native reads local files only +async function toImageInput( + file: ImageModelV4File +): Promise> { + if (file.type === 'file') { + return { imageBase64: toBase64(file.data) } + } + if (file.url.startsWith('data:')) { + return { imageBase64: file.url.slice(file.url.indexOf(',') + 1) } + } + if (/^https?:/.test(file.url)) { + const response = await fetch(file.url) + const data = new Uint8Array(await response.arrayBuffer()) + return { imageBase64: toBase64(data) } + } + return { imageUri: file.url } +} + +// Core AI edits one image at a time and has no inpainting +function getImageInputWarnings( + options: ImageModelV4CallOptions, + metadata: CoreAIImageGenerationResult['metadata'] +): SharedV4Warning[] { + const warnings: SharedV4Warning[] = [] + if (metadata?.imageToImage === false) { + warnings.push({ + type: 'unsupported', + feature: 'files', + details: + 'This export has no image-to-image components, so the image was generated from the prompt alone.', + }) + } else if (options.files && options.files.length > 1) { + warnings.push({ + type: 'unsupported', + feature: 'files', + details: 'Core AI image models edit the first image only.', + }) + } + if (options.mask) { + warnings.push({ + type: 'unsupported', + feature: 'mask', + details: + 'Core AI image models have no inpainting, so the mask was left out.', + }) + } + return warnings +} + // Exported models generate one fixed size, which the metadata reports function getImageWarnings( options: ImageModelV4CallOptions, diff --git a/packages/core-ai/src/types.ts b/packages/core-ai/src/types.ts index 35fb8096..c0cc700d 100644 --- a/packages/core-ai/src/types.ts +++ b/packages/core-ai/src/types.ts @@ -196,6 +196,13 @@ export interface CoreAIImageGenerationOptions { decodeResolution?: 'auto' | 'full' | 'half' | 'tiled' strength?: number lazyModelLoading?: boolean + // An image to edit, as a local file path or `file://` URI, or as base64 + // data. Needs an export with image-to-image components + imageUri?: string + imageBase64?: string + // How many tokens of the input image FLUX.2 attends to: more follows the + // image more closely and runs slower. Defaults to the finest grid the export has + referenceGrid?: 'full' | 'half' | 'quarter' } export interface CoreAIImageGenerationResult { @@ -205,6 +212,9 @@ export interface CoreAIImageGenerationResult { imageFormat: 'image/png' width: number height: number + // False when an input image was given but the export can't edit images, + // so the result was generated from the prompt alone + imageToImage?: boolean } } diff --git a/website/src/content/docs/docs/core-ai/images-and-video.md b/website/src/content/docs/docs/core-ai/images-and-video.md index 4eb5cd61..05a14d90 100644 --- a/website/src/content/docs/docs/core-ai/images-and-video.md +++ b/website/src/content/docs/docs/core-ai/images-and-video.md @@ -33,6 +33,25 @@ const { image } = await generateImage({ - `providerOptions['core-ai']` tunes the pipeline: `stepCount`, `guidanceScale`, `negativePrompt`, `schedulerType`, `decodeResolution` (`auto`, `full`, `half` or `tiled`) and `strength`. - The pipeline loads one part at a time, so a 4 GB model fits in an iPhone app's memory. `lazyModelLoading: false` loads it all at once. +## Image Editing + +Pass an image with the prompt to edit it. FLUX.2 starts from noise and attends to the image's tokens, so the result keeps its layout and follows the prompt: + +```typescript +const { image } = await generateImage({ + model, + prompt: { images: [photo], text: 'make the apple green' }, + seed: 1, + providerOptions: { 'core-ai': { stepCount: 4 } }, +}) +``` + +- `photo` is a `Uint8Array`, base64 string, `file://` URL or `https://` URL, which is downloaded first. Images are decoded with their EXIF orientation and resized to the model's size. +- The export needs a VAE encoder and an image-to-image transformer: `vae_encoder_half` and `transformer_512_img2img_half` in the iOS recipe. Without them the image is left out with a warning, and the result comes from the prompt alone. +- `providerOptions['core-ai'].referenceGrid` (`full`, `half` or `quarter`) sets how many of the image's tokens the model attends to: more follows the image more closely and runs slower. It defaults to the finest grid the export has. `strength` has no effect on FLUX.2. +- One image is edited per call, and there's no inpainting: extra images and a `mask` are left out with a warning. +- `coreAI.imageModel(config).generate(prompt, { imageUri })` (or `imageBase64`) edits outside the AI SDK. + ## Video Generation Pass `coreAI.videoModel(config)` to `experimental_generateVideo` to get a short MP4 clip.