diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d78904d..7de1023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: ${{ matrix.node }} - - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1 with: version: 12.16.1 - run: pnpm install --frozen-lockfile --ignore-scripts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 971f3ac..cc736a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24 - - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1 with: version: 12.16.1 - run: test "v$(jq -r .version package.json)" = "$GITHUB_REF_NAME" diff --git a/biome.json b/biome.json index 7089237..a11038a 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/2.6.4/schema.json", + "$schema": "https://biomejs.dev/schemas/2.6.5/schema.json", "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, "formatter": { "indentStyle": "space", "indentWidth": 2, "lineWidth": 110 }, "linter": { "enabled": true } diff --git a/package.json b/package.json index 4b953c3..e68f7a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-cliproxyapi-native", - "version": "0.3.4", + "version": "0.3.5", "description": "CLIProxyAPI chat discovery, image generation and editing, and video generation with Pi's native adapters", "publishConfig": { "access": "public" @@ -38,7 +38,7 @@ "check": "pnpm typecheck && pnpm lint && pnpm test" }, "devDependencies": { - "@biomejs/biome": "^2.6.4", + "@biomejs/biome": "^2.6.5", "@earendil-works/pi-ai": "^1.5.0", "@earendil-works/pi-coding-agent": "^1.5.0", "@earendil-works/pi-tui": "^1.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec61809..6498c2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@biomejs/biome': - specifier: ^2.6.4 - version: 2.6.4 + specifier: ^2.6.5 + version: 2.6.5 '@earendil-works/pi-ai': specifier: ^1.5.0 version: 1.5.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.12.3)(ws@8.21.3) @@ -125,71 +125,71 @@ packages: resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} - '@biomejs/biome@2.6.4': - resolution: {integrity: sha512-8sKKwaNW4ot22awZ3lBjwWLHnUFXgXnrBPYS2gNlBAXRbDC++7kqjK2H7dR/FgCDtPvO1xfKpgEhJkGxAKYkxA==} + '@biomejs/biome@2.6.5': + resolution: {integrity: sha512-n0sp74QbSVEkWXlH659TKv4oJNylwAvDeZ8K8vt/PbcF1xneuNWe57z/G0Bxfh00cufhWXF6DiStLM8+jcLUpg==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-android-arm64@2.6.4': - resolution: {integrity: sha512-jwrwigP4tBcHxzuV8izioX/BOUhJL/4e2rNSOE5nCjBkVC018LQAQUGiH+CNkFYvLtQ1dcZo5/xXERmaAGc7JQ==} + '@biomejs/cli-android-arm64@2.6.5': + resolution: {integrity: sha512-fkLzc1WRU7wRIUNmAm7cxILb4dd6eESz/VycWvVFKVn6tXx0gFQKdUbPnwT42xys8AEvIJQpFGcK00kURwqorg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [android] - '@biomejs/cli-android-x64@2.6.4': - resolution: {integrity: sha512-HdGInPJ4nZFwHxzvEtcp2nmSBtUyJrurmtIK43upFckuS3+ZqVM05HtsxQWnqGsuy/pxckIAjyKWlEKRlBXUyA==} + '@biomejs/cli-android-x64@2.6.5': + resolution: {integrity: sha512-fkZ9dPnDAzAcaPtPCiwqtkub5C9FQK/oGCGaU9sq+DysTaq13pE9FxBIMcEZ3ydn8/FfcMdXi/DDzKDP+qyn2Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [android] - '@biomejs/cli-darwin-arm64@2.6.4': - resolution: {integrity: sha512-0VvcUz46wbg59Q4TZ59KkIjDJrxEyu4Eoe79+F4SChxoWe+W7AEyIp6F5r72C9+p6MsB6dIovnZt1D30lGM3XA==} + '@biomejs/cli-darwin-arm64@2.6.5': + resolution: {integrity: sha512-bKd0EOREgnVjeZm4s9knQ+WdPOkWS3JHB+eWGHlx+Pq66bqQA1/KybwU0O+zuyAyXiphSCBcrLruZt3YtnJoaQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.6.4': - resolution: {integrity: sha512-fxmvkF81srlS3DiMyfm+aG3J0eG+6/BjaxRyjy57KbLSafnq0PO+hHfVWmtH2ty0S7nm6yDXBRwySfFeaRJKRQ==} + '@biomejs/cli-darwin-x64@2.6.5': + resolution: {integrity: sha512-UkT9BECNOXSiW6SGddjyaRACy362Sc9n3vDaTFkccL0bC2gW7JzM16i4+JKPbEaDn1Sa+oETHV8KungnI/y40Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.6.4': - resolution: {integrity: sha512-31ZaH/MHqPX+O+evRwRrCHQ8W5/9adY+/eMytGGWZnRx1T41QsWIF09+/WujynLBhWZUxNGS5CtsWLwgOFTAyg==} + '@biomejs/cli-linux-arm64-musl@2.6.5': + resolution: {integrity: sha512-kfc3AePW1a0/d/gUpcIz95nf3Ce8YY2/xL0Fk3fLqAhKsotqUi6PN6BaVWss4p6ijWbiuacZ9NQ4NiQtpjbYpw==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [musl] - '@biomejs/cli-linux-arm64@2.6.4': - resolution: {integrity: sha512-6+bK1Y/evmIUiJr7auertuMbiaYVicJhqA1bZyL47lCMWugv8l0gFQCqvR/gN1vZMHYm+HwoTKgeMxdNFCoQTg==} + '@biomejs/cli-linux-arm64@2.6.5': + resolution: {integrity: sha512-H9cVMWBqW/lkFK9NHJdt0bpM2q7PacUDEjQoff9+m0dCV09H/jmBykfTCkbDthHf8+jCuWDJWiQFJ7bDkiGOcA==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [glibc] - '@biomejs/cli-linux-x64-musl@2.6.4': - resolution: {integrity: sha512-bomSa2XdQAGGj6Cwdm4M13qE+lN9+AOlW8KtI4hdyVyRRF/Z71qPAi95T4wtdZK0QpBbkTW8hgbZZCwPi4xzIw==} + '@biomejs/cli-linux-x64-musl@2.6.5': + resolution: {integrity: sha512-0mHVpLNt7/FugfZ2VPugUIEIiooWIyHGGJUqU9StxamBln9hz+maXRFIECU50jaf5/ZB93veGIViN5VKgw2AEA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [musl] - '@biomejs/cli-linux-x64@2.6.4': - resolution: {integrity: sha512-tbFZMdL6dP/ws4edXxYG9WhvaVa1ZDzL4G5DzahnWW7Psib9H+VTdyU4XXVxpJk5j/nYZroSbvSNyYlpW3yCIw==} + '@biomejs/cli-linux-x64@2.6.5': + resolution: {integrity: sha512-e6/WMSN/+FhH+ZOoBevps/euxFQmv6BXqYNuTn1+ITR6O4XYFjHF5PeVSNZnjkgSYmvhcDkfepSX5lXczMijGA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [glibc] - '@biomejs/cli-win32-arm64@2.6.4': - resolution: {integrity: sha512-eCsqt8Ma3eVap+oyJCDnJZH60SIU464B66n3y38EUTJdM1T/HfZol1FJC9vb+sHToX0ix8XzqPcwJnOzyAJz0g==} + '@biomejs/cli-win32-arm64@2.6.5': + resolution: {integrity: sha512-uBbfpdR/cH5up1iRJFGAPP97xKMzrgMs685CSrY7NM+Q4Unw7chY0o3JulEvJdmU6c6hGQxwqotMq7DDH0DzxA==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.6.4': - resolution: {integrity: sha512-pYsMy5AThiHFOZAsngg8qN4PurM6nInKBhVX7sbj3eNpRQqo1Hb0MgCwPL7VqVMrmcEEMj9Ks3qbkallR65e8g==} + '@biomejs/cli-win32-x64@2.6.5': + resolution: {integrity: sha512-07LaCeN9vvmbAAMOMsUjMfL5UbsCDkvVeyMJIgwNRc0xYqhizBGkjVzx05LYEwCXl2AiHcdzqVPR4IOYAsI1QQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] @@ -920,47 +920,47 @@ snapshots: '@babel/runtime@7.29.7': {} - '@biomejs/biome@2.6.4': + '@biomejs/biome@2.6.5': optionalDependencies: - '@biomejs/cli-android-arm64': 2.6.4 - '@biomejs/cli-android-x64': 2.6.4 - '@biomejs/cli-darwin-arm64': 2.6.4 - '@biomejs/cli-darwin-x64': 2.6.4 - '@biomejs/cli-linux-arm64': 2.6.4 - '@biomejs/cli-linux-arm64-musl': 2.6.4 - '@biomejs/cli-linux-x64': 2.6.4 - '@biomejs/cli-linux-x64-musl': 2.6.4 - '@biomejs/cli-win32-arm64': 2.6.4 - '@biomejs/cli-win32-x64': 2.6.4 - - '@biomejs/cli-android-arm64@2.6.4': + '@biomejs/cli-android-arm64': 2.6.5 + '@biomejs/cli-android-x64': 2.6.5 + '@biomejs/cli-darwin-arm64': 2.6.5 + '@biomejs/cli-darwin-x64': 2.6.5 + '@biomejs/cli-linux-arm64': 2.6.5 + '@biomejs/cli-linux-arm64-musl': 2.6.5 + '@biomejs/cli-linux-x64': 2.6.5 + '@biomejs/cli-linux-x64-musl': 2.6.5 + '@biomejs/cli-win32-arm64': 2.6.5 + '@biomejs/cli-win32-x64': 2.6.5 + + '@biomejs/cli-android-arm64@2.6.5': optional: true - '@biomejs/cli-android-x64@2.6.4': + '@biomejs/cli-android-x64@2.6.5': optional: true - '@biomejs/cli-darwin-arm64@2.6.4': + '@biomejs/cli-darwin-arm64@2.6.5': optional: true - '@biomejs/cli-darwin-x64@2.6.4': + '@biomejs/cli-darwin-x64@2.6.5': optional: true - '@biomejs/cli-linux-arm64-musl@2.6.4': + '@biomejs/cli-linux-arm64-musl@2.6.5': optional: true - '@biomejs/cli-linux-arm64@2.6.4': + '@biomejs/cli-linux-arm64@2.6.5': optional: true - '@biomejs/cli-linux-x64-musl@2.6.4': + '@biomejs/cli-linux-x64-musl@2.6.5': optional: true - '@biomejs/cli-linux-x64@2.6.4': + '@biomejs/cli-linux-x64@2.6.5': optional: true - '@biomejs/cli-win32-arm64@2.6.4': + '@biomejs/cli-win32-arm64@2.6.5': optional: true - '@biomejs/cli-win32-x64@2.6.4': + '@biomejs/cli-win32-x64@2.6.5': optional: true '@earendil-works/pi-agent-core@1.5.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.12.3)(ws@8.21.3)': diff --git a/src/catalog.ts b/src/catalog.ts index 65189fe..a0a75db 100644 --- a/src/catalog.ts +++ b/src/catalog.ts @@ -118,6 +118,15 @@ const mediaModels = new Map< price?: readonly [number, number, number]; } >([ + [ + "gpt-image-2.6-flare", + { name: "GPT Image 2.6 Flare", purpose: "image", route: "openai", price: gptImage2Price }, + ], + [ + "gpt-image-2.6-sunburst", + { name: "GPT Image 2.6 Sunburst", purpose: "image", route: "openai", price: gptImage2Price }, + ], + ["gpt-image-2.6", { name: "GPT Image 2.6", purpose: "image", route: "openai" }], [ "gpt-image-2.5-flare", { name: "GPT Image 2.5 Flare", purpose: "image", route: "openai", price: gptImage2Price }, @@ -165,13 +174,12 @@ export function mediaCapability(id: string) { if (backend && (capability?.route === "xai" || capability?.route === "openai")) { return { ...capability, route: undefined, disabledReason: `Unsupported media execution on ${backend}.` }; } - // CLIProxyAPI 8.0.48 advertises it for API keys, but OpenAI's Images API rejects the bare 2.5 name. - if (capability && authType === "api-key" && metadataId === "gpt-image-2.5") { + // CLIProxyAPI advertises the plain names for API keys, but OpenAI's Images API has only Sunburst and Flare. + if (capability && authType === "api-key" && /^gpt-image-2\.[56]$/.test(metadataId)) { return { ...capability, route: undefined, - disabledReason: - "OpenAI's API has no gpt-image-2.5 model. Use api/gpt-image-2.5-sunburst or api/gpt-image-2.5-flare.", + disabledReason: `OpenAI's API has no ${metadataId} model. Use api/${metadataId}-sunburst or api/${metadataId}-flare.`, }; } return capability; @@ -185,6 +193,9 @@ export function mediaControls(id: string) { const capability = mediaCapability(id); const { metadataId, authType } = modelRoute(id); const openai = capability?.route === "openai"; + // Sunburst and Flare add quality tiers and markup edits; 2.6 adds complexity plus GIF and SVG output. + const tiered = openai && /^gpt-image-2\.[56]-(sunburst|flare)$/.test(metadataId); + const image26 = tiered && metadataId.startsWith("gpt-image-2.6-"); const commonRatios = ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]; let resolution: string[] = []; let aspect_ratio: string[] = []; @@ -214,24 +225,27 @@ export function mediaControls(id: string) { : undefined, resolution, aspect_ratio, - quality: openai - ? [ - "auto", - "low", - "medium", - "high", - ...(/^gpt-image-2\.5-(sunburst|flare)$/.test(metadataId) ? ["xhigh", "max"] : []), - ] - : [], + quality: openai ? ["auto", "low", "medium", "high", ...(tiered ? ["xhigh", "max"] : [])] : [], + complexity: image26 ? ["low", "medium", "high"] : [], background: openai ? ["auto", "opaque", "transparent"] : [], - output_format: openai ? ["png", "jpeg", "webp"] : [], + output_format: openai ? ["png", "jpeg", "webp", ...(image26 ? ["gif", "svg"] : [])] : [], + // GIF sizes are per frame and skip min_pixels; OpenAI checks the animation against its pixel budget. + gif: image26 + ? { frame_count: { min: 2, max: 16 }, frame_duration_ms: { min: 20, max: 2000 } } + : undefined, output_compression: openai ? { min: 0, max: 100, formats: ["jpeg", "webp"] } : undefined, moderation: openai ? ["auto", "low"] : [], // Subscription image-tool translation does not forward n. n: openai ? { min: 1, max: authType === "api-key" ? 10 : 1 } : undefined, partial_images: openai ? { min: 0, max: 3 } : undefined, editing: openai - ? { max_images: 16, max_file_bytes: 14 * 1024 * 1024, max_total_bytes: 20 * 1024 * 1024, mask: "png" } + ? { + max_images: 16, + max_file_bytes: 14 * 1024 * 1024, + max_total_bytes: 20 * 1024 * 1024, + mask: "png", + ...(tiered ? { markup: "png" } : {}), + } : undefined, input_fidelity: openai && metadataId === "gpt-image-1.5" ? ["low", "high"] : [], }; diff --git a/src/media-defaults.ts b/src/media-defaults.ts index 75f5dac..e656e0b 100644 --- a/src/media-defaults.ts +++ b/src/media-defaults.ts @@ -3,7 +3,7 @@ import { type MediaPurpose, mediaCapability, mediaPurpose, modelRoute } from "./ import { type Config, isModelId, isRecord, PROVIDER_ID } from "./config.ts"; export const MEDIA_DEFAULTS_ENTRY = "cliproxyapi-media-defaults"; -export const OPENAI_IMAGE_DEFAULT = "gpt-image-2.5-sunburst"; +export const OPENAI_IMAGE_DEFAULT = "gpt-image-2.6-sunburst"; // null blocks fallback for invalid selections, including after unrelated picker saves. export type MediaDefaults = Partial>; export type DefaultsContext = { diff --git a/src/media.ts b/src/media.ts index cb1d387..2e8f619 100644 --- a/src/media.ts +++ b/src/media.ts @@ -14,6 +14,7 @@ import type { TextContent, Usage, } from "@earendil-works/pi-ai"; +import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { type Static, Type } from "typebox"; import { fetchCatalog, mapMediaCatalog, mediaCapability, mediaControls } from "./catalog.ts"; @@ -65,13 +66,22 @@ const imageParameters = Type.Object({ description: "OpenAI quality from controls.quality; Sunburst and Flare also support xhigh and max.", }), ), + complexity: Type.Optional( + Type.String({ + description: + "GPT Image 2.6 Sunburst/Flare planning effort: low, medium, or high (upstream default high). Low suits quick edits.", + }), + ), background: Type.Optional( Type.String({ description: "OpenAI background: auto, opaque, or transparent. Transparency requires PNG or WebP.", }), ), output_format: Type.Optional( - Type.String({ description: "OpenAI output format: png, jpeg, or webp. Omit for the backend default." }), + Type.String({ + description: + "OpenAI output format: png, jpeg, or webp. GPT Image 2.6 Sunburst/Flare also generate gif and svg (svg needs n of 1). Omit for the backend default.", + }), ), output_compression: Type.Optional( Type.Integer({ @@ -80,6 +90,20 @@ const imageParameters = Type.Object({ description: "OpenAI JPEG/WebP compression, 0–100. Requires an explicit jpeg or webp output_format.", }), ), + frame_count: Type.Optional( + Type.Integer({ + minimum: 2, + maximum: 16, + description: "GIF output only: number of frames, 2–16 (upstream default 9).", + }), + ), + frame_duration_ms: Type.Optional( + Type.Integer({ + minimum: 20, + maximum: 2000, + description: "GIF output only: milliseconds per frame, 20–2000, rounded to 10 (upstream default 200).", + }), + ), moderation: Type.Optional(Type.String({ description: "OpenAI moderation: auto or low." })), n: Type.Optional( Type.Integer({ @@ -112,6 +136,13 @@ const imageParameters = Type.Object({ "Optional local PNG mask for images[0]. The first image must also be PNG with matching dimensions; mask must have an alpha channel. Transparent regions are edited.", }), ), + markup: Type.Optional( + Type.String({ + minLength: 1, + description: + "Optional local PNG annotation layer for images[0] with GPT Image 2.5/2.6 Sunburst or Flare: transparent except for marks showing where to edit. Same size as images[0]; cannot be combined with mask.", + }), + ), input_fidelity: Type.Optional( Type.String({ description: "Editing fidelity for GPT Image 1.5 only: low or high. Omit for other models.", @@ -122,7 +153,7 @@ const imageParameters = Type.Object({ minLength: 1, maxLength: 32, description: - "OpenAI image size: auto or WIDTHxHEIGHT. Use controls.size and controls.size_limits from cliproxyapi_media_models. Omit to preserve the backend default.", + "OpenAI image size: auto or WIDTHxHEIGHT. Use controls.size and controls.size_limits from cliproxyapi_media_models; GIF sizes are per frame and may go below min_pixels. Omit to preserve the backend default.", }), ), }); @@ -141,6 +172,7 @@ function mediaOutputOptions(model: string, params: Partial= limits.min_pixels && + (gif || pixels >= limits.min_pixels) && pixels <= limits.max_pixels ) continue; @@ -196,11 +234,24 @@ function mediaOutputOptions(model: string, params: Partial controls.partial_images.max || - (options.partial_images > 0 && (options.n ?? 1) !== 1)) + (options.partial_images > 0 && ((options.n ?? 1) !== 1 || generationOnly))) ) throw new Error( - "CLIProxyAPI partial_images requires an OpenAI image model, n of 1, and an integer from 0 to 3.", + "CLIProxyAPI partial_images requires an OpenAI image model, n of 1, PNG, JPEG, or WebP output, and an integer from 0 to 3.", ); + for (const field of ["frame_count", "frame_duration_ms"] as const) { + const value = options[field]; + const range = controls.gif?.[field]; + if ( + value !== undefined && + (!gif || !range || !Number.isInteger(value) || value < range.min || value > range.max) + ) + throw new Error(`CLIProxyAPI ${field} requires gif output_format and a value within controls.gif.`); + } + if (options.output_format === "svg" && (options.n ?? 1) !== 1) + throw new Error("CLIProxyAPI svg output_format requires n of 1."); + if (generationOnly && params.images !== undefined) + throw new Error("CLIProxyAPI gif and svg output_format are for generation, not editing."); if ( options.output_compression !== undefined && (!controls.output_compression || @@ -212,7 +263,11 @@ function mediaOutputOptions(model: string, params: Partial limits.max_images ) throw new Error("CLIProxyAPI editing requires 1–16 local images and a supported OpenAI image model."); + if (params.markup !== undefined && (params.mask !== undefined || !limits.markup)) + throw new Error( + "CLIProxyAPI markup requires GPT Image 2.5 or 2.6 Sunburst or Flare and cannot be combined with mask.", + ); + // The mask or markup layer follows the input images. + const layer = params.mask ?? params.markup; const images: { image_url: string }[] = []; let mask: { image_url: string } | undefined; + let markup: { image_url: string } | undefined; let firstPngDimensions: Buffer | undefined; let total = 0; - for (const [index, path] of [ - ...params.images, - ...(params.mask === undefined ? [] : [params.mask]), - ].entries()) { + for (const [index, path] of [...params.images, ...(layer === undefined ? [] : [layer])].entries()) { signal.throwIfAborted(); if ( typeof path !== "string" || @@ -285,22 +345,25 @@ async function imageInputs( } catch { throw new Error(`CLIProxyAPI editing input "${path}" is not a supported PNG, JPEG, or WebP image.`); } - const isMask = index === params.images.length; + const isLayer = index === params.images.length; if (index === 0 && mimeType === "image/png") firstPngDimensions = bytes.subarray(16, 24); + // Masks need a matching PNG first image; OpenAI checks markup on JPEG or WebP images after orientation. + const sameSize = firstPngDimensions?.equals(bytes.subarray(16, 24)); if ( - isMask && + isLayer && (mimeType !== "image/png" || - !firstPngDimensions?.equals(bytes.subarray(16, 24)) || - ![4, 6].includes(bytes[25])) + ![4, 6].includes(bytes[25]) || + (sameSize === undefined ? params.mask !== undefined : !sameSize)) ) throw new Error( - "CLIProxyAPI mask and first image must be matching-size PNGs; the mask needs an alpha channel.", + "CLIProxyAPI mask and markup layers must be alpha-channel PNGs matching a PNG first image's size; masks need a PNG first image.", ); const input = { image_url: `data:${mimeType};base64,${bytes.toString("base64")}` }; - if (isMask) mask = input; - else images.push(input); + if (!isLayer) images.push(input); + else if (params.mask !== undefined) mask = input; + else markup = input; } - return { images, mask }; + return { images, mask, markup }; } export async function mediaKey(ctx: Pick, signal: AbortSignal) { @@ -603,7 +666,9 @@ async function generationKey( const key = await mediaKey(ctx, signal); const available = mapMediaCatalog(await fetchCatalog(config, key, signal)); if (!available.some(({ id }) => id === params.model)) { - throw new Error("The selected media model is not available in the live CLIProxyAPI catalog."); + throw new Error( + "The selected media model is not available in the live CLIProxyAPI catalog. Update CLIProxyAPI, or choose an advertised model with /cli:model or an explicit model.", + ); } return key; } @@ -614,7 +679,7 @@ function imageType(bytes: Buffer) { if ( bytes.length > 24 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex")) && - bytes.toString("ascii", 12, 16) === "IHDR" + bytes.toString("latin1", 12, 16) === "IHDR" ) { extension = "png"; mimeType = "image/png"; @@ -630,8 +695,8 @@ function imageType(bytes: Buffer) { mimeType = "image/jpeg"; } else if ( bytes.length > 20 && - bytes.toString("ascii", 0, 4) === "RIFF" && - bytes.toString("ascii", 8, 12) === "WEBP" && + bytes.toString("latin1", 0, 4) === "RIFF" && + bytes.toString("latin1", 8, 12) === "WEBP" && bytes.readUInt32LE(4) + 8 === bytes.length ) { extension = "webp"; @@ -642,7 +707,71 @@ function imageType(bytes: Buffer) { return { extension, mimeType }; } -function parseImage(entry: unknown) { +// OpenAI documents static SVG without scripts, external resources, or raster images. Only static drawing +// elements with presentation attributes and same-document references pass; anything else fails closed. +const svgElements = new Set( + `svg g defs title desc symbol use path rect circle ellipse line polyline polygon text tspan textPath + linearGradient radialGradient stop pattern clipPath mask marker filter feBlend feColorMatrix feComponentTransfer + feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap feDistantLight feDropShadow feFlood feFuncA + feFuncB feFuncG feFuncR feGaussianBlur feMerge feMergeNode feMorphology feOffset fePointLight + feSpecularLighting feSpotLight feTile feTurbulence`.split(/\s+/), +); +const xmlEntities: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; +// Checked after decoding references: event handlers and style attributes after any separator HTML accepts, +// CSS escapes and image functions, xml:base, foreign namespaces, and references outside the document. The scan +// covers text too, so a label that looks like an assignment fails closed. CSS whitespace is ASCII only. +const unsafeSvg = + /[^\w.:-](?:on[a-z]+|style)\s*=|\\|(?:image-set|image|src)\(|xml:base|xmlns(?::[\w.-]+)?\s*=\s*["'](?!http:\/\/www\.w3\.org\/(?:2000\/svg|1999\/xlink)["'])|(?:href|src)\s*=\s*(?:"(?!#)|'(?!#)|[^"'\s#>])|url\([ \t\n\r\f]*(?:"(?!#)|'(?!#)|[^"'#) \t\n\r\f])/i; + +// Accept GIF or SVG only when the request asked GPT Image 2.6 for that format. +function outputType(bytes: Buffer, format?: string) { + if (format === "gif") { + if (bytes.length > 13 && /^GIF8[79]a$/.test(bytes.toString("latin1", 0, 6)) && bytes.at(-1) === 0x3b) + return { extension: "gif", mimeType: "image/gif" }; + throw new Error("CLIProxyAPI returned a malformed GIF."); + } + if (format !== "svg") return imageType(bytes); + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("CLIProxyAPI returned an SVG that is not valid UTF-8."); + } + // Only a UTF-8 XML declaration is removed: another encoding, such as ISO-2022-JP, makes renderers read + // different markup. Tags are checked before decoding because a decoded "<" in text is not markup. + const markup = text + .replace( + /^\s*<\?xml\s+version\s*=\s*(["'])1\.\d+\1(?:\s+encoding\s*=\s*(["'])utf-8\2)?(?:\s+standalone\s*=\s*(["'])(?:yes|no)\3)?\s*\?>/i, + "", + ) + .trim(); + // One decoding pass, so an escaped ampersand cannot start a second reference. + const decoded = markup.replace( + /&(?:#(x[\da-f]+|\d+)|(amp|lt|gt|quot|apos));?/gi, + (_, code?: string, name?: string) => { + if (code === undefined) return xmlEntities[name?.toLowerCase() ?? ""] ?? "\\"; + const point = /^x/i.test(code) ? Number.parseInt(code.slice(1), 16) : Number(code); + return point <= 0x10ffff ? String.fromCodePoint(point) : "\\"; + }, + ); + if ( + !/^]/.test(markup) || + !markup.endsWith("") || + // XML forbids these controls, and ESC can switch an encoding in the middle of an attribute name. + /(?![\t\n\r])\p{Cc}/u.test(text) || + // Comments, CDATA, declarations, and processing instructions. HTML parses comments inside SVG + // integration points such as desc and title differently from XML, so none are accepted. + /<[!?]/.test(markup) || + // XML defines only these named references; HTML would decode others such as ( into markup. + /&(?!(?:amp|lt|gt|quot|apos);|#)/.test(markup) || + [...markup.matchAll(/<\/?([^\s/>]*)/g)].some(([, name]) => !svgElements.has(name)) || + unsafeSvg.test(decoded) + ) + throw new Error("CLIProxyAPI returned a malformed SVG or one with active or external content."); + return { extension: "svg", mimeType: "image/svg+xml" }; +} + +function parseImage(entry: unknown, format?: string) { if (!isRecord(entry) || typeof entry.b64_json !== "string") { throw new Error("CLIProxyAPI did not return b64_json image data. URL-only images are not downloaded."); } @@ -651,12 +780,12 @@ function parseImage(entry: unknown) { if (!encoded || bytes.toString("base64") !== encoded) { throw new Error("CLIProxyAPI returned invalid base64 image data."); } - const { extension, mimeType } = imageType(bytes); + const { extension, mimeType } = outputType(bytes, format); const image: ImageContent = { type: "image", data: encoded, mimeType }; return { bytes, extension, image }; } -function parseImages(value: unknown, count: number) { +function parseImages(value: unknown, count: number, format?: string) { if ( !isRecord(value) || value.error !== undefined || @@ -664,7 +793,7 @@ function parseImages(value: unknown, count: number) { value.data.length !== count ) throw new Error(`CLIProxyAPI image response must contain ${count} image(s).`); - return value.data.map(parseImage); + return value.data.map((entry: unknown) => parseImage(entry, format)); } function parseGoogleImages(value: unknown) { @@ -727,7 +856,7 @@ function parseGoogleImages(value: unknown) { } // OpenAI reports image token usage. Costs use list prices and stay zero for unpriced models. -function imageUsage(model: string, value: unknown) { +function imageUsage(model: string, value: unknown, svg?: unknown) { const tokens = (field: unknown) => typeof field === "number" && Number.isSafeInteger(field) && field >= 0 ? field : undefined; if (!isRecord(value)) return undefined; @@ -746,8 +875,22 @@ function imageUsage(model: string, value: unknown) { cacheWrite: 0, total: 0, }; - cost.total = cost.input + cost.output; const usage: Usage = { input, output, cacheRead: 0, cacheWrite: 0, totalTokens: input + output, cost }; + // SVG output adds the authoring model's tokens, priced from Pi's OpenAI metadata when Pi knows it. + if (isRecord(svg)) { + const author = svg.model; + const svgInput = tokens(svg.input_tokens); + const svgOutput = tokens(svg.output_tokens); + if (svgInput !== undefined && svgOutput !== undefined) { + const price = getBuiltinModels("openai").find(({ id }) => id === author)?.cost; + usage.input += svgInput; + usage.output += svgOutput; + usage.totalTokens += svgInput + svgOutput; + cost.input += (svgInput * (price?.input ?? 0)) / 1e6; + cost.output += (svgOutput * (price?.output ?? 0)) / 1e6; + } + } + cost.total = cost.input + cost.output; return usage; } @@ -761,13 +904,14 @@ async function requestImages( output: Partial>; images?: { image_url: string }[]; mask?: { image_url: string }; + markup?: { image_url: string }; }, signal: AbortSignal, onPartial?: (image: ImageContent, index: number) => void, { onPayload, ...transport }: TransportOptions & { onPayload?: (payload: unknown) => unknown } = {}, ) { const route = mediaCapability(model)?.route; - const { prompt, images, mask } = request; + const { prompt, images, mask, markup } = request; const { partial_images = 0, ...output } = request.output; const n = output.n ?? 1; const stream = route === "openai" && partial_images > 0; @@ -782,7 +926,7 @@ async function requestImages( aspect_ratio: output.aspect_ratio, } : route === "openai" - ? { model, prompt, ...output, n, images, mask, ...(stream ? { stream, partial_images } : {}) } + ? { model, prompt, ...output, n, images, mask, markup, ...(stream ? { stream, partial_images } : {}) } : { contents: [{ role: "user", parts: [{ text: prompt }] }], generationConfig: { @@ -807,9 +951,17 @@ async function requestImages( body: payload, openaiImage: route === "openai", }); + // Decode the format actually requested, which a native payload hook may have changed. + const format = + route === "openai" && isRecord(payload) && typeof payload.output_format === "string" + ? payload.output_format + : undefined; return { - images: route === "gemini" ? parseGoogleImages(result) : parseImages(result, n), - usage: route === "openai" && isRecord(result) ? imageUsage(model, result.usage) : undefined, + images: route === "gemini" ? parseGoogleImages(result) : parseImages(result, n, format), + usage: + route === "openai" && isRecord(result) + ? imageUsage(model, result.usage, result.svg_usage) + : undefined, }; } // Each frame carries one full image, so the stream may hold the final image plus every preview. @@ -839,7 +991,10 @@ async function requestImages( (event) => isRecord(event) && typeof event.type === "string" && event.type.endsWith(".completed"), ); if (completed.length !== n) throw new Error(`CLIProxyAPI image stream must complete ${n} image(s).`); - return { images: completed.map(parseImage), usage: imageUsage(model, completed[0].usage) }; + return { + images: completed.map((event) => parseImage(event)), + usage: imageUsage(model, completed[0].usage), + }; } export async function generateNativeImages( @@ -967,11 +1122,14 @@ export async function generateImage( let previewBudget = MAX_INLINE_IMAGE_BASE64_BYTES; const content: (TextContent | ImageContent)[] = []; for (const [index, { image }] of images.entries()) { - const previewOmitted = !ctx.model?.input.includes("image") - ? "Preview omitted: the current chat model does not support images." - : image.data.length > previewBudget - ? "Preview omitted: image base64 exceeds the 4 MiB inline budget. Use read on the saved file." - : undefined; + // Chat providers disagree on animated GIF input and reject SVG, so both skip inline previews. + const previewOmitted = ["image/gif", "image/svg+xml"].includes(image.mimeType) + ? "Preview omitted: GIF and SVG results are saved without an inline preview." + : !ctx.model?.input.includes("image") + ? "Preview omitted: the current chat model does not support images." + : image.data.length > previewBudget + ? "Preview omitted: image base64 exceeds the 4 MiB inline budget. Use read on the saved file." + : undefined; content.push({ type: "text", text: `Saved image: ${files[index].path}${previewOmitted ? `\n${previewOmitted}` : ""}`, @@ -1076,7 +1234,7 @@ export function registerMediaTools(pi: ExtensionAPI, config: Config) { name: "cliproxyapi_generate_image", label: "CLIProxyAPI image", description: - "Generate images, or edit local images with OpenAI models by supplying images and an optional mask. Supports OpenAI size, quality, background, format/compression, image count and streamed partial previews from cliproxyapi_media_models. Uses the selected model or automatic OpenAI/GPT default without changing chat. Saves final images under a new .pi/ directory with optional previews. Maximum response: 32 MiB in total, plus 32 MiB per streamed preview; inline base64: 4 MiB. No automatic retries.", + "Generate images, or edit local images with OpenAI models by supplying images and an optional mask or markup layer. Supports OpenAI size, quality, complexity, background, format/compression, GIF/SVG output, image count and streamed partial previews; check cliproxyapi_media_models for each model's controls. Uses the selected model or automatic OpenAI/GPT default without changing chat. Saves final images under a new .pi/ directory with optional previews. Maximum response: 32 MiB in total, plus 32 MiB per streamed preview; inline base64: 4 MiB. No automatic retries.", promptSnippet: "Generate requested raster images through CLIProxyAPI using the effective image default.", promptGuidelines: [ "Use cliproxyapi_generate_image for requested raster images. Omit model to use the effective default unless a specific image model is requested; keep the current chat model.", diff --git a/test/auth-routes.test.ts b/test/auth-routes.test.ts index 7599a1e..071d983 100644 --- a/test/auth-routes.test.ts +++ b/test/auth-routes.test.ts @@ -88,17 +88,20 @@ test("automatic media selection remains separate from the chat credential route" } }); -test("the picker explains why API-key gpt-image-2.5 is disabled while other routes stay usable", () => { - const ids = ["api/gpt-image-2.5", "gpt-image-2.5", "oauth/gpt-image-2.5"]; - const catalog = { data: ids.map((id) => ({ id })) }; - assert.deepEqual( - mapMediaCatalog(catalog).map(({ id }) => id), - ids.slice(1), - ); - const row = pickerCatalog(catalog)[0]; - assert.equal(row.supported, false); - assert.match(row.disabledReason ?? "", /no gpt-image-2.5 model.*sunburst/); - assert.deepEqual(mediaControls(ids[0]).quality, []); - assert.throws(() => resolveMediaModel(parseConfig({}), {}, "image", ids[0]), /no gpt-image-2.5 model/); - assert.equal(resolveMediaModel(parseConfig({}), {}, "image", ids[2]), ids[2]); +test("the picker explains why API-key plain GPT Image 2.5 and 2.6 are disabled while other routes stay usable", () => { + for (const plain of ["gpt-image-2.5", "gpt-image-2.6"]) { + const ids = [`api/${plain}`, plain, `oauth/${plain}`]; + const catalog = { data: ids.map((id) => ({ id })) }; + assert.deepEqual( + mapMediaCatalog(catalog).map(({ id }) => id), + ids.slice(1), + ); + const row = pickerCatalog(catalog)[0]; + const reason = `OpenAI's API has no ${plain} model. Use api/${plain}-sunburst or api/${plain}-flare.`; + assert.equal(row.supported, false); + assert.equal(row.disabledReason, reason); + assert.deepEqual(mediaControls(ids[0]).quality, []); + assert.throws(() => resolveMediaModel(parseConfig({}), {}, "image", ids[0]), { message: reason }); + assert.equal(resolveMediaModel(parseConfig({}), {}, "image", ids[2]), ids[2]); + } }); diff --git a/test/media.test.ts b/test/media.test.ts index a4574bb..f918099 100644 --- a/test/media.test.ts +++ b/test/media.test.ts @@ -14,6 +14,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import type { ImageContent } from "@earendil-works/pi-ai"; +import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import { type ExtensionAPI, type ExtensionToolContext, @@ -48,6 +49,9 @@ const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwM const jpeg = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////2wBDAf//////////////////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdAAr/2Q=="; const webp = "UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA"; +const gif = "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"; +const svgDocument = (root: string) => `\n${root}\n`; +const svg = (root: string) => Buffer.from(svgDocument(root)).toString("base64"); async function home(t: TestContext) { const path = await mkdtemp(join(tmpdir(), "pi-cliproxyapi-media-test-")); @@ -856,6 +860,9 @@ test("recognized backends do not enable xAI media and unknown prefixes do not in }); const openaiIds = [ + "gpt-image-2.6-flare", + "gpt-image-2.6-sunburst", + "gpt-image-2.6", "gpt-image-2.5-flare", "gpt-image-2.5-sunburst", "gpt-image-2.5", @@ -899,7 +906,7 @@ test("OpenAI image IDs are image-only, preserve all wire IDs, and reject unsuppo } }); -test("all five OpenAI IDs POST one image without response_format and reuse bounded private artifact validation", async (t) => { +test("all eight OpenAI IDs POST one image without response_format and reuse bounded private artifact validation", async (t) => { const cwd = await home(t); const ctx = context(cwd); let response: unknown; @@ -1472,6 +1479,24 @@ test("media discovery reports model-specific output controls and preserves quali max_pixels: 8294400, max_aspect_ratio: 3, }); + for (const id of ["gpt-image-2.6-sunburst", "gpt-image-2.6-flare"]) { + assert.deepEqual(controls(id).size_limits, controls("gpt-image-2.5-sunburst").size_limits); + assert.deepEqual(controls(id).quality, controls("gpt-image-2.5-sunburst").quality); + assert.deepEqual(controls(id).complexity, ["low", "medium", "high"]); + assert.deepEqual(controls(id).output_format, ["png", "jpeg", "webp", "gif", "svg"]); + assert.deepEqual(controls(id).gif, { + frame_count: { min: 2, max: 16 }, + frame_duration_ms: { min: 20, max: 2000 }, + }); + assert.equal(controls(id).editing?.markup, "png"); + } + for (const id of ["gpt-image-2.6", "gpt-image-2.5-sunburst", "gpt-image-2.5", "gpt-image-2"]) { + assert.deepEqual(controls(id).complexity, []); + assert.deepEqual(controls(id).output_format, ["png", "jpeg", "webp"]); + assert.equal(controls(id).gif, undefined); + assert.equal(controls(id).editing?.markup, id === "gpt-image-2.5-sunburst" ? "png" : undefined); + } + assert.deepEqual(controls("gpt-image-2.6").quality, ["auto", "low", "medium", "high"]); assert.deepEqual(controls("gemini-2.5-flash-image").resolution, []); assert.deepEqual(controls("gemini-3-pro-image").resolution, ["1K", "2K", "4K"]); assert.deepEqual(controls("gemini-3.1-flash-image").resolution, ["512", "1K", "2K", "4K"]); @@ -1749,7 +1774,53 @@ test("OpenAI editing uploads bounded local files and a mask to the exact edit ro assert.equal((await readFile(join(ctx.cwd, name))).toString("base64"), data); }); +test("markup edits send a transparent PNG layer to Sunburst and Flare without a mask", async (t) => { + const ids = ["gpt-image-2.6-flare", "oauth/gpt-image-2.5-sunburst"]; + const posted: unknown[] = []; + const config = parseConfig({ + baseUrl: await server(t, (req, res) => { + if (req.method === "GET") return void res.end(JSON.stringify({ data: ids.map((id) => ({ id })) })); + assert.equal(req.url, "/v1/images/edits"); + void body(req).then((value) => { + posted.push(value); + res.end(JSON.stringify({ data: [{ b64_json: png }] })); + }); + }), + }); + const ctx = context(await home(t)); + for (const [name, data] of [ + ["kitchen.jpg", jpeg], + ["kitchen.png", png], + ["overlay.png", png], + ]) + await writeFile(join(ctx.cwd, name), Buffer.from(data, "base64")); + // OpenAI checks markup on a JPEG first image after orientation; the extension checks PNG sizes. + await generateImage( + config, + { model: ids[0], prompt: "fixture", images: ["kitchen.jpg"], markup: "overlay.png", complexity: "low" }, + ctx, + ); + await generateImage( + config, + { model: ids[1], prompt: "fixture", images: ["kitchen.png"], markup: "overlay.png" }, + ctx, + ); + const layer = { image_url: `data:image/png;base64,${png}` }; + assert.deepEqual(posted, [ + { + model: ids[0], + prompt: "fixture", + complexity: "low", + n: 1, + images: [{ image_url: `data:image/jpeg;base64,${jpeg}` }], + markup: layer, + }, + { model: ids[1], prompt: "fixture", n: 1, images: [layer], markup: layer }, + ]); +}); + test("invalid OpenAI controls and edit inputs fail before authentication or network access", async (t) => { + const image26 = "api/gpt-image-2.6-sunburst"; const config = parseConfig({}); const ctx = { ...context(await home(t)), @@ -1790,6 +1861,30 @@ test("invalid OpenAI controls and edit inputs fail before authentication or netw { partial_images: 1, n: 2 }, { model: imageModel, partial_images: 1 }, { model: "gemini-3-pro-image", partial_images: 1 }, + { complexity: "low" }, + { output_format: "svg" }, + { model: image26, complexity: "max" }, + { model: "oauth/gpt-image-2.6", complexity: "low" }, + { model: "api/gpt-image-2", output_format: "gif" }, + { model: image26, output_format: "svg", n: 2 }, + { model: image26, output_format: "gif", partial_images: 1 }, + { model: image26, output_format: "svg", partial_images: 2 }, + { model: image26, output_format: "gif", output_compression: 50 }, + { model: image26, output_format: "gif", background: "transparent" }, + { model: image26, output_format: "svg", background: "transparent" }, + { model: image26, frame_count: 4 }, + { model: image26, output_format: "png", frame_duration_ms: 100 }, + { model: image26, output_format: "gif", frame_count: 1 }, + { model: image26, output_format: "gif", frame_count: 17 }, + { model: image26, output_format: "gif", frame_count: 2.5 }, + { model: image26, output_format: "gif", frame_duration_ms: 10 }, + { model: image26, output_format: "gif", frame_duration_ms: 2001 }, + { model: image26, output_format: "gif", size: "520x512" }, + { model: image26, output_format: "gif", images: ["source.png"] }, + { model: image26, output_format: "svg", images: ["source.png"] }, + { markup: "mask.png" }, + { model: "api/gpt-image-2", images: ["source.png"], markup: "mask.png" }, + { images: ["source.png"], mask: "mask.png", markup: "mask.png" }, ]) await assert.rejects( generateImage(config, { model: "api/gpt-image-2.5-sunburst", prompt: "fixture", ...params }, ctx), @@ -1810,6 +1905,10 @@ test("invalid OpenAI controls and edit inputs fail before authentication or netw ctx, ), ); + for (const markup of ["invalid.png", "wrong-size.png", "no-alpha.png"]) + await assert.rejects( + generateImage(config, { model: image26, prompt: "fixture", images: ["source.png"], markup }, ctx), + ); await assert.rejects( generateImage(config, { model: "api/gpt-image-2", prompt: "fixture", images: ["invalid.png"] }, ctx), ); @@ -2068,6 +2167,171 @@ test("OpenAI image usage reaches the tool result with list-price estimates", asy } }); +test("GPT Image 2.6 GIF and SVG output saves validated files without previews and counts SVG authoring", async (t) => { + const model = "api/gpt-image-2.6-sunburst"; + const author = getBuiltinModels("openai").find(({ cost }) => cost.input > 0 && cost.output > 0); + assert.ok(author); + const root = + 'Field & Flour1 < 2 ©'; + const corrupt = (data: string, offset: number) => { + const bytes = Buffer.from(data, "base64"); + bytes[offset] |= 0x80; + return bytes.toString("base64"); + }; + const usage = { + input_tokens: 60, + input_tokens_details: { text_tokens: 10, image_tokens: 50 }, + output_tokens: 1000, + total_tokens: 1060, + }; + let response: unknown; + const posted: unknown[] = []; + const config = parseConfig({ + baseUrl: await server(t, (req, res) => { + if (req.method === "GET") return void res.end(JSON.stringify({ data: [{ id: model }] })); + assert.equal(req.url, "/v1/images/generations"); + void body(req).then((value) => { + posted.push(value); + res.end(JSON.stringify(response)); + }); + }), + }); + const ctx = context(await home(t)); + response = { data: [{ b64_json: gif }, { b64_json: gif }] }; + // GIF frames may go below the still-image minimum pixel count. + const params = { + model, + prompt: "fixture", + output_format: "gif", + frame_count: 4, + frame_duration_ms: 120, + size: "256x256", + complexity: "low", + n: 2, + }; + const animated = await generateImage(config, params, ctx); + assert.deepEqual(posted, [params]); + assert.deepEqual( + animated.details.files.map(({ mimeType }) => mimeType), + ["image/gif", "image/gif"], + ); + assert.ok(animated.details.files[1].path.endsWith("image-2.gif")); + assert.equal((await readFile(animated.details.files[1].path)).toString("base64"), gif); + assert.ok(animated.content.every((part) => part.type === "text")); + assert.match(JSON.stringify(animated.content), /GIF and SVG results are saved without an inline preview/); + for (const [svgModel, authorCost] of [ + [author.id, (100 * author.cost.input + 2000 * author.cost.output) / 1e6], + ["unknown-author", 0], + ] as const) { + response = { + data: [{ b64_json: svg(root) }], + usage, + svg_usage: { + model: svgModel, + input_tokens: 100, + output_tokens: 2000, + reasoning_tokens: 1500, + total_tokens: 2100, + }, + }; + const vector = await generateImage( + config, + { model, prompt: "fixture", output_format: "svg", quality: "low" }, + ctx, + ); + assert.deepEqual(posted.at(-1), { model, prompt: "fixture", output_format: "svg", quality: "low", n: 1 }); + assert.equal(vector.details.files[0].mimeType, "image/svg+xml"); + assert.ok(vector.details.files[0].path.endsWith("image.svg")); + assert.equal(await readFile(vector.details.files[0].path, "utf8"), svgDocument(root)); + assert.equal(vector.content.length, 1); + assert.equal(vector.usage?.totalTokens, 1060 + 2100); + assert.equal(vector.usage?.output, 3000); + assert.ok( + Math.abs((vector.usage?.cost.total ?? 0) - ((10 * 5 + 50 * 8 + 1000 * 30) / 1e6 + authorCost)) < 1e-12, + ); + } + const before = await readdir(join(ctx.cwd, ".pi")); + for (const [output_format, data] of [ + ["gif", png], + ["gif", Buffer.from("GIF89a").toString("base64")], + ["gif", corrupt(gif, 0)], + ["png", corrupt(png, 12)], + ["webp", corrupt(webp, 0)], + ["svg", png], + ["svg", Buffer.from("\u00ff", "latin1").toString("base64")], + ["svg", svg('')], + ["svg", svg('')], + ["svg", Buffer.from(']>&x;').toString("base64")], + [ + "svg", + Buffer.from( + '\n', + ).toString("base64"), + ], + [ + "svg", + svg( + '', + ), + ], + // An ISO-2022-JP renderer drops ESC ( B, turning the interrupted name into onload. + [ + "svg", + Buffer.from( + '', + ).toString("base64"), + ], + [ + "svg", + Buffer.from( + '', + ).toString("base64"), + ], + ["svg", svg('\u001b(B')], + [ + "svg", + svg(''), + ], + ...[ + "", + '', + '', + '', + '', + "", + '', + '', + "
", + 'alert(1)', + '