diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ef2e55f..116efd6 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -2107,6 +2107,16 @@ traced pixel that the fast mean's buffer had free: locked lettering of a still sign comes back from a motion's blur by the blend alone (the 0.2 on Kabukichō's signs); a lamp that comes on arrives over a quarter of a second. +**The measurements into the repo (2026-11-13, after the cleanup).** What the window's quality work of these weeks was +measured by lived in scratch scripts; now `tools/bench-window.ts` (the still frames, the Bistro slice, Tokyo at dusk, +Tokyo turning and orbiting as the play window's left drag does, each against offline), `tools/render-identity.ts` +(the byte gate for changes that must not change a picture) and an orbit move in `bench-ghosting` (reported, not yet +guarded). On the way: the moving lamps test (`upscale/lamps-in-motion-live`) never exercised the motion clamp. Its +lamps float before the black sky, a sliding camera does not move the sky, and a lamp no sample hit takes the +velocity of what is behind it: the upscaler saw no motion. Before a black wall 1 cm behind, the lamps keep 0.045 of +their 0.1 in motion as merged (no clamp at all: 0.105; before the lock: 0.014). The test changes with the fix of its +cause (a missed lamp reprojected by its own depth), so that main does not carry a failing test meanwhile. + **Cleanup: experiments and switches that outlived their measurement (2026-11-13, after it).** Removed, each off by default, so nothing a scene shows changes: diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index f9595bd..8ce0438 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -66,10 +66,17 @@ highest tier any of its changes needs. |---|---|---| | **D** docs | only `docs/`, READMEs, comments | CI | | **T** tools / TypeScript | `tools/`, `tests/` harness, `ue5/` scripts, `packages/*` without effect on what the renderer draws | `npm run typecheck`, `npm test`; `npm run test:render` if `packages/scene`, `packages/gameplay` or `tests/render` changed | -| **R1** renderer, existing scenes unchanged | new kernel / live code behind a feature bit, new DSL features, new passes that only run when a scene uses them | T, plus `cargo test --release`, `test:render` (no new failures against main), byte-identity of the catalog scenes (offline, software traversal), **perf smoke**, crops of the new feature | -| **R2** renderer, existing scenes change | anything that changes what existing scenes render, or the live passes they run (denoising, temporal, reflections, post) | R1 without byte-identity, plus `node tools/bench-ghosting.ts --quick --check` (Bistro) and `--scene kabukicho` (wet signs: the reflection stress case), `test:layer-b`, crops before / after | +| **R1** renderer, existing scenes unchanged | new kernel / live code behind a feature bit, new DSL features, new passes that only run when a scene uses them | T, plus `cargo test --release`, `test:render` (no new failures against main), byte-identity of the catalog scenes (offline, software traversal) and of every `test:render` output (`node tools/render-identity.ts --a
--b `: live tests included), **perf smoke**, crops of the new feature | +| **R2** renderer, existing scenes change | anything that changes what existing scenes render, or the live passes they run (denoising, temporal, reflections, post) | R1 without byte-identity, plus `node tools/bench-ghosting.ts --quick --check` (Bistro) and `--scene kabukicho` (wet signs: the reflection stress case), `test:layer-b`, crops before / after; a change to the live passes also `node tools/bench-window.ts` on main and the branch (the play window against offline: still, the Bistro slice, Tokyo at dusk and in motion) | | **R3** hot path or perf claim | the PR claims a speed-up, or changes the cost of work existing scenes do every frame (trace loop, light sampling, filter passes, resolution control) | R2, plus **full A/B timing** (Bistro night still and moving and the 3 × 3 Bistro district along its camera path at the 0.5 floor, main and branch alternated: `node tools/bench-district.ts`) | +- **render-identity** (R1, and any change that says it changes no picture): test-render whole, twice on each build, + every HDR output compared byte for byte. A test one build does not repeat to the byte (asynchronous builds, + physics, the radiance cache's training order: about 18 of 190) passes when the builds differ by no more than twice + their own run-to-run spread. `--env THREERT_MERGED=1` runs the browser's binding layout natively. +- **bench-window** (R2 for the live passes): the shots the window's quality work of 2026-11 was measured by, each + against an offline render of the same moment (cached in `out/cache/window-truth`), or for the dusk against the + same moment held. The Tokyo shots need the city (`tools/tokyo`); without it they are reported as not run. - **Perf smoke** (R1, R2): - Bistro night (`scenes/bistro.ts`) live at the fixed 0.5 floor, after warm-up: about 15 s still at its camera and 15 s on a scripted move (2.5 m down the street with 12° of yaw, ping-pong), GPU timestamps. Until 2026-11-08 diff --git a/tools/bench-ghosting.ts b/tools/bench-ghosting.ts index 20113f1..c40f2db 100644 --- a/tools/bench-ghosting.ts +++ b/tools/bench-ghosting.ts @@ -19,7 +19,11 @@ // history reprojects exactly, so what is left is the history's own blur, lag or doubling; // strafe: 2.5 m to the left (3 m to the right) in 0.4 s, hold, back (parallax, disocclusion, reflections moving // against their surface); -// stop: 8 m/s forward for 0.5 s, then still (what the history drags into the first frames after a sudden stop). +// stop: 8 m/s forward for 0.5 s, then still (what the history drags into the first frames after a sudden stop); +// orbit: the play window's left drag (view.rs `Orbit`: about a point ahead of the camera as far as the camera is +// from the origin, held to 1-50 m), 27.5 deg in 0.4 s, hold, back: rotation and translation at once, a +// degree a frame, where a feature finer than a pixel that the samples miss reprojects with what is behind +// it (a second copy of a lamp, 2026-11-13). Reported, not yet guarded. // Shots: each move's midpoint going out (vs the converged midpoint); the start pose 0.1 / 0.5 / 2 s after returning (counted in frames at 60 fps, not wall-clock time); // the stop pose 0 / 0.1 / 0.3 s after stopping. Per region (the wet street, where reflections are; the sign band; // the facades): mean dE, p95, p99, trail_px (p95 over rows of the longest horizontal run of pixels off by dE > 10) and @@ -144,6 +148,17 @@ function pose(yaw: number, side = 0, fwd = 0) { return { pos, target: pos.map((p, i) => p + dir[i] * 10) as V3, tan: view.tan }; } const turn = (f: number) => pose(view.turn * f); +/** The play window's orbit (view.rs `Orbit::from_camera`): about the point `dist` along the start pose's view, `dist` + * the start position's distance from the origin held to 1..50 m, `ORBIT_DEG` degrees at f = 1. */ +const ORBIT_DEG = 27.5; +function orbit(f: number) { + const p = pose(0); + const dir = p.target.map((t, i) => (t - p.pos[i]) / 10); + const n = Math.hypot(...dir), dist = Math.min(Math.max(Math.hypot(...view.p0), 1), 50); + const c = p.pos.map((x, i) => x + (dir[i] / n) * dist) as V3; + const a = (ORBIT_DEG * f * Math.PI) / 180, ox = p.pos[0] - c[0], oz = p.pos[2] - c[2]; + return { pos: [c[0] + ox * Math.cos(a) + oz * Math.sin(a), p.pos[1], c[2] - ox * Math.sin(a) + oz * Math.cos(a)] as V3, target: c, tan: view.tan }; +} const strafe = (f: number) => pose(0, view.strafe * f); const dolly = (f: number) => pose(0, 0, 4 * f); @@ -190,7 +205,7 @@ async function start(s: string) { // References: converged frames (history reset, then accumulated) at every pose a shot is taken at. const refFile = (n: string) => join(refs, `${n}.png`); -const refPoses: Record> = { start: pose(0), "turn-mid": turn(0.5), "strafe-mid": strafe(0.5), stop: dolly(1) }; +const refPoses: Record> = { start: pose(0), "turn-mid": turn(0.5), "strafe-mid": strafe(0.5), "orbit-mid": orbit(0.5), stop: dolly(1) }; if (!Object.keys(refPoses).every((n) => existsSync(refFile(n)))) { console.log(`rendering converged references (${refFrames} frames each at ${refScale})…`); const live = await start(refScale); @@ -227,7 +242,7 @@ async function sweep(at: (f: number) => ReturnType, f0: number, f1: for (rep = 0; rep < repeats; rep++) { if (rep === 0) await shot("settled", "start", 240, pose(0)); else await live.step(240, pose(0)); - for (const [name, at, n] of [["turn", turn, 30], ["strafe", strafe, 24]] as const) { + for (const [name, at, n] of [["turn", turn, 30], ["strafe", strafe, 24], ["orbit", orbit, 24]] as const) { await sweep(at, 0, 1, n, `${name}-mid`); await live.frames(30); await sweep(at, 1, 0, n); @@ -299,11 +314,11 @@ const tag = (sceneName === "kabukicho" ? "" : `${sceneName}-`) + (quick ? "ghost summary(`${tag} turn-mid ground excess`, value["turn-mid/ground"].mean, { unit: " dE" }); summary(`${tag} turn-mid ground trail`, value["turn-mid/ground"].trail, { unit: " px", digits: 0 }); summary(`${tag} strafe-return+0.5s excess`, value["strafe-return+0.5s/all"].mean, { unit: " dE" }); -for (const m of ["turn", "strafe"]) { +for (const m of ["turn", "strafe", "orbit"]) { execFileSync(bin, ["montage", join(out, `${m}.jpg`), "--width", "640", join(out, `${m}-mid.png`), refFile(`${m}-mid`), join(out, `${m}-return+0.1s.png`), refFile("start")]); } execFileSync(bin, ["montage", join(out, "stop.jpg"), "--width", "640", join(out, "stop+0.png"), join(out, "stop+0.1s.png"), refFile("stop")]); -console.log(`${out}/{turn,strafe,stop}.jpg (shot | converged)`); +console.log(`${out}/{turn,strafe,orbit,stop}.jpg (shot | converged)`); if (args.includes("--check") && failed) { console.error(`ghosting: ${failed} guarded value(s) over the limit`); process.exit(1); diff --git a/tools/bench-window.ts b/tools/bench-window.ts new file mode 100644 index 0000000..b81a2d5 --- /dev/null +++ b/tools/bench-window.ts @@ -0,0 +1,212 @@ +// The play window's picture against what it should be, in the shots the window's quality work of 2026-11 was measured +// by (docs/ROADMAP.md § 1a). Each shot bundles its game, drives one headless live session in lockstep and compares +// its frames with an offline render of the same moment (rendered once and kept: out/cache/window-truth, keyed by the +// bundle and the shot; --fresh-truth renders them again), or, for the dusk, with the same moment held. +// +// node tools/bench-window.ts [--shots still-bistro,still-kabukicho,slice,tokyo-dusk,tokyo-turn,tokyo-orbit] +// [--tag NAME] [--fresh-truth] +// +// Shots (the bin is THREERT_BIN, else the release build): +// still-bistro, still-kabukicho the scene held still at 2560 x 1440 (render scale 0.5, 0.375), animation frozen, +// 300 frames, then two consecutive frames: detail (the gradients' regression on offline's: 1 keeps every edge), +// ΔE, SSIM against offline (512 samples), and the RMS change of luma from one frame to the next (flicker). +// slice the Bistro slice half a second after its explosion (40.5 s, from the start in lockstep, scale 0.375, 1440p): +// the same four numbers against offline at 40.5 s. +// tokyo-dusk the Tokyo time-of-day reel's room (16:00 to 17:42 in 10.5 s), the camera still, at 6, 7, 8 s: the frame +// in real time against the same moment held (time all but stopped, 120 frames); the skyline and the whole frame. +// tokyo-turn, tokyo-orbit Tokyo at night (13 s), the camera turning 0.25 degrees a frame, or orbiting as the play +// window's left drag does (1.146 degrees a frame about a point 50 m ahead), for 24 frames, then still: the last +// moving frame and 1, 6, 30 and 150 frames after it against offline of the last pose (256 samples), over the +// tower and the whole frame. +// The Tokyo shots need the city (assets/city, tools/tokyo): without it they are reported as not run. +// Output: out/bench-window///*.png, out/bench-window//metrics.json, and a table. +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import sharp from "sharp"; +import { bundleGame } from "./bundle-game.ts"; +import { rendererBin, repoRoot, waitGpuLock } from "./lib/bench.ts"; +import { startLive, type Build, type Live } from "./lib/live.ts"; + +const args = process.argv.slice(2); +const flag = (k: string, d: string) => (args.includes(k) ? args[args.indexOf(k) + 1] : d); +const ALL = ["still-bistro", "still-kabukicho", "slice", "tokyo-dusk", "tokyo-turn", "tokyo-orbit"]; +const shots = flag("--shots", ALL.join(",")).split(","); +const tag = flag("--tag", new Date().toISOString().replace(/[:.]/g, "-")); +const freshTruth = args.includes("--fresh-truth"); +const out = join(repoRoot, "out/bench-window", tag); +const presets = join(repoRoot, "presets"); +const build: Build = { bin: rendererBin, assets: repoRoot, presets, bundles: join(repoRoot, "out/bundles"), label: "bench-window" }; +for (const s of shots) if (!ALL.includes(s)) throw new Error(`bench-window: no shot ${s} (${ALL.join(", ")})`); +await waitGpuLock("bench-window"); + +type Row = Record; +const rows: Row[] = []; +const bundles = new Map(); +async function bundle(src: string): Promise { + if (!bundles.has(src)) bundles.set(src, (await bundleGame(join(repoRoot, src))).out); + return bundles.get(src)!; +} + +/** The offline render of a bundle at these settings, rendered once (keyed by the bundle and the settings). */ +function truth(b: string, o: { width: number; height: number; spp: number; time?: number; room?: string; camera?: { position: number[]; target: number[]; fov: number } }): string { + const key = createHash("sha256").update(readFileSync(b)).update(JSON.stringify(o)).digest("hex").slice(0, 20); + const path = join(repoRoot, "out/cache/window-truth", `${key}.png`); + if (existsSync(path) && !freshTruth) return path; + mkdirSync(join(repoRoot, "out/cache/window-truth"), { recursive: true }); + console.log(`bench-window: rendering the offline truth (${o.width} x ${o.height}, ${o.spp} samples)…`); + const extra = [...(o.time !== undefined ? ["--time", String(o.time)] : []), ...(o.room ? ["--room", o.room] : []), ...(o.camera ? ["--camera", JSON.stringify(o.camera)] : [])]; + execFileSync(rendererBin, ["render", b, "-o", path, "--assets", repoRoot, "--presets", presets, "--width", String(o.width), "--height", String(o.height), "--spp", String(o.spp), ...extra], { stdio: ["ignore", "ignore", "inherit"] }); + return path; +} + +type M = { mean_delta_e: number; p99_delta_e: number; ssim: number; luminance_ratio: number }; +const compare = (ref: string, img: string, region?: string): M => + JSON.parse(execFileSync(rendererBin, ["compare", ref, img, ...(region ? ["--region", region] : [])]).toString().trim().split("\n").pop()!); +const luma = async (f: string) => { + const { data, info } = await sharp(f).removeAlpha().raw().toBuffer({ resolveWithObject: true }); + const l = new Float32Array(info.width * info.height); + for (let i = 0; i < l.length; i++) l[i] = 0.2126 * data[i * 3] + 0.7152 * data[i * 3 + 1] + 0.0722 * data[i * 3 + 2]; + return { l, w: info.width, h: info.height }; +}; +/** How much of the reference's detail a frame keeps: the regression of its luma gradients on the reference's. */ +async function detail(ref: string, img: string): Promise { + const R = await luma(ref), T = await luma(img); + let num = 0, den = 0; + for (let y = 1; y < R.h - 1; y++) { + for (let x = 1; x < R.w - 1; x++) { + const i = y * R.w + x, rx = R.l[i + 1] - R.l[i - 1], ry = R.l[i + R.w] - R.l[i - R.w]; + num += rx * (T.l[i + 1] - T.l[i - 1]) + ry * (T.l[i + R.w] - T.l[i - R.w]); + den += rx * rx + ry * ry; + } + } + return num / den; +} +/** RMS of the luma change between two frames (0-255). */ +async function flicker(a: string, b: string): Promise { + const A = await luma(a), B = await luma(b); + let s = 0; + for (let i = 0; i < A.l.length; i++) s += (A.l[i] - B.l[i]) ** 2; + return Math.sqrt(s / A.l.length); +} + +/** Lockstep frames on a session: `n` frames (then a snapshot to `file`, or a sync); a camera with the first. */ +let syncs = 0; +async function frames(live: Live, n: number, file?: string, cam?: object) { + const id = `w${++syncs}`; + const done = file ? live.wait((l) => l.includes(`snapshot ${file}`)) : live.wait((l) => l === `live: sync ${id}`); + live.send({ ...(cam ? { cmd: "camera", ...cam, frames: n } : { cmd: "frames", n }), ...(file ? { snapshot: file } : { sync: id }) }); + await done; +} +async function session(b: string, o: { width: number; height: number; scale: string; room?: string; env?: Record }) { + const live = startLive(build, b, o); + await live.firstFrame; + live.send({ cmd: "lockstep", dt: 1 / 60 }); + return live; +} +const r3 = (x: number) => Math.round(x * 1000) / 1000; + +/** Held still: 300 frames, then two consecutive frames, against offline. */ +async function still(name: string, src: string, scale: string, env: Record) { + const dir = join(out, name); + mkdirSync(dir, { recursive: true }); + const b = await bundle(src); + const ref = truth(b, { width: 2560, height: 1440, spp: 512 }); + const live = await session(b, { width: 2560, height: 1440, scale, env: { THREERT_FREEZE_TIME: "1", ...env } }); + const [a, c] = [join(dir, "a.png"), join(dir, "b.png")]; + await frames(live, 300, a); + await frames(live, 1, c); + await live.quit(); + const m = compare(ref, a); + rows.push({ shot: name, frame: "held", region: "all", detail: r3(await detail(ref, a)), dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio, flicker: r3(await flicker(a, c)) }); +} + +const TOKYO = "games/tokyo/tokyo-tod.game.ts", TOKYO_ROOM = "shiba-15.6-17.7-10.5-r1600", TOKYO_SETTLE = 2; +const tokyoMissing = () => !existsSync(join(repoRoot, "assets/city/glb/shiba")); + +async function slice() { + const dir = join(out, "slice"); + mkdirSync(dir, { recursive: true }); + const b = await bundle("games/bistro-slice/bistro-slice.game.ts"); + const ref = truth(b, { width: 2560, height: 1440, spp: 512, time: 40.5 }); + const live = await session(b, { width: 2560, height: 1440, scale: "0.375" }); + const [a, c] = [join(dir, "a.png"), join(dir, "b.png")]; + await frames(live, Math.round(40.5 * 60), a); + await frames(live, 1, c); + await live.quit(); + const m = compare(ref, a); + rows.push({ shot: "slice", frame: "40.5 s", region: "all", detail: r3(await detail(ref, a)), dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio, flicker: r3(await flicker(a, c)) }); +} + +/** The dusk: real time against the same moment held, a session a moment. */ +async function tokyoDusk() { + const dir = join(out, "tokyo-dusk"); + mkdirSync(dir, { recursive: true }); + const b = await bundle(TOKYO); + for (const t of [6, 7, 8]) { + const live = await session(b, { width: 1280, height: 720, scale: "0.75", room: TOKYO_ROOM }); + const [moving, held] = [join(dir, `t${t}.png`), join(dir, `t${t}-held.png`)]; + await frames(live, Math.round((TOKYO_SETTLE + t) * 60) - 1, moving); + live.send({ cmd: "lockstep", dt: 1e-5 }); + await frames(live, 120, held); + await live.quit(); + for (const [region, g] of [["skyline", "0.2,0.3,0.8,0.5"], ["all", undefined]] as const) { + const m = compare(held, moving, g); + rows.push({ shot: "tokyo-dusk", frame: `${t} s`, region, dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio }); + } + } +} + +/** Tokyo at night, the camera turning or orbiting for 24 frames, then still, against offline of the last pose. */ +async function tokyoMove(name: string, orbit: boolean) { + const dir = join(out, name); + mkdirSync(dir, { recursive: true }); + const b = await bundle(TOKYO); + // The scene's camera (tokyo-tod.game.ts): at (900, 260, -900) looking at (0, 180, 0), 60 degrees. + const P = [900, 260, -900], D = [-900, -80, 900], DL = Math.hypot(...D); + const T = P.map((p, i) => p + (D[i] / DL) * 50); + const deg = orbit ? 1.146 : 0.25; + const cam = (a: number) => { + const c = Math.cos((a * Math.PI) / 180), s = Math.sin((a * Math.PI) / 180); + if (orbit) { const ox = P[0] - T[0], oz = P[2] - T[2]; return { pos: [T[0] + ox * c + oz * s, P[1], T[2] - ox * s + oz * c], target: T, tan: 0 }; } + return { pos: P, target: [P[0] + D[0] * c + D[2] * s, P[1] + D[1], P[2] - D[0] * s + D[2] * c], tan: 0 }; + }; + const last = cam(deg * 24); + const ref = truth(b, { width: 1280, height: 720, spp: 256, time: 13, room: TOKYO_ROOM, camera: { position: last.pos, target: last.target, fov: 60 } }); + const live = await session(b, { width: 1280, height: 720, scale: "0.75", room: TOKYO_ROOM }); + await frames(live, (TOKYO_SETTLE + 11) * 60); + const f = (n: string) => join(dir, `${n}.png`); + for (let k = 1; k <= 24; k++) await frames(live, 1, k === 24 ? f("last") : undefined, cam(deg * k)); + await frames(live, 1, f("stop1")); + await frames(live, 5, f("stop6")); + await frames(live, 24, f("stop30")); + await frames(live, 120, f("settled")); + await live.quit(); + // Where the tower is in the last pose (the turn keeps it about the middle; the orbit carries it right). + const tower = orbit ? "0.68,0.35,0.8,0.64" : "0.3,0.25,0.7,0.65"; + for (const n of ["last", "stop1", "stop6", "stop30", "settled"]) { + for (const [region, g] of [["tower", tower], ["all", undefined]] as const) { + const m = compare(ref, f(n), g); + rows.push({ shot: name, frame: n, region, dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio }); + } + } +} + +mkdirSync(out, { recursive: true }); +for (const s of shots) { + console.log(`bench-window: ${s}`); + if (s.startsWith("tokyo") && tokyoMissing()) { + rows.push({ shot: s, frame: "not run", region: "the city is not here (assets/city)" }); + continue; + } + if (s === "still-bistro") await still(s, "scenes/bistro.ts", "0.5", {}); + else if (s === "still-kabukicho") await still(s, "games/kabukicho/kabukicho.game.ts", "0.375", { THREERT_NO_GRAIN: "1" }); + else if (s === "slice") await slice(); + else if (s === "tokyo-dusk") await tokyoDusk(); + else if (s === "tokyo-turn") await tokyoMove(s, false); + else if (s === "tokyo-orbit") await tokyoMove(s, true); +} +console.table(rows); +writeFileSync(join(out, "metrics.json"), JSON.stringify({ bin: rendererBin, shots, rows }, null, 1)); +console.log(`bench-window: ${join(out, "metrics.json")}`); diff --git a/tools/render-identity.ts b/tools/render-identity.ts new file mode 100644 index 0000000..17a3486 --- /dev/null +++ b/tools/render-identity.ts @@ -0,0 +1,108 @@ +// Whether a change leaves every picture as it was: test-render whole, `--runs` times on each of two builds, every +// test's HDR output compared byte for byte. The gate for changes that must not change a picture (a refactor, a +// cleanup). A test whose output a build does not repeat to the byte (asynchronous builds, physics, the radiance +// cache's training order) is judged by its spread instead: the outputs of the two builds may differ by no more than +// twice the larger of the two builds' own run-to-run differences (the largest relative difference of a value, under a +// floor of 1e-3). `--tolerance r` lets a difference up to r pass as rounding (a shader compiled from other source can +// round differently: 2e-6 is what removing unused kernel code did, 2026-11-13); it is reported apart. Exit 1 when a test +// differs beyond these or its output is missing on one side. +// +// node tools/render-identity.ts --a --b [--runs 2] [--tolerance 0] [--env K=V,K=V] +// [--filter id] [--out out/render-identity/] +// +// `--env` goes to both builds (THREERT_MERGED=1: the browser's binding layout, natively). Each run is test-render with +// an output directory of its own (it holds the GPU lock as it always does); the runs of one build follow each other. +// No outputs on a side, or no binary at --a / --b, is an error (exit 2): a gate that compared nothing has passed +// nothing. +import { execFileSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { repoRoot } from "./lib/bench.ts"; + +const args = process.argv.slice(2); +const flag = (k: string, d = "") => (args.includes(k) ? args[args.indexOf(k) + 1] : d); +const isFile = (p: string) => p !== "" && existsSync(p) && statSync(p).isFile(); +if (!isFile(flag("--a")) || !isFile(flag("--b"))) { + console.error("render-identity: --a and --b must name two threert-render binaries"); + process.exit(2); +} +const a = resolve(flag("--a")), b = resolve(flag("--b")); +const runs = Number(flag("--runs", "2")); +const tolerance = Number(flag("--tolerance", "0")); +const extra = Object.fromEntries(flag("--env").split(",").filter(Boolean).map((kv) => kv.split("=") as [string, string])); +const filter = flag("--filter"); +const out = resolve(flag("--out", join(repoRoot, "out/render-identity", new Date().toISOString().replace(/[:.]/g, "-")))); +/** One test-render run on `bin`, its outputs in `dir` (its own: nothing else writes there). */ +function run(bin: string, dir: string) { + rmSync(dir, { recursive: true, force: true }); + try { + execFileSync("node", [join(repoRoot, "tools/test-render.ts"), ...(filter ? [filter] : []), "--out", dir], { env: { ...process.env, ...extra, THREERT_BIN: bin }, stdio: ["ignore", "ignore", "inherit"] }); + } catch { + // (Failing tests still leave their outputs; what is compared here is the pictures, not the verdicts.) + } + const n = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".raw")).length : 0; + if (n === 0) { + console.error(`render-identity: test-render left no outputs in ${dir} (${bin})`); + process.exit(2); + } +} + +const dirs = { a: [] as string[], b: [] as string[] }; +for (const side of ["a", "b"] as const) { + for (let k = 0; k < runs; k++) { + const dir = join(out, `${side}${k + 1}`); + console.log(`render-identity: ${side} run ${k + 1} of ${runs}`); + run(side === "a" ? a : b, dir); + dirs[side].push(dir); + } +} + +const floats = (f: string) => { + const d = readFileSync(f); + return new Float32Array(d.buffer, d.byteOffset, d.byteLength / 4); +}; +/** The largest relative difference of a value between two outputs (0: identical to the byte). */ +function diff(x: string, y: string): number { + const bx = readFileSync(x), by = readFileSync(y); + if (bx.equals(by)) return 0; + if (bx.length !== by.length) return Infinity; + const fx = floats(x), fy = floats(y); + let m = 0; + for (let i = 0; i < fx.length; i++) m = Math.max(m, Math.abs(fx[i] - fy[i]) / Math.max(Math.abs(fx[i]), Math.abs(fy[i]), 1e-3)); + return m; +} +const spread = (ds: string[], f: string) => { + let m = 0; + for (let i = 0; i < ds.length; i++) for (let j = i + 1; j < ds.length; j++) m = Math.max(m, diff(join(ds[i], f), join(ds[j], f))); + return m; +}; + +const names = new Set([...dirs.a, ...dirs.b].flatMap((d) => readdirSync(d).filter((f) => f.endsWith(".raw")))); +let same = 0, within = 0, rounding = 0; +const changed: string[] = []; +for (const f of [...names].sort()) { + const all = [...dirs.a, ...dirs.b]; + if (!all.every((d) => existsSync(join(d, f)))) { + changed.push(`${f}: missing in ${all.filter((d) => !existsSync(join(d, f))).map((d) => d.split("/").pop()).join(", ")}`); + continue; + } + const between = Math.max(...dirs.a.flatMap((x) => dirs.b.map((y) => diff(join(x, f), join(y, f))))); + if (between === 0) { + same++; + continue; + } + const noise = Math.max(spread(dirs.a, f), spread(dirs.b, f)); + if (noise > 0 && between <= 2 * noise) { + within++; + continue; + } + if (between <= tolerance) { + rounding++; + console.log(` ${f}: ${between.toExponential(2)} (rounding, under --tolerance)`); + continue; + } + changed.push(`${f}: ${noise === 0 ? "repeatable, and" : `run to run ${noise.toExponential(2)},`} between the builds ${between.toExponential(2)}`); +} +console.log(`render-identity: ${names.size} outputs, ${same} byte-identical, ${within} within their run-to-run spread, ${rounding} within --tolerance, ${changed.length} changed (${out})`); +for (const c of changed) console.log(` ${c}`); +process.exit(changed.length ? 1 : 0); diff --git a/tools/test-render.ts b/tools/test-render.ts index b1fb1d9..354562e 100644 --- a/tools/test-render.ts +++ b/tools/test-render.ts @@ -2,7 +2,7 @@ // against a known answer (physics, a converged render or UE) and passes, fails or is unsupported. One warm renderer // process runs them all; each is sized to ~300 ms; the layer stays under a minute. // -// node tools/test-render.ts [filter] [--assets DIR] [--json out.json] +// node tools/test-render.ts [filter] [--assets DIR] [--json out.json] [--out DIR (default out/tests/render)] import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { createInterface } from "node:readline"; @@ -16,7 +16,7 @@ const flag = (k: string, d: string) => (args.includes(k) ? args[args.indexOf(k) const filter = args.find((a) => !a.startsWith("--") && !args[args.indexOf(a) - 1]?.startsWith("--")); const assets = resolve(flag("--assets", repoRoot)); const dir = join(repoRoot, "tests/render"); -const out = join(repoRoot, "out/tests/render"); +const out = resolve(flag("--out", join(repoRoot, "out/tests/render"))); mkdirSync(out, { recursive: true }); // Its renders would skew another run's timings: wait for the GPU lock (tools/lib/bench.ts) and hold it. await waitGpuLock("test-render");