From a4f4d9eca50a04602ad34db88590f09536977dc4 Mon Sep 17 00:00:00 2001 From: yujiaxu-ops Date: Fri, 13 Nov 2026 21:12:33 +0800 Subject: [PATCH 1/2] The window's quality measurements into the repo: bench-window, render-identity, an orbit in bench-ghosting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tools/bench-window.ts: the play window against offline renders of the same moment, in the shots the window's quality work of 2026-11 was measured by: Bistro and Kabukichō held still at 1440p (detail, dE, SSIM, flicker), the Bistro slice half a second after its explosion, Tokyo at dusk (real time against the moment held), Tokyo turning and orbiting as the play window's left drag does (the tower and the whole frame, against offline of the last pose). Truths cached in out/cache/window-truth; the Tokyo shots need the city. - tools/render-identity.ts: test-render twice on each of two builds, every HDR output compared byte for byte; a test a build does not repeat passes within twice its run-to-run spread. Against itself: 172 byte-identical, 18 within their spread, 0 changed. - bench-ghosting: the play window's orbit (view.rs Orbit), reported, not yet guarded. - docs/WORKFLOW.md names both in the merge gate (R1: render-identity; R2 for the live passes: bench-window). On the way: upscale/lamps-in-motion-live never exercised the motion clamp (its lamps float before the black sky, which a sliding camera does not move). It changes with the fix of that cause; ROADMAP § 1a. Co-Authored-By: Claude Opus 5.5 --- docs/ROADMAP.md | 10 ++ docs/WORKFLOW.md | 11 +- tools/bench-ghosting.ts | 25 ++++- tools/bench-window.ts | 212 +++++++++++++++++++++++++++++++++++++++ tools/render-identity.ts | 104 +++++++++++++++++++ 5 files changed, 355 insertions(+), 7 deletions(-) create mode 100644 tools/bench-window.ts create mode 100644 tools/render-identity.ts diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ef2e55f..116efd6 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -2107,6 +2107,16 @@ traced pixel that the fast mean's buffer had free: locked lettering of a still sign comes back from a motion's blur by the blend alone (the 0.2 on Kabukichō's signs); a lamp that comes on arrives over a quarter of a second. +**The measurements into the repo (2026-11-13, after the cleanup).** What the window's quality work of these weeks was +measured by lived in scratch scripts; now `tools/bench-window.ts` (the still frames, the Bistro slice, Tokyo at dusk, +Tokyo turning and orbiting as the play window's left drag does, each against offline), `tools/render-identity.ts` +(the byte gate for changes that must not change a picture) and an orbit move in `bench-ghosting` (reported, not yet +guarded). On the way: the moving lamps test (`upscale/lamps-in-motion-live`) never exercised the motion clamp. Its +lamps float before the black sky, a sliding camera does not move the sky, and a lamp no sample hit takes the +velocity of what is behind it: the upscaler saw no motion. Before a black wall 1 cm behind, the lamps keep 0.045 of +their 0.1 in motion as merged (no clamp at all: 0.105; before the lock: 0.014). The test changes with the fix of its +cause (a missed lamp reprojected by its own depth), so that main does not carry a failing test meanwhile. + **Cleanup: experiments and switches that outlived their measurement (2026-11-13, after it).** Removed, each off by default, so nothing a scene shows changes: diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index f9595bd..8ce0438 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -66,10 +66,17 @@ highest tier any of its changes needs. |---|---|---| | **D** docs | only `docs/`, READMEs, comments | CI | | **T** tools / TypeScript | `tools/`, `tests/` harness, `ue5/` scripts, `packages/*` without effect on what the renderer draws | `npm run typecheck`, `npm test`; `npm run test:render` if `packages/scene`, `packages/gameplay` or `tests/render` changed | -| **R1** renderer, existing scenes unchanged | new kernel / live code behind a feature bit, new DSL features, new passes that only run when a scene uses them | T, plus `cargo test --release`, `test:render` (no new failures against main), byte-identity of the catalog scenes (offline, software traversal), **perf smoke**, crops of the new feature | -| **R2** renderer, existing scenes change | anything that changes what existing scenes render, or the live passes they run (denoising, temporal, reflections, post) | R1 without byte-identity, plus `node tools/bench-ghosting.ts --quick --check` (Bistro) and `--scene kabukicho` (wet signs: the reflection stress case), `test:layer-b`, crops before / after | +| **R1** renderer, existing scenes unchanged | new kernel / live code behind a feature bit, new DSL features, new passes that only run when a scene uses them | T, plus `cargo test --release`, `test:render` (no new failures against main), byte-identity of the catalog scenes (offline, software traversal) and of every `test:render` output (`node tools/render-identity.ts --a
--b `: live tests included), **perf smoke**, crops of the new feature | +| **R2** renderer, existing scenes change | anything that changes what existing scenes render, or the live passes they run (denoising, temporal, reflections, post) | R1 without byte-identity, plus `node tools/bench-ghosting.ts --quick --check` (Bistro) and `--scene kabukicho` (wet signs: the reflection stress case), `test:layer-b`, crops before / after; a change to the live passes also `node tools/bench-window.ts` on main and the branch (the play window against offline: still, the Bistro slice, Tokyo at dusk and in motion) | | **R3** hot path or perf claim | the PR claims a speed-up, or changes the cost of work existing scenes do every frame (trace loop, light sampling, filter passes, resolution control) | R2, plus **full A/B timing** (Bistro night still and moving and the 3 × 3 Bistro district along its camera path at the 0.5 floor, main and branch alternated: `node tools/bench-district.ts`) | +- **render-identity** (R1, and any change that says it changes no picture): test-render whole, twice on each build, + every HDR output compared byte for byte. A test one build does not repeat to the byte (asynchronous builds, + physics, the radiance cache's training order: about 18 of 190) passes when the builds differ by no more than twice + their own run-to-run spread. `--env THREERT_MERGED=1` runs the browser's binding layout natively. +- **bench-window** (R2 for the live passes): the shots the window's quality work of 2026-11 was measured by, each + against an offline render of the same moment (cached in `out/cache/window-truth`), or for the dusk against the + same moment held. The Tokyo shots need the city (`tools/tokyo`); without it they are reported as not run. - **Perf smoke** (R1, R2): - Bistro night (`scenes/bistro.ts`) live at the fixed 0.5 floor, after warm-up: about 15 s still at its camera and 15 s on a scripted move (2.5 m down the street with 12° of yaw, ping-pong), GPU timestamps. Until 2026-11-08 diff --git a/tools/bench-ghosting.ts b/tools/bench-ghosting.ts index 20113f1..c40f2db 100644 --- a/tools/bench-ghosting.ts +++ b/tools/bench-ghosting.ts @@ -19,7 +19,11 @@ // history reprojects exactly, so what is left is the history's own blur, lag or doubling; // strafe: 2.5 m to the left (3 m to the right) in 0.4 s, hold, back (parallax, disocclusion, reflections moving // against their surface); -// stop: 8 m/s forward for 0.5 s, then still (what the history drags into the first frames after a sudden stop). +// stop: 8 m/s forward for 0.5 s, then still (what the history drags into the first frames after a sudden stop); +// orbit: the play window's left drag (view.rs `Orbit`: about a point ahead of the camera as far as the camera is +// from the origin, held to 1-50 m), 27.5 deg in 0.4 s, hold, back: rotation and translation at once, a +// degree a frame, where a feature finer than a pixel that the samples miss reprojects with what is behind +// it (a second copy of a lamp, 2026-11-13). Reported, not yet guarded. // Shots: each move's midpoint going out (vs the converged midpoint); the start pose 0.1 / 0.5 / 2 s after returning (counted in frames at 60 fps, not wall-clock time); // the stop pose 0 / 0.1 / 0.3 s after stopping. Per region (the wet street, where reflections are; the sign band; // the facades): mean dE, p95, p99, trail_px (p95 over rows of the longest horizontal run of pixels off by dE > 10) and @@ -144,6 +148,17 @@ function pose(yaw: number, side = 0, fwd = 0) { return { pos, target: pos.map((p, i) => p + dir[i] * 10) as V3, tan: view.tan }; } const turn = (f: number) => pose(view.turn * f); +/** The play window's orbit (view.rs `Orbit::from_camera`): about the point `dist` along the start pose's view, `dist` + * the start position's distance from the origin held to 1..50 m, `ORBIT_DEG` degrees at f = 1. */ +const ORBIT_DEG = 27.5; +function orbit(f: number) { + const p = pose(0); + const dir = p.target.map((t, i) => (t - p.pos[i]) / 10); + const n = Math.hypot(...dir), dist = Math.min(Math.max(Math.hypot(...view.p0), 1), 50); + const c = p.pos.map((x, i) => x + (dir[i] / n) * dist) as V3; + const a = (ORBIT_DEG * f * Math.PI) / 180, ox = p.pos[0] - c[0], oz = p.pos[2] - c[2]; + return { pos: [c[0] + ox * Math.cos(a) + oz * Math.sin(a), p.pos[1], c[2] - ox * Math.sin(a) + oz * Math.cos(a)] as V3, target: c, tan: view.tan }; +} const strafe = (f: number) => pose(0, view.strafe * f); const dolly = (f: number) => pose(0, 0, 4 * f); @@ -190,7 +205,7 @@ async function start(s: string) { // References: converged frames (history reset, then accumulated) at every pose a shot is taken at. const refFile = (n: string) => join(refs, `${n}.png`); -const refPoses: Record> = { start: pose(0), "turn-mid": turn(0.5), "strafe-mid": strafe(0.5), stop: dolly(1) }; +const refPoses: Record> = { start: pose(0), "turn-mid": turn(0.5), "strafe-mid": strafe(0.5), "orbit-mid": orbit(0.5), stop: dolly(1) }; if (!Object.keys(refPoses).every((n) => existsSync(refFile(n)))) { console.log(`rendering converged references (${refFrames} frames each at ${refScale})…`); const live = await start(refScale); @@ -227,7 +242,7 @@ async function sweep(at: (f: number) => ReturnType, f0: number, f1: for (rep = 0; rep < repeats; rep++) { if (rep === 0) await shot("settled", "start", 240, pose(0)); else await live.step(240, pose(0)); - for (const [name, at, n] of [["turn", turn, 30], ["strafe", strafe, 24]] as const) { + for (const [name, at, n] of [["turn", turn, 30], ["strafe", strafe, 24], ["orbit", orbit, 24]] as const) { await sweep(at, 0, 1, n, `${name}-mid`); await live.frames(30); await sweep(at, 1, 0, n); @@ -299,11 +314,11 @@ const tag = (sceneName === "kabukicho" ? "" : `${sceneName}-`) + (quick ? "ghost summary(`${tag} turn-mid ground excess`, value["turn-mid/ground"].mean, { unit: " dE" }); summary(`${tag} turn-mid ground trail`, value["turn-mid/ground"].trail, { unit: " px", digits: 0 }); summary(`${tag} strafe-return+0.5s excess`, value["strafe-return+0.5s/all"].mean, { unit: " dE" }); -for (const m of ["turn", "strafe"]) { +for (const m of ["turn", "strafe", "orbit"]) { execFileSync(bin, ["montage", join(out, `${m}.jpg`), "--width", "640", join(out, `${m}-mid.png`), refFile(`${m}-mid`), join(out, `${m}-return+0.1s.png`), refFile("start")]); } execFileSync(bin, ["montage", join(out, "stop.jpg"), "--width", "640", join(out, "stop+0.png"), join(out, "stop+0.1s.png"), refFile("stop")]); -console.log(`${out}/{turn,strafe,stop}.jpg (shot | converged)`); +console.log(`${out}/{turn,strafe,orbit,stop}.jpg (shot | converged)`); if (args.includes("--check") && failed) { console.error(`ghosting: ${failed} guarded value(s) over the limit`); process.exit(1); diff --git a/tools/bench-window.ts b/tools/bench-window.ts new file mode 100644 index 0000000..b81a2d5 --- /dev/null +++ b/tools/bench-window.ts @@ -0,0 +1,212 @@ +// The play window's picture against what it should be, in the shots the window's quality work of 2026-11 was measured +// by (docs/ROADMAP.md § 1a). Each shot bundles its game, drives one headless live session in lockstep and compares +// its frames with an offline render of the same moment (rendered once and kept: out/cache/window-truth, keyed by the +// bundle and the shot; --fresh-truth renders them again), or, for the dusk, with the same moment held. +// +// node tools/bench-window.ts [--shots still-bistro,still-kabukicho,slice,tokyo-dusk,tokyo-turn,tokyo-orbit] +// [--tag NAME] [--fresh-truth] +// +// Shots (the bin is THREERT_BIN, else the release build): +// still-bistro, still-kabukicho the scene held still at 2560 x 1440 (render scale 0.5, 0.375), animation frozen, +// 300 frames, then two consecutive frames: detail (the gradients' regression on offline's: 1 keeps every edge), +// ΔE, SSIM against offline (512 samples), and the RMS change of luma from one frame to the next (flicker). +// slice the Bistro slice half a second after its explosion (40.5 s, from the start in lockstep, scale 0.375, 1440p): +// the same four numbers against offline at 40.5 s. +// tokyo-dusk the Tokyo time-of-day reel's room (16:00 to 17:42 in 10.5 s), the camera still, at 6, 7, 8 s: the frame +// in real time against the same moment held (time all but stopped, 120 frames); the skyline and the whole frame. +// tokyo-turn, tokyo-orbit Tokyo at night (13 s), the camera turning 0.25 degrees a frame, or orbiting as the play +// window's left drag does (1.146 degrees a frame about a point 50 m ahead), for 24 frames, then still: the last +// moving frame and 1, 6, 30 and 150 frames after it against offline of the last pose (256 samples), over the +// tower and the whole frame. +// The Tokyo shots need the city (assets/city, tools/tokyo): without it they are reported as not run. +// Output: out/bench-window///*.png, out/bench-window//metrics.json, and a table. +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import sharp from "sharp"; +import { bundleGame } from "./bundle-game.ts"; +import { rendererBin, repoRoot, waitGpuLock } from "./lib/bench.ts"; +import { startLive, type Build, type Live } from "./lib/live.ts"; + +const args = process.argv.slice(2); +const flag = (k: string, d: string) => (args.includes(k) ? args[args.indexOf(k) + 1] : d); +const ALL = ["still-bistro", "still-kabukicho", "slice", "tokyo-dusk", "tokyo-turn", "tokyo-orbit"]; +const shots = flag("--shots", ALL.join(",")).split(","); +const tag = flag("--tag", new Date().toISOString().replace(/[:.]/g, "-")); +const freshTruth = args.includes("--fresh-truth"); +const out = join(repoRoot, "out/bench-window", tag); +const presets = join(repoRoot, "presets"); +const build: Build = { bin: rendererBin, assets: repoRoot, presets, bundles: join(repoRoot, "out/bundles"), label: "bench-window" }; +for (const s of shots) if (!ALL.includes(s)) throw new Error(`bench-window: no shot ${s} (${ALL.join(", ")})`); +await waitGpuLock("bench-window"); + +type Row = Record; +const rows: Row[] = []; +const bundles = new Map(); +async function bundle(src: string): Promise { + if (!bundles.has(src)) bundles.set(src, (await bundleGame(join(repoRoot, src))).out); + return bundles.get(src)!; +} + +/** The offline render of a bundle at these settings, rendered once (keyed by the bundle and the settings). */ +function truth(b: string, o: { width: number; height: number; spp: number; time?: number; room?: string; camera?: { position: number[]; target: number[]; fov: number } }): string { + const key = createHash("sha256").update(readFileSync(b)).update(JSON.stringify(o)).digest("hex").slice(0, 20); + const path = join(repoRoot, "out/cache/window-truth", `${key}.png`); + if (existsSync(path) && !freshTruth) return path; + mkdirSync(join(repoRoot, "out/cache/window-truth"), { recursive: true }); + console.log(`bench-window: rendering the offline truth (${o.width} x ${o.height}, ${o.spp} samples)…`); + const extra = [...(o.time !== undefined ? ["--time", String(o.time)] : []), ...(o.room ? ["--room", o.room] : []), ...(o.camera ? ["--camera", JSON.stringify(o.camera)] : [])]; + execFileSync(rendererBin, ["render", b, "-o", path, "--assets", repoRoot, "--presets", presets, "--width", String(o.width), "--height", String(o.height), "--spp", String(o.spp), ...extra], { stdio: ["ignore", "ignore", "inherit"] }); + return path; +} + +type M = { mean_delta_e: number; p99_delta_e: number; ssim: number; luminance_ratio: number }; +const compare = (ref: string, img: string, region?: string): M => + JSON.parse(execFileSync(rendererBin, ["compare", ref, img, ...(region ? ["--region", region] : [])]).toString().trim().split("\n").pop()!); +const luma = async (f: string) => { + const { data, info } = await sharp(f).removeAlpha().raw().toBuffer({ resolveWithObject: true }); + const l = new Float32Array(info.width * info.height); + for (let i = 0; i < l.length; i++) l[i] = 0.2126 * data[i * 3] + 0.7152 * data[i * 3 + 1] + 0.0722 * data[i * 3 + 2]; + return { l, w: info.width, h: info.height }; +}; +/** How much of the reference's detail a frame keeps: the regression of its luma gradients on the reference's. */ +async function detail(ref: string, img: string): Promise { + const R = await luma(ref), T = await luma(img); + let num = 0, den = 0; + for (let y = 1; y < R.h - 1; y++) { + for (let x = 1; x < R.w - 1; x++) { + const i = y * R.w + x, rx = R.l[i + 1] - R.l[i - 1], ry = R.l[i + R.w] - R.l[i - R.w]; + num += rx * (T.l[i + 1] - T.l[i - 1]) + ry * (T.l[i + R.w] - T.l[i - R.w]); + den += rx * rx + ry * ry; + } + } + return num / den; +} +/** RMS of the luma change between two frames (0-255). */ +async function flicker(a: string, b: string): Promise { + const A = await luma(a), B = await luma(b); + let s = 0; + for (let i = 0; i < A.l.length; i++) s += (A.l[i] - B.l[i]) ** 2; + return Math.sqrt(s / A.l.length); +} + +/** Lockstep frames on a session: `n` frames (then a snapshot to `file`, or a sync); a camera with the first. */ +let syncs = 0; +async function frames(live: Live, n: number, file?: string, cam?: object) { + const id = `w${++syncs}`; + const done = file ? live.wait((l) => l.includes(`snapshot ${file}`)) : live.wait((l) => l === `live: sync ${id}`); + live.send({ ...(cam ? { cmd: "camera", ...cam, frames: n } : { cmd: "frames", n }), ...(file ? { snapshot: file } : { sync: id }) }); + await done; +} +async function session(b: string, o: { width: number; height: number; scale: string; room?: string; env?: Record }) { + const live = startLive(build, b, o); + await live.firstFrame; + live.send({ cmd: "lockstep", dt: 1 / 60 }); + return live; +} +const r3 = (x: number) => Math.round(x * 1000) / 1000; + +/** Held still: 300 frames, then two consecutive frames, against offline. */ +async function still(name: string, src: string, scale: string, env: Record) { + const dir = join(out, name); + mkdirSync(dir, { recursive: true }); + const b = await bundle(src); + const ref = truth(b, { width: 2560, height: 1440, spp: 512 }); + const live = await session(b, { width: 2560, height: 1440, scale, env: { THREERT_FREEZE_TIME: "1", ...env } }); + const [a, c] = [join(dir, "a.png"), join(dir, "b.png")]; + await frames(live, 300, a); + await frames(live, 1, c); + await live.quit(); + const m = compare(ref, a); + rows.push({ shot: name, frame: "held", region: "all", detail: r3(await detail(ref, a)), dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio, flicker: r3(await flicker(a, c)) }); +} + +const TOKYO = "games/tokyo/tokyo-tod.game.ts", TOKYO_ROOM = "shiba-15.6-17.7-10.5-r1600", TOKYO_SETTLE = 2; +const tokyoMissing = () => !existsSync(join(repoRoot, "assets/city/glb/shiba")); + +async function slice() { + const dir = join(out, "slice"); + mkdirSync(dir, { recursive: true }); + const b = await bundle("games/bistro-slice/bistro-slice.game.ts"); + const ref = truth(b, { width: 2560, height: 1440, spp: 512, time: 40.5 }); + const live = await session(b, { width: 2560, height: 1440, scale: "0.375" }); + const [a, c] = [join(dir, "a.png"), join(dir, "b.png")]; + await frames(live, Math.round(40.5 * 60), a); + await frames(live, 1, c); + await live.quit(); + const m = compare(ref, a); + rows.push({ shot: "slice", frame: "40.5 s", region: "all", detail: r3(await detail(ref, a)), dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio, flicker: r3(await flicker(a, c)) }); +} + +/** The dusk: real time against the same moment held, a session a moment. */ +async function tokyoDusk() { + const dir = join(out, "tokyo-dusk"); + mkdirSync(dir, { recursive: true }); + const b = await bundle(TOKYO); + for (const t of [6, 7, 8]) { + const live = await session(b, { width: 1280, height: 720, scale: "0.75", room: TOKYO_ROOM }); + const [moving, held] = [join(dir, `t${t}.png`), join(dir, `t${t}-held.png`)]; + await frames(live, Math.round((TOKYO_SETTLE + t) * 60) - 1, moving); + live.send({ cmd: "lockstep", dt: 1e-5 }); + await frames(live, 120, held); + await live.quit(); + for (const [region, g] of [["skyline", "0.2,0.3,0.8,0.5"], ["all", undefined]] as const) { + const m = compare(held, moving, g); + rows.push({ shot: "tokyo-dusk", frame: `${t} s`, region, dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio }); + } + } +} + +/** Tokyo at night, the camera turning or orbiting for 24 frames, then still, against offline of the last pose. */ +async function tokyoMove(name: string, orbit: boolean) { + const dir = join(out, name); + mkdirSync(dir, { recursive: true }); + const b = await bundle(TOKYO); + // The scene's camera (tokyo-tod.game.ts): at (900, 260, -900) looking at (0, 180, 0), 60 degrees. + const P = [900, 260, -900], D = [-900, -80, 900], DL = Math.hypot(...D); + const T = P.map((p, i) => p + (D[i] / DL) * 50); + const deg = orbit ? 1.146 : 0.25; + const cam = (a: number) => { + const c = Math.cos((a * Math.PI) / 180), s = Math.sin((a * Math.PI) / 180); + if (orbit) { const ox = P[0] - T[0], oz = P[2] - T[2]; return { pos: [T[0] + ox * c + oz * s, P[1], T[2] - ox * s + oz * c], target: T, tan: 0 }; } + return { pos: P, target: [P[0] + D[0] * c + D[2] * s, P[1] + D[1], P[2] - D[0] * s + D[2] * c], tan: 0 }; + }; + const last = cam(deg * 24); + const ref = truth(b, { width: 1280, height: 720, spp: 256, time: 13, room: TOKYO_ROOM, camera: { position: last.pos, target: last.target, fov: 60 } }); + const live = await session(b, { width: 1280, height: 720, scale: "0.75", room: TOKYO_ROOM }); + await frames(live, (TOKYO_SETTLE + 11) * 60); + const f = (n: string) => join(dir, `${n}.png`); + for (let k = 1; k <= 24; k++) await frames(live, 1, k === 24 ? f("last") : undefined, cam(deg * k)); + await frames(live, 1, f("stop1")); + await frames(live, 5, f("stop6")); + await frames(live, 24, f("stop30")); + await frames(live, 120, f("settled")); + await live.quit(); + // Where the tower is in the last pose (the turn keeps it about the middle; the orbit carries it right). + const tower = orbit ? "0.68,0.35,0.8,0.64" : "0.3,0.25,0.7,0.65"; + for (const n of ["last", "stop1", "stop6", "stop30", "settled"]) { + for (const [region, g] of [["tower", tower], ["all", undefined]] as const) { + const m = compare(ref, f(n), g); + rows.push({ shot: name, frame: n, region, dE: m.mean_delta_e, p99: m.p99_delta_e, ssim: m.ssim, lum: m.luminance_ratio }); + } + } +} + +mkdirSync(out, { recursive: true }); +for (const s of shots) { + console.log(`bench-window: ${s}`); + if (s.startsWith("tokyo") && tokyoMissing()) { + rows.push({ shot: s, frame: "not run", region: "the city is not here (assets/city)" }); + continue; + } + if (s === "still-bistro") await still(s, "scenes/bistro.ts", "0.5", {}); + else if (s === "still-kabukicho") await still(s, "games/kabukicho/kabukicho.game.ts", "0.375", { THREERT_NO_GRAIN: "1" }); + else if (s === "slice") await slice(); + else if (s === "tokyo-dusk") await tokyoDusk(); + else if (s === "tokyo-turn") await tokyoMove(s, false); + else if (s === "tokyo-orbit") await tokyoMove(s, true); +} +console.table(rows); +writeFileSync(join(out, "metrics.json"), JSON.stringify({ bin: rendererBin, shots, rows }, null, 1)); +console.log(`bench-window: ${join(out, "metrics.json")}`); diff --git a/tools/render-identity.ts b/tools/render-identity.ts new file mode 100644 index 0000000..1ac49fe --- /dev/null +++ b/tools/render-identity.ts @@ -0,0 +1,104 @@ +// Whether a change leaves every picture as it was: test-render whole, `--runs` times on each of two builds, every +// test's HDR output compared byte for byte. The gate for changes that must not change a picture (a refactor, a +// cleanup). A test whose output a build does not repeat to the byte (asynchronous builds, physics, the radiance +// cache's training order) is judged by its spread instead: the outputs of the two builds may differ by no more than +// twice the larger of the two builds' own run-to-run differences (the largest relative difference of a value, under a +// floor of 1e-3). `--tolerance r` lets a difference up to r pass as rounding (a shader compiled from other source can +// round differently: 2e-6 is what removing unused kernel code did, 2026-11-13); it is reported apart. Exit 1 when a test +// differs beyond these or its output is missing on one side. +// +// node tools/render-identity.ts --a --b [--runs 2] [--tolerance 0] [--env K=V,K=V] +// [--filter id] [--out out/render-identity/] +// +// `--env` goes to both builds (THREERT_MERGED=1: the browser's binding layout, natively). Each run holds the GPU lock +// as test-render does; the runs of one build follow each other. +import { execFileSync } from "node:child_process"; +import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { repoRoot } from "./lib/bench.ts"; + +const args = process.argv.slice(2); +const flag = (k: string, d = "") => (args.includes(k) ? args[args.indexOf(k) + 1] : d); +const a = resolve(flag("--a")), b = resolve(flag("--b")); +if (!existsSync(a) || !existsSync(b)) { + console.error("render-identity: --a and --b must name two threert-render binaries"); + process.exit(2); +} +const runs = Number(flag("--runs", "2")); +const tolerance = Number(flag("--tolerance", "0")); +const extra = Object.fromEntries(flag("--env").split(",").filter(Boolean).map((kv) => kv.split("=") as [string, string])); +const filter = flag("--filter"); +const out = resolve(flag("--out", join(repoRoot, "out/render-identity", new Date().toISOString().replace(/[:.]/g, "-")))); +const rendered = join(repoRoot, "out/tests/render"); + +/** One test-render run on `bin`, its HDR outputs copied to `dir`. */ +function run(bin: string, dir: string) { + for (const f of existsSync(rendered) ? readdirSync(rendered) : []) if (f.endsWith(".raw")) rmSync(join(rendered, f)); + try { + execFileSync("node", [join(repoRoot, "tools/test-render.ts"), ...(filter ? [filter] : [])], { env: { ...process.env, ...extra, THREERT_BIN: bin }, stdio: ["ignore", "ignore", "inherit"] }); + } catch { + // (Failing tests still leave their outputs; what is compared here is the pictures, not the verdicts.) + } + mkdirSync(dir, { recursive: true }); + for (const f of readdirSync(rendered)) if (f.endsWith(".raw")) cpSync(join(rendered, f), join(dir, f)); +} + +const dirs = { a: [] as string[], b: [] as string[] }; +for (const side of ["a", "b"] as const) { + for (let k = 0; k < runs; k++) { + const dir = join(out, `${side}${k + 1}`); + console.log(`render-identity: ${side} run ${k + 1} of ${runs}`); + run(side === "a" ? a : b, dir); + dirs[side].push(dir); + } +} + +const floats = (f: string) => { + const d = readFileSync(f); + return new Float32Array(d.buffer, d.byteOffset, d.byteLength / 4); +}; +/** The largest relative difference of a value between two outputs (0: identical to the byte). */ +function diff(x: string, y: string): number { + const bx = readFileSync(x), by = readFileSync(y); + if (bx.equals(by)) return 0; + if (bx.length !== by.length) return Infinity; + const fx = floats(x), fy = floats(y); + let m = 0; + for (let i = 0; i < fx.length; i++) m = Math.max(m, Math.abs(fx[i] - fy[i]) / Math.max(Math.abs(fx[i]), Math.abs(fy[i]), 1e-3)); + return m; +} +const spread = (ds: string[], f: string) => { + let m = 0; + for (let i = 0; i < ds.length; i++) for (let j = i + 1; j < ds.length; j++) m = Math.max(m, diff(join(ds[i], f), join(ds[j], f))); + return m; +}; + +const names = new Set([...dirs.a, ...dirs.b].flatMap((d) => readdirSync(d))); +let same = 0, within = 0, rounding = 0; +const changed: string[] = []; +for (const f of [...names].sort()) { + const all = [...dirs.a, ...dirs.b]; + if (!all.every((d) => existsSync(join(d, f)))) { + changed.push(`${f}: missing in ${all.filter((d) => !existsSync(join(d, f))).map((d) => d.split("/").pop()).join(", ")}`); + continue; + } + const between = Math.max(...dirs.a.flatMap((x) => dirs.b.map((y) => diff(join(x, f), join(y, f))))); + if (between === 0) { + same++; + continue; + } + const noise = Math.max(spread(dirs.a, f), spread(dirs.b, f)); + if (noise > 0 && between <= 2 * noise) { + within++; + continue; + } + if (between <= tolerance) { + rounding++; + console.log(` ${f}: ${between.toExponential(2)} (rounding, under --tolerance)`); + continue; + } + changed.push(`${f}: ${noise === 0 ? "repeatable, and" : `run to run ${noise.toExponential(2)},`} between the builds ${between.toExponential(2)}`); +} +console.log(`render-identity: ${names.size} outputs, ${same} byte-identical, ${within} within their run-to-run spread, ${rounding} within --tolerance, ${changed.length} changed (${out})`); +for (const c of changed) console.log(` ${c}`); +process.exit(changed.length ? 1 : 0); From ea1ada9ffb01d2e526ba11a469bc39b8b777d626 Mon Sep 17 00:00:00 2001 From: yujiaxu-ops Date: Fri, 13 Nov 2026 21:37:57 +0800 Subject: [PATCH 2/2] render-identity: its own output directories (test-render --out), and no pass on nothing compared Bugbot on #64: the gate passed when --a / --b were missing (they resolved to the working directory, test-render then left no outputs, and nothing compared counted as nothing changed); and it emptied and copied the shared out/tests/render without the GPU lock, so another test-render run could lose its outputs. test-render takes --out DIR (default out/tests/render); each run of the gate writes its own directory and touches nothing else. Binaries that are not files, or a run that leaves no outputs, exit 2. Co-Authored-By: Claude Opus 5.5 --- tools/render-identity.ts | 30 +++++++++++++++++------------- tools/test-render.ts | 4 ++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tools/render-identity.ts b/tools/render-identity.ts index 1ac49fe..17a3486 100644 --- a/tools/render-identity.ts +++ b/tools/render-identity.ts @@ -10,37 +10,41 @@ // node tools/render-identity.ts --a --b [--runs 2] [--tolerance 0] [--env K=V,K=V] // [--filter id] [--out out/render-identity/] // -// `--env` goes to both builds (THREERT_MERGED=1: the browser's binding layout, natively). Each run holds the GPU lock -// as test-render does; the runs of one build follow each other. +// `--env` goes to both builds (THREERT_MERGED=1: the browser's binding layout, natively). Each run is test-render with +// an output directory of its own (it holds the GPU lock as it always does); the runs of one build follow each other. +// No outputs on a side, or no binary at --a / --b, is an error (exit 2): a gate that compared nothing has passed +// nothing. import { execFileSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs"; import { join, resolve } from "node:path"; import { repoRoot } from "./lib/bench.ts"; const args = process.argv.slice(2); const flag = (k: string, d = "") => (args.includes(k) ? args[args.indexOf(k) + 1] : d); -const a = resolve(flag("--a")), b = resolve(flag("--b")); -if (!existsSync(a) || !existsSync(b)) { +const isFile = (p: string) => p !== "" && existsSync(p) && statSync(p).isFile(); +if (!isFile(flag("--a")) || !isFile(flag("--b"))) { console.error("render-identity: --a and --b must name two threert-render binaries"); process.exit(2); } +const a = resolve(flag("--a")), b = resolve(flag("--b")); const runs = Number(flag("--runs", "2")); const tolerance = Number(flag("--tolerance", "0")); const extra = Object.fromEntries(flag("--env").split(",").filter(Boolean).map((kv) => kv.split("=") as [string, string])); const filter = flag("--filter"); const out = resolve(flag("--out", join(repoRoot, "out/render-identity", new Date().toISOString().replace(/[:.]/g, "-")))); -const rendered = join(repoRoot, "out/tests/render"); - -/** One test-render run on `bin`, its HDR outputs copied to `dir`. */ +/** One test-render run on `bin`, its outputs in `dir` (its own: nothing else writes there). */ function run(bin: string, dir: string) { - for (const f of existsSync(rendered) ? readdirSync(rendered) : []) if (f.endsWith(".raw")) rmSync(join(rendered, f)); + rmSync(dir, { recursive: true, force: true }); try { - execFileSync("node", [join(repoRoot, "tools/test-render.ts"), ...(filter ? [filter] : [])], { env: { ...process.env, ...extra, THREERT_BIN: bin }, stdio: ["ignore", "ignore", "inherit"] }); + execFileSync("node", [join(repoRoot, "tools/test-render.ts"), ...(filter ? [filter] : []), "--out", dir], { env: { ...process.env, ...extra, THREERT_BIN: bin }, stdio: ["ignore", "ignore", "inherit"] }); } catch { // (Failing tests still leave their outputs; what is compared here is the pictures, not the verdicts.) } - mkdirSync(dir, { recursive: true }); - for (const f of readdirSync(rendered)) if (f.endsWith(".raw")) cpSync(join(rendered, f), join(dir, f)); + const n = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".raw")).length : 0; + if (n === 0) { + console.error(`render-identity: test-render left no outputs in ${dir} (${bin})`); + process.exit(2); + } } const dirs = { a: [] as string[], b: [] as string[] }; @@ -73,7 +77,7 @@ const spread = (ds: string[], f: string) => { return m; }; -const names = new Set([...dirs.a, ...dirs.b].flatMap((d) => readdirSync(d))); +const names = new Set([...dirs.a, ...dirs.b].flatMap((d) => readdirSync(d).filter((f) => f.endsWith(".raw")))); let same = 0, within = 0, rounding = 0; const changed: string[] = []; for (const f of [...names].sort()) { diff --git a/tools/test-render.ts b/tools/test-render.ts index b1fb1d9..354562e 100644 --- a/tools/test-render.ts +++ b/tools/test-render.ts @@ -2,7 +2,7 @@ // against a known answer (physics, a converged render or UE) and passes, fails or is unsupported. One warm renderer // process runs them all; each is sized to ~300 ms; the layer stays under a minute. // -// node tools/test-render.ts [filter] [--assets DIR] [--json out.json] +// node tools/test-render.ts [filter] [--assets DIR] [--json out.json] [--out DIR (default out/tests/render)] import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { createInterface } from "node:readline"; @@ -16,7 +16,7 @@ const flag = (k: string, d: string) => (args.includes(k) ? args[args.indexOf(k) const filter = args.find((a) => !a.startsWith("--") && !args[args.indexOf(a) - 1]?.startsWith("--")); const assets = resolve(flag("--assets", repoRoot)); const dir = join(repoRoot, "tests/render"); -const out = join(repoRoot, "out/tests/render"); +const out = resolve(flag("--out", join(repoRoot, "out/tests/render"))); mkdirSync(out, { recursive: true }); // Its renders would skew another run's timings: wait for the GPU lock (tools/lib/bench.ts) and hold it. await waitGpuLock("test-render");