diff --git a/src/engine/panel-protection/archive-background.test.ts b/src/engine/panel-protection/archive-background.test.ts index c97aa35..31b137e 100644 --- a/src/engine/panel-protection/archive-background.test.ts +++ b/src/engine/panel-protection/archive-background.test.ts @@ -1,5 +1,5 @@ import { test, expect } from "bun:test"; -import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { mkdtempSync, writeFileSync, rmSync, openSync, ftruncateSync, closeSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { randomBytes } from "node:crypto"; @@ -21,3 +21,17 @@ test("backup compression leaves the request event loop responsive and preserves expect(Buffer.from(decryptArchive(encrypted, key).database, "base64").equals(database)).toBe(true); } finally { rmSync(dir, { recursive: true, force: true }); } }); + +test("a large database snapshot does not require database-sized JSON copies in memory", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "ocd-large-backup-")); + try { + const snapshotPath = path.join(dir, "deploy.db"); + const fd = openSync(snapshotPath, "w"); + ftruncateSync(fd, 160 * 1024 * 1024); + closeSync(fd); + const before = process.resourceUsage().maxRSS; + const encrypted = await encryptSnapshot({ snapshotPath, key: generateRecoveryKey(), archive: { version: 1, createdAt: new Date().toISOString(), image: "test", schemaVersion: 1, jwtSecret: "test", ssh: {} } }); + expect(encrypted.length).toBeGreaterThan(0); + expect((process.resourceUsage().maxRSS - before) * 1024).toBeLessThan(128 * 1024 * 1024); + } finally { rmSync(dir, { recursive: true, force: true }); } +}, 30_000); diff --git a/src/engine/panel-protection/archive-background.ts b/src/engine/panel-protection/archive-background.ts index e38f341..4d695f2 100644 --- a/src/engine/panel-protection/archive-background.ts +++ b/src/engine/panel-protection/archive-background.ts @@ -6,14 +6,21 @@ import type { PanelArchive } from "./archive.ts"; export function encryptSnapshot(input: { snapshotPath: string; databasePath?: string; key: string; archive: Omit }): Promise { return new Promise((resolve, reject) => { const worker = new Worker(new URL("./archive-worker.ts", import.meta.url), { workerData: input }); + const timeout = setTimeout(() => { + void worker.terminate(); + reject(new Error("Panel archive worker timed out")); + }, 120_000); + timeout.unref(); let received = false; worker.once("message", (bytes: Uint8Array) => { + clearTimeout(timeout); received = true; resolve(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)); void worker.terminate(); }); - worker.once("error", () => reject(new Error("Panel archive encryption failed"))); + worker.once("error", () => { clearTimeout(timeout); reject(new Error("Panel archive encryption failed")); }); worker.once("exit", () => { + clearTimeout(timeout); if (!received) reject(new Error("Panel archive worker exited without a backup")); }); }); diff --git a/src/engine/panel-protection/archive-worker.ts b/src/engine/panel-protection/archive-worker.ts index 62d8007..f63faae 100644 --- a/src/engine/panel-protection/archive-worker.ts +++ b/src/engine/panel-protection/archive-worker.ts @@ -1,7 +1,7 @@ import { parentPort, workerData } from "node:worker_threads"; -import { readFileSync, statSync } from "node:fs"; +import { statSync } from "node:fs"; import { Database } from "bun:sqlite"; -import { encryptArchive, sha256, MAX_ARCHIVE_BYTES, type PanelArchive } from "./archive.ts"; +import { encryptSnapshotArchive, MAX_ARCHIVE_BYTES, type PanelArchive } from "./archive.ts"; const input = workerData as { snapshotPath: string; databasePath?: string; key: string; archive: Omit }; try { @@ -18,8 +18,7 @@ try { } finally { snapshot.close(); } } if (statSync(input.snapshotPath).size > MAX_ARCHIVE_BYTES / 2) throw new Error("Snapshot exceeds backup size limit"); - const database = readFileSync(input.snapshotPath); - const encrypted = encryptArchive({ ...input.archive, database: database.toString("base64"), databaseSha256: sha256(database) }, input.key); + const encrypted = await encryptSnapshotArchive(input.snapshotPath, input.archive, input.key); const bytes = new Uint8Array(encrypted); parentPort!.postMessage(bytes, [bytes.buffer]); } catch { diff --git a/src/engine/panel-protection/archive.ts b/src/engine/panel-protection/archive.ts index 86c580f..4d61384 100644 --- a/src/engine/panel-protection/archive.ts +++ b/src/engine/panel-protection/archive.ts @@ -1,5 +1,8 @@ import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; -import { gzipSync, gunzipSync } from "node:zlib"; +import { gzipSync, gunzipSync, createGzip } from "node:zlib"; +import { createReadStream } from "node:fs"; +import { Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; const MAGIC = Buffer.from("OCD-BACKUP-1\n"); export const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024; @@ -26,6 +29,41 @@ export function encryptArchive(archive: PanelArchive, key: string): Buffer { const compressed = gzipSync(JSON.stringify(archive)); return Buffer.concat([MAGIC, iv, cipher.update(compressed), cipher.final(), cipher.getAuthTag()]); } + +/** Preserve the archive format without materializing the database, its base64 + * string and a second JSON copy together on a small production panel. */ +export async function encryptSnapshotArchive(snapshotPath: string, archive: Omit, key: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(snapshotPath)) hash.update(chunk); + const metadata = { ...archive, databaseSha256: hash.digest("hex") }; + async function* json() { + yield JSON.stringify(metadata).slice(0, -1) + ',"database":"'; + let remainder = Buffer.alloc(0); + for await (const chunk of createReadStream(snapshotPath)) { + const bytes = remainder.length ? Buffer.concat([remainder, chunk]) : chunk as Buffer; + const end = bytes.length - bytes.length % 3; + if (end) yield bytes.subarray(0, end).toString("base64"); + remainder = Buffer.from(bytes.subarray(end)); + } + if (remainder.length) yield remainder.toString("base64"); + yield '"}'; + } + const iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", keyBytes(key), iv); + cipher.setAAD(MAGIC); + const chunks: Buffer[] = [MAGIC, iv]; + let size = MAGIC.length + iv.length + 16; + await pipeline(Readable.from(json()), createGzip(), cipher, new Writable({ + write(chunk: Buffer, _encoding, done) { + size += chunk.length; + if (size > MAX_ARCHIVE_BYTES) return done(new Error("Backup exceeds maximum archive size")); + chunks.push(Buffer.from(chunk)); + done(); + }, + })); + chunks.push(cipher.getAuthTag()); + return Buffer.concat(chunks); +} export function decryptArchive(bytes: Buffer, key: string): PanelArchive { if (bytes.length > MAX_ARCHIVE_BYTES || !bytes.subarray(0, MAGIC.length).equals(MAGIC) || bytes.length < MAGIC.length + 28) { throw new Error("Unsupported panel backup");