diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daba610..3161ded 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,3 +21,11 @@ jobs: - uses: actions/checkout@v7.0.1 - uses: ./.github/actions/setup - run: pnpm test + + server-e2e: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7.0.1 + - uses: ./.github/actions/setup + - run: pnpm test:e2e diff --git a/README.md b/README.md index b059b36..2f38558 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ pnpm run typecheck # core を build してから各 package を型チェッ pnpm run lint # biome pnpm run test:unit # 純粋ロジック pnpm run test:integration # 実モデルロード+実 classify(モデルが無ければ skip) +pnpm run test:e2e # ビルド済み server に実画像を送信(事前に pnpm build が必要) # ローカル起動(config に modelDir を設定) pnpm --filter @misskey-sensitive-detector/server dev -- --config ./config.dev.mjs @@ -93,6 +94,16 @@ pnpm --filter @misskey-sensitive-detector/server dev -- --config ./config.dev.mj 統合テストは `SENSITIVE_DETECTOR_TEST_MODEL_DIR` でモデルディレクトリを上書きできる (既定: `./nsfw-model`)。モデルが無い環境では自動的に skip する。 +E2Eはリポジトリ同梱のモデルと[固定画像](apps/server/e2e/fixtures/README.md)を必須とし、 +欠落時は失敗する。別プロセスでビルド済みserverを起動し、Bearer認証付きの実HTTP通信で +AI生成の猫画像が `Neutral`、架空の成人を描いたAI生成の水着イラストが `Sexy` を最大確率として返すことと、5クラスの確率が +有限値かつ0〜1であることを確認する。モデル全体の精度評価や本番のセンシティブ判定ではない。 +起動待ちは最大120秒、HTTP通信は応答本文を含め最大60秒。成功・失敗時ともserverを終了する。 +CIではPR、mainへのpush、手動実行時に独立した `server-e2e` ジョブとして実行する。 + +`apps/server/e2e/check-images.ts` の `checkImages({ baseUrl, apiKey })` は起動処理と独立しており、 +Docker等で起動したserverにも同じ画像・検証を使用できる。 + ## 構成 - `packages/core` (`@misskey-sensitive-detector/core`): 推論エンジン。重いネイティブ実依存(onnxruntime-node)はここに集約。 diff --git a/apps/server/e2e/check-images.ts b/apps/server/e2e/check-images.ts new file mode 100644 index 0000000..20b3e0e --- /dev/null +++ b/apps/server/e2e/check-images.ts @@ -0,0 +1,90 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { basename } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const CLASSES = ['Drawing', 'Hentai', 'Neutral', 'Porn', 'Sexy'] as const; +export type Fixture = { file: URL; expectedClass: (typeof CLASSES)[number] }; +export const FIXTURES = [ + { file: new URL('./fixtures/neutral-synthetic-cat.png', import.meta.url), expectedClass: 'Neutral' }, + { file: new URL('./fixtures/sexy-synthetic-swimwear.png', import.meta.url), expectedClass: 'Sexy' }, +] as const satisfies readonly Fixture[]; +export const HTTP_TIMEOUT_MS = 60_000; + +/** These are regression expectations for fixed images, not a sensitive-content policy. */ +export function assertPredictions(body: unknown, fixtures: readonly Fixture[]): void { + type Prediction = { className: string; probability: number }; + type Item = { success?: boolean; predictions?: Prediction[] }; + const batch = body as { + success?: boolean; + result?: { results?: Item[] }; + } | null; + assert.equal(batch?.success, true, 'batch did not succeed'); + const results = batch?.result?.results; + assert(Array.isArray(results), 'missing results array'); + assert.equal(results.length, fixtures.length, 'wrong result count'); + for (const [index, fixture] of fixtures.entries()) { + const label = basename(fileURLToPath(fixture.file)); + const item: Item | undefined = results[index]; + assert.equal(item?.success, true, `${label}: inference failed`); + const predictions: Prediction[] | undefined = item?.predictions; + assert(Array.isArray(predictions), `${label}: missing predictions`); + assert.equal(predictions.length, 5, `${label}: expected five classes`); + for (const prediction of predictions) { + assert(prediction && typeof prediction === 'object', `${label}: invalid prediction`); + assert( + Number.isFinite(prediction.probability) && prediction.probability >= 0 && prediction.probability <= 1, + `${label}: invalid probability for ${prediction.className}`, + ); + } + assert.deepEqual( + predictions.map((p) => p.className).sort(), + [...CLASSES].sort(), + `${label}: unexpected or duplicate classes`, + ); + const expected = predictions.find((p) => p.className === fixture.expectedClass); + assert( + expected && predictions.every((p) => p === expected || p.probability < expected.probability), + `${label}: expected ${fixture.expectedClass} to have the highest probability: ${JSON.stringify(predictions)}`, + ); + } +} + +/** Works against either a local server or a published Docker port. No model access is needed here. */ +export async function checkImages({ + baseUrl, + apiKey, + fixtures = FIXTURES, + signal, + requestTimeoutMs = HTTP_TIMEOUT_MS, +}: { + baseUrl: string; + apiKey: string; + fixtures?: readonly Fixture[]; + signal?: AbortSignal; + requestTimeoutMs?: number; +}): Promise { + assert(fixtures.length > 0, 'at least one fixture is required'); + const form = new FormData(); + for (const [index, fixture] of fixtures.entries()) { + const data = await readFile(fixture.file); // Missing fixtures must fail, never skip. + form.append( + `image${index}`, + new Blob([new Uint8Array(data)], { type: 'image/png' }), + basename(fileURLToPath(fixture.file)), + ); + } + const timeout = AbortSignal.timeout(requestTimeoutMs); + const response = await fetch(new URL('/v1/detect-images', baseUrl), { + method: 'POST', + headers: { authorization: `Bearer ${apiKey}` }, + body: form, + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + }); + try { + assert.equal(response.status, 200, `detect-images returned HTTP ${response.status}`); + assertPredictions(await response.json(), fixtures); + } finally { + await response.body?.cancel().catch(() => {}); + } +} diff --git a/apps/server/e2e/fixtures/README.md b/apps/server/e2e/fixtures/README.md new file mode 100644 index 0000000..099cdb6 --- /dev/null +++ b/apps/server/e2e/fixtures/README.md @@ -0,0 +1,41 @@ +# E2E image fixtures + +These fixed AI-generated images exercise real ONNX inference. They are regression +examples, not a dataset for measuring model accuracy or a definition of sensitive +content. Both were generated for this project with the built-in OpenAI +`image_gen` tool on 2026-11-06. No photograph or other reference image was supplied +for either generation. + +| File | Expected top class | Subject | Prompt | +| ----------------------------- | ------------------ | --------------------------------------------------------- | ------------------------------------------------------- | +| `neutral-synthetic-cat.png` | `Neutral` | Photograph-style rendering of a fictional adult tabby cat | [Generation prompt](neutral-synthetic-cat.prompt.txt) | +| `sexy-synthetic-swimwear.png` | `Sexy` | 3D illustration of a fictional adult in opaque swimwear | [Generation prompt](sexy-synthetic-swimwear.prompt.txt) | + +## Provenance and reuse + +The cat prompt requests a generic fictional animal without a specific real pet, +reference photograph, branded mascot or existing fictional character. The swimwear +prompt requests an entirely fictional adult, a visibly synthetic 3D style, a +neutral pose, opaque clothing and no real-person or existing-character references. +The committed prompts record the inputs used to generate the selected images. +Neither test fixture is a third-party photograph. + +Both project-generated fixtures are included under the repository's +[MIT license](../../../../LICENSE), to the extent the contributors hold rights in +them. This does not assert exclusive copyright in AI-generated output. + +## Normalization and expectations + +Both selected images were resized to 299×299 (whole image, no crop) using FFmpeg's +Lanczos filter and converted to an 8-bit RGB PNG. Metadata is not copied. To +reproduce normalization from a source image: + +```sh +ffmpeg -i original.png -vf scale=299:299:flags=lanczos -pix_fmt rgb24 -frames:v 1 fixture.png +``` + +CI reads the committed PNGs and never downloads, generates or converts images. +Both were verified against the real bundled ONNX model: the cat's winning class +was `Neutral` (approximately 0.999855 at selection time), and the swimwear +illustration's winning class was `Sexy` (approximately 0.967). Only the winning +class and probability validity are asserted, not exact floats. diff --git a/apps/server/e2e/fixtures/neutral-synthetic-cat.png b/apps/server/e2e/fixtures/neutral-synthetic-cat.png new file mode 100644 index 0000000..8099bb1 Binary files /dev/null and b/apps/server/e2e/fixtures/neutral-synthetic-cat.png differ diff --git a/apps/server/e2e/fixtures/neutral-synthetic-cat.prompt.txt b/apps/server/e2e/fixtures/neutral-synthetic-cat.prompt.txt new file mode 100644 index 0000000..e011740 --- /dev/null +++ b/apps/server/e2e/fixtures/neutral-synthetic-cat.prompt.txt @@ -0,0 +1 @@ +Use case: photorealistic-natural. Asset type: original square raster fixture for an image-classifier regression test. Generate a natural-looking photograph-style image of one entirely fictional adult domestic tabby cat, sitting calmly on a light wooden floor in a simple sunlit room. The full cat is clearly visible and fills most of the square, with realistic fur, natural proportions, soft window light, and a quiet uncluttered background. Design a new generic cat without using any photograph, reference image, specific real animal, branded mascot, or existing fictional character. No people, text, logos, collars with identifying tags, or watermarks. This is an AI-generated fictional animal, not a depiction of a specific actual pet. diff --git a/apps/server/e2e/fixtures/sexy-synthetic-swimwear.png b/apps/server/e2e/fixtures/sexy-synthetic-swimwear.png new file mode 100644 index 0000000..6116b03 Binary files /dev/null and b/apps/server/e2e/fixtures/sexy-synthetic-swimwear.png differ diff --git a/apps/server/e2e/fixtures/sexy-synthetic-swimwear.prompt.txt b/apps/server/e2e/fixtures/sexy-synthetic-swimwear.prompt.txt new file mode 100644 index 0000000..1c60318 --- /dev/null +++ b/apps/server/e2e/fixtures/sexy-synthetic-swimwear.prompt.txt @@ -0,0 +1 @@ +Use case: stylized-concept. Asset type: original square raster fixture for a local image-classifier regression test. Create an unmistakably computer-rendered 3D illustration of one entirely fictional adult woman, approximately 30 years old, wearing an opaque simple two-piece swimsuit, standing upright with arms relaxed in a neutral swimwear-catalog pose. Frame the entire person from head to feet, filling most of the square. Simple warm studio background and soft even lighting. The face and materials must look visibly stylized and synthetic, like an original 3D character, not a photograph. Design a new generic character without using any real person, celebrity, photograph, reference image, or existing fictional character. Non-erotic presentation. No nudity, transparent clothing, suggestive posing, sexual activity, logos, text, or watermarks. This is a swimwear illustration, not a portrait of any actual person. diff --git a/apps/server/e2e/run.ts b/apps/server/e2e/run.ts new file mode 100644 index 0000000..e889246 --- /dev/null +++ b/apps/server/e2e/run.ts @@ -0,0 +1,13 @@ +import { checkImages } from './check-images.js'; +import { API_KEY, withServer } from './server.js'; +import { withSignals } from './signals.js'; + +try { + await withSignals((signal) => + withServer((baseUrl, running) => checkImages({ baseUrl, apiKey: API_KEY, signal: running }), { signal }), + ); + console.log('Server E2E passed: Neutral synthetic-cat and Sexy synthetic-swimwear predictions.'); +} catch (error) { + console.error(error); + process.exitCode = 1; +} diff --git a/apps/server/e2e/server.ts b/apps/server/e2e/server.ts new file mode 100644 index 0000000..0db3219 --- /dev/null +++ b/apps/server/e2e/server.ts @@ -0,0 +1,126 @@ +import { spawn } from 'node:child_process'; +import { access, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createInterface } from 'node:readline'; +import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; +import { HTTP_TIMEOUT_MS } from './check-images.js'; + +export const API_KEY = 'e2e-only-api-key-not-a-secret'; +const ROOT = new URL('../../../', import.meta.url); + +async function unusedPort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + if (!address || typeof address === 'string') throw new Error('could not allocate an E2E port'); + return address.port; +} + +async function waitForHealth(baseUrl: string, hasBound: () => boolean, signal: AbortSignal): Promise { + while (true) { + signal.throwIfAborted(); + if (hasBound()) { + try { + const response = await fetch(new URL('/health', baseUrl), { + signal: AbortSignal.any([signal, AbortSignal.timeout(HTTP_TIMEOUT_MS)]), + }); + await response.body?.cancel(); + if (response.status === 200) return; + } catch (error) { + signal.throwIfAborted(); + // Retry refused connections while the model loads; HTTP timeouts are failures. + if (!(error instanceof TypeError)) throw error; + } + } + try { + await delay(100, undefined, { signal }); + } catch (error) { + signal.throwIfAborted(); + throw error; + } + } +} + +/** Start the built entrypoint with isolated configuration and always reap its process. */ +export async function withServer( + check: (baseUrl: string, signal: AbortSignal) => Promise, + { + entry = new URL('../dist/main.mjs', import.meta.url), + modelDir = fileURLToPath(new URL('nsfw-model/', ROOT)), + startupTimeoutMs = 120_000, + signal = new AbortController().signal, + }: { entry?: URL; modelDir?: string; startupTimeoutMs?: number; signal?: AbortSignal } = {}, +): Promise { + signal.throwIfAborted(); + await access(entry); + await access(join(modelDir, 'nsfw_model.onnx')); // E2E must never skip a missing model. + const dir = await mkdtemp(join(tmpdir(), 'sensitive-detector-e2e-')); + try { + const port = await unusedPort(); + const configPath = join(dir, 'config.mjs'); + await writeFile( + configPath, + `export default ${JSON.stringify({ port, host: '127.0.0.1', modelDir, apiKey: API_KEY })};\n`, + ); + const child = spawn(process.execPath, [fileURLToPath(entry), '--config', configPath], { + cwd: fileURLToPath(ROOT), + env: { ...process.env, NODE_ENV: 'production', LOG_LEVEL: 'info' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const exited = new AbortController(); + let logs = ''; + const capture = (chunk: Buffer) => { + logs = (logs + chunk.toString()).slice(-64 * 1024); + }; + child.stdout.on('data', capture); + child.stderr.on('data', capture); + // A released ephemeral port may be claimed by another process. Accept health + // only after this child confirms its successful bind via the existing log. + let hasBound = false; + const lines = createInterface({ input: child.stdout }); + lines.on('line', (line) => { + try { + if (JSON.parse(line)?.msg === `listening on http://127.0.0.1:${port}`) hasBound = true; + } catch { + // Non-JSON diagnostic output is still retained in the captured logs. + } + }); + child.once('error', (error) => exited.abort(error)); + const closed = new Promise((resolve) => { + child.once('close', (code, exitSignal) => { + exited.abort(new Error(`server exited (code ${code}, signal ${exitSignal})`)); + resolve(); + }); + }); + const running = AbortSignal.any([signal, exited.signal]); + try { + const baseUrl = `http://127.0.0.1:${port}`; + await waitForHealth(baseUrl, () => hasBound, AbortSignal.any([running, AbortSignal.timeout(startupTimeoutMs)])); + running.throwIfAborted(); + const result = await check(baseUrl, running); + running.throwIfAborted(); + return result; + } catch (error) { + process.stderr.write(`E2E server logs:\n${logs}\n`); + throw error; + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM'); + const forceKill = setTimeout(() => child.kill('SIGKILL'), 5_000); + try { + await closed; + } finally { + clearTimeout(forceKill); + lines.close(); + } + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +} diff --git a/apps/server/e2e/signals.ts b/apps/server/e2e/signals.ts new file mode 100644 index 0000000..6934647 --- /dev/null +++ b/apps/server/e2e/signals.ts @@ -0,0 +1,16 @@ +/** Keep signal handlers installed until the task has finished its cleanup. */ +export async function withSignals(run: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + const onInterrupt = () => controller.abort(new Error('E2E interrupted by SIGINT')); + const onTerminate = () => controller.abort(new Error('E2E interrupted by SIGTERM')); + process.on('SIGINT', onInterrupt); + process.on('SIGTERM', onTerminate); + try { + const result = await run(controller.signal); + controller.signal.throwIfAborted(); + return result; + } finally { + process.removeListener('SIGINT', onInterrupt); + process.removeListener('SIGTERM', onTerminate); + } +} diff --git a/apps/server/package.json b/apps/server/package.json index 4a90421..a04beb2 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -9,7 +9,7 @@ "clean": "rm -rf dist", "dev": "tsx watch src/main.ts", "start": "node dist/main.mjs", - "typecheck": "tsc -p tsconfig.json --noEmit" + "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.e2e.json" }, "dependencies": { "@hono/node-server": "catalog:", diff --git a/apps/server/test/e2e-check.test.ts b/apps/server/test/e2e-check.test.ts new file mode 100644 index 0000000..b110912 --- /dev/null +++ b/apps/server/test/e2e-check.test.ts @@ -0,0 +1,111 @@ +import { createServer, type RequestListener } from 'node:http'; +import { afterEach, describe, expect, it } from 'vitest'; +import { assertPredictions, checkImages, FIXTURES } from '../e2e/check-images.js'; + +const predictions = [ + { className: 'Neutral', probability: 0.9 }, + { className: 'Drawing', probability: 0.04 }, + { className: 'Hentai', probability: 0.03 }, + { className: 'Porn', probability: 0.02 }, + { className: 'Sexy', probability: 0.01 }, +]; +const fixture = { file: FIXTURES[0].file, expectedClass: 'Neutral' as const }; +const response = (values: unknown = predictions) => ({ + success: true, + result: { results: [{ success: true, predictions: values }] }, +}); + +describe('E2E prediction assertions', () => { + it('accepts valid predictions regardless of response order', () => { + expect(() => assertPredictions(response([...predictions].reverse()), [fixture])).not.toThrow(); + }); + + it('rejects a prediction of the wrong class', () => { + expect(() => assertPredictions(response(), [{ ...fixture, expectedClass: 'Sexy' }])).toThrow(/expected Sexy/); + }); + + it.each([NaN, Infinity, -0.01, 1.01, '0.9', null])('rejects invalid probability %s', (probability) => { + expect(() => + assertPredictions(response([{ className: 'Neutral', probability }, ...predictions.slice(1)]), [fixture]), + ).toThrow(/probability/); + }); + + it('rejects ties for the highest probability', () => { + expect(() => + assertPredictions(response([{ ...predictions[0], probability: 0.04 }, ...predictions.slice(1)]), [fixture]), + ).toThrow(/expected Neutral/); + }); + + it.each([ + undefined, + { success: false }, + { success: true, result: { results: [] } }, + { success: true, result: { results: [{ success: false }] } }, + response(predictions.slice(1)), + response([...predictions.slice(0, 4), predictions[0]]), + response([...predictions.slice(0, 4), { className: 'Unknown', probability: 0.01 }]), + ])('rejects malformed, partial, or failed results %#', (body) => { + expect(() => assertPredictions(body, [fixture])).toThrow(); + }); +}); + +describe('E2E HTTP checks', () => { + let server: ReturnType | undefined; + afterEach(async () => { + server?.closeAllConnections(); + if (server) await new Promise((resolve) => server?.close(() => resolve())); + }); + + async function listen(handler: RequestListener) { + server = createServer(handler); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('missing TCP address'); + return `http://127.0.0.1:${address.port}`; + } + + it('sends the real fixture as multipart with bearer authentication', async () => { + let received = ''; + let authorization: string | undefined; + let contentType: string | undefined; + const baseUrl = await listen(async (req, res) => { + authorization = req.headers.authorization; + contentType = req.headers['content-type']; + for await (const chunk of req) received += chunk.toString('latin1'); + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify(response())); + }); + await checkImages({ baseUrl, apiKey: 'test-api-key', fixtures: [fixture] }); + expect(authorization).toBe('Bearer test-api-key'); + expect(contentType).toMatch(/^multipart\/form-data; boundary=/); + expect(received).toContain('neutral-synthetic-cat.png'); + expect(received).toContain('\x89PNG\r\n\x1a\n'); + }); + + it('fails for a missing image instead of skipping it', async () => { + await expect( + checkImages({ + baseUrl: 'http://127.0.0.1:1', + apiKey: 'test', + fixtures: [{ ...fixture, file: new URL('file:///missing-e2e-image.png') }], + }), + ).rejects.toThrow(/ENOENT/); + }); + + it('fails on an HTTP error', async () => { + const baseUrl = await listen((_req, res) => { + res.writeHead(401).end(); + }); + await expect(checkImages({ baseUrl, apiKey: 'test', fixtures: [fixture] })).rejects.toThrow(/401/); + }); + + it('times out even when headers arrive but the response body stalls', async () => { + const baseUrl = await listen((_req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{'); + }); + await expect(checkImages({ baseUrl, apiKey: 'test', fixtures: [fixture], requestTimeoutMs: 50 })).rejects.toThrow( + /timeout/i, + ); + }); +}); diff --git a/apps/server/test/e2e-server.test.ts b/apps/server/test/e2e-server.test.ts new file mode 100644 index 0000000..6a7a8a0 --- /dev/null +++ b/apps/server/test/e2e-server.test.ts @@ -0,0 +1,138 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { withServer } from '../e2e/server.js'; + +// A real child process isolates lifecycle failures from expensive ONNX inference. +describe('E2E server lifecycle', () => { + let dir: string; + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'e2e-lifecycle-test-')); + await writeFile(join(dir, 'nsfw_model.onnx'), 'model presence sentinel'); + }); + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + async function entry(body: string) { + const file = join(dir, 'server.mjs'); + await writeFile( + file, + `import {writeFileSync} from 'node:fs'; + writeFileSync(${JSON.stringify(join(dir, 'pid'))}, String(process.pid)); + ${body}`, + ); + return pathToFileURL(file); + } + const healthy = ` + import {createServer} from 'node:http'; + const {default: config} = await import(process.argv[3]); + createServer((req, res) => res.end('ok')).listen(config.port, config.host, () => { + console.log(JSON.stringify({msg: 'listening on http://' + config.host + ':' + config.port})); + }); + `; + async function expectChildStopped() { + const pid = Number(await readFile(join(dir, 'pid'), 'utf8')); + expect(() => process.kill(pid, 0)).toThrow(/ESRCH/); + } + + it('stops the server after a successful check', async () => { + await withServer(async () => {}, { entry: await entry(healthy), modelDir: dir }); + await expectChildStopped(); + }); + + it('stops the server after an assertion failure', async () => { + await expect( + withServer( + async () => { + throw new Error('wrong prediction'); + }, + { entry: await entry(healthy), modelDir: dir }, + ), + ).rejects.toThrow('wrong prediction'); + await expectChildStopped(); + }); + + it('fails and stops a server that never becomes healthy', async () => { + await expect( + withServer(async () => {}, { + entry: await entry('setInterval(() => {}, 1000);'), + modelDir: dir, + startupTimeoutMs: 1_000, + }), + ).rejects.toThrow(/timeout/i); + await expectChildStopped(); + }); + + it('bounds a stalled health response by the startup deadline', async () => { + await expect( + withServer(async () => {}, { + entry: await entry(healthy.replace("res.end('ok')", "{ res.writeHead(503); res.write('unavailable'); }")), + modelDir: dir, + startupTimeoutMs: 1_000, + }), + ).rejects.toThrow(/timeout/i); + await expectChildStopped(); + }); + + it('reports an early server exit without waiting for startup timeout', async () => { + await expect(withServer(async () => {}, { entry: await entry('process.exit(7);'), modelDir: dir })).rejects.toThrow( + /exited.*7/, + ); + await expectChildStopped(); + }); + + it('stops the server when the caller cancels an active check', async () => { + const controller = new AbortController(); + await expect( + withServer( + async (_url, signal) => { + controller.abort(new Error('cancelled test')); + signal.throwIfAborted(); + }, + { entry: await entry(healthy), modelDir: dir, signal: controller.signal }, + ), + ).rejects.toThrow('cancelled test'); + await expectChildStopped(); + }); + + it('rejects a missing model without starting the server', async () => { + const options = { entry: await entry(healthy), modelDir: join(dir, 'missing') }; + await expect(withServer(async () => {}, options)).rejects.toThrow(/ENOENT/); + await expect(readFile(join(dir, 'pid'))).rejects.toThrow(/ENOENT/); + }); + + it('does not accept a healthy competing listener before its own child binds', async () => { + const competing = createServer((_req, res) => res.end('ok')); + const portFile = join(dir, 'port'); + const selectedEntry = await entry(` + const {default: config} = await import(process.argv[3]); + writeFileSync(${JSON.stringify(portFile)}, String(config.port)); + setInterval(() => {}, 1000); + `); + const checked = withServer(async () => {}, { entry: selectedEntry, modelDir: dir, startupTimeoutMs: 2_000 }); + const rejected = expect(checked).rejects.toThrow(/timeout/i); + try { + let port = 0; + for (let attempt = 0; attempt < 100 && !port; attempt++) { + port = Number(await readFile(portFile, 'utf8').catch(() => '0')); + if (!port) await delay(10); + } + expect(port).toBeGreaterThan(0); + await new Promise((resolve, reject) => { + competing.once('error', reject); + competing.listen(port, '127.0.0.1', resolve); + }); + await rejected; + await expectChildStopped(); + } finally { + competing.closeAllConnections(); + await new Promise((resolve) => competing.close(() => resolve())); + await checked.catch(() => {}); + } + }); +}); diff --git a/apps/server/test/e2e-signals.test.ts b/apps/server/test/e2e-signals.test.ts new file mode 100644 index 0000000..7cdd98b --- /dev/null +++ b/apps/server/test/e2e-signals.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { expect, it } from 'vitest'; + +it.each([ + ['SIGINT', false], + ['SIGTERM', false], + ['SIGINT', true], + ['SIGTERM', true], +] as const)('finishes cleanup and fails after %s during teardown (repeated: %s)', async (signal, repeated) => { + const moduleUrl = new URL('../e2e/signals.ts', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--import', + 'tsx', + '--input-type=module', + '--eval', + `import {withSignals} from ${JSON.stringify(moduleUrl)}; + try { + await withSignals(async signal => { + const finishCleanup = new Promise(resolve => process.once('message', resolve)); + signal.addEventListener('abort', () => console.log('interrupted'), {once:true}); + console.log('ready'); + console.log('cleaning'); + await finishCleanup; + console.log('cleaned'); + }); + } catch { process.exitCode = 1; } + finally { process.disconnect(); }`, + ], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }, + ); + const stdout = child.stdout; + assert(stdout, 'child stdout must be piped'); + const closed = once(child, 'close'); + let output = ''; + const waitFor = (marker: string) => + new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`child never reported ${marker}: ${output}`)), 3_000); + const inspect = () => { + if (output.includes(marker)) { + clearTimeout(timer); + stdout.off('data', inspect); + resolve(); + } + }; + stdout.on('data', inspect); + inspect(); + }); + stdout.on('data', (chunk) => { + output += chunk.toString(); + }); + try { + await waitFor('ready'); + child.kill(signal); + await waitFor('interrupted'); + if (repeated) child.kill(signal); + child.send('finish cleanup', () => {}); + const [code, exitSignal] = await closed; + expect(exitSignal).toBe(null); + expect(code).toBe(1); + expect(output).toContain('cleaned'); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + await closed; + } +}); diff --git a/apps/server/tsconfig.e2e.json b/apps/server/tsconfig.e2e.json new file mode 100644 index 0000000..8154160 --- /dev/null +++ b/apps/server/tsconfig.e2e.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": false, + "noEmit": true, + "types": ["node"] + }, + "include": ["e2e/**/*.ts", "test/e2e-*.test.ts"] +} diff --git a/docs/superpowers/plans/2026-11-06-server-e2e.md b/docs/superpowers/plans/2026-11-06-server-e2e.md new file mode 100644 index 0000000..a5c1fc7 --- /dev/null +++ b/docs/superpowers/plans/2026-11-06-server-e2e.md @@ -0,0 +1,49 @@ +# Server E2E CI Implementation Plan + +> Use superpowers:executing-plans to implement the tasks in order. + +**Goal:** Run fixed real-image regression checks against the built server in CI. + +**Architecture:** A shared TypeScript HTTP checker owns the fixtures and prediction assertions. A separate runner starts the built server with temporary configuration, waits for health, invokes the checker, and always stops its child. Existing Vitest tests cover checker failures; actual E2E and deliberate failure runs verify the process lifecycle. + +**Tech Stack:** Node.js 22+, tsx, Vitest, GitHub Actions. + +**Spec:** https://github.com/misskey-dev/sensitive-detector/issues/38 + +## Global Constraints + +- Commit redistributable 299×299 PNG fixtures with source, license, and expected class. +- Five unique classes, finite probabilities in [0, 1], expected class strictly highest; no exact probabilities. +- Missing images/model fail; startup limit 120 seconds, HTTP limit 60 seconds. +- Always terminate the server, including errors and signals. +- Reuse the HTTP checker from Docker issue #39. No production API or release workflow changes. + +## Review Focus + +- Health connection attempts and body reads must remain bounded by the startup deadline. +- Failed inference and malformed responses must fail even when HTTP status is 200. +- Child exit, spawn errors, cancellation, and assertion failures must release the process and temporary files. +- Fixture licenses and expected classes must have independent provenance. +- E2E must not resolve the application entrypoint to source or skip missing inputs. + +### Task 1: Shared real-image checker and fixtures + +**Files:** apps/server/e2e/check-images.ts, apps/server/e2e/fixtures/*, apps/server/test/e2e-check.test.ts + +**Interfaces:** Produces FIXTURES and checkImages({ baseUrl, apiKey, signal?, fixtures?, requestTimeoutMs? }). + +- [x] Write response-validation and HTTP failure tests; run and observe RED. +- [x] Add generated fixtures with prompts/license documentation, implement checker; observe GREEN. +- [x] Confirm real model classifies a generated cat as Neutral and a generated fictional-adult swimwear illustration as Sexy. + +### Task 2: Built-server runner and CI + +**Files:** apps/server/e2e/server.ts, apps/server/e2e/run.ts, package.json, .github/workflows/ci.yml, README.md + +**Interfaces:** Consumes Task 1 checker and fixtures; produces pnpm test:e2e. + +- [x] Run test:e2e before implementation and observe absent command. +- [x] Implement child lifecycle, temporary config, readiness, bounded requests and cleanup. +- [x] Add dedicated CI job and usage documentation. +- [x] Run E2E, intentional wrong expectation, missing image/model, readiness/request timeouts, cancellation, and process cleanup checks. +- [x] Run unit/integration tests, typecheck and lint; request whole-branch review and address findings. diff --git a/package.json b/package.json index 1bbd19d..b7e6fd8 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test": "vitest run", "test:unit": "vitest run --project unit", "test:integration": "vitest run --project integration", + "test:e2e": "node --import tsx apps/server/e2e/run.ts", "lint": "pnpm exec -- biome check .", "format": "pnpm exec -- biome format --write ." },