From a394bb88a77dc49c54183c4f41b69684a4a590ed Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 11:22:12 +0200 Subject: [PATCH 1/8] ci: run the Studio browser suite through Endform with OIDC Keep four isolated application shards and cap each Endform run at one concurrent test because fixture cleanup clears shared storage. Preserve suite coverage and retries, transfer ARIA snapshots, and retain failed-attempt traces. --- .github/workflows/fork-studio-browser.yml | 12 ++-- packages/playground/e2e/endform.config.ts | 9 +++ packages/playground/e2e/playwright.config.ts | 4 +- packages/playground/package.json | 3 +- pnpm-lock.yaml | 64 ++++++++++++++++++++ 5 files changed, 83 insertions(+), 9 deletions(-) create mode 100644 packages/playground/e2e/endform.config.ts diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index 5f102b7e8e3e..cd61c030457a 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -9,7 +9,7 @@ permissions: {} jobs: e2e-kitchen-sink: - name: E2E kitchen-sink (${{ matrix.shard }}/${{ strategy.job-total }}) + name: Endform kitchen-sink (${{ matrix.shard }}/${{ strategy.job-total }}) runs-on: ubuntu-24.04 timeout-minutes: 45 strategy: @@ -18,6 +18,7 @@ jobs: shard: [1, 2, 3, 4] permissions: contents: read + id-token: write env: TURBO_CACHE: local:rw @@ -51,17 +52,16 @@ jobs: - name: Setup e2e project dependencies working-directory: ./packages/playground - run: pnpm test:e2e:setup + run: pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile - name: Test E2E - working-directory: ./packages/playground - run: pnpm exec playwright test -c e2e/playwright.config.ts --shard=${{ matrix.shard }}/${{ strategy.job-total }} + working-directory: ./packages/playground/e2e + run: npx endform@latest test --organization-id 2G1ZCj7X --shard=${{ matrix.shard }}/${{ strategy.job-total }} - name: Upload test artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: - name: studio-playwright-${{ matrix.shard }} + name: studio-endform-${{ matrix.shard }} path: packages/playground/test-results/ if-no-files-found: ignore - diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts new file mode 100644 index 000000000000..3ccedc268b8a --- /dev/null +++ b/packages/playground/e2e/endform.config.ts @@ -0,0 +1,9 @@ +import { defineEndformConfig } from 'endform'; + +export default defineEndformConfig({ + // ARIA snapshots are read from disk rather than imported by the specs. + additionalFiles: ['tests/**/*.aria.yml'], + // Tests clear the whole kitchen-sink database, so each application must serve + // one test at a time. CI retains four disjoint shards with isolated servers. + concurrentTestLimits: [{ scope: 'within-suite-run', limit: 1 }], +}); diff --git a/packages/playground/e2e/playwright.config.ts b/packages/playground/e2e/playwright.config.ts index 8168188310e1..ad8e9aba7846 100644 --- a/packages/playground/e2e/playwright.config.ts +++ b/packages/playground/e2e/playwright.config.ts @@ -24,7 +24,7 @@ if (PORT) { export default defineConfig({ testDir: './tests', - fullyParallel: false, + fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 1, workers: 1, @@ -32,7 +32,7 @@ export default defineConfig({ use: { baseURL: BASE_URL, - trace: 'on-first-retry', + trace: 'retain-on-failure', reducedMotion: 'reduce', }, diff --git a/packages/playground/package.json b/packages/playground/package.json index 915639ddf9fc..29ac777afe9c 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -110,14 +110,15 @@ "@types/react-syntax-highlighter": "^15.5.13", "@types/semver": "^7.7.1", "@vitejs/plugin-react": "^5.2.0", + "endform": "0.81.1", "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.5.3", "fake-indexeddb": "^6.2.5", "jsdom": "^27", + "msw": "^2.6.0", "react-grab": "^0.1.37", "tailwindcss": "4.3.3", "tw-animate-css": "^1.4.0", - "msw": "^2.6.0", "typescript": "catalog:ts6", "vite": "^7.3.1", "vitest": "catalog:" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a12391ec2ee1..d4cdd43c4f7e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5893,6 +5893,9 @@ importers: '@vitejs/plugin-react': specifier: ^5.2.0 version: 5.2.0(vite@7.3.6(@types/node@22.19.15)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.44.1)(tsx@4.23.1)(yaml@2.9.0)) + endform: + specifier: 0.81.1 + version: 0.81.1 eslint-plugin-react-hooks: specifier: ^7.1.1 version: 7.1.1(eslint@10.7.0(jiti@2.7.0)(supports-color@10.2.2)) @@ -25458,6 +25461,40 @@ packages: end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + endform-darwin-arm64@0.81.1: + resolution: {integrity: sha512-duBYoWEdBEdk9UHUrUA9ZYP3RqxXGrjS6plkrwFmdnhFAph+6gopXUljw6d1pJB0j/kP25LPBEPK+5iQ1rx5ng==} + cpu: [arm64] + os: [darwin] + + endform-darwin-x86@0.81.1: + resolution: {integrity: sha512-an6GjutD6PZAF5PLlWIEWTMOkLZx5LrVZ4OmukL1eQd8WWITsFi42VGfxYpYJFXe7GdQaCDrEaJSLCdsL0Iqyw==} + cpu: [x64] + os: [darwin] + + endform-linux-arm64@0.81.1: + resolution: {integrity: sha512-gxhTi3jsRtzA4ZhibK4idNAcp0cb0V4idZkJxLCq9ao8FUMUiiVHhjhtiFNO7eUGgsA8+DfFnRdiPa2zTePWjQ==} + cpu: [arm64] + os: [linux] + + endform-linux-x86@0.81.1: + resolution: {integrity: sha512-aXZP8Wds4bzcCRMFOL64/B7jQLIP7ECrDW8H6vjl6TZTjg7lipfPLams8R2RDigRUjB5i8RDwuusoeOMvYf2gA==} + cpu: [x64] + os: [linux] + + endform-win32-arm64@0.81.1: + resolution: {integrity: sha512-ckWkuUgGj9bPzMfTJOLX2n6YgWjPxhJWtcKMQX6it5OhB8bDjHA2vnhOowPZlLoPwLtjT3NaEUnh81NdHFVhDw==} + cpu: [arm64] + os: [win32] + + endform-win32-x86@0.81.1: + resolution: {integrity: sha512-I5cXm6huxhcs5ZTH+U4O0Ehx6kvpKH4HVATPT5y4C+GliAZVY+rcrB2NBIl+61xWsjHmoC6tkn+HX6lzKJlCGw==} + cpu: [x64] + os: [win32] + + endform@0.81.1: + resolution: {integrity: sha512-dLWKcpN6FfIEcxzzlJkBBDLcxll7xkS36tuKadt13lM8ZwbcmytgLHHgJ1USLtLuUWCrsPq172qtiZHN3sufXQ==} + hasBin: true + engine.io-client@6.6.6: resolution: {integrity: sha512-iY6QdftLQ9pyiPoX082bpf/u1UewnOaJrtJIF9T0++QB34lZrj0uP+Q/bj8AlUsAxqhnkTV2BS8SBZSxOmoV5Q==} @@ -51624,6 +51661,33 @@ snapshots: dependencies: once: 1.4.0 + endform-darwin-arm64@0.81.1: + optional: true + + endform-darwin-x86@0.81.1: + optional: true + + endform-linux-arm64@0.81.1: + optional: true + + endform-linux-x86@0.81.1: + optional: true + + endform-win32-arm64@0.81.1: + optional: true + + endform-win32-x86@0.81.1: + optional: true + + endform@0.81.1: + optionalDependencies: + endform-darwin-arm64: 0.81.1 + endform-darwin-x86: 0.81.1 + endform-linux-arm64: 0.81.1 + endform-linux-x86: 0.81.1 + endform-win32-arm64: 0.81.1 + endform-win32-x86: 0.81.1 + engine.io-client@6.6.6(bufferutil@4.1.0)(supports-color@10.2.2): dependencies: '@socket.io/component-emitter': 3.1.2 From 899827d0f6ad39bac8d1fd72247529e19fff3cb6 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 14:38:51 +0200 Subject: [PATCH 2/8] test(playground): isolate applications for a single Endform run --- .github/workflows/fork-studio-browser.yml | 14 +- packages/playground/e2e/README.md | 44 ++++ packages/playground/e2e/endform.config.ts | 7 +- packages/playground/e2e/isolation-server.mjs | 217 ++++++++++++++++++ .../playground/e2e/isolation-server.test.mjs | 61 +++++ .../e2e/kitchen-sink/src/mastra/index.ts | 1 + packages/playground/e2e/playwright.config.ts | 14 +- .../playground/e2e/tests/__utils__/test.ts | 43 ++++ .../deterministic-builder.spec.ts | 2 +- .../agents/$agentId/browser-stream.spec.ts | 2 +- .../agents/$agentId/composer-ring.spec.ts | 2 +- .../$agentId/history-pagination.spec.ts | 2 +- .../agents/$agentId/ime-composition.spec.ts | 2 +- .../e2e/tests/agents/$agentId/page.spec.ts | 2 +- .../e2e/tests/agents/$agentId/session.spec.ts | 2 +- .../e2e/tests/agents/$agentId/stream.spec.ts | 2 +- .../$agentId/tools/$toolId/page.spec.ts | 2 +- .../tests/agents/observability-tabs.spec.ts | 2 +- .../tests/agents/observational-memory.spec.ts | 2 +- .../playground/e2e/tests/agents/page.spec.ts | 2 +- .../e2e/tests/agents/recovery.spec.ts | 2 +- .../tests/agents/refresh-reconnect.spec.ts | 2 +- .../e2e/tests/auth/admin-role.spec.ts | 2 +- .../e2e/tests/auth/infrastructure.spec.ts | 2 +- .../e2e/tests/auth/login-flow.spec.ts | 2 +- .../e2e/tests/auth/member-role.spec.ts | 2 +- .../e2e/tests/auth/viewer-role.spec.ts | 2 +- .../cms/agents/code-agent-override.spec.ts | 2 +- .../e2e/tests/cms/agents/create/page.spec.ts | 2 +- .../e2e/tests/cms/scorers/create/page.spec.ts | 2 +- .../e2e/tests/cms/scorers/edit/page.spec.ts | 2 +- .../datasets/compare-experiments.spec.ts | 2 +- .../tests/datasets/dataset-items-list.spec.ts | 2 +- .../item-review-panels-layout.spec.ts | 2 +- .../e2e/tests/datasets/pagination.spec.ts | 2 +- .../e2e/tests/mcps/$serverId/page.spec.ts | 2 +- .../mcps/$serverId/tools/$toolId/page.spec.ts | 2 +- .../playground/e2e/tests/mcps/page.spec.ts | 2 +- .../e2e/tests/metrics/drilldown.spec.ts | 2 +- .../tests/metrics/filter-persistence.spec.ts | 2 +- .../playground/e2e/tests/metrics/page.spec.ts | 2 +- .../processors/$processorId/page.spec.ts | 2 +- .../e2e/tests/processors/page.spec.ts | 2 +- packages/playground/e2e/tests/root.spec.ts | 2 +- .../e2e/tests/scorers/$scorerId/page.spec.ts | 2 +- .../playground/e2e/tests/scorers/page.spec.ts | 2 +- .../e2e/tests/settings/api-prefix.spec.ts | 2 +- .../e2e/tests/settings/page.spec.ts | 2 +- .../tests/sidebar/cold-load-layout.spec.ts | 2 +- .../tests/sidebar/navigation-scroll.spec.ts | 2 +- .../templates/$templateSlug/page.spec.ts | 2 +- .../e2e/tests/templates/page.spec.ts | 2 +- .../e2e/tests/tools/$toolId/page.spec.ts | 2 +- .../playground/e2e/tests/tools/page.spec.ts | 2 +- .../playground/e2e/tests/traces/page.spec.ts | 2 +- .../e2e/tests/traces/partial-thread.spec.ts | 2 +- .../e2e/tests/traces/query-filters.spec.ts | 2 +- ...-step-by-step-conditional-run-page.spec.ts | 2 +- .../debug-step-by-step-edges.spec.ts | 2 +- .../debug-step-by-step-run-page.spec.ts | 2 +- .../$workflowId/debug-step-by-step.spec.ts | 2 +- .../$workflowId/nested-graph.spec.ts | 2 +- .../tests/workflows/$workflowId/page.spec.ts | 2 +- .../e2e/tests/workflows/page.spec.ts | 2 +- .../e2e/tests/workflows/schedules.spec.ts | 2 +- 65 files changed, 436 insertions(+), 79 deletions(-) create mode 100644 packages/playground/e2e/README.md create mode 100644 packages/playground/e2e/isolation-server.mjs create mode 100644 packages/playground/e2e/isolation-server.test.mjs create mode 100644 packages/playground/e2e/tests/__utils__/test.ts diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index cd61c030457a..cea214799f6c 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -9,18 +9,15 @@ permissions: {} jobs: e2e-kitchen-sink: - name: Endform kitchen-sink (${{ matrix.shard }}/${{ strategy.job-total }}) + name: Endform kitchen-sink runs-on: ubuntu-24.04 timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4] permissions: contents: read id-token: write env: TURBO_CACHE: local:rw + E2E_APP_POOL_SIZE: '8' steps: - name: Checkout repo @@ -54,14 +51,17 @@ jobs: working-directory: ./packages/playground run: pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile + - name: Verify application isolation + run: node --test packages/playground/e2e/isolation-server.test.mjs + - name: Test E2E working-directory: ./packages/playground/e2e - run: npx endform@latest test --organization-id 2G1ZCj7X --shard=${{ matrix.shard }}/${{ strategy.job-total }} + run: npx endform@latest test --organization-id 2G1ZCj7X - name: Upload test artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: - name: studio-endform-${{ matrix.shard }} + name: studio-endform path: packages/playground/test-results/ if-no-files-found: ignore diff --git a/packages/playground/e2e/README.md b/packages/playground/e2e/README.md new file mode 100644 index 000000000000..bc698c1f4cdc --- /dev/null +++ b/packages/playground/e2e/README.md @@ -0,0 +1,44 @@ +# Studio browser tests on Endform + +Run the existing suite from this directory after building the workspace dependencies +and installing `kitchen-sink` dependencies: + +```sh +E2E_APP_POOL_SIZE=8 npx endform@latest test --organization-id 2G1ZCj7X +``` + +The fork workflow runs this command once, without a shard matrix. Endform schedules +all 335 Chromium tests. `E2E_APP_POOL_SIZE` controls both application capacity and +the Endform concurrency limit (default 4, supported range 1–16). + +Tests import `test` and `expect` from `tests/__utils__/test`. Its automatic fixture +leases one kitchen-sink instance for the whole test attempt, including hooks. The +lease header is attached to browser contexts, API request contexts and Node fetch +calls to the test origin. Existing test URLs and assertions stay unchanged. + +`isolation-server.mjs` runs `mastra dev` once to generate the development server +and Studio, then starts isolated copies of that output behind one gateway. Each +instance has its own database, source tree and process. After every attempt, the +instance is stopped, its database and source tree are reset, and it is restarted +before another test can lease it. This also isolates tests that save agent +configuration to disk. Dependency builds and browser installation are not +repeated per application instance; Endform provides the browsers. + +The gateway refuses application requests without an active lease. Do not bypass +it by calling a backend port, or remove a reset to increase concurrency. A hung +or failing instance is not returned to the pool. The server is a local test +harness, not a production application service. + +Playwright explicitly uses the playground `tsconfig.json` so imported fixtures +retain their `@/*` mappings on remote runners. ARIA snapshot files are transferred +explicitly because tests read them at runtime. + +Verify the lease and reset behavior before browser experiments: + +```sh +node --test isolation-server.test.mjs +``` + +For targeted experiments, use Endform's normal Playwright file or grep filters; +do not replace full-suite verification with a smaller passing subset. Changing +pool size changes application capacity, so report it with benchmark results. diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts index 3ccedc268b8a..abc34584ce5f 100644 --- a/packages/playground/e2e/endform.config.ts +++ b/packages/playground/e2e/endform.config.ts @@ -2,8 +2,7 @@ import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ // ARIA snapshots are read from disk rather than imported by the specs. - additionalFiles: ['tests/**/*.aria.yml'], - // Tests clear the whole kitchen-sink database, so each application must serve - // one test at a time. CI retains four disjoint shards with isolated servers. - concurrentTestLimits: [{ scope: 'within-suite-run', limit: 1 }], + additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json'], + // Each test leases an isolated application; Endform schedules the whole suite. + concurrentTestLimits: [{ scope: 'within-suite-run', limit: Number(process.env.E2E_APP_POOL_SIZE || 4) }], }); diff --git a/packages/playground/e2e/isolation-server.mjs b/packages/playground/e2e/isolation-server.mjs new file mode 100644 index 000000000000..e5e96053be4d --- /dev/null +++ b/packages/playground/e2e/isolation-server.mjs @@ -0,0 +1,217 @@ +import { spawn } from 'node:child_process'; +import { cp, mkdir, mkdtemp, rm, symlink } from 'node:fs/promises'; +import { createServer, request } from 'node:http'; +import { connect } from 'node:net'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const SESSION_HEADER = 'x-mastra-e2e-session'; + +// A lease covers one test attempt, including its hooks and browser/API requests. +// Each backend owns its database, source tree and in-memory Mastra registries. +export function createGateway(backends) { + const sessions = new Map(); + const available = [...backends]; + const waiting = []; + const server = createServer(async (req, res) => { + const send = (status, data) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(data)); + }; + if (req.url === '/__e2e/health') return send(200, { instances: backends.length }); + if (req.url === '/__e2e/lease' && req.method === 'POST') { + const session = req.headers[SESSION_HEADER]; + if (typeof session !== 'string' || !/^[a-zA-Z0-9-]{1,100}$/.test(session)) { + return send(400, { error: 'A valid session header is required' }); + } + if (sessions.has(session)) return send(409, { error: 'Session already leased' }); + const allocate = backend => { + sessions.set(session, backend); + send(201, { session }); + }; + const backend = available.shift(); + if (backend) allocate(backend); + else { + const pending = { allocate }; + waiting.push(pending); + res.on('close', () => { + const index = waiting.indexOf(pending); + if (index >= 0) waiting.splice(index, 1); + }); + } + return; + } + if (req.url === '/__e2e/lease' && req.method === 'DELETE') { + const session = req.headers[SESSION_HEADER]; + const backend = sessions.get(session); + if (!backend) return send(404, { error: 'Unknown session' }); + sessions.delete(session); + // Reset after every attempt, even when a test has no afterEach reset or fails. + // Filesystem changes require a fresh process/source tree as well as a database reset. + try { + await backend.reset(); + const pending = waiting.shift(); + if (pending) pending.allocate(backend); + else available.push(backend); + send(200, { released: true }); + } catch (error) { + send(500, { error: `Could not reset isolated instance: ${error.message}` }); + } + return; + } + const backend = sessions.get(req.headers[SESSION_HEADER]); + if (!backend) return send(409, { error: 'Application requests require an active test lease' }); + const upstream = request( + { + hostname: '127.0.0.1', + port: backend.port, + path: req.url, + method: req.method, + headers: { ...req.headers, host: `localhost:${backend.port}` }, + }, + response => { + res.writeHead(response.statusCode, response.headers); + response.pipe(res); + }, + ); + upstream.on('error', error => { + if (!res.headersSent) send(502, { error: error.message }); + else res.destroy(error); + }); + res.on('close', () => upstream.destroy()); + req.pipe(upstream); + }); + server.on('upgrade', (req, socket, head) => { + const backend = sessions.get(req.headers[SESSION_HEADER]); + if (!backend) return socket.destroy(); + const upstream = connect(backend.port, '127.0.0.1', () => { + upstream.write(`${req.method} ${req.url} HTTP/${req.httpVersion}\r\n`); + for (const [name, value] of Object.entries(req.headers)) { + upstream.write(`${name}: ${value}\r\n`); + } + upstream.write('\r\n'); + upstream.write(head); + socket.pipe(upstream).pipe(socket); + }); + upstream.on('error', () => socket.destroy()); + socket.on('error', () => upstream.destroy()); + socket.on('close', () => upstream.destroy()); + }); + return server; +} + +async function waitForServer(port, child) { + for (let attempt = 0; attempt < 600; attempt++) { + if (child.exitCode !== null) throw new Error(`Kitchen-sink exited with ${child.exitCode}`); + try { + const response = await fetch(`http://127.0.0.1:${port}/health`); + if (response.ok) return; + } catch {} + await new Promise(resolve => setTimeout(resolve, 100)); + } + throw new Error(`Kitchen-sink on port ${port} did not become ready`); +} + +async function stop(child) { + if (!child || child.exitCode !== null) return; + await new Promise(resolve => { + const timeout = setTimeout(() => child.kill('SIGKILL'), 5000); + child.once('exit', () => { + clearTimeout(timeout); + resolve(); + }); + child.kill('SIGTERM'); + }); +} + +export async function startIsolatedApps({ size, port, kitchenSink }) { + const root = await mkdtemp(join(tmpdir(), 'mastra-endform-')); + const backends = []; + let templateProcess; + try { + // Use the supported dev command once to generate the development server and Studio. + templateProcess = spawn('pnpm', ['dev'], { + cwd: kitchenSink, + detached: true, + env: { ...process.env, PORT: String(port + 100), E2E_GATEWAY_PORT: String(port) }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + await waitForServer(port + 100, templateProcess); + // pnpm owns a CLI and server process tree; terminate only this process group. + process.kill(-templateProcess.pid, 'SIGTERM'); + await stop(templateProcess); + templateProcess = undefined; + for (let index = 0; index < size; index++) { + const directory = join(root, String(index)); + const output = join(directory, '.mastra/output'); + await mkdir(directory, { recursive: true }); + await cp(join(kitchenSink, '.mastra/output'), output, { recursive: true }); + await symlink(join(kitchenSink, 'node_modules'), join(directory, 'node_modules')); + await cp(join(kitchenSink, 'package.json'), join(directory, 'package.json')); + await mkdir(join(output, 'public'), { recursive: true }); + const backend = { port: port + index + 1, child: undefined }; + const start = async () => { + backend.child = spawn(process.execPath, [join(output, 'index.mjs')], { + cwd: join(output, 'public'), + env: { + ...process.env, + PORT: String(backend.port), + E2E_GATEWAY_PORT: String(port), + MASTRA_DEV: 'true', + MASTRA_PROJECT_ROOT: join(directory, '.mastra'), + }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + await waitForServer(backend.port, backend.child); + }; + backend.reset = async () => { + await stop(backend.child); + await rm(join(directory, 'src'), { recursive: true, force: true }); + await cp(join(kitchenSink, 'src'), join(directory, 'src'), { recursive: true }); + await rm(join(output, 'public'), { recursive: true, force: true }); + await mkdir(join(output, 'public'), { recursive: true }); + await start(); + }; + backends.push(backend); + await backend.reset(); + } + return { + backends, + close: async () => { + await Promise.all(backends.map(backend => stop(backend.child))); + await rm(root, { recursive: true, force: true }); + }, + }; + } catch (error) { + if (templateProcess) { + try { + process.kill(-templateProcess.pid, 'SIGTERM'); + } catch {} + } + await Promise.all(backends.map(backend => stop(backend.child))); + await rm(root, { recursive: true, force: true }); + throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const port = Number(process.env.E2E_PORT || 4111); + const size = Number(process.env.E2E_APP_POOL_SIZE || 4); + if (!Number.isInteger(size) || size < 1 || size > 16) throw new Error('E2E_APP_POOL_SIZE must be 1–16'); + const kitchenSink = join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'); + const pool = await startIsolatedApps({ size, port, kitchenSink }); + const gateway = createGateway(pool.backends); + gateway.listen(port, '0.0.0.0', () => console.log(`Isolated kitchen-sink ready: ${size} instances on :${port}`)); + let stopping = false; + const shutdown = async () => { + if (stopping) return; + stopping = true; + gateway.closeAllConnections(); + gateway.close(); + await pool.close(); + process.exit(0); + }; + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); +} diff --git a/packages/playground/e2e/isolation-server.test.mjs b/packages/playground/e2e/isolation-server.test.mjs new file mode 100644 index 000000000000..27ee0e60db42 --- /dev/null +++ b/packages/playground/e2e/isolation-server.test.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { test } from 'node:test'; +import { createGateway, SESSION_HEADER } from './isolation-server.mjs'; + +async function listen(server) { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return server.address().port; +} + +test('concurrent leases isolate writes and resets, and queued tests reuse a clean instance', async t => { + const backends = []; + for (let index = 0; index < 2; index++) { + let value = ''; + const app = createServer((req, res) => { + if (req.method === 'POST') value = req.headers['x-value']; + res.end(JSON.stringify({ instance: index, value })); + }); + backends.push({ + port: await listen(app), + reset: async () => { + value = ''; + }, + }); + t.after(() => { + app.closeAllConnections(); + app.close(); + }); + } + const gateway = createGateway(backends); + const url = `http://127.0.0.1:${await listen(gateway)}`; + t.after(() => { + gateway.closeAllConnections(); + gateway.close(); + }); + const headers = session => ({ [SESSION_HEADER]: session }); + const lease = session => fetch(`${url}/__e2e/lease`, { method: 'POST', headers: headers(session) }); + const release = session => fetch(`${url}/__e2e/lease`, { method: 'DELETE', headers: headers(session) }); + const read = session => fetch(url, { headers: headers(session) }).then(response => response.json()); + assert.equal((await fetch(url)).status, 409); + assert.equal((await lease('a')).status, 201); + assert.equal((await lease('b')).status, 201); + await Promise.all( + ['a', 'b'].map(session => fetch(url, { method: 'POST', headers: { ...headers(session), 'x-value': session } })), + ); + const [a, b] = await Promise.all([read('a'), read('b')]); + assert.notEqual(a.instance, b.instance); + assert.equal(a.value, 'a'); + assert.equal(b.value, 'b'); + assert.equal((await lease('a')).status, 409); + const queued = lease('c'); + assert.equal((await release('a')).status, 200); + assert.equal((await queued).status, 201); + assert.deepEqual(await read('c'), { instance: a.instance, value: '' }); + assert.deepEqual(await read('b'), b); + assert.equal((await fetch(url, { headers: headers('a') })).status, 409); + assert.equal((await release('c')).status, 200); + assert.equal((await release('b')).status, 200); +}); diff --git a/packages/playground/e2e/kitchen-sink/src/mastra/index.ts b/packages/playground/e2e/kitchen-sink/src/mastra/index.ts index 26505603e741..bc79a8c79647 100644 --- a/packages/playground/e2e/kitchen-sink/src/mastra/index.ts +++ b/packages/playground/e2e/kitchen-sink/src/mastra/index.ts @@ -62,6 +62,7 @@ export const mastra = new Mastra({ contentFilterProcessor, }, server: { + ...(process.env.E2E_GATEWAY_PORT ? { studioPort: Number(process.env.E2E_GATEWAY_PORT) } : {}), ...(process.env.E2E_STUDIO_BASE_PATH ? { studioBase: process.env.E2E_STUDIO_BASE_PATH } : {}), apiRoutes: [ // Seeds a weather-agent thread with `count` user messages ("seed message N", diff --git a/packages/playground/e2e/playwright.config.ts b/packages/playground/e2e/playwright.config.ts index ad8e9aba7846..a56ab193a825 100644 --- a/packages/playground/e2e/playwright.config.ts +++ b/packages/playground/e2e/playwright.config.ts @@ -7,22 +7,14 @@ const webservers: PlaywrightTestConfig['webServer'] = [ { // UI tests use route interception for auth mocking - no server auth needed // Server-side permission tests are in server-adapters/hono - command: `pnpm -C ./kitchen-sink dev`, - url: `http://localhost:4111`, + command: 'node ./isolation-server.mjs', + url: `${BASE_URL}/__e2e/health`, timeout: 120_000, }, ]; -if (PORT) { - webservers.push({ - command: `echo "App is running on :${PORT}"`, - url: `http://localhost:${PORT}`, - timeout: 120_000, - reuseExistingServer: true, - }); -} - export default defineConfig({ + tsconfig: '../tsconfig.json', testDir: './tests', fullyParallel: true, forbidOnly: !!process.env.CI, diff --git a/packages/playground/e2e/tests/__utils__/test.ts b/packages/playground/e2e/tests/__utils__/test.ts new file mode 100644 index 000000000000..32b452271507 --- /dev/null +++ b/packages/playground/e2e/tests/__utils__/test.ts @@ -0,0 +1,43 @@ +import { randomUUID } from 'node:crypto'; +import { test as base } from '@playwright/test'; + +const BASE_URL = `http://localhost:${process.env.E2E_PORT || '4111'}`; +const SESSION_HEADER = 'x-mastra-e2e-session'; +const SESSION_ANNOTATION = 'mastra-e2e-session'; + +// Node fetch helpers and browser/APIRequestContext traffic must use the same lease. +// Resolve the annotation per call rather than storing mutable process-wide test state. +const originalFetch = globalThis.fetch; +globalThis.fetch = (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin !== BASE_URL || url.pathname.startsWith('/__e2e/')) return originalFetch(input, init); + const session = base.info().annotations.find(annotation => annotation.type === SESSION_ANNOTATION)?.description; + if (!session) throw new Error('Kitchen-sink request made without an isolated test session'); + const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); + headers.set(SESSION_HEADER, session); + return originalFetch(input, { ...init, headers }); +}; + +export const test = base.extend<{ isolatedSession: string }>({ + isolatedSession: [ + async ({}, use, testInfo) => { + const session = randomUUID(); + const headers = { [SESSION_HEADER]: session }; + const response = await originalFetch(`${BASE_URL}/__e2e/lease`, { method: 'POST', headers }); + if (!response.ok) throw new Error(`Could not lease kitchen-sink: ${response.status}`); + testInfo.annotations.push({ type: SESSION_ANNOTATION, description: session }); + try { + await use(session); + } finally { + const release = await originalFetch(`${BASE_URL}/__e2e/lease`, { method: 'DELETE', headers }); + if (!release.ok) throw new Error(`Could not reset kitchen-sink: ${release.status}`); + } + }, + { auto: true }, + ], + extraHTTPHeaders: async ({ isolatedSession, extraHTTPHeaders }, use) => { + await use({ ...extraHTTPHeaders, [SESSION_HEADER]: isolatedSession }); + }, +}); + +export { expect } from '@playwright/test'; diff --git a/packages/playground/e2e/tests/agent-builder/deterministic-builder.spec.ts b/packages/playground/e2e/tests/agent-builder/deterministic-builder.spec.ts index 3859e0d61a18..2279d919e071 100644 --- a/packages/playground/e2e/tests/agent-builder/deterministic-builder.spec.ts +++ b/packages/playground/e2e/tests/agent-builder/deterministic-builder.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test } from '../__utils__/test'; import type { Page } from '@playwright/test'; import { resetStorage } from '../__utils__/reset-storage'; import { selectFixture } from '../__utils__/select-fixture'; diff --git a/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts b/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts index 1809af867e7c..51a967714bae 100644 --- a/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import type { Page } from '@playwright/test'; import { resetStorage } from '../../__utils__/reset-storage'; diff --git a/packages/playground/e2e/tests/agents/$agentId/composer-ring.spec.ts b/packages/playground/e2e/tests/agents/$agentId/composer-ring.spec.ts index c4710025eb23..826a0bbca084 100644 --- a/packages/playground/e2e/tests/agents/$agentId/composer-ring.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/composer-ring.spec.ts @@ -1,5 +1,5 @@ import type { Locator } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; /** diff --git a/packages/playground/e2e/tests/agents/$agentId/history-pagination.spec.ts b/packages/playground/e2e/tests/agents/$agentId/history-pagination.spec.ts index f39c55cf9b4a..f5630dc88786 100644 --- a/packages/playground/e2e/tests/agents/$agentId/history-pagination.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/history-pagination.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { expect, test } from '@playwright/test'; +import { expect, test } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { seedThread } from '../../__utils__/seed-thread'; diff --git a/packages/playground/e2e/tests/agents/$agentId/ime-composition.spec.ts b/packages/playground/e2e/tests/agents/$agentId/ime-composition.spec.ts index 3350e941b4ee..ed9013ef1af6 100644 --- a/packages/playground/e2e/tests/agents/$agentId/ime-composition.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/ime-composition.spec.ts @@ -1,5 +1,5 @@ import type { Page, BrowserContext } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { selectFixture } from '../../__utils__/select-fixture'; diff --git a/packages/playground/e2e/tests/agents/$agentId/page.spec.ts b/packages/playground/e2e/tests/agents/$agentId/page.spec.ts index 8ec17e0fde26..da8dce0bbf94 100644 --- a/packages/playground/e2e/tests/agents/$agentId/page.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; test.describe('Agent detail page', () => { diff --git a/packages/playground/e2e/tests/agents/$agentId/session.spec.ts b/packages/playground/e2e/tests/agents/$agentId/session.spec.ts index 0a95b4ec809d..459774f7969c 100644 --- a/packages/playground/e2e/tests/agents/$agentId/session.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/session.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; /** diff --git a/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts b/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts index 292b6faacc1b..3ad7ffe07d12 100644 --- a/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts @@ -1,5 +1,5 @@ import type { Page, BrowserContext } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { selectFixture } from '../../__utils__/select-fixture'; diff --git a/packages/playground/e2e/tests/agents/$agentId/tools/$toolId/page.spec.ts b/packages/playground/e2e/tests/agents/$agentId/tools/$toolId/page.spec.ts index 53bdca6a0b14..1c5d92ae1a69 100644 --- a/packages/playground/e2e/tests/agents/$agentId/tools/$toolId/page.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/tools/$toolId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../../../__utils__/test'; import { resetStorage } from '../../../../__utils__/reset-storage'; test.afterEach(async () => { diff --git a/packages/playground/e2e/tests/agents/observability-tabs.spec.ts b/packages/playground/e2e/tests/agents/observability-tabs.spec.ts index 80ceae76be48..300a53882c38 100644 --- a/packages/playground/e2e/tests/agents/observability-tabs.spec.ts +++ b/packages/playground/e2e/tests/agents/observability-tabs.spec.ts @@ -1,6 +1,6 @@ import type { MastraClient } from '@mastra/client-js'; import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { mockTraceQueryCapabilities } from '../__utils__/mock-trace-query-capabilities'; import { resetStorage } from '../__utils__/reset-storage'; diff --git a/packages/playground/e2e/tests/agents/observational-memory.spec.ts b/packages/playground/e2e/tests/agents/observational-memory.spec.ts index a31e41144362..2bba4db47cbd 100644 --- a/packages/playground/e2e/tests/agents/observational-memory.spec.ts +++ b/packages/playground/e2e/tests/agents/observational-memory.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import type { Page } from '@playwright/test'; import { resetStorage } from '../__utils__/reset-storage'; import { selectFixture } from '../__utils__/select-fixture'; diff --git a/packages/playground/e2e/tests/agents/page.spec.ts b/packages/playground/e2e/tests/agents/page.spec.ts index 9259558a23ec..f45616fa04de 100644 --- a/packages/playground/e2e/tests/agents/page.spec.ts +++ b/packages/playground/e2e/tests/agents/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/agents/recovery.spec.ts b/packages/playground/e2e/tests/agents/recovery.spec.ts index b4a96ae15cd6..f29757615afe 100644 --- a/packages/playground/e2e/tests/agents/recovery.spec.ts +++ b/packages/playground/e2e/tests/agents/recovery.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test } from '../__utils__/test'; // A missing agent must not strand users in a dead chat. Recovery either reloads // the resource from the server or returns to the registered agents list. diff --git a/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts b/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts index 19bf46ee4c67..88ff63ddcff3 100644 --- a/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts +++ b/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test } from '../__utils__/test'; const chatPath = '/agents/weather-agent/threads/new'; const agentDetails = /\/api\/agents\/weather-agent(?:\?.*)?$/; diff --git a/packages/playground/e2e/tests/auth/admin-role.spec.ts b/packages/playground/e2e/tests/auth/admin-role.spec.ts index 8d425dd035f9..983a9d71b151 100644 --- a/packages/playground/e2e/tests/auth/admin-role.spec.ts +++ b/packages/playground/e2e/tests/auth/admin-role.spec.ts @@ -11,7 +11,7 @@ * - Settings and configuration */ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { setupAdminAuth, setupMockAuth, MOCK_USERS } from '../__utils__/auth'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/auth/infrastructure.spec.ts b/packages/playground/e2e/tests/auth/infrastructure.spec.ts index 6f0e165e9fd3..eb0826fc4ea4 100644 --- a/packages/playground/e2e/tests/auth/infrastructure.spec.ts +++ b/packages/playground/e2e/tests/auth/infrastructure.spec.ts @@ -10,7 +10,7 @@ * - Auth fixtures provide correct permission data */ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { setupMockAuth, setupAdminAuth, diff --git a/packages/playground/e2e/tests/auth/login-flow.spec.ts b/packages/playground/e2e/tests/auth/login-flow.spec.ts index 3bdd05134ce1..2768c2ed884e 100644 --- a/packages/playground/e2e/tests/auth/login-flow.spec.ts +++ b/packages/playground/e2e/tests/auth/login-flow.spec.ts @@ -11,7 +11,7 @@ * - Login state reflected in UI (user avatar, name display) */ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { setupMockAuth, setupUnauthenticated, setupAdminAuth, clearMockAuth } from '../__utils__/auth'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/auth/member-role.spec.ts b/packages/playground/e2e/tests/auth/member-role.spec.ts index 5b45955094eb..fe2ffae6777c 100644 --- a/packages/playground/e2e/tests/auth/member-role.spec.ts +++ b/packages/playground/e2e/tests/auth/member-role.spec.ts @@ -11,7 +11,7 @@ * - Cannot access admin settings */ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { setupMemberAuth, setupMockAuth } from '../__utils__/auth'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/auth/viewer-role.spec.ts b/packages/playground/e2e/tests/auth/viewer-role.spec.ts index ef8bf0eb4cc7..69a33e2ee053 100644 --- a/packages/playground/e2e/tests/auth/viewer-role.spec.ts +++ b/packages/playground/e2e/tests/auth/viewer-role.spec.ts @@ -14,7 +14,7 @@ * - Action buttons are hidden or disabled */ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { setupViewerAuth, setupMockAuth } from '../__utils__/auth'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts b/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts index 3d3f998490fa..c9dcd3379adb 100644 --- a/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts +++ b/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; // These tests cover the code-mode override product behavior: diff --git a/packages/playground/e2e/tests/cms/agents/create/page.spec.ts b/packages/playground/e2e/tests/cms/agents/create/page.spec.ts index a8a485a5195c..4b31d50fb321 100644 --- a/packages/playground/e2e/tests/cms/agents/create/page.spec.ts +++ b/packages/playground/e2e/tests/cms/agents/create/page.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../../__utils__/test'; import { resetStorage } from '../../../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../../../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/cms/scorers/create/page.spec.ts b/packages/playground/e2e/tests/cms/scorers/create/page.spec.ts index 60a292c08cc3..e8b1be95060b 100644 --- a/packages/playground/e2e/tests/cms/scorers/create/page.spec.ts +++ b/packages/playground/e2e/tests/cms/scorers/create/page.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../../__utils__/test'; import { resetStorage } from '../../../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../../../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/cms/scorers/edit/page.spec.ts b/packages/playground/e2e/tests/cms/scorers/edit/page.spec.ts index 847d78625731..d3f73b759814 100644 --- a/packages/playground/e2e/tests/cms/scorers/edit/page.spec.ts +++ b/packages/playground/e2e/tests/cms/scorers/edit/page.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../../__utils__/test'; import { resetStorage } from '../../../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../../../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/datasets/compare-experiments.spec.ts b/packages/playground/e2e/tests/datasets/compare-experiments.spec.ts index 2d32999fc43c..cc707899ff60 100644 --- a/packages/playground/e2e/tests/datasets/compare-experiments.spec.ts +++ b/packages/playground/e2e/tests/datasets/compare-experiments.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__'; const PORT = process.env.E2E_PORT || '4111'; diff --git a/packages/playground/e2e/tests/datasets/dataset-items-list.spec.ts b/packages/playground/e2e/tests/datasets/dataset-items-list.spec.ts index 0ec021748f81..684b4ed222c4 100644 --- a/packages/playground/e2e/tests/datasets/dataset-items-list.spec.ts +++ b/packages/playground/e2e/tests/datasets/dataset-items-list.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage, seedDatasetWithItems } from '../__utils__'; const PORT = process.env.E2E_PORT || '4111'; diff --git a/packages/playground/e2e/tests/datasets/item-review-panels-layout.spec.ts b/packages/playground/e2e/tests/datasets/item-review-panels-layout.spec.ts index f7be202f08ef..3c4b96baf2b2 100644 --- a/packages/playground/e2e/tests/datasets/item-review-panels-layout.spec.ts +++ b/packages/playground/e2e/tests/datasets/item-review-panels-layout.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test } from '../__utils__/test'; import type { Locator, Page } from '@playwright/test'; import { longPanelItems, longPanelResults, mockPanelRequests } from './__tests__/fixtures/item-review-panels'; diff --git a/packages/playground/e2e/tests/datasets/pagination.spec.ts b/packages/playground/e2e/tests/datasets/pagination.spec.ts index 1818b21d9d3d..c731f5684e7d 100644 --- a/packages/playground/e2e/tests/datasets/pagination.spec.ts +++ b/packages/playground/e2e/tests/datasets/pagination.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage, seedDatasets } from '../__utils__'; test.afterEach(async () => { diff --git a/packages/playground/e2e/tests/mcps/$serverId/page.spec.ts b/packages/playground/e2e/tests/mcps/$serverId/page.spec.ts index cb7c1e073838..82b6e34f5866 100644 --- a/packages/playground/e2e/tests/mcps/$serverId/page.spec.ts +++ b/packages/playground/e2e/tests/mcps/$serverId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { expectBreadcrumbLink } from '../../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/mcps/$serverId/tools/$toolId/page.spec.ts b/packages/playground/e2e/tests/mcps/$serverId/tools/$toolId/page.spec.ts index 9430219ebaee..e23b08c5de8d 100644 --- a/packages/playground/e2e/tests/mcps/$serverId/tools/$toolId/page.spec.ts +++ b/packages/playground/e2e/tests/mcps/$serverId/tools/$toolId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../../../__utils__/test'; import { resetStorage } from '../../../../__utils__/reset-storage'; test.describe('MCP server tool detail page', () => { diff --git a/packages/playground/e2e/tests/mcps/page.spec.ts b/packages/playground/e2e/tests/mcps/page.spec.ts index 6497fb1df1a6..676c04cc53e4 100644 --- a/packages/playground/e2e/tests/mcps/page.spec.ts +++ b/packages/playground/e2e/tests/mcps/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/metrics/drilldown.spec.ts b/packages/playground/e2e/tests/metrics/drilldown.spec.ts index 8f0f62d1f66e..cc4ec357e7aa 100644 --- a/packages/playground/e2e/tests/metrics/drilldown.spec.ts +++ b/packages/playground/e2e/tests/metrics/drilldown.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import type { Locator, Page } from '@playwright/test'; import { resetStorage } from '../__utils__/reset-storage'; diff --git a/packages/playground/e2e/tests/metrics/filter-persistence.spec.ts b/packages/playground/e2e/tests/metrics/filter-persistence.spec.ts index 32c870a7e92b..cf3fa466ee2e 100644 --- a/packages/playground/e2e/tests/metrics/filter-persistence.spec.ts +++ b/packages/playground/e2e/tests/metrics/filter-persistence.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; const METRICS_FILTERS_STORAGE_KEY = 'mastra:metrics:saved-filters'; diff --git a/packages/playground/e2e/tests/metrics/page.spec.ts b/packages/playground/e2e/tests/metrics/page.spec.ts index f24c04991c77..ac2fc342266f 100644 --- a/packages/playground/e2e/tests/metrics/page.spec.ts +++ b/packages/playground/e2e/tests/metrics/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/processors/$processorId/page.spec.ts b/packages/playground/e2e/tests/processors/$processorId/page.spec.ts index 8647420bceaf..69b50c72a828 100644 --- a/packages/playground/e2e/tests/processors/$processorId/page.spec.ts +++ b/packages/playground/e2e/tests/processors/$processorId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { expectBreadcrumbLink } from '../../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/processors/page.spec.ts b/packages/playground/e2e/tests/processors/page.spec.ts index 373c356bb9d9..f144d22fbf16 100644 --- a/packages/playground/e2e/tests/processors/page.spec.ts +++ b/packages/playground/e2e/tests/processors/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/root.spec.ts b/packages/playground/e2e/tests/root.spec.ts index 5b57cd2a3397..c53f9bdaa58d 100644 --- a/packages/playground/e2e/tests/root.spec.ts +++ b/packages/playground/e2e/tests/root.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from './__utils__/test'; import { resetStorage } from './__utils__/reset-storage'; test.describe('Root path', () => { diff --git a/packages/playground/e2e/tests/scorers/$scorerId/page.spec.ts b/packages/playground/e2e/tests/scorers/$scorerId/page.spec.ts index e562b4b09647..e53e8cef1bdd 100644 --- a/packages/playground/e2e/tests/scorers/$scorerId/page.spec.ts +++ b/packages/playground/e2e/tests/scorers/$scorerId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { expectBreadcrumbLink, expectCurrentBreadcrumb } from '../../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/scorers/page.spec.ts b/packages/playground/e2e/tests/scorers/page.spec.ts index bd366bc92e0b..1538a092affd 100644 --- a/packages/playground/e2e/tests/scorers/page.spec.ts +++ b/packages/playground/e2e/tests/scorers/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; test.describe('Scorers list page', () => { diff --git a/packages/playground/e2e/tests/settings/api-prefix.spec.ts b/packages/playground/e2e/tests/settings/api-prefix.spec.ts index 1e161c2dd47d..4e8d795139de 100644 --- a/packages/playground/e2e/tests/settings/api-prefix.spec.ts +++ b/packages/playground/e2e/tests/settings/api-prefix.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; /** diff --git a/packages/playground/e2e/tests/settings/page.spec.ts b/packages/playground/e2e/tests/settings/page.spec.ts index 4df4e6d3a0f7..ca35e272f4b9 100644 --- a/packages/playground/e2e/tests/settings/page.spec.ts +++ b/packages/playground/e2e/tests/settings/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; test.describe('Settings page', () => { diff --git a/packages/playground/e2e/tests/sidebar/cold-load-layout.spec.ts b/packages/playground/e2e/tests/sidebar/cold-load-layout.spec.ts index d11c181e2c09..81900540d6aa 100644 --- a/packages/playground/e2e/tests/sidebar/cold-load-layout.spec.ts +++ b/packages/playground/e2e/tests/sidebar/cold-load-layout.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { buildAuthCapabilities, buildCurrentUserResponse } from '../__utils__/auth'; import type { MockAuthConfig } from '../__utils__/auth'; import { resetStorage } from '../__utils__/reset-storage'; diff --git a/packages/playground/e2e/tests/sidebar/navigation-scroll.spec.ts b/packages/playground/e2e/tests/sidebar/navigation-scroll.spec.ts index 918d59356b7e..bb53cf9773e6 100644 --- a/packages/playground/e2e/tests/sidebar/navigation-scroll.spec.ts +++ b/packages/playground/e2e/tests/sidebar/navigation-scroll.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; /** diff --git a/packages/playground/e2e/tests/templates/$templateSlug/page.spec.ts b/packages/playground/e2e/tests/templates/$templateSlug/page.spec.ts index fd54ce196f80..1936f853850a 100644 --- a/packages/playground/e2e/tests/templates/$templateSlug/page.spec.ts +++ b/packages/playground/e2e/tests/templates/$templateSlug/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; test.describe('Template detail page', () => { diff --git a/packages/playground/e2e/tests/templates/page.spec.ts b/packages/playground/e2e/tests/templates/page.spec.ts index 5231927612f4..c1278ad5ee52 100644 --- a/packages/playground/e2e/tests/templates/page.spec.ts +++ b/packages/playground/e2e/tests/templates/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/tools/$toolId/page.spec.ts b/packages/playground/e2e/tests/tools/$toolId/page.spec.ts index 6d7bb708807f..bb46671e68f7 100644 --- a/packages/playground/e2e/tests/tools/$toolId/page.spec.ts +++ b/packages/playground/e2e/tests/tools/$toolId/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; test.describe('Tool detail page', () => { diff --git a/packages/playground/e2e/tests/tools/page.spec.ts b/packages/playground/e2e/tests/tools/page.spec.ts index 8afe2e8727a1..7f1524317ecb 100644 --- a/packages/playground/e2e/tests/tools/page.spec.ts +++ b/packages/playground/e2e/tests/tools/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; test.describe('Tools list page', () => { diff --git a/packages/playground/e2e/tests/traces/page.spec.ts b/packages/playground/e2e/tests/traces/page.spec.ts index 4547b6ad42ef..05280005336a 100644 --- a/packages/playground/e2e/tests/traces/page.spec.ts +++ b/packages/playground/e2e/tests/traces/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; import { expectCurrentBreadcrumb } from '../__utils__/route-header'; diff --git a/packages/playground/e2e/tests/traces/partial-thread.spec.ts b/packages/playground/e2e/tests/traces/partial-thread.spec.ts index 12fde14d46d9..3729a0fc9e0d 100644 --- a/packages/playground/e2e/tests/traces/partial-thread.spec.ts +++ b/packages/playground/e2e/tests/traces/partial-thread.spec.ts @@ -1,6 +1,6 @@ import type { MastraClient } from '@mastra/client-js'; import type { Page } from '@playwright/test'; -import { expect, test } from '@playwright/test'; +import { expect, test } from '../__utils__/test'; import { mockTraceQueryCapabilities } from '../__utils__/mock-trace-query-capabilities'; import { resetStorage } from '../__utils__/reset-storage'; diff --git a/packages/playground/e2e/tests/traces/query-filters.spec.ts b/packages/playground/e2e/tests/traces/query-filters.spec.ts index 16c00cb0f9d9..5f6ed432ed4e 100644 --- a/packages/playground/e2e/tests/traces/query-filters.spec.ts +++ b/packages/playground/e2e/tests/traces/query-filters.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from '@playwright/test'; +import { expect, test } from '../__utils__/test'; import { traceQueryPage } from '../../../src/pages/traces/__tests__/fixtures/trace-query'; import { mockTraceQueryCapabilities } from '../__utils__/mock-trace-query-capabilities'; diff --git a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-conditional-run-page.spec.ts b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-conditional-run-page.spec.ts index f8b3d42c4ab0..8f698240be03 100644 --- a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-conditional-run-page.spec.ts +++ b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-conditional-run-page.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { topLevelWorkflowNodes } from '../../__utils__/workflow-nodes'; diff --git a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-edges.spec.ts b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-edges.spec.ts index 71c38d29c8be..c8d008644abf 100644 --- a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-edges.spec.ts +++ b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-edges.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { expectExactEdgeStatuses } from '../../__utils__/workflow-edges'; import type { EdgeExpectation } from '../../__utils__/workflow-edges'; diff --git a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-run-page.spec.ts b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-run-page.spec.ts index 37ed3353ff1c..c3d15525b121 100644 --- a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-run-page.spec.ts +++ b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step-run-page.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { topLevelWorkflowNodes } from '../../__utils__/workflow-nodes'; diff --git a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step.spec.ts b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step.spec.ts index d650a193464f..c8775722541e 100644 --- a/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step.spec.ts +++ b/packages/playground/e2e/tests/workflows/$workflowId/debug-step-by-step.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { topLevelWorkflowNodes } from '../../__utils__/workflow-nodes'; diff --git a/packages/playground/e2e/tests/workflows/$workflowId/nested-graph.spec.ts b/packages/playground/e2e/tests/workflows/$workflowId/nested-graph.spec.ts index 225797be3612..036519da24f7 100644 --- a/packages/playground/e2e/tests/workflows/$workflowId/nested-graph.spec.ts +++ b/packages/playground/e2e/tests/workflows/$workflowId/nested-graph.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { topLevelWorkflowNodes } from '../../__utils__/workflow-nodes'; diff --git a/packages/playground/e2e/tests/workflows/$workflowId/page.spec.ts b/packages/playground/e2e/tests/workflows/$workflowId/page.spec.ts index 82a191fb5a2d..efc868e7426f 100644 --- a/packages/playground/e2e/tests/workflows/$workflowId/page.spec.ts +++ b/packages/playground/e2e/tests/workflows/$workflowId/page.spec.ts @@ -1,5 +1,5 @@ import type { Page } from '@playwright/test'; -import { test, expect } from '@playwright/test'; +import { test, expect } from '../../__utils__/test'; import { resetStorage } from '../../__utils__/reset-storage'; import { topLevelWorkflowNodes } from '../../__utils__/workflow-nodes'; diff --git a/packages/playground/e2e/tests/workflows/page.spec.ts b/packages/playground/e2e/tests/workflows/page.spec.ts index 790823f22e18..110d0f24b87e 100644 --- a/packages/playground/e2e/tests/workflows/page.spec.ts +++ b/packages/playground/e2e/tests/workflows/page.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; test.afterEach(async () => { diff --git a/packages/playground/e2e/tests/workflows/schedules.spec.ts b/packages/playground/e2e/tests/workflows/schedules.spec.ts index 7436c474d1c3..d2231e5afaae 100644 --- a/packages/playground/e2e/tests/workflows/schedules.spec.ts +++ b/packages/playground/e2e/tests/workflows/schedules.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '../__utils__/test'; import { resetStorage } from '../__utils__/reset-storage'; /** From 32452bcffdf47dd1d91eaf9d16bed673bb979192 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 14:45:50 +0200 Subject: [PATCH 3/8] test(playground): make Endform primary and propagate isolated contexts --- .github/workflows/fork-studio-browser.yml | 4 ++-- packages/playground/e2e/README.md | 5 ++++ packages/playground/e2e/isolated-app.mjs | 19 +++++++++++++++ packages/playground/e2e/isolation-server.mjs | 24 +++++++++++-------- .../playground/e2e/tests/__utils__/test.ts | 11 ++++++++- packages/playground/package.json | 4 +++- 6 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 packages/playground/e2e/isolated-app.mjs diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index cea214799f6c..674d28dad60e 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -55,8 +55,8 @@ jobs: run: node --test packages/playground/e2e/isolation-server.test.mjs - name: Test E2E - working-directory: ./packages/playground/e2e - run: npx endform@latest test --organization-id 2G1ZCj7X + working-directory: ./packages/playground + run: pnpm test:e2e:endform - name: Upload test artifacts if: always() diff --git a/packages/playground/e2e/README.md b/packages/playground/e2e/README.md index bc698c1f4cdc..1a7f8d081c71 100644 --- a/packages/playground/e2e/README.md +++ b/packages/playground/e2e/README.md @@ -7,6 +7,11 @@ and installing `kitchen-sink` dependencies: E2E_APP_POOL_SIZE=8 npx endform@latest test --organization-id 2G1ZCj7X ``` +`pnpm --filter @internal/playground test:e2e` uses Endform as the default runner. +`test:e2e:endform` runs the same command without dependency installation. +The previous native runner remains available as `test:e2e:playwright`, including +its separate Studio base-path test; that extra test is outside this benchmark. + The fork workflow runs this command once, without a shard matrix. Endform schedules all 335 Chromium tests. `E2E_APP_POOL_SIZE` controls both application capacity and the Endform concurrency limit (default 4, supported range 1–16). diff --git a/packages/playground/e2e/isolated-app.mjs b/packages/playground/e2e/isolated-app.mjs new file mode 100644 index 000000000000..f3d058c7d50a --- /dev/null +++ b/packages/playground/e2e/isolated-app.mjs @@ -0,0 +1,19 @@ +import { pathToFileURL } from 'node:url'; + +// Tests can configure MCP clients with the public gateway URL. Calls originating +// inside an isolated application must return to that same application, not rely +// on a browser lease header that the MCP SDK does not know about. +const gatewayPort = process.env.E2E_GATEWAY_PORT; +const port = process.env.PORT; +const originalFetch = globalThis.fetch; +globalThis.fetch = (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if ((url.hostname === 'localhost' || url.hostname === '127.0.0.1') && url.port === gatewayPort) { + url.port = port; + const redirected = input instanceof Request ? new Request(url, input) : url; + return originalFetch(redirected, init); + } + return originalFetch(input, init); +}; + +await import(pathToFileURL(process.argv[2]).href); diff --git a/packages/playground/e2e/isolation-server.mjs b/packages/playground/e2e/isolation-server.mjs index e5e96053be4d..1bb5b2fdfa76 100644 --- a/packages/playground/e2e/isolation-server.mjs +++ b/packages/playground/e2e/isolation-server.mjs @@ -152,17 +152,21 @@ export async function startIsolatedApps({ size, port, kitchenSink }) { await mkdir(join(output, 'public'), { recursive: true }); const backend = { port: port + index + 1, child: undefined }; const start = async () => { - backend.child = spawn(process.execPath, [join(output, 'index.mjs')], { - cwd: join(output, 'public'), - env: { - ...process.env, - PORT: String(backend.port), - E2E_GATEWAY_PORT: String(port), - MASTRA_DEV: 'true', - MASTRA_PROJECT_ROOT: join(directory, '.mastra'), + backend.child = spawn( + process.execPath, + [join(dirname(fileURLToPath(import.meta.url)), 'isolated-app.mjs'), join(output, 'index.mjs')], + { + cwd: join(output, 'public'), + env: { + ...process.env, + PORT: String(backend.port), + E2E_GATEWAY_PORT: String(port), + MASTRA_DEV: 'true', + MASTRA_PROJECT_ROOT: join(directory, '.mastra'), + }, + stdio: ['ignore', 'inherit', 'inherit'], }, - stdio: ['ignore', 'inherit', 'inherit'], - }); + ); await waitForServer(backend.port, backend.child); }; backend.reset = async () => { diff --git a/packages/playground/e2e/tests/__utils__/test.ts b/packages/playground/e2e/tests/__utils__/test.ts index 32b452271507..d1ade819ed2b 100644 --- a/packages/playground/e2e/tests/__utils__/test.ts +++ b/packages/playground/e2e/tests/__utils__/test.ts @@ -20,15 +20,24 @@ globalThis.fetch = (input, init) => { export const test = base.extend<{ isolatedSession: string }>({ isolatedSession: [ - async ({}, use, testInfo) => { + async ({ browser }, use, testInfo) => { const session = randomUUID(); const headers = { [SESSION_HEADER]: session }; const response = await originalFetch(`${BASE_URL}/__e2e/lease`, { method: 'POST', headers }); if (!response.ok) throw new Error(`Could not lease kitchen-sink: ${response.status}`); testInfo.annotations.push({ type: SESSION_ANNOTATION, description: session }); + // Existing streaming/IME specs create contexts explicitly. Browser is a + // worker fixture, so patch only for this test attempt and restore on exit. + const newContext = browser.newContext.bind(browser); + browser.newContext = options => + newContext({ + ...options, + extraHTTPHeaders: { ...options?.extraHTTPHeaders, [SESSION_HEADER]: session }, + }); try { await use(session); } finally { + browser.newContext = newContext; const release = await originalFetch(`${BASE_URL}/__e2e/lease`, { method: 'DELETE', headers }); if (!release.ok) throw new Error(`Could not reset kitchen-sink: ${release.status}`); } diff --git a/packages/playground/package.json b/packages/playground/package.json index 29ac777afe9c..b76b57e84c35 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -37,7 +37,9 @@ "test": "vitest", "test:mutate": "stryker run --mutate", "test:run": "vitest run", - "test:e2e": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts && playwright test -c e2e/playwright.studio-base.config.ts", + "test:e2e": "pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile && pnpm test:e2e:endform", + "test:e2e:endform": "cd e2e && npx endform@latest test --organization-id 2G1ZCj7X", + "test:e2e:playwright": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts && playwright test -c e2e/playwright.studio-base.config.ts", "test:e2e:studio-base": "pnpm test:e2e:setup && playwright test -c e2e/playwright.studio-base.config.ts", "test:e2e:ui": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts --ui", "test:e2e:dev:ui": "pnpm test:e2e:setup && E2E_PORT=5173 playwright test -c e2e/playwright.config.ts --ui", From 5ce58fb0943d842af93e40d5de91c280511bc2be Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 14:55:44 +0200 Subject: [PATCH 4/8] test(playground): tune Endform concurrency by workload --- .github/workflows/fork-studio-browser.yml | 4 +- packages/playground/e2e/README.md | 11 +++- packages/playground/e2e/endform.config.ts | 6 +- .../e2e/isolation-server.integration.test.mjs | 63 +++++++++++++++++++ packages/playground/e2e/isolation-server.mjs | 11 +++- .../playground/e2e/isolation-server.test.mjs | 3 + .../agents/$agentId/browser-stream.spec.ts | 2 +- .../e2e/tests/agents/$agentId/stream.spec.ts | 2 +- .../tests/agents/observational-memory.spec.ts | 4 +- .../tests/agents/refresh-reconnect.spec.ts | 2 +- .../cms/agents/code-agent-override.spec.ts | 2 +- 11 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 packages/playground/e2e/isolation-server.integration.test.mjs diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index 674d28dad60e..7b4d0842a0e9 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -17,7 +17,7 @@ jobs: id-token: write env: TURBO_CACHE: local:rw - E2E_APP_POOL_SIZE: '8' + E2E_APP_POOL_SIZE: '16' steps: - name: Checkout repo @@ -52,7 +52,7 @@ jobs: run: pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile - name: Verify application isolation - run: node --test packages/playground/e2e/isolation-server.test.mjs + run: node --test packages/playground/e2e/isolation-server.test.mjs packages/playground/e2e/isolation-server.integration.test.mjs - name: Test E2E working-directory: ./packages/playground diff --git a/packages/playground/e2e/README.md b/packages/playground/e2e/README.md index 1a7f8d081c71..3ce7b3a1e3db 100644 --- a/packages/playground/e2e/README.md +++ b/packages/playground/e2e/README.md @@ -4,7 +4,7 @@ Run the existing suite from this directory after building the workspace dependen and installing `kitchen-sink` dependencies: ```sh -E2E_APP_POOL_SIZE=8 npx endform@latest test --organization-id 2G1ZCj7X +E2E_APP_POOL_SIZE=16 npx endform@latest test --organization-id 2G1ZCj7X ``` `pnpm --filter @internal/playground test:e2e` uses Endform as the default runner. @@ -15,6 +15,10 @@ its separate Studio base-path test; that extra test is outside this benchmark. The fork workflow runs this command once, without a shard matrix. Endform schedules all 335 Chromium tests. `E2E_APP_POOL_SIZE` controls both application capacity and the Endform concurrency limit (default 4, supported range 1–16). +Endform also caps `@streaming` tests at four concurrent attempts and the +`@filesystem` test at one. These are scheduling limits, not test filters; all +tests and retries still execute. The streaming cap leaves application capacity +for ordinary UI cases rather than letting long streams occupy every instance. Tests import `test` and `expect` from `tests/__utils__/test`. Its automatic fixture leases one kitchen-sink instance for the whole test attempt, including hooks. The @@ -42,8 +46,13 @@ Verify the lease and reset behavior before browser experiments: ```sh node --test isolation-server.test.mjs +node --test isolation-server.integration.test.mjs ``` +The integration check starts real kitchen-sink instances, seeds the same thread +ID with different data concurrently, resets one lease, and verifies that the +other lease's data survives and the first instance's source tree is restored. + For targeted experiments, use Endform's normal Playwright file or grep filters; do not replace full-suite verification with a smaller passing subset. Changing pool size changes application capacity, so report it with benchmark results. diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts index abc34584ce5f..9eefe8d82a98 100644 --- a/packages/playground/e2e/endform.config.ts +++ b/packages/playground/e2e/endform.config.ts @@ -4,5 +4,9 @@ export default defineEndformConfig({ // ARIA snapshots are read from disk rather than imported by the specs. additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json'], // Each test leases an isolated application; Endform schedules the whole suite. - concurrentTestLimits: [{ scope: 'within-suite-run', limit: Number(process.env.E2E_APP_POOL_SIZE || 4) }], + concurrentTestLimits: [ + { scope: 'within-suite-run', limit: Number(process.env.E2E_APP_POOL_SIZE || 4) }, + { scope: 'within-suite-run', label: 'tag:@streaming', limit: 4 }, + { scope: 'within-suite-run', label: 'tag:@filesystem', limit: 1 }, + ], }); diff --git a/packages/playground/e2e/isolation-server.integration.test.mjs b/packages/playground/e2e/isolation-server.integration.test.mjs new file mode 100644 index 000000000000..ec81e662e4a4 --- /dev/null +++ b/packages/playground/e2e/isolation-server.integration.test.mjs @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { access, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { createGateway, startIsolatedApps, SESSION_HEADER } from './isolation-server.mjs'; + +test('real kitchen-sink instances isolate identical thread IDs and restore source files between leases', async t => { + const port = 49151; + const pool = await startIsolatedApps({ + size: 2, + port, + kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), + }); + const gateway = createGateway(pool.backends); + gateway.listen(port, '127.0.0.1'); + await once(gateway, 'listening'); + t.after(async () => { + gateway.closeAllConnections(); + gateway.close(); + await pool.close(); + }); + const url = `http://127.0.0.1:${port}`; + const headers = session => ({ [SESSION_HEADER]: session, 'content-type': 'application/json' }); + const lease = session => fetch(`${url}/__e2e/lease`, { method: 'POST', headers: headers(session) }); + const release = session => fetch(`${url}/__e2e/lease`, { method: 'DELETE', headers: headers(session) }); + const messages = session => + fetch(`${url}/api/memory/threads/identical-thread/messages?agentId=weather-agent`, { + headers: headers(session), + }).then(response => response.json()); + assert.equal((await lease('a')).status, 201); + assert.equal((await lease('b')).status, 201); + const seeds = await Promise.all( + [ + ['a', 2], + ['b', 3], + ].map(([session, count]) => + fetch(`${url}/e2e/seed-thread`, { + method: 'POST', + headers: headers(session), + body: JSON.stringify({ threadId: 'identical-thread', count }), + }), + ), + ); + assert.ok(seeds.every(response => response.status === 201)); + assert.equal((await messages('a')).messages.length, 2); + assert.equal((await messages('b')).messages.length, 3); + const entry = pool.backends[0].child.spawnargs[2]; + const directory = dirname(dirname(dirname(entry))); + const sourceFile = join(directory, 'src/mastra/e2e-reset-proof.json'); + await writeFile(sourceFile, JSON.stringify({ changed: true })); + assert.equal((await release('a')).status, 200); + await assert.rejects(access(sourceFile)); + assert.equal((await messages('b')).messages.length, 3); + assert.equal((await lease('c')).status, 201); + const cleared = await fetch(`${url}/api/memory/threads/identical-thread/messages?agentId=weather-agent`, { + headers: headers('c'), + }); + assert.equal(cleared.status, 404); + assert.equal((await release('c')).status, 200); + assert.equal((await release('b')).status, 200); +}); diff --git a/packages/playground/e2e/isolation-server.mjs b/packages/playground/e2e/isolation-server.mjs index 1bb5b2fdfa76..91bca756eaf8 100644 --- a/packages/playground/e2e/isolation-server.mjs +++ b/packages/playground/e2e/isolation-server.mjs @@ -14,12 +14,13 @@ export function createGateway(backends) { const sessions = new Map(); const available = [...backends]; const waiting = []; + const metrics = { leases: 0, maxActive: 0, resets: 0, resetMilliseconds: 0 }; const server = createServer(async (req, res) => { const send = (status, data) => { res.writeHead(status, { 'content-type': 'application/json' }); res.end(JSON.stringify(data)); }; - if (req.url === '/__e2e/health') return send(200, { instances: backends.length }); + if (req.url === '/__e2e/health') return send(200, { instances: backends.length, ...metrics }); if (req.url === '/__e2e/lease' && req.method === 'POST') { const session = req.headers[SESSION_HEADER]; if (typeof session !== 'string' || !/^[a-zA-Z0-9-]{1,100}$/.test(session)) { @@ -28,6 +29,8 @@ export function createGateway(backends) { if (sessions.has(session)) return send(409, { error: 'Session already leased' }); const allocate = backend => { sessions.set(session, backend); + metrics.leases++; + metrics.maxActive = Math.max(metrics.maxActive, sessions.size); send(201, { session }); }; const backend = available.shift(); @@ -50,7 +53,10 @@ export function createGateway(backends) { // Reset after every attempt, even when a test has no afterEach reset or fails. // Filesystem changes require a fresh process/source tree as well as a database reset. try { + const started = Date.now(); await backend.reset(); + metrics.resets++; + metrics.resetMilliseconds += Date.now() - started; const pending = waiting.shift(); if (pending) pending.allocate(backend); else available.push(backend); @@ -98,7 +104,7 @@ export function createGateway(backends) { socket.on('error', () => upstream.destroy()); socket.on('close', () => upstream.destroy()); }); - return server; + return Object.assign(server, { isolationMetrics: metrics }); } async function waitForServer(port, child) { @@ -211,6 +217,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1] const shutdown = async () => { if (stopping) return; stopping = true; + console.log(`Isolation metrics: ${JSON.stringify(gateway.isolationMetrics)}`); gateway.closeAllConnections(); gateway.close(); await pool.close(); diff --git a/packages/playground/e2e/isolation-server.test.mjs b/packages/playground/e2e/isolation-server.test.mjs index 27ee0e60db42..e7ab7a63ea7a 100644 --- a/packages/playground/e2e/isolation-server.test.mjs +++ b/packages/playground/e2e/isolation-server.test.mjs @@ -58,4 +58,7 @@ test('concurrent leases isolate writes and resets, and queued tests reuse a clea assert.equal((await fetch(url, { headers: headers('a') })).status, 409); assert.equal((await release('c')).status, 200); assert.equal((await release('b')).status, 200); + assert.equal(gateway.isolationMetrics.maxActive, 2); + assert.equal(gateway.isolationMetrics.leases, 3); + assert.equal(gateway.isolationMetrics.resets, 3); }); diff --git a/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts b/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts index 51a967714bae..05226ee25a55 100644 --- a/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/browser-stream.spec.ts @@ -33,7 +33,7 @@ function observeBrowserTraffic(page: Page): ObservedTraffic { return observed; } -test.describe('Browser stream WebSocket gating', () => { +test.describe('Browser stream WebSocket gating', { tag: '@streaming' }, () => { test.afterEach(async () => { await resetStorage(); }); diff --git a/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts b/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts index 3ad7ffe07d12..d066b455bccd 100644 --- a/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts +++ b/packages/playground/e2e/tests/agents/$agentId/stream.spec.ts @@ -45,7 +45,7 @@ async function assertToolStream(page: Page) { await expect(page.getByTestId('tool-result')).toContainText(`"location":`); } -test.describe('Agent chat streaming', () => { +test.describe('Agent chat streaming', { tag: '@streaming' }, () => { test.beforeEach(async ({ browser }) => { await resetStorage(); context = await browser.newContext(); diff --git a/packages/playground/e2e/tests/agents/observational-memory.spec.ts b/packages/playground/e2e/tests/agents/observational-memory.spec.ts index 2bba4db47cbd..f371baa49412 100644 --- a/packages/playground/e2e/tests/agents/observational-memory.spec.ts +++ b/packages/playground/e2e/tests/agents/observational-memory.spec.ts @@ -41,7 +41,7 @@ async function openMemorySidebar(page: Page) { await expect(memoryCard).toHaveAttribute('aria-pressed', 'true'); } -test.describe('Observational Memory - Behavior Tests', () => { +test.describe('Observational Memory - Behavior Tests', { tag: '@streaming' }, () => { test.afterEach(async () => { await resetStorage(); }); @@ -352,7 +352,7 @@ test.describe('Observational Memory - Behavior Tests', () => { }); }); -test.describe('Observational Memory - Edge Cases', () => { +test.describe('Observational Memory - Edge Cases', { tag: '@streaming' }, () => { test.afterEach(async () => { await resetStorage(); }); diff --git a/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts b/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts index 88ff63ddcff3..9efe9bba524e 100644 --- a/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts +++ b/packages/playground/e2e/tests/agents/refresh-reconnect.spec.ts @@ -3,7 +3,7 @@ import { expect, test } from '../__utils__/test'; const chatPath = '/agents/weather-agent/threads/new'; const agentDetails = /\/api\/agents\/weather-agent(?:\?.*)?$/; -test.describe('Studio refresh connection', () => { +test.describe('Studio refresh connection', { tag: '@streaming' }, () => { for (const hasHandshake of [true, false]) { test.describe('when the server restarts while disconnected', () => { test(`recovers the stale chat ${hasHandshake ? 'after an established connection' : 'before the first handshake'} without a broadcast`, async ({ diff --git a/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts b/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts index c9dcd3379adb..72ad1a17b0bc 100644 --- a/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts +++ b/packages/playground/e2e/tests/cms/agents/code-agent-override.spec.ts @@ -13,7 +13,7 @@ test.describe('code-mode agent override', () => { await resetStorage(); }); - test.describe('when an editable code-mode agent is opened', () => { + test.describe('when an editable code-mode agent is opened', { tag: '@filesystem' }, () => { test('editable local code agent saves to filesystem and can download JSON', async ({ page, request }) => { await page.goto('/agents/code-override-editable/editor'); From 05c5de4443f30b91fc6ae912b016ad968de55826 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 15:20:48 +0200 Subject: [PATCH 5/8] test(playground): experiment with runner-local applications --- .github/workflows/fork-studio-browser.yml | 7 ++- packages/playground/e2e/endform.config.ts | 13 ++++- packages/playground/e2e/playwright.config.ts | 2 +- .../playground/e2e/prepare-remote-app.mjs | 11 ++++ .../e2e/tests/__utils__/remote-test.ts | 53 +++++++++++++++++++ .../playground/e2e/tests/__utils__/test.ts | 5 +- 6 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 packages/playground/e2e/prepare-remote-app.mjs create mode 100644 packages/playground/e2e/tests/__utils__/remote-test.ts diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index 7b4d0842a0e9..7b712423de31 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -17,7 +17,8 @@ jobs: id-token: write env: TURBO_CACHE: local:rw - E2E_APP_POOL_SIZE: '16' + E2E_REMOTE_APP: 'true' + E2E_CONCURRENCY: '40' steps: - name: Checkout repo @@ -54,6 +55,10 @@ jobs: - name: Verify application isolation run: node --test packages/playground/e2e/isolation-server.test.mjs packages/playground/e2e/isolation-server.integration.test.mjs + - name: Prepare remote Studio assets + working-directory: ./packages/playground/e2e + run: node prepare-remote-app.mjs + - name: Test E2E working-directory: ./packages/playground run: pnpm test:e2e:endform diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts index 9eefe8d82a98..27aacced2c49 100644 --- a/packages/playground/e2e/endform.config.ts +++ b/packages/playground/e2e/endform.config.ts @@ -2,10 +2,19 @@ import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ // ARIA snapshots are read from disk rather than imported by the specs. - additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json'], + additionalFiles: [ + 'tests/**/*.aria.yml', + '../tsconfig.json', + ...(process.env.E2E_REMOTE_APP === 'true' ? ['kitchen-sink/.mastra/output/studio/**/*'] : []), + ], // Each test leases an isolated application; Endform schedules the whole suite. concurrentTestLimits: [ - { scope: 'within-suite-run', limit: Number(process.env.E2E_APP_POOL_SIZE || 4) }, + { + scope: 'within-suite-run', + limit: Number( + process.env.E2E_REMOTE_APP === 'true' ? process.env.E2E_CONCURRENCY || 40 : process.env.E2E_APP_POOL_SIZE || 4, + ), + }, { scope: 'within-suite-run', label: 'tag:@streaming', limit: 4 }, { scope: 'within-suite-run', label: 'tag:@filesystem', limit: 1 }, ], diff --git a/packages/playground/e2e/playwright.config.ts b/packages/playground/e2e/playwright.config.ts index a56ab193a825..df025b84a9ca 100644 --- a/packages/playground/e2e/playwright.config.ts +++ b/packages/playground/e2e/playwright.config.ts @@ -35,5 +35,5 @@ export default defineConfig({ }, ], - webServer: webservers, + webServer: process.env.E2E_REMOTE_APP === 'true' ? undefined : webservers, }); diff --git a/packages/playground/e2e/prepare-remote-app.mjs b/packages/playground/e2e/prepare-remote-app.mjs new file mode 100644 index 000000000000..564b957fa911 --- /dev/null +++ b/packages/playground/e2e/prepare-remote-app.mjs @@ -0,0 +1,11 @@ +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { startIsolatedApps } from './isolation-server.mjs'; + +// Generate Studio assets with the supported dev command; no app pool is retained. +const prepared = await startIsolatedApps({ + size: 0, + port: 4111, + kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), +}); +await prepared.close(); diff --git a/packages/playground/e2e/tests/__utils__/remote-test.ts b/packages/playground/e2e/tests/__utils__/remote-test.ts new file mode 100644 index 000000000000..dc5c33b57383 --- /dev/null +++ b/packages/playground/e2e/tests/__utils__/remote-test.ts @@ -0,0 +1,53 @@ +import { cp, mkdir, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test as base } from '@playwright/test'; +import { createNodeServer } from '../../../../deployer/dist/server/index.js'; + +// Experimental mode: Endform gives each parallel test its own remote runner. +// Put the application next to its browser rather than proxying to a shared CI host. +export const remoteTest = base.extend<{ isolatedSession: string }>({ + isolatedSession: [ + async ({}, use) => { + const oldDirectory = process.cwd(); + const directory = await mkdtemp(join(tmpdir(), 'mastra-remote-test-')); + process.chdir(directory); + process.env.MASTRA_DEV = 'true'; + process.env.MASTRA_STUDIO_PATH = fileURLToPath( + new URL('../../kitchen-sink/.mastra/output/studio', import.meta.url), + ); + await mkdir(join(directory, '.mastra'), { recursive: true }); + await cp(fileURLToPath(new URL('../../kitchen-sink/src', import.meta.url)), join(directory, 'src'), { + recursive: true, + }); + process.env.MASTRA_PROJECT_ROOT = join(directory, '.mastra'); + delete process.env.E2E_GATEWAY_PORT; + const { mastra } = await import('../../kitchen-sink/src/mastra/index'); + const tools = await import('../../kitchen-sink/src/mastra/tools'); + // Mastra dev emits a CLI readiness message over process.send. This process + // belongs to Playwright, so keep that message out of its worker IPC protocol. + const send = process.send; + if (send) { + process.send = ((message, ...args) => { + if (typeof message === 'object' && message !== null && 'type' in message && message.type === 'server-ready') + return true; + return send.call(process, message, ...args); + }) as typeof process.send; + } + const server = await createNodeServer(mastra, { studio: true, isDev: true, tools }); + await fetch('http://localhost:4111/health'); + process.send = send; + try { + await use('remote'); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + await mastra.shutdown(); + process.chdir(oldDirectory); + await rm(directory, { recursive: true, force: true }); + } + }, + { auto: true, timeout: 60_000 }, + ], +}); diff --git a/packages/playground/e2e/tests/__utils__/test.ts b/packages/playground/e2e/tests/__utils__/test.ts index d1ade819ed2b..ec8241697172 100644 --- a/packages/playground/e2e/tests/__utils__/test.ts +++ b/packages/playground/e2e/tests/__utils__/test.ts @@ -9,6 +9,7 @@ const SESSION_ANNOTATION = 'mastra-e2e-session'; // Resolve the annotation per call rather than storing mutable process-wide test state. const originalFetch = globalThis.fetch; globalThis.fetch = (input, init) => { + if (process.env.E2E_REMOTE_APP === 'true') return originalFetch(input, init); const url = new URL(input instanceof Request ? input.url : String(input)); if (url.origin !== BASE_URL || url.pathname.startsWith('/__e2e/')) return originalFetch(input, init); const session = base.info().annotations.find(annotation => annotation.type === SESSION_ANNOTATION)?.description; @@ -18,7 +19,7 @@ globalThis.fetch = (input, init) => { return originalFetch(input, { ...init, headers }); }; -export const test = base.extend<{ isolatedSession: string }>({ +const isolatedTest = base.extend<{ isolatedSession: string }>({ isolatedSession: [ async ({ browser }, use, testInfo) => { const session = randomUUID(); @@ -49,4 +50,6 @@ export const test = base.extend<{ isolatedSession: string }>({ }, }); +export const test = process.env.E2E_REMOTE_APP === 'true' ? (await import('./remote-test')).remoteTest : isolatedTest; + export { expect } from '@playwright/test'; From 2fb782c6fd606db00dc289a8c2570256a978116a Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 16:27:44 +0200 Subject: [PATCH 6/8] test(playground): make Endform the primary runner with isolated remote apps --- .github/workflows/fork-studio-browser.yml | 10 +- packages/playground/e2e/README.md | 88 ++++--- packages/playground/e2e/endform.config.ts | 12 +- packages/playground/e2e/isolated-app.mjs | 19 -- .../e2e/isolation-server.integration.test.mjs | 63 ----- packages/playground/e2e/isolation-server.mjs | 228 ------------------ .../playground/e2e/isolation-server.test.mjs | 64 ----- .../e2e/kitchen-sink/src/mastra/index.ts | 1 - packages/playground/e2e/playwright.config.ts | 13 +- .../playground/e2e/prepare-remote-app.mjs | 50 +++- .../e2e/tests/__utils__/remote-test.ts | 107 +++++--- .../playground/e2e/tests/__utils__/test.ts | 54 +---- packages/playground/package.json | 2 +- 13 files changed, 175 insertions(+), 536 deletions(-) delete mode 100644 packages/playground/e2e/isolated-app.mjs delete mode 100644 packages/playground/e2e/isolation-server.integration.test.mjs delete mode 100644 packages/playground/e2e/isolation-server.mjs delete mode 100644 packages/playground/e2e/isolation-server.test.mjs diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index 7b712423de31..0dab9895a795 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -17,8 +17,7 @@ jobs: id-token: write env: TURBO_CACHE: local:rw - E2E_REMOTE_APP: 'true' - E2E_CONCURRENCY: '40' + E2E_CONCURRENCY: '20' steps: - name: Checkout repo @@ -52,13 +51,6 @@ jobs: working-directory: ./packages/playground run: pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile - - name: Verify application isolation - run: node --test packages/playground/e2e/isolation-server.test.mjs packages/playground/e2e/isolation-server.integration.test.mjs - - - name: Prepare remote Studio assets - working-directory: ./packages/playground/e2e - run: node prepare-remote-app.mjs - - name: Test E2E working-directory: ./packages/playground run: pnpm test:e2e:endform diff --git a/packages/playground/e2e/README.md b/packages/playground/e2e/README.md index 3ce7b3a1e3db..c7b5ea2f4898 100644 --- a/packages/playground/e2e/README.md +++ b/packages/playground/e2e/README.md @@ -1,58 +1,56 @@ # Studio browser tests on Endform -Run the existing suite from this directory after building the workspace dependencies -and installing `kitchen-sink` dependencies: +After building the workspace dependencies, run: ```sh -E2E_APP_POOL_SIZE=16 npx endform@latest test --organization-id 2G1ZCj7X +pnpm --filter @internal/playground test:e2e ``` -`pnpm --filter @internal/playground test:e2e` uses Endform as the default runner. -`test:e2e:endform` runs the same command without dependency installation. -The previous native runner remains available as `test:e2e:playwright`, including -its separate Studio base-path test; that extra test is outside this benchmark. - -The fork workflow runs this command once, without a shard matrix. Endform schedules -all 335 Chromium tests. `E2E_APP_POOL_SIZE` controls both application capacity and -the Endform concurrency limit (default 4, supported range 1–16). -Endform also caps `@streaming` tests at four concurrent attempts and the -`@filesystem` test at one. These are scheduling limits, not test filters; all -tests and retries still execute. The streaming cap leaves application capacity -for ordinary UI cases rather than letting long streams occupy every instance. - -Tests import `test` and `expect` from `tests/__utils__/test`. Its automatic fixture -leases one kitchen-sink instance for the whole test attempt, including hooks. The -lease header is attached to browser contexts, API request contexts and Node fetch -calls to the test origin. Existing test URLs and assertions stay unchanged. - -`isolation-server.mjs` runs `mastra dev` once to generate the development server -and Studio, then starts isolated copies of that output behind one gateway. Each -instance has its own database, source tree and process. After every attempt, the -instance is stopped, its database and source tree are reset, and it is restarted -before another test can lease it. This also isolates tests that save agent -configuration to disk. Dependency builds and browser installation are not -repeated per application instance; Endform provides the browsers. - -The gateway refuses application requests without an active lease. Do not bypass -it by calling a backend port, or remove a reset to increase concurrency. A hung -or failing instance is not returned to the pool. The server is a local test -harness, not a production application service. +This installs the kitchen-sink dependencies, generates Studio assets once using +its supported `mastra dev` command, and runs one Endform command for the existing +335 Chromium tests. Endform supplies the browsers. CI uses the same path after +its workspace build and frozen fixture installation; there is no shard matrix. +`test:e2e:endform` skips the dependency installation. -Playwright explicitly uses the playground `tsconfig.json` so imported fixtures -retain their `@/*` mappings on remote runners. ARIA snapshot files are transferred -explicitly because tests read them at runtime. +Each Endform test runs on its own remote machine. The automatic fixture in +`tests/__utils__/remote-test.ts` starts the existing kitchen-sink application using +Mastra's Node server and the same agents, tools, storage and workflow definitions. +It gives that attempt a temporary database directory and source tree. Browser, +API request context and Node fetch traffic stays on that machine's localhost. +The server, storage and temporary files are cleaned up after the attempt. +Retries run in fresh Playwright workers. The application starts before test hooks +and has a separate 60-second infrastructure budget; existing test bodies, +assertions, assertion timeouts and retry policy remain unchanged. -Verify the lease and reset behavior before browser experiments: +The default Endform concurrency limit is 20. Override it for experiments with: ```sh -node --test isolation-server.test.mjs -node --test isolation-server.integration.test.mjs +E2E_CONCURRENCY=40 pnpm --filter @internal/playground test:e2e:endform ``` -The integration check starts real kitchen-sink instances, seeds the same thread -ID with different data concurrently, resets one lease, and verifies that the -other lease's data survives and the first instance's source tree is restored. +`concurrentTestLimits` also caps `@streaming` tests at four concurrent attempts +and the single `@filesystem` test at one. All matching limits apply; these are +scheduling limits, not test filters. The filesystem cap currently adds no extra +restriction because only one test has that tag. Limits apply within each suite +run because applications are isolated across runs too. + +The native runner is available as `test:e2e:playwright`, including its separate +Studio base-path test, which is outside this benchmark. It uses the original +shared development server and one Playwright worker. `test:e2e:ui` and the other +native debugging scripts remain available. The common fixture chooses the +remote application only when `E2E_REMOTE_APP=true`, set by the Endform script. -For targeted experiments, use Endform's normal Playwright file or grep filters; -do not replace full-suite verification with a smaller passing subset. Changing -pool size changes application capacity, so report it with benchmark results. +Playwright explicitly uses the playground `tsconfig.json` so imported fixtures +retain their `@/*` mappings on remote runners. Endform transfers imports and +directly referenced environment variables automatically. ARIA snapshots, +`tsconfig.json` and generated Studio assets are transferred explicitly because +these are runtime file reads. There is no broad environment capture or API key. +The fork CI job authenticates through GitHub OIDC. + +Full-suite experiments compared a shared CI application pool of 8 and 16 +instances with applications on Endform runners. The pool saturated the CI host +and was removed from the final setup. Runner-local execution reduced measured +feedback time, but completed runs still contain assertion and timing failures. +Those failures stay visible; this is not a passing migration yet. See the fork +PR for exact commits, completed runs, outcomes and timing comparisons. Do not +use a smaller passing subset to represent the full suite. diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts index 27aacced2c49..ca21b254e9cf 100644 --- a/packages/playground/e2e/endform.config.ts +++ b/packages/playground/e2e/endform.config.ts @@ -2,18 +2,12 @@ import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ // ARIA snapshots are read from disk rather than imported by the specs. - additionalFiles: [ - 'tests/**/*.aria.yml', - '../tsconfig.json', - ...(process.env.E2E_REMOTE_APP === 'true' ? ['kitchen-sink/.mastra/output/studio/**/*'] : []), - ], - // Each test leases an isolated application; Endform schedules the whole suite. + additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json', 'kitchen-sink/.mastra/output/studio/**/*'], + // Each remote runner owns its application. Endform schedules the whole suite. concurrentTestLimits: [ { scope: 'within-suite-run', - limit: Number( - process.env.E2E_REMOTE_APP === 'true' ? process.env.E2E_CONCURRENCY || 40 : process.env.E2E_APP_POOL_SIZE || 4, - ), + limit: Number(process.env.E2E_CONCURRENCY || 20), }, { scope: 'within-suite-run', label: 'tag:@streaming', limit: 4 }, { scope: 'within-suite-run', label: 'tag:@filesystem', limit: 1 }, diff --git a/packages/playground/e2e/isolated-app.mjs b/packages/playground/e2e/isolated-app.mjs deleted file mode 100644 index f3d058c7d50a..000000000000 --- a/packages/playground/e2e/isolated-app.mjs +++ /dev/null @@ -1,19 +0,0 @@ -import { pathToFileURL } from 'node:url'; - -// Tests can configure MCP clients with the public gateway URL. Calls originating -// inside an isolated application must return to that same application, not rely -// on a browser lease header that the MCP SDK does not know about. -const gatewayPort = process.env.E2E_GATEWAY_PORT; -const port = process.env.PORT; -const originalFetch = globalThis.fetch; -globalThis.fetch = (input, init) => { - const url = new URL(input instanceof Request ? input.url : String(input)); - if ((url.hostname === 'localhost' || url.hostname === '127.0.0.1') && url.port === gatewayPort) { - url.port = port; - const redirected = input instanceof Request ? new Request(url, input) : url; - return originalFetch(redirected, init); - } - return originalFetch(input, init); -}; - -await import(pathToFileURL(process.argv[2]).href); diff --git a/packages/playground/e2e/isolation-server.integration.test.mjs b/packages/playground/e2e/isolation-server.integration.test.mjs deleted file mode 100644 index ec81e662e4a4..000000000000 --- a/packages/playground/e2e/isolation-server.integration.test.mjs +++ /dev/null @@ -1,63 +0,0 @@ -import assert from 'node:assert/strict'; -import { once } from 'node:events'; -import { access, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { test } from 'node:test'; -import { fileURLToPath } from 'node:url'; -import { createGateway, startIsolatedApps, SESSION_HEADER } from './isolation-server.mjs'; - -test('real kitchen-sink instances isolate identical thread IDs and restore source files between leases', async t => { - const port = 49151; - const pool = await startIsolatedApps({ - size: 2, - port, - kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), - }); - const gateway = createGateway(pool.backends); - gateway.listen(port, '127.0.0.1'); - await once(gateway, 'listening'); - t.after(async () => { - gateway.closeAllConnections(); - gateway.close(); - await pool.close(); - }); - const url = `http://127.0.0.1:${port}`; - const headers = session => ({ [SESSION_HEADER]: session, 'content-type': 'application/json' }); - const lease = session => fetch(`${url}/__e2e/lease`, { method: 'POST', headers: headers(session) }); - const release = session => fetch(`${url}/__e2e/lease`, { method: 'DELETE', headers: headers(session) }); - const messages = session => - fetch(`${url}/api/memory/threads/identical-thread/messages?agentId=weather-agent`, { - headers: headers(session), - }).then(response => response.json()); - assert.equal((await lease('a')).status, 201); - assert.equal((await lease('b')).status, 201); - const seeds = await Promise.all( - [ - ['a', 2], - ['b', 3], - ].map(([session, count]) => - fetch(`${url}/e2e/seed-thread`, { - method: 'POST', - headers: headers(session), - body: JSON.stringify({ threadId: 'identical-thread', count }), - }), - ), - ); - assert.ok(seeds.every(response => response.status === 201)); - assert.equal((await messages('a')).messages.length, 2); - assert.equal((await messages('b')).messages.length, 3); - const entry = pool.backends[0].child.spawnargs[2]; - const directory = dirname(dirname(dirname(entry))); - const sourceFile = join(directory, 'src/mastra/e2e-reset-proof.json'); - await writeFile(sourceFile, JSON.stringify({ changed: true })); - assert.equal((await release('a')).status, 200); - await assert.rejects(access(sourceFile)); - assert.equal((await messages('b')).messages.length, 3); - assert.equal((await lease('c')).status, 201); - const cleared = await fetch(`${url}/api/memory/threads/identical-thread/messages?agentId=weather-agent`, { - headers: headers('c'), - }); - assert.equal(cleared.status, 404); - assert.equal((await release('c')).status, 200); - assert.equal((await release('b')).status, 200); -}); diff --git a/packages/playground/e2e/isolation-server.mjs b/packages/playground/e2e/isolation-server.mjs deleted file mode 100644 index 91bca756eaf8..000000000000 --- a/packages/playground/e2e/isolation-server.mjs +++ /dev/null @@ -1,228 +0,0 @@ -import { spawn } from 'node:child_process'; -import { cp, mkdir, mkdtemp, rm, symlink } from 'node:fs/promises'; -import { createServer, request } from 'node:http'; -import { connect } from 'node:net'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; - -export const SESSION_HEADER = 'x-mastra-e2e-session'; - -// A lease covers one test attempt, including its hooks and browser/API requests. -// Each backend owns its database, source tree and in-memory Mastra registries. -export function createGateway(backends) { - const sessions = new Map(); - const available = [...backends]; - const waiting = []; - const metrics = { leases: 0, maxActive: 0, resets: 0, resetMilliseconds: 0 }; - const server = createServer(async (req, res) => { - const send = (status, data) => { - res.writeHead(status, { 'content-type': 'application/json' }); - res.end(JSON.stringify(data)); - }; - if (req.url === '/__e2e/health') return send(200, { instances: backends.length, ...metrics }); - if (req.url === '/__e2e/lease' && req.method === 'POST') { - const session = req.headers[SESSION_HEADER]; - if (typeof session !== 'string' || !/^[a-zA-Z0-9-]{1,100}$/.test(session)) { - return send(400, { error: 'A valid session header is required' }); - } - if (sessions.has(session)) return send(409, { error: 'Session already leased' }); - const allocate = backend => { - sessions.set(session, backend); - metrics.leases++; - metrics.maxActive = Math.max(metrics.maxActive, sessions.size); - send(201, { session }); - }; - const backend = available.shift(); - if (backend) allocate(backend); - else { - const pending = { allocate }; - waiting.push(pending); - res.on('close', () => { - const index = waiting.indexOf(pending); - if (index >= 0) waiting.splice(index, 1); - }); - } - return; - } - if (req.url === '/__e2e/lease' && req.method === 'DELETE') { - const session = req.headers[SESSION_HEADER]; - const backend = sessions.get(session); - if (!backend) return send(404, { error: 'Unknown session' }); - sessions.delete(session); - // Reset after every attempt, even when a test has no afterEach reset or fails. - // Filesystem changes require a fresh process/source tree as well as a database reset. - try { - const started = Date.now(); - await backend.reset(); - metrics.resets++; - metrics.resetMilliseconds += Date.now() - started; - const pending = waiting.shift(); - if (pending) pending.allocate(backend); - else available.push(backend); - send(200, { released: true }); - } catch (error) { - send(500, { error: `Could not reset isolated instance: ${error.message}` }); - } - return; - } - const backend = sessions.get(req.headers[SESSION_HEADER]); - if (!backend) return send(409, { error: 'Application requests require an active test lease' }); - const upstream = request( - { - hostname: '127.0.0.1', - port: backend.port, - path: req.url, - method: req.method, - headers: { ...req.headers, host: `localhost:${backend.port}` }, - }, - response => { - res.writeHead(response.statusCode, response.headers); - response.pipe(res); - }, - ); - upstream.on('error', error => { - if (!res.headersSent) send(502, { error: error.message }); - else res.destroy(error); - }); - res.on('close', () => upstream.destroy()); - req.pipe(upstream); - }); - server.on('upgrade', (req, socket, head) => { - const backend = sessions.get(req.headers[SESSION_HEADER]); - if (!backend) return socket.destroy(); - const upstream = connect(backend.port, '127.0.0.1', () => { - upstream.write(`${req.method} ${req.url} HTTP/${req.httpVersion}\r\n`); - for (const [name, value] of Object.entries(req.headers)) { - upstream.write(`${name}: ${value}\r\n`); - } - upstream.write('\r\n'); - upstream.write(head); - socket.pipe(upstream).pipe(socket); - }); - upstream.on('error', () => socket.destroy()); - socket.on('error', () => upstream.destroy()); - socket.on('close', () => upstream.destroy()); - }); - return Object.assign(server, { isolationMetrics: metrics }); -} - -async function waitForServer(port, child) { - for (let attempt = 0; attempt < 600; attempt++) { - if (child.exitCode !== null) throw new Error(`Kitchen-sink exited with ${child.exitCode}`); - try { - const response = await fetch(`http://127.0.0.1:${port}/health`); - if (response.ok) return; - } catch {} - await new Promise(resolve => setTimeout(resolve, 100)); - } - throw new Error(`Kitchen-sink on port ${port} did not become ready`); -} - -async function stop(child) { - if (!child || child.exitCode !== null) return; - await new Promise(resolve => { - const timeout = setTimeout(() => child.kill('SIGKILL'), 5000); - child.once('exit', () => { - clearTimeout(timeout); - resolve(); - }); - child.kill('SIGTERM'); - }); -} - -export async function startIsolatedApps({ size, port, kitchenSink }) { - const root = await mkdtemp(join(tmpdir(), 'mastra-endform-')); - const backends = []; - let templateProcess; - try { - // Use the supported dev command once to generate the development server and Studio. - templateProcess = spawn('pnpm', ['dev'], { - cwd: kitchenSink, - detached: true, - env: { ...process.env, PORT: String(port + 100), E2E_GATEWAY_PORT: String(port) }, - stdio: ['ignore', 'inherit', 'inherit'], - }); - await waitForServer(port + 100, templateProcess); - // pnpm owns a CLI and server process tree; terminate only this process group. - process.kill(-templateProcess.pid, 'SIGTERM'); - await stop(templateProcess); - templateProcess = undefined; - for (let index = 0; index < size; index++) { - const directory = join(root, String(index)); - const output = join(directory, '.mastra/output'); - await mkdir(directory, { recursive: true }); - await cp(join(kitchenSink, '.mastra/output'), output, { recursive: true }); - await symlink(join(kitchenSink, 'node_modules'), join(directory, 'node_modules')); - await cp(join(kitchenSink, 'package.json'), join(directory, 'package.json')); - await mkdir(join(output, 'public'), { recursive: true }); - const backend = { port: port + index + 1, child: undefined }; - const start = async () => { - backend.child = spawn( - process.execPath, - [join(dirname(fileURLToPath(import.meta.url)), 'isolated-app.mjs'), join(output, 'index.mjs')], - { - cwd: join(output, 'public'), - env: { - ...process.env, - PORT: String(backend.port), - E2E_GATEWAY_PORT: String(port), - MASTRA_DEV: 'true', - MASTRA_PROJECT_ROOT: join(directory, '.mastra'), - }, - stdio: ['ignore', 'inherit', 'inherit'], - }, - ); - await waitForServer(backend.port, backend.child); - }; - backend.reset = async () => { - await stop(backend.child); - await rm(join(directory, 'src'), { recursive: true, force: true }); - await cp(join(kitchenSink, 'src'), join(directory, 'src'), { recursive: true }); - await rm(join(output, 'public'), { recursive: true, force: true }); - await mkdir(join(output, 'public'), { recursive: true }); - await start(); - }; - backends.push(backend); - await backend.reset(); - } - return { - backends, - close: async () => { - await Promise.all(backends.map(backend => stop(backend.child))); - await rm(root, { recursive: true, force: true }); - }, - }; - } catch (error) { - if (templateProcess) { - try { - process.kill(-templateProcess.pid, 'SIGTERM'); - } catch {} - } - await Promise.all(backends.map(backend => stop(backend.child))); - await rm(root, { recursive: true, force: true }); - throw error; - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { - const port = Number(process.env.E2E_PORT || 4111); - const size = Number(process.env.E2E_APP_POOL_SIZE || 4); - if (!Number.isInteger(size) || size < 1 || size > 16) throw new Error('E2E_APP_POOL_SIZE must be 1–16'); - const kitchenSink = join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'); - const pool = await startIsolatedApps({ size, port, kitchenSink }); - const gateway = createGateway(pool.backends); - gateway.listen(port, '0.0.0.0', () => console.log(`Isolated kitchen-sink ready: ${size} instances on :${port}`)); - let stopping = false; - const shutdown = async () => { - if (stopping) return; - stopping = true; - console.log(`Isolation metrics: ${JSON.stringify(gateway.isolationMetrics)}`); - gateway.closeAllConnections(); - gateway.close(); - await pool.close(); - process.exit(0); - }; - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); -} diff --git a/packages/playground/e2e/isolation-server.test.mjs b/packages/playground/e2e/isolation-server.test.mjs deleted file mode 100644 index e7ab7a63ea7a..000000000000 --- a/packages/playground/e2e/isolation-server.test.mjs +++ /dev/null @@ -1,64 +0,0 @@ -import assert from 'node:assert/strict'; -import { createServer } from 'node:http'; -import { once } from 'node:events'; -import { test } from 'node:test'; -import { createGateway, SESSION_HEADER } from './isolation-server.mjs'; - -async function listen(server) { - server.listen(0, '127.0.0.1'); - await once(server, 'listening'); - return server.address().port; -} - -test('concurrent leases isolate writes and resets, and queued tests reuse a clean instance', async t => { - const backends = []; - for (let index = 0; index < 2; index++) { - let value = ''; - const app = createServer((req, res) => { - if (req.method === 'POST') value = req.headers['x-value']; - res.end(JSON.stringify({ instance: index, value })); - }); - backends.push({ - port: await listen(app), - reset: async () => { - value = ''; - }, - }); - t.after(() => { - app.closeAllConnections(); - app.close(); - }); - } - const gateway = createGateway(backends); - const url = `http://127.0.0.1:${await listen(gateway)}`; - t.after(() => { - gateway.closeAllConnections(); - gateway.close(); - }); - const headers = session => ({ [SESSION_HEADER]: session }); - const lease = session => fetch(`${url}/__e2e/lease`, { method: 'POST', headers: headers(session) }); - const release = session => fetch(`${url}/__e2e/lease`, { method: 'DELETE', headers: headers(session) }); - const read = session => fetch(url, { headers: headers(session) }).then(response => response.json()); - assert.equal((await fetch(url)).status, 409); - assert.equal((await lease('a')).status, 201); - assert.equal((await lease('b')).status, 201); - await Promise.all( - ['a', 'b'].map(session => fetch(url, { method: 'POST', headers: { ...headers(session), 'x-value': session } })), - ); - const [a, b] = await Promise.all([read('a'), read('b')]); - assert.notEqual(a.instance, b.instance); - assert.equal(a.value, 'a'); - assert.equal(b.value, 'b'); - assert.equal((await lease('a')).status, 409); - const queued = lease('c'); - assert.equal((await release('a')).status, 200); - assert.equal((await queued).status, 201); - assert.deepEqual(await read('c'), { instance: a.instance, value: '' }); - assert.deepEqual(await read('b'), b); - assert.equal((await fetch(url, { headers: headers('a') })).status, 409); - assert.equal((await release('c')).status, 200); - assert.equal((await release('b')).status, 200); - assert.equal(gateway.isolationMetrics.maxActive, 2); - assert.equal(gateway.isolationMetrics.leases, 3); - assert.equal(gateway.isolationMetrics.resets, 3); -}); diff --git a/packages/playground/e2e/kitchen-sink/src/mastra/index.ts b/packages/playground/e2e/kitchen-sink/src/mastra/index.ts index bc79a8c79647..26505603e741 100644 --- a/packages/playground/e2e/kitchen-sink/src/mastra/index.ts +++ b/packages/playground/e2e/kitchen-sink/src/mastra/index.ts @@ -62,7 +62,6 @@ export const mastra = new Mastra({ contentFilterProcessor, }, server: { - ...(process.env.E2E_GATEWAY_PORT ? { studioPort: Number(process.env.E2E_GATEWAY_PORT) } : {}), ...(process.env.E2E_STUDIO_BASE_PATH ? { studioBase: process.env.E2E_STUDIO_BASE_PATH } : {}), apiRoutes: [ // Seeds a weather-agent thread with `count` user messages ("seed message N", diff --git a/packages/playground/e2e/playwright.config.ts b/packages/playground/e2e/playwright.config.ts index df025b84a9ca..73a6711cb222 100644 --- a/packages/playground/e2e/playwright.config.ts +++ b/packages/playground/e2e/playwright.config.ts @@ -7,12 +7,21 @@ const webservers: PlaywrightTestConfig['webServer'] = [ { // UI tests use route interception for auth mocking - no server auth needed // Server-side permission tests are in server-adapters/hono - command: 'node ./isolation-server.mjs', - url: `${BASE_URL}/__e2e/health`, + command: 'pnpm -C ./kitchen-sink dev', + url: 'http://localhost:4111', timeout: 120_000, }, ]; +if (PORT) { + webservers.push({ + command: `echo "App is running on :${PORT}"`, + url: BASE_URL, + timeout: 120_000, + reuseExistingServer: true, + }); +} + export default defineConfig({ tsconfig: '../tsconfig.json', testDir: './tests', diff --git a/packages/playground/e2e/prepare-remote-app.mjs b/packages/playground/e2e/prepare-remote-app.mjs index 564b957fa911..40cbc3988771 100644 --- a/packages/playground/e2e/prepare-remote-app.mjs +++ b/packages/playground/e2e/prepare-remote-app.mjs @@ -1,11 +1,45 @@ -import { dirname, join } from 'node:path'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; import { fileURLToPath } from 'node:url'; -import { startIsolatedApps } from './isolation-server.mjs'; -// Generate Studio assets with the supported dev command; no app pool is retained. -const prepared = await startIsolatedApps({ - size: 0, - port: 4111, - kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), +// Generate Studio with the supported dev command once before uploading tests. +// The server is stopped here; applications run beside the remote browsers. +const child = spawn('pnpm', ['dev'], { + cwd: fileURLToPath(new URL('./kitchen-sink', import.meta.url)), + detached: true, + env: { ...process.env, PORT: '4211' }, + stdio: ['ignore', 'inherit', 'inherit'], }); -await prepared.close(); + +try { + let ready = false; + for (let attempt = 0; attempt < 600; attempt++) { + if (child.exitCode !== null || child.signalCode !== null) throw new Error('Studio preparation server exited'); + try { + const response = await fetch('http://127.0.0.1:4211/health', { signal: AbortSignal.timeout(1000) }); + if (response.ok) { + ready = true; + break; + } + } catch {} + await new Promise(resolve => setTimeout(resolve, 100)); + } + if (!ready) throw new Error('Studio preparation server did not become ready'); +} finally { + if (child.exitCode === null && child.signalCode === null) { + const exited = once(child, 'exit'); + process.kill(-child.pid, 'SIGTERM'); + const timeout = setTimeout(() => { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if (error.code !== 'ESRCH') throw error; + } + }, 5000); + try { + await exited; + } finally { + clearTimeout(timeout); + } + } +} diff --git a/packages/playground/e2e/tests/__utils__/remote-test.ts b/packages/playground/e2e/tests/__utils__/remote-test.ts index dc5c33b57383..9901afe6c14d 100644 --- a/packages/playground/e2e/tests/__utils__/remote-test.ts +++ b/packages/playground/e2e/tests/__utils__/remote-test.ts @@ -5,49 +5,84 @@ import { fileURLToPath } from 'node:url'; import { test as base } from '@playwright/test'; import { createNodeServer } from '../../../../deployer/dist/server/index.js'; -// Experimental mode: Endform gives each parallel test its own remote runner. -// Put the application next to its browser rather than proxying to a shared CI host. -export const remoteTest = base.extend<{ isolatedSession: string }>({ - isolatedSession: [ +// Endform gives each test its own remote runner. Start the existing kitchen-sink +// application there, so resets and source writes cannot affect other tests. +export const remoteTest = base.extend<{ isolatedApplication: void }>({ + isolatedApplication: [ async ({}, use) => { const oldDirectory = process.cwd(); const directory = await mkdtemp(join(tmpdir(), 'mastra-remote-test-')); - process.chdir(directory); - process.env.MASTRA_DEV = 'true'; - process.env.MASTRA_STUDIO_PATH = fileURLToPath( - new URL('../../kitchen-sink/.mastra/output/studio', import.meta.url), - ); - await mkdir(join(directory, '.mastra'), { recursive: true }); - await cp(fileURLToPath(new URL('../../kitchen-sink/src', import.meta.url)), join(directory, 'src'), { - recursive: true, - }); - process.env.MASTRA_PROJECT_ROOT = join(directory, '.mastra'); - delete process.env.E2E_GATEWAY_PORT; - const { mastra } = await import('../../kitchen-sink/src/mastra/index'); - const tools = await import('../../kitchen-sink/src/mastra/tools'); - // Mastra dev emits a CLI readiness message over process.send. This process - // belongs to Playwright, so keep that message out of its worker IPC protocol. - const send = process.send; - if (send) { - process.send = ((message, ...args) => { - if (typeof message === 'object' && message !== null && 'type' in message && message.type === 'server-ready') - return true; - return send.call(process, message, ...args); - }) as typeof process.send; - } - const server = await createNodeServer(mastra, { studio: true, isDev: true, tools }); - await fetch('http://localhost:4111/health'); - process.send = send; + const environment = { + MASTRA_DEV: process.env.MASTRA_DEV, + MASTRA_STUDIO_PATH: process.env.MASTRA_STUDIO_PATH, + MASTRA_PROJECT_ROOT: process.env.MASTRA_PROJECT_ROOT, + PORT: process.env.PORT, + }; + let server: Awaited> | undefined; + let shutdown: (() => Promise) | undefined; try { - await use('remote'); + process.chdir(directory); + process.env.MASTRA_DEV = 'true'; + process.env.PORT = process.env.E2E_PORT || '4111'; + process.env.MASTRA_STUDIO_PATH = fileURLToPath( + new URL('../../kitchen-sink/.mastra/output/studio', import.meta.url), + ); + await mkdir(join(directory, '.mastra'), { recursive: true }); + await cp(fileURLToPath(new URL('../../kitchen-sink/src', import.meta.url)), join(directory, 'src'), { + recursive: true, + }); + process.env.MASTRA_PROJECT_ROOT = join(directory, '.mastra'); + const { mastra } = await import('../../kitchen-sink/src/mastra/index'); + const tools = await import('../../kitchen-sink/src/mastra/tools'); + shutdown = () => mastra.shutdown(); + // Mastra dev emits a CLI readiness message over process.send. This process + // belongs to Playwright, so keep that message out of its worker IPC protocol. + const send = process.send; + if (send) { + process.send = ((message, ...args) => { + if (typeof message === 'object' && message !== null && 'type' in message && message.type === 'server-ready') + return true; + return Reflect.apply(send, process, [message, ...args]); + }) as typeof process.send; + } + try { + // The server's public type erases tool input/output generics; these + // are the same schema-backed exports passed by the development CLI. + server = await createNodeServer(mastra, { + studio: true, + isDev: true, + tools, + } as NonNullable[1]>); + // createNodeServer resolves before its listen callback emits readiness. + // Keep the IPC guard until a health request proves that callback ran. + const ready = await fetch(`http://localhost:${process.env.PORT}/health`); + if (!ready.ok) throw new Error(`Kitchen-sink health check failed: ${ready.status}`); + } finally { + process.send = send; + } + await use(); } finally { - server.closeAllConnections(); - await new Promise(resolve => server.close(() => resolve())); - await mastra.shutdown(); - process.chdir(oldDirectory); - await rm(directory, { recursive: true, force: true }); + try { + if (server) { + if ('closeAllConnections' in server) server.closeAllConnections(); + const runningServer = server; + await new Promise((resolve, reject) => + runningServer.close(error => (error ? reject(error) : resolve())), + ); + } + await shutdown?.(); + } finally { + for (const [name, value] of Object.entries(environment)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + process.chdir(oldDirectory); + await rm(directory, { recursive: true, force: true }); + } } }, + // Infrastructure setup/teardown has its own budget; existing journeys keep + // their original test and assertion timeouts. { auto: true, timeout: 60_000 }, ], }); diff --git a/packages/playground/e2e/tests/__utils__/test.ts b/packages/playground/e2e/tests/__utils__/test.ts index ec8241697172..80fdfd7cb1c5 100644 --- a/packages/playground/e2e/tests/__utils__/test.ts +++ b/packages/playground/e2e/tests/__utils__/test.ts @@ -1,55 +1,7 @@ -import { randomUUID } from 'node:crypto'; import { test as base } from '@playwright/test'; -const BASE_URL = `http://localhost:${process.env.E2E_PORT || '4111'}`; -const SESSION_HEADER = 'x-mastra-e2e-session'; -const SESSION_ANNOTATION = 'mastra-e2e-session'; - -// Node fetch helpers and browser/APIRequestContext traffic must use the same lease. -// Resolve the annotation per call rather than storing mutable process-wide test state. -const originalFetch = globalThis.fetch; -globalThis.fetch = (input, init) => { - if (process.env.E2E_REMOTE_APP === 'true') return originalFetch(input, init); - const url = new URL(input instanceof Request ? input.url : String(input)); - if (url.origin !== BASE_URL || url.pathname.startsWith('/__e2e/')) return originalFetch(input, init); - const session = base.info().annotations.find(annotation => annotation.type === SESSION_ANNOTATION)?.description; - if (!session) throw new Error('Kitchen-sink request made without an isolated test session'); - const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); - headers.set(SESSION_HEADER, session); - return originalFetch(input, { ...init, headers }); -}; - -const isolatedTest = base.extend<{ isolatedSession: string }>({ - isolatedSession: [ - async ({ browser }, use, testInfo) => { - const session = randomUUID(); - const headers = { [SESSION_HEADER]: session }; - const response = await originalFetch(`${BASE_URL}/__e2e/lease`, { method: 'POST', headers }); - if (!response.ok) throw new Error(`Could not lease kitchen-sink: ${response.status}`); - testInfo.annotations.push({ type: SESSION_ANNOTATION, description: session }); - // Existing streaming/IME specs create contexts explicitly. Browser is a - // worker fixture, so patch only for this test attempt and restore on exit. - const newContext = browser.newContext.bind(browser); - browser.newContext = options => - newContext({ - ...options, - extraHTTPHeaders: { ...options?.extraHTTPHeaders, [SESSION_HEADER]: session }, - }); - try { - await use(session); - } finally { - browser.newContext = newContext; - const release = await originalFetch(`${BASE_URL}/__e2e/lease`, { method: 'DELETE', headers }); - if (!release.ok) throw new Error(`Could not reset kitchen-sink: ${release.status}`); - } - }, - { auto: true }, - ], - extraHTTPHeaders: async ({ isolatedSession, extraHTTPHeaders }, use) => { - await use({ ...extraHTTPHeaders, [SESSION_HEADER]: isolatedSession }); - }, -}); - -export const test = process.env.E2E_REMOTE_APP === 'true' ? (await import('./remote-test')).remoteTest : isolatedTest; +// Native Playwright keeps its existing dev server. Endform starts an isolated +// application in the remote test process, beside its browser. +export const test = process.env.E2E_REMOTE_APP === 'true' ? (await import('./remote-test')).remoteTest : base; export { expect } from '@playwright/test'; diff --git a/packages/playground/package.json b/packages/playground/package.json index b76b57e84c35..d215f5f32cb0 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -38,7 +38,7 @@ "test:mutate": "stryker run --mutate", "test:run": "vitest run", "test:e2e": "pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile && pnpm test:e2e:endform", - "test:e2e:endform": "cd e2e && npx endform@latest test --organization-id 2G1ZCj7X", + "test:e2e:endform": "cd e2e && node prepare-remote-app.mjs && E2E_REMOTE_APP=true npx endform@latest test --organization-id 2G1ZCj7X", "test:e2e:playwright": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts && playwright test -c e2e/playwright.studio-base.config.ts", "test:e2e:studio-base": "pnpm test:e2e:setup && playwright test -c e2e/playwright.studio-base.config.ts", "test:e2e:ui": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts --ui", From c7c31d24e9d3342f415453198acaa3b6282bc620 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Tue, 6 Oct 2026 11:13:26 -0700 Subject: [PATCH 7/8] test(playground): proxy Endform directly to isolated local applications --- .github/workflows/fork-studio-browser.yml | 17 +- packages/playground/e2e/README.md | 99 ++++---- packages/playground/e2e/endform.config.ts | 17 +- .../e2e/local-app-pool.integration.test.mjs | 65 +++++ packages/playground/e2e/local-app-pool.mjs | 235 ++++++++++++++++++ .../playground/e2e/local-app-pool.test.mjs | 41 +++ packages/playground/e2e/local-app.mjs | 15 ++ packages/playground/e2e/playwright.config.ts | 5 +- .../playground/e2e/prepare-remote-app.mjs | 45 ---- .../e2e/tests/__utils__/local-test.ts | 45 ++++ .../e2e/tests/__utils__/remote-test.ts | 88 ------- .../playground/e2e/tests/__utils__/test.ts | 6 +- packages/playground/package.json | 2 +- 13 files changed, 490 insertions(+), 190 deletions(-) create mode 100644 packages/playground/e2e/local-app-pool.integration.test.mjs create mode 100644 packages/playground/e2e/local-app-pool.mjs create mode 100644 packages/playground/e2e/local-app-pool.test.mjs create mode 100644 packages/playground/e2e/local-app.mjs delete mode 100644 packages/playground/e2e/prepare-remote-app.mjs create mode 100644 packages/playground/e2e/tests/__utils__/local-test.ts delete mode 100644 packages/playground/e2e/tests/__utils__/remote-test.ts diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index 0dab9895a795..c82d35a7a05f 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -4,6 +4,16 @@ on: pull_request: branches: [main, 'codex/mastra-playwright-baseline'] workflow_dispatch: + inputs: + app_pool_size: + description: 'Local applications on the CI host' + default: '8' + concurrency: + description: 'Endform concurrent test limit' + default: '8' + streaming_concurrency: + description: 'Endform streaming test limit' + default: '12' permissions: {} @@ -17,7 +27,9 @@ jobs: id-token: write env: TURBO_CACHE: local:rw - E2E_CONCURRENCY: '20' + E2E_APP_POOL_SIZE: ${{ inputs.app_pool_size || '8' }} + E2E_CONCURRENCY: ${{ inputs.concurrency || '8' }} + E2E_STREAMING_CONCURRENCY: ${{ inputs.streaming_concurrency || '12' }} steps: - name: Checkout repo @@ -51,6 +63,9 @@ jobs: working-directory: ./packages/playground run: pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile + - name: Verify local application isolation + run: node --test packages/playground/e2e/local-app-pool.test.mjs packages/playground/e2e/local-app-pool.integration.test.mjs + - name: Test E2E working-directory: ./packages/playground run: pnpm test:e2e:endform diff --git a/packages/playground/e2e/README.md b/packages/playground/e2e/README.md index c7b5ea2f4898..8125eb2a457f 100644 --- a/packages/playground/e2e/README.md +++ b/packages/playground/e2e/README.md @@ -1,56 +1,69 @@ # Studio browser tests on Endform -After building the workspace dependencies, run: +Build the relevant workspace dependencies, then run: ```sh pnpm --filter @internal/playground test:e2e ``` -This installs the kitchen-sink dependencies, generates Studio assets once using -its supported `mastra dev` command, and runs one Endform command for the existing -335 Chromium tests. Endform supplies the browsers. CI uses the same path after -its workspace build and frozen fixture installation; there is no shard matrix. -`test:e2e:endform` skips the dependency installation. +The command installs kitchen-sink dependencies and runs the existing 335 Chromium +cases once on Endform. All applications run locally on the machine running the +CLI. Endform provides the remote browsers and proxies HTTP, streaming and +WebSocket traffic back to the assigned local ports. There is no CI shard matrix. -Each Endform test runs on its own remote machine. The automatic fixture in -`tests/__utils__/remote-test.ts` starts the existing kitchen-sink application using -Mastra's Node server and the same agents, tools, storage and workflow definitions. -It gives that attempt a temporary database directory and source tree. Browser, -API request context and Node fetch traffic stays on that machine's localhost. -The server, storage and temporary files are cleaned up after the attempt. -Retries run in fresh Playwright workers. The application starts before test hooks -and has a separate 60-second infrastructure budget; existing test bodies, -assertions, assertion timeouts and retry policy remain unchanged. +The local application pool starts the supported `mastra dev` command once to +produce the native generated entrypoint, Studio assets and package metadata. +Each pool slot runs that entrypoint in a separate Node process, with its own +source tree and database working directory. Studio assets are immutable and +shared locally. Application configuration, Node environment and runtime directory +layout follow the native CLI startup path. Nothing starts an application inside +the remote Playwright worker, and Studio/server dependencies are not transferred +to remote machines. -The default Endform concurrency limit is 20. Override it for experiments with: +The automatic fixture leases a slot for the whole attempt, including test hooks. +Browser and API request contexts use the slot's base URL. Existing Node fetch +helpers targeting the configured test origin are redirected to the same slot. +The control server only allocates leases; application traffic goes directly +through Endform's proxy to the application's port, without a gateway forwarding +hop. Explicit browser contexts in streaming/IME journeys use the same slot. +The existing MCP server call to localhost:4111 is mapped back to its own slot. + +After an attempt, the application process stops, its database and source tree +are recreated, and it restarts before reuse. A failed reset removes the slot +from circulation. A crashed remote attempt's lease expires after three minutes. +The fixture has a separate 60-second infrastructure budget; assertions, journey +timeouts, skip declarations and retries remain unchanged. + +Tune host capacity and scheduling together: + +```sh +E2E_APP_POOL_SIZE=16 E2E_CONCURRENCY=16 E2E_STREAMING_CONCURRENCY=12 \ + pnpm --filter @internal/playground test:e2e:endform +``` + +Defaults are eight applications, eight concurrent tests and a streaming cap of +12 (which has no additional effect when the total limit is eight). Increasing +Endform concurrency beyond local application capacity queues leases rather than +creating more applications. All matching native Endform concurrency limits apply. +The single filesystem case remains tagged with limit one; it adds no restriction +beyond that case being the only one with the tag. CI's manual workflow exposes +application, total and streaming limits as inputs for measured experiments. + +The native runner remains available as `test:e2e:playwright`, with its original +shared development server and one worker, including the separate Studio +base-path test outside this benchmark. Native UI/debugging scripts are preserved. +The common fixture chooses local leases only when `E2E_LOCAL_APP=true`, set by +the Endform script. Remote fixture alias imports use the explicit playground +tsconfig; runtime ARIA snapshots and tsconfig are transferred separately. + +Check real state isolation before browser experiments: ```sh -E2E_CONCURRENCY=40 pnpm --filter @internal/playground test:e2e:endform +node --test local-app-pool.test.mjs local-app-pool.integration.test.mjs ``` -`concurrentTestLimits` also caps `@streaming` tests at four concurrent attempts -and the single `@filesystem` test at one. All matching limits apply; these are -scheduling limits, not test filters. The filesystem cap currently adds no extra -restriction because only one test has that tag. Limits apply within each suite -run because applications are isolated across runs too. - -The native runner is available as `test:e2e:playwright`, including its separate -Studio base-path test, which is outside this benchmark. It uses the original -shared development server and one Playwright worker. `test:e2e:ui` and the other -native debugging scripts remain available. The common fixture chooses the -remote application only when `E2E_REMOTE_APP=true`, set by the Endform script. - -Playwright explicitly uses the playground `tsconfig.json` so imported fixtures -retain their `@/*` mappings on remote runners. Endform transfers imports and -directly referenced environment variables automatically. ARIA snapshots, -`tsconfig.json` and generated Studio assets are transferred explicitly because -these are runtime file reads. There is no broad environment capture or API key. -The fork CI job authenticates through GitHub OIDC. - -Full-suite experiments compared a shared CI application pool of 8 and 16 -instances with applications on Endform runners. The pool saturated the CI host -and was removed from the final setup. Runner-local execution reduced measured -feedback time, but completed runs still contain assertion and timing failures. -Those failures stay visible; this is not a passing migration yet. See the fork -PR for exact commits, completed runs, outcomes and timing comparisons. Do not -use a smaller passing subset to represent the full suite. +Report application preparation, test-stage wall time, outcomes, host CPU/memory +and concurrency together. Local Mac results are not an equivalent benchmark to +four-core GitHub-hosted CI. Every full-suite run must account for all 335 cases, +including the existing skips; a focused passing probe is not suite success. +GitHub CI authenticates with job-scoped OIDC; no API key is added. diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts index ca21b254e9cf..85cba6531a13 100644 --- a/packages/playground/e2e/endform.config.ts +++ b/packages/playground/e2e/endform.config.ts @@ -1,15 +1,16 @@ import { defineEndformConfig } from 'endform'; +const port = Number(process.env.E2E_PORT || 4111); +const capacity = Number(process.env.E2E_APP_POOL_SIZE || 8); + export default defineEndformConfig({ - // ARIA snapshots are read from disk rather than imported by the specs. - additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json', 'kitchen-sink/.mastra/output/studio/**/*'], - // Each remote runner owns its application. Endform schedules the whole suite. + additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json'], + // Applications stay on the CLI host; only browser/test code is distributed. + proxyNetworkHosts: [''], + proxyNetworkPorts: [port, ...Array.from({ length: capacity }, (_, index) => port + 1000 + index)], concurrentTestLimits: [ - { - scope: 'within-suite-run', - limit: Number(process.env.E2E_CONCURRENCY || 20), - }, - { scope: 'within-suite-run', label: 'tag:@streaming', limit: 4 }, + { scope: 'within-suite-run', limit: Number(process.env.E2E_CONCURRENCY || capacity) }, + { scope: 'within-suite-run', label: 'tag:@streaming', limit: Number(process.env.E2E_STREAMING_CONCURRENCY || 12) }, { scope: 'within-suite-run', label: 'tag:@filesystem', limit: 1 }, ], }); diff --git a/packages/playground/e2e/local-app-pool.integration.test.mjs b/packages/playground/e2e/local-app-pool.integration.test.mjs new file mode 100644 index 000000000000..dbfa06ad2d43 --- /dev/null +++ b/packages/playground/e2e/local-app-pool.integration.test.mjs @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { access, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { createLeaseServer, startLocalApps } from './local-app-pool.mjs'; + +test('local native entrypoints isolate database/source writes and expose CLI package metadata', async t => { + const port = 49151; + const pool = await startLocalApps({ + size: 2, + port, + kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), + }); + const server = createLeaseServer(pool.apps); + server.listen(port, '127.0.0.1'); + await once(server, 'listening'); + t.after(async () => { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + await pool.close(); + }); + const url = `http://127.0.0.1:${port}`; + const lease = async id => { + const response = await fetch(`${url}/__e2e/lease`, { method: 'POST', headers: { 'x-mastra-e2e-session': id } }); + assert.equal(response.status, 201); + return (await response.json()).baseURL; + }; + const release = id => fetch(`${url}/__e2e/lease`, { method: 'DELETE', headers: { 'x-mastra-e2e-session': id } }); + const [a, b] = await Promise.all([lease('a'), lease('b')]); + assert.notEqual(a, b); + const metadata = await fetch(`${a}/api/system/packages`).then(r => r.json()); + assert.ok(metadata.packages.some(p => p.name === '@mastra/memory')); + assert.equal(metadata.isDev, true); + const seeds = await Promise.all( + [ + [a, 2], + [b, 3], + ].map(([origin, count]) => + fetch(`${origin}/e2e/seed-thread`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ threadId: 'identical-thread', count }), + }), + ), + ); + assert.ok(seeds.every(r => r.status === 201)); + const messages = origin => + fetch(`${origin}/api/memory/threads/identical-thread/messages?agentId=weather-agent`).then(r => r.json()); + assert.equal((await messages(a)).messages.length, 2); + assert.equal((await messages(b)).messages.length, 3); + const app = pool.apps.find(app => `http://localhost:${app.port}` === a); + const directory = dirname(dirname(dirname(app.child.spawnargs[2]))); + const source = join(directory, 'src/mastra/reset-proof.json'); + await writeFile(source, '{"changed":true}'); + assert.equal((await release('a')).status, 200); + await assert.rejects(access(source)); + assert.equal((await messages(b)).messages.length, 3); + const c = await lease('c'); + assert.equal(c, a); + assert.equal((await fetch(`${c}/api/memory/threads/identical-thread/messages?agentId=weather-agent`)).status, 404); + assert.equal((await release('c')).status, 200); + assert.equal((await release('b')).status, 200); +}); diff --git a/packages/playground/e2e/local-app-pool.mjs b/packages/playground/e2e/local-app-pool.mjs new file mode 100644 index 000000000000..fd73e8254267 --- /dev/null +++ b/packages/playground/e2e/local-app-pool.mjs @@ -0,0 +1,235 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { cp, mkdir, mkdtemp, rm, symlink } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export function createLeaseServer(apps, { leaseMilliseconds = 180_000 } = {}) { + const available = [...apps]; + const sessions = new Map(); + const waiting = []; + const metrics = { leases: 0, maxActive: 0, resets: 0, resetMilliseconds: 0, expired: 0, resetFailures: 0 }; + let closing = false; + const returnApp = app => { + if (closing) return; + const pending = waiting.shift(); + if (pending) pending.allocate(app); + else available.push(app); + }; + const reset = async app => { + const started = Date.now(); + try { + await app.reset(); + metrics.resets++; + metrics.resetMilliseconds += Date.now() - started; + returnApp(app); + } catch (error) { + metrics.resetFailures++; + throw error; + } + }; + const server = createServer(async (req, res) => { + const send = (status, body) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + if (req.url === '/__e2e/health') + return send(200, { instances: apps.length, available: available.length, ...metrics }); + if (req.url !== '/__e2e/lease') return send(404, { error: 'Control server does not forward application traffic' }); + const id = req.headers['x-mastra-e2e-session']; + if (typeof id !== 'string' || !/^[a-zA-Z0-9-]{1,100}$/.test(id)) return send(400, { error: 'Session required' }); + if (req.method === 'POST') { + if (closing) return send(503, { error: 'Pool is closing' }); + if (sessions.has(id)) return send(409, { error: 'Session already leased' }); + const allocate = app => { + sessions.set(id, { app, expires: Date.now() + leaseMilliseconds }); + metrics.leases++; + metrics.maxActive = Math.max(metrics.maxActive, sessions.size); + send(201, { baseURL: `http://localhost:${app.port}` }); + }; + const app = available.shift(); + if (app) allocate(app); + else { + const pending = { allocate }; + waiting.push(pending); + res.on('close', () => { + const index = waiting.indexOf(pending); + if (index >= 0) waiting.splice(index, 1); + }); + } + return; + } + if (req.method === 'DELETE') { + const session = sessions.get(id); + if (!session) return send(404, { error: 'Unknown session' }); + sessions.delete(id); + try { + await reset(session.app); + send(200, { released: true }); + } catch (error) { + send(500, { error: `Application reset failed: ${error.message}` }); + } + return; + } + send(405, { error: 'Method not allowed' }); + }); + const reap = setInterval( + () => { + for (const [id, session] of sessions) { + if (session.expires > Date.now()) continue; + sessions.delete(id); + metrics.expired++; + void reset(session.app).catch(error => console.error(`Expired lease reset failed: ${error.message}`)); + } + }, + Math.min(1000, leaseMilliseconds), + ); + reap.unref(); + server.on('close', () => { + closing = true; + clearInterval(reap); + }); + return Object.assign(server, { isolationMetrics: metrics }); +} + +async function stop(child, group = false) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, 'exit'); + const kill = signal => { + try { + if (group) process.kill(-child.pid, signal); + else child.kill(signal); + } catch (error) { + if (error.code !== 'ESRCH') throw error; + } + }; + kill('SIGTERM'); + const timeout = setTimeout(() => kill('SIGKILL'), 5000); + try { + await exited; + } finally { + clearTimeout(timeout); + } +} + +async function ready(port, child) { + for (let attempt = 0; attempt < 600; attempt++) { + if (child.exitCode !== null || child.signalCode !== null) throw new Error(`Application on :${port} exited`); + try { + const response = await fetch(`http://127.0.0.1:${port}/health`, { signal: AbortSignal.timeout(1000) }); + if (response.ok) return; + } catch {} + await new Promise(resolve => setTimeout(resolve, 100)); + } + throw new Error(`Application on :${port} did not become ready`); +} + +export async function startLocalApps({ size, port = 4111, kitchenSink }) { + const root = await mkdtemp(join(tmpdir(), 'mastra-local-pool-')); + const apps = []; + let template; + try { + // Generate exactly the entrypoint and package metadata used by native CI. + template = spawn('pnpm', ['dev'], { + cwd: kitchenSink, + detached: true, + env: { ...process.env, PORT: String(port + 100) }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + await ready(port + 100, template); + await stop(template, true); + template = undefined; + const generated = join(kitchenSink, '.mastra/output'); + const prepare = async index => { + const directory = join(root, String(index)); + const output = join(directory, '.mastra/output'); + await mkdir(directory, { recursive: true }); + await cp(generated, output, { + recursive: true, + filter: source => source !== join(generated, 'studio') && source !== join(generated, 'public'), + }); + // Immutable Studio assets are shared locally, never uploaded to test VMs. + await symlink(join(generated, 'studio'), join(output, 'studio')); + await symlink(join(kitchenSink, 'node_modules'), join(directory, 'node_modules')); + await cp(join(kitchenSink, 'package.json'), join(directory, 'package.json')); + const packagesFile = join(directory, '.mastra/mastra-packages.json'); + await cp(join(kitchenSink, '.mastra/mastra-packages.json'), packagesFile); + const app = { port: port + 1000 + index, child: undefined }; + const publicDir = join(directory, 'src/mastra/public'); + app.reset = async () => { + await stop(app.child); + await rm(join(directory, 'src'), { recursive: true, force: true }); + await cp(join(kitchenSink, 'src'), join(directory, 'src'), { + recursive: true, + filter: source => source !== join(kitchenSink, 'src/mastra/public'), + }); + await mkdir(publicDir, { recursive: true }); + app.child = spawn( + process.execPath, + [join(dirname(fileURLToPath(import.meta.url)), 'local-app.mjs'), join(output, 'index.mjs')], + { + cwd: publicDir, + env: { + ...process.env, + NODE_ENV: 'production', + MASTRA_DEV: 'true', + PORT: String(app.port), + MASTRA_PROJECT_ROOT: join(directory, '.mastra'), + MASTRA_PACKAGES_FILE: packagesFile, + MASTRA_TELEMETRY_COMMAND: 'dev', + E2E_CONTROL_PORT: String(port), + }, + stdio: ['ignore', 'inherit', 'inherit'], + }, + ); + await ready(app.port, app.child); + }; + apps.push(app); + await app.reset(); + }; + // Avoid starting every Node process at once on a small CI host. + for (let index = 0; index < size; index += 4) { + await Promise.all(Array.from({ length: Math.min(4, size - index) }, (_, offset) => prepare(index + offset))); + } + return { + apps, + close: async () => { + await Promise.all(apps.map(app => stop(app.child))); + await rm(root, { recursive: true, force: true }); + }, + }; + } catch (error) { + await stop(template, true); + await Promise.all(apps.map(app => stop(app.child))); + await rm(root, { recursive: true, force: true }); + throw error; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const port = Number(process.env.E2E_PORT || 4111); + const size = Number(process.env.E2E_APP_POOL_SIZE || 8); + if (!Number.isInteger(size) || size < 1 || size > 100) throw new Error('E2E_APP_POOL_SIZE must be 1–100'); + const pool = await startLocalApps({ + size, + port, + kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), + }); + const leases = createLeaseServer(pool.apps); + await new Promise(resolve => leases.listen(port, '127.0.0.1', resolve)); + console.log(`Local applications ready: ${size}, control port ${port}`); + let stopping = false; + const shutdown = async () => { + if (stopping) return; + stopping = true; + console.log(`Local application metrics: ${JSON.stringify(leases.isolationMetrics)}`); + leases.closeAllConnections(); + await new Promise(resolve => leases.close(resolve)); + await pool.close(); + process.exit(0); + }; + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); +} diff --git a/packages/playground/e2e/local-app-pool.test.mjs b/packages/playground/e2e/local-app-pool.test.mjs new file mode 100644 index 000000000000..6629bec2740c --- /dev/null +++ b/packages/playground/e2e/local-app-pool.test.mjs @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { test } from 'node:test'; +import { createLeaseServer } from './local-app-pool.mjs'; + +test('leases distinct ports and queues reuse until a failed attempt has been reset', async t => { + let finishReset; + const reset = new Promise(resolve => { + finishReset = resolve; + }); + const server = createLeaseServer([ + { port: 5111, reset: async () => reset }, + { port: 5112, reset: async () => {} }, + ]); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const url = `http://127.0.0.1:${server.address().port}`; + const request = (id, method) => fetch(`${url}/__e2e/lease`, { method, headers: { 'x-mastra-e2e-session': id } }); + assert.equal((await fetch(`${url}/api/agents`)).status, 404); + assert.equal((await request('a', 'POST')).status, 201); + const b = await request('b', 'POST'); + assert.equal((await b.json()).baseURL, 'http://localhost:5112'); + assert.equal((await request('a', 'POST')).status, 409); + let allocated = false; + const waiting = request('c', 'POST').then(response => { + allocated = true; + return response; + }); + const released = request('a', 'DELETE'); + await new Promise(resolve => setTimeout(resolve, 30)); + assert.equal(allocated, false); + finishReset(); + assert.equal((await released).status, 200); + assert.equal((await (await waiting).json()).baseURL, 'http://localhost:5111'); + assert.equal((await request('missing', 'DELETE')).status, 404); + assert.equal(server.isolationMetrics.maxActive, 2); +}); diff --git a/packages/playground/e2e/local-app.mjs b/packages/playground/e2e/local-app.mjs new file mode 100644 index 000000000000..366578ae64c2 --- /dev/null +++ b/packages/playground/e2e/local-app.mjs @@ -0,0 +1,15 @@ +import { pathToFileURL } from 'node:url'; + +// The existing MCP journey enters localhost:4111 in Studio. Server-side calls +// to that origin belong to this isolated application, not the pool controller. +const originalFetch = globalThis.fetch; +globalThis.fetch = (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.hostname === 'localhost' && url.port === process.env.E2E_CONTROL_PORT) { + url.port = process.env.PORT; + return originalFetch(input instanceof Request ? new Request(url, input) : url, init); + } + return originalFetch(input, init); +}; + +await import(pathToFileURL(process.argv[2]).href); diff --git a/packages/playground/e2e/playwright.config.ts b/packages/playground/e2e/playwright.config.ts index 73a6711cb222..e39c71c0e15f 100644 --- a/packages/playground/e2e/playwright.config.ts +++ b/packages/playground/e2e/playwright.config.ts @@ -44,5 +44,8 @@ export default defineConfig({ }, ], - webServer: process.env.E2E_REMOTE_APP === 'true' ? undefined : webservers, + webServer: + process.env.E2E_LOCAL_APP === 'true' + ? [{ command: 'node ./local-app-pool.mjs', url: `${BASE_URL}/__e2e/health`, timeout: 180_000 }] + : webservers, }); diff --git a/packages/playground/e2e/prepare-remote-app.mjs b/packages/playground/e2e/prepare-remote-app.mjs deleted file mode 100644 index 40cbc3988771..000000000000 --- a/packages/playground/e2e/prepare-remote-app.mjs +++ /dev/null @@ -1,45 +0,0 @@ -import { spawn } from 'node:child_process'; -import { once } from 'node:events'; -import { fileURLToPath } from 'node:url'; - -// Generate Studio with the supported dev command once before uploading tests. -// The server is stopped here; applications run beside the remote browsers. -const child = spawn('pnpm', ['dev'], { - cwd: fileURLToPath(new URL('./kitchen-sink', import.meta.url)), - detached: true, - env: { ...process.env, PORT: '4211' }, - stdio: ['ignore', 'inherit', 'inherit'], -}); - -try { - let ready = false; - for (let attempt = 0; attempt < 600; attempt++) { - if (child.exitCode !== null || child.signalCode !== null) throw new Error('Studio preparation server exited'); - try { - const response = await fetch('http://127.0.0.1:4211/health', { signal: AbortSignal.timeout(1000) }); - if (response.ok) { - ready = true; - break; - } - } catch {} - await new Promise(resolve => setTimeout(resolve, 100)); - } - if (!ready) throw new Error('Studio preparation server did not become ready'); -} finally { - if (child.exitCode === null && child.signalCode === null) { - const exited = once(child, 'exit'); - process.kill(-child.pid, 'SIGTERM'); - const timeout = setTimeout(() => { - try { - process.kill(-child.pid, 'SIGKILL'); - } catch (error) { - if (error.code !== 'ESRCH') throw error; - } - }, 5000); - try { - await exited; - } finally { - clearTimeout(timeout); - } - } -} diff --git a/packages/playground/e2e/tests/__utils__/local-test.ts b/packages/playground/e2e/tests/__utils__/local-test.ts new file mode 100644 index 000000000000..4987c0bb304b --- /dev/null +++ b/packages/playground/e2e/tests/__utils__/local-test.ts @@ -0,0 +1,45 @@ +import { randomUUID } from 'node:crypto'; +import { test as base } from '@playwright/test'; + +const CONTROL_URL = `http://localhost:${process.env.E2E_PORT || '4111'}`; +const APP_ANNOTATION = 'mastra-e2e-app-url'; +const originalFetch = globalThis.fetch; + +// Preserve existing helpers that seed/reset the configured localhost origin. +// Each attempt redirects those calls to its assigned local application port. +globalThis.fetch = (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin !== CONTROL_URL || url.pathname.startsWith('/__e2e/')) return originalFetch(input, init); + const target = base.info().annotations.find(annotation => annotation.type === APP_ANNOTATION)?.description; + if (!target) throw new Error('Kitchen-sink request made without an application lease'); + const redirect = new URL(url.pathname + url.search, target); + return originalFetch(input instanceof Request ? new Request(redirect, input) : redirect, init); +}; + +export const localTest = base.extend<{ appLease: string }>({ + appLease: [ + async ({ browser }, use, testInfo) => { + const headers = { 'x-mastra-e2e-session': randomUUID() }; + const lease = await originalFetch(`${CONTROL_URL}/__e2e/lease`, { + method: 'POST', + headers, + signal: AbortSignal.timeout(55_000), + }); + if (!lease.ok) throw new Error(`Application lease failed: ${lease.status}`); + const { baseURL } = (await lease.json()) as { baseURL: string }; + testInfo.annotations.push({ type: APP_ANNOTATION, description: baseURL }); + const newContext = browser.newContext.bind(browser); + // Streaming and IME journeys also create browser contexts explicitly. + browser.newContext = options => newContext({ ...options, baseURL }); + try { + await use(baseURL); + } finally { + browser.newContext = newContext; + const released = await originalFetch(`${CONTROL_URL}/__e2e/lease`, { method: 'DELETE', headers }); + if (!released.ok) throw new Error(`Application reset failed: ${released.status}`); + } + }, + { auto: true, timeout: 60_000 }, + ], + baseURL: async ({ appLease }, use) => use(appLease), +}); diff --git a/packages/playground/e2e/tests/__utils__/remote-test.ts b/packages/playground/e2e/tests/__utils__/remote-test.ts deleted file mode 100644 index 9901afe6c14d..000000000000 --- a/packages/playground/e2e/tests/__utils__/remote-test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { cp, mkdir, mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { test as base } from '@playwright/test'; -import { createNodeServer } from '../../../../deployer/dist/server/index.js'; - -// Endform gives each test its own remote runner. Start the existing kitchen-sink -// application there, so resets and source writes cannot affect other tests. -export const remoteTest = base.extend<{ isolatedApplication: void }>({ - isolatedApplication: [ - async ({}, use) => { - const oldDirectory = process.cwd(); - const directory = await mkdtemp(join(tmpdir(), 'mastra-remote-test-')); - const environment = { - MASTRA_DEV: process.env.MASTRA_DEV, - MASTRA_STUDIO_PATH: process.env.MASTRA_STUDIO_PATH, - MASTRA_PROJECT_ROOT: process.env.MASTRA_PROJECT_ROOT, - PORT: process.env.PORT, - }; - let server: Awaited> | undefined; - let shutdown: (() => Promise) | undefined; - try { - process.chdir(directory); - process.env.MASTRA_DEV = 'true'; - process.env.PORT = process.env.E2E_PORT || '4111'; - process.env.MASTRA_STUDIO_PATH = fileURLToPath( - new URL('../../kitchen-sink/.mastra/output/studio', import.meta.url), - ); - await mkdir(join(directory, '.mastra'), { recursive: true }); - await cp(fileURLToPath(new URL('../../kitchen-sink/src', import.meta.url)), join(directory, 'src'), { - recursive: true, - }); - process.env.MASTRA_PROJECT_ROOT = join(directory, '.mastra'); - const { mastra } = await import('../../kitchen-sink/src/mastra/index'); - const tools = await import('../../kitchen-sink/src/mastra/tools'); - shutdown = () => mastra.shutdown(); - // Mastra dev emits a CLI readiness message over process.send. This process - // belongs to Playwright, so keep that message out of its worker IPC protocol. - const send = process.send; - if (send) { - process.send = ((message, ...args) => { - if (typeof message === 'object' && message !== null && 'type' in message && message.type === 'server-ready') - return true; - return Reflect.apply(send, process, [message, ...args]); - }) as typeof process.send; - } - try { - // The server's public type erases tool input/output generics; these - // are the same schema-backed exports passed by the development CLI. - server = await createNodeServer(mastra, { - studio: true, - isDev: true, - tools, - } as NonNullable[1]>); - // createNodeServer resolves before its listen callback emits readiness. - // Keep the IPC guard until a health request proves that callback ran. - const ready = await fetch(`http://localhost:${process.env.PORT}/health`); - if (!ready.ok) throw new Error(`Kitchen-sink health check failed: ${ready.status}`); - } finally { - process.send = send; - } - await use(); - } finally { - try { - if (server) { - if ('closeAllConnections' in server) server.closeAllConnections(); - const runningServer = server; - await new Promise((resolve, reject) => - runningServer.close(error => (error ? reject(error) : resolve())), - ); - } - await shutdown?.(); - } finally { - for (const [name, value] of Object.entries(environment)) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - process.chdir(oldDirectory); - await rm(directory, { recursive: true, force: true }); - } - } - }, - // Infrastructure setup/teardown has its own budget; existing journeys keep - // their original test and assertion timeouts. - { auto: true, timeout: 60_000 }, - ], -}); diff --git a/packages/playground/e2e/tests/__utils__/test.ts b/packages/playground/e2e/tests/__utils__/test.ts index 80fdfd7cb1c5..302cf8611524 100644 --- a/packages/playground/e2e/tests/__utils__/test.ts +++ b/packages/playground/e2e/tests/__utils__/test.ts @@ -1,7 +1,7 @@ import { test as base } from '@playwright/test'; -// Native Playwright keeps its existing dev server. Endform starts an isolated -// application in the remote test process, beside its browser. -export const test = process.env.E2E_REMOTE_APP === 'true' ? (await import('./remote-test')).remoteTest : base; +// Native Playwright keeps its existing dev server; Endform leases a local app +// and proxies directly to that app's port through the CLI. +export const test = process.env.E2E_LOCAL_APP === 'true' ? (await import('./local-test')).localTest : base; export { expect } from '@playwright/test'; diff --git a/packages/playground/package.json b/packages/playground/package.json index d215f5f32cb0..e3ec0c2f7439 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -38,7 +38,7 @@ "test:mutate": "stryker run --mutate", "test:run": "vitest run", "test:e2e": "pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile && pnpm test:e2e:endform", - "test:e2e:endform": "cd e2e && node prepare-remote-app.mjs && E2E_REMOTE_APP=true npx endform@latest test --organization-id 2G1ZCj7X", + "test:e2e:endform": "cd e2e && E2E_LOCAL_APP=true npx endform@latest test --organization-id 2G1ZCj7X", "test:e2e:playwright": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts && playwright test -c e2e/playwright.studio-base.config.ts", "test:e2e:studio-base": "pnpm test:e2e:setup && playwright test -c e2e/playwright.studio-base.config.ts", "test:e2e:ui": "pnpm test:e2e:setup && playwright test -c e2e/playwright.config.ts --ui", From fee14a1aafcf413201cd35e08b8d05c705fa7ee9 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Tue, 6 Oct 2026 12:48:52 -0700 Subject: [PATCH 8/8] test(playground): compare warm local apps with HTTP-only Endform proxying --- .github/workflows/fork-studio-browser.yml | 4 ++ packages/playground/e2e/README.md | 25 +++++--- packages/playground/e2e/endform.config.ts | 6 +- .../e2e/local-app-pool.integration.test.mjs | 38 ++++++++++++ packages/playground/e2e/local-app-pool.mjs | 61 +++++++++++++------ .../e2e/tests/__utils__/local-test.ts | 9 ++- .../e2e/tests/__utils__/reset-storage.ts | 5 +- .../e2e/tests/__utils__/seed-dataset-items.ts | 7 +-- .../e2e/tests/__utils__/seed-datasets.ts | 5 +- .../e2e/tests/__utils__/seed-thread.ts | 5 +- .../e2e/tests/__utils__/test-origin.ts | 10 +++ 11 files changed, 132 insertions(+), 43 deletions(-) create mode 100644 packages/playground/e2e/tests/__utils__/test-origin.ts diff --git a/.github/workflows/fork-studio-browser.yml b/.github/workflows/fork-studio-browser.yml index c82d35a7a05f..fa145e12ee88 100644 --- a/.github/workflows/fork-studio-browser.yml +++ b/.github/workflows/fork-studio-browser.yml @@ -14,6 +14,9 @@ on: streaming_concurrency: description: 'Endform streaming test limit' default: '12' + warm_apps: + description: 'Reuse successful ordinary applications with storage resets' + default: 'true' permissions: {} @@ -30,6 +33,7 @@ jobs: E2E_APP_POOL_SIZE: ${{ inputs.app_pool_size || '8' }} E2E_CONCURRENCY: ${{ inputs.concurrency || '8' }} E2E_STREAMING_CONCURRENCY: ${{ inputs.streaming_concurrency || '12' }} + E2E_WARM_APPS: ${{ inputs.warm_apps || 'true' }} steps: - name: Checkout repo diff --git a/packages/playground/e2e/README.md b/packages/playground/e2e/README.md index 8125eb2a457f..bc0fcaac76bc 100644 --- a/packages/playground/e2e/README.md +++ b/packages/playground/e2e/README.md @@ -28,26 +28,29 @@ through Endform's proxy to the application's port, without a gateway forwarding hop. Explicit browser contexts in streaming/IME journeys use the same slot. The existing MCP server call to localhost:4111 is mapped back to its own slot. -After an attempt, the application process stops, its database and source tree -are recreated, and it restarts before reuse. A failed reset removes the slot -from circulation. A crashed remote attempt's lease expires after three minutes. +With `E2E_WARM_APPS=true`, successful ordinary tests return their slot after +running the native storage-reset endpoint. The application stays warm, matching +the native suite's reuse between tests. Failed attempts, filesystem cases, +workflow journeys, agent-builder journeys and CMS agent journeys stop the process, +restore its source/database, and restart before reuse. `E2E_WARM_APPS=false` +restarts every attempt. A failed reset removes the slot from circulation. A +crashed remote attempt's lease expires after three minutes and is restarted. The fixture has a separate 60-second infrastructure budget; assertions, journey timeouts, skip declarations and retries remain unchanged. Tune host capacity and scheduling together: ```sh -E2E_APP_POOL_SIZE=16 E2E_CONCURRENCY=16 E2E_STREAMING_CONCURRENCY=12 \ +E2E_WARM_APPS=true E2E_APP_POOL_SIZE=16 E2E_CONCURRENCY=16 E2E_STREAMING_CONCURRENCY=12 \ pnpm --filter @internal/playground test:e2e:endform ``` Defaults are eight applications, eight concurrent tests and a streaming cap of -12 (which has no additional effect when the total limit is eight). Increasing -Endform concurrency beyond local application capacity queues leases rather than -creating more applications. All matching native Endform concurrency limits apply. +12 (which has no additional effect when the total limit is eight). The effective Endform limit is capped at local application capacity; increasing +the requested concurrency does not create additional applications. All matching native Endform concurrency limits apply. The single filesystem case remains tagged with limit one; it adds no restriction beyond that case being the only one with the tag. CI's manual workflow exposes -application, total and streaming limits as inputs for measured experiments. +application, total and streaming limits, plus warm reuse, as inputs for measured experiments. The native runner remains available as `test:e2e:playwright`, with its original shared development server and one worker, including the separate Studio @@ -67,3 +70,9 @@ and concurrency together. Local Mac results are not an equivalent benchmark to four-core GitHub-hosted CI. Every full-suite run must account for all 335 cases, including the existing skips; a focused passing probe is not suite success. GitHub CI authenticates with job-scoped OIDC; no API key is added. + +Proxy transport and asset-cache probes did not establish an improvement. The +final setup uses HTTP host interception without an additional raw-port tunnel or +remote Studio cache. Failed experimental variants are recorded with the fork PR. +Completed proxy runs have not met the sub-five-minute target; performance and +reliability require further work rather than weakening assertions or retries. diff --git a/packages/playground/e2e/endform.config.ts b/packages/playground/e2e/endform.config.ts index 85cba6531a13..9a476b900167 100644 --- a/packages/playground/e2e/endform.config.ts +++ b/packages/playground/e2e/endform.config.ts @@ -1,15 +1,13 @@ import { defineEndformConfig } from 'endform'; -const port = Number(process.env.E2E_PORT || 4111); const capacity = Number(process.env.E2E_APP_POOL_SIZE || 8); export default defineEndformConfig({ additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json'], - // Applications stay on the CLI host; only browser/test code is distributed. + // Browser, Node fetch, SSE and WebSocket traffic returns to the CLI host. proxyNetworkHosts: [''], - proxyNetworkPorts: [port, ...Array.from({ length: capacity }, (_, index) => port + 1000 + index)], concurrentTestLimits: [ - { scope: 'within-suite-run', limit: Number(process.env.E2E_CONCURRENCY || capacity) }, + { scope: 'within-suite-run', limit: Math.min(Number(process.env.E2E_CONCURRENCY || capacity), capacity) }, { scope: 'within-suite-run', label: 'tag:@streaming', limit: Number(process.env.E2E_STREAMING_CONCURRENCY || 12) }, { scope: 'within-suite-run', label: 'tag:@filesystem', limit: 1 }, ], diff --git a/packages/playground/e2e/local-app-pool.integration.test.mjs b/packages/playground/e2e/local-app-pool.integration.test.mjs index dbfa06ad2d43..8563599885ba 100644 --- a/packages/playground/e2e/local-app-pool.integration.test.mjs +++ b/packages/playground/e2e/local-app-pool.integration.test.mjs @@ -63,3 +63,41 @@ test('local native entrypoints isolate database/source writes and expose CLI pac assert.equal((await release('c')).status, 200); assert.equal((await release('b')).status, 200); }); + +test('warm reuse clears its database without restarting, while a requested restart restores source', async t => { + const previous = process.env.E2E_WARM_APPS; + process.env.E2E_WARM_APPS = 'true'; + const pool = await startLocalApps({ + size: 1, + port: 49251, + generate: false, + kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'), + }); + t.after(async () => { + await pool.close(); + if (previous === undefined) delete process.env.E2E_WARM_APPS; + else process.env.E2E_WARM_APPS = previous; + }); + const app = pool.apps[0]; + const pid = app.child.pid; + const origin = `http://localhost:${app.port}`; + assert.equal( + ( + await fetch(`${origin}/e2e/seed-thread`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ threadId: 'warm-proof', count: 2 }), + }) + ).status, + 201, + ); + await app.reset(false); + assert.equal(app.child.pid, pid); + assert.equal((await fetch(`${origin}/api/memory/threads/warm-proof/messages?agentId=weather-agent`)).status, 404); + const directory = dirname(dirname(dirname(app.child.spawnargs[2]))); + const source = join(directory, 'src/mastra/reset-proof.json'); + await writeFile(source, '{"changed":true}'); + await app.reset(true); + assert.notEqual(app.child.pid, pid); + await assert.rejects(access(source)); +}); diff --git a/packages/playground/e2e/local-app-pool.mjs b/packages/playground/e2e/local-app-pool.mjs index fd73e8254267..ce3e3dff7f3a 100644 --- a/packages/playground/e2e/local-app-pool.mjs +++ b/packages/playground/e2e/local-app-pool.mjs @@ -10,7 +10,16 @@ export function createLeaseServer(apps, { leaseMilliseconds = 180_000 } = {}) { const available = [...apps]; const sessions = new Map(); const waiting = []; - const metrics = { leases: 0, maxActive: 0, resets: 0, resetMilliseconds: 0, expired: 0, resetFailures: 0 }; + const metrics = { + leases: 0, + maxActive: 0, + resets: 0, + resetMilliseconds: 0, + expired: 0, + resetFailures: 0, + warmResets: 0, + restartResets: 0, + }; let closing = false; const returnApp = app => { if (closing) return; @@ -18,10 +27,12 @@ export function createLeaseServer(apps, { leaseMilliseconds = 180_000 } = {}) { if (pending) pending.allocate(app); else available.push(app); }; - const reset = async app => { + const reset = async (app, forceRestart = true) => { const started = Date.now(); try { - await app.reset(); + const kind = await app.reset(forceRestart); + if (kind === 'warm') metrics.warmResets++; + else metrics.restartResets++; metrics.resets++; metrics.resetMilliseconds += Date.now() - started; returnApp(app); @@ -66,7 +77,7 @@ export function createLeaseServer(apps, { leaseMilliseconds = 180_000 } = {}) { if (!session) return send(404, { error: 'Unknown session' }); sessions.delete(id); try { - await reset(session.app); + await reset(session.app, req.headers['x-mastra-e2e-restart'] !== 'false'); send(200, { released: true }); } catch (error) { send(500, { error: `Application reset failed: ${error.message}` }); @@ -126,21 +137,23 @@ async function ready(port, child) { throw new Error(`Application on :${port} did not become ready`); } -export async function startLocalApps({ size, port = 4111, kitchenSink }) { +export async function startLocalApps({ size, port = 4111, kitchenSink, generate = true }) { const root = await mkdtemp(join(tmpdir(), 'mastra-local-pool-')); const apps = []; let template; try { // Generate exactly the entrypoint and package metadata used by native CI. - template = spawn('pnpm', ['dev'], { - cwd: kitchenSink, - detached: true, - env: { ...process.env, PORT: String(port + 100) }, - stdio: ['ignore', 'inherit', 'inherit'], - }); - await ready(port + 100, template); - await stop(template, true); - template = undefined; + if (generate) { + template = spawn('pnpm', ['dev'], { + cwd: kitchenSink, + detached: true, + env: { ...process.env, PORT: String(port + 100) }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + await ready(port + 100, template); + await stop(template, true); + template = undefined; + } const generated = join(kitchenSink, '.mastra/output'); const prepare = async index => { const directory = join(root, String(index)); @@ -158,7 +171,20 @@ export async function startLocalApps({ size, port = 4111, kitchenSink }) { await cp(join(kitchenSink, '.mastra/mastra-packages.json'), packagesFile); const app = { port: port + 1000 + index, child: undefined }; const publicDir = join(directory, 'src/mastra/public'); - app.reset = async () => { + app.reset = async (forceRestart = true) => { + if ( + process.env.E2E_WARM_APPS === 'true' && + !forceRestart && + app.child?.exitCode === null && + app.child.signalCode === null + ) { + const response = await fetch(`http://127.0.0.1:${app.port}/e2e/reset-storage`, { + method: 'POST', + signal: AbortSignal.timeout(20_000), + }); + if (!response.ok) throw new Error(`Storage reset failed: ${response.status}`); + return 'warm'; + } await stop(app.child); await rm(join(directory, 'src'), { recursive: true, force: true }); await cp(join(kitchenSink, 'src'), join(directory, 'src'), { @@ -185,6 +211,7 @@ export async function startLocalApps({ size, port = 4111, kitchenSink }) { }, ); await ready(app.port, app.child); + return 'restart'; }; apps.push(app); await app.reset(); @@ -219,12 +246,12 @@ if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1] }); const leases = createLeaseServer(pool.apps); await new Promise(resolve => leases.listen(port, '127.0.0.1', resolve)); - console.log(`Local applications ready: ${size}, control port ${port}`); + console.error(`Local applications ready: ${size}, control port ${port}`); let stopping = false; const shutdown = async () => { if (stopping) return; stopping = true; - console.log(`Local application metrics: ${JSON.stringify(leases.isolationMetrics)}`); + console.error(`Local application metrics: ${JSON.stringify(leases.isolationMetrics)}`); leases.closeAllConnections(); await new Promise(resolve => leases.close(resolve)); await pool.close(); diff --git a/packages/playground/e2e/tests/__utils__/local-test.ts b/packages/playground/e2e/tests/__utils__/local-test.ts index 4987c0bb304b..3a052cf91e50 100644 --- a/packages/playground/e2e/tests/__utils__/local-test.ts +++ b/packages/playground/e2e/tests/__utils__/local-test.ts @@ -35,7 +35,14 @@ export const localTest = base.extend<{ appLease: string }>({ await use(baseURL); } finally { browser.newContext = newContext; - const released = await originalFetch(`${CONTROL_URL}/__e2e/lease`, { method: 'DELETE', headers }); + const requiresFreshProcess = + testInfo.status !== testInfo.expectedStatus || + testInfo.tags.includes('@filesystem') || + /\/(workflows|agent-builder|cms\/agents)\//.test(testInfo.file); + const released = await originalFetch(`${CONTROL_URL}/__e2e/lease`, { + method: 'DELETE', + headers: { ...headers, 'x-mastra-e2e-restart': String(requiresFreshProcess) }, + }); if (!released.ok) throw new Error(`Application reset failed: ${released.status}`); } }, diff --git a/packages/playground/e2e/tests/__utils__/reset-storage.ts b/packages/playground/e2e/tests/__utils__/reset-storage.ts index 997b8cce4fc4..70b87da51c89 100644 --- a/packages/playground/e2e/tests/__utils__/reset-storage.ts +++ b/packages/playground/e2e/tests/__utils__/reset-storage.ts @@ -1,8 +1,7 @@ -const PORT = process.env.E2E_PORT || '4111'; -const BASE_URL = `http://localhost:${PORT}`; +import { testOrigin } from './test-origin'; export const resetStorage = async () => { - return fetch(`${BASE_URL}/e2e/reset-storage`, { + return fetch(`${testOrigin()}/e2e/reset-storage`, { method: 'POST', }).then(res => { if (!res.ok) { diff --git a/packages/playground/e2e/tests/__utils__/seed-dataset-items.ts b/packages/playground/e2e/tests/__utils__/seed-dataset-items.ts index f885c12921e9..185b39a77517 100644 --- a/packages/playground/e2e/tests/__utils__/seed-dataset-items.ts +++ b/packages/playground/e2e/tests/__utils__/seed-dataset-items.ts @@ -1,5 +1,4 @@ -const PORT = process.env.E2E_PORT || '4111'; -const BASE_URL = `http://localhost:${PORT}`; +import { testOrigin } from './test-origin'; export interface SeededDataset { id: string; @@ -15,7 +14,7 @@ export const seedDatasetWithItems = async ( itemCount: number, datasetName = 'E2E Items Dataset', ): Promise => { - const datasetRes = await fetch(`${BASE_URL}/api/datasets`, { + const datasetRes = await fetch(`${testOrigin()}/api/datasets`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: datasetName }), @@ -32,7 +31,7 @@ export const seedDatasetWithItems = async ( groundTruth: `Expected output ${i + 1}`, })); - const itemsRes = await fetch(`${BASE_URL}/api/datasets/${dataset.id}/items/batch`, { + const itemsRes = await fetch(`${testOrigin()}/api/datasets/${dataset.id}/items/batch`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ items }), diff --git a/packages/playground/e2e/tests/__utils__/seed-datasets.ts b/packages/playground/e2e/tests/__utils__/seed-datasets.ts index f55d83d1be5d..e453c42f6470 100644 --- a/packages/playground/e2e/tests/__utils__/seed-datasets.ts +++ b/packages/playground/e2e/tests/__utils__/seed-datasets.ts @@ -1,5 +1,4 @@ -const PORT = process.env.E2E_PORT || '4111'; -const BASE_URL = `http://localhost:${PORT}`; +import { testOrigin } from './test-origin'; /** * Seeds datasets via the Studio API. Returns the names created. @@ -10,7 +9,7 @@ export const seedDatasets = async (count: number, namePrefix = 'E2E Dataset'): P const names: string[] = []; for (let i = 0; i < count; i++) { const name = `${namePrefix} ${String(i + 1).padStart(2, '0')}`; - const res = await fetch(`${BASE_URL}/api/datasets`, { + const res = await fetch(`${testOrigin()}/api/datasets`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name }), diff --git a/packages/playground/e2e/tests/__utils__/seed-thread.ts b/packages/playground/e2e/tests/__utils__/seed-thread.ts index 686102ad2c48..28edfc39291a 100644 --- a/packages/playground/e2e/tests/__utils__/seed-thread.ts +++ b/packages/playground/e2e/tests/__utils__/seed-thread.ts @@ -1,9 +1,8 @@ -const PORT = process.env.E2E_PORT || '4111'; -const BASE_URL = `http://localhost:${PORT}`; +import { testOrigin } from './test-origin'; /** Seeds a weather-agent thread with `count` user messages "seed message 0..count-1", oldest first. */ export const seedThread = async (threadId: string, count: number) => { - const res = await fetch(`${BASE_URL}/e2e/seed-thread`, { + const res = await fetch(`${testOrigin()}/e2e/seed-thread`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ threadId, count }), diff --git a/packages/playground/e2e/tests/__utils__/test-origin.ts b/packages/playground/e2e/tests/__utils__/test-origin.ts new file mode 100644 index 000000000000..429986ae17a6 --- /dev/null +++ b/packages/playground/e2e/tests/__utils__/test-origin.ts @@ -0,0 +1,10 @@ +import { test } from '@playwright/test'; + +export function testOrigin() { + if (process.env.E2E_LOCAL_APP === 'true') { + const origin = test.info().annotations.find(annotation => annotation.type === 'mastra-e2e-app-url')?.description; + if (!origin) throw new Error('Kitchen-sink helper called without an application lease'); + return origin; + } + return `http://localhost:${process.env.E2E_PORT || '4111'}`; +}