test(playground): proxy the Studio suite to isolated local apps with Endform

.github/workflows/fork-studio-browser.yml

@@ -4,22 +4,36 @@ on:
   pull_request:
     branches: [main, 'codex/mastra-playwright-baseline']
   workflow_dispatch:
+    inputs:
+      app_pool_size:
+        description: 'Local applications on the CI host'
+        default: '8'
+      concurrency:
+        description: 'Endform concurrent test limit'
+        default: '8'
+      streaming_concurrency:
+        description: 'Endform streaming test limit'
+        default: '12'
+      warm_apps:
+        description: 'Reuse successful ordinary applications with storage resets'
+        default: 'true'
 
 permissions: {}
 
 jobs:
   e2e-kitchen-sink:
-    name: E2E kitchen-sink (${{ matrix.shard }}/${{ strategy.job-total }})
+    name: Endform kitchen-sink
     runs-on: ubuntu-24.04
     timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [1, 2, 3, 4]
     permissions:
       contents: read
+      id-token: write
     env:
       TURBO_CACHE: local:rw
+      E2E_APP_POOL_SIZE: ${{ inputs.app_pool_size || '8' }}
+      E2E_CONCURRENCY: ${{ inputs.concurrency || '8' }}
+      E2E_STREAMING_CONCURRENCY: ${{ inputs.streaming_concurrency || '12' }}
+      E2E_WARM_APPS: ${{ inputs.warm_apps || 'true' }}
 
     steps:
       - name: Checkout repo
@@ -51,17 +65,19 @@ jobs:
 
       - name: Setup e2e project dependencies
         working-directory: ./packages/playground
-        run: pnpm test:e2e:setup
+        run: pnpm install --dir ./e2e/kitchen-sink --frozen-lockfile
+
+      - name: Verify local application isolation
+        run: node --test packages/playground/e2e/local-app-pool.test.mjs packages/playground/e2e/local-app-pool.integration.test.mjs
 
       - name: Test E2E
         working-directory: ./packages/playground
-        run: pnpm exec playwright test -c e2e/playwright.config.ts --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+        run: pnpm test:e2e:endform
 
       - name: Upload test artifacts
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
         with:
-          name: studio-playwright-${{ matrix.shard }}
+          name: studio-endform
           path: packages/playground/test-results/
           if-no-files-found: ignore
-

packages/playground/e2e/README.md

@@ -0,0 +1,78 @@
+# Studio browser tests on Endform
+
+Build the relevant workspace dependencies, then run:
+
+```sh
+pnpm --filter @internal/playground test:e2e
+```
+
+The command installs kitchen-sink dependencies and runs the existing 335 Chromium
+cases once on Endform. All applications run locally on the machine running the
+CLI. Endform provides the remote browsers and proxies HTTP, streaming and
+WebSocket traffic back to the assigned local ports. There is no CI shard matrix.
+
+The local application pool starts the supported `mastra dev` command once to
+produce the native generated entrypoint, Studio assets and package metadata.
+Each pool slot runs that entrypoint in a separate Node process, with its own
+source tree and database working directory. Studio assets are immutable and
+shared locally. Application configuration, Node environment and runtime directory
+layout follow the native CLI startup path. Nothing starts an application inside
+the remote Playwright worker, and Studio/server dependencies are not transferred
+to remote machines.
+
+The automatic fixture leases a slot for the whole attempt, including test hooks.
+Browser and API request contexts use the slot's base URL. Existing Node fetch
+helpers targeting the configured test origin are redirected to the same slot.
+The control server only allocates leases; application traffic goes directly
+through Endform's proxy to the application's port, without a gateway forwarding
+hop. Explicit browser contexts in streaming/IME journeys use the same slot.
+The existing MCP server call to localhost:4111 is mapped back to its own slot.
+
+With `E2E_WARM_APPS=true`, successful ordinary tests return their slot after
+running the native storage-reset endpoint. The application stays warm, matching
+the native suite's reuse between tests. Failed attempts, filesystem cases,
+workflow journeys, agent-builder journeys and CMS agent journeys stop the process,
+restore its source/database, and restart before reuse. `E2E_WARM_APPS=false`
+restarts every attempt. A failed reset removes the slot from circulation. A
+crashed remote attempt's lease expires after three minutes and is restarted.
+The fixture has a separate 60-second infrastructure budget; assertions, journey
+timeouts, skip declarations and retries remain unchanged.
+
+Tune host capacity and scheduling together:
+
+```sh
+E2E_WARM_APPS=true E2E_APP_POOL_SIZE=16 E2E_CONCURRENCY=16 E2E_STREAMING_CONCURRENCY=12 \
+  pnpm --filter @internal/playground test:e2e:endform
+```
+
+Defaults are eight applications, eight concurrent tests and a streaming cap of
+12 (which has no additional effect when the total limit is eight). The effective Endform limit is capped at local application capacity; increasing
+the requested concurrency does not create additional applications. All matching native Endform concurrency limits apply.
+The single filesystem case remains tagged with limit one; it adds no restriction
+beyond that case being the only one with the tag. CI's manual workflow exposes
+application, total and streaming limits, plus warm reuse, as inputs for measured experiments.
+
+The native runner remains available as `test:e2e:playwright`, with its original
+shared development server and one worker, including the separate Studio
+base-path test outside this benchmark. Native UI/debugging scripts are preserved.
+The common fixture chooses local leases only when `E2E_LOCAL_APP=true`, set by
+the Endform script. Remote fixture alias imports use the explicit playground
+tsconfig; runtime ARIA snapshots and tsconfig are transferred separately.
+
+Check real state isolation before browser experiments:
+
+```sh
+node --test local-app-pool.test.mjs local-app-pool.integration.test.mjs
+```
+
+Report application preparation, test-stage wall time, outcomes, host CPU/memory
+and concurrency together. Local Mac results are not an equivalent benchmark to
+four-core GitHub-hosted CI. Every full-suite run must account for all 335 cases,
+including the existing skips; a focused passing probe is not suite success.
+GitHub CI authenticates with job-scoped OIDC; no API key is added.
+
+Proxy transport and asset-cache probes did not establish an improvement. The
+final setup uses HTTP host interception without an additional raw-port tunnel or
+remote Studio cache. Failed experimental variants are recorded with the fork PR.
+Completed proxy runs have not met the sub-five-minute target; performance and
+reliability require further work rather than weakening assertions or retries.

packages/playground/e2e/endform.config.ts

@@ -0,0 +1,14 @@
+import { defineEndformConfig } from 'endform';
+
+const capacity = Number(process.env.E2E_APP_POOL_SIZE || 8);
+
+export default defineEndformConfig({
+  additionalFiles: ['tests/**/*.aria.yml', '../tsconfig.json'],
+  // Browser, Node fetch, SSE and WebSocket traffic returns to the CLI host.
+  proxyNetworkHosts: ['<loopback>'],
+  concurrentTestLimits: [
+    { scope: 'within-suite-run', limit: Math.min(Number(process.env.E2E_CONCURRENCY || capacity), capacity) },
+    { scope: 'within-suite-run', label: 'tag:@streaming', limit: Number(process.env.E2E_STREAMING_CONCURRENCY || 12) },
+    { scope: 'within-suite-run', label: 'tag:@filesystem', limit: 1 },
+  ],
+});

packages/playground/e2e/local-app-pool.integration.test.mjs

@@ -0,0 +1,103 @@
+import assert from 'node:assert/strict';
+import { once } from 'node:events';
+import { access, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+import { createLeaseServer, startLocalApps } from './local-app-pool.mjs';
+
+test('local native entrypoints isolate database/source writes and expose CLI package metadata', async t => {
+  const port = 49151;
+  const pool = await startLocalApps({
+    size: 2,
+    port,
+    kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'),
+  });
+  const server = createLeaseServer(pool.apps);
+  server.listen(port, '127.0.0.1');
+  await once(server, 'listening');
+  t.after(async () => {
+    server.closeAllConnections();
+    await new Promise(resolve => server.close(resolve));
+    await pool.close();
+  });
+  const url = `http://127.0.0.1:${port}`;
+  const lease = async id => {
+    const response = await fetch(`${url}/__e2e/lease`, { method: 'POST', headers: { 'x-mastra-e2e-session': id } });
+    assert.equal(response.status, 201);
+    return (await response.json()).baseURL;
+  };
+  const release = id => fetch(`${url}/__e2e/lease`, { method: 'DELETE', headers: { 'x-mastra-e2e-session': id } });
+  const [a, b] = await Promise.all([lease('a'), lease('b')]);
+  assert.notEqual(a, b);
+  const metadata = await fetch(`${a}/api/system/packages`).then(r => r.json());
+  assert.ok(metadata.packages.some(p => p.name === '@mastra/memory'));
+  assert.equal(metadata.isDev, true);
+  const seeds = await Promise.all(
+    [
+      [a, 2],
+      [b, 3],
+    ].map(([origin, count]) =>
+      fetch(`${origin}/e2e/seed-thread`, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ threadId: 'identical-thread', count }),
+      }),
+    ),
+  );
+  assert.ok(seeds.every(r => r.status === 201));
+  const messages = origin =>
+    fetch(`${origin}/api/memory/threads/identical-thread/messages?agentId=weather-agent`).then(r => r.json());
+  assert.equal((await messages(a)).messages.length, 2);
+  assert.equal((await messages(b)).messages.length, 3);
+  const app = pool.apps.find(app => `http://localhost:${app.port}` === a);
+  const directory = dirname(dirname(dirname(app.child.spawnargs[2])));
+  const source = join(directory, 'src/mastra/reset-proof.json');
+  await writeFile(source, '{"changed":true}');
+  assert.equal((await release('a')).status, 200);
+  await assert.rejects(access(source));
+  assert.equal((await messages(b)).messages.length, 3);
+  const c = await lease('c');
+  assert.equal(c, a);
+  assert.equal((await fetch(`${c}/api/memory/threads/identical-thread/messages?agentId=weather-agent`)).status, 404);
+  assert.equal((await release('c')).status, 200);
+  assert.equal((await release('b')).status, 200);
+});
+
+test('warm reuse clears its database without restarting, while a requested restart restores source', async t => {
+  const previous = process.env.E2E_WARM_APPS;
+  process.env.E2E_WARM_APPS = 'true';
+  const pool = await startLocalApps({
+    size: 1,
+    port: 49251,
+    generate: false,
+    kitchenSink: join(dirname(fileURLToPath(import.meta.url)), 'kitchen-sink'),
+  });
+  t.after(async () => {
+    await pool.close();
+    if (previous === undefined) delete process.env.E2E_WARM_APPS;
+    else process.env.E2E_WARM_APPS = previous;
+  });
+  const app = pool.apps[0];
+  const pid = app.child.pid;
+  const origin = `http://localhost:${app.port}`;
+  assert.equal(
+    (
+      await fetch(`${origin}/e2e/seed-thread`, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ threadId: 'warm-proof', count: 2 }),
+      })
+    ).status,
+    201,
+  );
+  await app.reset(false);
+  assert.equal(app.child.pid, pid);
+  assert.equal((await fetch(`${origin}/api/memory/threads/warm-proof/messages?agentId=weather-agent`)).status, 404);
+  const directory = dirname(dirname(dirname(app.child.spawnargs[2])));
+  const source = join(directory, 'src/mastra/reset-proof.json');
+  await writeFile(source, '{"changed":true}');
+  await app.reset(true);
+  assert.notEqual(app.child.pid, pid);
+  await assert.rejects(access(source));
+});