feat(dev): set up the Cloud Agent sandbox for pnpm dev:start

.kilo/skills/cloud-agent-sandbox/SKILL.md

@@ -0,0 +1,207 @@
+---
+name: cloud-agent-sandbox
+description: Sets up and runs this monorepo inside a memory-constrained Kilo Cloud Agent sandbox with `.kilo/cloud-agent-setup.sh`. Use when working in a Cloud Agent sandbox, before starting local services there, after a sandbox restart, or when debugging the dev memory cap, Docker, DNS, fake login, agent-browser, or fake-LLM Cloud Agent sessions in the sandbox.
+---
+
+# Cloud Agent sandbox
+
+`.kilo/cloud-agent-setup.sh` prepares a Debian/Ubuntu Cloud Agent sandbox for
+`pnpm dev:start`. It usually runs automatically when the machine starts. You can
+run it at any time: reruns are safe and take about 20 s once the machine is set up.
+
+Follow the `local-development` skill for ports, fake login, and service
+management. This skill covers what is different in the sandbox.
+
+## Is setup done?
+
+Sandbox restarts wipe Docker, the pnpm wrapper, the memory cgroup, and
+`.wrangler/kilo-startup/`. The repository and `node_modules` survive. Check
+before starting services:
+
+```bash
+test -f .wrangler/kilo-startup/env \
+  && grep -q kilo-cloud-agent-pnpm-wrapper "$(command -v pnpm)" \
+  && docker info >/dev/null 2>&1 && echo ready
+```
+
+If this does not print `ready`, run setup from the repository root:
+
+```bash
+bash .kilo/cloud-agent-setup.sh
+```
+
+A restarted machine takes about 6 minutes, mostly building the Cloud Agent
+sandbox images; `node_modules` survives the restart. A rerun on a running
+machine takes under 2 minutes, because images come from the build cache. A
+machine without dependencies adds about 3 minutes for `pnpm install`. Setup stops at the first failure and prints
+the line number.
+
+## What setup does
+
+- Creates a memory cgroup for all dev workloads, capped at the sandbox memory
+  minus 2 GiB. Override the cap in MiB with `KILO_STARTUP_MEMORY_MB`; the
+  minimum is 3072.
+- Installs Docker, Compose v2, tmux, Chromium, agent-browser, and dnsmasq.
+  Starts dockerd with its containers inside the cgroup and pulls images through
+  `mirror.gcr.io`.
+- Installs a global `pnpm` wrapper. Inside this repository it moves the command
+  into the capped cgroup and loads `.wrangler/kilo-startup/env`. Outside the
+  repository it runs the real pnpm, saved at
+  `/usr/local/lib/kilo-cloud-agent/real-pnpm`, unchanged.
+- Installs dependencies, creates `.env.local`, runs `pnpm test:db`, and seeds a
+  fake-login user with credits.
+- Builds the eight Cloud Agent sandbox images by running `wrangler dev` for
+  `cloud-agent-next` on spare ports until `Container image(s) ready`. The log is
+  `.wrangler/kilo-startup/sandbox-images.log`. Skip this with
+  `KILO_STARTUP_SANDBOX_IMAGES=0`.
+- It does not start the dev stack.
+
+`.wrangler/kilo-startup/env` only sets defaults. Override a value by exporting
+it before running pnpm. Do not edit the file: setup rewrites it.
+
+## Start services
+
+Run every command from the repository root, so that pnpm is capped.
+
+```bash
+pnpm dev:start --no-attach app               # web app
+pnpm dev:start --no-attach agents fake-llm   # Cloud Agents with local fake inference
+pnpm dev:status                              # services and ports
+pnpm dev:stop
+```
+
+- `KILO_DEV_WITHOUT` in the env file skips agents services that fake-LLM sessions
+  on public repositories do not need: notifications, event-service,
+  webhook-agent-ingest, container-usage-meter, and git-token-service. That
+  leaves 7 services, about 3 GB when idle. Add services back with
+  `--without=<smaller list>`. `--without=` starts everything, but the full agents
+  stack does not fit in the cap.
+- Expected warnings without those services: billing-heartbeat errors from the
+  skipped usage meter. GitHub-backed repositories need git-token-service and
+  GitHub App credentials, which the sandbox does not have.
+- `--no-attach` returns once services are up: about 50 s for `app`. The first
+  page load compiles with Turbopack and takes about 30 s more.
+- Wrangler rebuilds every sandbox image each time `cloud-agent-next` starts.
+  Images build in dockerd's own BuildKit, so unchanged images come from the
+  layer cache. Setup prebuilds them; without that, or after a Dockerfile change,
+  the build takes minutes. `cloud-agent-next` reports `up` before images are
+  ready, and sessions created meanwhile fail (`fetch failed` in the harness).
+  Wait for the build to finish:
+
+  ```bash
+  until grep -q 'Container image(s) ready' dev/logs/cloud-agent-next.log; do sleep 15; done
+  ```
+- `--reuse-running` currently refuses to reuse a session because the Stripe
+  forwarder is always skipped. Check `pnpm dev:status` instead.
+
+## Log in and use the browser
+
+Setup seeds a verified user with credits. Its email is `KILO_TEST_USER_EMAIL` in
+the env file. Shells outside the pnpm wrapper must load the env file first. It
+also puts setup's `docker` wrapper on `PATH`, which keeps builds serialized and
+inside the cap:
+
+```bash
+source .wrangler/kilo-startup/env
+agent-browser --session main open "http://localhost:3000/users/sign_in?fakeUser=$KILO_TEST_USER_EMAIL&callbackPath=/profile"
+```
+
+- Read the real port from `pnpm dev:status`. It is 3000 unless an offset applies.
+- The env file points agent-browser at the installed Chromium with a 120 s
+  timeout. Confirm login with
+  `agent-browser --session main eval '(async () => (await (await fetch("/api/auth/session")).json()).user?.email)()'`.
+- If a click fails because the element is covered, focus the input and use
+  `agent-browser press Enter`.
+- Starting a Cloud Agent session from `/cloud` requires a connected GitHub or
+  GitLab provider, which the sandbox cannot set up. Create sessions with the
+  fake-LLM harness instead, then view them in the browser.
+
+## Fake-LLM Cloud Agent sessions
+
+Start `agents fake-llm`, then follow `services/cloud-agent-next/test/e2e/README.md`.
+Ports below are the defaults; check `pnpm dev:status`.
+
+```bash
+WORKER_URL=http://localhost:8794 FAKE_LLM_URL=http://localhost:8811 \
+  pnpm -s exec tsx services/cloud-agent-next/test/e2e/run.ts cold echo:hi
+```
+
+- Verified passing on the minimal stack: `cold echo:hi`, `cold-hot echo:hi`,
+  `chunked-streaming slow:5:50`, `queue-while-busy`, and
+  `--api=legacy cold-hot echo:legacy`. `llm-error boom` fails a retry-status
+  assertion that is unrelated to the sandbox.
+- Each run leaves a sandbox container of about 750 MB until it stops for being
+  idle. Remove them between runs:
+  `docker rm -f $(docker ps -q --filter name=workerd-cloud-agent-next-dev-Sandbox)`.
+- The first session after `dev:start` can fail model validation with a 503
+  ("Model availability could not be verified"). Turbopack is still compiling
+  the validation route; retry once.
+- To view harness sessions in the browser, set `E2E_USER_EMAIL` so runs reuse
+  one driver user. Mark that user verified, then fake-login as it and open
+  `/cloud/sessions`:
+
+  ```bash
+  docker compose -f dev/docker-compose.yml exec -T postgres psql -U postgres -d postgres -c \
+    "UPDATE kilocode_users SET has_validation_stytch = true, completed_welcome_form = true WHERE google_user_email = '<driver email>'"
+  ```
+
+  Sending a message to a harness session from the UI fails with
+  "Session not found".
+
+## Memory
+
+The cap is a hard limit with no swap. When the workload nears it, the kernel
+reclaims memory instead of killing processes: everything slows, and
+`docker`, `tmux`, and `pnpm dev:stop` can hang. Check pressure with:
+
+```bash
+cg=/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD")
+echo "$(( $(cat $cg/memory.current) / 1048576 )) MiB of $(( $(cat $cg/memory.max) / 1048576 )) MiB"
+grep -E '^(high|max|oom_kill) ' $cg/memory.events
+```
+
+If the `high` or `max` count keeps rising, you are near the cap. Stop work you do
+not need, remove idle sandbox containers, or start fewer services. Dev tooling is
+heavy: wrangler and workerd use about 3 GB, and the pnpm parents and log filters
+about 2 GB.
+
+- Next.js dev (Turbopack) grows by 1 to 3 GB while compiling a route and gives
+  most of it back after about two minutes idle: 4.2 GB after login fell to
+  2.8 GB, and 4.4 GB after `/cloud` fell to 2.0 GB. Opening many routes back to
+  back fills the cap before that happens. Pause between heavy routes. Next 16.3
+  already defaults `experimental.turbopackMemoryEviction` to `auto`.
+- High Redis, redis-http, or Postgres CPU means memory pressure, not load. Under
+  pressure, Redis used 6 s of user CPU and 1590 s of system CPU in 90 minutes:
+  the kernel kept evicting and re-reading its code pages. Compare with
+  `cat $cg/containers/*/cpu.stat`. Idle Redis without pressure uses under 1%.
+- If Next.js is stuck above the cap, `pkill -9 -f '^[n]ext-server'`, then
+  `pnpm dev:restart nextjs`.
+
+Run heavy commands such as builds, tests, and typechecks from inside the
+repository, so that the wrapper caps them. pnpm outside the repository and
+direct `node` or `npx` processes are not capped.
+
+## Docker networking and DNS
+
+`/proc/sys` is read-only, so dockerd runs with `--ip-forward=false`: bridge
+containers can reach only the host. Workerd also pins sandbox containers to
+1.1.1.1 and 8.8.8.8. Setup runs dnsmasq on the bridge gateway, 172.17.0.1, and
+uses iptables to redirect all DNS from `docker0` to it.
+
+A `git_network_failed` clone failure, or `Could not resolve host` inside a
+sandbox, means this redirect is missing. Rerun setup, then check:
+
+```bash
+iptables -t nat -S PREROUTING | grep 'dport 53'
+pgrep -a dnsmasq
+docker exec <sandbox container> git ls-remote https://github.com/octocat/Hello-World.git
+```
+
+## Pitfalls
+
+- `pkill -f next-server` also matches the shell that runs it. Use
+  `pkill -f '[n]ext-server'`.
+- Do not prune Docker images or build cache. Rebuilding the Cloud Agent images
+  costs about 12 minutes.
+- Lint setup changes with `shellcheck -S warning -e SC1090 .kilo/cloud-agent-setup.sh`.
+  Setup does not install ShellCheck; use `apt-get install -y shellcheck`.

.kilo/skills/local-development/SKILL.md

@@ -5,6 +5,8 @@ description: Start, reuse, inspect, or browser-test local apps and services in t
 
 # Local development
 
+In a Kilo Cloud Agent sandbox, load the `cloud-agent-sandbox` skill first.
+
 Read `DEVELOPMENT.md` for human setup and service procedures. Read `ENVIRONMENT.md` for the environment-variable inventory. Shared web environment mutations are governed by `apps/web/AGENTS.md`; do not use this skill for that workflow.
 
 ## Start or reuse services

AGENTS.md

@@ -59,6 +59,7 @@ package manifests before running repository JavaScript or package scripts. Load
 | TypeScript implementation or review | `code-quality` skill |
 | Verification or pre-commit checks | `repository-verification` skill |
 | Local services, ports, and fake login | `local-development` skill |
+| Cloud Agent sandbox setup, memory cap, and fake-LLM sessions | `cloud-agent-sandbox` skill and `.kilo/cloud-agent-setup.sh` |
 | Shared web environment changes | `apps/web/AGENTS.md` and `DEVELOPMENT.md` |
 | PostgreSQL schema or migration work | `packages/db/AGENTS.md` and `database-migrations` skill |
 | Service, Durable Object, or Worker code | `services/AGENTS.md`, nearest owning service's `AGENTS.md`, and relevant Durable Objects or Workers skills |

dev/local/cli.ts

@@ -6,6 +6,8 @@ import {
   applyPortOffset,
   candidatePortOffsets,
   clearDevLogs,
+  excludeServices,
+  parseServiceList,
   resolveTargets,
   getService,
   getGroups,
@@ -433,7 +435,14 @@ function removeStaleComposeProject(repoRoot: string, previousOffset: number | un
 async function cmdUp(args: string[], repoRoot: string): Promise<string | undefined> {
   const noAttach = args.includes('--no-attach');
   const reuseRunning = args.includes('--reuse-running');
-  const targets = args.filter(arg => arg !== '--no-attach' && arg !== '--reuse-running');
+  const withoutArg = args.findLast(arg => arg.startsWith('--without='));
+  const withoutSource = withoutArg === undefined ? 'KILO_DEV_WITHOUT' : '--without';
+  const without = parseServiceList(
+    withoutArg === undefined ? process.env.KILO_DEV_WITHOUT : withoutArg.slice('--without='.length)
+  );
+  const targets = args.filter(
+    arg => arg !== '--no-attach' && arg !== '--reuse-running' && !arg.startsWith('--without=')
+  );
 
   // --- Preflight checks ---
   if (!isTmuxAvailable()) {
@@ -465,6 +474,16 @@ async function cmdUp(args: string[], repoRoot: string): Promise<string | undefin
   const coreServices = resolveGroups(getAlwaysOnGroupIds());
   const extraServices = targets.length === 0 ? [] : resolveTargets(targets);
   let serviceNames = topologicalSort([...new Set([...coreServices, ...extraServices])]);
+  if (without.length > 0) {
+    const exclusion = excludeServices(serviceNames, without);
+    serviceNames = exclusion.serviceNames;
+    if (exclusion.skipped.length > 0) {
+      console.log(`${DIM}Skipping (${withoutSource}): ${exclusion.skipped.join(', ')}${RESET}`);
+    }
+    for (const [name, missing] of exclusion.dependents) {
+      console.warn(`⚠ ${name} runs without ${missing.join(', ')}; calls to them will fail.`);
+    }
+  }
 
   const sessionName = getSessionName();
   let sessionAlreadyRunning = sessionExists(sessionName);
@@ -1488,9 +1507,11 @@ async function cmdEnv(args: string[], repoRoot: string): Promise<void> {
 function printUsage(): void {
   console.log(`
 Usage:
-  dev:start [--no-attach] [--reuse-running] [targets...]
+  dev:start [--no-attach] [--reuse-running] [--without=a,b] [targets...]
                           Start services (default: core)
                           --reuse-running never restarts an existing complete stack
+                          --without skips the named services (default: $KILO_DEV_WITHOUT;
+                          --without= starts everything)
   dev:stop [--force]      Stop all services (skips shared Docker infra if
                           other kilo-dev sessions are running; --force overrides)
   dev:status [--json]     Show running services and their ports

dev/local/services.test.ts

@@ -9,8 +9,10 @@ import {
   candidatePortOffsets,
   clearDevLogs,
   computePortOffset,
+  excludeServices,
   getAlwaysOnGroupIds,
   getService,
+  parseServiceList,
   portOffset,
   readPersistedPortOffset,
   resolveGroups,
@@ -577,3 +579,33 @@ test('a tunnels restart keeps the live selection and reloads the HTTP worker', (
   );
   assert.equal(planTunnelRestart(['cloud-agent-public-tunnels']).reloadTarget, undefined);
 });
+
+test('excludes unwanted services and reports dependents that lose them', () => {
+  const selection = resolveTargets(['agents', 'fake-llm']);
+  const exclusion = excludeServices(selection, ['notifications', 'event-service']);
+
+  assert.ok(!exclusion.serviceNames.includes('notifications'));
+  assert.ok(!exclusion.serviceNames.includes('event-service'));
+  assert.ok(exclusion.serviceNames.includes('cloud-agent-next'));
+  assert.deepEqual(exclusion.skipped.toSorted(), ['event-service', 'notifications']);
+  assert.deepEqual(exclusion.dependents.get('cloud-agent-next'), ['notifications']);
+});
+
+test('ignores excluded services that are not selected', () => {
+  const exclusion = excludeServices(['postgres', 'nextjs'], ['notifications']);
+
+  assert.deepEqual(exclusion.serviceNames, ['postgres', 'nextjs']);
+  assert.deepEqual(exclusion.skipped, []);
+});
+
+test('rejects unknown excluded services instead of starting everything', () => {
+  assert.throws(() => excludeServices(['postgres'], ['notifcations']), /Unknown service/);
+});
+
+test('parses comma-separated service lists', () => {
+  assert.deepEqual(parseServiceList(' notifications, event-service ,,'), [
+    'notifications',
+    'event-service',
+  ]);
+  assert.deepEqual(parseServiceList(undefined), []);
+});

dev/local/services.ts

@@ -1160,6 +1160,44 @@ export function resolveTargets(targets: string[]): string[] {
   return topologicalSort(resolveTransitiveDeps(allNames));
 }
 
+export type ServiceExclusion = {
+  serviceNames: string[];
+  skipped: string[];
+  /** Selected services that declare a dependency on a skipped service. */
+  dependents: Map<string, string[]>;
+};
+
+/**
+ * Drop explicitly unwanted services from a resolved selection, for memory-
+ * constrained environments that run only what they exercise. Unknown names
+ * throw so a typo cannot silently start the full stack.
+ */
+export function excludeServices(
+  serviceNames: readonly string[],
+  excluded: readonly string[]
+): ServiceExclusion {
+  for (const name of excluded) getService(name);
+  const excludedSet = new Set(excluded);
+  const kept = serviceNames.filter(name => !excludedSet.has(name));
+  const dependents = new Map<string, string[]>();
+  for (const name of kept) {
+    const missing = getService(name).dependsOn.filter(dep => excludedSet.has(dep));
+    if (missing.length > 0) dependents.set(name, missing);
+  }
+  return {
+    serviceNames: kept,
+    skipped: serviceNames.filter(name => excludedSet.has(name)),
+    dependents,
+  };
+}
+
+export function parseServiceList(value: string | undefined): string[] {
+  return (value ?? '')
+    .split(',')
+    .map(name => name.trim())
+    .filter(name => name !== '');
+}
+
 export function getService(name: string): ServiceDef {
   const svc = services.get(name);
   if (!svc) throw new Error(`Unknown service: ${name}`);