From 447382c25e31fd8b7bbb1598539a1e72e70d2ecd Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:15:39 +0000 Subject: [PATCH 01/35] feat(dev): add Cloud Agent sandbox startup script --- .kilo/cloud-agent-startup.sh | 160 +++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100755 .kilo/cloud-agent-startup.sh diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh new file mode 100755 index 0000000000..89e0c795fa --- /dev/null +++ b/.kilo/cloud-agent-startup.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +trap 'printf "Startup failed at line %s. Check dev/logs/ and pnpm dev:status --json.\n" "$LINENO" >&2' ERR + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +export CI=true +export KILO_PORT_OFFSET="${KILO_PORT_OFFSET:-auto}" +export NEXT_TELEMETRY_DISABLED=1 +export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" + +if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then + printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 + exit 1 +fi + +root=() +if (( EUID != 0 )); then + if ! command -v sudo >/dev/null || ! sudo -n true; then + printf 'Root or passwordless sudo is required to install sandbox prerequisites.\n' >&2 + exit 1 + fi + root=(sudo -n) +fi + +"${root[@]}" apt-get update +"${root[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates chromium curl git git-lfs openssl sudo unzip tmux docker.io docker-compose + +if ! command -v node >/dev/null || [[ $(node -p 'process.versions.node.split(".")[0]') != 24 ]]; then + printf 'The sandbox image must provide Node.js 24 and Corepack before running this script.\n' >&2 + exit 1 +fi +"${root[@]}" corepack enable +corepack install + +tools=() +command -v bun >/dev/null || tools+=(bun@1.3.13) +command -v agent-browser >/dev/null || tools+=(agent-browser@0.38.2) +if (( ${#tools[@]} )); then + "${root[@]}" corepack pnpm add --global --global-dir /opt/kilo-startup-tools \ + --global-bin-dir /usr/local/bin "${tools[@]}" +fi +export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium +export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" + +if ! docker info >/dev/null 2>&1; then + if [[ -n ${DOCKER_HOST:-} ]] || [[ -S /var/run/docker.sock ]]; then + printf 'The supplied Docker daemon is inaccessible; fix Docker access and rerun.\n' >&2 + exit 1 + fi + # Cloudflare sandboxes mount /proc/sys read-only and cannot use overlay-on-overlay. + "${root[@]}" tmux new-session -d -s kilo-startup-docker \ + 'env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --storage-driver=vfs --ip-forward=false' + for (( attempt=0; attempt<30; attempt++ )); do + docker info >/dev/null 2>&1 && break + sleep 1 + done + if ! docker info >/dev/null 2>&1; then + printf 'Docker could not start. This sandbox must support nested containers or supply a Docker socket.\n' >&2 + "${root[@]}" tmux capture-pane -p -t kilo-startup-docker || true + exit 1 + fi +fi +docker compose version +export WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1 +if [[ -n ${NODE_EXTRA_CA_CERTS:-} && -f $NODE_EXTRA_CA_CERTS ]]; then + mkdir -p dev/logs + export WRANGLER_DOCKER_BIN="$PWD/dev/logs/sandbox-docker.cjs" + cat > "$WRANGLER_DOCKER_BIN" <<'JS' +#!/usr/bin/env node +const fs = require('node:fs'); +const { spawn } = require('node:child_process'); +const args = process.argv.slice(2); +const stdinDockerfile = args[0] === 'build' && args.some((arg, i) => + (arg === '-f' || arg === '--file') && args[i + 1] === '-'); +function run(input) { + const child = spawn('docker', args, { stdio: [input === undefined ? 'inherit' : 'pipe', 'inherit', 'inherit'] }); + for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); + child.on('error', error => { console.error(error.message); process.exitCode = 1; }); + child.on('exit', code => { process.exitCode = code ?? 1; }); + if (input !== undefined) child.stdin.end(input); +} +if (!stdinDockerfile) { + run(); +} else { + let dockerfile = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', chunk => { dockerfile += chunk; }); + process.stdin.on('end', () => { + const cert = fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64'); + // Trust the sandbox's HTTPS interception CA inside development images, not production sources. + dockerfile = dockerfile.replace(/^FROM (?:docker.io\/)?cloudflare\/sandbox:[^\n]+/m, from => + `${from}\nRUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`); + run(dockerfile); + }); +} +JS + chmod +x "$WRANGLER_DOCKER_BIN" +fi +if tmux list-sessions >/dev/null 2>&1; then + tmux set-environment -g WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 + if [[ -n ${WRANGLER_DOCKER_BIN:-} ]]; then + tmux set-environment -g WRANGLER_DOCKER_BIN "$WRANGLER_DOCKER_BIN" + tmux set-environment -g NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" + fi +fi + +pnpm install --frozen-lockfile +if [[ ! -s .env.local ]]; then + (umask 077; pnpm dev:setup-env --ci) + printf 'Created local-only credentials. Real payment, model, and Git integrations require supplied secrets.\n' +fi + +# Avoid Docker Hub's shared unauthenticated pull limit without changing Compose files. +while IFS= read -r image; do + if ! docker image inspect "$image" >/dev/null 2>&1; then + if docker pull "mirror.gcr.io/$image"; then + docker tag "mirror.gcr.io/$image" "$image" + else + docker pull "$image" + fi + fi +done < <(docker compose -f dev/docker-compose.yml config --images | sort -u) + +if (( $# == 0 )); then + set -- cloud-agent +fi +pnpm dev:start --no-attach --reuse-running "$@" +pnpm test:db + +web_port=$(node -e ' + const fs = require("node:fs"); + const manifest = JSON.parse(fs.readFileSync("dev/logs/manifest.json", "utf8")); + const service = manifest.services.find(service => service.name === "nextjs"); + if (!service?.port) throw new Error("The selected stack must include nextjs"); + console.log(service.port); +') +web_url="http://localhost:$web_port" +ready=false +for (( attempt=0; attempt<90; attempt++ )); do + if curl --fail --silent --output /dev/null --max-time 10 "$web_url/users/sign_in"; then + ready=true + break + fi + sleep 2 +done +if [[ $ready != true ]]; then + printf 'Web app did not become ready at %s.\n' "$web_url" >&2 + pnpm dev:status --json + exit 1 +fi + +pnpm dev:status --json +printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=kilo-%s-%s@example.com&callbackPath=/profile\n' \ + "$web_url" "$web_url" "$(basename "$HOME")" "$(date -u +%Y%m%d%H%M%S)" +printf 'Browser setup: export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium AGENT_BROWSER_SOCKET_DIR=%q\n' "$AGENT_BROWSER_SOCKET_DIR" +printf 'Browser: agent-browser open , then agent-browser snapshot -i\n' +printf 'Cloud Agent testing: select kilo/fake-deterministic for local inference; real inference needs provider credentials.\n' +printf 'Manage services with pnpm dev:status, pnpm dev:restart , and pnpm dev:stop.\n' From 9082c33f7f268c817b7ec78670508b999448cec7 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:20:12 +0000 Subject: [PATCH 02/35] fix(dev): preserve sandbox trust wrapper and bound compiler threads --- .kilo/cloud-agent-startup.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 89e0c795fa..6dd33e3e2b 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -8,6 +8,8 @@ export CI=true export KILO_PORT_OFFSET="${KILO_PORT_OFFSET:-auto}" export NEXT_TELEMETRY_DISABLED=1 export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" +export GOMAXPROCS="${GOMAXPROCS:-2}" +export RAYON_NUM_THREADS="${RAYON_NUM_THREADS:-2}" if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 @@ -65,8 +67,8 @@ fi docker compose version export WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1 if [[ -n ${NODE_EXTRA_CA_CERTS:-} && -f $NODE_EXTRA_CA_CERTS ]]; then - mkdir -p dev/logs - export WRANGLER_DOCKER_BIN="$PWD/dev/logs/sandbox-docker.cjs" + mkdir -p .wrangler/kilo-startup + export WRANGLER_DOCKER_BIN="$PWD/.wrangler/kilo-startup/sandbox-docker.cjs" cat > "$WRANGLER_DOCKER_BIN" <<'JS' #!/usr/bin/env node const fs = require('node:fs'); @@ -100,6 +102,8 @@ JS fi if tmux list-sessions >/dev/null 2>&1; then tmux set-environment -g WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 + tmux set-environment -g GOMAXPROCS "$GOMAXPROCS" + tmux set-environment -g RAYON_NUM_THREADS "$RAYON_NUM_THREADS" if [[ -n ${WRANGLER_DOCKER_BIN:-} ]]; then tmux set-environment -g WRANGLER_DOCKER_BIN "$WRANGLER_DOCKER_BIN" tmux set-environment -g NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" @@ -152,7 +156,7 @@ if [[ $ready != true ]]; then fi pnpm dev:status --json -printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=kilo-%s-%s@example.com&callbackPath=/profile\n' \ +printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=kilo-%s-%s%%2Bstytchpass@example.com&callbackPath=/profile\n' \ "$web_url" "$web_url" "$(basename "$HOME")" "$(date -u +%Y%m%d%H%M%S)" printf 'Browser setup: export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium AGENT_BROWSER_SOCKET_DIR=%q\n' "$AGENT_BROWSER_SOCKET_DIR" printf 'Browser: agent-browser open , then agent-browser snapshot -i\n' From b283575975d0f9f227c56e5b318e3af9104bc680 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:27:28 +0000 Subject: [PATCH 03/35] fix(dev): verify sandbox images and seed an offline test account --- .kilo/cloud-agent-startup.sh | 64 +++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 6dd33e3e2b..0f2a7cd8f6 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -45,6 +45,7 @@ if (( ${#tools[@]} )); then fi export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" +export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" if ! docker info >/dev/null 2>&1; then if [[ -n ${DOCKER_HOST:-} ]] || [[ -S /var/run/docker.sock ]]; then @@ -91,6 +92,8 @@ if (!stdinDockerfile) { process.stdin.on('data', chunk => { dockerfile += chunk; }); process.stdin.on('end', () => { const cert = fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64'); + dockerfile = dockerfile.replace(/^FROM (?:docker.io\/library\/)?docker:dind-rootless/m, + 'FROM mirror.gcr.io/library/docker:dind-rootless'); // Trust the sandbox's HTTPS interception CA inside development images, not production sources. dockerfile = dockerfile.replace(/^FROM (?:docker.io\/)?cloudflare\/sandbox:[^\n]+/m, from => `${from}\nRUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`); @@ -115,6 +118,14 @@ if [[ ! -s .env.local ]]; then (umask 077; pnpm dev:setup-env --ci) printf 'Created local-only credentials. Real payment, model, and Git integrations require supplied secrets.\n' fi +node -e ' + const fs = require("node:fs"); + const match = fs.readFileSync(".env.local", "utf8").match(/^POSTGRES_URL\s*=\s*(.*)$/m); + const value = process.env.POSTGRES_URL || match?.[1].trim().replace(/^(["\x27])(.*)\1$/, (_, quote, inner) => inner); + if (!value || !["localhost", "127.0.0.1", "[::1]"].includes(new URL(value).hostname)) { + throw new Error("Sandbox startup requires a local POSTGRES_URL; remote databases will not be migrated or seeded"); + } +' # Avoid Docker Hub's shared unauthenticated pull limit without changing Compose files. while IFS= read -r image; do @@ -130,7 +141,19 @@ done < <(docker compose -f dev/docker-compose.yml config --images | sort -u) if (( $# == 0 )); then set -- cloud-agent fi -pnpm dev:start --no-attach --reuse-running "$@" +mkdir -p .wrangler/kilo-startup +selection=$(printf '%s\n' "$@") +status=$(pnpm -s dev:status --json) +if node -e 'process.exit(JSON.parse(process.argv[1]).services.length ? 0 : 1)' "$status"; then + if [[ ! -f .wrangler/kilo-startup/selection || $(< .wrangler/kilo-startup/selection) != "$selection" ]]; then + printf 'A different dev stack is already running. Stop it with pnpm dev:stop before changing the selection.\n' >&2 + exit 1 + fi + printf 'Reusing this sandbox startup script\x27s existing dev stack.\n' +else + pnpm dev:start --no-attach "$@" + printf '%s\n' "$selection" > .wrangler/kilo-startup/selection +fi pnpm test:db web_port=$(node -e ' @@ -155,10 +178,43 @@ if [[ $ready != true ]]; then exit 1 fi +if node -e 'const m = require("./dev/logs/manifest.json"); process.exit(m.services.some(s => s.name === "cloud-agent-next") ? 0 : 1)'; then + ready=false + for (( attempt=0; attempt<450; attempt++ )); do + if grep -Fq 'Container image(s) ready' dev/logs/cloud-agent-next.log; then + ready=true + break + fi + if grep -Fq '[ERROR]' dev/logs/cloud-agent-next.log; then + printf 'Cloud Agent image preparation failed. See dev/logs/cloud-agent-next.log.\n' >&2 + exit 1 + fi + sleep 2 + done + if [[ $ready != true ]]; then + printf 'Cloud Agent images did not become ready within 15 minutes. See dev/logs/cloud-agent-next.log.\n' >&2 + exit 1 + fi +fi + +test_email="kilo-$(basename "$HOME")-$(date -u +%Y%m%d%H%M%S)@example.com" +test_user_id=$(docker compose -f dev/docker-compose.yml exec -T postgres \ + psql -U postgres -d postgres -X -qAt -v ON_ERROR_STOP=1 -v email="$test_email" <<'SQL' +INSERT INTO kilocode_users ( + id, google_user_email, google_user_name, google_user_image_url, hosted_domain, + stripe_customer_id, completed_welcome_form, has_validation_stytch, customer_source +) VALUES ( + gen_random_uuid()::text, :'email', 'Sandbox Test User', '', '@@fake@@', + 'cus_local_sandbox', true, true, 'dev-seed' +) RETURNING id; +SQL +) +pnpm dev:seed app:add-credits "$test_user_id" 100 --free + pnpm dev:status --json -printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=kilo-%s-%s%%2Bstytchpass@example.com&callbackPath=/profile\n' \ - "$web_url" "$web_url" "$(basename "$HOME")" "$(date -u +%Y%m%d%H%M%S)" -printf 'Browser setup: export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium AGENT_BROWSER_SOCKET_DIR=%q\n' "$AGENT_BROWSER_SOCKET_DIR" +printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/profile\n' \ + "$web_url" "$web_url" "$test_email" +printf 'Browser setup: export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q\n' "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" printf 'Browser: agent-browser open , then agent-browser snapshot -i\n' printf 'Cloud Agent testing: select kilo/fake-deterministic for local inference; real inference needs provider credentials.\n' printf 'Manage services with pnpm dev:status, pnpm dev:restart , and pnpm dev:stop.\n' From 6e06b90823a4d27823861cbbf5429aa3c38d91d7 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:09:56 +0000 Subject: [PATCH 04/35] fix(dev): install Docker CLI and trust all sandbox image stages --- .kilo/cloud-agent-startup.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 0f2a7cd8f6..b7e2dbf38c 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -26,8 +26,19 @@ if (( EUID != 0 )); then fi "${root[@]}" apt-get update +docker_packages=(docker.io) +for package in docker-cli docker-buildx; do + if apt-cache show "$package" >/dev/null 2>&1; then + docker_packages+=("$package") + fi +done +if apt-cache show docker-compose-v2 >/dev/null 2>&1; then + docker_packages+=(docker-compose-v2) +else + docker_packages+=(docker-compose) +fi "${root[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates chromium curl git git-lfs openssl sudo unzip tmux docker.io docker-compose + ca-certificates chromium curl git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" if ! command -v node >/dev/null || [[ $(node -p 'process.versions.node.split(".")[0]') != 24 ]]; then printf 'The sandbox image must provide Node.js 24 and Corepack before running this script.\n' >&2 @@ -92,11 +103,12 @@ if (!stdinDockerfile) { process.stdin.on('data', chunk => { dockerfile += chunk; }); process.stdin.on('end', () => { const cert = fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64'); - dockerfile = dockerfile.replace(/^FROM (?:docker.io\/library\/)?docker:dind-rootless/m, - 'FROM mirror.gcr.io/library/docker:dind-rootless'); // Trust the sandbox's HTTPS interception CA inside development images, not production sources. + const trust = `RUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && (command -v update-ca-certificates || (apt-get update && apt-get install -y --no-install-recommends ca-certificates)) && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`; + dockerfile = dockerfile.replace(/^FROM (?:docker.io\/library\/)?(docker:dind-rootless|debian:trixie-slim)([^\n]*)/gm, + (_, image, suffix) => `FROM mirror.gcr.io/library/${image}${suffix}\nUSER root\n${trust}`); dockerfile = dockerfile.replace(/^FROM (?:docker.io\/)?cloudflare\/sandbox:[^\n]+/m, from => - `${from}\nRUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`); + `${from}\n${trust}`); run(dockerfile); }); } From ab7881948ac7d54d7ceb4bd031bd35c92597c3c7 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:32:41 +0000 Subject: [PATCH 05/35] feat(dev): persist browser setup and smoke-test sandbox login --- .kilo/cloud-agent-startup.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index b7e2dbf38c..f66c231cf3 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -57,6 +57,7 @@ fi export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" +export AGENT_BROWSER_DEFAULT_TIMEOUT="${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" if ! docker info >/dev/null 2>&1; then if [[ -n ${DOCKER_HOST:-} ]] || [[ -S /var/run/docker.sock ]]; then @@ -223,10 +224,27 @@ SQL ) pnpm dev:seed app:add-credits "$test_user_id" 100 --free -pnpm dev:status --json +status=$(pnpm -s dev:status --json) +node -e ' + const status = JSON.parse(process.argv[1]); + const unavailable = status.services.filter(service => service.status !== "up"); + if (unavailable.length) throw new Error(`Services are not ready: ${unavailable.map(s => s.name).join(", ")}`); +' "$status" +printf '%s\n' "$status" +export KILO_DEV_WEB_URL="$web_url" +export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/profile" +printf 'export AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q\n' \ + "$AGENT_BROWSER_EXECUTABLE_PATH" "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" \ + "$AGENT_BROWSER_DEFAULT_TIMEOUT" "$KILO_DEV_WEB_URL" "$KILO_TEST_LOGIN_URL" \ + > .wrangler/kilo-startup/browser.env +if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then + agent-browser --session kilo-startup open "$KILO_TEST_LOGIN_URL" + agent-browser --session kilo-startup wait --fn 'window.location.pathname === "/profile"' + agent-browser --session kilo-startup snapshot -i +fi printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/profile\n' \ "$web_url" "$web_url" "$test_email" -printf 'Browser setup: export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q\n' "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" -printf 'Browser: agent-browser open , then agent-browser snapshot -i\n' +printf 'Browser setup: source .wrangler/kilo-startup/browser.env\n' +printf 'Browser: agent-browser --session kilo-startup open %q, then agent-browser --session kilo-startup snapshot -i\n' "$KILO_TEST_LOGIN_URL" printf 'Cloud Agent testing: select kilo/fake-deterministic for local inference; real inference needs provider credentials.\n' printf 'Manage services with pnpm dev:status, pnpm dev:restart , and pnpm dev:stop.\n' From 921ab3d82e81e3e1cfb7e826270ed76507e8ed62 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:53:21 +0000 Subject: [PATCH 06/35] fix(dev): bound sandbox setup waits and report image progress --- .kilo/cloud-agent-startup.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index f66c231cf3..04c0f464e0 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -25,7 +25,7 @@ if (( EUID != 0 )); then root=(sudo -n) fi -"${root[@]}" apt-get update +"${root[@]}" timeout --foreground 5m apt-get -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update docker_packages=(docker.io) for package in docker-cli docker-buildx; do if apt-cache show "$package" >/dev/null 2>&1; then @@ -37,7 +37,7 @@ if apt-cache show docker-compose-v2 >/dev/null 2>&1; then else docker_packages+=(docker-compose) fi -"${root[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ +"${root[@]}" env DEBIAN_FRONTEND=noninteractive timeout --foreground 10m apt-get install -y --no-install-recommends \ ca-certificates chromium curl git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" if ! command -v node >/dev/null || [[ $(node -p 'process.versions.node.split(".")[0]') != 24 ]]; then @@ -105,7 +105,7 @@ if (!stdinDockerfile) { process.stdin.on('end', () => { const cert = fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64'); // Trust the sandbox's HTTPS interception CA inside development images, not production sources. - const trust = `RUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && (command -v update-ca-certificates || (apt-get update && apt-get install -y --no-install-recommends ca-certificates)) && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`; + const trust = `RUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && (if command -v apt-get >/dev/null; then printf 'Acquire::http::Timeout "30";\\nAcquire::https::Timeout "30";\\nAcquire::Retries "2";\\n' > /etc/apt/apt.conf.d/99-kilo-startup-timeouts; fi) && (command -v update-ca-certificates || (apt-get update && apt-get install -y --no-install-recommends ca-certificates)) && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`; dockerfile = dockerfile.replace(/^FROM (?:docker.io\/library\/)?(docker:dind-rootless|debian:trixie-slim)([^\n]*)/gm, (_, image, suffix) => `FROM mirror.gcr.io/library/${image}${suffix}\nUSER root\n${trust}`); dockerfile = dockerfile.replace(/^FROM (?:docker.io\/)?cloudflare\/sandbox:[^\n]+/m, from => @@ -126,7 +126,7 @@ if tmux list-sessions >/dev/null 2>&1; then fi fi -pnpm install --frozen-lockfile +timeout --foreground 15m pnpm install --frozen-lockfile if [[ ! -s .env.local ]]; then (umask 077; pnpm dev:setup-env --ci) printf 'Created local-only credentials. Real payment, model, and Git integrations require supplied secrets.\n' @@ -164,10 +164,11 @@ if node -e 'process.exit(JSON.parse(process.argv[1]).services.length ? 0 : 1)' " fi printf 'Reusing this sandbox startup script\x27s existing dev stack.\n' else - pnpm dev:start --no-attach "$@" + timeout --foreground 5m pnpm dev:start --no-attach "$@" printf '%s\n' "$selection" > .wrangler/kilo-startup/selection fi -pnpm test:db +printf 'Preparing the local database (up to 5 minutes).\n' +timeout --foreground 5m pnpm test:db web_port=$(node -e ' const fs = require("node:fs"); @@ -192,6 +193,7 @@ if [[ $ready != true ]]; then fi if node -e 'const m = require("./dev/logs/manifest.json"); process.exit(m.services.some(s => s.name === "cloud-agent-next") ? 0 : 1)'; then + printf 'Waiting for Cloud Agent images (up to 15 minutes). Build output: dev/logs/cloud-agent-next.log\n' ready=false for (( attempt=0; attempt<450; attempt++ )); do if grep -Fq 'Container image(s) ready' dev/logs/cloud-agent-next.log; then @@ -202,6 +204,9 @@ if node -e 'const m = require("./dev/logs/manifest.json"); process.exit(m.servic printf 'Cloud Agent image preparation failed. See dev/logs/cloud-agent-next.log.\n' >&2 exit 1 fi + if (( attempt > 0 && attempt % 30 == 0 )); then + printf 'Cloud Agent image preparation is still running (%s seconds).\n' "$(( attempt * 2 ))" + fi sleep 2 done if [[ $ready != true ]]; then From 7c237ed3f375a7d2b63ac945ed7a246502030772 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:20:29 +0000 Subject: [PATCH 07/35] fix(dev): enforce sandbox memory budgets and default to web stack --- .kilo/cloud-agent-startup.sh | 163 ++++++++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 20 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 04c0f464e0..74ea7e18a3 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -10,6 +10,16 @@ export NEXT_TELEMETRY_DISABLED=1 export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" export GOMAXPROCS="${GOMAXPROCS:-2}" export RAYON_NUM_THREADS="${RAYON_NUM_THREADS:-2}" +export KILO_ENV_SYNC_CONCURRENCY=1 +export NODE_OPTIONS=--max-old-space-size=512 +if (( $# == 0 )); then + set -- app +fi +cloud_agents=false +if [[ "$*" != app ]]; then + cloud_agents=true +fi +export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-$([[ $cloud_agents == true ]] && printf 6144 || printf 4096)}" if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 @@ -25,6 +35,38 @@ if (( EUID != 0 )); then root=(sudo -n) fi +if [[ ! $KILO_STARTUP_MEMORY_MB =~ ^[0-9]+$ ]] || (( KILO_STARTUP_MEMORY_MB < 3072 )); then + printf 'KILO_STARTUP_MEMORY_MB must be an integer of at least 3072 MiB.\n' >&2 + exit 1 +fi +export KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD")" +if [[ ! -f /sys/fs/cgroup/kilo-workloads/memory.max ]]; then + printf 'Memory-safe startup requires the sandbox\x27s delegated cgroup v2 memory controller. No workloads were started.\n' >&2 + exit 1 +fi +node -e ' + const fs = require("node:fs"); + const parent = "/sys/fs/cgroup/kilo-workloads"; + const stat = Object.fromEntries(fs.readFileSync(parent + "/memory.stat", "utf8").trim().split("\n").map(line => line.split(" "))); + const maximum = Number(fs.readFileSync(parent + "/memory.max", "utf8")); + const current = Number(fs.readFileSync(parent + "/memory.current", "utf8")); + const available = Number(fs.readFileSync("/proc/meminfo", "utf8").match(/^MemAvailable:\s+(\d+)/m)[1]) * 1024; + const protectedBytes = current - Number(stat.inactive_file || 0) + Number(stat.file_dirty || 0) + Number(stat.file_writeback || 0); + const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? Number(fs.readFileSync(process.env.KILO_STARTUP_CGROUP + "/memory.current", "utf8")) : 0; + const additional = Math.max(0, Number(process.env.KILO_STARTUP_MEMORY_MB) * 1048576 - existing); + const safeBytes = Math.min(available, Number.isFinite(maximum) ? maximum - protectedBytes : available) - 2048 * 1048576; + if (additional > safeBytes) throw new Error("Insufficient memory headroom: use the app profile, stop other workloads, or use a larger sandbox"); +' +"${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP" +printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.max" >/dev/null +printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 * 85 / 100 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.high" >/dev/null +printf '0\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.swap.max" >/dev/null +printf '1\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.oom.group" >/dev/null +printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/cgroup.subtree_control" >/dev/null +"${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP/processes" "$KILO_STARTUP_CGROUP/containers" +printf '%s\n' "$$" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/processes/cgroup.procs" >/dev/null +printf 'Startup workload capped at %s MiB, with 2048 MiB reserved for other sandbox workloads.\n' "$KILO_STARTUP_MEMORY_MB" + "${root[@]}" timeout --foreground 5m apt-get -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update docker_packages=(docker.io) for package in docker-cli docker-buildx; do @@ -38,7 +80,7 @@ else docker_packages+=(docker-compose) fi "${root[@]}" env DEBIAN_FRONTEND=noninteractive timeout --foreground 10m apt-get install -y --no-install-recommends \ - ca-certificates chromium curl git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" + ca-certificates chromium curl fuse-overlayfs git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" if ! command -v node >/dev/null || [[ $(node -p 'process.versions.node.split(".")[0]') != 24 ]]; then printf 'The sandbox image must provide Node.js 24 and Corepack before running this script.\n' >&2 @@ -56,7 +98,7 @@ if (( ${#tools[@]} )); then fi export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" -export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" +export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu,--renderer-process-limit=2,--js-flags=--max-old-space-size=256}" export AGENT_BROWSER_DEFAULT_TIMEOUT="${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" if ! docker info >/dev/null 2>&1; then @@ -64,9 +106,14 @@ if ! docker info >/dev/null 2>&1; then printf 'The supplied Docker daemon is inaccessible; fix Docker access and rerun.\n' >&2 exit 1 fi - # Cloudflare sandboxes mount /proc/sys read-only and cannot use overlay-on-overlay. + # These sandboxes mount /proc/sys read-only; overlay2 is supported by the current kernel. + storage_driver="${KILO_STARTUP_DOCKER_STORAGE_DRIVER:-overlay2}" + if [[ $storage_driver != overlay2 && $storage_driver != fuse-overlayfs ]]; then + printf 'Only overlay2 or fuse-overlayfs is allowed; vfs layer copies are unsafe for this sandbox.\n' >&2 + exit 1 + fi "${root[@]}" tmux new-session -d -s kilo-startup-docker \ - 'env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --storage-driver=vfs --ip-forward=false' + "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --storage-driver=$storage_driver --ip-forward=false --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" for (( attempt=0; attempt<30; attempt++ )); do docker info >/dev/null 2>&1 && break sleep 1 @@ -78,19 +125,72 @@ if ! docker info >/dev/null 2>&1; then fi fi docker compose version +if [[ $(docker info --format '{{.Driver}}') == vfs ]]; then + printf 'The existing Docker daemon uses vfs. Stop it and restart with overlay2 or fuse-overlayfs before running this workload.\n' >&2 + exit 1 +fi +if tmux list-sessions >/dev/null 2>&1; then + tmux_pid=$(tmux display-message -p '#{pid}') + if ! node -e 'const fs = require("node:fs"); process.exit(fs.readFileSync("/proc/" + process.argv[1] + "/cgroup", "utf8").includes(process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/processes") ? 0 : 1)' "$tmux_pid"; then + printf 'The existing tmux server is outside the startup memory budget. Stop its sessions before running this script.\n' >&2 + exit 1 + fi +fi +mkdir -p .wrangler/kilo-startup/bin +real_docker=${KILO_STARTUP_REAL_DOCKER:-$(command -v docker)} +real_pnpm=${KILO_STARTUP_REAL_PNPM:-$(command -v pnpm)} +export KILO_STARTUP_REAL_PNPM="$real_pnpm" +cat > .wrangler/kilo-startup/compose.memory.yml < .wrangler/kilo-startup/bin/pnpm < "$WRANGLER_DOCKER_BIN" <<'JS' #!/usr/bin/env node const fs = require('node:fs'); +const path = require('node:path'); const { spawn } = require('node:child_process'); const args = process.argv.slice(2); const stdinDockerfile = args[0] === 'build' && args.some((arg, i) => (arg === '-f' || arg === '--file') && args[i + 1] === '-'); function run(input) { - const child = spawn('docker', args, { stdio: [input === undefined ? 'inherit' : 'pipe', 'inherit', 'inherit'] }); + const build = args[0] === 'build'; + if (args[0] === 'compose') { + const source = args.findIndex((arg, i) => arg === '-f' && path.resolve(args[i + 1]) === path.resolve(__dirname, '../../dev/docker-compose.yml')); + if (source !== -1) args.splice(source + 2, 0, '-f', path.join(__dirname, 'compose.memory.yml')); + } + if (build) args.splice(0, 1, 'buildx', 'build', '--builder', process.env.KILO_STARTUP_BUILDER, '--allow=network.host'); + const command = build ? 'flock' : process.env.KILO_STARTUP_REAL_DOCKER; + const commandArgs = build ? [path.join(__dirname, 'image-build.lock'), process.env.KILO_STARTUP_REAL_DOCKER, ...args] : args; + const child = spawn(command, commandArgs, { stdio: [input === undefined ? 'inherit' : 'pipe', 'inherit', 'inherit'] }); for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); child.on('error', error => { console.error(error.message); process.exitCode = 1; }); child.on('exit', code => { process.exitCode = code ?? 1; }); @@ -103,30 +203,52 @@ if (!stdinDockerfile) { process.stdin.setEncoding('utf8'); process.stdin.on('data', chunk => { dockerfile += chunk; }); process.stdin.on('end', () => { - const cert = fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64'); + const cert = process.env.NODE_EXTRA_CA_CERTS && fs.existsSync(process.env.NODE_EXTRA_CA_CERTS) ? fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64') : null; // Trust the sandbox's HTTPS interception CA inside development images, not production sources. - const trust = `RUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && (if command -v apt-get >/dev/null; then printf 'Acquire::http::Timeout "30";\\nAcquire::https::Timeout "30";\\nAcquire::Retries "2";\\n' > /etc/apt/apt.conf.d/99-kilo-startup-timeouts; fi) && (command -v update-ca-certificates || (apt-get update && apt-get install -y --no-install-recommends ca-certificates)) && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt`; + const trust = cert ? `RUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && (if command -v apt-get >/dev/null; then printf 'Acquire::http::Timeout "30";\\nAcquire::https::Timeout "30";\\nAcquire::Retries "2";\\n' > /etc/apt/apt.conf.d/99-kilo-startup-timeouts; fi) && (command -v update-ca-certificates || (apt-get update && apt-get install -y --no-install-recommends ca-certificates)) && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt` : ''; + const limits = 'ENV NODE_OPTIONS=--max-old-space-size=768 GOMAXPROCS=2 npm_config_jobs=1'; dockerfile = dockerfile.replace(/^FROM (?:docker.io\/library\/)?(docker:dind-rootless|debian:trixie-slim)([^\n]*)/gm, - (_, image, suffix) => `FROM mirror.gcr.io/library/${image}${suffix}\nUSER root\n${trust}`); + (_, image, suffix) => `FROM mirror.gcr.io/library/${image}${suffix}\nUSER root\n${trust}\n${limits}`); dockerfile = dockerfile.replace(/^FROM (?:docker.io\/)?cloudflare\/sandbox:[^\n]+/m, from => - `${from}\n${trust}`); + `${from}\n${trust}\n${limits}`); run(dockerfile); }); } JS - chmod +x "$WRANGLER_DOCKER_BIN" +chmod +x "$WRANGLER_DOCKER_BIN" +ln -sf "$WRANGLER_DOCKER_BIN" .wrangler/kilo-startup/bin/docker +export PATH="$PWD/.wrangler/kilo-startup/bin:$PATH" +if [[ $cloud_agents == true ]]; then + cat > .wrangler/kilo-startup/buildkitd.toml <<'TOML' +[worker.oci] + max-parallelism = 1 + networkMode = "host" +TOML + if ! docker buildx inspect "$KILO_STARTUP_BUILDER" >/dev/null 2>&1; then + docker buildx create --name "$KILO_STARTUP_BUILDER" --driver docker-container \ + --driver-opt "image=mirror.gcr.io/moby/buildkit:v0.16.0,memory=2g,memory-swap=2g,network=host,cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" \ + --buildkitd-config "$PWD/.wrangler/kilo-startup/buildkitd.toml" \ + --buildkitd-flags '--allow-insecure-entitlement network.host' + fi + timeout 3m docker buildx inspect --bootstrap "$KILO_STARTUP_BUILDER" fi if tmux list-sessions >/dev/null 2>&1; then tmux set-environment -g WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 tmux set-environment -g GOMAXPROCS "$GOMAXPROCS" tmux set-environment -g RAYON_NUM_THREADS "$RAYON_NUM_THREADS" + tmux set-environment -g NODE_OPTIONS "$NODE_OPTIONS" + tmux set-environment -g KILO_ENV_SYNC_CONCURRENCY 1 + tmux set-environment -g KILO_STARTUP_BUILDER "$KILO_STARTUP_BUILDER" + tmux set-environment -g KILO_STARTUP_REAL_DOCKER "$KILO_STARTUP_REAL_DOCKER" if [[ -n ${WRANGLER_DOCKER_BIN:-} ]]; then tmux set-environment -g WRANGLER_DOCKER_BIN "$WRANGLER_DOCKER_BIN" - tmux set-environment -g NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" + if [[ -n ${NODE_EXTRA_CA_CERTS:-} ]]; then + tmux set-environment -g NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" + fi fi fi -timeout --foreground 15m pnpm install --frozen-lockfile +timeout 15m pnpm install --frozen-lockfile --child-concurrency=1 --network-concurrency=4 if [[ ! -s .env.local ]]; then (umask 077; pnpm dev:setup-env --ci) printf 'Created local-only credentials. Real payment, model, and Git integrations require supplied secrets.\n' @@ -151,9 +273,6 @@ while IFS= read -r image; do fi done < <(docker compose -f dev/docker-compose.yml config --images | sort -u) -if (( $# == 0 )); then - set -- cloud-agent -fi mkdir -p .wrangler/kilo-startup selection=$(printf '%s\n' "$@") status=$(pnpm -s dev:status --json) @@ -233,23 +352,27 @@ status=$(pnpm -s dev:status --json) node -e ' const status = JSON.parse(process.argv[1]); const unavailable = status.services.filter(service => service.status !== "up"); - if (unavailable.length) throw new Error(`Services are not ready: ${unavailable.map(s => s.name).join(", ")}`); + if (unavailable.length) throw new Error("Services are not ready: " + unavailable.map(s => s.name).join(", ")); ' "$status" printf '%s\n' "$status" export KILO_DEV_WEB_URL="$web_url" export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/profile" -printf 'export AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q\n' \ +printf 'export AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q\n' \ "$AGENT_BROWSER_EXECUTABLE_PATH" "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" \ "$AGENT_BROWSER_DEFAULT_TIMEOUT" "$KILO_DEV_WEB_URL" "$KILO_TEST_LOGIN_URL" \ + "$PATH" "$KILO_STARTUP_REAL_DOCKER" "$KILO_STARTUP_REAL_PNPM" "$KILO_STARTUP_BUILDER" "$NODE_OPTIONS" \ > .wrangler/kilo-startup/browser.env if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then agent-browser --session kilo-startup open "$KILO_TEST_LOGIN_URL" agent-browser --session kilo-startup wait --fn 'window.location.pathname === "/profile"' agent-browser --session kilo-startup snapshot -i + agent-browser --session kilo-startup close fi printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/profile\n' \ "$web_url" "$web_url" "$test_email" printf 'Browser setup: source .wrangler/kilo-startup/browser.env\n' printf 'Browser: agent-browser --session kilo-startup open %q, then agent-browser --session kilo-startup snapshot -i\n' "$KILO_TEST_LOGIN_URL" printf 'Cloud Agent testing: select kilo/fake-deterministic for local inference; real inference needs provider credentials.\n' +printf 'Default profile is app. For Cloud Agents: bash .kilo/cloud-agent-startup.sh agents fake-llm (after pnpm dev:stop).\n' +printf 'Memory peak: %s MiB / %s MiB hard limit.\n' "$(( $(< "$KILO_STARTUP_CGROUP/memory.peak") / 1048576 ))" "$KILO_STARTUP_MEMORY_MB" printf 'Manage services with pnpm dev:status, pnpm dev:restart , and pnpm dev:stop.\n' From cddb66ff499933a1af3417c90733bd2b3410c176 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:40:17 +0000 Subject: [PATCH 08/35] fix(dev): avoid premature reclaim stalls within the hard memory cap --- .kilo/cloud-agent-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 74ea7e18a3..bc311a93fe 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -59,7 +59,7 @@ node -e ' ' "${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP" printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.max" >/dev/null -printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 * 85 / 100 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.high" >/dev/null +printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 * 95 / 100 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.high" >/dev/null printf '0\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.swap.max" >/dev/null printf '1\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.oom.group" >/dev/null printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/cgroup.subtree_control" >/dev/null From 00f491dc2b195eb044b7ff934e0c5b25e7b3cb6a Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:46:01 +0000 Subject: [PATCH 09/35] fix(dev): rely on workload cap instead of restrictive Chromium flags --- .kilo/cloud-agent-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index bc311a93fe..bafcbd63fa 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -98,7 +98,7 @@ if (( ${#tools[@]} )); then fi export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" -export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu,--renderer-process-limit=2,--js-flags=--max-old-space-size=256}" +export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" export AGENT_BROWSER_DEFAULT_TIMEOUT="${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" if ! docker info >/dev/null 2>&1; then From 805d1d308c28661c508174fa7b76f73ee1114f4e Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:58:05 +0000 Subject: [PATCH 10/35] fix(dev): reserve enough web compile memory and batch browser smoke checks --- .kilo/cloud-agent-startup.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index bafcbd63fa..6508250b35 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -19,7 +19,7 @@ cloud_agents=false if [[ "$*" != app ]]; then cloud_agents=true fi -export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-$([[ $cloud_agents == true ]] && printf 6144 || printf 4096)}" +export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-$([[ $cloud_agents == true ]] && printf 6144 || printf 5120)}" if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 @@ -97,6 +97,7 @@ if (( ${#tools[@]} )); then --global-bin-dir /usr/local/bin "${tools[@]}" fi export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium +export AGENT_BROWSER_ENGINE=chromium export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" export AGENT_BROWSER_DEFAULT_TIMEOUT="${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" @@ -357,16 +358,17 @@ node -e ' printf '%s\n' "$status" export KILO_DEV_WEB_URL="$web_url" export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/profile" -printf 'export AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q\n' \ +printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q\n' \ + "$AGENT_BROWSER_ENGINE" \ "$AGENT_BROWSER_EXECUTABLE_PATH" "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" \ "$AGENT_BROWSER_DEFAULT_TIMEOUT" "$KILO_DEV_WEB_URL" "$KILO_TEST_LOGIN_URL" \ "$PATH" "$KILO_STARTUP_REAL_DOCKER" "$KILO_STARTUP_REAL_PNPM" "$KILO_STARTUP_BUILDER" "$NODE_OPTIONS" \ > .wrangler/kilo-startup/browser.env if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then - agent-browser --session kilo-startup open "$KILO_TEST_LOGIN_URL" - agent-browser --session kilo-startup wait --fn 'window.location.pathname === "/profile"' - agent-browser --session kilo-startup snapshot -i - agent-browser --session kilo-startup close + agent-browser --session kilo-startup batch --bail \ + "open $KILO_TEST_LOGIN_URL" \ + "wait --fn 'window.location.pathname === \"/profile\"'" \ + 'snapshot -i' 'close' fi printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/profile\n' \ "$web_url" "$web_url" "$test_email" From ddeb9482ab75120a0cd9734355586539c4d2bac6 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:02:51 +0000 Subject: [PATCH 11/35] fix(dev): select the web heap budget by working directory --- .kilo/cloud-agent-startup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 6508250b35..ef48e4c1ef 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -162,6 +162,9 @@ YAML cat > .wrangler/kilo-startup/bin/pnpm < Date: Thu, 8 Oct 2026 09:06:28 +0000 Subject: [PATCH 12/35] fix(dev): use the supported chrome browser engine name --- .kilo/cloud-agent-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index ef48e4c1ef..cb98fbe278 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -97,7 +97,7 @@ if (( ${#tools[@]} )); then --global-bin-dir /usr/local/bin "${tools[@]}" fi export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium -export AGENT_BROWSER_ENGINE=chromium +export AGENT_BROWSER_ENGINE=chrome export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" export AGENT_BROWSER_DEFAULT_TIMEOUT="${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" From e7d694853d9eec369320026bd2b9f22022d29ccb Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:12:50 +0000 Subject: [PATCH 13/35] fix(dev): preserve resource wrappers through tmux service shells --- .kilo/cloud-agent-startup.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index cb98fbe278..5dc1ba9fdd 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -172,7 +172,15 @@ for arg in "\$@"; do done exec "$real_pnpm" "\$@" SH -chmod +x .wrangler/kilo-startup/bin/pnpm +cat > .wrangler/kilo-startup/bin/kilo-shell <<'SH' +#!/usr/bin/env bash +if [[ ${1:-} == -lc ]]; then + shift + exec /bin/bash --noprofile --norc -c "$@" +fi +exec /bin/bash "$@" +SH +chmod +x .wrangler/kilo-startup/bin/pnpm .wrangler/kilo-startup/bin/kilo-shell export WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1 export KILO_STARTUP_REAL_DOCKER="$real_docker" export KILO_STARTUP_BUILDER="kilo-lowmem-$(basename "$PWD")" @@ -222,6 +230,8 @@ JS chmod +x "$WRANGLER_DOCKER_BIN" ln -sf "$WRANGLER_DOCKER_BIN" .wrangler/kilo-startup/bin/docker export PATH="$PWD/.wrangler/kilo-startup/bin:$PATH" +export SHELL="$PWD/.wrangler/kilo-startup/bin/kilo-shell" +"$SHELL" -lc "cd $(printf '%q' "$PWD/apps/web") && pnpm exec node -e 'if (process.env.NODE_OPTIONS !== \"--max-old-space-size=1536\") throw new Error(\"Web heap budget is not applied\")'" if [[ $cloud_agents == true ]]; then cat > .wrangler/kilo-startup/buildkitd.toml <<'TOML' [worker.oci] @@ -241,6 +251,7 @@ if tmux list-sessions >/dev/null 2>&1; then tmux set-environment -g GOMAXPROCS "$GOMAXPROCS" tmux set-environment -g RAYON_NUM_THREADS "$RAYON_NUM_THREADS" tmux set-environment -g NODE_OPTIONS "$NODE_OPTIONS" + tmux set-environment -g SHELL "$SHELL" tmux set-environment -g KILO_ENV_SYNC_CONCURRENCY 1 tmux set-environment -g KILO_STARTUP_BUILDER "$KILO_STARTUP_BUILDER" tmux set-environment -g KILO_STARTUP_REAL_DOCKER "$KILO_STARTUP_REAL_DOCKER" @@ -361,11 +372,11 @@ node -e ' printf '%s\n' "$status" export KILO_DEV_WEB_URL="$web_url" export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/profile" -printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q\n' \ +printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q SHELL=%q\n' \ "$AGENT_BROWSER_ENGINE" \ "$AGENT_BROWSER_EXECUTABLE_PATH" "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" \ "$AGENT_BROWSER_DEFAULT_TIMEOUT" "$KILO_DEV_WEB_URL" "$KILO_TEST_LOGIN_URL" \ - "$PATH" "$KILO_STARTUP_REAL_DOCKER" "$KILO_STARTUP_REAL_PNPM" "$KILO_STARTUP_BUILDER" "$NODE_OPTIONS" \ + "$PATH" "$KILO_STARTUP_REAL_DOCKER" "$KILO_STARTUP_REAL_PNPM" "$KILO_STARTUP_BUILDER" "$NODE_OPTIONS" "$SHELL" \ > .wrangler/kilo-startup/browser.env if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then agent-browser --session kilo-startup batch --bail \ From bf896aa14e9383ca32056acec7fc8340baddfcca Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:25:10 +0000 Subject: [PATCH 14/35] fix(dev): validate fresh test login and constrained builder configuration --- .kilo/cloud-agent-startup.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 5dc1ba9fdd..242c33d8ce 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -237,7 +237,13 @@ if [[ $cloud_agents == true ]]; then [worker.oci] max-parallelism = 1 networkMode = "host" +[registry."docker.io"] + mirrors = ["mirror.gcr.io"] TOML + if [[ -n ${NODE_EXTRA_CA_CERTS:-} && -f $NODE_EXTRA_CA_CERTS ]]; then + ca_path=$(node -p 'JSON.stringify(process.env.NODE_EXTRA_CA_CERTS)') + printf ' ca = [%s]\n[registry."mirror.gcr.io"]\n ca = [%s]\n' "$ca_path" "$ca_path" >> .wrangler/kilo-startup/buildkitd.toml + fi if ! docker buildx inspect "$KILO_STARTUP_BUILDER" >/dev/null 2>&1; then docker buildx create --name "$KILO_STARTUP_BUILDER" --driver docker-container \ --driver-opt "image=mirror.gcr.io/moby/buildkit:v0.16.0,memory=2g,memory-swap=2g,network=host,cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" \ @@ -245,6 +251,15 @@ TOML --buildkitd-flags '--allow-insecure-entitlement network.host' fi timeout 3m docker buildx inspect --bootstrap "$KILO_STARTUP_BUILDER" + docker inspect "buildx_buildkit_${KILO_STARTUP_BUILDER}0" --format '{{json .HostConfig}}' | node -e ' + let input = ""; + process.stdin.on("data", chunk => { input += chunk; }); + process.stdin.on("end", () => { + const config = JSON.parse(input); + const parent = process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/containers"; + if (config.Memory !== 2147483648 || config.CgroupParent !== parent) throw new Error("BuildKit is not inside its required memory budget"); + }); + ' fi if tmux list-sessions >/dev/null 2>&1; then tmux set-environment -g WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 @@ -347,6 +362,7 @@ if node -e 'const m = require("./dev/logs/manifest.json"); process.exit(m.servic printf 'Cloud Agent images did not become ready within 15 minutes. See dev/logs/cloud-agent-next.log.\n' >&2 exit 1 fi + docker stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" fi test_email="kilo-$(basename "$HOME")-$(date -u +%Y%m%d%H%M%S)@example.com" @@ -380,8 +396,9 @@ printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BR > .wrangler/kilo-startup/browser.env if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then agent-browser --session kilo-startup batch --bail \ + 'cookies clear' \ "open $KILO_TEST_LOGIN_URL" \ - "wait --fn 'window.location.pathname === \"/profile\"'" \ + "wait --fn 'window.location.pathname === \"/profile\" && document.body.innerText.includes(\"$test_email\")'" \ 'snapshot -i' 'close' fi printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/profile\n' \ From 8d0fad3581840c8e90cc69a78ccd7146041be820 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:29:16 +0000 Subject: [PATCH 15/35] fix(dev): account for protected memory and recover stopped services --- .kilo/cloud-agent-startup.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 242c33d8ce..d5bd66673e 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -47,12 +47,15 @@ fi node -e ' const fs = require("node:fs"); const parent = "/sys/fs/cgroup/kilo-workloads"; - const stat = Object.fromEntries(fs.readFileSync(parent + "/memory.stat", "utf8").trim().split("\n").map(line => line.split(" "))); + function protectedMemory(directory) { + const stat = Object.fromEntries(fs.readFileSync(directory + "/memory.stat", "utf8").trim().split("\n").map(line => line.split(" "))); + const current = Number(fs.readFileSync(directory + "/memory.current", "utf8")); + return current - Number(stat.inactive_file || 0) + Number(stat.file_dirty || 0) + Number(stat.file_writeback || 0); + } const maximum = Number(fs.readFileSync(parent + "/memory.max", "utf8")); - const current = Number(fs.readFileSync(parent + "/memory.current", "utf8")); const available = Number(fs.readFileSync("/proc/meminfo", "utf8").match(/^MemAvailable:\s+(\d+)/m)[1]) * 1024; - const protectedBytes = current - Number(stat.inactive_file || 0) + Number(stat.file_dirty || 0) + Number(stat.file_writeback || 0); - const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? Number(fs.readFileSync(process.env.KILO_STARTUP_CGROUP + "/memory.current", "utf8")) : 0; + const protectedBytes = protectedMemory(parent); + const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? protectedMemory(process.env.KILO_STARTUP_CGROUP) : 0; const additional = Math.max(0, Number(process.env.KILO_STARTUP_MEMORY_MB) * 1048576 - existing); const safeBytes = Math.min(available, Number.isFinite(maximum) ? maximum - protectedBytes : available) - 2048 * 1048576; if (additional > safeBytes) throw new Error("Insufficient memory headroom: use the app profile, stop other workloads, or use a larger sandbox"); @@ -312,6 +315,10 @@ if node -e 'process.exit(JSON.parse(process.argv[1]).services.length ? 0 : 1)' " exit 1 fi printf 'Reusing this sandbox startup script\x27s existing dev stack.\n' + while IFS= read -r service; do + printf 'Restarting unavailable service: %s\n' "$service" + timeout 1m pnpm dev:restart "$service" + done < <(node -e 'for (const s of JSON.parse(process.argv[1]).services) if (s.status !== "up") console.log(s.name)' "$status") else timeout --foreground 5m pnpm dev:start --no-attach "$@" printf '%s\n' "$selection" > .wrangler/kilo-startup/selection From 4e6629a34c66366af0aae4c3fcbe43c60619febf Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:40:01 +0000 Subject: [PATCH 16/35] fix(dev): use heap-bounded webpack for sandbox web development --- .kilo/cloud-agent-startup.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index d5bd66673e..7c00738c54 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -166,11 +166,14 @@ cat > .wrangler/kilo-startup/bin/pnpm < .wrangler/kilo-startup/buildkitd.toml <<'TOML' [worker.oci] From b20016fdf24190ec5a292a55b406227957c61e0b Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:55:13 +0000 Subject: [PATCH 17/35] fix(dev): select webpack for filtered web package commands --- .kilo/cloud-agent-startup.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 7c00738c54..85b704b591 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -165,17 +165,21 @@ YAML cat > .wrangler/kilo-startup/bin/pnpm < .wrangler/kilo-startup/bin/kilo-shell <<'SH' From bc11ebca1f7bf6f6a396fced6bfc3d61903e016c Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:08:35 +0000 Subject: [PATCH 18/35] fix(dev): consistently apply sufficient web heap within a six GiB cap --- .kilo/cloud-agent-startup.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 85b704b591..ab7423633e 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -19,7 +19,7 @@ cloud_agents=false if [[ "$*" != app ]]; then cloud_agents=true fi -export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-$([[ $cloud_agents == true ]] && printf 6144 || printf 5120)}" +export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-6144}" if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 @@ -142,7 +142,7 @@ if tmux list-sessions >/dev/null 2>&1; then fi mkdir -p .wrangler/kilo-startup/bin real_docker=${KILO_STARTUP_REAL_DOCKER:-$(command -v docker)} -real_pnpm=${KILO_STARTUP_REAL_PNPM:-$(command -v pnpm)} +real_pnpm=$(readlink -f "${KILO_STARTUP_REAL_PNPM:-$(command -v pnpm)}") export KILO_STARTUP_REAL_PNPM="$real_pnpm" cat > .wrangler/kilo-startup/compose.memory.yml < .wrangler/kilo-startup/bin/pnpm < .wrangler/kilo-startup/buildkitd.toml <<'TOML' [worker.oci] From f3bc0dc6dafcbed4a47daf26d6af93191edfda9b Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:15:10 +0000 Subject: [PATCH 19/35] fix(dev): reserve bounded capacity for the full web profile compile --- .kilo/cloud-agent-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index ab7423633e..0f229afb92 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -19,7 +19,7 @@ cloud_agents=false if [[ "$*" != app ]]; then cloud_agents=true fi -export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-6144}" +export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-7168}" if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 From c99b43beee74e885f23f337652acbdddacd40243 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:22:29 +0000 Subject: [PATCH 20/35] fix(dev): report sandbox startup memory admission requirements --- .kilo/cloud-agent-startup.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 0f229afb92..b97eb14991 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -39,7 +39,8 @@ if [[ ! $KILO_STARTUP_MEMORY_MB =~ ^[0-9]+$ ]] || (( KILO_STARTUP_MEMORY_MB < 30 printf 'KILO_STARTUP_MEMORY_MB must be an integer of at least 3072 MiB.\n' >&2 exit 1 fi -export KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD")" +KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD")" +export KILO_STARTUP_CGROUP if [[ ! -f /sys/fs/cgroup/kilo-workloads/memory.max ]]; then printf 'Memory-safe startup requires the sandbox\x27s delegated cgroup v2 memory controller. No workloads were started.\n' >&2 exit 1 @@ -58,7 +59,10 @@ node -e ' const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? protectedMemory(process.env.KILO_STARTUP_CGROUP) : 0; const additional = Math.max(0, Number(process.env.KILO_STARTUP_MEMORY_MB) * 1048576 - existing); const safeBytes = Math.min(available, Number.isFinite(maximum) ? maximum - protectedBytes : available) - 2048 * 1048576; - if (additional > safeBytes) throw new Error("Insufficient memory headroom: use the app profile, stop other workloads, or use a larger sandbox"); + if (additional > safeBytes) { + const mib = bytes => Math.floor(bytes / 1048576); + throw new Error("Insufficient memory headroom: " + Math.ceil(additional / 1048576) + " MiB additional capacity required, " + mib(Math.max(0, safeBytes)) + " MiB available after the 2048 MiB reserve (" + mib(protectedBytes) + " MiB parent protected memory). No workloads were started. Stop other workloads or use a larger sandbox; do not lower the reserve."); + } ' "${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP" printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.max" >/dev/null @@ -197,7 +201,8 @@ chmod +x .wrangler/kilo-startup/bin/pnpm .wrangler/kilo-startup/bin/kilo-shell "${root[@]}" ln -sfn "$PWD/.wrangler/kilo-startup/bin/pnpm" /usr/local/bin/pnpm export WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1 export KILO_STARTUP_REAL_DOCKER="$real_docker" -export KILO_STARTUP_BUILDER="kilo-lowmem-$(basename "$PWD")" +KILO_STARTUP_BUILDER="kilo-lowmem-$(basename "$PWD")" +export KILO_STARTUP_BUILDER export WRANGLER_DOCKER_BIN="$PWD/.wrangler/kilo-startup/sandbox-docker.cjs" cat > "$WRANGLER_DOCKER_BIN" <<'JS' #!/usr/bin/env node From 41c51e0e437fcdb9cace9c1193c5f0f2181475db Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:24:20 +0000 Subject: [PATCH 21/35] fix(dev): fit default app budget within sandbox headroom --- .kilo/cloud-agent-startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index b97eb14991..54e5488295 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -19,7 +19,7 @@ cloud_agents=false if [[ "$*" != app ]]; then cloud_agents=true fi -export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-7168}" +export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-6912}" if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 From 8fffb9a1c1a2e5ab293e7284aed8139dd40973f1 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:32:33 +0000 Subject: [PATCH 22/35] fix(dev): smoke test lightweight landing and authenticated session --- .kilo/cloud-agent-startup.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 54e5488295..0023433b7b 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -410,7 +410,7 @@ node -e ' ' "$status" printf '%s\n' "$status" export KILO_DEV_WEB_URL="$web_url" -export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/profile" +export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/" printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q SHELL=%q\n' \ "$AGENT_BROWSER_ENGINE" \ "$AGENT_BROWSER_EXECUTABLE_PATH" "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" \ @@ -421,10 +421,11 @@ if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then agent-browser --session kilo-startup batch --bail \ 'cookies clear' \ "open $KILO_TEST_LOGIN_URL" \ - "wait --fn 'window.location.pathname === \"/profile\" && document.body.innerText.includes(\"$test_email\")'" \ + "wait --fn '(async () => window.location.pathname === \"/\" && (await (await fetch(\"/api/auth/session\")).json()).user?.email === \"$test_email\")()'" \ 'snapshot -i' 'close' + printf 'Verified authenticated browser session email: %s\n' "$test_email" fi -printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/profile\n' \ +printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/\n' \ "$web_url" "$web_url" "$test_email" printf 'Browser setup: source .wrangler/kilo-startup/browser.env\n' printf 'Browser: agent-browser --session kilo-startup open %q, then agent-browser --session kilo-startup snapshot -i\n' "$KILO_TEST_LOGIN_URL" From 4d6ce053088ef369a891e23deaf2aa78c6e81972 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:35:26 +0000 Subject: [PATCH 23/35] fix(dev): explicitly assert seeded browser session authentication --- .kilo/cloud-agent-startup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 0023433b7b..af94610112 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -421,7 +421,8 @@ if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then agent-browser --session kilo-startup batch --bail \ 'cookies clear' \ "open $KILO_TEST_LOGIN_URL" \ - "wait --fn '(async () => window.location.pathname === \"/\" && (await (await fetch(\"/api/auth/session\")).json()).user?.email === \"$test_email\")()'" \ + "wait --fn 'window.location.pathname === \"/\"'" \ + "eval '(async () => { const session = await (await fetch(\"/api/auth/session\")).json(); if (window.location.pathname !== \"/\" || session.user?.email !== \"$test_email\") throw new Error(\"Seeded browser authentication did not match\"); return true; })()'" \ 'snapshot -i' 'close' printf 'Verified authenticated browser session email: %s\n' "$test_email" fi From b68d970e5964f4b494b7054eea3b9bfe6a53bfa0 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:38:45 +0000 Subject: [PATCH 24/35] fix(dev): require explicit browser authentication result before success --- .kilo/cloud-agent-startup.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index af94610112..5f88754aa1 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -418,12 +418,18 @@ printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BR "$PATH" "$KILO_STARTUP_REAL_DOCKER" "$KILO_STARTUP_REAL_PNPM" "$KILO_STARTUP_BUILDER" "$NODE_OPTIONS" "$SHELL" \ > .wrangler/kilo-startup/browser.env if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then - agent-browser --session kilo-startup batch --bail \ - 'cookies clear' \ - "open $KILO_TEST_LOGIN_URL" \ - "wait --fn 'window.location.pathname === \"/\"'" \ - "eval '(async () => { const session = await (await fetch(\"/api/auth/session\")).json(); if (window.location.pathname !== \"/\" || session.user?.email !== \"$test_email\") throw new Error(\"Seeded browser authentication did not match\"); return true; })()'" \ - 'snapshot -i' 'close' + agent-browser --session kilo-startup cookies clear + agent-browser --session kilo-startup open "$KILO_TEST_LOGIN_URL" + agent-browser --session kilo-startup wait --fn 'window.location.pathname === "/"' + authenticated=$(agent-browser --session kilo-startup eval \ + "(async () => { const session = await (await fetch(\"/api/auth/session\")).json(); return window.location.pathname === \"/\" && session.user?.email === \"$test_email\"; })()") + if [[ $authenticated != true ]]; then + agent-browser --session kilo-startup close + printf 'Seeded browser authentication did not match.\n' >&2 + exit 1 + fi + agent-browser --session kilo-startup snapshot -i + agent-browser --session kilo-startup close printf 'Verified authenticated browser session email: %s\n' "$test_email" fi printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/\n' \ From 063d164cde333929375c3cf8740c9c467ad51cfc Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:56:57 +0000 Subject: [PATCH 25/35] fix(dev): cap the startup workload at total sandbox memory minus the reserve --- .kilo/cloud-agent-startup.sh | 43 +++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-startup.sh index 5f88754aa1..1f5ce8ecce 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-startup.sh @@ -19,7 +19,7 @@ cloud_agents=false if [[ "$*" != app ]]; then cloud_agents=true fi -export KILO_STARTUP_MEMORY_MB="${KILO_STARTUP_MEMORY_MB:-6912}" +KILO_STARTUP_RESERVE_MB=2048 if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 @@ -35,35 +35,42 @@ if (( EUID != 0 )); then root=(sudo -n) fi -if [[ ! $KILO_STARTUP_MEMORY_MB =~ ^[0-9]+$ ]] || (( KILO_STARTUP_MEMORY_MB < 3072 )); then - printf 'KILO_STARTUP_MEMORY_MB must be an integer of at least 3072 MiB.\n' >&2 - exit 1 -fi KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD")" -export KILO_STARTUP_CGROUP +export KILO_STARTUP_CGROUP KILO_STARTUP_RESERVE_MB if [[ ! -f /sys/fs/cgroup/kilo-workloads/memory.max ]]; then printf 'Memory-safe startup requires the sandbox\x27s delegated cgroup v2 memory controller. No workloads were started.\n' >&2 exit 1 fi -node -e ' +KILO_STARTUP_MEMORY_MB=$(node -e ' const fs = require("node:fs"); + const MiB = 1048576; const parent = "/sys/fs/cgroup/kilo-workloads"; + function fail(message) { + console.error(message + " No workloads were started."); + process.exit(1); + } function protectedMemory(directory) { const stat = Object.fromEntries(fs.readFileSync(directory + "/memory.stat", "utf8").trim().split("\n").map(line => line.split(" "))); const current = Number(fs.readFileSync(directory + "/memory.current", "utf8")); return current - Number(stat.inactive_file || 0) + Number(stat.file_dirty || 0) + Number(stat.file_writeback || 0); } - const maximum = Number(fs.readFileSync(parent + "/memory.max", "utf8")); - const available = Number(fs.readFileSync("/proc/meminfo", "utf8").match(/^MemAvailable:\s+(\d+)/m)[1]) * 1024; - const protectedBytes = protectedMemory(parent); + const meminfo = fs.readFileSync("/proc/meminfo", "utf8"); + const meminfoBytes = key => Number(meminfo.match(new RegExp("^" + key + ":\\s+(\\d+)", "m"))[1]) * 1024; + const parentMax = Number(fs.readFileSync(parent + "/memory.max", "utf8")); + const total = Math.min(meminfoBytes("MemTotal"), Number.isFinite(parentMax) ? parentMax : Infinity); + const reserve = Number(process.env.KILO_STARTUP_RESERVE_MB) * MiB; + const requested = process.env.KILO_STARTUP_MEMORY_MB; + if (requested !== undefined && !/^[0-9]+$/.test(requested)) fail("KILO_STARTUP_MEMORY_MB must be an integer number of MiB."); + const limit = requested === undefined ? Math.floor((total - reserve) / MiB) * MiB : Number(requested) * MiB; + if (limit < 3072 * MiB) fail("The startup workload needs at least 3072 MiB, but its cap is " + Math.floor(limit / MiB) + " MiB (" + Math.floor(total / MiB) + " MiB total, " + Math.floor(reserve / MiB) + " MiB reserve)."); const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? protectedMemory(process.env.KILO_STARTUP_CGROUP) : 0; - const additional = Math.max(0, Number(process.env.KILO_STARTUP_MEMORY_MB) * 1048576 - existing); - const safeBytes = Math.min(available, Number.isFinite(maximum) ? maximum - protectedBytes : available) - 2048 * 1048576; - if (additional > safeBytes) { - const mib = bytes => Math.floor(bytes / 1048576); - throw new Error("Insufficient memory headroom: " + Math.ceil(additional / 1048576) + " MiB additional capacity required, " + mib(Math.max(0, safeBytes)) + " MiB available after the 2048 MiB reserve (" + mib(protectedBytes) + " MiB parent protected memory). No workloads were started. Stop other workloads or use a larger sandbox; do not lower the reserve."); - } -' + const others = Math.max(0, protectedMemory(parent) - existing); + if (limit + others > total) fail("Insufficient memory headroom: other sandbox workloads hold " + Math.ceil(others / MiB) + " MiB, so a " + Math.floor(limit / MiB) + " MiB startup cap exceeds the " + Math.floor(total / MiB) + " MiB total. Stop other workloads or use a larger sandbox."); + const additional = Math.max(0, limit - existing); + if (additional > meminfoBytes("MemAvailable")) fail("Insufficient host memory: " + Math.ceil(additional / MiB) + " MiB additional capacity required, " + Math.floor(meminfoBytes("MemAvailable") / MiB) + " MiB available."); + console.log(Math.floor(limit / MiB)); +') +export KILO_STARTUP_MEMORY_MB "${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP" printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.max" >/dev/null printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 * 95 / 100 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.high" >/dev/null @@ -72,7 +79,7 @@ printf '1\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.oom.group" >/dev/nu printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/cgroup.subtree_control" >/dev/null "${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP/processes" "$KILO_STARTUP_CGROUP/containers" printf '%s\n' "$$" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/processes/cgroup.procs" >/dev/null -printf 'Startup workload capped at %s MiB, with 2048 MiB reserved for other sandbox workloads.\n' "$KILO_STARTUP_MEMORY_MB" +printf 'Startup workload capped at %s MiB, with %s MiB reserved for other sandbox workloads.\n' "$KILO_STARTUP_MEMORY_MB" "$KILO_STARTUP_RESERVE_MB" "${root[@]}" timeout --foreground 5m apt-get -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update docker_packages=(docker.io) From 62dfa348fc277af0b01a3c21103948ce0a346528 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:06:15 +0000 Subject: [PATCH 26/35] feat(dev): split sandbox setup from running the stack with pnpm dev:start --- ...-agent-startup.sh => cloud-agent-setup.sh} | 430 ++++++++---------- 1 file changed, 186 insertions(+), 244 deletions(-) rename .kilo/{cloud-agent-startup.sh => cloud-agent-setup.sh} (50%) diff --git a/.kilo/cloud-agent-startup.sh b/.kilo/cloud-agent-setup.sh similarity index 50% rename from .kilo/cloud-agent-startup.sh rename to .kilo/cloud-agent-setup.sh index 1f5ce8ecce..a37a2c1253 100755 --- a/.kilo/cloud-agent-startup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -1,28 +1,32 @@ #!/usr/bin/env bash set -Eeuo pipefail -trap 'printf "Startup failed at line %s. Check dev/logs/ and pnpm dev:status --json.\n" "$LINENO" >&2' ERR +trap 'status=$?; printf "Setup failed at line %s (exit %s).\n" "$LINENO" "$status" >&2; exit "$status"' ERR cd "$(dirname "${BASH_SOURCE[0]}")/.." +repo=$PWD +state_dir="$repo/.wrangler/kilo-startup" +startup_bin="$state_dir/bin" +env_file="$state_dir/env" +global_state_dir=/usr/local/lib/kilo-cloud-agent +pnpm_wrapper_marker='# kilo-cloud-agent-pnpm-wrapper' + +clean_path= +IFS=: read -ra path_entries <<< "$PATH" +for entry in "${path_entries[@]}"; do + [[ $entry == "$startup_bin" ]] || clean_path+=${clean_path:+:}$entry +done +PATH=$clean_path + export CI=true -export KILO_PORT_OFFSET="${KILO_PORT_OFFSET:-auto}" export NEXT_TELEMETRY_DISABLED=1 -export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" -export GOMAXPROCS="${GOMAXPROCS:-2}" -export RAYON_NUM_THREADS="${RAYON_NUM_THREADS:-2}" export KILO_ENV_SYNC_CONCURRENCY=1 -export NODE_OPTIONS=--max-old-space-size=512 -if (( $# == 0 )); then - set -- app -fi -cloud_agents=false -if [[ "$*" != app ]]; then - cloud_agents=true -fi +export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" +export KILO_PORT_OFFSET="${KILO_PORT_OFFSET:-auto}" KILO_STARTUP_RESERVE_MB=2048 if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then - printf 'This startup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 + printf 'This setup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 exit 1 fi @@ -35,10 +39,10 @@ if (( EUID != 0 )); then root=(sudo -n) fi -KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD")" +KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$repo")" export KILO_STARTUP_CGROUP KILO_STARTUP_RESERVE_MB if [[ ! -f /sys/fs/cgroup/kilo-workloads/memory.max ]]; then - printf 'Memory-safe startup requires the sandbox\x27s delegated cgroup v2 memory controller. No workloads were started.\n' >&2 + printf 'Memory-safe setup requires the sandbox\x27s delegated cgroup v2 memory controller. No workloads were started.\n' >&2 exit 1 fi KILO_STARTUP_MEMORY_MB=$(node -e ' @@ -62,10 +66,10 @@ KILO_STARTUP_MEMORY_MB=$(node -e ' const requested = process.env.KILO_STARTUP_MEMORY_MB; if (requested !== undefined && !/^[0-9]+$/.test(requested)) fail("KILO_STARTUP_MEMORY_MB must be an integer number of MiB."); const limit = requested === undefined ? Math.floor((total - reserve) / MiB) * MiB : Number(requested) * MiB; - if (limit < 3072 * MiB) fail("The startup workload needs at least 3072 MiB, but its cap is " + Math.floor(limit / MiB) + " MiB (" + Math.floor(total / MiB) + " MiB total, " + Math.floor(reserve / MiB) + " MiB reserve)."); + if (limit < 3072 * MiB) fail("The dev workload needs at least 3072 MiB, but its cap is " + Math.floor(limit / MiB) + " MiB (" + Math.floor(total / MiB) + " MiB total, " + Math.floor(reserve / MiB) + " MiB reserve)."); const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? protectedMemory(process.env.KILO_STARTUP_CGROUP) : 0; const others = Math.max(0, protectedMemory(parent) - existing); - if (limit + others > total) fail("Insufficient memory headroom: other sandbox workloads hold " + Math.ceil(others / MiB) + " MiB, so a " + Math.floor(limit / MiB) + " MiB startup cap exceeds the " + Math.floor(total / MiB) + " MiB total. Stop other workloads or use a larger sandbox."); + if (limit + others > total) fail("Insufficient memory headroom: other sandbox workloads hold " + Math.ceil(others / MiB) + " MiB, so a " + Math.floor(limit / MiB) + " MiB dev workload cap exceeds the " + Math.floor(total / MiB) + " MiB total. Stop other workloads or use a larger sandbox."); const additional = Math.max(0, limit - existing); if (additional > meminfoBytes("MemAvailable")) fail("Insufficient host memory: " + Math.ceil(additional / MiB) + " MiB additional capacity required, " + Math.floor(meminfoBytes("MemAvailable") / MiB) + " MiB available."); console.log(Math.floor(limit / MiB)); @@ -79,7 +83,7 @@ printf '1\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.oom.group" >/dev/nu printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/cgroup.subtree_control" >/dev/null "${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP/processes" "$KILO_STARTUP_CGROUP/containers" printf '%s\n' "$$" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/processes/cgroup.procs" >/dev/null -printf 'Startup workload capped at %s MiB, with %s MiB reserved for other sandbox workloads.\n' "$KILO_STARTUP_MEMORY_MB" "$KILO_STARTUP_RESERVE_MB" +printf 'Dev workload capped at %s MiB, with %s MiB reserved for other sandbox workloads.\n' "$KILO_STARTUP_MEMORY_MB" "$KILO_STARTUP_RESERVE_MB" "${root[@]}" timeout --foreground 5m apt-get -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update docker_packages=(docker.io) @@ -91,71 +95,61 @@ done if apt-cache show docker-compose-v2 >/dev/null 2>&1; then docker_packages+=(docker-compose-v2) else + # Debian trixie's docker-compose package ships Compose v2 as the docker CLI plugin. docker_packages+=(docker-compose) fi "${root[@]}" env DEBIAN_FRONTEND=noninteractive timeout --foreground 10m apt-get install -y --no-install-recommends \ ca-certificates chromium curl fuse-overlayfs git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" +if ! docker compose version >/dev/null 2>&1; then + printf 'Docker Compose v2 (the docker compose CLI plugin) is required, but %s did not provide it.\n' "${docker_packages[-1]}" >&2 + exit 1 +fi if ! command -v node >/dev/null || [[ $(node -p 'process.versions.node.split(".")[0]') != 24 ]]; then printf 'The sandbox image must provide Node.js 24 and Corepack before running this script.\n' >&2 exit 1 fi -"${root[@]}" corepack enable -corepack install + +global_pnpm=$(command -v pnpm || true) +if [[ -n $global_pnpm ]] && grep -qF "$pnpm_wrapper_marker" "$global_pnpm" 2>/dev/null; then + real_pnpm=$(< "$global_state_dir/real-pnpm") +else + if [[ -n $global_pnpm && $(readlink -f "$global_pnpm") == "$state_dir"/* ]]; then + "${root[@]}" rm -f "$global_pnpm" + global_pnpm= + fi + if [[ -z $global_pnpm ]]; then + "${root[@]}" corepack enable + corepack install + global_pnpm=$(command -v pnpm) + fi + real_pnpm=$(readlink -f "$global_pnpm") +fi +if [[ ! -x $real_pnpm ]] || grep -qF "$pnpm_wrapper_marker" "$real_pnpm"; then + printf 'Could not locate the real pnpm executable (resolved %s).\n' "$real_pnpm" >&2 + exit 1 +fi tools=() command -v bun >/dev/null || tools+=(bun@1.3.13) command -v agent-browser >/dev/null || tools+=(agent-browser@0.38.2) if (( ${#tools[@]} )); then - "${root[@]}" corepack pnpm add --global --global-dir /opt/kilo-startup-tools \ - --global-bin-dir /usr/local/bin "${tools[@]}" + "${root[@]}" npm install --global --no-audit --no-fund "${tools[@]}" fi -export AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium -export AGENT_BROWSER_ENGINE=chrome -export AGENT_BROWSER_SOCKET_DIR="${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" -export AGENT_BROWSER_ARGS="${AGENT_BROWSER_ARGS:---disable-gpu}" -export AGENT_BROWSER_DEFAULT_TIMEOUT="${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" -if ! docker info >/dev/null 2>&1; then - if [[ -n ${DOCKER_HOST:-} ]] || [[ -S /var/run/docker.sock ]]; then - printf 'The supplied Docker daemon is inaccessible; fix Docker access and rerun.\n' >&2 - exit 1 - fi - # These sandboxes mount /proc/sys read-only; overlay2 is supported by the current kernel. - storage_driver="${KILO_STARTUP_DOCKER_STORAGE_DRIVER:-overlay2}" - if [[ $storage_driver != overlay2 && $storage_driver != fuse-overlayfs ]]; then - printf 'Only overlay2 or fuse-overlayfs is allowed; vfs layer copies are unsafe for this sandbox.\n' >&2 - exit 1 - fi - "${root[@]}" tmux new-session -d -s kilo-startup-docker \ - "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --storage-driver=$storage_driver --ip-forward=false --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" - for (( attempt=0; attempt<30; attempt++ )); do - docker info >/dev/null 2>&1 && break - sleep 1 - done - if ! docker info >/dev/null 2>&1; then - printf 'Docker could not start. This sandbox must support nested containers or supply a Docker socket.\n' >&2 - "${root[@]}" tmux capture-pane -p -t kilo-startup-docker || true - exit 1 - fi -fi -docker compose version -if [[ $(docker info --format '{{.Driver}}') == vfs ]]; then - printf 'The existing Docker daemon uses vfs. Stop it and restart with overlay2 or fuse-overlayfs before running this workload.\n' >&2 - exit 1 -fi if tmux list-sessions >/dev/null 2>&1; then tmux_pid=$(tmux display-message -p '#{pid}') if ! node -e 'const fs = require("node:fs"); process.exit(fs.readFileSync("/proc/" + process.argv[1] + "/cgroup", "utf8").includes(process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/processes") ? 0 : 1)' "$tmux_pid"; then - printf 'The existing tmux server is outside the startup memory budget. Stop its sessions before running this script.\n' >&2 + printf 'The existing tmux server is outside the dev workload memory budget. Stop its sessions before running this script.\n' >&2 exit 1 fi fi -mkdir -p .wrangler/kilo-startup/bin -real_docker=${KILO_STARTUP_REAL_DOCKER:-$(command -v docker)} -real_pnpm=$(readlink -f "${KILO_STARTUP_REAL_PNPM:-$(command -v pnpm)}") -export KILO_STARTUP_REAL_PNPM="$real_pnpm" -cat > .wrangler/kilo-startup/compose.memory.yml < "$state_dir/compose.memory.yml" < .wrangler/kilo-startup/bin/pnpm < .wrangler/kilo-startup/bin/kilo-shell <<'SH' +cat > "$startup_bin/kilo-shell" <<'SH' #!/usr/bin/env bash if [[ ${1:-} == -lc ]]; then shift @@ -204,14 +175,8 @@ if [[ ${1:-} == -lc ]]; then fi exec /bin/bash "$@" SH -chmod +x .wrangler/kilo-startup/bin/pnpm .wrangler/kilo-startup/bin/kilo-shell -"${root[@]}" ln -sfn "$PWD/.wrangler/kilo-startup/bin/pnpm" /usr/local/bin/pnpm -export WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1 -export KILO_STARTUP_REAL_DOCKER="$real_docker" -KILO_STARTUP_BUILDER="kilo-lowmem-$(basename "$PWD")" -export KILO_STARTUP_BUILDER -export WRANGLER_DOCKER_BIN="$PWD/.wrangler/kilo-startup/sandbox-docker.cjs" - cat > "$WRANGLER_DOCKER_BIN" <<'JS' +chmod +x "$startup_bin/kilo-shell" +cat > "$state_dir/sandbox-docker.cjs" <<'JS' #!/usr/bin/env node const fs = require('node:fs'); const path = require('node:path'); @@ -253,56 +218,118 @@ if (!stdinDockerfile) { }); } JS -chmod +x "$WRANGLER_DOCKER_BIN" -ln -sf "$WRANGLER_DOCKER_BIN" .wrangler/kilo-startup/bin/docker -export PATH="$PWD/.wrangler/kilo-startup/bin:$PATH" -export SHELL="$PWD/.wrangler/kilo-startup/bin/kilo-shell" -"$SHELL" -lc "cd $(printf '%q' "$PWD/apps/web") && pnpm exec node -e 'if (process.env.NODE_OPTIONS !== \"--max-old-space-size=3072\") throw new Error(\"Web heap budget is not applied\")'" -if [[ $cloud_agents == true ]]; then - cat > .wrangler/kilo-startup/buildkitd.toml <<'TOML' +chmod +x "$state_dir/sandbox-docker.cjs" +ln -sf "$state_dir/sandbox-docker.cjs" "$startup_bin/docker" + +{ + printf 'case ":$PATH:" in *:%q:*) ;; *) export PATH=%q:"$PATH" ;; esac\n' "$startup_bin" "$startup_bin" + for name in NEXT_TELEMETRY_DISABLED SKIP_STRIPE_API KILO_PORT_OFFSET KILO_ENV_SYNC_CONCURRENCY KILO_STARTUP_CGROUP; do + printf 'export %s=%q\n' "$name" "${!name}" + done + printf 'export KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_BUILDER=%q\n' "$real_docker" "$KILO_STARTUP_BUILDER" + printf 'export WRANGLER_DOCKER_BIN=%q WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1\n' "$state_dir/sandbox-docker.cjs" + printf 'export SHELL=%q\n' "$startup_bin/kilo-shell" + printf 'export AGENT_BROWSER_ENGINE=chrome AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium\n' + printf 'export AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q\n' \ + "${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" "${AGENT_BROWSER_ARGS:---disable-gpu}" "${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" + if [[ -n ${NODE_EXTRA_CA_CERTS:-} ]]; then + printf 'export NODE_EXTRA_CA_CERTS=%q\n' "$NODE_EXTRA_CA_CERTS" + fi +} > "$env_file" + +cgroup_writer=(tee) +if (( ${#root[@]} )); then + cgroup_writer=("${root[@]}" tee) +fi +"${root[@]}" mkdir -p "$global_state_dir" +printf '%s\n' "$real_pnpm" | "${root[@]}" tee "$global_state_dir/real-pnpm" >/dev/null +{ + printf '#!/usr/bin/env bash\n%s\n' "$pnpm_wrapper_marker" + printf 'repo=%q\nenv_file=%q\n' "$repo" "$env_file" + printf 'cgroup_writer=(%s)\n' "$(printf '%q ' "${cgroup_writer[@]}")" + cat <<'SH' +real_pnpm=$(< /usr/local/lib/kilo-cloud-agent/real-pnpm) +if [[ ($PWD == "$repo" || $PWD == "$repo"/*) && -f $env_file ]]; then + source "$env_file" + cgroup=${KILO_STARTUP_CGROUP#/sys/fs/cgroup} + if [[ $(< /proc/self/cgroup) != "0::$cgroup/"* ]]; then + if [[ ! -d $KILO_STARTUP_CGROUP/processes ]]; then + printf 'The sandbox dev memory cgroup is missing. Rerun bash %q/.kilo/cloud-agent-setup.sh.\n' "$repo" >&2 + exit 1 + fi + printf '%s\n' "$$" | "${cgroup_writer[@]}" "$KILO_STARTUP_CGROUP/processes/cgroup.procs" >/dev/null + fi +fi +exec "$real_pnpm" "$@" +SH +} > "$state_dir/pnpm-wrapper" +"${root[@]}" install -m 0755 "$state_dir/pnpm-wrapper" "${global_pnpm:-/usr/local/bin/pnpm}" +hash -r +source "$env_file" + +if ! docker info >/dev/null 2>&1; then + if [[ -n ${DOCKER_HOST:-} ]] || [[ -S /var/run/docker.sock ]]; then + printf 'The supplied Docker daemon is inaccessible; fix Docker access and rerun.\n' >&2 + exit 1 + fi + # These sandboxes mount /proc/sys read-only; overlay2 is supported by the current kernel. + storage_driver="${KILO_STARTUP_DOCKER_STORAGE_DRIVER:-overlay2}" + if [[ $storage_driver != overlay2 && $storage_driver != fuse-overlayfs ]]; then + printf 'Only overlay2 or fuse-overlayfs is allowed; vfs layer copies are unsafe for this sandbox.\n' >&2 + exit 1 + fi + # Own the socket by the invoking user's group so a non-root sandbox can use the daemon it starts. + "${root[@]}" tmux new-session -d -s kilo-startup-docker \ + "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" + for (( attempt=0; attempt<30; attempt++ )); do + docker info >/dev/null 2>&1 && break + sleep 1 + done + if ! docker info >/dev/null 2>&1; then + printf 'Docker could not start. This sandbox must support nested containers or supply a Docker socket.\n' >&2 + "${root[@]}" tmux capture-pane -p -t kilo-startup-docker || true + exit 1 + fi +fi +if [[ $(docker info --format '{{.Driver}}') == vfs ]]; then + printf 'The existing Docker daemon uses vfs. Stop it and restart with overlay2 or fuse-overlayfs before running this workload.\n' >&2 + exit 1 +fi +if tmux list-sessions >/dev/null 2>&1; then + while IFS= read -r line; do + tmux set-environment -g "${line%%=*}" "${line#*=}" + done < <(bash -c 'source "$1"; for name in PATH SHELL KILO_PORT_OFFSET KILO_STARTUP_CGROUP KILO_STARTUP_BUILDER KILO_STARTUP_REAL_DOCKER WRANGLER_DOCKER_BIN WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST NODE_EXTRA_CA_CERTS; do [[ -n ${!name:-} ]] && printf "%s=%s\n" "$name" "${!name}"; done' _ "$env_file") +fi + +cat > "$state_dir/buildkitd.toml" <<'TOML' [worker.oci] max-parallelism = 1 networkMode = "host" [registry."docker.io"] mirrors = ["mirror.gcr.io"] TOML - if [[ -n ${NODE_EXTRA_CA_CERTS:-} && -f $NODE_EXTRA_CA_CERTS ]]; then - ca_path=$(node -p 'JSON.stringify(process.env.NODE_EXTRA_CA_CERTS)') - printf ' ca = [%s]\n[registry."mirror.gcr.io"]\n ca = [%s]\n' "$ca_path" "$ca_path" >> .wrangler/kilo-startup/buildkitd.toml - fi - if ! docker buildx inspect "$KILO_STARTUP_BUILDER" >/dev/null 2>&1; then - docker buildx create --name "$KILO_STARTUP_BUILDER" --driver docker-container \ - --driver-opt "image=mirror.gcr.io/moby/buildkit:v0.16.0,memory=2g,memory-swap=2g,network=host,cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" \ - --buildkitd-config "$PWD/.wrangler/kilo-startup/buildkitd.toml" \ - --buildkitd-flags '--allow-insecure-entitlement network.host' - fi - timeout 3m docker buildx inspect --bootstrap "$KILO_STARTUP_BUILDER" - docker inspect "buildx_buildkit_${KILO_STARTUP_BUILDER}0" --format '{{json .HostConfig}}' | node -e ' - let input = ""; - process.stdin.on("data", chunk => { input += chunk; }); - process.stdin.on("end", () => { - const config = JSON.parse(input); - const parent = process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/containers"; - if (config.Memory !== 2147483648 || config.CgroupParent !== parent) throw new Error("BuildKit is not inside its required memory budget"); - }); - ' +if [[ -n ${NODE_EXTRA_CA_CERTS:-} && -f $NODE_EXTRA_CA_CERTS ]]; then + ca_path=$(node -p 'JSON.stringify(process.env.NODE_EXTRA_CA_CERTS)') + printf ' ca = [%s]\n[registry."mirror.gcr.io"]\n ca = [%s]\n' "$ca_path" "$ca_path" >> "$state_dir/buildkitd.toml" fi -if tmux list-sessions >/dev/null 2>&1; then - tmux set-environment -g WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 - tmux set-environment -g GOMAXPROCS "$GOMAXPROCS" - tmux set-environment -g RAYON_NUM_THREADS "$RAYON_NUM_THREADS" - tmux set-environment -g NODE_OPTIONS "$NODE_OPTIONS" - tmux set-environment -g SHELL "$SHELL" - tmux set-environment -g KILO_ENV_SYNC_CONCURRENCY 1 - tmux set-environment -g KILO_STARTUP_BUILDER "$KILO_STARTUP_BUILDER" - tmux set-environment -g KILO_STARTUP_REAL_DOCKER "$KILO_STARTUP_REAL_DOCKER" - if [[ -n ${WRANGLER_DOCKER_BIN:-} ]]; then - tmux set-environment -g WRANGLER_DOCKER_BIN "$WRANGLER_DOCKER_BIN" - if [[ -n ${NODE_EXTRA_CA_CERTS:-} ]]; then - tmux set-environment -g NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" - fi - fi +if ! docker buildx inspect "$KILO_STARTUP_BUILDER" >/dev/null 2>&1; then + docker buildx create --name "$KILO_STARTUP_BUILDER" --driver docker-container \ + --driver-opt "image=mirror.gcr.io/moby/buildkit:v0.16.0,memory=2g,memory-swap=2g,network=host,cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" \ + --buildkitd-config "$state_dir/buildkitd.toml" \ + --buildkitd-flags '--allow-insecure-entitlement network.host' fi +timeout 3m docker buildx inspect --bootstrap "$KILO_STARTUP_BUILDER" >/dev/null +docker inspect "buildx_buildkit_${KILO_STARTUP_BUILDER}0" --format '{{json .HostConfig}}' | node -e ' + let input = ""; + process.stdin.on("data", chunk => { input += chunk; }); + process.stdin.on("end", () => { + const config = JSON.parse(input); + const parent = process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/containers"; + if (config.Memory !== 2147483648 || config.CgroupParent !== parent) throw new Error("BuildKit is not inside its required memory budget"); + }); +' +# Buildx restarts the builder on demand; keep its 2 GiB out of the budget until an image build needs it. +docker stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" >/dev/null timeout 15m pnpm install --frozen-lockfile --child-concurrency=1 --network-concurrency=4 if [[ ! -s .env.local ]]; then @@ -314,87 +341,32 @@ node -e ' const match = fs.readFileSync(".env.local", "utf8").match(/^POSTGRES_URL\s*=\s*(.*)$/m); const value = process.env.POSTGRES_URL || match?.[1].trim().replace(/^(["\x27])(.*)\1$/, (_, quote, inner) => inner); if (!value || !["localhost", "127.0.0.1", "[::1]"].includes(new URL(value).hostname)) { - throw new Error("Sandbox startup requires a local POSTGRES_URL; remote databases will not be migrated or seeded"); + throw new Error("Sandbox setup requires a local POSTGRES_URL; remote databases will not be migrated or seeded"); } ' # Avoid Docker Hub's shared unauthenticated pull limit without changing Compose files. while IFS= read -r image; do - if ! docker image inspect "$image" >/dev/null 2>&1; then - if docker pull "mirror.gcr.io/$image"; then - docker tag "mirror.gcr.io/$image" "$image" - else - docker pull "$image" - fi + if docker image inspect "$image" >/dev/null 2>&1; then + continue + fi + hub_image=${image#docker.io/} + registry=${hub_image%%/*} + if [[ $hub_image == */* && ($registry == *.* || $registry == *:* || $registry == localhost) ]]; then + docker pull "$image" + continue + fi + [[ $hub_image == */* ]] || hub_image="library/$hub_image" + if docker pull "mirror.gcr.io/$hub_image"; then + docker tag "mirror.gcr.io/$hub_image" "$image" + else + docker pull "$image" fi done < <(docker compose -f dev/docker-compose.yml config --images | sort -u) -mkdir -p .wrangler/kilo-startup -selection=$(printf '%s\n' "$@") -status=$(pnpm -s dev:status --json) -if node -e 'process.exit(JSON.parse(process.argv[1]).services.length ? 0 : 1)' "$status"; then - if [[ ! -f .wrangler/kilo-startup/selection || $(< .wrangler/kilo-startup/selection) != "$selection" ]]; then - printf 'A different dev stack is already running. Stop it with pnpm dev:stop before changing the selection.\n' >&2 - exit 1 - fi - printf 'Reusing this sandbox startup script\x27s existing dev stack.\n' - while IFS= read -r service; do - printf 'Restarting unavailable service: %s\n' "$service" - timeout 1m pnpm dev:restart "$service" - done < <(node -e 'for (const s of JSON.parse(process.argv[1]).services) if (s.status !== "up") console.log(s.name)' "$status") -else - timeout --foreground 5m pnpm dev:start --no-attach "$@" - printf '%s\n' "$selection" > .wrangler/kilo-startup/selection -fi printf 'Preparing the local database (up to 5 minutes).\n' timeout --foreground 5m pnpm test:db -web_port=$(node -e ' - const fs = require("node:fs"); - const manifest = JSON.parse(fs.readFileSync("dev/logs/manifest.json", "utf8")); - const service = manifest.services.find(service => service.name === "nextjs"); - if (!service?.port) throw new Error("The selected stack must include nextjs"); - console.log(service.port); -') -web_url="http://localhost:$web_port" -ready=false -for (( attempt=0; attempt<90; attempt++ )); do - if curl --fail --silent --output /dev/null --max-time 10 "$web_url/users/sign_in"; then - ready=true - break - fi - sleep 2 -done -if [[ $ready != true ]]; then - printf 'Web app did not become ready at %s.\n' "$web_url" >&2 - pnpm dev:status --json - exit 1 -fi - -if node -e 'const m = require("./dev/logs/manifest.json"); process.exit(m.services.some(s => s.name === "cloud-agent-next") ? 0 : 1)'; then - printf 'Waiting for Cloud Agent images (up to 15 minutes). Build output: dev/logs/cloud-agent-next.log\n' - ready=false - for (( attempt=0; attempt<450; attempt++ )); do - if grep -Fq 'Container image(s) ready' dev/logs/cloud-agent-next.log; then - ready=true - break - fi - if grep -Fq '[ERROR]' dev/logs/cloud-agent-next.log; then - printf 'Cloud Agent image preparation failed. See dev/logs/cloud-agent-next.log.\n' >&2 - exit 1 - fi - if (( attempt > 0 && attempt % 30 == 0 )); then - printf 'Cloud Agent image preparation is still running (%s seconds).\n' "$(( attempt * 2 ))" - fi - sleep 2 - done - if [[ $ready != true ]]; then - printf 'Cloud Agent images did not become ready within 15 minutes. See dev/logs/cloud-agent-next.log.\n' >&2 - exit 1 - fi - docker stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" -fi - test_email="kilo-$(basename "$HOME")-$(date -u +%Y%m%d%H%M%S)@example.com" test_user_id=$(docker compose -f dev/docker-compose.yml exec -T postgres \ psql -U postgres -d postgres -X -qAt -v ON_ERROR_STOP=1 -v email="$test_email" <<'SQL' @@ -408,42 +380,12 @@ INSERT INTO kilocode_users ( SQL ) pnpm dev:seed app:add-credits "$test_user_id" 100 --free +printf 'export KILO_TEST_USER_EMAIL=%q\n' "$test_email" >> "$env_file" -status=$(pnpm -s dev:status --json) -node -e ' - const status = JSON.parse(process.argv[1]); - const unavailable = status.services.filter(service => service.status !== "up"); - if (unavailable.length) throw new Error("Services are not ready: " + unavailable.map(s => s.name).join(", ")); -' "$status" -printf '%s\n' "$status" -export KILO_DEV_WEB_URL="$web_url" -export KILO_TEST_LOGIN_URL="$web_url/users/sign_in?fakeUser=$test_email&callbackPath=/" -printf 'export AGENT_BROWSER_ENGINE=%q AGENT_BROWSER_EXECUTABLE_PATH=%q AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q KILO_DEV_WEB_URL=%q KILO_TEST_LOGIN_URL=%q PATH=%q KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_REAL_PNPM=%q KILO_STARTUP_BUILDER=%q NODE_OPTIONS=%q SHELL=%q\n' \ - "$AGENT_BROWSER_ENGINE" \ - "$AGENT_BROWSER_EXECUTABLE_PATH" "$AGENT_BROWSER_SOCKET_DIR" "$AGENT_BROWSER_ARGS" \ - "$AGENT_BROWSER_DEFAULT_TIMEOUT" "$KILO_DEV_WEB_URL" "$KILO_TEST_LOGIN_URL" \ - "$PATH" "$KILO_STARTUP_REAL_DOCKER" "$KILO_STARTUP_REAL_PNPM" "$KILO_STARTUP_BUILDER" "$NODE_OPTIONS" "$SHELL" \ - > .wrangler/kilo-startup/browser.env -if [[ ${KILO_STARTUP_BROWSER_SMOKE:-true} == true ]]; then - agent-browser --session kilo-startup cookies clear - agent-browser --session kilo-startup open "$KILO_TEST_LOGIN_URL" - agent-browser --session kilo-startup wait --fn 'window.location.pathname === "/"' - authenticated=$(agent-browser --session kilo-startup eval \ - "(async () => { const session = await (await fetch(\"/api/auth/session\")).json(); return window.location.pathname === \"/\" && session.user?.email === \"$test_email\"; })()") - if [[ $authenticated != true ]]; then - agent-browser --session kilo-startup close - printf 'Seeded browser authentication did not match.\n' >&2 - exit 1 - fi - agent-browser --session kilo-startup snapshot -i - agent-browser --session kilo-startup close - printf 'Verified authenticated browser session email: %s\n' "$test_email" -fi -printf '\nWeb app: %s\nFake test-account login: %s/users/sign_in?fakeUser=%s&callbackPath=/\n' \ - "$web_url" "$web_url" "$test_email" -printf 'Browser setup: source .wrangler/kilo-startup/browser.env\n' -printf 'Browser: agent-browser --session kilo-startup open %q, then agent-browser --session kilo-startup snapshot -i\n' "$KILO_TEST_LOGIN_URL" -printf 'Cloud Agent testing: select kilo/fake-deterministic for local inference; real inference needs provider credentials.\n' -printf 'Default profile is app. For Cloud Agents: bash .kilo/cloud-agent-startup.sh agents fake-llm (after pnpm dev:stop).\n' -printf 'Memory peak: %s MiB / %s MiB hard limit.\n' "$(( $(< "$KILO_STARTUP_CGROUP/memory.peak") / 1048576 ))" "$KILO_STARTUP_MEMORY_MB" -printf 'Manage services with pnpm dev:status, pnpm dev:restart , and pnpm dev:stop.\n' +printf '\nSetup complete. Dev workload memory peak so far: %s MiB / %s MiB.\n' \ + "$(( $(< "$KILO_STARTUP_CGROUP/memory.peak") / 1048576 ))" "$KILO_STARTUP_MEMORY_MB" +printf 'Run from %s (pnpm there joins the capped cgroup and loads %s):\n' "$repo" "$env_file" +printf ' pnpm dev:start --no-attach app # web app\n' +printf ' pnpm dev:start --no-attach agents fake-llm # Cloud Agents with local fake inference\n' +printf 'Then pnpm dev:status for ports; log in at http://localhost:/users/sign_in?fakeUser=%s&callbackPath=/profile\n' "$test_email" +printf 'Other shells (docker, agent-browser): source %q\n' "$env_file" From 572535c18c832746733b3b43ae29c6ae1ae6c587 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:12:07 +0000 Subject: [PATCH 27/35] fix(dev): keep values dev:start resolves when pnpm loads the sandbox env --- .kilo/cloud-agent-setup.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index a37a2c1253..01a145dafd 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -1,7 +1,12 @@ #!/usr/bin/env bash set -Eeuo pipefail -trap 'status=$?; printf "Setup failed at line %s (exit %s).\n" "$LINENO" "$status" >&2; exit "$status"' ERR +on_error() { + local status=$? + printf 'Setup failed at line %s (exit %s).\n' "$1" "$status" >&2 + exit "$status" +} +trap 'on_error "$LINENO"' ERR cd "$(dirname "${BASH_SOURCE[0]}")/.." repo=$PWD @@ -221,19 +226,25 @@ JS chmod +x "$state_dir/sandbox-docker.cjs" ln -sf "$state_dir/sandbox-docker.cjs" "$startup_bin/docker" +env_default() { + printf '[[ -n ${%s:-} ]] || %s=%q; export %s\n' "$1" "$1" "$2" "$1" +} { printf 'case ":$PATH:" in *:%q:*) ;; *) export PATH=%q:"$PATH" ;; esac\n' "$startup_bin" "$startup_bin" - for name in NEXT_TELEMETRY_DISABLED SKIP_STRIPE_API KILO_PORT_OFFSET KILO_ENV_SYNC_CONCURRENCY KILO_STARTUP_CGROUP; do - printf 'export %s=%q\n' "$name" "${!name}" - done - printf 'export KILO_STARTUP_REAL_DOCKER=%q KILO_STARTUP_BUILDER=%q\n' "$real_docker" "$KILO_STARTUP_BUILDER" - printf 'export WRANGLER_DOCKER_BIN=%q WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST=1\n' "$state_dir/sandbox-docker.cjs" printf 'export SHELL=%q\n' "$startup_bin/kilo-shell" - printf 'export AGENT_BROWSER_ENGINE=chrome AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium\n' - printf 'export AGENT_BROWSER_SOCKET_DIR=%q AGENT_BROWSER_ARGS=%q AGENT_BROWSER_DEFAULT_TIMEOUT=%q\n' \ - "${AGENT_BROWSER_SOCKET_DIR:-/tmp/kilo-browser}" "${AGENT_BROWSER_ARGS:---disable-gpu}" "${AGENT_BROWSER_DEFAULT_TIMEOUT:-120000}" + for name in NEXT_TELEMETRY_DISABLED SKIP_STRIPE_API KILO_PORT_OFFSET KILO_ENV_SYNC_CONCURRENCY KILO_STARTUP_CGROUP KILO_STARTUP_BUILDER; do + env_default "$name" "${!name}" + done + env_default KILO_STARTUP_REAL_DOCKER "$real_docker" + env_default WRANGLER_DOCKER_BIN "$state_dir/sandbox-docker.cjs" + env_default WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 + env_default AGENT_BROWSER_ENGINE chrome + env_default AGENT_BROWSER_EXECUTABLE_PATH /usr/bin/chromium + env_default AGENT_BROWSER_SOCKET_DIR /tmp/kilo-browser + env_default AGENT_BROWSER_ARGS --disable-gpu + env_default AGENT_BROWSER_DEFAULT_TIMEOUT 120000 if [[ -n ${NODE_EXTRA_CA_CERTS:-} ]]; then - printf 'export NODE_EXTRA_CA_CERTS=%q\n' "$NODE_EXTRA_CA_CERTS" + env_default NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" fi } > "$env_file" @@ -380,7 +391,7 @@ INSERT INTO kilocode_users ( SQL ) pnpm dev:seed app:add-credits "$test_user_id" 100 --free -printf 'export KILO_TEST_USER_EMAIL=%q\n' "$test_email" >> "$env_file" +env_default KILO_TEST_USER_EMAIL "$test_email" >> "$env_file" printf '\nSetup complete. Dev workload memory peak so far: %s MiB / %s MiB.\n' \ "$(( $(< "$KILO_STARTUP_CGROUP/memory.peak") / 1048576 ))" "$KILO_STARTUP_MEMORY_MB" From 9ef02957ed7700d6fe09c6488f41bd821c43fc33 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:31:52 +0000 Subject: [PATCH 28/35] fix(dev): release the BuildKit budget after each sandbox image build --- .kilo/cloud-agent-setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index 01a145dafd..a5e5dbf251 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -196,8 +196,10 @@ function run(input) { if (source !== -1) args.splice(source + 2, 0, '-f', path.join(__dirname, 'compose.memory.yml')); } if (build) args.splice(0, 1, 'buildx', 'build', '--builder', process.env.KILO_STARTUP_BUILDER, '--allow=network.host'); + // Stop the builder after each build so its 2 GiB does not stay inside the dev workload budget; buildx restarts it on demand. + const buildScript = '"$KILO_STARTUP_REAL_DOCKER" "$@"; status=$?; "$KILO_STARTUP_REAL_DOCKER" stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" >/dev/null 2>&1; exit $status'; const command = build ? 'flock' : process.env.KILO_STARTUP_REAL_DOCKER; - const commandArgs = build ? [path.join(__dirname, 'image-build.lock'), process.env.KILO_STARTUP_REAL_DOCKER, ...args] : args; + const commandArgs = build ? [path.join(__dirname, 'image-build.lock'), 'sh', '-c', buildScript, 'sandbox-docker-build', ...args] : args; const child = spawn(command, commandArgs, { stdio: [input === undefined ? 'inherit' : 'pipe', 'inherit', 'inherit'] }); for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); child.on('error', error => { console.error(error.message); process.exitCode = 1; }); @@ -339,7 +341,6 @@ docker inspect "buildx_buildkit_${KILO_STARTUP_BUILDER}0" --format '{{json .Host if (config.Memory !== 2147483648 || config.CgroupParent !== parent) throw new Error("BuildKit is not inside its required memory budget"); }); ' -# Buildx restarts the builder on demand; keep its 2 GiB out of the budget until an image build needs it. docker stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" >/dev/null timeout 15m pnpm install --frozen-lockfile --child-concurrency=1 --network-concurrency=4 From a924ee5693556d3f5f25f0efc866907e787b4cc1 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:34:55 +0000 Subject: [PATCH 29/35] fix(dev): preserve a standalone pnpm binary before installing the wrapper --- .kilo/cloud-agent-setup.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index a5e5dbf251..34a9e3fb95 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -129,6 +129,12 @@ else global_pnpm=$(command -v pnpm) fi real_pnpm=$(readlink -f "$global_pnpm") + if [[ ! -L $global_pnpm ]]; then + # The wrapper replaces this path, so move a standalone pnpm binary out of the way first. + "${root[@]}" mkdir -p "$global_state_dir" + "${root[@]}" mv "$global_pnpm" "$global_state_dir/pnpm" + real_pnpm="$global_state_dir/pnpm" + fi fi if [[ ! -x $real_pnpm ]] || grep -qF "$pnpm_wrapper_marker" "$real_pnpm"; then printf 'Could not locate the real pnpm executable (resolved %s).\n' "$real_pnpm" >&2 From 7909ddc13fdd49643a1e4fe37aad39e99be891b2 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:57:50 +0000 Subject: [PATCH 30/35] fix(dev): resolve DNS for sandbox containers without IP forwarding --- .kilo/cloud-agent-setup.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index 34a9e3fb95..b2e18aed80 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -29,6 +29,7 @@ export KILO_ENV_SYNC_CONCURRENCY=1 export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" export KILO_PORT_OFFSET="${KILO_PORT_OFFSET:-auto}" KILO_STARTUP_RESERVE_MB=2048 +bridge_gateway=172.17.0.1 if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then printf 'This setup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 @@ -104,7 +105,7 @@ else docker_packages+=(docker-compose) fi "${root[@]}" env DEBIAN_FRONTEND=noninteractive timeout --foreground 10m apt-get install -y --no-install-recommends \ - ca-certificates chromium curl fuse-overlayfs git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" + ca-certificates chromium curl dnsmasq-base fuse-overlayfs git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" if ! docker compose version >/dev/null 2>&1; then printf 'Docker Compose v2 (the docker compose CLI plugin) is required, but %s did not provide it.\n' "${docker_packages[-1]}" >&2 exit 1 @@ -298,8 +299,9 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi # Own the socket by the invoking user's group so a non-root sandbox can use the daemon it starts. + # Without IP forwarding, bridge containers reach only the host, so their DNS goes to a forwarder on the bridge gateway. "${root[@]}" tmux new-session -d -s kilo-startup-docker \ - "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" + "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --bip=$bridge_gateway/16 --dns=$bridge_gateway --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" for (( attempt=0; attempt<30; attempt++ )); do docker info >/dev/null 2>&1 && break sleep 1 @@ -310,6 +312,9 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi fi +if pgrep -a -x dockerd | grep -qF -- "--dns=$bridge_gateway" && ! pgrep -f "dnsmasq .*--listen-address=$bridge_gateway" >/dev/null; then + "${root[@]}" dnsmasq --conf-file=/dev/null --no-hosts --bind-interfaces --listen-address="$bridge_gateway" +fi if [[ $(docker info --format '{{.Driver}}') == vfs ]]; then printf 'The existing Docker daemon uses vfs. Stop it and restart with overlay2 or fuse-overlayfs before running this workload.\n' >&2 exit 1 From f910993c6fe941cf2df6097d4724519db263daee Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:57:51 +0000 Subject: [PATCH 31/35] feat(dev): let dev:start skip services with --without or KILO_DEV_WITHOUT --- dev/local/cli.ts | 25 +++++++++++++++++++++++-- dev/local/services.test.ts | 32 ++++++++++++++++++++++++++++++++ dev/local/services.ts | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/dev/local/cli.ts b/dev/local/cli.ts index 2867776134..4c7613b9dd 100644 --- a/dev/local/cli.ts +++ b/dev/local/cli.ts @@ -6,6 +6,8 @@ import { applyPortOffset, candidatePortOffsets, clearDevLogs, + excludeServices, + parseServiceList, resolveTargets, getService, getGroups, @@ -433,7 +435,14 @@ function removeStaleComposeProject(repoRoot: string, previousOffset: number | un async function cmdUp(args: string[], repoRoot: string): Promise { const noAttach = args.includes('--no-attach'); const reuseRunning = args.includes('--reuse-running'); - const targets = args.filter(arg => arg !== '--no-attach' && arg !== '--reuse-running'); + const withoutArg = args.findLast(arg => arg.startsWith('--without=')); + const withoutSource = withoutArg === undefined ? 'KILO_DEV_WITHOUT' : '--without'; + const without = parseServiceList( + withoutArg === undefined ? process.env.KILO_DEV_WITHOUT : withoutArg.slice('--without='.length) + ); + const targets = args.filter( + arg => arg !== '--no-attach' && arg !== '--reuse-running' && !arg.startsWith('--without=') + ); // --- Preflight checks --- if (!isTmuxAvailable()) { @@ -465,6 +474,16 @@ async function cmdUp(args: string[], repoRoot: string): Promise 0) { + const exclusion = excludeServices(serviceNames, without); + serviceNames = exclusion.serviceNames; + if (exclusion.skipped.length > 0) { + console.log(`${DIM}Skipping (${withoutSource}): ${exclusion.skipped.join(', ')}${RESET}`); + } + for (const [name, missing] of exclusion.dependents) { + console.warn(`⚠ ${name} runs without ${missing.join(', ')}; calls to them will fail.`); + } + } const sessionName = getSessionName(); let sessionAlreadyRunning = sessionExists(sessionName); @@ -1488,9 +1507,11 @@ async function cmdEnv(args: string[], repoRoot: string): Promise { function printUsage(): void { console.log(` Usage: - dev:start [--no-attach] [--reuse-running] [targets...] + dev:start [--no-attach] [--reuse-running] [--without=a,b] [targets...] Start services (default: core) --reuse-running never restarts an existing complete stack + --without skips the named services (default: $KILO_DEV_WITHOUT; + --without= starts everything) dev:stop [--force] Stop all services (skips shared Docker infra if other kilo-dev sessions are running; --force overrides) dev:status [--json] Show running services and their ports diff --git a/dev/local/services.test.ts b/dev/local/services.test.ts index b612781c37..04b449721d 100644 --- a/dev/local/services.test.ts +++ b/dev/local/services.test.ts @@ -9,8 +9,10 @@ import { candidatePortOffsets, clearDevLogs, computePortOffset, + excludeServices, getAlwaysOnGroupIds, getService, + parseServiceList, portOffset, readPersistedPortOffset, resolveGroups, @@ -577,3 +579,33 @@ test('a tunnels restart keeps the live selection and reloads the HTTP worker', ( ); assert.equal(planTunnelRestart(['cloud-agent-public-tunnels']).reloadTarget, undefined); }); + +test('excludes unwanted services and reports dependents that lose them', () => { + const selection = resolveTargets(['agents', 'fake-llm']); + const exclusion = excludeServices(selection, ['notifications', 'event-service']); + + assert.ok(!exclusion.serviceNames.includes('notifications')); + assert.ok(!exclusion.serviceNames.includes('event-service')); + assert.ok(exclusion.serviceNames.includes('cloud-agent-next')); + assert.deepEqual(exclusion.skipped.toSorted(), ['event-service', 'notifications']); + assert.deepEqual(exclusion.dependents.get('cloud-agent-next'), ['notifications']); +}); + +test('ignores excluded services that are not selected', () => { + const exclusion = excludeServices(['postgres', 'nextjs'], ['notifications']); + + assert.deepEqual(exclusion.serviceNames, ['postgres', 'nextjs']); + assert.deepEqual(exclusion.skipped, []); +}); + +test('rejects unknown excluded services instead of starting everything', () => { + assert.throws(() => excludeServices(['postgres'], ['notifcations']), /Unknown service/); +}); + +test('parses comma-separated service lists', () => { + assert.deepEqual(parseServiceList(' notifications, event-service ,,'), [ + 'notifications', + 'event-service', + ]); + assert.deepEqual(parseServiceList(undefined), []); +}); diff --git a/dev/local/services.ts b/dev/local/services.ts index e169b39690..444e94dbdd 100644 --- a/dev/local/services.ts +++ b/dev/local/services.ts @@ -1160,6 +1160,44 @@ export function resolveTargets(targets: string[]): string[] { return topologicalSort(resolveTransitiveDeps(allNames)); } +export type ServiceExclusion = { + serviceNames: string[]; + skipped: string[]; + /** Selected services that declare a dependency on a skipped service. */ + dependents: Map; +}; + +/** + * Drop explicitly unwanted services from a resolved selection, for memory- + * constrained environments that run only what they exercise. Unknown names + * throw so a typo cannot silently start the full stack. + */ +export function excludeServices( + serviceNames: readonly string[], + excluded: readonly string[] +): ServiceExclusion { + for (const name of excluded) getService(name); + const excludedSet = new Set(excluded); + const kept = serviceNames.filter(name => !excludedSet.has(name)); + const dependents = new Map(); + for (const name of kept) { + const missing = getService(name).dependsOn.filter(dep => excludedSet.has(dep)); + if (missing.length > 0) dependents.set(name, missing); + } + return { + serviceNames: kept, + skipped: serviceNames.filter(name => excludedSet.has(name)), + dependents, + }; +} + +export function parseServiceList(value: string | undefined): string[] { + return (value ?? '') + .split(',') + .map(name => name.trim()) + .filter(name => name !== ''); +} + export function getService(name: string): ServiceDef { const svc = services.get(name); if (!svc) throw new Error(`Unknown service: ${name}`); From 59b810bec4504d070c084a0b8bb285b42a9a75f1 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:19:18 +0000 Subject: [PATCH 32/35] fix(dev): redirect pinned sandbox DNS and skip unneeded agents services --- .kilo/cloud-agent-setup.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index b2e18aed80..be267ef840 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -252,6 +252,8 @@ env_default() { env_default AGENT_BROWSER_SOCKET_DIR /tmp/kilo-browser env_default AGENT_BROWSER_ARGS --disable-gpu env_default AGENT_BROWSER_DEFAULT_TIMEOUT 120000 + # The full agents stack does not fit the sandbox budget; these are not needed for fake-LLM sessions on public repositories. + env_default KILO_DEV_WITHOUT notifications,event-service,cloudflare-webhook-agent-ingest,container-usage-meter,cloudflare-git-token-service if [[ -n ${NODE_EXTRA_CA_CERTS:-} ]]; then env_default NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" fi @@ -299,9 +301,8 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi # Own the socket by the invoking user's group so a non-root sandbox can use the daemon it starts. - # Without IP forwarding, bridge containers reach only the host, so their DNS goes to a forwarder on the bridge gateway. "${root[@]}" tmux new-session -d -s kilo-startup-docker \ - "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --bip=$bridge_gateway/16 --dns=$bridge_gateway --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" + "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --bip=$bridge_gateway/16 --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" for (( attempt=0; attempt<30; attempt++ )); do docker info >/dev/null 2>&1 && break sleep 1 @@ -312,8 +313,16 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi fi -if pgrep -a -x dockerd | grep -qF -- "--dns=$bridge_gateway" && ! pgrep -f "dnsmasq .*--listen-address=$bridge_gateway" >/dev/null; then - "${root[@]}" dnsmasq --conf-file=/dev/null --no-hosts --bind-interfaces --listen-address="$bridge_gateway" +if pgrep -a -x dockerd | grep -qF -- "--bip=$bridge_gateway/16"; then + # Without IP forwarding, bridge containers reach only the host. Workerd pins sandbox + # containers to public resolvers, so redirect all their DNS to a forwarder on the gateway. + if ! pgrep -f "dnsmasq .*--listen-address=$bridge_gateway" >/dev/null; then + "${root[@]}" dnsmasq --conf-file=/dev/null --no-hosts --bind-interfaces --listen-address="$bridge_gateway" + fi + for proto in udp tcp; do + dns_redirect=(PREROUTING -i docker0 -p "$proto" --dport 53 -j DNAT --to-destination "$bridge_gateway:53") + "${root[@]}" iptables -t nat -C "${dns_redirect[@]}" 2>/dev/null || "${root[@]}" iptables -t nat -I "${dns_redirect[@]}" + done fi if [[ $(docker info --format '{{.Driver}}') == vfs ]]; then printf 'The existing Docker daemon uses vfs. Stop it and restart with overlay2 or fuse-overlayfs before running this workload.\n' >&2 @@ -410,5 +419,6 @@ printf '\nSetup complete. Dev workload memory peak so far: %s MiB / %s MiB.\n' \ printf 'Run from %s (pnpm there joins the capped cgroup and loads %s):\n' "$repo" "$env_file" printf ' pnpm dev:start --no-attach app # web app\n' printf ' pnpm dev:start --no-attach agents fake-llm # Cloud Agents with local fake inference\n' +printf 'KILO_DEV_WITHOUT skips services the sandbox does not need; pass --without= to start everything.\n' printf 'Then pnpm dev:status for ports; log in at http://localhost:/users/sign_in?fakeUser=%s&callbackPath=/profile\n' "$test_email" printf 'Other shells (docker, agent-browser): source %q\n' "$env_file" From ff097ab75fcb7f45016bb367da421cf9fe7f09f5 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:30:40 +0000 Subject: [PATCH 33/35] docs(dev): add cloud-agent-sandbox skill for the sandbox setup --- .kilo/cloud-agent-setup.sh | 1 + .kilo/skills/cloud-agent-sandbox/SKILL.md | 181 ++++++++++++++++++++++ .kilo/skills/local-development/SKILL.md | 2 + AGENTS.md | 1 + 4 files changed, 185 insertions(+) create mode 100644 .kilo/skills/cloud-agent-sandbox/SKILL.md diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index be267ef840..9831db215d 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -422,3 +422,4 @@ printf ' pnpm dev:start --no-attach agents fake-llm # Cloud Agents with loc printf 'KILO_DEV_WITHOUT skips services the sandbox does not need; pass --without= to start everything.\n' printf 'Then pnpm dev:status for ports; log in at http://localhost:/users/sign_in?fakeUser=%s&callbackPath=/profile\n' "$test_email" printf 'Other shells (docker, agent-browser): source %q\n' "$env_file" +printf 'Usage, limits, and troubleshooting: .kilo/skills/cloud-agent-sandbox/SKILL.md\n' diff --git a/.kilo/skills/cloud-agent-sandbox/SKILL.md b/.kilo/skills/cloud-agent-sandbox/SKILL.md new file mode 100644 index 0000000000..6d6ba335ad --- /dev/null +++ b/.kilo/skills/cloud-agent-sandbox/SKILL.md @@ -0,0 +1,181 @@ +--- +name: cloud-agent-sandbox +description: Sets up and runs this monorepo inside a memory-constrained Kilo Cloud Agent sandbox with `.kilo/cloud-agent-setup.sh`. Use when working in a Cloud Agent sandbox, before starting local services there, after a sandbox restart, or when debugging the dev memory cap, Docker, DNS, fake login, agent-browser, or fake-LLM Cloud Agent sessions in the sandbox. +--- + +# Cloud Agent sandbox + +`.kilo/cloud-agent-setup.sh` prepares a Debian/Ubuntu Cloud Agent sandbox for +`pnpm dev:start`. It usually runs automatically when the machine starts. You can +run it at any time: reruns are safe and take about 20 s once the machine is set up. + +Follow the `local-development` skill for ports, fake login, and service +management. This skill covers what is different in the sandbox. + +## Is setup done? + +Sandbox restarts wipe Docker, the pnpm wrapper, the memory cgroup, and +`.wrangler/kilo-startup/`. The repository and `node_modules` survive. Check +before starting services: + +```bash +test -f .wrangler/kilo-startup/env \ + && grep -q kilo-cloud-agent-pnpm-wrapper "$(command -v pnpm)" \ + && docker info >/dev/null 2>&1 && echo ready +``` + +If this does not print `ready`, run setup from the repository root: + +```bash +bash .kilo/cloud-agent-setup.sh +``` + +A restarted machine takes about 90 s, since `node_modules` survives. A machine +without dependencies takes about 4 minutes: apt packages, `pnpm install`, +Compose image pulls, and migrations. Setup stops at the first failure and prints +the line number. + +## What setup does + +- Creates a memory cgroup for all dev workloads, capped at the sandbox memory + minus 2 GiB. Override the cap in MiB with `KILO_STARTUP_MEMORY_MB`; the + minimum is 3072. +- Installs Docker, Compose v2, tmux, Chromium, agent-browser, and dnsmasq. + Starts dockerd with its containers inside the cgroup and pulls images through + `mirror.gcr.io`. +- Installs a global `pnpm` wrapper. Inside this repository it moves the command + into the capped cgroup and loads `.wrangler/kilo-startup/env`. Outside the + repository it runs the real pnpm, saved at + `/usr/local/lib/kilo-cloud-agent/real-pnpm`, unchanged. +- Installs dependencies, creates `.env.local`, runs `pnpm test:db`, and seeds a + fake-login user with credits. +- It does not start the dev stack. + +`.wrangler/kilo-startup/env` only sets defaults. Override a value by exporting +it before running pnpm. Do not edit the file: setup rewrites it. + +## Start services + +Run every command from the repository root, so that pnpm is capped. + +```bash +pnpm dev:start --no-attach app # web app +pnpm dev:start --no-attach agents fake-llm # Cloud Agents with local fake inference +pnpm dev:status # services and ports +pnpm dev:stop +``` + +- `KILO_DEV_WITHOUT` in the env file skips agents services that fake-LLM sessions + on public repositories do not need: notifications, event-service, + webhook-agent-ingest, container-usage-meter, and git-token-service. That + leaves 7 services, about 3 GB when idle. Add services back with + `--without=`. `--without=` starts everything, but the full agents + stack does not fit in the cap. +- Expected warnings without those services: billing-heartbeat errors from the + skipped usage meter. GitHub-backed repositories need git-token-service and + GitHub App credentials, which the sandbox does not have. +- `--no-attach` returns once services are up: about 50 s for `app`. The first + page load compiles with Turbopack and takes about 30 s more. +- The first agents start after a restart builds the Cloud Agent sandbox images. + This can take 10 to 15 minutes. Later starts reuse them. +- `--reuse-running` currently refuses to reuse a session because the Stripe + forwarder is always skipped. Check `pnpm dev:status` instead. + +## Log in and use the browser + +Setup seeds a verified user with credits. Its email is `KILO_TEST_USER_EMAIL` in +the env file. Shells outside the pnpm wrapper must load the env file first. It +also puts setup's `docker` wrapper on `PATH`, which keeps builds serialized and +inside the cap: + +```bash +source .wrangler/kilo-startup/env +agent-browser --session main open "http://localhost:3000/users/sign_in?fakeUser=$KILO_TEST_USER_EMAIL&callbackPath=/profile" +``` + +- Read the real port from `pnpm dev:status`. It is 3000 unless an offset applies. +- The env file points agent-browser at the installed Chromium with a 120 s + timeout. Confirm login with + `agent-browser --session main eval '(async () => (await (await fetch("/api/auth/session")).json()).user?.email)()'`. +- If a click fails because the element is covered, focus the input and use + `agent-browser press Enter`. +- Starting a Cloud Agent session from `/cloud` requires a connected GitHub or + GitLab provider, which the sandbox cannot set up. Create sessions with the + fake-LLM harness instead, then view them in the browser. + +## Fake-LLM Cloud Agent sessions + +Start `agents fake-llm`, then follow `services/cloud-agent-next/test/e2e/README.md`. +Ports below are the defaults; check `pnpm dev:status`. + +```bash +WORKER_URL=http://localhost:8794 FAKE_LLM_URL=http://localhost:8811 \ + pnpm -s exec tsx services/cloud-agent-next/test/e2e/run.ts cold echo:hi +``` + +- Verified passing on the minimal stack: `cold echo:hi`, `cold-hot echo:hi`, + `chunked-streaming slow:5:50`, `queue-while-busy`, and + `--api=legacy cold-hot echo:legacy`. `llm-error boom` fails a retry-status + assertion that is unrelated to the sandbox. +- Each run leaves a sandbox container of about 750 MB until it stops for being + idle. Remove them between runs: + `docker rm -f $(docker ps -q --filter name=workerd-cloud-agent-next-dev-Sandbox)`. +- The first session after `dev:start` can fail model validation with a 503 + ("Model availability could not be verified"). Turbopack is still compiling + the validation route; retry once. +- To view harness sessions in the browser, set `E2E_USER_EMAIL` so runs reuse + one driver user. Mark that user verified, then fake-login as it and open + `/cloud/sessions`: + + ```bash + docker compose -f dev/docker-compose.yml exec -T postgres psql -U postgres -d postgres -c \ + "UPDATE kilocode_users SET has_validation_stytch = true, completed_welcome_form = true WHERE google_user_email = ''" + ``` + + Sending a message to a harness session from the UI fails with + "Session not found". + +## Memory + +The cap is a hard limit with no swap. When the workload nears it, the kernel +reclaims memory instead of killing processes: everything slows, and +`docker`, `tmux`, and `pnpm dev:stop` can hang. Check pressure with: + +```bash +cg=/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD") +echo "$(( $(cat $cg/memory.current) / 1048576 )) MiB of $(( $(cat $cg/memory.max) / 1048576 )) MiB" +grep -E '^(high|max|oom_kill) ' $cg/memory.events +``` + +If the `max` count keeps rising, you are near the cap. Stop work you do not +need, remove idle sandbox containers, or start fewer services. Dev tooling is +heavy: wrangler and workerd use about 3 GB, and the pnpm parents and log filters +about 2 GB. Redis is not the problem; it idles under 1% CPU. + +Run heavy commands such as builds, tests, and typechecks from inside the +repository, so that the wrapper caps them. pnpm outside the repository and +direct `node` or `npx` processes are not capped. + +## Docker networking and DNS + +`/proc/sys` is read-only, so dockerd runs with `--ip-forward=false`: bridge +containers can reach only the host. Workerd also pins sandbox containers to +1.1.1.1 and 8.8.8.8. Setup runs dnsmasq on the bridge gateway, 172.17.0.1, and +uses iptables to redirect all DNS from `docker0` to it. + +A `git_network_failed` clone failure, or `Could not resolve host` inside a +sandbox, means this redirect is missing. Rerun setup, then check: + +```bash +iptables -t nat -S PREROUTING | grep 'dport 53' +pgrep -a dnsmasq +docker exec git ls-remote https://github.com/octocat/Hello-World.git +``` + +## Pitfalls + +- `pkill -f next-server` also matches the shell that runs it. Use + `pkill -f '[n]ext-server'`. +- Do not prune Docker images, volumes, or the BuildKit builder. Rebuilding the + Cloud Agent images costs 10 to 15 minutes. +- Lint setup changes with `shellcheck -S warning -e SC1090 .kilo/cloud-agent-setup.sh`. diff --git a/.kilo/skills/local-development/SKILL.md b/.kilo/skills/local-development/SKILL.md index 688a541e0b..ef0269223d 100644 --- a/.kilo/skills/local-development/SKILL.md +++ b/.kilo/skills/local-development/SKILL.md @@ -5,6 +5,8 @@ description: Start, reuse, inspect, or browser-test local apps and services in t # Local development +In a Kilo Cloud Agent sandbox, load the `cloud-agent-sandbox` skill first. + Read `DEVELOPMENT.md` for human setup and service procedures. Read `ENVIRONMENT.md` for the environment-variable inventory. Shared web environment mutations are governed by `apps/web/AGENTS.md`; do not use this skill for that workflow. ## Start or reuse services diff --git a/AGENTS.md b/AGENTS.md index 54619e40fe..b198eb49db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,7 @@ package manifests before running repository JavaScript or package scripts. Load | TypeScript implementation or review | `code-quality` skill | | Verification or pre-commit checks | `repository-verification` skill | | Local services, ports, and fake login | `local-development` skill | +| Cloud Agent sandbox setup, memory cap, and fake-LLM sessions | `cloud-agent-sandbox` skill and `.kilo/cloud-agent-setup.sh` | | Shared web environment changes | `apps/web/AGENTS.md` and `DEVELOPMENT.md` | | PostgreSQL schema or migration work | `packages/db/AGENTS.md` and `database-migrations` skill | | Service, Durable Object, or Worker code | `services/AGENTS.md`, nearest owning service's `AGENTS.md`, and relevant Durable Objects or Workers skills | From 8b4c0fe421400c60227ffeea64dc356aa318c377 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:44:00 +0000 Subject: [PATCH 34/35] docs(dev): document sandbox image readiness in cloud-agent-sandbox skill --- .kilo/skills/cloud-agent-sandbox/SKILL.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.kilo/skills/cloud-agent-sandbox/SKILL.md b/.kilo/skills/cloud-agent-sandbox/SKILL.md index 6d6ba335ad..10b1a8a8a6 100644 --- a/.kilo/skills/cloud-agent-sandbox/SKILL.md +++ b/.kilo/skills/cloud-agent-sandbox/SKILL.md @@ -76,8 +76,14 @@ pnpm dev:stop GitHub App credentials, which the sandbox does not have. - `--no-attach` returns once services are up: about 50 s for `app`. The first page load compiles with Turbopack and takes about 30 s more. -- The first agents start after a restart builds the Cloud Agent sandbox images. - This can take 10 to 15 minutes. Later starts reuse them. +- The first agents start after a restart builds eight Cloud Agent sandbox + images, about 12 minutes. `cloud-agent-next` reports `up` before they are + ready, and sessions created meanwhile fail (`fetch failed` in the harness). + Wait for the build to finish: + + ```bash + until grep -q 'Container image(s) ready' dev/logs/cloud-agent-next.log; do sleep 15; done + ``` - `--reuse-running` currently refuses to reuse a session because the Stripe forwarder is always skipped. Check `pnpm dev:status` instead. @@ -177,5 +183,6 @@ docker exec git ls-remote https://github.com/octocat/Hello-W - `pkill -f next-server` also matches the shell that runs it. Use `pkill -f '[n]ext-server'`. - Do not prune Docker images, volumes, or the BuildKit builder. Rebuilding the - Cloud Agent images costs 10 to 15 minutes. + Cloud Agent images costs about 12 minutes. - Lint setup changes with `shellcheck -S warning -e SC1090 .kilo/cloud-agent-setup.sh`. + Setup does not install ShellCheck; use `apt-get install -y shellcheck`. From 54841732e5f6ae012b223cd671a07ae06af50742 Mon Sep 17 00:00:00 2001 From: eshurakov <54751+eshurakov@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:07:31 +0000 Subject: [PATCH 35/35] feat(dev): prebuild Cloud Agent sandbox images during sandbox setup Build images with dockerd's own BuildKit in a capped builds cgroup so unchanged images come from the layer cache, and run wrangler dev once during setup to build them. Document Turbopack memory and Redis CPU findings. --- .kilo/cloud-agent-setup.sh | 71 ++++++++++++----------- .kilo/skills/cloud-agent-sandbox/SKILL.md | 39 +++++++++---- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh index 9831db215d..e2fb26d2be 100755 --- a/.kilo/cloud-agent-setup.sh +++ b/.kilo/cloud-agent-setup.sh @@ -160,7 +160,6 @@ fi mkdir -p "$startup_bin" rm -f "$startup_bin/pnpm" "$state_dir/browser.env" "$state_dir/selection" real_docker=$(readlink -f "$(command -v docker)") -KILO_STARTUP_BUILDER="kilo-lowmem-$(basename "$repo")" cat > "$state_dir/compose.memory.yml" < arg === '-f' && path.resolve(args[i + 1]) === path.resolve(__dirname, '../../dev/docker-compose.yml')); if (source !== -1) args.splice(source + 2, 0, '-f', path.join(__dirname, 'compose.memory.yml')); } - if (build) args.splice(0, 1, 'buildx', 'build', '--builder', process.env.KILO_STARTUP_BUILDER, '--allow=network.host'); - // Stop the builder after each build so its 2 GiB does not stay inside the dev workload budget; buildx restarts it on demand. - const buildScript = '"$KILO_STARTUP_REAL_DOCKER" "$@"; status=$?; "$KILO_STARTUP_REAL_DOCKER" stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" >/dev/null 2>&1; exit $status'; + // dockerd's own BuildKit keeps images in the daemon store, so unchanged images rebuild from cache in seconds. + if (build) args.splice(1, 0, `--cgroup-parent=${process.env.KILO_STARTUP_CGROUP.replace('/sys/fs/cgroup', '')}/containers/builds`); const command = build ? 'flock' : process.env.KILO_STARTUP_REAL_DOCKER; - const commandArgs = build ? [path.join(__dirname, 'image-build.lock'), 'sh', '-c', buildScript, 'sandbox-docker-build', ...args] : args; + const commandArgs = build ? [path.join(__dirname, 'image-build.lock'), process.env.KILO_STARTUP_REAL_DOCKER, ...args] : args; const child = spawn(command, commandArgs, { stdio: [input === undefined ? 'inherit' : 'pipe', 'inherit', 'inherit'] }); for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); child.on('error', error => { console.error(error.message); process.exitCode = 1; }); @@ -241,7 +239,7 @@ env_default() { { printf 'case ":$PATH:" in *:%q:*) ;; *) export PATH=%q:"$PATH" ;; esac\n' "$startup_bin" "$startup_bin" printf 'export SHELL=%q\n' "$startup_bin/kilo-shell" - for name in NEXT_TELEMETRY_DISABLED SKIP_STRIPE_API KILO_PORT_OFFSET KILO_ENV_SYNC_CONCURRENCY KILO_STARTUP_CGROUP KILO_STARTUP_BUILDER; do + for name in NEXT_TELEMETRY_DISABLED SKIP_STRIPE_API KILO_PORT_OFFSET KILO_ENV_SYNC_CONCURRENCY KILO_STARTUP_CGROUP; do env_default "$name" "${!name}" done env_default KILO_STARTUP_REAL_DOCKER "$real_docker" @@ -302,7 +300,7 @@ if ! docker info >/dev/null 2>&1; then fi # Own the socket by the invoking user's group so a non-root sandbox can use the daemon it starts. "${root[@]}" tmux new-session -d -s kilo-startup-docker \ - "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --bip=$bridge_gateway/16 --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" + "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --bip=$bridge_gateway/16 --registry-mirror=https://mirror.gcr.io --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" for (( attempt=0; attempt<30; attempt++ )); do docker info >/dev/null 2>&1 && break sleep 1 @@ -331,37 +329,19 @@ fi if tmux list-sessions >/dev/null 2>&1; then while IFS= read -r line; do tmux set-environment -g "${line%%=*}" "${line#*=}" - done < <(bash -c 'source "$1"; for name in PATH SHELL KILO_PORT_OFFSET KILO_STARTUP_CGROUP KILO_STARTUP_BUILDER KILO_STARTUP_REAL_DOCKER WRANGLER_DOCKER_BIN WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST NODE_EXTRA_CA_CERTS; do [[ -n ${!name:-} ]] && printf "%s=%s\n" "$name" "${!name}"; done' _ "$env_file") + done < <(bash -c 'source "$1"; for name in PATH SHELL KILO_PORT_OFFSET KILO_STARTUP_CGROUP KILO_STARTUP_REAL_DOCKER WRANGLER_DOCKER_BIN WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST NODE_EXTRA_CA_CERTS; do [[ -n ${!name:-} ]] && printf "%s=%s\n" "$name" "${!name}"; done' _ "$env_file") fi -cat > "$state_dir/buildkitd.toml" <<'TOML' -[worker.oci] - max-parallelism = 1 - networkMode = "host" -[registry."docker.io"] - mirrors = ["mirror.gcr.io"] -TOML -if [[ -n ${NODE_EXTRA_CA_CERTS:-} && -f $NODE_EXTRA_CA_CERTS ]]; then - ca_path=$(node -p 'JSON.stringify(process.env.NODE_EXTRA_CA_CERTS)') - printf ' ca = [%s]\n[registry."mirror.gcr.io"]\n ca = [%s]\n' "$ca_path" "$ca_path" >> "$state_dir/buildkitd.toml" +legacy_builder="kilo-lowmem-$(basename "$repo")" +if docker buildx inspect "$legacy_builder" >/dev/null 2>&1; then + docker buildx rm --force "$legacy_builder" >/dev/null fi -if ! docker buildx inspect "$KILO_STARTUP_BUILDER" >/dev/null 2>&1; then - docker buildx create --name "$KILO_STARTUP_BUILDER" --driver docker-container \ - --driver-opt "image=mirror.gcr.io/moby/buildkit:v0.16.0,memory=2g,memory-swap=2g,network=host,cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" \ - --buildkitd-config "$state_dir/buildkitd.toml" \ - --buildkitd-flags '--allow-insecure-entitlement network.host' -fi -timeout 3m docker buildx inspect --bootstrap "$KILO_STARTUP_BUILDER" >/dev/null -docker inspect "buildx_buildkit_${KILO_STARTUP_BUILDER}0" --format '{{json .HostConfig}}' | node -e ' - let input = ""; - process.stdin.on("data", chunk => { input += chunk; }); - process.stdin.on("end", () => { - const config = JSON.parse(input); - const parent = process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/containers"; - if (config.Memory !== 2147483648 || config.CgroupParent !== parent) throw new Error("BuildKit is not inside its required memory budget"); - }); -' -docker stop "buildx_buildkit_${KILO_STARTUP_BUILDER}0" >/dev/null +# Image build steps run in their own 2 GiB slice of the dev workload budget. +builds_cgroup="$KILO_STARTUP_CGROUP/containers/builds" +printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/containers/cgroup.subtree_control" >/dev/null +"${root[@]}" mkdir -p "$builds_cgroup" +printf '%s\n' "$(( 2048 * 1048576 ))" | "${root[@]}" tee "$builds_cgroup/memory.max" >/dev/null +printf '0\n' | "${root[@]}" tee "$builds_cgroup/memory.swap.max" >/dev/null timeout 15m pnpm install --frozen-lockfile --child-concurrency=1 --network-concurrency=4 if [[ ! -s .env.local ]]; then @@ -414,6 +394,27 @@ SQL pnpm dev:seed app:add-credits "$test_user_id" 100 --free env_default KILO_TEST_USER_EMAIL "$test_email" >> "$env_file" +# wrangler dev builds every Cloud Agent sandbox image on start. Build them now so dev:start only hits the cache. +if [[ ${KILO_STARTUP_SANDBOX_IMAGES:-1} != 0 ]]; then + printf 'Building Cloud Agent sandbox images (about 10 minutes on a new machine, under a minute when cached).\n' + images_log="$state_dir/sandbox-images.log" + setsid bash -c 'source "$1"; cd services/cloud-agent-next; exec pnpm run dev --port 28794 --inspector-port 38794 --ip 127.0.0.1' \ + _ "$env_file" > "$images_log" 2>&1 < /dev/null & + images_pid=$! + images_deadline=$(( SECONDS + 2400 )) + until grep -qF 'Container image(s) ready' "$images_log"; do + if ! kill -0 "$images_pid" 2>/dev/null || (( SECONDS > images_deadline )); then + kill -KILL -- "-$images_pid" 2>/dev/null || true + tail -20 "$images_log" >&2 + printf 'Building Cloud Agent sandbox images failed; see %s. Rerun setup, or set KILO_STARTUP_SANDBOX_IMAGES=0 to skip.\n' "$images_log" >&2 + exit 1 + fi + sleep 5 + done + kill -TERM -- "-$images_pid" + wait "$images_pid" 2>/dev/null || true +fi + printf '\nSetup complete. Dev workload memory peak so far: %s MiB / %s MiB.\n' \ "$(( $(< "$KILO_STARTUP_CGROUP/memory.peak") / 1048576 ))" "$KILO_STARTUP_MEMORY_MB" printf 'Run from %s (pnpm there joins the capped cgroup and loads %s):\n' "$repo" "$env_file" diff --git a/.kilo/skills/cloud-agent-sandbox/SKILL.md b/.kilo/skills/cloud-agent-sandbox/SKILL.md index 10b1a8a8a6..b14cc04a32 100644 --- a/.kilo/skills/cloud-agent-sandbox/SKILL.md +++ b/.kilo/skills/cloud-agent-sandbox/SKILL.md @@ -30,9 +30,10 @@ If this does not print `ready`, run setup from the repository root: bash .kilo/cloud-agent-setup.sh ``` -A restarted machine takes about 90 s, since `node_modules` survives. A machine -without dependencies takes about 4 minutes: apt packages, `pnpm install`, -Compose image pulls, and migrations. Setup stops at the first failure and prints +A restarted machine takes about 6 minutes, mostly building the Cloud Agent +sandbox images; `node_modules` survives the restart. A rerun on a running +machine takes under 2 minutes, because images come from the build cache. A +machine without dependencies adds about 3 minutes for `pnpm install`. Setup stops at the first failure and prints the line number. ## What setup does @@ -49,6 +50,10 @@ the line number. `/usr/local/lib/kilo-cloud-agent/real-pnpm`, unchanged. - Installs dependencies, creates `.env.local`, runs `pnpm test:db`, and seeds a fake-login user with credits. +- Builds the eight Cloud Agent sandbox images by running `wrangler dev` for + `cloud-agent-next` on spare ports until `Container image(s) ready`. The log is + `.wrangler/kilo-startup/sandbox-images.log`. Skip this with + `KILO_STARTUP_SANDBOX_IMAGES=0`. - It does not start the dev stack. `.wrangler/kilo-startup/env` only sets defaults. Override a value by exporting @@ -76,8 +81,10 @@ pnpm dev:stop GitHub App credentials, which the sandbox does not have. - `--no-attach` returns once services are up: about 50 s for `app`. The first page load compiles with Turbopack and takes about 30 s more. -- The first agents start after a restart builds eight Cloud Agent sandbox - images, about 12 minutes. `cloud-agent-next` reports `up` before they are +- Wrangler rebuilds every sandbox image each time `cloud-agent-next` starts. + Images build in dockerd's own BuildKit, so unchanged images come from the + layer cache. Setup prebuilds them; without that, or after a Dockerfile change, + the build takes minutes. `cloud-agent-next` reports `up` before images are ready, and sessions created meanwhile fail (`fetch failed` in the harness). Wait for the build to finish: @@ -153,10 +160,22 @@ echo "$(( $(cat $cg/memory.current) / 1048576 )) MiB of $(( $(cat $cg/memory.max grep -E '^(high|max|oom_kill) ' $cg/memory.events ``` -If the `max` count keeps rising, you are near the cap. Stop work you do not -need, remove idle sandbox containers, or start fewer services. Dev tooling is +If the `high` or `max` count keeps rising, you are near the cap. Stop work you do +not need, remove idle sandbox containers, or start fewer services. Dev tooling is heavy: wrangler and workerd use about 3 GB, and the pnpm parents and log filters -about 2 GB. Redis is not the problem; it idles under 1% CPU. +about 2 GB. + +- Next.js dev (Turbopack) grows by 1 to 3 GB while compiling a route and gives + most of it back after about two minutes idle: 4.2 GB after login fell to + 2.8 GB, and 4.4 GB after `/cloud` fell to 2.0 GB. Opening many routes back to + back fills the cap before that happens. Pause between heavy routes. Next 16.3 + already defaults `experimental.turbopackMemoryEviction` to `auto`. +- High Redis, redis-http, or Postgres CPU means memory pressure, not load. Under + pressure, Redis used 6 s of user CPU and 1590 s of system CPU in 90 minutes: + the kernel kept evicting and re-reading its code pages. Compare with + `cat $cg/containers/*/cpu.stat`. Idle Redis without pressure uses under 1%. +- If Next.js is stuck above the cap, `pkill -9 -f '^[n]ext-server'`, then + `pnpm dev:restart nextjs`. Run heavy commands such as builds, tests, and typechecks from inside the repository, so that the wrapper caps them. pnpm outside the repository and @@ -182,7 +201,7 @@ docker exec git ls-remote https://github.com/octocat/Hello-W - `pkill -f next-server` also matches the shell that runs it. Use `pkill -f '[n]ext-server'`. -- Do not prune Docker images, volumes, or the BuildKit builder. Rebuilding the - Cloud Agent images costs about 12 minutes. +- Do not prune Docker images or build cache. Rebuilding the Cloud Agent images + costs about 12 minutes. - Lint setup changes with `shellcheck -S warning -e SC1090 .kilo/cloud-agent-setup.sh`. Setup does not install ShellCheck; use `apt-get install -y shellcheck`.