diff --git a/.kilo/cloud-agent-setup.sh b/.kilo/cloud-agent-setup.sh new file mode 100755 index 0000000000..e2fb26d2be --- /dev/null +++ b/.kilo/cloud-agent-setup.sh @@ -0,0 +1,426 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +on_error() { + local status=$? + printf 'Setup failed at line %s (exit %s).\n' "$1" "$status" >&2 + exit "$status" +} +trap 'on_error "$LINENO"' ERR + +cd "$(dirname "${BASH_SOURCE[0]}")/.." +repo=$PWD +state_dir="$repo/.wrangler/kilo-startup" +startup_bin="$state_dir/bin" +env_file="$state_dir/env" +global_state_dir=/usr/local/lib/kilo-cloud-agent +pnpm_wrapper_marker='# kilo-cloud-agent-pnpm-wrapper' + +clean_path= +IFS=: read -ra path_entries <<< "$PATH" +for entry in "${path_entries[@]}"; do + [[ $entry == "$startup_bin" ]] || clean_path+=${clean_path:+:}$entry +done +PATH=$clean_path + +export CI=true +export NEXT_TELEMETRY_DISABLED=1 +export KILO_ENV_SYNC_CONCURRENCY=1 +export SKIP_STRIPE_API="${SKIP_STRIPE_API:-true}" +export KILO_PORT_OFFSET="${KILO_PORT_OFFSET:-auto}" +KILO_STARTUP_RESERVE_MB=2048 +bridge_gateway=172.17.0.1 + +if [[ $(uname -s) != Linux ]] || ! command -v apt-get >/dev/null; then + printf 'This setup script requires a Debian/Ubuntu Linux sandbox.\n' >&2 + exit 1 +fi + +root=() +if (( EUID != 0 )); then + if ! command -v sudo >/dev/null || ! sudo -n true; then + printf 'Root or passwordless sudo is required to install sandbox prerequisites.\n' >&2 + exit 1 + fi + root=(sudo -n) +fi + +KILO_STARTUP_CGROUP="/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$repo")" +export KILO_STARTUP_CGROUP KILO_STARTUP_RESERVE_MB +if [[ ! -f /sys/fs/cgroup/kilo-workloads/memory.max ]]; then + printf 'Memory-safe setup requires the sandbox\x27s delegated cgroup v2 memory controller. No workloads were started.\n' >&2 + exit 1 +fi +KILO_STARTUP_MEMORY_MB=$(node -e ' + const fs = require("node:fs"); + const MiB = 1048576; + const parent = "/sys/fs/cgroup/kilo-workloads"; + function fail(message) { + console.error(message + " No workloads were started."); + process.exit(1); + } + function protectedMemory(directory) { + const stat = Object.fromEntries(fs.readFileSync(directory + "/memory.stat", "utf8").trim().split("\n").map(line => line.split(" "))); + const current = Number(fs.readFileSync(directory + "/memory.current", "utf8")); + return current - Number(stat.inactive_file || 0) + Number(stat.file_dirty || 0) + Number(stat.file_writeback || 0); + } + const meminfo = fs.readFileSync("/proc/meminfo", "utf8"); + const meminfoBytes = key => Number(meminfo.match(new RegExp("^" + key + ":\\s+(\\d+)", "m"))[1]) * 1024; + const parentMax = Number(fs.readFileSync(parent + "/memory.max", "utf8")); + const total = Math.min(meminfoBytes("MemTotal"), Number.isFinite(parentMax) ? parentMax : Infinity); + const reserve = Number(process.env.KILO_STARTUP_RESERVE_MB) * MiB; + const requested = process.env.KILO_STARTUP_MEMORY_MB; + if (requested !== undefined && !/^[0-9]+$/.test(requested)) fail("KILO_STARTUP_MEMORY_MB must be an integer number of MiB."); + const limit = requested === undefined ? Math.floor((total - reserve) / MiB) * MiB : Number(requested) * MiB; + if (limit < 3072 * MiB) fail("The dev workload needs at least 3072 MiB, but its cap is " + Math.floor(limit / MiB) + " MiB (" + Math.floor(total / MiB) + " MiB total, " + Math.floor(reserve / MiB) + " MiB reserve)."); + const existing = fs.existsSync(process.env.KILO_STARTUP_CGROUP + "/memory.current") ? protectedMemory(process.env.KILO_STARTUP_CGROUP) : 0; + const others = Math.max(0, protectedMemory(parent) - existing); + if (limit + others > total) fail("Insufficient memory headroom: other sandbox workloads hold " + Math.ceil(others / MiB) + " MiB, so a " + Math.floor(limit / MiB) + " MiB dev workload cap exceeds the " + Math.floor(total / MiB) + " MiB total. Stop other workloads or use a larger sandbox."); + const additional = Math.max(0, limit - existing); + if (additional > meminfoBytes("MemAvailable")) fail("Insufficient host memory: " + Math.ceil(additional / MiB) + " MiB additional capacity required, " + Math.floor(meminfoBytes("MemAvailable") / MiB) + " MiB available."); + console.log(Math.floor(limit / MiB)); +') +export KILO_STARTUP_MEMORY_MB +"${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP" +printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.max" >/dev/null +printf '%s\n' "$(( KILO_STARTUP_MEMORY_MB * 1048576 * 95 / 100 ))" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.high" >/dev/null +printf '0\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.swap.max" >/dev/null +printf '1\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/memory.oom.group" >/dev/null +printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/cgroup.subtree_control" >/dev/null +"${root[@]}" mkdir -p "$KILO_STARTUP_CGROUP/processes" "$KILO_STARTUP_CGROUP/containers" +printf '%s\n' "$$" | "${root[@]}" tee "$KILO_STARTUP_CGROUP/processes/cgroup.procs" >/dev/null +printf 'Dev workload capped at %s MiB, with %s MiB reserved for other sandbox workloads.\n' "$KILO_STARTUP_MEMORY_MB" "$KILO_STARTUP_RESERVE_MB" + +"${root[@]}" timeout --foreground 5m apt-get -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update +docker_packages=(docker.io) +for package in docker-cli docker-buildx; do + if apt-cache show "$package" >/dev/null 2>&1; then + docker_packages+=("$package") + fi +done +if apt-cache show docker-compose-v2 >/dev/null 2>&1; then + docker_packages+=(docker-compose-v2) +else + # Debian trixie's docker-compose package ships Compose v2 as the docker CLI plugin. + docker_packages+=(docker-compose) +fi +"${root[@]}" env DEBIAN_FRONTEND=noninteractive timeout --foreground 10m apt-get install -y --no-install-recommends \ + ca-certificates chromium curl dnsmasq-base fuse-overlayfs git git-lfs openssl sudo unzip tmux "${docker_packages[@]}" +if ! docker compose version >/dev/null 2>&1; then + printf 'Docker Compose v2 (the docker compose CLI plugin) is required, but %s did not provide it.\n' "${docker_packages[-1]}" >&2 + exit 1 +fi + +if ! command -v node >/dev/null || [[ $(node -p 'process.versions.node.split(".")[0]') != 24 ]]; then + printf 'The sandbox image must provide Node.js 24 and Corepack before running this script.\n' >&2 + exit 1 +fi + +global_pnpm=$(command -v pnpm || true) +if [[ -n $global_pnpm ]] && grep -qF "$pnpm_wrapper_marker" "$global_pnpm" 2>/dev/null; then + real_pnpm=$(< "$global_state_dir/real-pnpm") +else + if [[ -n $global_pnpm && $(readlink -f "$global_pnpm") == "$state_dir"/* ]]; then + "${root[@]}" rm -f "$global_pnpm" + global_pnpm= + fi + if [[ -z $global_pnpm ]]; then + "${root[@]}" corepack enable + corepack install + global_pnpm=$(command -v pnpm) + fi + real_pnpm=$(readlink -f "$global_pnpm") + if [[ ! -L $global_pnpm ]]; then + # The wrapper replaces this path, so move a standalone pnpm binary out of the way first. + "${root[@]}" mkdir -p "$global_state_dir" + "${root[@]}" mv "$global_pnpm" "$global_state_dir/pnpm" + real_pnpm="$global_state_dir/pnpm" + fi +fi +if [[ ! -x $real_pnpm ]] || grep -qF "$pnpm_wrapper_marker" "$real_pnpm"; then + printf 'Could not locate the real pnpm executable (resolved %s).\n' "$real_pnpm" >&2 + exit 1 +fi + +tools=() +command -v bun >/dev/null || tools+=(bun@1.3.13) +command -v agent-browser >/dev/null || tools+=(agent-browser@0.38.2) +if (( ${#tools[@]} )); then + "${root[@]}" npm install --global --no-audit --no-fund "${tools[@]}" +fi + +if tmux list-sessions >/dev/null 2>&1; then + tmux_pid=$(tmux display-message -p '#{pid}') + if ! node -e 'const fs = require("node:fs"); process.exit(fs.readFileSync("/proc/" + process.argv[1] + "/cgroup", "utf8").includes(process.env.KILO_STARTUP_CGROUP.replace("/sys/fs/cgroup", "") + "/processes") ? 0 : 1)' "$tmux_pid"; then + printf 'The existing tmux server is outside the dev workload memory budget. Stop its sessions before running this script.\n' >&2 + exit 1 + fi +fi + +mkdir -p "$startup_bin" +rm -f "$startup_bin/pnpm" "$state_dir/browser.env" "$state_dir/selection" +real_docker=$(readlink -f "$(command -v docker)") +cat > "$state_dir/compose.memory.yml" < "$startup_bin/kilo-shell" <<'SH' +#!/usr/bin/env bash +if [[ ${1:-} == -lc ]]; then + shift + exec /bin/bash --noprofile --norc -c "$@" +fi +exec /bin/bash "$@" +SH +chmod +x "$startup_bin/kilo-shell" +cat > "$state_dir/sandbox-docker.cjs" <<'JS' +#!/usr/bin/env node +const fs = require('node:fs'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const args = process.argv.slice(2); +const stdinDockerfile = args[0] === 'build' && args.some((arg, i) => + (arg === '-f' || arg === '--file') && args[i + 1] === '-'); +function run(input) { + const build = args[0] === 'build'; + if (args[0] === 'compose') { + const source = args.findIndex((arg, i) => arg === '-f' && path.resolve(args[i + 1]) === path.resolve(__dirname, '../../dev/docker-compose.yml')); + if (source !== -1) args.splice(source + 2, 0, '-f', path.join(__dirname, 'compose.memory.yml')); + } + // dockerd's own BuildKit keeps images in the daemon store, so unchanged images rebuild from cache in seconds. + if (build) args.splice(1, 0, `--cgroup-parent=${process.env.KILO_STARTUP_CGROUP.replace('/sys/fs/cgroup', '')}/containers/builds`); + const command = build ? 'flock' : process.env.KILO_STARTUP_REAL_DOCKER; + const commandArgs = build ? [path.join(__dirname, 'image-build.lock'), process.env.KILO_STARTUP_REAL_DOCKER, ...args] : args; + const child = spawn(command, commandArgs, { stdio: [input === undefined ? 'inherit' : 'pipe', 'inherit', 'inherit'] }); + for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => child.kill(signal)); + child.on('error', error => { console.error(error.message); process.exitCode = 1; }); + child.on('exit', code => { process.exitCode = code ?? 1; }); + if (input !== undefined) child.stdin.end(input); +} +if (!stdinDockerfile) { + run(); +} else { + let dockerfile = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', chunk => { dockerfile += chunk; }); + process.stdin.on('end', () => { + const cert = process.env.NODE_EXTRA_CA_CERTS && fs.existsSync(process.env.NODE_EXTRA_CA_CERTS) ? fs.readFileSync(process.env.NODE_EXTRA_CA_CERTS).toString('base64') : null; + // Trust the sandbox's HTTPS interception CA inside development images, not production sources. + const trust = cert ? `RUN mkdir -p /usr/local/share/ca-certificates && printf '%s' '${cert}' | base64 -d > /usr/local/share/ca-certificates/kilo-sandbox.crt && (if command -v apt-get >/dev/null; then printf 'Acquire::http::Timeout "30";\\nAcquire::https::Timeout "30";\\nAcquire::Retries "2";\\n' > /etc/apt/apt.conf.d/99-kilo-startup-timeouts; fi) && (command -v update-ca-certificates || (apt-get update && apt-get install -y --no-install-recommends ca-certificates)) && update-ca-certificates\nENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/kilo-sandbox.crt` : ''; + const limits = 'ENV NODE_OPTIONS=--max-old-space-size=768 GOMAXPROCS=2 npm_config_jobs=1'; + dockerfile = dockerfile.replace(/^FROM (?:docker.io\/library\/)?(docker:dind-rootless|debian:trixie-slim)([^\n]*)/gm, + (_, image, suffix) => `FROM mirror.gcr.io/library/${image}${suffix}\nUSER root\n${trust}\n${limits}`); + dockerfile = dockerfile.replace(/^FROM (?:docker.io\/)?cloudflare\/sandbox:[^\n]+/m, from => + `${from}\n${trust}\n${limits}`); + run(dockerfile); + }); +} +JS +chmod +x "$state_dir/sandbox-docker.cjs" +ln -sf "$state_dir/sandbox-docker.cjs" "$startup_bin/docker" + +env_default() { + printf '[[ -n ${%s:-} ]] || %s=%q; export %s\n' "$1" "$1" "$2" "$1" +} +{ + printf 'case ":$PATH:" in *:%q:*) ;; *) export PATH=%q:"$PATH" ;; esac\n' "$startup_bin" "$startup_bin" + printf 'export SHELL=%q\n' "$startup_bin/kilo-shell" + for name in NEXT_TELEMETRY_DISABLED SKIP_STRIPE_API KILO_PORT_OFFSET KILO_ENV_SYNC_CONCURRENCY KILO_STARTUP_CGROUP; do + env_default "$name" "${!name}" + done + env_default KILO_STARTUP_REAL_DOCKER "$real_docker" + env_default WRANGLER_DOCKER_BIN "$state_dir/sandbox-docker.cjs" + env_default WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST 1 + env_default AGENT_BROWSER_ENGINE chrome + env_default AGENT_BROWSER_EXECUTABLE_PATH /usr/bin/chromium + env_default AGENT_BROWSER_SOCKET_DIR /tmp/kilo-browser + env_default AGENT_BROWSER_ARGS --disable-gpu + env_default AGENT_BROWSER_DEFAULT_TIMEOUT 120000 + # The full agents stack does not fit the sandbox budget; these are not needed for fake-LLM sessions on public repositories. + env_default KILO_DEV_WITHOUT notifications,event-service,cloudflare-webhook-agent-ingest,container-usage-meter,cloudflare-git-token-service + if [[ -n ${NODE_EXTRA_CA_CERTS:-} ]]; then + env_default NODE_EXTRA_CA_CERTS "$NODE_EXTRA_CA_CERTS" + fi +} > "$env_file" + +cgroup_writer=(tee) +if (( ${#root[@]} )); then + cgroup_writer=("${root[@]}" tee) +fi +"${root[@]}" mkdir -p "$global_state_dir" +printf '%s\n' "$real_pnpm" | "${root[@]}" tee "$global_state_dir/real-pnpm" >/dev/null +{ + printf '#!/usr/bin/env bash\n%s\n' "$pnpm_wrapper_marker" + printf 'repo=%q\nenv_file=%q\n' "$repo" "$env_file" + printf 'cgroup_writer=(%s)\n' "$(printf '%q ' "${cgroup_writer[@]}")" + cat <<'SH' +real_pnpm=$(< /usr/local/lib/kilo-cloud-agent/real-pnpm) +if [[ ($PWD == "$repo" || $PWD == "$repo"/*) && -f $env_file ]]; then + source "$env_file" + cgroup=${KILO_STARTUP_CGROUP#/sys/fs/cgroup} + if [[ $(< /proc/self/cgroup) != "0::$cgroup/"* ]]; then + if [[ ! -d $KILO_STARTUP_CGROUP/processes ]]; then + printf 'The sandbox dev memory cgroup is missing. Rerun bash %q/.kilo/cloud-agent-setup.sh.\n' "$repo" >&2 + exit 1 + fi + printf '%s\n' "$$" | "${cgroup_writer[@]}" "$KILO_STARTUP_CGROUP/processes/cgroup.procs" >/dev/null + fi +fi +exec "$real_pnpm" "$@" +SH +} > "$state_dir/pnpm-wrapper" +"${root[@]}" install -m 0755 "$state_dir/pnpm-wrapper" "${global_pnpm:-/usr/local/bin/pnpm}" +hash -r +source "$env_file" + +if ! docker info >/dev/null 2>&1; then + if [[ -n ${DOCKER_HOST:-} ]] || [[ -S /var/run/docker.sock ]]; then + printf 'The supplied Docker daemon is inaccessible; fix Docker access and rerun.\n' >&2 + exit 1 + fi + # These sandboxes mount /proc/sys read-only; overlay2 is supported by the current kernel. + storage_driver="${KILO_STARTUP_DOCKER_STORAGE_DRIVER:-overlay2}" + if [[ $storage_driver != overlay2 && $storage_driver != fuse-overlayfs ]]; then + printf 'Only overlay2 or fuse-overlayfs is allowed; vfs layer copies are unsafe for this sandbox.\n' >&2 + exit 1 + fi + # Own the socket by the invoking user's group so a non-root sandbox can use the daemon it starts. + "${root[@]}" tmux new-session -d -s kilo-startup-docker \ + "env DOCKER_ALLOW_IPV6_ON_IPV4_INTERFACE=1 dockerd --group=$(id -gn) --storage-driver=$storage_driver --ip-forward=false --bip=$bridge_gateway/16 --registry-mirror=https://mirror.gcr.io --cgroup-parent=${KILO_STARTUP_CGROUP#/sys/fs/cgroup}/containers" + for (( attempt=0; attempt<30; attempt++ )); do + docker info >/dev/null 2>&1 && break + sleep 1 + done + if ! docker info >/dev/null 2>&1; then + printf 'Docker could not start. This sandbox must support nested containers or supply a Docker socket.\n' >&2 + "${root[@]}" tmux capture-pane -p -t kilo-startup-docker || true + exit 1 + fi +fi +if pgrep -a -x dockerd | grep -qF -- "--bip=$bridge_gateway/16"; then + # Without IP forwarding, bridge containers reach only the host. Workerd pins sandbox + # containers to public resolvers, so redirect all their DNS to a forwarder on the gateway. + if ! pgrep -f "dnsmasq .*--listen-address=$bridge_gateway" >/dev/null; then + "${root[@]}" dnsmasq --conf-file=/dev/null --no-hosts --bind-interfaces --listen-address="$bridge_gateway" + fi + for proto in udp tcp; do + dns_redirect=(PREROUTING -i docker0 -p "$proto" --dport 53 -j DNAT --to-destination "$bridge_gateway:53") + "${root[@]}" iptables -t nat -C "${dns_redirect[@]}" 2>/dev/null || "${root[@]}" iptables -t nat -I "${dns_redirect[@]}" + done +fi +if [[ $(docker info --format '{{.Driver}}') == vfs ]]; then + printf 'The existing Docker daemon uses vfs. Stop it and restart with overlay2 or fuse-overlayfs before running this workload.\n' >&2 + exit 1 +fi +if tmux list-sessions >/dev/null 2>&1; then + while IFS= read -r line; do + tmux set-environment -g "${line%%=*}" "${line#*=}" + done < <(bash -c 'source "$1"; for name in PATH SHELL KILO_PORT_OFFSET KILO_STARTUP_CGROUP KILO_STARTUP_REAL_DOCKER WRANGLER_DOCKER_BIN WRANGLER_CI_OVERRIDE_NETWORK_MODE_HOST NODE_EXTRA_CA_CERTS; do [[ -n ${!name:-} ]] && printf "%s=%s\n" "$name" "${!name}"; done' _ "$env_file") +fi + +legacy_builder="kilo-lowmem-$(basename "$repo")" +if docker buildx inspect "$legacy_builder" >/dev/null 2>&1; then + docker buildx rm --force "$legacy_builder" >/dev/null +fi +# Image build steps run in their own 2 GiB slice of the dev workload budget. +builds_cgroup="$KILO_STARTUP_CGROUP/containers/builds" +printf '+memory +cpu\n' | "${root[@]}" tee "$KILO_STARTUP_CGROUP/containers/cgroup.subtree_control" >/dev/null +"${root[@]}" mkdir -p "$builds_cgroup" +printf '%s\n' "$(( 2048 * 1048576 ))" | "${root[@]}" tee "$builds_cgroup/memory.max" >/dev/null +printf '0\n' | "${root[@]}" tee "$builds_cgroup/memory.swap.max" >/dev/null + +timeout 15m pnpm install --frozen-lockfile --child-concurrency=1 --network-concurrency=4 +if [[ ! -s .env.local ]]; then + (umask 077; pnpm dev:setup-env --ci) + printf 'Created local-only credentials. Real payment, model, and Git integrations require supplied secrets.\n' +fi +node -e ' + const fs = require("node:fs"); + const match = fs.readFileSync(".env.local", "utf8").match(/^POSTGRES_URL\s*=\s*(.*)$/m); + const value = process.env.POSTGRES_URL || match?.[1].trim().replace(/^(["\x27])(.*)\1$/, (_, quote, inner) => inner); + if (!value || !["localhost", "127.0.0.1", "[::1]"].includes(new URL(value).hostname)) { + throw new Error("Sandbox setup requires a local POSTGRES_URL; remote databases will not be migrated or seeded"); + } +' + +# Avoid Docker Hub's shared unauthenticated pull limit without changing Compose files. +while IFS= read -r image; do + if docker image inspect "$image" >/dev/null 2>&1; then + continue + fi + hub_image=${image#docker.io/} + registry=${hub_image%%/*} + if [[ $hub_image == */* && ($registry == *.* || $registry == *:* || $registry == localhost) ]]; then + docker pull "$image" + continue + fi + [[ $hub_image == */* ]] || hub_image="library/$hub_image" + if docker pull "mirror.gcr.io/$hub_image"; then + docker tag "mirror.gcr.io/$hub_image" "$image" + else + docker pull "$image" + fi +done < <(docker compose -f dev/docker-compose.yml config --images | sort -u) + +printf 'Preparing the local database (up to 5 minutes).\n' +timeout --foreground 5m pnpm test:db + +test_email="kilo-$(basename "$HOME")-$(date -u +%Y%m%d%H%M%S)@example.com" +test_user_id=$(docker compose -f dev/docker-compose.yml exec -T postgres \ + psql -U postgres -d postgres -X -qAt -v ON_ERROR_STOP=1 -v email="$test_email" <<'SQL' +INSERT INTO kilocode_users ( + id, google_user_email, google_user_name, google_user_image_url, hosted_domain, + stripe_customer_id, completed_welcome_form, has_validation_stytch, customer_source +) VALUES ( + gen_random_uuid()::text, :'email', 'Sandbox Test User', '', '@@fake@@', + 'cus_local_sandbox', true, true, 'dev-seed' +) RETURNING id; +SQL +) +pnpm dev:seed app:add-credits "$test_user_id" 100 --free +env_default KILO_TEST_USER_EMAIL "$test_email" >> "$env_file" + +# wrangler dev builds every Cloud Agent sandbox image on start. Build them now so dev:start only hits the cache. +if [[ ${KILO_STARTUP_SANDBOX_IMAGES:-1} != 0 ]]; then + printf 'Building Cloud Agent sandbox images (about 10 minutes on a new machine, under a minute when cached).\n' + images_log="$state_dir/sandbox-images.log" + setsid bash -c 'source "$1"; cd services/cloud-agent-next; exec pnpm run dev --port 28794 --inspector-port 38794 --ip 127.0.0.1' \ + _ "$env_file" > "$images_log" 2>&1 < /dev/null & + images_pid=$! + images_deadline=$(( SECONDS + 2400 )) + until grep -qF 'Container image(s) ready' "$images_log"; do + if ! kill -0 "$images_pid" 2>/dev/null || (( SECONDS > images_deadline )); then + kill -KILL -- "-$images_pid" 2>/dev/null || true + tail -20 "$images_log" >&2 + printf 'Building Cloud Agent sandbox images failed; see %s. Rerun setup, or set KILO_STARTUP_SANDBOX_IMAGES=0 to skip.\n' "$images_log" >&2 + exit 1 + fi + sleep 5 + done + kill -TERM -- "-$images_pid" + wait "$images_pid" 2>/dev/null || true +fi + +printf '\nSetup complete. Dev workload memory peak so far: %s MiB / %s MiB.\n' \ + "$(( $(< "$KILO_STARTUP_CGROUP/memory.peak") / 1048576 ))" "$KILO_STARTUP_MEMORY_MB" +printf 'Run from %s (pnpm there joins the capped cgroup and loads %s):\n' "$repo" "$env_file" +printf ' pnpm dev:start --no-attach app # web app\n' +printf ' pnpm dev:start --no-attach agents fake-llm # Cloud Agents with local fake inference\n' +printf 'KILO_DEV_WITHOUT skips services the sandbox does not need; pass --without= to start everything.\n' +printf 'Then pnpm dev:status for ports; log in at http://localhost:/users/sign_in?fakeUser=%s&callbackPath=/profile\n' "$test_email" +printf 'Other shells (docker, agent-browser): source %q\n' "$env_file" +printf 'Usage, limits, and troubleshooting: .kilo/skills/cloud-agent-sandbox/SKILL.md\n' diff --git a/.kilo/skills/cloud-agent-sandbox/SKILL.md b/.kilo/skills/cloud-agent-sandbox/SKILL.md new file mode 100644 index 0000000000..b14cc04a32 --- /dev/null +++ b/.kilo/skills/cloud-agent-sandbox/SKILL.md @@ -0,0 +1,207 @@ +--- +name: cloud-agent-sandbox +description: Sets up and runs this monorepo inside a memory-constrained Kilo Cloud Agent sandbox with `.kilo/cloud-agent-setup.sh`. Use when working in a Cloud Agent sandbox, before starting local services there, after a sandbox restart, or when debugging the dev memory cap, Docker, DNS, fake login, agent-browser, or fake-LLM Cloud Agent sessions in the sandbox. +--- + +# Cloud Agent sandbox + +`.kilo/cloud-agent-setup.sh` prepares a Debian/Ubuntu Cloud Agent sandbox for +`pnpm dev:start`. It usually runs automatically when the machine starts. You can +run it at any time: reruns are safe and take about 20 s once the machine is set up. + +Follow the `local-development` skill for ports, fake login, and service +management. This skill covers what is different in the sandbox. + +## Is setup done? + +Sandbox restarts wipe Docker, the pnpm wrapper, the memory cgroup, and +`.wrangler/kilo-startup/`. The repository and `node_modules` survive. Check +before starting services: + +```bash +test -f .wrangler/kilo-startup/env \ + && grep -q kilo-cloud-agent-pnpm-wrapper "$(command -v pnpm)" \ + && docker info >/dev/null 2>&1 && echo ready +``` + +If this does not print `ready`, run setup from the repository root: + +```bash +bash .kilo/cloud-agent-setup.sh +``` + +A restarted machine takes about 6 minutes, mostly building the Cloud Agent +sandbox images; `node_modules` survives the restart. A rerun on a running +machine takes under 2 minutes, because images come from the build cache. A +machine without dependencies adds about 3 minutes for `pnpm install`. Setup stops at the first failure and prints +the line number. + +## What setup does + +- Creates a memory cgroup for all dev workloads, capped at the sandbox memory + minus 2 GiB. Override the cap in MiB with `KILO_STARTUP_MEMORY_MB`; the + minimum is 3072. +- Installs Docker, Compose v2, tmux, Chromium, agent-browser, and dnsmasq. + Starts dockerd with its containers inside the cgroup and pulls images through + `mirror.gcr.io`. +- Installs a global `pnpm` wrapper. Inside this repository it moves the command + into the capped cgroup and loads `.wrangler/kilo-startup/env`. Outside the + repository it runs the real pnpm, saved at + `/usr/local/lib/kilo-cloud-agent/real-pnpm`, unchanged. +- Installs dependencies, creates `.env.local`, runs `pnpm test:db`, and seeds a + fake-login user with credits. +- Builds the eight Cloud Agent sandbox images by running `wrangler dev` for + `cloud-agent-next` on spare ports until `Container image(s) ready`. The log is + `.wrangler/kilo-startup/sandbox-images.log`. Skip this with + `KILO_STARTUP_SANDBOX_IMAGES=0`. +- It does not start the dev stack. + +`.wrangler/kilo-startup/env` only sets defaults. Override a value by exporting +it before running pnpm. Do not edit the file: setup rewrites it. + +## Start services + +Run every command from the repository root, so that pnpm is capped. + +```bash +pnpm dev:start --no-attach app # web app +pnpm dev:start --no-attach agents fake-llm # Cloud Agents with local fake inference +pnpm dev:status # services and ports +pnpm dev:stop +``` + +- `KILO_DEV_WITHOUT` in the env file skips agents services that fake-LLM sessions + on public repositories do not need: notifications, event-service, + webhook-agent-ingest, container-usage-meter, and git-token-service. That + leaves 7 services, about 3 GB when idle. Add services back with + `--without=`. `--without=` starts everything, but the full agents + stack does not fit in the cap. +- Expected warnings without those services: billing-heartbeat errors from the + skipped usage meter. GitHub-backed repositories need git-token-service and + GitHub App credentials, which the sandbox does not have. +- `--no-attach` returns once services are up: about 50 s for `app`. The first + page load compiles with Turbopack and takes about 30 s more. +- Wrangler rebuilds every sandbox image each time `cloud-agent-next` starts. + Images build in dockerd's own BuildKit, so unchanged images come from the + layer cache. Setup prebuilds them; without that, or after a Dockerfile change, + the build takes minutes. `cloud-agent-next` reports `up` before images are + ready, and sessions created meanwhile fail (`fetch failed` in the harness). + Wait for the build to finish: + + ```bash + until grep -q 'Container image(s) ready' dev/logs/cloud-agent-next.log; do sleep 15; done + ``` +- `--reuse-running` currently refuses to reuse a session because the Stripe + forwarder is always skipped. Check `pnpm dev:status` instead. + +## Log in and use the browser + +Setup seeds a verified user with credits. Its email is `KILO_TEST_USER_EMAIL` in +the env file. Shells outside the pnpm wrapper must load the env file first. It +also puts setup's `docker` wrapper on `PATH`, which keeps builds serialized and +inside the cap: + +```bash +source .wrangler/kilo-startup/env +agent-browser --session main open "http://localhost:3000/users/sign_in?fakeUser=$KILO_TEST_USER_EMAIL&callbackPath=/profile" +``` + +- Read the real port from `pnpm dev:status`. It is 3000 unless an offset applies. +- The env file points agent-browser at the installed Chromium with a 120 s + timeout. Confirm login with + `agent-browser --session main eval '(async () => (await (await fetch("/api/auth/session")).json()).user?.email)()'`. +- If a click fails because the element is covered, focus the input and use + `agent-browser press Enter`. +- Starting a Cloud Agent session from `/cloud` requires a connected GitHub or + GitLab provider, which the sandbox cannot set up. Create sessions with the + fake-LLM harness instead, then view them in the browser. + +## Fake-LLM Cloud Agent sessions + +Start `agents fake-llm`, then follow `services/cloud-agent-next/test/e2e/README.md`. +Ports below are the defaults; check `pnpm dev:status`. + +```bash +WORKER_URL=http://localhost:8794 FAKE_LLM_URL=http://localhost:8811 \ + pnpm -s exec tsx services/cloud-agent-next/test/e2e/run.ts cold echo:hi +``` + +- Verified passing on the minimal stack: `cold echo:hi`, `cold-hot echo:hi`, + `chunked-streaming slow:5:50`, `queue-while-busy`, and + `--api=legacy cold-hot echo:legacy`. `llm-error boom` fails a retry-status + assertion that is unrelated to the sandbox. +- Each run leaves a sandbox container of about 750 MB until it stops for being + idle. Remove them between runs: + `docker rm -f $(docker ps -q --filter name=workerd-cloud-agent-next-dev-Sandbox)`. +- The first session after `dev:start` can fail model validation with a 503 + ("Model availability could not be verified"). Turbopack is still compiling + the validation route; retry once. +- To view harness sessions in the browser, set `E2E_USER_EMAIL` so runs reuse + one driver user. Mark that user verified, then fake-login as it and open + `/cloud/sessions`: + + ```bash + docker compose -f dev/docker-compose.yml exec -T postgres psql -U postgres -d postgres -c \ + "UPDATE kilocode_users SET has_validation_stytch = true, completed_welcome_form = true WHERE google_user_email = ''" + ``` + + Sending a message to a harness session from the UI fails with + "Session not found". + +## Memory + +The cap is a hard limit with no swap. When the workload nears it, the kernel +reclaims memory instead of killing processes: everything slows, and +`docker`, `tmux`, and `pnpm dev:stop` can hang. Check pressure with: + +```bash +cg=/sys/fs/cgroup/kilo-workloads/kilo-dev-$(basename "$PWD") +echo "$(( $(cat $cg/memory.current) / 1048576 )) MiB of $(( $(cat $cg/memory.max) / 1048576 )) MiB" +grep -E '^(high|max|oom_kill) ' $cg/memory.events +``` + +If the `high` or `max` count keeps rising, you are near the cap. Stop work you do +not need, remove idle sandbox containers, or start fewer services. Dev tooling is +heavy: wrangler and workerd use about 3 GB, and the pnpm parents and log filters +about 2 GB. + +- Next.js dev (Turbopack) grows by 1 to 3 GB while compiling a route and gives + most of it back after about two minutes idle: 4.2 GB after login fell to + 2.8 GB, and 4.4 GB after `/cloud` fell to 2.0 GB. Opening many routes back to + back fills the cap before that happens. Pause between heavy routes. Next 16.3 + already defaults `experimental.turbopackMemoryEviction` to `auto`. +- High Redis, redis-http, or Postgres CPU means memory pressure, not load. Under + pressure, Redis used 6 s of user CPU and 1590 s of system CPU in 90 minutes: + the kernel kept evicting and re-reading its code pages. Compare with + `cat $cg/containers/*/cpu.stat`. Idle Redis without pressure uses under 1%. +- If Next.js is stuck above the cap, `pkill -9 -f '^[n]ext-server'`, then + `pnpm dev:restart nextjs`. + +Run heavy commands such as builds, tests, and typechecks from inside the +repository, so that the wrapper caps them. pnpm outside the repository and +direct `node` or `npx` processes are not capped. + +## Docker networking and DNS + +`/proc/sys` is read-only, so dockerd runs with `--ip-forward=false`: bridge +containers can reach only the host. Workerd also pins sandbox containers to +1.1.1.1 and 8.8.8.8. Setup runs dnsmasq on the bridge gateway, 172.17.0.1, and +uses iptables to redirect all DNS from `docker0` to it. + +A `git_network_failed` clone failure, or `Could not resolve host` inside a +sandbox, means this redirect is missing. Rerun setup, then check: + +```bash +iptables -t nat -S PREROUTING | grep 'dport 53' +pgrep -a dnsmasq +docker exec git ls-remote https://github.com/octocat/Hello-World.git +``` + +## Pitfalls + +- `pkill -f next-server` also matches the shell that runs it. Use + `pkill -f '[n]ext-server'`. +- Do not prune Docker images or build cache. Rebuilding the Cloud Agent images + costs about 12 minutes. +- Lint setup changes with `shellcheck -S warning -e SC1090 .kilo/cloud-agent-setup.sh`. + Setup does not install ShellCheck; use `apt-get install -y shellcheck`. diff --git a/.kilo/skills/local-development/SKILL.md b/.kilo/skills/local-development/SKILL.md index 688a541e0b..ef0269223d 100644 --- a/.kilo/skills/local-development/SKILL.md +++ b/.kilo/skills/local-development/SKILL.md @@ -5,6 +5,8 @@ description: Start, reuse, inspect, or browser-test local apps and services in t # Local development +In a Kilo Cloud Agent sandbox, load the `cloud-agent-sandbox` skill first. + Read `DEVELOPMENT.md` for human setup and service procedures. Read `ENVIRONMENT.md` for the environment-variable inventory. Shared web environment mutations are governed by `apps/web/AGENTS.md`; do not use this skill for that workflow. ## Start or reuse services diff --git a/AGENTS.md b/AGENTS.md index 54619e40fe..b198eb49db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,7 @@ package manifests before running repository JavaScript or package scripts. Load | TypeScript implementation or review | `code-quality` skill | | Verification or pre-commit checks | `repository-verification` skill | | Local services, ports, and fake login | `local-development` skill | +| Cloud Agent sandbox setup, memory cap, and fake-LLM sessions | `cloud-agent-sandbox` skill and `.kilo/cloud-agent-setup.sh` | | Shared web environment changes | `apps/web/AGENTS.md` and `DEVELOPMENT.md` | | PostgreSQL schema or migration work | `packages/db/AGENTS.md` and `database-migrations` skill | | Service, Durable Object, or Worker code | `services/AGENTS.md`, nearest owning service's `AGENTS.md`, and relevant Durable Objects or Workers skills | diff --git a/dev/local/cli.ts b/dev/local/cli.ts index 2867776134..4c7613b9dd 100644 --- a/dev/local/cli.ts +++ b/dev/local/cli.ts @@ -6,6 +6,8 @@ import { applyPortOffset, candidatePortOffsets, clearDevLogs, + excludeServices, + parseServiceList, resolveTargets, getService, getGroups, @@ -433,7 +435,14 @@ function removeStaleComposeProject(repoRoot: string, previousOffset: number | un async function cmdUp(args: string[], repoRoot: string): Promise { const noAttach = args.includes('--no-attach'); const reuseRunning = args.includes('--reuse-running'); - const targets = args.filter(arg => arg !== '--no-attach' && arg !== '--reuse-running'); + const withoutArg = args.findLast(arg => arg.startsWith('--without=')); + const withoutSource = withoutArg === undefined ? 'KILO_DEV_WITHOUT' : '--without'; + const without = parseServiceList( + withoutArg === undefined ? process.env.KILO_DEV_WITHOUT : withoutArg.slice('--without='.length) + ); + const targets = args.filter( + arg => arg !== '--no-attach' && arg !== '--reuse-running' && !arg.startsWith('--without=') + ); // --- Preflight checks --- if (!isTmuxAvailable()) { @@ -465,6 +474,16 @@ async function cmdUp(args: string[], repoRoot: string): Promise 0) { + const exclusion = excludeServices(serviceNames, without); + serviceNames = exclusion.serviceNames; + if (exclusion.skipped.length > 0) { + console.log(`${DIM}Skipping (${withoutSource}): ${exclusion.skipped.join(', ')}${RESET}`); + } + for (const [name, missing] of exclusion.dependents) { + console.warn(`⚠ ${name} runs without ${missing.join(', ')}; calls to them will fail.`); + } + } const sessionName = getSessionName(); let sessionAlreadyRunning = sessionExists(sessionName); @@ -1488,9 +1507,11 @@ async function cmdEnv(args: string[], repoRoot: string): Promise { function printUsage(): void { console.log(` Usage: - dev:start [--no-attach] [--reuse-running] [targets...] + dev:start [--no-attach] [--reuse-running] [--without=a,b] [targets...] Start services (default: core) --reuse-running never restarts an existing complete stack + --without skips the named services (default: $KILO_DEV_WITHOUT; + --without= starts everything) dev:stop [--force] Stop all services (skips shared Docker infra if other kilo-dev sessions are running; --force overrides) dev:status [--json] Show running services and their ports diff --git a/dev/local/services.test.ts b/dev/local/services.test.ts index b612781c37..04b449721d 100644 --- a/dev/local/services.test.ts +++ b/dev/local/services.test.ts @@ -9,8 +9,10 @@ import { candidatePortOffsets, clearDevLogs, computePortOffset, + excludeServices, getAlwaysOnGroupIds, getService, + parseServiceList, portOffset, readPersistedPortOffset, resolveGroups, @@ -577,3 +579,33 @@ test('a tunnels restart keeps the live selection and reloads the HTTP worker', ( ); assert.equal(planTunnelRestart(['cloud-agent-public-tunnels']).reloadTarget, undefined); }); + +test('excludes unwanted services and reports dependents that lose them', () => { + const selection = resolveTargets(['agents', 'fake-llm']); + const exclusion = excludeServices(selection, ['notifications', 'event-service']); + + assert.ok(!exclusion.serviceNames.includes('notifications')); + assert.ok(!exclusion.serviceNames.includes('event-service')); + assert.ok(exclusion.serviceNames.includes('cloud-agent-next')); + assert.deepEqual(exclusion.skipped.toSorted(), ['event-service', 'notifications']); + assert.deepEqual(exclusion.dependents.get('cloud-agent-next'), ['notifications']); +}); + +test('ignores excluded services that are not selected', () => { + const exclusion = excludeServices(['postgres', 'nextjs'], ['notifications']); + + assert.deepEqual(exclusion.serviceNames, ['postgres', 'nextjs']); + assert.deepEqual(exclusion.skipped, []); +}); + +test('rejects unknown excluded services instead of starting everything', () => { + assert.throws(() => excludeServices(['postgres'], ['notifcations']), /Unknown service/); +}); + +test('parses comma-separated service lists', () => { + assert.deepEqual(parseServiceList(' notifications, event-service ,,'), [ + 'notifications', + 'event-service', + ]); + assert.deepEqual(parseServiceList(undefined), []); +}); diff --git a/dev/local/services.ts b/dev/local/services.ts index e169b39690..444e94dbdd 100644 --- a/dev/local/services.ts +++ b/dev/local/services.ts @@ -1160,6 +1160,44 @@ export function resolveTargets(targets: string[]): string[] { return topologicalSort(resolveTransitiveDeps(allNames)); } +export type ServiceExclusion = { + serviceNames: string[]; + skipped: string[]; + /** Selected services that declare a dependency on a skipped service. */ + dependents: Map; +}; + +/** + * Drop explicitly unwanted services from a resolved selection, for memory- + * constrained environments that run only what they exercise. Unknown names + * throw so a typo cannot silently start the full stack. + */ +export function excludeServices( + serviceNames: readonly string[], + excluded: readonly string[] +): ServiceExclusion { + for (const name of excluded) getService(name); + const excludedSet = new Set(excluded); + const kept = serviceNames.filter(name => !excludedSet.has(name)); + const dependents = new Map(); + for (const name of kept) { + const missing = getService(name).dependsOn.filter(dep => excludedSet.has(dep)); + if (missing.length > 0) dependents.set(name, missing); + } + return { + serviceNames: kept, + skipped: serviceNames.filter(name => excludedSet.has(name)), + dependents, + }; +} + +export function parseServiceList(value: string | undefined): string[] { + return (value ?? '') + .split(',') + .map(name => name.trim()) + .filter(name => name !== ''); +} + export function getService(name: string): ServiceDef { const svc = services.get(name); if (!svc) throw new Error(`Unknown service: ${name}`);