diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 273fa1dd33..fb781b1102 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -613,6 +613,7 @@ jobs: - 'scripts/tests/test-agentic-engineer-schedule.sh' - 'scripts/tests/test-agentic-engineer-lane-store.sh' - 'scripts/tests/test-agentic-engineer-lane-store-lease.sh' + - 'scripts/tests/test-agentic-engineer-lane-store-gateway.sh' - 'scripts/tests/test-kagent-claude-harness-wrapper.sh' - 'scripts/kagent-claude-harness/**' - 'scripts/tests/test-anthropic-proxy-envoy-config.sh' @@ -2899,6 +2900,15 @@ jobs: shellcheck scripts/tests/test-agentic-engineer-lane-store-lease.sh bash scripts/tests/test-agentic-engineer-lane-store-lease.sh + - name: 🚪 Validate the lane store through Substrate's egress gateway + if: needs.changes.outputs.k8s == 'true' + # Runs the gateway image Substrate deploys in front of the store's pinned + # rclone: a WebDAV method that opens a connection gets no reply, and the + # run message's own helper reaches PROPFIND, MKCOL, LOCK and UNLOCK. + run: | + shellcheck scripts/tests/test-agentic-engineer-lane-store-gateway.sh + bash scripts/tests/test-agentic-engineer-lane-store-gateway.sh + - name: 🧠 Validate the kagent harness plugin wrapper if: needs.changes.outputs.k8s == 'true' # Runs the image's real `claude` wrapper against a stub Claude Code: a diff --git a/k8s/providers/hetzner/apps/agentic-engineer/egress-policy-lane-store.yaml b/k8s/providers/hetzner/apps/agentic-engineer/egress-policy-lane-store.yaml index f3977cbf3f..10212b13c5 100644 --- a/k8s/providers/hetzner/apps/agentic-engineer/egress-policy-lane-store.yaml +++ b/k8s/providers/hetzner/apps/agentic-engineer/egress-policy-lane-store.yaml @@ -1,6 +1,12 @@ # The lane store for the agent (deployment-lane-store.yaml). The gateway # replaces the Authorization header with the store credential, so the agent # sends only `Authorization: Basic kagent-credential-injected`. +# +# The gateway takes a plaintext connection for HTTP only when its first +# request uses one of nine standard methods; any other first method is +# dropped as opaque TCP. So the agent sends a WebDAV method (PROPFIND, MKCOL, +# LOCK, UNLOCK) only on a connection an OPTIONS opened, through the run +# message's `dav` helper (scripts/tests/test-agentic-engineer-lane-store-gateway.sh). apiVersion: api.kagent.dev/v1alpha3 kind: EgressPolicy metadata: diff --git a/k8s/providers/hetzner/apps/agentic-engineer/job-schedule.yaml b/k8s/providers/hetzner/apps/agentic-engineer/job-schedule.yaml index fd2db6dcff..072361a34c 100644 --- a/k8s/providers/hetzner/apps/agentic-engineer/job-schedule.yaml +++ b/k8s/providers/hetzner/apps/agentic-engineer/job-schedule.yaml @@ -97,17 +97,29 @@ spec: Scheduled run of the devantler-tech Agentic Engineer on its kagent lane: instance `kagent-cluster`, writer namespace `kagent/*`, one run a day. + The lane store is a WebDAV server. Its gateway treats a connection as + HTTP only when the connection's first request uses a standard method, + so a PROPFIND, MKCOL, LOCK or UNLOCK sent first gets an empty reply. + Send each of those through this helper, which opens its connection + with an OPTIONS and then prints the reply's status, headers and body: + + S=http://lane-store.agentic-engineer.svc.cluster.local:8080 + A='Authorization: Basic kagent-credential-injected' + dav() { m=$1 p=$2; shift 2; curl -sS -o /dev/null -X OPTIONS -H "$A" "$S/" --next -sS -D - -X "$m" -H "$A" "$@" "$S/$p"; } + + Send a GET or PUT with curl directly, with the same header. Read every + status with `-sS -D -`, never -f, which hides it. + 1. Take the lane store's single-writer lease: an exclusive WebDAV lock on memory/ that also fences every write under it, so that overlapping runs never overwrite each other's memory (devantler-tech/monorepo#4910). - Read every status with `curl -sS -D -`, not -f, which hides it. - a. Send a PROPFIND to memory/ with the store header and `Depth: 0`. - A 404 means a first run: create memory/ with a MKCOL. Any reply - but 207 or that 404 stops the run. - b. Send a LOCK to memory/ with the store header, `Timeout: Infinite` - and `Content-Type: application/xml`, and this body with OWNER - replaced by `agentic-engineer`, your UTC start time and a random - 16-hex-digit nonce, which is your owner: + a. Run `dav PROPFIND memory/ -H 'Depth: 0'`. A 404 means a first run: + create memory/ with `dav MKCOL memory/`. Any reply but 207 or that + 404 stops the run. + b. Take the lease with `dav LOCK memory/ -H 'Timeout: Infinite'`, + plus `-H 'Content-Type: application/xml'` and `--data-binary` + with this body, OWNER replaced by `agentic-engineer`, your UTC + start time and a random 16-hex-digit nonce, which is your owner: `OWNER` A 200 means you hold the lease: keep the reply's Lock-Token value, angle brackets included. A 423 means another run holds it, or a @@ -126,39 +138,38 @@ spec: step 4 and check again. If it is still not safe, or the refresh fails or names another owner, upload nothing, leave the lease alone and report it. - 2. Restore your native memory from the lane store: list memory/ with a - PROPFIND (`-X PROPFIND -H 'Depth: 1'` plus the store header), list - each folder it shows the same way, and download every file into - ~/lane-memory/, which stays empty on a first run. A listing or - download you cannot complete stops the run: release the lease with - the UNLOCK of step 4, upload nothing and report it. + 2. Restore your native memory from the lane store: list memory/ with + `dav PROPFIND memory/ -H 'Depth: 1'`, list each folder it shows the + same way, and download every file into ~/lane-memory/ with a GET; + ~/lane-memory/ stays empty on a first run. A listing or download you + cannot complete stops the run: release the lease with the UNLOCK of + step 4, upload nothing and report it. 3. Shallow-clone devantler-tech/monorepo, read its AGENTS.md, load the agentic-engineering plugin and invoke its agentic-engineer entrypoint as this scheduled run. ~/lane-memory/ is your native memory (the legacy layout), as the durable-memory guide says for this lane. Follow the consumer contract and fail closed on missing deployment facts. Do not duplicate the role here. - 4. Before you stop, however the run ended, refresh the lease: a LOCK to - memory/ with the store header, `Timeout: Infinite`, no body, and your - Lock-Token value in parentheses as the If header (`If: ()` for - `Lock-Token: `). You still hold the lease only if the reply is 200 - and its owner is yours. On any other reply, upload nothing, leave the - lease alone and report it. Otherwise upload ~/lane-memory/ back to - memory/ one file at a time, with that same If header on every PUT and - on the MKCOL of any folder inside memory/: the store refuses a write - under memory/ that lacks the lease's token, and a second write with - one token while the first is still running. A 412 or 423 means you - lost the lease: stop uploading, leave the lease alone and report it. - Then upload this session's transcript (the newest - Claude Code session JSONL under your Claude config directory) and your - run report, with no If header, to runs//transcript.jsonl and - runs//report.md. If an earlier run that day already wrote - them, never replace them: write yours beside them as - transcript-.jsonl and report-.md. - Create a missing folder with MKCOL before writing into it, and check - that every upload succeeded. Last, when every write under memory/ - succeeded, release the lease: an UNLOCK to memory/ with the store - header and your Lock-Token value as the Lock-Token header. + 4. Before you stop, however the run ended, refresh the lease with + `dav LOCK memory/ -H 'Timeout: Infinite' -H 'If: ()'`, no body, + where `` is your Lock-Token value. You still hold the lease only + if the reply is 200 and its owner is yours. On any other reply, + upload nothing, leave the lease alone and report it. Otherwise upload + ~/lane-memory/ back to memory/ one file at a time, with that same If + header on every PUT and on the `dav MKCOL` of any folder inside + memory/: the store refuses a write under memory/ that lacks the + lease's token, and a second write with one token while the first is + still running. A 412 or 423 means you lost the lease: stop uploading, + leave the lease alone and report it. Then upload this session's + transcript (the newest Claude Code session JSONL under your Claude + config directory) and your run report, with no If header, to + runs//transcript.jsonl and runs//report.md. If an + earlier run that day already wrote them, never replace them: write + yours beside them as transcript-.jsonl and + report-.md. Create a missing folder with `dav MKCOL` + before writing into it, and check that every upload succeeded. Last, + when every write under memory/ succeeded, release the lease with + `dav UNLOCK memory/ -H 'Lock-Token: '`. This run may open, update and review devantler-tech pull requests and issues, per AGENTS.md. It never merges. Only when genuinely blocked on a diff --git a/scripts/tests/test-agentic-engineer-lane-store-gateway.sh b/scripts/tests/test-agentic-engineer-lane-store-gateway.sh new file mode 100755 index 0000000000..4e5747bb76 --- /dev/null +++ b/scripts/tests/test-agentic-engineer-lane-store-gateway.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Prove the kagent lane reaches its store's WebDAV methods through Substrate's egress gateway. +# +# A run reaches the lane store only through Substrate's egress gateway, agentgateway in AUTO mode. +# It decides whether a plaintext connection is HTTP from the connection's first bytes, and knows +# only nine standard methods there (crates/agentgateway/src/proxy/gateway.rs, `METHODS`). A +# PROPFIND, MKCOL, LOCK or UNLOCK that opens a connection is taken for opaque TCP and dropped, so the +# lane's first full run stopped at its first store request with an empty reply. After that first +# request the gateway parses every request on the connection as HTTP, whatever its method. The run +# message therefore sends each WebDAV method through a `dav` helper that opens its connection with +# an OPTIONS. +# +# This runs the gateway image Substrate deploys, with the AUTO bind Substrate configures for its +# plaintext egress, in front of the store's own pinned rclone, and proves: +# - a WebDAV method that opens a connection gets no reply at all, which is why the helper exists; +# - the run message's own helper reaches PROPFIND, MKCOL, LOCK and UNLOCK through the gateway, with +# their real statuses, headers and bodies, and a standard PUT that carries the lease's token lands. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +dir="${root}/k8s/providers/hetzner/apps/agentic-engineer" +# The gateway image Substrate's control-plane chart runs as atenet-egress. The chart, not this +# repository, chooses it, so the pin below is the one chart 1.0.0 deploys; moving the chart fails +# this test until the pin follows it. +gateway_image='cr.agentgateway.dev/agentgateway:v1.7.0@sha256:60216aafb03d37f94e952ff3d51d2f630a65b10ecaebdc0a864882f92610c45b' +gateway_chart='1.0.0' +work="$(mktemp -d)" +network='' +containers=() +completed=0 +# Remove the containers, their network and the work directory on exit, and fail any exit that did +# not reach the PASS. +cleanup() { + local name + for name in "${containers[@]}"; do + docker rm -f "${name}" >/dev/null 2>&1 || true + done + if [[ -n "${network}" ]]; then + docker network rm "${network}" >/dev/null 2>&1 || true + fi + rm -rf "${work}" + if [[ "${completed}" != 1 ]]; then + printf 'FAIL: agentic-engineer lane store gateway: aborted before completion\n' >&2 + exit 1 + fi +} +trap cleanup EXIT +# Print one failure reason and stop. +fail() { + printf 'FAIL: agentic-engineer lane store gateway: %s\n' "$1" >&2 + exit 1 +} +for tool in curl docker yq; do + command -v "${tool}" >/dev/null || fail "${tool} is required" +done + +chart="$(yq '.spec.ref.tag' "${root}/k8s/bases/infrastructure/substrate/oci-repository-control-plane.yaml")" +[[ "${chart}" == "${gateway_chart}" ]] || + fail "Substrate's chart is ${chart}, not ${gateway_chart}: pin the gateway image that chart deploys as atenet-egress here" + +# The run message's store address, header and helper, exactly as a run defines them. +prompt="$(yq '.spec.template.spec.containers[] | select(.name == "schedule") | .env[] | + select(.name == "PROMPT") | .value' "${dir}/job-schedule.yaml")" +helper="$(grep -E '^ *(S=|A=|dav\(\) )' <<<"${prompt}" || true)" +[[ "$(grep -c . <<<"${helper}")" == 3 ]] || fail "the run message does not define S, A and dav exactly once: ${helper}" +[[ "${helper}" == *'S=http://lane-store.agentic-engineer.svc.cluster.local:8080'* ]] || + fail 'the helper does not address the lane store' +[[ "${helper}" == *"A='Authorization: Basic kagent-credential-injected'"* ]] || + fail "the helper does not send the placeholder the store's egress policy replaces" +[[ "${helper}" == *'-X OPTIONS'*'--next'* ]] || fail 'the helper does not open its connection with an OPTIONS' +pattern='(<[?]xml [^`]*)' +[[ "${prompt}" =~ ${pattern} ]] || fail 'the run message gives no lock body' +lockinfo="${BASH_REMATCH[1]}" + +# The store's image and arguments, without the credential: the gateway's egress policy adds that, +# and it has no part in which methods pass. +deployment="${dir}/deployment-lane-store.yaml" +store_image="$(yq '.spec.template.spec.containers[] | select(.name == "webdav") | .image' "${deployment}")" +[[ "${store_image}" =~ ^docker\.io/rclone/rclone:[^@]+@sha256:[0-9a-f]{64}$ ]] || + fail "the store does not run a digest-pinned rclone: ${store_image}" +args=() +while IFS= read -r arg; do + case "${arg}" in + --htpasswd=*) ;; + *) args+=("${arg}") ;; + esac +done < <(yq '.spec.template.spec.containers[] | select(.name == "webdav") | .args[]' "${deployment}") +[[ "${args[*]:0:4}" == 'serve webdav /data --addr=:8080' ]] || + fail "the store no longer serves WebDAV from /data on :8080: ${args[*]}" + +network="lane-store-gateway-$$-${RANDOM}" +docker network create "${network}" >/dev/null +store="${network}-store" +containers+=("${store}") +mkdir -p "${work}/data" "${work}/gateway" +docker run -d --name "${store}" --network "${network}" --user "$(id -u):$(id -g)" --read-only --tmpfs /tmp \ + --cap-drop ALL --security-opt no-new-privileges --mount "type=bind,src=${work}/data,dst=/data" \ + "${store_image}" "${args[@]}" >/dev/null +store_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${store}")" +[[ -n "${store_ip}" ]] || fail 'the store got no address on the test network' +# Substrate's egress bind is AUTO; its HTTP listener carries every plaintext request a run sends. +# The HTTPS and TLS listeners beside it take only TLS, so they play no part in what follows. +cat >"${work}/gateway/config.yaml" </dev/null + +# Wait until a standard request crosses the gateway to the store. +url='' +for attempt in $(seq 1 30); do + ports="$(docker port "${gateway}" 3000/tcp 2>/dev/null || true)" + address="${ports%%$'\n'*}" + if [[ -n "${address}" ]] && + [[ "$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -X OPTIONS "http://${address}/" 2>/dev/null || true)" == 200 ]]; then + url="http://${address}" + break + fi + sleep 1 +done +[[ -n "${url}" ]] || fail "the gateway did not reach the store after ${attempt} tries: $(docker logs "${gateway}" 2>&1 | tail -c 800)" + +# A WebDAV method that opens a connection is taken for opaque TCP and dropped: curl gets no reply. +status=0 +curl -sS --max-time 20 -o /dev/null -X PROPFIND -H 'Depth: 0' "${url}/" 2>"${work}/bare.err" || status=$? +[[ "${status}" == 52 ]] || + fail "a PROPFIND that opens a connection now gets a reply (curl ${status}): the gateway may pass WebDAV methods, so revisit the helper" + +# The run message's helper, pointed at this gateway in place of the cluster's. The helper reads S. +eval "${helper}" +# shellcheck disable=SC2034 +S="${url}" +# reply : runs the helper, keeps its output as , prints the status. +reply() { + local name="$1" + shift + dav "$@" | tr -d '\r' >"${work}/${name}" || fail "the helper failed for $*: $(cat "${work}/${name}")" + sed -n '1s/^HTTP\/[0-9.]* \([0-9][0-9][0-9]\).*/\1/p' "${work}/${name}" +} +# header
: one header of a kept reply. +header() { awk -v h="$2" 'BEGIN {h = tolower(h) ":"} tolower(substr($0, 1, length(h))) == h { + sub(/^[^:]*:[[:space:]]*/, ""); print; exit}' "${work}/$1"; } +owner='agentic-engineer 2026-11-11T05:15:00Z 0123456789abcdef' + +[[ "$(reply missing PROPFIND memory/ -H 'Depth: 0')" == 404 ]] || fail "PROPFIND of a missing memory/ did not answer 404: $(head -c 300 "${work}/missing")" +[[ "$(reply created MKCOL memory/)" == 201 ]] || fail "MKCOL did not create memory/: $(head -c 300 "${work}/created")" +[[ "$(reply listed PROPFIND memory/ -H 'Depth: 1')" == 207 ]] || fail "PROPFIND did not list memory/: $(head -c 300 "${work}/listed")" +grep -q 'multistatus' "${work}/listed" || fail 'the listing has no multistatus body' +[[ "$(reply locked LOCK memory/ -H 'Timeout: Infinite' -H 'Content-Type: application/xml' --data-binary "${lockinfo/OWNER/${owner}}")" == 200 ]] || + fail "LOCK did not take the lease: $(head -c 300 "${work}/locked")" +token="$(header locked Lock-Token)" +[[ "${token}" =~ ^\<[0-9]+\>$ ]] || fail "the lease reply carried no Lock-Token: ${token}" +grep -Fq ">${owner}<" "${work}/locked" || fail 'the lease reply does not name its holder' +[[ "$(reply refreshed LOCK memory/ -H 'Timeout: Infinite' -H "If: (${token})")" == 200 ]] || fail 'the refresh did not hold the lease' +grep -Fq ">${owner}<" "${work}/refreshed" || fail 'the refresh does not name the holder' +put="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT -H "${A}" -H "If: (${token})" --data-binary memory "${url}/memory/MEMORY.md")" +[[ "${put}" == 201 ]] || fail "a PUT with the lease's token did not land: ${put}" +[[ "$(reply folder MKCOL memory/archive/ -H "If: (${token})")" == 201 ]] || fail 'MKCOL with the lease token did not create a folder' +[[ "$(reply released UNLOCK memory/ -H "Lock-Token: ${token}")" == 204 ]] || fail "UNLOCK did not release the lease: $(head -c 300 "${work}/released")" + +grep -Fq 'bash scripts/tests/test-agentic-engineer-lane-store-gateway.sh' "${root}/.github/workflows/ci.yaml" || + fail 'CI does not run this test' + +completed=1 +printf 'agentic-engineer lane store gateway: PASS\n' diff --git a/scripts/tests/test-agentic-engineer-lane-store-lease.sh b/scripts/tests/test-agentic-engineer-lane-store-lease.sh index ccf664fe58..e9a8895c34 100755 --- a/scripts/tests/test-agentic-engineer-lane-store-lease.sh +++ b/scripts/tests/test-agentic-engineer-lane-store-lease.sh @@ -57,16 +57,16 @@ at() { if [[ "${before}" == "${prompt}" ]]; then printf '%s' -1; else printf '%s' "${#before}"; fi } previous=-1 -for step in 'Send a PROPFIND to memory/' 'Send a LOCK to memory/' 'at least 60 below a reply' \ - 'download every file' 'however the run ended, refresh the lease' 'one file at a time' 'UNLOCK to memory/'; do +for step in "dav PROPFIND memory/ -H 'Depth: 0'" 'Take the lease with' 'at least 60 below a reply' \ + "dav PROPFIND memory/ -H 'Depth: 1'" 'download every file' 'however the run ended, refresh the lease' \ + "dav LOCK memory/ -H 'Timeout: Infinite' -H 'If:" 'one file at a time' "dav UNLOCK memory/ -H 'Lock-Token:"; do position="$(at "${step}")" [[ "${position}" -gt "${previous}" ]] || fail "the run message does not say, in order: ${step}" previous="${position}" done for rule in 'Timeout: Infinite' 'A 423 means another run holds it' 'Never remove a lease you did not take' \ - 'its owner is yours' 'If header on every PUT' 'on the MKCOL of any folder inside memory/' \ - 'each folder it shows the same way' 'leave the lease alone' 'A 412' \ - 'never UNLOCK it' 'wait out the difference'; do + 'its owner is yours' 'header on every PUT' 'of any folder inside' 'list each folder it shows the' \ + 'leave the lease alone' 'A 412' 'never UNLOCK it' 'wait out the difference'; do [[ "$(at "${rule}")" -ge 0 ]] || fail "the run message no longer says: ${rule}" done pattern='(<[?]xml [^`]*)'