From 1fa5aa2644a514149a9258948c04141abb04d35b Mon Sep 17 00:00:00 2001 From: kperry Date: Sun, 8 Nov 2026 14:02:18 -0600 Subject: [PATCH] feat(examples): score the smoke run's behavior into a pinned Trust Index Add make openshell-smoke-trust-index, the OpenShell smoke run plus five checks that take the run's policy decisions to a Trust Index score: the gateway's OCSF JSON export is switched on, the approved sandbox's supervisor records are collected, ans-ocsf-hydrator scores them and imports the score into a pinned agent-trust-discovery release the harness starts on a loopback port, and the score is read back from the agent's trust evaluation at weight 0. make openshell-smoke is unchanged. The behavior path lives in trust-index.sh, sourced by smoke.sh only under --with-trust-index. The Trust Index loads the signal registration this repository ships for operators and a profile naming only that signal. The per-run admin key travels through the server's environment and files for the hydrator and curl, never a command line or a retained log. The decision count the assertions expect comes from sandbox_curl, so it cannot drift from the checks that run. Release downloads use a per-run partial file, which also closes a latent race between two cold-cache OpenShell runs. CI runs both targets as matrix legs with per-leg cache keys; the trust-index leg runs the gateway at info because the supervisor's lossy console queue is shared with the records it counts. Co-Authored-By: Claude Fable 5.1 Signed-off-by: kperry --- .github/workflows/openshell-smoke.yml | 44 ++- Makefile | 7 +- README.md | 9 +- docs/hydrator-activity.md | 13 +- examples/openshell-smoke/README.md | 64 +++- examples/openshell-smoke/smoke.sh | 137 ++++++-- .../openshell-smoke/trust-index-release.env | 10 + examples/openshell-smoke/trust-index.sh | 304 ++++++++++++++++++ 8 files changed, 530 insertions(+), 58 deletions(-) create mode 100644 examples/openshell-smoke/trust-index-release.env create mode 100644 examples/openshell-smoke/trust-index.sh diff --git a/.github/workflows/openshell-smoke.yml b/.github/workflows/openshell-smoke.yml index 939fb23..8d378c4 100644 --- a/.github/workflows/openshell-smoke.yml +++ b/.github/workflows/openshell-smoke.yml @@ -1,7 +1,9 @@ # Runs the extension against the pinned OpenShell release with a real gateway, -# supervisor and sandbox on the Docker driver. Slower than the unit suite and -# dependent on GitHub's Docker daemon and ghcr.io, so it runs on pushes to main, -# on pull requests that touch what it exercises, and on demand. +# supervisor and sandbox on the Docker driver, once for the identity checks and +# once with the run's behavior scored into a pinned agent-trust-discovery. +# Slower than the unit suite and dependent on GitHub's Docker daemon and +# ghcr.io, so it runs on pushes to main, on pull requests that touch what it +# exercises, and on demand. Neither job is a required status check. name: OpenShell smoke on: @@ -12,7 +14,10 @@ on: paths: - "cmd/ans-openshell-middleware/**" - "cmd/ans-reference-callee/**" + - "cmd/ans-ocsf-hydrator/**" - "internal/**" + - "pkg/**" + - "deploy/examples/trust-index-score-signals.yaml" - "gen/**" - "third_party/openshell/**" - "examples/openshell-smoke/**" @@ -27,34 +32,49 @@ permissions: jobs: smoke: - name: Gateway, supervisor and sandbox on Docker + name: ${{ matrix.target }} runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + target: [openshell-smoke, openshell-smoke-trust-index] + include: + # The gateway runs at debug so a middleware_failed verdict on CI + # explains itself; smoke.sh refuses debug under --with-trust-index + # (see its usage text), so that run stays at info. + - target: openshell-smoke + log_level: debug + - target: openshell-smoke-trust-index + log_level: info steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e #v7.0.0 with: go-version: "1.26.9" - - name: Cache OpenShell release assets + - name: Cache release assets + # Keyed per target: the two jobs cache different asset sets, and a shared + # key would be saved by whichever job finished first and never refilled. uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0 with: path: ~/.cache/ans-openshell-smoke - key: openshell-${{ hashFiles('examples/openshell-smoke/openshell-release.env') }}-${{ runner.os }}-${{ runner.arch }} + key: ${{ matrix.target }}-${{ hashFiles('examples/openshell-smoke/*-release.env') }}-${{ runner.os }}-${{ runner.arch }} - name: Run the smoke test # A fixed TMPDIR makes the retained log directory addressable by the - # upload step below; the gateway runs at debug so a middleware_failed - # verdict on CI explains itself. + # upload step below. env: TMPDIR: ${{ runner.temp }}/smoke - ANS_SMOKE_LOG_LEVEL: debug + ANS_SMOKE_LOG_LEVEL: ${{ matrix.log_level }} run: | mkdir -p "$TMPDIR" - make openshell-smoke + make ${{ matrix.target }} - name: Upload retained logs - if: failure() + # Also on cancellation, so a run cut off by timeout-minutes still + # leaves its logs behind. + if: failure() || cancelled() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 #v7.0.2 with: - name: openshell-smoke-logs + name: ${{ matrix.target }}-logs path: ${{ runner.temp }}/smoke/ans-openshell-smoke.*/logs/ if-no-files-found: ignore retention-days: 7 diff --git a/Makefile b/Makefile index 352b0b2..d8f2b54 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ LINT_GOTOOLCHAIN ?= local BIN := bin CMDS := ans-openshell-middleware ans-openshell-interceptor ans-ocsf-hydrator ans-reference-callee -.PHONY: all build proto test cover lint vet demo openshell-smoke clean +.PHONY: all build proto test cover lint vet demo openshell-smoke openshell-smoke-trust-index clean all: build @@ -48,3 +48,8 @@ clean: # End-to-end check against a real OpenShell gateway (Docker). See examples/openshell-smoke/README.md. openshell-smoke: examples/openshell-smoke/smoke.sh + +# The same run, then its OCSF records scored by the hydrator and imported into a pinned +# agent-trust-discovery. See "Scoring the run's behavior" in the same README. +openshell-smoke-trust-index: + examples/openshell-smoke/smoke.sh --with-trust-index diff --git a/README.md b/README.md index 19bc111..5081055 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,11 @@ down. See [examples/openshell-smoke](examples/openshell-smoke/README.md): make openshell-smoke ``` +`make openshell-smoke-trust-index` runs the same test and then scores the run's behavior: it +enables OpenShell's OCSF export, feeds the supervisor's records through the hydrator into a pinned +`agent-trust-discovery` release it downloads and runs, and checks the score on the agent's trust +evaluation. + ## Running the services Every service takes one YAML file. Flags: `--config` (required), `--listen` to override the @@ -75,7 +80,9 @@ configured address, `--log-level` (`debug`, `info`, `warn`, `error`), `--log-for ``` The hydrator is a batch job over the sandbox supervisors' OCSF records. Enable OpenShell's JSON -export with `openshell settings set --global --key ocsf_json_enabled --value true`; each +export with `openshell settings set --global --key ocsf_json_enabled --value true` (at a terminal +the CLI asks you to confirm that a global value disables per-sandbox control of the key; pass +`--yes` only where no terminal is attached); each supervisor then writes one `OCSF-JSON` line per event to its stderr and, where its `/var/log` is writable, to `/var/log/openshell-ocsf..jsonl` inside the supervisor container. Collect either with your log pipeline into a file the hydrator can read, one bare line per record: it diff --git a/docs/hydrator-activity.md b/docs/hydrator-activity.md index 5488bb0..a020dde 100644 --- a/docs/hydrator-activity.md +++ b/docs/hydrator-activity.md @@ -178,7 +178,12 @@ the example leaves it unset. The evidence output keeps the refusal counts. - The score is rounded, so a single violation among 200 or more decisions still reports 100. The evidence output keeps the exact counts. -- Rollout behavior (weight 0 has no effect on trust decisions, retained scores, - whole-batch rejection) belongs to the consumer and is not tested in this - repository. Verify it against the deployed `agent-trust-discovery` before - changing the weight. +- Rollout behavior belongs to the consumer. `make openshell-smoke-trust-index` + exercises the registration from + [trust-index-score-signals.yaml](../deploy/examples/trust-index-score-signals.yaml), + the import, and the weight-0 state (the score is listed on the agent and the + behavior dimension stays 0) against the pinned `agent-trust-discovery` release + named in [trust-index-release.env](../examples/openshell-smoke/trust-index-release.env). + Retention of stale scores and whole-batch rejection are not exercised there; + verify both against the deployed `agent-trust-discovery` before changing the + weight. diff --git a/examples/openshell-smoke/README.md b/examples/openshell-smoke/README.md index 4721ac0..064a489 100644 --- a/examples/openshell-smoke/README.md +++ b/examples/openshell-smoke/README.md @@ -7,7 +7,9 @@ that OpenShell invokes the service, applies its header mutations before the requ and blocks the request when the service is down. > OpenShell's supervisor-middleware contract is a research preview upstream and may change. -> The test pins one release; see [Updating the OpenShell pin](#updating-the-openshell-pin). +> The test pins one OpenShell release, and one agent-trust-discovery release for the behavior +> path; see [Updating the OpenShell pin](#updating-the-openshell-pin) and +> [Updating the Trust Index pin](#updating-the-trust-index-pin). ## What it does @@ -46,6 +48,41 @@ make openshell-smoke Replay and body tampering are covered by the verifier's own tests and by `make demo`, and are reported there, not here. +## Scoring the run's behavior + +```bash +make openshell-smoke-trust-index +``` + +The same run with `--with-trust-index`, which adds the behavior path: the policy decisions +OpenShell makes for the sandbox become a Trust Index score. Before the callee starts, the harness +starts a pinned [agent-trust-discovery](https://github.com/agentnameservice/agent-trust-discovery) +release on a loopback port, registers the fixture agent with it, and, once the gateway is up, +turns on OpenShell's OCSF JSON export for the run's sandboxes. The Trust Index loads the signal +registration this repository ships for operators, +[deploy/examples/trust-index-score-signals.yaml](../../deploy/examples/trust-index-score-signals.yaml), +and a profile that names only that signal, at weight 0. Right after the sandbox is created the +run proves the export is live, so that check prints before the identity checks; after them it +makes one request the policy does not allow, collects the supervisor's OCSF records, scores them +with the hydrator built from this checkout and reads the score back. Five more checks: + +| Check | Expected observation | +|-------|----------------------| +| Export live | Before the first request, the approved sandbox's supervisor log already carries `OCSF-JSON` records; the supervisor applies `ocsf_json_enabled` on its first settings poll. | +| Policy-denied request | A `DELETE /v1/tasks`, which the policy does not allow, gets `403 policy_denied` from the supervisor, and the callee's log does not grow. | +| Attributable records | Every collected record names the approved sandbox by its gateway-issued id, and the records hold exactly the run's L7 decisions: one `allow` per request the sandbox made, including the ones answered `middleware_failed`, and the one `deny`. | +| Hydrator score | The hydrator's evidence shows those counts and a score that is not withheld; the import to the Trust Index succeeds. The score's formula belongs to the hydrator and its unit tests, so the harness reads the computed value rather than restating the formula. | +| Trust Index read-back | The agent's trust evaluation lists `openshell.behavior.activity` under `behavior` with the imported score at weight 0, and the behavior dimension stays 0. | + +Only the approved sandbox is mapped in the hydrator configuration, by its gateway-issued id, and +only its supervisor's records are collected; the unapproved sandbox never enters the behavior +path. The import route runs with a per-run +admin key that reaches the server through its environment and the hydrator and `curl` through +files, so no command line and no retained log carries it. The Trust Index listens on +`127.0.0.1` only, which is the one control on its unauthenticated read routes, and the hydrator's +hop to it is plaintext on that interface. A run under the flag downloads and executes a second +pinned third-party binary, verified the same way as the OpenShell assets. + ## Requirements - Linux (x86_64 or arm64) or macOS on Apple silicon, with Bash, `curl`, `jq`, `tar`, @@ -73,11 +110,13 @@ the next run removes everything under a label whose pid no longer runs. | Flag or variable | Effect | |------------------|--------| -| `--keep-running` | After the assertions pass, keep the gateway, middleware, callee and sandbox up until Ctrl-C and print the commands to use them. | -| `--print-config` | Print the rendered `gateway.toml` and `middleware.yaml` and exit. | +| `--keep-running` | After the assertions pass, keep the gateway, middleware, callee and sandbox up until Ctrl-C and print the commands to use them. Under `--with-trust-index` the Trust Index stays up too and its detail query is printed. | +| `--print-config` | Print the rendered `gateway.toml` and `middleware.yaml` and exit. Under `--with-trust-index` the Trust Index and hydrator configuration follow. | +| `--with-trust-index` | Add the behavior checks: OCSF export, the hydrator, and a pinned `agent-trust-discovery` (see [Scoring the run's behavior](#scoring-the-runs-behavior)). | | `ANS_SMOKE_HOST` | Address the gateway and sandboxes use to reach the middleware and callee. | | `ANS_SMOKE_CACHE` | Where release assets are cached. | -| `ANS_SMOKE_LOG_LEVEL` | Gateway log level, default `info`. | +| `ANS_SMOKE_LOG_LEVEL` | Gateway log level, default `info`. The Docker driver forwards it to the supervisors, whose console queue is lossy and carries the OCSF records the behavior path counts, so `--with-trust-index` refuses `debug` and `trace`. | +| `ANS_SMOKE_TRUST_INDEX_LOG_LEVEL` | Log level of the Trust Index and the hydrator under `--with-trust-index`, default `info`. | ## Fixture trust versus production trust @@ -102,6 +141,7 @@ fixtures minted so the middleware's JWKS matches. | Vendored extension protos | `OPENSHELL_COMMIT` in the same file | CI's proto-drift job diffs them against that commit | | Supervisor and sandbox runtime images | manifest-list digests in the same file, passed to the gateway as `supervisor_image` and `sandbox_runtime_image` | the container engine, by digest | | Workload base image | manifest-list digest in [workload/Dockerfile](workload/Dockerfile) | the container engine, by digest | +| agent-trust-discovery tarball (`--with-trust-index`) | SHA-256 per platform in [trust-index-release.env](trust-index-release.env), copied from the release's `checksums.txt` (the release publishes no signature or attestation, so the digest and the tarball come from the same page); `TRUST_INDEX_COMMIT` is printed in the run summary | `smoke.sh`, every run, before extraction into the run directory | The `apt-get install curl` layer in the workload image is the one unpinned input; the identity assertions do not rest on it, because the callee's own log corroborates the two denial checks and @@ -120,8 +160,8 @@ its rejection logging is proven live first. ## Ownership -The harness, its OpenShell pin and the `OpenShell smoke` workflow are maintained by this -repository's maintainers under its normal [contribution process](../../CONTRIBUTING.md); a +The harness, its OpenShell and agent-trust-discovery pins and the `OpenShell smoke` workflow are +maintained by this repository's maintainers under its normal [contribution process](../../CONTRIBUTING.md); a maintainer roster is proposed in the governance change under review. File problems as issues on this repository. The link from OpenShell's supervisor-middleware documentation belongs to the upstream contribution and is not part of this repository. @@ -134,3 +174,15 @@ the upstream contribution and is not part of this repository. 2. Re-pin the vendored contract (`third_party/openshell`, then `make proto`) if the proto-drift job reports a difference at the new commit. 3. Run `make openshell-smoke`. + +## Updating the Trust Index pin + +1. Edit [trust-index-release.env](trust-index-release.env): the version, the tag's commit, and + the SHA-256 of each platform's tarball from the release's `checksums.txt`. +2. The release tarballs ship no configuration, so [trust-index.sh](trust-index.sh) carries the + contract the harness relies on: the `runtime.yaml` and profile keys it writes, the agent + import body it posts, and the trust-evaluation fields it reads back. Compare them with the + new release's `config/runtime.yaml`, `config/default-profile.yaml`, agent import handler and + agent detail response, and adjust the script where they differ. The signal registration stays + `deploy/examples/trust-index-score-signals.yaml`. +3. Run `make openshell-smoke-trust-index`. diff --git a/examples/openshell-smoke/smoke.sh b/examples/openshell-smoke/smoke.sh index 82c91d7..8fa64fb 100755 --- a/examples/openshell-smoke/smoke.sh +++ b/examples/openshell-smoke/smoke.sh @@ -20,33 +20,47 @@ # The in-process demo (make demo) covers replay and body tampering; this script # covers what only a real runtime can show: that OpenShell invokes the service, # applies its header mutations, and blocks traffic when it is down. +# +# --with-trust-index adds five checks from trust-index.sh: the run's policy +# decisions are exported as OCSF records, scored by the hydrator and imported +# into a pinned agent-trust-discovery, which lists the score on the agent. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" EXAMPLE_DIR="$ROOT/examples/openshell-smoke" KEEP_RUNNING=0 PRINT_CONFIG=0 +WITH_TRUST_INDEX=0 usage() { cat <&2; usage >&2; exit 2 ;; esac @@ -62,6 +77,14 @@ done # shellcheck source=examples/openshell-smoke/openshell-release.env source "$EXAMPLE_DIR/openshell-release.env" +if [[ $WITH_TRUST_INDEX -eq 1 ]]; then + if [[ "${ANS_SMOKE_LOG_LEVEL:-info}" =~ ^(debug|trace)$ ]]; then + echo "FAIL --with-trust-index needs ANS_SMOKE_LOG_LEVEL=info or quieter: the supervisor's console queue is lossy and carries the OCSF records check 8 counts" >&2 + exit 2 + fi + # shellcheck source=examples/openshell-smoke/trust-index.sh + source "$EXAMPLE_DIR/trust-index.sh" +fi require_command() { command -v "$1" >/dev/null 2>&1 || { echo "FAIL missing required command: $1" >&2; exit 1; } @@ -300,16 +323,18 @@ cleanup() { sleep 0.5 done fi - for pid in "$GATEWAY_PID" "$MIDDLEWARE_PID" "$CALLEE_PID"; do + for pid in "$GATEWAY_PID" "$MIDDLEWARE_PID" "$CALLEE_PID" "${TRUST_INDEX_PID:-}"; do [[ -z "$pid" ]] || stop_process "$pid" done label_resources | awk -v label="$DOCKER_SANDBOX_LABEL" '$3 == label' | remove_resources if [[ $status -eq 0 || -z "$(ls -A "$LOG_DIR" 2>/dev/null)" ]]; then rm -rf "$WORK" else - # Keep the logs for diagnosis, never the minted private keys, the gateway - # database or the CLI client certificate. - rm -rf "$FIXTURES" "$OPENSHELL_LOCAL_TLS_DIR" "$WORK/xdg" "$BIN" "$WORK"/gateway.db* + # Keep the logs for diagnosis and nothing else: the minted private keys, + # the gateway database, the CLI client certificate and whatever a sourced + # extension put under WORK all go. Anything worth keeping is written under + # LOG_DIR by the code that produces it. + find "$WORK" -mindepth 1 -maxdepth 1 ! -name logs -exec rm -rf {} + echo "logs retained in $LOG_DIR" >&2 fi exit "$status" @@ -339,12 +364,13 @@ choose_port_block() { exit 1 } -PORT_BASE="$(choose_port_block 5)" +PORT_BASE="$(choose_port_block $((5 + WITH_TRUST_INDEX)))" MIDDLEWARE_PORT="$PORT_BASE" MIDDLEWARE_HEALTH_PORT="$((PORT_BASE + 1))" CALLEE_PORT="$((PORT_BASE + 2))" GATEWAY_PORT="$((PORT_BASE + 3))" GATEWAY_HEALTH_PORT="$((PORT_BASE + 4))" +[[ $WITH_TRUST_INDEX -eq 0 ]] || TRUST_INDEX_PORT="$((PORT_BASE + 5))" GATEWAY_ENDPOINT="https://127.0.0.1:$GATEWAY_PORT" # The sandbox dials the callee by this authority; the middleware signs for it # and the callee verifies against it (pop trusted host). @@ -354,7 +380,8 @@ fail() { printf 'FAIL %s\n' "$1" >&2 capture_container_logs local log - for log in "$LOG_DIR"/*.log; do + for log in "$LOG_DIR"/*.log "$LOG_DIR"/*.jsonl; do + [[ -f "$log" ]] || continue printf '\n--- %s ---\n' "$log" >&2 tail -n 60 "$log" >&2 done @@ -399,22 +426,26 @@ case "$(uname -s)-$(uname -m)" in *) echo "FAIL unsupported platform $(uname -s)-$(uname -m)" >&2; exit 1 ;; esac -CACHE="${ANS_SMOKE_CACHE:-${XDG_CACHE_HOME:-${HOME}/.cache}/ans-openshell-smoke}/$OPENSHELL_VERSION" +CACHE_ROOT="${ANS_SMOKE_CACHE:-${XDG_CACHE_HOME:-${HOME}/.cache}/ans-openshell-smoke}" +CACHE="$CACHE_ROOT/$OPENSHELL_VERSION" mkdir -p "$CACHE" -# fetch_asset NAME SHA BINARY: downloads the release tarball once, verifies it -# against the pinned digest, and extracts BINARY into the run directory. -# Everything it prints goes to stderr so the callers below can read nothing -# but the binary path from stdout. -fetch_asset() { - local name="$1" sha="$2" bin="$BIN/$3" - local dest="$CACHE/$name.tar.gz" +# fetch_release_asset BASE_URL VERSION CACHE_DIR NAME SHA BINARY: downloads the +# release tarball BASE_URL/VERSION/NAME.tar.gz once into CACHE_DIR, verifies it +# against the pinned digest, and extracts BINARY into the run directory. The +# download lands in a per-run partial file so two cold-cache runs cannot +# corrupt each other's copy. Everything it prints goes to stderr so the callers +# below can read nothing but the binary path from stdout. +fetch_release_asset() { + local base_url="$1" version="$2" cache_dir="$3" name="$4" sha="$5" bin="$BIN/$6" + local dest="$cache_dir/$name.tar.gz" + mkdir -p "$cache_dir" if [[ ! -f "$dest" ]]; then - printf 'INFO downloading %s %s\n' "$OPENSHELL_VERSION" "$name" >&2 + printf 'INFO downloading %s %s\n' "$version" "$name" >&2 curl -fsSL --retry 3 --retry-max-time 600 --connect-timeout 15 \ - --speed-limit 1024 --speed-time 30 --max-time 600 -o "$dest.partial" \ - "https://github.com/NVIDIA/OpenShell/releases/download/$OPENSHELL_VERSION/$name.tar.gz" - mv "$dest.partial" "$dest" + --speed-limit 1024 --speed-time 30 --max-time 600 -o "$dest.partial.$$" \ + "$base_url/$version/$name.tar.gz" || { rm -f "$dest.partial.$$"; fail "download of $name failed"; } + mv "$dest.partial.$$" "$dest" fi if ! sha256_matches "$sha" "$dest"; then rm -f "$dest" @@ -422,11 +453,15 @@ fetch_asset() { fi # The binary is extracted from the verified tarball on every run, into the # run's own directory, so what executes is always what the checksum covered. - tar -xzf "$dest" -C "$BIN" "$3" + tar -xzf "$dest" -C "$BIN" "$6" chmod 0755 "$bin" printf '%s\n' "$bin" } +fetch_asset() { + fetch_release_asset "https://github.com/NVIDIA/OpenShell/releases/download" "$OPENSHELL_VERSION" "$CACHE" "$@" +} + CLI_BIN="$(fetch_asset "$CLI_ASSET" "$CLI_SHA" openshell)" GATEWAY_BIN="$(fetch_asset "$GW_ASSET" "$GW_SHA" openshell-gateway)" [[ -x "$CLI_BIN" && -x "$GATEWAY_BIN" ]] || fail "release binaries are not executable: $CLI_BIN $GATEWAY_BIN" @@ -435,7 +470,14 @@ printf 'INFO openshell %s\n' "$("$CLI_BIN" --version 2>&1 | head -n 1)" # --------------------------------------------------------------------------- # Build the extension and mint the fixtures # --------------------------------------------------------------------------- -step "building the extension" go build -C "$ROOT" -o "$BIN/" ./cmd/ans-openshell-middleware ./cmd/ans-reference-callee ./examples/openshell-smoke/fixtures +BUILD_PACKAGES=(./cmd/ans-openshell-middleware ./cmd/ans-reference-callee ./examples/openshell-smoke/fixtures) +if [[ $WITH_TRUST_INDEX -eq 1 ]]; then + BUILD_PACKAGES+=(./cmd/ans-ocsf-hydrator) + TRUST_INDEX_BIN="$(fetch_release_asset "https://github.com/agentnameservice/agent-trust-discovery/releases/download" \ + "$TRUST_INDEX_VERSION" "$CACHE_ROOT/trust-index/$TRUST_INDEX_VERSION" "$TRUST_INDEX_ASSET" "$TRUST_INDEX_SHA" agent-trust-discovery)" + [[ -x "$TRUST_INDEX_BIN" ]] || fail "release binary is not executable: $TRUST_INDEX_BIN" +fi +step "building the extension" go build -C "$ROOT" -o "$BIN/" "${BUILD_PACKAGES[@]}" step "minting fixtures" "$BIN/fixtures" -out "$FIXTURES" -middleware-host "$SERVICE_HOST" if ! docker image inspect "$WORKLOAD_IMAGE" >/dev/null 2>&1; then step "building the workload image $WORKLOAD_IMAGE" run_bounded 600 docker build -q -t "$WORKLOAD_IMAGE" "$EXAMPLE_DIR/workload" @@ -525,8 +567,15 @@ TOML write_gateway_config write_middleware_config "{}" +[[ $WITH_TRUST_INDEX -eq 0 ]] || trust_index_prepare if [[ $PRINT_CONFIG -eq 1 ]]; then cat "$WORK/gateway.toml" "$WORK/middleware.yaml" + if [[ $WITH_TRUST_INDEX -eq 1 ]]; then + # The sandbox id only exists once the sandbox does; a placeholder stands + # in for it so the printed file is valid YAML. + write_hydrator_config "00000000-0000-0000-0000-000000000000" + trust_index_print_config + fi exit 0 fi @@ -565,6 +614,7 @@ stop_middleware() { fail "middleware port did not close" } +[[ $WITH_TRUST_INDEX -eq 0 ]] || trust_index_start start_callee # The callee has no health route; its first signed request is the readiness check. for _ in {1..30}; do @@ -589,6 +639,7 @@ middleware_negotiated() { | jq -e --arg name "$MIDDLEWARE_NAME" '.extensions[] | select(.configured_name == $name)' >/dev/null } step "verifying the gateway negotiated the middleware" middleware_negotiated +[[ $WITH_TRUST_INDEX -eq 0 ]] || trust_index_enable_export # --------------------------------------------------------------------------- # Sandboxes @@ -608,6 +659,11 @@ create_sandbox() { sandbox_curl() { local name="$1" shift + # Every request is journaled, one line per call, so a failure shows what the + # sandboxes were asked to do. A file rather than a counter because callers + # run this inside a command substitution, where a variable never reaches + # the parent shell. + printf '%s %s\n' "$name" "$*" >>"$LOG_DIR/requests.log" run_bounded 90 "${CLI[@]}" sandbox exec --name "$name" --no-tty -- \ curl -sS --max-time 20 -w '\n%{http_code}' "$@" "http://$CALLEE_AUTHORITY/v1/tasks" \ -H 'content-type: application/json' --data '{"task":"settle invoice 42"}' @@ -622,18 +678,20 @@ SANDBOX_ID="$(create_sandbox "$SANDBOX_NAME")" [[ -n "$SANDBOX_ID" && "$SANDBOX_ID" != null ]] || fail "could not read the sandbox id" printf 'INFO sandbox %s has id %s\n' "$SANDBOX_NAME" "$SANDBOX_ID" -# Approve exactly this sandbox for the identity; the operator-owned map is -# the only thing that grants signing. -write_middleware_config "{$SANDBOX_ID: $IDENTITY}" -stop_middleware -start_middleware - PASS=0 check() { printf 'PASS %s\n' "$1" PASS=$((PASS + 1)) } +[[ $WITH_TRUST_INDEX -eq 0 ]] || trust_index_check_export_live + +# Approve exactly this sandbox for the identity; the operator-owned map is +# the only thing that grants signing. +write_middleware_config "{$SANDBOX_ID: $IDENTITY}" +stop_middleware +start_middleware + # 1. Authorized sandbox: the callee returns the assigned identity. The # middleware was just restarted to approve the sandbox, and the supervisor's # gRPC channel to it (5 s connect timeout, 10 s keepalive) takes a few seconds @@ -701,7 +759,16 @@ grep -q 'middleware_failed' <<<"$BODY" || fail "middleware down: not a middlewar check "request is blocked while the middleware is unavailable (fail_closed)" start_middleware -printf 'ALL PASS %d checks against OpenShell %s (%s)\n' "$PASS" "$OPENSHELL_VERSION" "${OPENSHELL_COMMIT:0:7}" +if [[ $WITH_TRUST_INDEX -eq 1 ]]; then + trust_index_check_policy_denied + trust_index_check_records + trust_index_check_hydrator + trust_index_check_readback + printf 'ALL PASS %d checks against OpenShell %s (%s) and %s\n' \ + "$PASS" "$OPENSHELL_VERSION" "${OPENSHELL_COMMIT:0:7}" "$(trust_index_summary)" +else + printf 'ALL PASS %d checks against OpenShell %s (%s)\n' "$PASS" "$OPENSHELL_VERSION" "${OPENSHELL_COMMIT:0:7}" +fi if [[ $KEEP_RUNNING -eq 1 ]]; then cat </dev/null && kill -0 "$MIDDLEWARE_PID" 2>/dev/null; do sleep 1; done + [[ $WITH_TRUST_INDEX -eq 0 ]] || trust_index_print_ready + while kill -0 "$GATEWAY_PID" 2>/dev/null && kill -0 "$MIDDLEWARE_PID" 2>/dev/null \ + && { [[ -z "${TRUST_INDEX_PID:-}" ]] || kill -0 "$TRUST_INDEX_PID" 2>/dev/null; }; do sleep 1; done fail "a process exited" fi diff --git a/examples/openshell-smoke/trust-index-release.env b/examples/openshell-smoke/trust-index-release.env new file mode 100644 index 0000000..5ade637 --- /dev/null +++ b/examples/openshell-smoke/trust-index-release.env @@ -0,0 +1,10 @@ +# agent-trust-discovery release the smoke test imports the behavior score into +# under --with-trust-index. smoke.sh downloads the asset for this platform from +# https://github.com/agentnameservice/agent-trust-discovery/releases/download// +# and refuses a file whose SHA-256 differs. The digests are copied from the +# release's checksums.txt; the commit is recorded for reference only. +TRUST_INDEX_VERSION=v0.1.4 +TRUST_INDEX_COMMIT=27a30a1c7e692ec45ae36563125836460195a375 +TRUST_INDEX_SHA256_linux_amd64=d05829e4ba5f9fd826820b413d44ab9af9b06949f0b95b1a5385dbed0d2ca4c3 +TRUST_INDEX_SHA256_linux_arm64=75a68633400f71a5694597a32cef74a75ecaabd736567ad32609185579dfe69a +TRUST_INDEX_SHA256_darwin_arm64=a120d329a56e0bf01be2e9be797153d517c5532fb294ae08b9fb3ba1471b6f05 diff --git a/examples/openshell-smoke/trust-index.sh b/examples/openshell-smoke/trust-index.sh new file mode 100644 index 0000000..2986ef2 --- /dev/null +++ b/examples/openshell-smoke/trust-index.sh @@ -0,0 +1,304 @@ +# shellcheck shell=bash +# Sourced by smoke.sh under --with-trust-index. Adds the behavior path to the +# run: the gateway's OCSF JSON export is switched on, the approved sandbox's +# supervisor records are collected, the hydrator scores them and imports the +# score into a pinned agent-trust-discovery that this file starts, and the +# score is read back from the agent's trust evaluation. The functions below +# are called from smoke.sh at the points marked there; nothing here runs on +# its own. +# +# Checks added, continuing smoke.sh's numbering: +# 6. OCSF JSON export is live for the sandbox before the first request; +# 7. a request the policy does not allow is denied by the supervisor and +# never reaches the callee; +# 8. the collected records all carry the gateway-issued sandbox id and hold +# exactly the policy decisions the run made; +# 9. the hydrator computes the expected score from them and imports it; +# 10. the Trust Index lists the score on the agent at weight 0. +# +# Requires from smoke.sh: ROOT, EXAMPLE_DIR, WORK, BIN, LOG_DIR, CLI, +# DOCKER_SANDBOX_LABEL, SANDBOX_NAME, SANDBOX_ID, AGENT_ID, ANS_NAME, +# IDENTITY, TRUST_INDEX_PORT, the request journal $LOG_DIR/requests.log, +# and the functions fail, check, step, run_bounded, wait_for_http, +# sandbox_curl, response_status, response_body. +# Provides to smoke.sh: TRUST_INDEX_PID, TRUST_INDEX_VERSION, TRUST_INDEX_ASSET, +# TRUST_INDEX_SHA, and the trust_index_* functions it calls. + +# shellcheck source=examples/openshell-smoke/trust-index-release.env +source "$EXAMPLE_DIR/trust-index-release.env" + +TRUST_INDEX_PID="" +TRUST_INDEX_PORT="" +TRUST_INDEX_DIR="" +TRUST_INDEX_URL="" +SUPERVISOR_CONTAINER="" +EXPECTED_SCORE="" + +# The checksum variables are assigned by trust-index-release.env; smoke.sh +# passes the asset name and digest chosen here to fetch_release_asset. +# shellcheck disable=SC2154,SC2034 +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) TRUST_INDEX_ASSET="agent-trust-discovery_${TRUST_INDEX_VERSION#v}_linux_amd64"; TRUST_INDEX_SHA="$TRUST_INDEX_SHA256_linux_amd64" ;; + Linux-aarch64 | Linux-arm64) TRUST_INDEX_ASSET="agent-trust-discovery_${TRUST_INDEX_VERSION#v}_linux_arm64"; TRUST_INDEX_SHA="$TRUST_INDEX_SHA256_linux_arm64" ;; + Darwin-arm64) TRUST_INDEX_ASSET="agent-trust-discovery_${TRUST_INDEX_VERSION#v}_darwin_arm64"; TRUST_INDEX_SHA="$TRUST_INDEX_SHA256_darwin_arm64" ;; + *) echo "FAIL unsupported platform $(uname -s)-$(uname -m)" >&2; exit 1 ;; +esac + +# trust_index_prepare: called once the run directory and ports exist. Mints the +# admin key the import route requires and writes the Trust Index's own +# configuration. The key reaches the server through its environment and the +# hydrator and curl through files, so no command line and no log carries it. +trust_index_prepare() { + TRUST_INDEX_DIR="$WORK/trust-index" + TRUST_INDEX_URL="http://127.0.0.1:$TRUST_INDEX_PORT" + mkdir -p "$TRUST_INDEX_DIR/profiles" + ( + umask 077 + od -An -tx1 -N32 /dev/urandom | tr -d ' \n' >"$TRUST_INDEX_DIR/admin-key" + printf 'Authorization: Bearer %s\n' "$(<"$TRUST_INDEX_DIR/admin-key")" >"$TRUST_INDEX_DIR/auth-header" + ) + # The signal registration under test is the file this repository ships for + # operators; the profile names only that signal, so a Trust Index release + # that renames a built-in signal cannot stop the server from starting. + cat >"$TRUST_INDEX_DIR/runtime.yaml" <"$TRUST_INDEX_DIR/default-profile.yaml" <"$WORK/hydrator.yaml" </dev/null || printf '0' +} + +trust_index_print_config() { + cat "$TRUST_INDEX_DIR/runtime.yaml" "$TRUST_INDEX_DIR/default-profile.yaml" "$WORK/hydrator.yaml" +} + +# trust_index_retain_configs: copies the rendered configuration next to the +# logs so a failed run's artifact explains what the processes were given. +# Neither file holds a secret; the admin key travels by environment and by +# files that are removed with the other key material. +trust_index_retain_configs() { + cp "$TRUST_INDEX_DIR/runtime.yaml" "$LOG_DIR/trust-index-runtime.yaml" + cp "$TRUST_INDEX_DIR/default-profile.yaml" "$LOG_DIR/trust-index-default-profile.yaml" + cp "$WORK/hydrator.yaml" "$LOG_DIR/hydrator.yaml" +} + +# trust_index_start: starts the server on the loopback port and registers the +# fixture agent. The server logs JSON to stdout. The registration curl is +# called directly, not through step, which would echo its arguments into +# setup.log; the key travels in a header file instead. +trust_index_start() { + printf 'INFO starting agent-trust-discovery %s on %s\n' "$TRUST_INDEX_VERSION" "$TRUST_INDEX_URL" + ANS_ADMIN_KEY="$(<"$TRUST_INDEX_DIR/admin-key")" "$BIN/agent-trust-discovery" \ + -config "$TRUST_INDEX_DIR/runtime.yaml" >"$LOG_DIR/trust-index.log" 2>&1 & + TRUST_INDEX_PID=$! + wait_for_http "trust index" "$TRUST_INDEX_URL/health" "$TRUST_INDEX_PID" 30 + local now status + now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + status="$(run_bounded 30 curl -sS -o "$LOG_DIR/agent-import.out" -w '%{http_code}' --max-time 20 \ + -H @"$TRUST_INDEX_DIR/auth-header" -H 'content-type: application/json' \ + -X POST "$TRUST_INDEX_URL/v1/internal/agents/import" \ + --data "$(jq -n --arg id "$AGENT_ID" --arg name "$ANS_NAME" --arg display "$IDENTITY" --arg now "$now" \ + '{agents: [{agentId: $id, dnsName: $name, displayName: $display, status: "ACTIVE", firstSeen: $now, lastUpdated: $now}]}')")" \ + || fail "agent import request did not complete" + [[ "$status" == 200 ]] || fail "agent import: status $status, body: $(cat "$LOG_DIR/agent-import.out")" + printf 'INFO agent %s registered with the trust index\n' "$AGENT_ID" +} + +trust_index_enable_export() { + step "enabling OCSF JSON export" run_bounded 60 "${CLI[@]}" settings set --global --key ocsf_json_enabled --value true --yes +} + +# supervisor_container NAME: the one container of the sandbox that runs the +# supervisor, whose stderr carries the OCSF JSON records. +supervisor_container() { + run_bounded 30 docker ps -aq --filter "label=openshell.ai/sandbox-namespace=$DOCKER_SANDBOX_LABEL" \ + --filter "label=openshell.ai/sandbox-name=$1" --filter "label=openshell.ai/isolation-role=supervisor" 2>/dev/null | head -n 1 +} + +# ocsf_records CONTAINER: the OCSF JSON lines of the container's log, framing +# intact, so the hydrator later parses exactly what a log collector would hand +# it: a timestamp, the OCSF-JSON marker, then the record. Everything else the +# supervisor writes to stderr is left out here. A docker logs failure is an +# error, not an empty log. +ocsf_records() { + local lines + lines="$(run_bounded 30 docker logs "$1" 2>&1)" || fail "docker logs $1 failed: $(tail -n 3 <<<"$lines" | tr '\n' ';')" + grep -E '^[^ ]+ OCSF-JSON \{' <<<"$lines" || true +} + +# ocsf_payloads FILE: the JSON records in FILE without their framing. +ocsf_payloads() { + sed -E 's/^[^ ]+ OCSF-JSON //' "$1" +} + +# l7_decisions FILE: the L7 policy decisions among the records in FILE. A +# record may carry no message at all, so the field is defaulted before the +# prefix test. +l7_decisions() { + ocsf_payloads "$1" | jq -r 'select((.message // "") | startswith("L7_REQUEST ")) | .message' +} + +# 6. The supervisor applies ocsf_json_enabled on its first settings poll, so a +# sandbox created after the global set exports from its first request. This +# proves it before any request is made: records the sandbox exec produces +# (SSH activity, no HTTP decision) must appear in the supervisor's log. +trust_index_check_export_live() { + local container count deadline=$((SECONDS + 60)) + container="$(supervisor_container "$SANDBOX_NAME")" + [[ -n "$container" ]] || fail "no supervisor container for $SANDBOX_NAME" + SUPERVISOR_CONTAINER="$container" + while ((SECONDS < deadline)); do + run_bounded 30 "${CLI[@]}" sandbox exec --name "$SANDBOX_NAME" --no-tty -- true >>"$LOG_DIR/setup.log" 2>&1 || true + count="$(ocsf_records "$container" | wc -l | tr -d ' ')" + if [[ "$count" -gt 0 ]]; then + check "OCSF JSON export is live for the sandbox ($count records before the first request)" + return + fi + sleep 2 + done + fail "no OCSF JSON record from the supervisor within 60 s; settings: $(run_bounded 30 "${CLI[@]}" settings get --global --json 2>&1 | tr '\n' ' '); toggles: $(run_bounded 30 docker logs "$container" 2>&1 | grep -c 'OCSF JSONL logging toggled' || true); log lines: $(run_bounded 30 docker logs "$container" 2>&1 | wc -l | tr -d ' ')" +} + +# 7. A request the policy does not allow (the policy allows POST /v1/tasks +# only) is denied by the supervisor itself; no middleware runs and the callee +# is never reached. This is the run's one denied decision. +trust_index_check_policy_denied() { + local out status body lines_before + lines_before="$(wc -l <"$LOG_DIR/callee.log")" + out="$(sandbox_curl "$SANDBOX_NAME" -X DELETE)" || true + status="$(response_status "$out")" + body="$(response_body "$out")" + [[ "$status" == 403 ]] || fail "policy-denied request: status $status, body: $body" + [[ "$(jq -r .error <<<"$body" 2>/dev/null)" == policy_denied ]] || fail "policy-denied request: not a policy denial: $body" + [[ "$(wc -l <"$LOG_DIR/callee.log")" -eq "$lines_before" ]] || fail "the callee saw the policy-denied request" + check "policy-denied request is blocked by the supervisor and the callee is not reached" +} + +# 8. Collect the supervisor's records once every decision the run made is in +# them, and prove they are attributable: each carries the gateway-issued +# sandbox id, and the decisions are exactly the run's requests. +trust_index_check_records() { + local expected allowed + expected="$(authorized_decisions)" + allowed=$((expected - 1)) + local records="$LOG_DIR/supervisor-ocsf.jsonl" count=0 deadline=$((SECONDS + 30)) + while ((SECONDS < deadline)); do + ocsf_records "$SUPERVISOR_CONTAINER" >"$records" + count="$(l7_decisions "$records" | wc -l | tr -d ' ')" + [[ "$count" -ge "$expected" ]] && break + sleep 1 + done + [[ "$count" -eq "$expected" ]] \ + || fail "L7 decisions in the supervisor log: want $expected, got $count: $(l7_decisions "$records" | tr '\n' ';')" + local foreign + foreign="$(ocsf_payloads "$records" \ + | jq -r --arg uid "$SANDBOX_ID" --arg name "$SANDBOX_NAME" \ + '(.device.container // .container) as $c | select(($c.uid // "") != $uid or ($c.name // "") != $name) | .message' \ + | head -n 3)" + [[ -z "$foreign" ]] || fail "records not attributed to $SANDBOX_NAME ($SANDBOX_ID): $foreign" + local denied allowed_seen transformed + denied="$(l7_decisions "$records" | grep -c '^L7_REQUEST deny DELETE ' || true)" + allowed_seen="$(l7_decisions "$records" | grep -c '^L7_REQUEST allow POST ' || true)" + transformed="$(ocsf_payloads "$records" | jq -r 'select((.message // "") | startswith("L7_REQUEST_TRANSFORMED")) | .message' | wc -l | tr -d ' ')" + [[ "$denied" -eq 1 && "$allowed_seen" -eq "$allowed" && "$transformed" -eq 0 ]] \ + || fail "decisions: want $allowed allow POST, 1 deny DELETE, 0 transformed; got $allowed_seen, $denied, $transformed" + check "supervisor OCSF export carries $expected policy decisions, every record attributed to sandbox $SANDBOX_ID" +} + +# 9. The hydrator reads the collected records, scores the agent and imports +# the score. The counts are asserted exactly; the score itself is the +# hydrator's to compute (its formula is pinned by its unit tests and may be +# recalibrated), so it is read from the evidence and carried to check 10, +# which proves the Trust Index shows what was imported. +trust_index_check_hydrator() { + local allowed evidence="$LOG_DIR/activity-evidence.jsonl" + allowed=$(($(authorized_decisions) - 1)) + write_hydrator_config "$SANDBOX_ID" + trust_index_retain_configs + run_bounded 60 "$BIN/ans-ocsf-hydrator" --config "$WORK/hydrator.yaml" --input "$LOG_DIR/supervisor-ocsf.jsonl" \ + --evidence-output "$evidence" --log-format text --log-level "${ANS_SMOKE_TRUST_INDEX_LOG_LEVEL:-info}" 2>"$LOG_DIR/hydrator.log" \ + || fail "hydrator exited non-zero: $(tail -n 5 "$LOG_DIR/hydrator.log" | tr '\n' ';')" + [[ "$(wc -l <"$evidence" | tr -d ' ')" -eq 1 ]] || fail "hydrator evidence: want 1 agent, got $(wc -l <"$evidence" | tr -d ' ') lines" + jq -e --arg agent "$AGENT_ID" --argjson allowed "$allowed" ' + .agentId == $agent + and .measurements.httpInitial == {allowed: $allowed, denied: 1, audited: 0, unknown: 0} + and .measurements.connections.denied == 0 + and (.score | type) == "number" and .score > 0 and .score < 100 + and (has("unavailableReason") | not)' "$evidence" >/dev/null \ + || fail "hydrator evidence: $(jq -c '{agentId, score, unavailableReason, measurements, diagnostics}' "$evidence"); hydrator log: $(tail -n 3 "$LOG_DIR/hydrator.log" | tr '\n' ';')" + EXPECTED_SCORE="$(jq -r .score "$evidence")" + check "hydrator scores $EXPECTED_SCORE from $allowed allowed and 1 denied decision and imports it" +} + +# 10. The Trust Index shows the score on the agent's trust evaluation, listed +# under the behavior dimension at weight 0: visible, contributing nothing to +# trust decisions, the rollout state docs/hydrator-activity.md asks for. +trust_index_check_readback() { + local body + body="$(run_bounded 30 curl -sS --max-time 20 "$TRUST_INDEX_URL/v1/ans/registered-agents/$AGENT_ID")" \ + || fail "trust index detail request did not complete" + jq -e --argjson score "$EXPECTED_SCORE" ' + (.trustEvaluation.dimensions[] | select(.dimension == "behavior") | .signalScores[] + | select(.signalId == "openshell.behavior.activity")) as $s + | $s.rawScore == $score and $s.weight == 0 and ($s.explanation | length) > 0 + and .trustEvaluation.trustVector.behavior == 0' <<<"$body" >/dev/null \ + || fail "trust index detail: $(jq -c '{trustVector: .trustEvaluation.trustVector, behavior: (.trustEvaluation.dimensions[]? | select(.dimension == "behavior"))}' <<<"$body" 2>/dev/null || printf '%s' "$body")" + check "trust index shows openshell.behavior.activity = $EXPECTED_SCORE at weight 0" +} + +# trust_index_summary: the pinned release as the ALL PASS line names it. +trust_index_summary() { + printf 'agent-trust-discovery %s (%s)' "$TRUST_INDEX_VERSION" "${TRUST_INDEX_COMMIT:0:7}" +} + +trust_index_print_ready() { + cat <