pimlock · Nov 9, 2026 · Nov 9, 2026 · Nov 9, 2026 · Nov 9, 2026
@@ -70,6 +70,10 @@ jobs:
       - name: Build, verify, and install the wheel and sdist
         run: mise run build
 
+  e2e:
+    name: e2e
+    uses: ./.github/workflows/e2e.yml
+
   # The one required status check. Its name matches ai-openshell/sdk-go, so
   # rulesets can require `ci` in every SDK repository regardless of the jobs
   # each one runs.

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# Runs the E2E suite against the published gateway of a core version, started
+# from core's deploy/docker files (scripts/e2e/start-gateway.sh). Sandboxes
+# run the image from tests/e2e/Dockerfile.workload, which has the same Python
+# as the client, so exec_python can unpickle what the client pickled.
+name: e2e
+
+on:
+  workflow_call:
+    inputs:
+      core-version:
+        description: Core version (default the pin, OPENSHELL_CORE_REF in mise.toml)
+        type: string
+        default: ""
+  workflow_dispatch:
+    inputs:
+      core-version:
+        description: Core version, for example v0.1.7 (default the pin, OPENSHELL_CORE_REF in mise.toml)
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    name: e2e
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0
+
+      - name: Build the sandbox image
+        run: mise run e2e:image
+
+      - name: Start the gateway
+        env:
+          CORE_VERSION: ${{ inputs.core-version }}
+        run: |
+          set -euo pipefail
+          version=${CORE_VERSION:-$(mise run -q core:version)}
+          echo "Core version: $version" >>"$GITHUB_STEP_SUMMARY"
+          scripts/e2e/start-gateway.sh "$version" --default-image openshell/e2e-python:dev
+
+      - name: Run the E2E suite
+        env:
+          OPENSHELL_GATEWAY: e2e
+        run: mise run e2e
+
+      - name: Stop the gateway
+        if: always()
+        run: scripts/e2e/stop-gateway.sh
+
+      - name: Upload the gateway log
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: gateway-log
+          path: ${{ runner.temp }}/openshell-e2e/gateway.log
+          if-no-files-found: ignore
@@ -20,6 +20,7 @@ virtual environment. `mise tasks` lists every task. The ones you need most:
 | `mise run format` | Format the code |
 | `mise run test` | Unit tests; `UV_PYTHON=3.11` picks the interpreter |
 | `mise run build` | Build the wheel and sdist, verify them, and install them on Python 3.11 |
+| `mise run e2e` | E2E tests against a running gateway; see the README for setup |
 | `mise run proto:gen` | Regenerate the bindings from core at the pinned ref |
 | `mise run core:version` | Print the pinned core version |
 

@@ -40,7 +40,29 @@ reads them, and `grpcio-tools` generates the bindings:
 - A change of the pin shows the effect on the generated API in its diff.
 - `mise run proto:check` fails when the committed bindings differ from the
   pinned protos, and `mise run conformance:check` does the same for the
-  vendored test fixtures.
+  vendored test fixtures and `tests/e2e/Dockerfile.workload`.
+
+## E2E tests
+
+`tests/e2e` runs against a real gateway; `mise run test` skips it. CI runs it
+against the published gateway of the pinned core version, with the Docker
+compute driver. To run it locally, you need Docker with the compose plugin:
+
+```shell
+mise run e2e:image
+scripts/e2e/start-gateway.sh "$(mise run -q core:version)" --default-image openshell/e2e-python:dev
+OPENSHELL_GATEWAY=e2e mise run e2e
+scripts/e2e/stop-gateway.sh
+```
+
+`e2e:image` builds the sandbox image from `tests/e2e/Dockerfile.workload`, a
+copy of core's, with the Python version from `.python-version` and
+`cloudpickle`, so `Sandbox.exec_python` can run the client's callables.
+
+`start-gateway.sh` is meant for throwaway hosts. It registers `e2e` as the
+active gateway, and `stop-gateway.sh` removes every sandbox container labelled
+with core's default namespace. Set `XDG_CONFIG_HOME` to keep your own gateway
+registrations apart.
 
 ## History
 

@@ -65,16 +65,17 @@ fi
 """
 
 [tasks."conformance:check"]
-description = "Fail if vendored fixtures differ from core at the pinned ref"
+description = "Fail if vendored fixtures or the E2E workload Dockerfile differ from core at the pinned ref"
 run = """
 #!/usr/bin/env bash
 set -euo pipefail
 core=$(mktemp -d)
 trap 'rm -rf "$core"' EXIT
-scripts/fetch-core.sh "$OPENSHELL_CORE_REF" "$core" sdk/conformance
+scripts/fetch-core.sh "$OPENSHELL_CORE_REF" "$core" sdk/conformance e2e/python/Dockerfile.workload
 for fixture in oauth-client-credentials.json oauth-client-credentials.json.license; do
   diff -u "$core/sdk/conformance/$fixture" "tests/fixtures/$fixture"
 done
+diff -u "$core/e2e/python/Dockerfile.workload" tests/e2e/Dockerfile.workload
 """
 
 [tasks.lint]
@@ -100,7 +101,18 @@ run = "uv run --frozen ty check {{vars.python_paths}}"
 [tasks.test]
 description = "Run unit tests (set UV_PYTHON to pick the interpreter)"
 depends = ["sync", "proto:fetch"]
-run = "uv run --frozen pytest -n auto"
+run = "uv run --frozen pytest -n auto --ignore=tests/e2e"
+
+[tasks."e2e:image"]
+description = "Build the E2E sandbox image: Python from .python-version plus cloudpickle, for exec_python"
+run = """
+docker build --file tests/e2e/Dockerfile.workload   --build-arg "PYTHON_VERSION=$(cat .python-version)"   --tag openshell/e2e-python:dev tests/e2e
+"""
+
+[tasks.e2e]
+description = "Run the E2E suite against the gateway named in OPENSHELL_GATEWAY (see scripts/e2e/start-gateway.sh)"
+depends = ["sync"]
+run = "uv run --frozen pytest tests/e2e -n ${E2E_PARALLEL:-5}"
 
 [tasks.check]
 description = "Run formatting, lint, type, and drift checks"

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+
+# start-gateway.sh: run the published OpenShell gateway for a core version
+# with the Docker compute driver, for the E2E lane.
+#
+# Usage: start-gateway.sh <core-version> [--state-dir <dir>] [--default-image <image>]
+#
+# Uses core's own deployment files, deploy/docker/docker-compose.yml and
+# gateway.toml, at the core tag: an init container generates the supervisor
+# JWT keys, and the gateway serves plaintext gRPC on 127.0.0.1:8080 and
+# /readyz on 127.0.0.1:8081. The supervisor and sandbox images are pinned to
+# the same version instead of :latest. --default-image replaces the image that
+# sandboxes run when they don't name one (core's default is plain Ubuntu); the
+# gateway pulls it only when it isn't present locally. Then registers the
+# gateway as "e2e" for the SDK in $XDG_CONFIG_HOME/openshell. Waits up to 15
+# minutes for the gateway image of a fresh tag to be published.
+#
+# Needs docker with the compose plugin and a host path /var/lib/openshell
+# (core's compose file bind-mounts it at the same path). Meant for throwaway
+# CI hosts: it registers the gateway as the active one, and stop-gateway.sh
+# removes every container labelled with core's default sandbox namespace,
+# including sandboxes of another local gateway that uses the same namespace.
+set -euo pipefail
+
+core_repo=${CORE_REPO:-NVIDIA/OpenShell}
+registry=ghcr.io/nvidia/openshell
+name=e2e
+port=8080
+health_port=8081
+
+tag=
+default_image=
+state=${OPENSHELL_E2E_STATE:-${RUNNER_TEMP:-/tmp}/openshell-e2e}
+while [ $# -gt 0 ]; do
+  case $1 in
+  --state-dir) state=${2:?--state-dir needs a path}; shift 2 ;;
+  --default-image) default_image=${2:?--default-image needs an image}; shift 2 ;;
+  -*) echo "usage: $0 <core-version> [--state-dir <dir>] [--default-image <image>]" >&2; exit 2 ;;
+  *) tag=$1; shift ;;
+  esac
+done
+[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || {
+  echo "usage: $0 <core-version>   (vX.Y.Z or vX.Y.Z-pre.N)" >&2
+  exit 2
+}
+semver=${tag#v}
+mkdir -p "$state"
+echo "$tag" >"$state/core-version"
+
+# Core pushes images after tagging; a proposal can be faster than that.
+manifest_exists() {
+  local token
+  token=$(curl -fsS "https://ghcr.io/token?scope=repository:nvidia/openshell/gateway:pull" | jq -r .token) || return 1
+  curl -fsS -o /dev/null -H "Authorization: Bearer $token" \
+    -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json' \
+    "https://ghcr.io/v2/nvidia/openshell/gateway/manifests/$semver"
+}
+for i in $(seq 1 15); do
+  manifest_exists && break
+  if [ "$i" -eq 15 ]; then
+    echo "ERROR: gateway image for $semver not published yet, re-run later" >&2
+    exit 1
+  fi
+  echo "Waiting for $registry/gateway:$semver ($i/15) ..."
+  sleep 60
+done
+
+# Core's deployment files at the tag.
+base="https://raw.githubusercontent.com/$core_repo/$tag/deploy/docker"
+curl -fsSL "$base/docker-compose.yml" -o "$state/docker-compose.yml"
+curl -fsSL "$base/gateway.toml" -o "$state/gateway.toml"
+sed -i.bak \
+  -e "s#\"$registry/supervisor:latest\"#\"$registry/supervisor:$semver\"#" \
+  -e "s#\"$registry/sandbox:latest\"#\"$registry/sandbox:$semver\"#" \
+  "$state/gateway.toml" && rm -f "$state/gateway.toml.bak"
+for img in supervisor sandbox; do
+  grep -q "\"$registry/$img:$semver\"" "$state/gateway.toml" || {
+    echo "ERROR: could not pin the $img image in core's gateway.toml at $tag" >&2
+    exit 1
+  }
+done
+# Core's file binds the health server to loopback inside the container, where
+# the published port can't reach it. Bind it to all container interfaces; the
+# compose file publishes the port on the host's 127.0.0.1 only.
+sed -i.bak 's#^health_bind_address *= *"127\.0\.0\.1:8081"#health_bind_address = "0.0.0.0:8081"#' \
+  "$state/gateway.toml" && rm -f "$state/gateway.toml.bak"
+grep -q '^health_bind_address = "0.0.0.0:8081"' "$state/gateway.toml" || {
+  echo "ERROR: could not rebind the health server in core's gateway.toml at $tag" >&2
+  exit 1
+}
+
+if [ -n "$default_image" ]; then
+  sed -i.bak "s#^default_image *= *\".*\"#default_image     = \"$default_image\"#" \
+    "$state/gateway.toml" && rm -f "$state/gateway.toml.bak"
+  grep -qF "default_image     = \"$default_image\"" "$state/gateway.toml" || {
+    echo "ERROR: could not set default_image in core's gateway.toml at $tag" >&2
+    exit 1
+  }
+fi
+
+compose() { IMAGE_TAG=$semver docker compose -p openshell-e2e -f "$state/docker-compose.yml" "$@"; }
+echo "Starting gateway $semver ..."
+compose up -d --quiet-pull
+
+for i in $(seq 1 90); do
+  if curl -fsS -o /dev/null "http://127.0.0.1:$health_port/readyz"; then
+    break
+  fi
+  if [ "$i" -eq 90 ]; then
+    echo "ERROR: gateway not ready after 180 s" >&2
+    compose logs --no-color gateway | tail -n 80 >&2 || true
+    exit 1
+  fi
+  sleep 2
+done
+echo "Gateway $semver ready on 127.0.0.1:$port."
+
+config_home=${XDG_CONFIG_HOME:-$HOME/.config}
+dir="$config_home/openshell/gateways/$name"
+mkdir -p "$dir"
+cat >"$dir/metadata.json" <<JSON
+{
+  "name": "$name",
+  "gateway_endpoint": "http://127.0.0.1:$port",
+  "is_remote": false,
+  "gateway_port": $port,
+  "auth_mode": "plaintext"
+}
+JSON
+printf '%s' "$name" >"$config_home/openshell/active_gateway"
+echo "Registered gateway '$name' in $config_home/openshell; run the suite with OPENSHELL_GATEWAY=$name."
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+
+# stop-gateway.sh: stop the E2E gateway started by start-gateway.sh, save its
+# log, remove the sandbox containers, and remove the "e2e" gateway
+# registration (and the active-gateway pointer when it still names "e2e").
+#
+# Usage: stop-gateway.sh [--state-dir <dir>]
+#
+# Writes <state-dir>/gateway.log. Safe to run when nothing is running. Runs
+# every cleanup step even when one fails, then exits nonzero if any failed.
+set -uo pipefail
+
+state=${OPENSHELL_E2E_STATE:-${RUNNER_TEMP:-/tmp}/openshell-e2e}
+[ "${1:-}" != --state-dir ] || state=${2:?--state-dir needs a path}
+[ -f "$state/docker-compose.yml" ] || { echo "No E2E gateway state in $state."; exit 0; }
+
+semver=$(sed 's/^v//' "$state/core-version" 2>/dev/null || echo latest)
+compose() { IMAGE_TAG=$semver docker compose -p openshell-e2e -f "$state/docker-compose.yml" "$@"; }
+
+failed=0
+fail() { echo "stop-gateway: $*" >&2; failed=1; }
+
+if compose logs --no-color --timestamps >"$state/gateway.log" 2>&1; then
+  echo "Gateway log: $state/gateway.log"
+else
+  fail "could not save the gateway log; see $state/gateway.log"
+fi
+
+# The Docker driver labels sandbox and supervisor containers with core's
+# openshell.ai/managed-by and openshell.ai/sandbox-namespace labels. Compose
+# doesn't know about them, so remove them before taking the project down.
+if ids=$(docker ps -aq \
+  --filter label=openshell.ai/managed-by=openshell \
+  --filter label=openshell.ai/sandbox-namespace=openshell); then
+  # shellcheck disable=SC2086 # one container ID per word
+  [ -z "$ids" ] || docker rm -f $ids >/dev/null || fail "could not remove sandbox containers"
+else
+  fail "could not list sandbox containers"
+fi
+compose down --volumes --remove-orphans >/dev/null || fail "docker compose down failed"
+
+config_home=${XDG_CONFIG_HOME:-$HOME/.config}
+rm -rf "$config_home/openshell/gateways/e2e" || fail "could not remove the e2e gateway registration"
+if [ "$(cat "$config_home/openshell/active_gateway" 2>/dev/null)" = e2e ]; then
+  rm -f "$config_home/openshell/active_gateway" || fail "could not remove the active-gateway pointer"
+fi
+
+if [ "$failed" -ne 0 ]; then
+  echo "Gateway cleanup failed; the gateway or its sandboxes may still be running." >&2
+  exit 1
+fi
+echo "Gateway stopped."
@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+ARG WORKLOAD_BASE=nvcr.io/nvidia/base/ubuntu:24.04
+FROM ghcr.io/astral-sh/uv:0.12.17@sha256:10787c682e4184e4f290de1171fd4703dc63de99221f10fe1c99002ce7fa9acc AS uv
+FROM ${WORKLOAD_BASE}
+
+COPY --from=uv /uv /uvx /usr/local/bin/
+# Supplied from .python-version by the build task for cloudpickle compatibility.
+ARG PYTHON_VERSION
+# curl and openssl also support the Rust Docker tests sharing this fixture.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        bash ca-certificates curl git openssl python3 python3-pip python3-venv python-is-python3 \
+    && rm -rf /var/lib/apt/lists/* \
+    && if getent passwd ubuntu >/dev/null; then userdel ubuntu; fi \
+    && if getent group ubuntu >/dev/null; then groupdel ubuntu; fi \
+    && groupadd --gid 1000 sandbox \
+    && useradd --uid 1000 --gid sandbox --create-home --shell /bin/bash sandbox \
+    && UV_PYTHON_INSTALL_DIR=/sandbox/.uv/python uv python install "${PYTHON_VERSION}" \
+    && UV_PYTHON_INSTALL_DIR=/sandbox/.uv/python uv venv --python "${PYTHON_VERSION}" --seed /sandbox/.venv \
+    && uv pip install --python /sandbox/.venv/bin/python cloudpickle==3.1.2 google-auth==2.40.3 requests==2.32.5 \
+    && chown -R sandbox:sandbox /sandbox \
+    && uv cache clean
+
+ENV VIRTUAL_ENV=/sandbox/.venv
+ENV PATH="/sandbox/.venv/bin:${PATH}"
+WORKDIR /sandbox
+USER sandbox:sandbox