From eb0d1bf657eccc37a2b4274cb5795cdea0b90336 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 26 Oct 2026 21:07:09 -0700 Subject: [PATCH 1/2] feat(proxy): add discovery-driven application TLS passthrough Route application TLS directly to discovered supervisor listeners, with bounded caches and connections, expiry and withdrawal, health endpoints, container/release packaging, and a standalone Helm chart. Support one workspace, a finite selected set, or all active workspaces through exclusive CLI and Helm settings. Validate complete snapshot scope before atomic cache replacement and retain credential reload on reconnect. Closes #4954 Refs #4668, #4669 Signed-off-by: Taylor Mutch --- .github/workflows/build-binaries.yml | 8 + .github/workflows/build-images.yml | 9 + .../workflows/package-release-binaries.yml | 6 + .github/workflows/release-dev.yml | 16 +- .github/workflows/release-publish.yml | 18 +- AGENTS.md | 1 + CONTRIBUTING.md | 2 + Cargo.lock | 17 + crates/openshell-sandbox-proxy/Cargo.toml | 30 ++ crates/openshell-sandbox-proxy/README.md | 94 +++++ crates/openshell-sandbox-proxy/src/cache.rs | 205 ++++++++++ .../openshell-sandbox-proxy/src/discovery.rs | 306 ++++++++++++++ crates/openshell-sandbox-proxy/src/main.rs | 376 ++++++++++++++++++ deploy/docker/Dockerfile.sandbox-proxy | 8 + .../helm/openshell-sandbox-proxy/Chart.yaml | 8 + deploy/helm/openshell-sandbox-proxy/README.md | 45 +++ .../templates/proxy.yaml | 145 +++++++ .../tests/proxy_test.yaml | 65 +++ .../helm/openshell-sandbox-proxy/values.yaml | 25 ++ docs/kubernetes/ingress.mdx | 25 ++ e2e/kubernetes/app-listener/README.md | 27 ++ skills/debug-openshell-cluster/SKILL.md | 17 + tasks/docker.toml | 4 + tasks/helm.toml | 3 +- tasks/scripts/docker-build-image.sh | 11 +- tasks/scripts/docker-publish-multiarch.sh | 8 +- tasks/scripts/stage-prebuilt-binaries.sh | 13 +- 27 files changed, 1481 insertions(+), 11 deletions(-) create mode 100644 crates/openshell-sandbox-proxy/Cargo.toml create mode 100644 crates/openshell-sandbox-proxy/README.md create mode 100644 crates/openshell-sandbox-proxy/src/cache.rs create mode 100644 crates/openshell-sandbox-proxy/src/discovery.rs create mode 100644 crates/openshell-sandbox-proxy/src/main.rs create mode 100644 deploy/docker/Dockerfile.sandbox-proxy create mode 100644 deploy/helm/openshell-sandbox-proxy/Chart.yaml create mode 100644 deploy/helm/openshell-sandbox-proxy/README.md create mode 100644 deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml create mode 100644 deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml create mode 100644 deploy/helm/openshell-sandbox-proxy/values.yaml diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 1289968d14..11dcb071f4 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -114,6 +114,14 @@ jobs: binary: openshell-sandbox triple: aarch64-unknown-linux-musl runner: linux-arm64-cpu8 + - package: openshell-sandbox-proxy + binary: openshell-sandbox-proxy + triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + - package: openshell-sandbox-proxy + binary: openshell-sandbox-proxy + triple: aarch64-unknown-linux-gnu + runner: linux-arm64-cpu8 - package: openshell-supervisor binary: openshell-supervisor triple: x86_64-unknown-linux-gnu diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e43b5bf62e..799fb2df98 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -64,3 +64,12 @@ jobs: runs-on: linux-amd64-cpu8 timeout-minutes: 20 steps: *build_steps + + sandbox-proxy: + env: + COMPONENT: sandbox-proxy + BINARY: openshell-sandbox-proxy + TARGET_SUFFIX: unknown-linux-gnu + runs-on: linux-amd64-cpu8 + timeout-minutes: 20 + steps: *build_steps diff --git a/.github/workflows/package-release-binaries.yml b/.github/workflows/package-release-binaries.yml index fb8df4ada2..9124ecb228 100644 --- a/.github/workflows/package-release-binaries.yml +++ b/.github/workflows/package-release-binaries.yml @@ -54,6 +54,12 @@ jobs: - artifact: openshell-supervisor-aarch64-unknown-linux-gnu binary: openshell-supervisor package: supervisor-binary-linux-arm64 + - artifact: openshell-sandbox-proxy-x86_64-unknown-linux-gnu + binary: openshell-sandbox-proxy + package: sandbox-proxy-binary-linux-amd64 + - artifact: openshell-sandbox-proxy-aarch64-unknown-linux-gnu + binary: openshell-sandbox-proxy + package: sandbox-proxy-binary-linux-arm64 - artifact: openshell-driver-vm-x86_64-unknown-linux-gnu binary: openshell-driver-vm package: driver-vm-linux-amd64 diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 71db45ffb1..560fa68298 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -177,7 +177,7 @@ jobs: run: | set -euo pipefail REGISTRY="ghcr.io/nvidia/openshell" - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do echo "Tagging ${REGISTRY}/${component}:${{ github.sha }} as dev..." docker buildx imagetools create \ --prefer-index=false \ @@ -310,6 +310,13 @@ jobs: path: release/ merge-multiple: true + - name: Download sandbox proxy binary artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sandbox-proxy-binary-* + path: release/ + merge-multiple: true + - name: Download sandbox binary artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -427,6 +434,10 @@ jobs: openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz \ openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz > openshell-supervisor-checksums-sha256.txt cat openshell-supervisor-checksums-sha256.txt + sha256sum -- \ + openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz \ + openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-proxy-checksums-sha256.txt + cat openshell-sandbox-proxy-checksums-sha256.txt - name: Generate Homebrew formula run: | @@ -533,7 +544,9 @@ jobs: release/openshell-sandbox-x86_64-unknown-linux-musl.tar.gz release/openshell-sandbox-aarch64-unknown-linux-musl.tar.gz release/openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz release/openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-apple-darwin.tar.gz @@ -546,6 +559,7 @@ jobs: release/openshell-gateway-checksums-sha256.txt release/openshell-sandbox-checksums-sha256.txt release/openshell-supervisor-checksums-sha256.txt + release/openshell-sandbox-proxy-checksums-sha256.txt release/openshell-prover-checksums-sha256.txt release-helm: diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index ed6e63827c..e1a9bc238e 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -89,6 +89,13 @@ jobs: path: release/ merge-multiple: true + - name: Download sandbox proxy binary artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sandbox-proxy-binary-* + path: release/ + merge-multiple: true + - name: Download sandbox binary artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -181,6 +188,10 @@ jobs: openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz \ openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz > openshell-supervisor-checksums-sha256.txt cat openshell-supervisor-checksums-sha256.txt + sha256sum -- \ + openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz \ + openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-proxy-checksums-sha256.txt + cat openshell-sandbox-proxy-checksums-sha256.txt - name: Generate Homebrew formula run: | @@ -245,7 +256,7 @@ jobs: VERSION: ${{ inputs.semver }} run: | set -euo pipefail - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do image="ghcr.io/nvidia/openshell/${component}:${VERSION}" # Never replace a published stable image; a rerun keeps the original. # Only a missing manifest means unpublished; any other lookup error @@ -291,7 +302,7 @@ jobs: run: | set -euo pipefail REGISTRY="ghcr.io/nvidia/openshell" - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do if [[ "${IS_PRERELEASE}" == "true" ]]; then echo "Tagging ${REGISTRY}/${component}:${SOURCE_TAG} as ${VERSION}..." docker buildx imagetools create \ @@ -421,7 +432,9 @@ jobs: release/openshell-sandbox-x86_64-unknown-linux-musl.tar.gz release/openshell-sandbox-aarch64-unknown-linux-musl.tar.gz release/openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz release/openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-apple-darwin.tar.gz @@ -434,6 +447,7 @@ jobs: release/openshell-gateway-checksums-sha256.txt release/openshell-sandbox-checksums-sha256.txt release/openshell-supervisor-checksums-sha256.txt + release/openshell-sandbox-proxy-checksums-sha256.txt release/openshell-prover-checksums-sha256.txt - name: Upload amd64 Debian prerelease package diff --git a/AGENTS.md b/AGENTS.md index 8866ead489..90904ecb38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ Do not rely on this file for a full inventory. The detailed public and contribut | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-credential-issuer/` | Embedded credential issuer | Credential storage, built-in refresh execution/recovery/scheduling, typed secret owners, and existing driver transport | | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | +| `crates/openshell-sandbox-proxy/` | Optional application proxy | Discovery-driven TLS passthrough directly to supervisor app listeners | | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | | `crates/openshell-supervisor-control/` | Shared supervisor control | Gateway-owned assignment controller, launch registrations, recovery coordination, and host observations for a supervisor shared by many sandboxes | | `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e800debdf..36a52bcfdd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -346,6 +346,8 @@ still miss across worktrees. ## Container Image Base Overrides +The optional application proxy image can be built with `mise run docker:build:sandbox-proxy`. Its standalone deployment chart is in `deploy/helm/openshell-sandbox-proxy`. + The gateway and supervisor Dockerfiles accept `GATEWAY_BASE_IMAGE` and `SUPERVISOR_BASE_IMAGE` build arguments. Omitting them keeps the pinned distroless defaults. To build with your own bases, pass the argument through diff --git a/Cargo.lock b/Cargo.lock index d178930a4a..34784003f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5077,6 +5077,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "openshell-sandbox-proxy" +version = "0.0.0" +dependencies = [ + "axum", + "clap", + "openshell-core", + "openshell-crypto", + "openshell-sdk", + "rustls", + "tokio", + "tonic", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" diff --git a/crates/openshell-sandbox-proxy/Cargo.toml b/crates/openshell-sandbox-proxy/Cargo.toml new file mode 100644 index 0000000000..1951a176b0 --- /dev/null +++ b/crates/openshell-sandbox-proxy/Cargo.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-sandbox-proxy" +description = "Optional discovery-driven TLS passthrough proxy for sandbox applications" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core" } +openshell-sdk = { path = "../openshell-sdk" } +axum = { workspace = true } +clap = { workspace = true } +rustls = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } +openshell-crypto = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-sandbox-proxy/README.md b/crates/openshell-sandbox-proxy/README.md new file mode 100644 index 0000000000..8436019183 --- /dev/null +++ b/crates/openshell-sandbox-proxy/README.md @@ -0,0 +1,94 @@ + + +# Sandbox application proxy + +`openshell-sandbox-proxy` is an optional consumer of the gateway service discovery +API. It selects the exact discovered hostname from TLS SNI and forwards TLS bytes +to the owning supervisor's app listener. Application certificates and private +keys remain at supervisors. HTTP, WebSockets, application authentication, and +sandbox policy remain enforced by the existing listener and application. + +```shell +openshell-sandbox-proxy \ + --gateway-endpoint https://gateway.example.com \ + --workspace default \ + --gateway-ca /run/discovery/ca.crt \ + --discovery-token-file /run/discovery/token \ + --listen-address 0.0.0.0:8443 +``` + +Repeat `--workspace` for a finite set (up to 64 distinct names), or use +`--all-workspaces` to include every active workspace, including future ones. +These modes are mutually exclusive. For example: + +```shell +openshell-sandbox-proxy --gateway-endpoint https://gateway.example.com \ + --workspace team-a --workspace team-b \ + --discovery-client-cert /run/discovery/tls.crt \ + --discovery-client-key /run/discovery/tls.key --gateway-ca /run/discovery/ca.crt +``` + +Use a discovery-only `OU=discovery` certificate with an administrator-managed +fingerprint grant, or an existing identity authorized for the selection. Client mTLS can +use `--discovery-client-cert` and `--discovery-client-key`. Credentials and CA +files reload on reconnect. The gateway must authorize every selected workspace; +a finite grant cannot request platform scope. Unauthorized selections fail as a +whole. All-workspaces output is bounded by the gateway's 64-workspace scan and +shared route/byte limits. The proxy does not require operator credentials. Plaintext +credential-free loopback development requires `--allow-insecure-gateway`. + +A snapshot initializes the cache and bounded watches replace it. Reconnects +relist; transient errors retain routes only until their published expiry. +Authorization/configuration/data-integrity failures clear the cache. Monotonic +local deadlines are derived from absolute expiries and capped at fifteen seconds; +operators must synchronize gateway and proxy clocks. Snapshot revisions are +opaque and installation versions identify replacement independently of renewal. + +Only exact, current routes can dial a backend. A route withdrawal, installation +replacement, or expiry closes affected TCP streams; renewal preserves them. +There is no gateway payload fallback, wildcard route fallback, or caller-selected +backend. Limits are 4,096 cached routes, 1,024 concurrent connections, 64 per +hostname, a 64 KiB ClientHello, and five-second hello/connect/setup deadlines. +Each relay uses Tokio's bounded bidirectional copy buffers. ClientHello parsing +is dependency-owned rustls protocol parsing, not a first-party cryptographic +operation. Authenticated discovery transport uses the shared SDK/crypto facade. + +The health listener defaults to port 9091: `/healthz` is process liveness, +`/readyz` reflects discovery availability, and `/metrics` exposes connection +counts. During transient discovery failures readiness fails while existing +unexpired connections can continue. Expiry still closes streams independently +of the discovery connection. Multiple replicas keep independent caches and need +no Kubernetes ownership or leader election. + +Build with `mise run docker:build:sandbox-proxy`. The standalone Helm chart at +`deploy/helm/openshell-sandbox-proxy` mounts only discovery identity/trust Secrets; +it needs no Kubernetes API permissions. Configure NetworkPolicy to allow the +proxy to reach the gateway API and discovered supervisor app endpoints, and +restrict supervisor ingress where edge authentication is required. The proxy +never authenticates application users using its discovery identity. + +For Gateway API TLS passthrough, attach a TLSRoute to a Gateway listener in +`Passthrough` mode and target the proxy Service on port 443. Application TLS +terminates at the supervisor. An HTTPRoute requires a separate edge termination +and backend-TLS contract. A controller can instead consume the same discovery +API and reconcile exact-host routes directly to supervisor endpoints; it does +not need this proxy. + +## Verification + +```shell +cargo test -p openshell-sandbox-proxy +cargo clippy -p openshell-sandbox-proxy --all-targets -- -D warnings +helm lint deploy/helm/openshell-sandbox-proxy \ + --set gatewayEndpoint=https://gateway.example.com --set workspace=default +``` + +Kubernetes discovery resolves dedicated supervisors and acknowledged shared-pool +owners. During HA takeover the proxy withdraws the old installation and connects +to the new owner after it advertises an accepted installation. Existing streams +may close; clients reconnect without restarting the sandbox. Readable and hashed +hostnames use the same discovery path. The proxy consumes returned hostnames +and never reconstructs them or needs the gateway's hashing configuration. diff --git a/crates/openshell-sandbox-proxy/src/cache.rs b/crates/openshell-sandbox-proxy/src/cache.rs new file mode 100644 index 0000000000..78bfc2f406 --- /dev/null +++ b/crates/openshell-sandbox-proxy/src/cache.rs @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use openshell_core::proto::ServiceRouteSnapshot; +use tokio::sync::{Semaphore, watch}; +use tokio::time::Instant; + +const MAX_ROUTES: usize = 4096; +const MAX_METADATA_BYTES: usize = 4 * 1024 * 1024; +const MAX_LEASE: Duration = Duration::from_secs(15); + +#[derive(Clone)] +pub struct Route { + pub endpoint: SocketAddr, + pub version: String, + pub deadline: Instant, + pub connections: Arc, +} + +pub type Routes = BTreeMap; +pub type RouteCache = watch::Sender; + +pub fn valid_hostname(host: &str) -> bool { + !host.is_empty() + && host.len() <= 253 + && host.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }) +} + +pub fn valid_workspace(name: &str) -> bool { + valid_hostname(name) && name.len() <= 19 && !name.contains('.') +} + +#[cfg(test)] +fn replace( + cache: &RouteCache, + snapshot: ServiceRouteSnapshot, + workspace: &str, +) -> Result<(), &'static str> { + replace_selected(cache, snapshot, &[workspace.to_string()], false) +} + +/// A complete view is validated before replacement. Receipt never extends a +/// gateway's absolute expiry; a monotonic deadline prevents local clock rollback. +pub fn replace_selected( + cache: &RouteCache, + snapshot: ServiceRouteSnapshot, + workspaces: &[String], + all_workspaces: bool, +) -> Result<(), &'static str> { + if snapshot.routes.len() > MAX_ROUTES || snapshot.revision.is_empty() { + return Err("invalid snapshot bounds or revision"); + } + let now_ms = i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "invalid system clock")? + .as_millis(), + ) + .map_err(|_| "invalid system clock")?; + let now = Instant::now(); + let old = cache.borrow(); + let mut next = Routes::new(); + let mut metadata_bytes = snapshot.revision.len(); + for route in snapshot.routes { + metadata_bytes = metadata_bytes + .saturating_add(route.hostname.len()) + .saturating_add(route.installation_version.len()) + .saturating_add(route.workspace.len()) + .saturating_add(route.sandbox.len()) + .saturating_add(route.name.len()); + if metadata_bytes > MAX_METADATA_BYTES + || !valid_workspace(&route.workspace) + || (!all_workspaces && !workspaces.contains(&route.workspace)) + || !valid_hostname(&route.hostname) + || route.installation_version.is_empty() + { + return Err("invalid discovered route identity"); + } + let endpoint = route.endpoint.ok_or("missing supervisor endpoint")?; + let host: IpAddr = endpoint.host.parse().map_err(|_| "invalid supervisor IP")?; + let port = u16::try_from(endpoint.port) + .ok() + .filter(|port| *port != 0) + .ok_or("invalid supervisor port")?; + let expiry = route + .expires_at + .as_ref() + .and_then(|value| openshell_core::time::timestamp_to_millis(value).ok()) + .ok_or("invalid installation expiry")?; + let remaining = expiry.saturating_sub(now_ms); + if remaining <= 0 { + continue; + } + let deadline = now + Duration::from_millis(remaining.unsigned_abs()).min(MAX_LEASE); + let connections = old.get(&route.hostname).map_or_else( + || Arc::new(Semaphore::new(64)), + |previous| previous.connections.clone(), + ); + if next + .insert( + route.hostname, + Route { + endpoint: SocketAddr::new(host, port), + version: route.installation_version, + deadline, + connections, + }, + ) + .is_some() + { + return Err("duplicate discovered hostname"); + } + } + drop(old); + cache.send_replace(next); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::{DiscoveredServiceRoute, ServiceRouteEndpoint}; + + fn snapshot() -> ServiceRouteSnapshot { + let now = openshell_core::time::timestamp_from_millis(crate::now_ms() + 10_000).unwrap(); + ServiceRouteSnapshot { + revision: "revision".into(), + routes: vec![DiscoveredServiceRoute { + workspace: "default".into(), + sandbox: "demo".into(), + name: "web".into(), + hostname: "demo.example.com".into(), + installation_version: "owner-1".into(), + endpoint: Some(ServiceRouteEndpoint { + host: "127.0.0.1".into(), + port: 9443, + }), + expires_at: Some(now), + ..Default::default() + }], + } + } + + #[test] + fn renewal_preserves_capacity_and_errors_do_not_publish_partial_views() { + let (cache, _) = watch::channel(Routes::new()); + replace(&cache, snapshot(), "default").unwrap(); + let connections = cache.borrow()["demo.example.com"].connections.clone(); + replace(&cache, snapshot(), "default").unwrap(); + assert!(Arc::ptr_eq( + &connections, + &cache.borrow()["demo.example.com"].connections + )); + let mut duplicate = snapshot(); + duplicate.routes.push(duplicate.routes[0].clone()); + assert!(replace(&cache, duplicate, "default").is_err()); + assert_eq!(cache.borrow().len(), 1); + assert!(replace(&cache, snapshot(), "other").is_err()); + let mut expired = snapshot(); + expired.routes[0].expires_at = + openshell_core::time::timestamp_from_millis(crate::now_ms() - 1).ok(); + replace(&cache, expired, "default").unwrap(); + assert!(cache.borrow().is_empty()); + } + #[test] + fn combined_views_validate_every_workspace_before_atomic_replacement() { + let (cache, _) = watch::channel(Routes::new()); + let mut view = snapshot(); + let mut other = view.routes[0].clone(); + other.workspace = "team-b".into(); + other.hostname = "team-b.example.com".into(); + view.routes.push(other); + let scopes = vec!["default".to_string(), "team-b".to_string()]; + replace_selected(&cache, view.clone(), &scopes, false).unwrap(); + assert_eq!(cache.borrow().len(), 2); + let previous = cache.borrow()["team-b.example.com"].connections.clone(); + replace_selected(&cache, view.clone(), &[], true).unwrap(); + assert!(Arc::ptr_eq( + &previous, + &cache.borrow()["team-b.example.com"].connections + )); + view.routes[1].workspace = "outside".into(); + assert!(replace_selected(&cache, view, &scopes, false).is_err()); + assert_eq!(cache.borrow().len(), 2); + let empty = ServiceRouteSnapshot { + revision: "empty".into(), + routes: vec![], + }; + replace_selected(&cache, empty, &[], true).unwrap(); + assert!(cache.borrow().is_empty()); + } +} diff --git a/crates/openshell-sandbox-proxy/src/discovery.rs b/crates/openshell-sandbox-proxy/src/discovery.rs new file mode 100644 index 0000000000..54709b6274 --- /dev/null +++ b/crates/openshell-sandbox-proxy/src/discovery.rs @@ -0,0 +1,306 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, atomic::Ordering}; +use std::time::Duration; + +use clap::Parser; +use openshell_core::proto::{ + GetServiceRouteSnapshotRequest, WatchServiceRoutesRequest, workspace_selector, +}; +use openshell_sdk::{AuthConfig, ClientConfig, ClientIdentity, OpenShellClient}; +use tonic::{Code, Status}; + +use crate::{Health, cache}; + +#[derive(Clone, Parser)] +#[command( + version, + about = "Route application TLS directly to discovered supervisor listeners" +)] +pub struct Args { + #[arg(long)] + pub gateway_endpoint: String, + /// Repeat for a finite set. Exclusive with --all-workspaces. + #[arg( + long, + required_unless_present = "all_workspaces", + conflicts_with = "all_workspaces" + )] + pub workspace: Vec, + /// Discover all active workspaces, including newly created ones. + #[arg(long, conflicts_with = "workspace")] + pub all_workspaces: bool, + #[arg(long, default_value = "0.0.0.0:8443")] + pub listen_address: std::net::SocketAddr, + #[arg(long, default_value = "0.0.0.0:9091")] + pub health_address: std::net::SocketAddr, + #[arg(long)] + pub gateway_ca: Option, + #[arg(long, requires = "discovery_client_key")] + pub discovery_client_cert: Option, + #[arg(long, requires = "discovery_client_cert")] + pub discovery_client_key: Option, + #[arg(long)] + pub discovery_token_file: Option, + /// Permit a plaintext loopback development gateway without credentials. + #[arg(long)] + pub allow_insecure_gateway: bool, +} + +async fn read_file(path: &Path) -> Result, Status> { + let metadata = tokio::fs::metadata(path) + .await + .map_err(|_| Status::unavailable("credential file unavailable"))?; + if metadata.len() > 1024 * 1024 { + return Err(Status::invalid_argument("credential file too large")); + } + tokio::fs::read(path) + .await + .map_err(|_| Status::unavailable("credential file unavailable")) +} + +impl Args { + pub fn validate(&self) -> Result<(), &'static str> { + let gateway = url::Url::parse(&self.gateway_endpoint).map_err(|_| "invalid gateway URL")?; + if !gateway.username().is_empty() + || gateway.password().is_some() + || gateway.query().is_some() + || gateway.fragment().is_some() + || gateway.path() != "/" + || self.all_workspaces != self.workspace.is_empty() + || self.workspace.len() > 64 + || self + .workspace + .iter() + .any(|name| !cache::valid_workspace(name)) + || self + .workspace + .iter() + .collect::>() + .len() + != self.workspace.len() + { + return Err("invalid gateway origin or workspace"); + } + if gateway.scheme() != "https" { + let loopback = gateway.host_str().is_some_and(|host| { + host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }); + if gateway.scheme() != "http" + || !loopback + || !self.allow_insecure_gateway + || self.discovery_token_file.is_some() + || self.discovery_client_cert.is_some() + || self.discovery_client_key.is_some() + || self.gateway_ca.is_some() + { + return Err( + "discovery requires HTTPS; plaintext is limited to credential-free loopback development", + ); + } + } + if self.discovery_token_file.is_some() + && (self.discovery_client_cert.is_some() || self.discovery_client_key.is_some()) + { + return Err("choose bearer credentials or a discovery client certificate"); + } + if self.discovery_client_cert.is_some() != self.discovery_client_key.is_some() { + return Err("client certificate and key must be configured together"); + } + Ok(()) + } + + fn selector(&self) -> Option { + if self.all_workspaces { + Some(openshell_core::proto::all_workspaces_selector()) + } else if self.workspace.len() == 1 { + Some(workspace_selector(&self.workspace[0])) + } else { + None + } + } + + fn selected_workspaces(&self) -> Vec { + if self.workspace.len() > 1 { + self.workspace.clone() + } else { + Vec::new() + } + } + + /// Files are read on every reconnect so mounted credentials can rotate. + async fn client(&self) -> Result { + let mut config = ClientConfig::new(&self.gateway_endpoint); + if let Some(path) = &self.gateway_ca { + config.ca_cert = Some(read_file(path).await?); + } + if let (Some(cert), Some(key)) = (&self.discovery_client_cert, &self.discovery_client_key) { + config.client_identity = Some(ClientIdentity::new( + read_file(cert).await?, + read_file(key).await?, + )); + } + if let Some(path) = &self.discovery_token_file { + let bytes = read_file(path).await?; + let token = std::str::from_utf8(&bytes) + .map_err(|_| Status::invalid_argument("invalid token encoding"))? + .trim(); + if token.is_empty() { + return Err(Status::invalid_argument("empty discovery token")); + } + config.auth = Some(AuthConfig::oidc(token)); + } + OpenShellClient::connect(config) + .await + .map_err(|_| Status::unavailable("gateway connection failed")) + } +} + +async fn consume(args: &Args, routes: &cache::RouteCache, health: &Health) -> Result<(), Status> { + let client = args.client().await?; + let mut grpc = client.raw_grpc(); + let request = GetServiceRouteSnapshotRequest { + workspace_scope: args.selector(), + workspaces: args.selected_workspaces(), + }; + let initial = tokio::time::timeout( + Duration::from_secs(10), + grpc.get_service_route_snapshot(request), + ) + .await + .map_err(|_| Status::deadline_exceeded("snapshot timeout"))?? + .into_inner() + .snapshot + .ok_or_else(|| Status::data_loss("snapshot missing"))?; + cache::replace_selected(routes, initial, &args.workspace, args.all_workspaces) + .map_err(Status::data_loss)?; + health.discovery_ready.store(true, Ordering::Release); + let mut stream = grpc + .watch_service_routes(WatchServiceRoutesRequest { + workspace_scope: args.selector(), + workspaces: args.selected_workspaces(), + }) + .await? + .into_inner(); + loop { + // The gateway ends bounded watches after one minute. A deadline also + // protects against a stalled intermediary that never closes the stream. + let message = tokio::time::timeout(Duration::from_secs(70), stream.message()) + .await + .map_err(|_| Status::deadline_exceeded("watch stalled"))??; + let Some(message) = message else { + return Ok(()); + }; + cache::replace_selected( + routes, + message + .snapshot + .ok_or_else(|| Status::data_loss("snapshot missing"))?, + &args.workspace, + args.all_workspaces, + ) + .map_err(Status::data_loss)?; + health.discovery_ready.store(true, Ordering::Release); + } +} + +pub async fn run(args: Args, routes: cache::RouteCache, health: Arc) { + let mut backoff = Duration::from_secs(1); + loop { + let result = consume(&args, &routes, &health).await; + health.discovery_ready.store(false, Ordering::Release); + match result { + Ok(()) => { + backoff = Duration::from_secs(1); + tokio::time::sleep(backoff).await; + } + Err(error) => { + if matches!( + error.code(), + Code::PermissionDenied + | Code::Unauthenticated + | Code::InvalidArgument + | Code::FailedPrecondition + | Code::DataLoss + ) { + routes.send_replace(cache::Routes::new()); + } + tracing::warn!(code = ?error.code(), "discovery reconnect required"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(Duration::from_secs(30)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn credentials_require_tls_and_workspace_selection_is_explicit() { + let args = |endpoint: &str| { + Args::try_parse_from([ + "proxy", + "--gateway-endpoint", + endpoint, + "--workspace", + "default", + "--allow-insecure-gateway", + ]) + .unwrap() + }; + assert!(args("https://gateway.example.com").validate().is_ok()); + assert!(args("http://localhost:8080").validate().is_ok()); + assert!(args("http://gateway.example.com").validate().is_err()); + let mut plaintext = args("http://localhost:8080"); + plaintext.discovery_token_file = Some("/run/token".into()); + assert!(plaintext.validate().is_err()); + assert!( + args("https://user:secret@gateway.example.com") + .validate() + .is_err() + ); + assert!( + Args::try_parse_from(["proxy", "--gateway-endpoint", "https://gateway.example.com"]) + .is_err() + ); + } + #[test] + fn workspace_modes_are_exclusive_bounded_and_preserve_single_workspace_wire_calls() { + let parse = |flags: &[&str]| { + let mut input = vec!["proxy", "--gateway-endpoint", "https://gateway.example.com"]; + input.extend_from_slice(flags); + Args::try_parse_from(input) + }; + let single = parse(&["--workspace", "team-a"]).unwrap(); + assert!(single.validate().is_ok()); + assert_eq!( + single.selector().unwrap().selection, + Some(workspace_selector::Selection::Workspace("team-a".into())) + ); + let set = parse(&["--workspace", "team-a", "--workspace", "team-b"]).unwrap(); + assert!(set.validate().is_ok()); + assert!(set.selector().is_none()); + assert_eq!(set.selected_workspaces(), vec!["team-a", "team-b"]); + let all = parse(&["--all-workspaces"]).unwrap(); + assert!(all.validate().is_ok()); + assert!(all.workspace.is_empty()); + assert!(all.selector().is_some()); + assert!(parse(&["--workspace", "team-a", "--all-workspaces"]).is_err()); + assert!( + parse(&["--workspace", "team-a", "--workspace", "team-a"]) + .unwrap() + .validate() + .is_err() + ); + let mut too_many = set; + too_many.workspace = (0..65).map(|i| format!("team-{i}")).collect(); + assert!(too_many.validate().is_err()); + } +} diff --git a/crates/openshell-sandbox-proxy/src/main.rs b/crates/openshell-sandbox-proxy/src/main.rs new file mode 100644 index 0000000000..1e22e07c76 --- /dev/null +++ b/crates/openshell-sandbox-proxy/src/main.rs @@ -0,0 +1,376 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! TLS passthrough: discovery credentials never enter application streams. + +mod cache; +mod discovery; + +use std::io::{self, Cursor}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, +}; +use std::time::Duration; + +use axum::{Router, extract::State, http::StatusCode, routing::get}; +use clap::Parser as _; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::{Semaphore, watch}; +use tokio::task::JoinSet; +use tokio::time::Instant; + +const SETUP_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CLIENT_HELLO: usize = 64 * 1024; + +#[derive(Default)] +pub struct Health { + discovery_ready: AtomicBool, + accepted: AtomicU64, + rejected: AtomicU64, + active: AtomicU64, +} + +#[cfg(test)] +fn now_ms() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_millis(), + ) + .expect("bounded timestamp") +} + +fn denied(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::PermissionDenied, message) +} + +/// rustls owns `ClientHello` parsing. No cryptographic handshake or TLS termination +/// happens here; every received byte is forwarded unchanged after SNI selection. +async fn client_hello(stream: &mut TcpStream) -> io::Result<(String, Vec)> { + let mut acceptor = rustls::server::Acceptor::default(); + let mut hello = Vec::new(); + loop { + let mut bytes = [0; 4096]; + let count = stream.read(&mut bytes).await?; + if count == 0 { + return Err(denied("incomplete ClientHello")); + } + if hello.len() + count > MAX_CLIENT_HELLO { + return Err(denied("ClientHello too large")); + } + hello.extend_from_slice(&bytes[..count]); + acceptor.read_tls(&mut Cursor::new(&bytes[..count]))?; + if let Some(accepted) = acceptor + .accept() + .map_err(|_| denied("invalid ClientHello"))? + { + let hostname = accepted + .client_hello() + .server_name() + .filter(|name| cache::valid_hostname(name)) + .ok_or_else(|| denied("service SNI required"))? + .to_string(); + return Ok((hostname, hello)); + } + } +} + +async fn connection( + mut incoming: TcpStream, + mut updates: watch::Receiver, +) -> io::Result<()> { + openshell_core::net::set_tcp_nodelay_best_effort(&incoming); + let (host, hello) = tokio::time::timeout(SETUP_TIMEOUT, client_hello(&mut incoming)) + .await + .map_err(|_| denied("ClientHello timeout"))??; + let route = updates + .borrow() + .get(&host) + .filter(|route| route.deadline > Instant::now()) + .cloned() + .ok_or_else(|| denied("unknown or expired service"))?; + let _permit = route + .connections + .clone() + .try_acquire_owned() + .map_err(|_| denied("service connection limit"))?; + let mut outgoing = tokio::time::timeout( + SETUP_TIMEOUT, + openshell_core::net::connect_tcp_nodelay_best_effort(&[route.endpoint]), + ) + .await + .map_err(|_| denied("backend connect timeout"))??; + let still_current = updates.borrow().get(&host).is_some_and(|current| { + current.version == route.version + && current.endpoint == route.endpoint + && current.deadline > Instant::now() + }); + if !still_current || route.deadline <= Instant::now() { + return Err(denied("installation changed during setup")); + } + tokio::time::timeout_at( + route.deadline.min(Instant::now() + SETUP_TIMEOUT), + outgoing.write_all(&hello), + ) + .await + .map_err(|_| denied("backend setup timeout"))??; + let transfer = tokio::io::copy_bidirectional(&mut incoming, &mut outgoing); + tokio::pin!(transfer); + let mut deadline = route.deadline; + loop { + // Even a queued renewal cannot resurrect a connection after its + // previously observed lease has lapsed. + if Instant::now() >= deadline { + return Ok(()); + } + let current = updates.borrow().get(&host).cloned(); + let Some(current) = current.filter(|current| { + current.version == route.version + && current.endpoint == route.endpoint + && current.deadline > Instant::now() + }) else { + return Ok(()); + }; + deadline = current.deadline; + tokio::select! { + result = &mut transfer => return result.map(|_| ()), + () = tokio::time::sleep_until(deadline) => return Ok(()), + result = updates.changed() => { if result.is_err() { return Ok(()); } } + } + } +} + +async fn serve( + listener: TcpListener, + routes: cache::RouteCache, + health: Arc, +) -> io::Result<()> { + let capacity = Arc::new(Semaphore::new(1024)); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let (stream, _) = accepted?; + let Ok(permit) = capacity.clone().try_acquire_owned() else { + health.rejected.fetch_add(1, Ordering::Relaxed); + continue; + }; + let updates = routes.subscribe(); + let health = health.clone(); + connections.spawn(async move { + let _permit = permit; + health.accepted.fetch_add(1, Ordering::Relaxed); + health.active.fetch_add(1, Ordering::Relaxed); + let result = connection(stream, updates).await; + health.active.fetch_sub(1, Ordering::Relaxed); + if result.is_err() { health.rejected.fetch_add(1, Ordering::Relaxed); } + }); + } + _ = connections.join_next(), if !connections.is_empty() => {} + } + } +} + +async fn readiness(State(health): State>) -> StatusCode { + if health.discovery_ready.load(Ordering::Acquire) { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + +async fn metrics(State(health): State>) -> String { + format!( + "# TYPE openshell_proxy_connections_total counter\nopenshell_proxy_connections_total {}\n# TYPE openshell_proxy_rejected_total counter\nopenshell_proxy_rejected_total {}\n# TYPE openshell_proxy_active gauge\nopenshell_proxy_active {}\n", + health.accepted.load(Ordering::Relaxed), + health.rejected.load(Ordering::Relaxed), + health.active.load(Ordering::Relaxed) + ) +} + +async fn shutdown() -> io::Result<()> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { result = tokio::signal::ctrl_c() => result, _ = terminate.recv() => Ok(()) } + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + let args = discovery::Args::parse(); + args.validate()?; + let listener = TcpListener::bind(args.listen_address).await?; + let health_listener = TcpListener::bind(args.health_address).await?; + let health = Arc::new(Health::default()); + let (routes, _) = watch::channel(cache::Routes::new()); + let discovery = tokio::spawn(discovery::run(args, routes.clone(), health.clone())); + let diagnostics = Router::new() + .route("/healthz", get(|| async { StatusCode::OK })) + .route("/readyz", get(readiness)) + .route("/metrics", get(metrics)) + .with_state(health.clone()); + tokio::select! { + result = serve(listener, routes, health) => result?, + result = axum::serve(health_listener, diagnostics) => result?, + result = shutdown() => result?, + } + discovery.abort(); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hello(host: &str) -> Vec { + let config = openshell_crypto::tls::client_builder() + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(); + let mut client = + rustls::ClientConnection::new(Arc::new(config), host.to_owned().try_into().unwrap()) + .unwrap(); + let mut bytes = Vec::new(); + client.write_tls(&mut bytes).unwrap(); + bytes + } + + #[tokio::test] + async fn passthrough_preserves_bytes_and_revokes_replaced_installations() { + let backend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let frontend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = frontend.local_addr().unwrap(); + let (routes, _) = watch::channel(cache::Routes::from([( + "demo.example.com".into(), + cache::Route { + endpoint: backend.local_addr().unwrap(), + version: "owner-1".into(), + deadline: Instant::now() + Duration::from_secs(10), + connections: Arc::new(Semaphore::new(64)), + }, + )])); + let updates = routes.subscribe(); + let server = tokio::spawn(async move { + let (stream, _) = frontend.accept().await.unwrap(); + connection(stream, updates).await.unwrap(); + }); + let mut client = TcpStream::connect(address).await.unwrap(); + let bytes = hello("demo.example.com"); + client.write_all(&bytes).await.unwrap(); + let (mut peer, _) = backend.accept().await.unwrap(); + let mut received = vec![0; bytes.len()]; + peer.read_exact(&mut received).await.unwrap(); + assert_eq!(received, bytes); + peer.write_all(b"tls-stream").await.unwrap(); + let mut response = [0; 10]; + client.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"tls-stream"); + routes.send_modify(|view| { + view.get_mut("demo.example.com").unwrap().deadline = + Instant::now() + Duration::from_secs(14); + }); + peer.write_all(b"renewed").await.unwrap(); + let mut renewed = [0; 7]; + client.read_exact(&mut renewed).await.unwrap(); + assert_eq!(&renewed, b"renewed"); + routes.send_modify(|view| { + view.get_mut("demo.example.com").unwrap().version = "owner-2".into(); + }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), client.read(&mut response)) + .await + .unwrap() + .unwrap(), + 0 + ); + server.await.unwrap(); + } + + #[tokio::test] + async fn unknown_sni_never_dials() { + let frontend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = frontend.local_addr().unwrap(); + let (routes, _) = watch::channel(cache::Routes::new()); + let server = tokio::spawn(async move { + let (stream, _) = frontend.accept().await.unwrap(); + connection(stream, routes.subscribe()).await.unwrap_err() + }); + let mut client = TcpStream::connect(address).await.unwrap(); + client + .write_all(&hello("unknown.example.com")) + .await + .unwrap(); + assert_eq!( + server.await.unwrap().kind(), + io::ErrorKind::PermissionDenied + ); + } + #[tokio::test] + async fn incomplete_hello_has_a_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let _client = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let (routes, _) = watch::channel(cache::Routes::new()); + tokio::time::pause(); + let result = tokio::time::timeout( + SETUP_TIMEOUT + Duration::from_secs(1), + connection(stream, routes.subscribe()), + ) + .await + .unwrap(); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::PermissionDenied); + } + #[tokio::test] + async fn cached_lease_expiry_closes_a_live_backend_stream() { + let backend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let frontend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = frontend.local_addr().unwrap(); + let (routes, _) = watch::channel(cache::Routes::from([( + "demo.example.com".into(), + cache::Route { + endpoint: backend.local_addr().unwrap(), + version: "owner-1".into(), + deadline: Instant::now() + Duration::from_secs(10), + connections: Arc::new(Semaphore::new(64)), + }, + )])); + let updates = routes.subscribe(); + let server = tokio::spawn(async move { + let (stream, _) = frontend.accept().await.unwrap(); + connection(stream, updates).await.unwrap(); + }); + let mut client = TcpStream::connect(address).await.unwrap(); + let bytes = hello("demo.example.com"); + client.write_all(&bytes).await.unwrap(); + let (mut peer, _) = backend.accept().await.unwrap(); + let mut received = vec![0; bytes.len()]; + peer.read_exact(&mut received).await.unwrap(); + routes.send_modify(|view| { + view.get_mut("demo.example.com").unwrap().deadline = + Instant::now() + Duration::from_millis(100); + }); + let mut buffer = [0; 1]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), client.read(&mut buffer)) + .await + .unwrap() + .unwrap(), + 0 + ); + server.await.unwrap(); + drop(peer); + } +} diff --git a/deploy/docker/Dockerfile.sandbox-proxy b/deploy/docker/Dockerfile.sandbox-proxy new file mode 100644 index 0000000000..f2ffde7aba --- /dev/null +++ b/deploy/docker/Dockerfile.sandbox-proxy @@ -0,0 +1,8 @@ +# syntax=docker/dockerfile:1.4 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +FROM gcr.io/distroless/base-nossl-debian13@sha256:af5cb8dd589b8520b8c06bebb9efb73d7e16406cab58e85c51761fff49d370a0 AS sandbox-proxy +ARG TARGETARCH +COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox-proxy /openshell-sandbox-proxy +USER 65532:65532 +ENTRYPOINT ["/openshell-sandbox-proxy"] diff --git a/deploy/helm/openshell-sandbox-proxy/Chart.yaml b/deploy/helm/openshell-sandbox-proxy/Chart.yaml new file mode 100644 index 0000000000..13148e19f4 --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/Chart.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v2 +name: openshell-sandbox-proxy +description: Optional discovery-driven application TLS passthrough proxy +type: application +version: 0.1.0 +appVersion: dev diff --git a/deploy/helm/openshell-sandbox-proxy/README.md b/deploy/helm/openshell-sandbox-proxy/README.md new file mode 100644 index 0000000000..03104efe0a --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/README.md @@ -0,0 +1,45 @@ + + +# Optional sandbox application proxy + +This standalone chart deploys the discovery-driven TLS passthrough proxy. It +requires an enabled supervisor app listener, accepted advertisements, the service +discovery API, and an existing identity authorized for one explicit workspace. + +```shell +helm upgrade --install app-proxy ./deploy/helm/openshell-sandbox-proxy \ + --namespace openshell \ + --set gatewayEndpoint=https://openshell.openshell.svc:8080 \ + --set workspace=default \ + --set gatewayCASecret=discovery-ca \ + --set tokenSecret=discovery-token \ + --set image.tag=YOUR_VERSION +``` + +The CA Secret uses `ca.crt`; the token Secret uses `token`. Set +`clientIdentitySecret` for a Secret containing discovery `tls.crt` and `tls.key`. +These are discovery credentials, not the application TLS private key. Secret +volumes use directory mounts so rotations are visible on reconnect. + +The Service exposes port 443 and forwards to the nonroot proxy's port 8443. +Attach your TLS-passthrough ingress and configure application DNS/public origin +separately. `/readyz` on port 9091 checks discovery availability; `/healthz` checks +process liveness, and `/metrics` reports connection counters. Two replicas are +deployed by default, each with its own cache. No Kubernetes API access is needed. + +Provide NetworkPolicy permitting discovery and supervisor app connections and +restrict direct supervisor ingress if your edge applies application access +controls. Configure discovery-only credentials through the gateway's certificate +grant mapping. Shared-supervisor publication still requires the HA integration. See the +[proxy README](../../../crates/openshell-sandbox-proxy/README.md) for limits and +cache/expiry behavior. + +Select exactly one workspace mode: existing `workspace: team-a`, +`workspaces: [team-a, team-b]`, or `allWorkspaces: true`. Match the mounted +client identity to an administrator-managed discovery grant for that scope. +A platform grant can still be used by a proxy configured for a narrower set. +The gateway rejects partially authorized selections; cache/connection limits +apply to the combined view, not independently to each workspace. diff --git a/deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml b/deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml new file mode 100644 index 0000000000..d3a407c820 --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml @@ -0,0 +1,145 @@ +{{- $modes := 0 -}} +{{- if .Values.workspace }}{{ $modes = add $modes 1 }}{{ end -}} +{{- if .Values.workspaces }}{{ $modes = add $modes 1 }}{{ end -}} +{{- if .Values.allWorkspaces }}{{ $modes = add $modes 1 }}{{ end -}} +{{- if ne (int $modes) 1 }}{{ fail "select exactly one of workspace, workspaces, or allWorkspaces" }}{{ end -}} +{{- if gt (len .Values.workspaces) 64 }}{{ fail "select at most 64 workspaces" }}{{ end -}} +{{- if ne (len .Values.workspaces) (len (uniq .Values.workspaces)) }}{{ fail "duplicate workspaces are not allowed" }}{{ end -}} +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: openshell-sandbox-proxy + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: openshell-sandbox-proxy + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: proxy + image: {{ printf "%s:%s" .Values.image.repository .Values.image.tag | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --gateway-endpoint + - {{ required "gatewayEndpoint is required" .Values.gatewayEndpoint | quote }} + {{- if .Values.allWorkspaces }} + - --all-workspaces + {{- else if .Values.workspace }} + - --workspace + - {{ .Values.workspace | quote }} + {{- else }} + {{- range .Values.workspaces }} + - --workspace + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.gatewayCASecret }} + - --gateway-ca + - /run/discovery/ca/ca.crt + {{- end }} + {{- if .Values.clientIdentitySecret }} + - --discovery-client-cert + - /run/discovery/client/tls.crt + - --discovery-client-key + - /run/discovery/client/tls.key + {{- end }} + {{- if .Values.tokenSecret }} + - --discovery-token-file + - /run/discovery/token/token + {{- end }} + ports: + - name: app + containerPort: 8443 + - name: health + containerPort: 9091 + readinessProbe: + httpGet: + path: /readyz + port: health + livenessProbe: + httpGet: + path: /healthz + port: health + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + {{- if .Values.gatewayCASecret }} + - name: ca + mountPath: /run/discovery/ca + readOnly: true + {{- end }} + {{- if .Values.clientIdentitySecret }} + - name: client + mountPath: /run/discovery/client + readOnly: true + {{- end }} + {{- if .Values.tokenSecret }} + - name: token + mountPath: /run/discovery/token + readOnly: true + {{- end }} + volumes: + {{- if .Values.gatewayCASecret }} + - name: ca + secret: + secretName: {{ .Values.gatewayCASecret }} + items: + - key: ca.crt + path: ca.crt + defaultMode: 0440 + {{- end }} + {{- if .Values.clientIdentitySecret }} + - name: client + secret: + secretName: {{ .Values.clientIdentitySecret }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + defaultMode: 0440 + {{- end }} + {{- if .Values.tokenSecret }} + - name: token + secret: + secretName: {{ .Values.tokenSecret }} + items: + - key: token + path: token + defaultMode: 0440 + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} +spec: + type: {{ .Values.service.type }} + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: openshell-sandbox-proxy + ports: + - name: app + port: {{ .Values.service.port }} + targetPort: app diff --git a/deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml b/deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml new file mode 100644 index 0000000000..95637fa8ac --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: discovery proxy deployment +templates: + - templates/proxy.yaml +set: + gatewayEndpoint: https://gateway.example.com + clientIdentitySecret: discovery-reader +tests: + - it: renders a Kubernetes Deployment for one workspace + set: + workspace: default + asserts: + - equal: + path: apiVersion + value: apps/v1 + documentIndex: 0 + - isKind: + of: Deployment + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].args + content: default + documentIndex: 0 + - it: renders a Deployment for a finite workspace set + set: + workspaces: [team-a, team-b] + asserts: + - equal: + path: apiVersion + value: apps/v1 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].args + content: team-b + documentIndex: 0 + - it: renders a Deployment for all workspaces + set: + allWorkspaces: true + asserts: + - equal: + path: apiVersion + value: apps/v1 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].args + content: --all-workspaces + documentIndex: 0 + - it: rejects ambiguous scope + set: + workspace: default + allWorkspaces: true + asserts: + - failedTemplate: + errorMessage: select exactly one of workspace, workspaces, or allWorkspaces + - it: rejects missing scope + asserts: + - failedTemplate: + errorMessage: select exactly one of workspace, workspaces, or allWorkspaces + - it: rejects duplicate workspaces + set: + workspaces: [team-a, team-a] + asserts: + - failedTemplate: + errorMessage: duplicate workspaces are not allowed diff --git a/deploy/helm/openshell-sandbox-proxy/values.yaml b/deploy/helm/openshell-sandbox-proxy/values.yaml new file mode 100644 index 0000000000..f4834722c8 --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/values.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +replicaCount: 2 +image: + repository: ghcr.io/nvidia/openshell/sandbox-proxy + tag: dev + pullPolicy: IfNotPresent +gatewayEndpoint: "" +# Exactly one of workspace, workspaces, or allWorkspaces must be configured. +workspace: "" +workspaces: [] +allWorkspaces: false +# Secrets are mounted as directories so projected credential rotation is visible. +gatewayCASecret: "" +clientIdentitySecret: "" +tokenSecret: "" +service: + type: ClusterIP + port: 443 +resources: + requests: + cpu: 100m + memory: 64Mi + limits: + memory: 256Mi diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index f2dcabde28..bb2a74f818 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -314,3 +314,28 @@ for the routing-name field, its validation, and the effects of changing it. ## Next Steps Return to [Setup](/kubernetes/setup) to complete the installation. + +## Independent application proxy + +With supervisor application listeners and service discovery enabled, the optional +`openshell-sandbox-proxy` forwards TLS directly to the discovered supervisor. +Application payloads bypass the gateway. Configure the application public origin +with `supervisor.app_listener.public_url`; the gateway API endpoint and internal +supervisor listener port can differ from that origin. + +Deploy the standalone chart in `deploy/helm/openshell-sandbox-proxy` with a gateway +HTTPS endpoint, a workspace selection, and discovery trust/identity Secrets. +Select `workspace`, a `workspaces` list, or `allWorkspaces: true`. Use a +discovery-only certificate whose administrator-managed grant covers the selection; +see [gateway configuration](../how-it-works/gateways/configuration#discovery-reader-grants). +The proxy does not need Kubernetes API +permissions or application private keys; application TLS terminates at supervisors. + +Attach TLSRoute passthrough to the proxy Service's port 443 and configure DNS +for the application domain. HTTPRoute requires an explicit edge-termination and +backend-TLS design. Allow proxy egress to the gateway API and supervisor app +ports, and restrict direct supervisor ingress when edge authentication is required. +Discovery failures retain routes only through their expiry; withdrawals, +replacement, and expiry close affected streams. Multiple proxy replicas maintain +independent caches. A separate ingress controller can consume the same API and +route directly to supervisors without deploying this proxy. diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md index 93dc17c0b6..080a3d13fc 100644 --- a/e2e/kubernetes/app-listener/README.md +++ b/e2e/kubernetes/app-listener/README.md @@ -175,6 +175,33 @@ The local Rust listener tests use a real TLS client and backend RPC connector. They supplement this live cluster check; they do not establish HA pool recovery or real Kubernetes reachability. +## Discovery-driven proxy path + +Build the gateway, CLI, and optional proxy from the same stack. Deploy dedicated +supervisors with the listener enabled and set `public_url` to an HTTPS origin +whose port differs from both the gateway port-forward and listener port. Deploy +`deploy/helm/openshell-sandbox-proxy` with two replicas, the gateway CA, and one +explicit workspace. Provide a workspace-authorized discovery identity when the +gateway requires authentication. Permit the proxy Pods to reach supervisor app +ports through NetworkPolicy; the proxy needs no Kubernetes API access. + +Create a sandbox with an unnamed service and expose a second named service. +Verify `service expose`, API get/list, create-time response maps, and SDK results +retain the public origin. The gateway alternative must remain independently +available. From a permitted client Pod, use the returned hostname as SNI and +HTTP authority while connecting to each proxy Pod and then its Service. Verify +the supervisor certificate using only the public application CA. No application +private key is mounted in the proxy. + +Hold named and unnamed WebSockets through the proxy beyond the thirty-second +listener freshness interval. Delete the named declaration and require that +stream to close while the unnamed stream survives. Verify both proxy replicas +observe the same accepted installation. Repeat reconnect/expiry checks during +discovery interruption: unexpired cached routes may be used, authorization loss +clears them, and expired routes never open or retain a connection. Test malformed +or missing SNI without backend fallback. Inspect `/readyz`, `/healthz`, and +`/metrics` separately from application health. + ## Shared-supervisor HA and discovery Use the shared-supervisor HA overlay with the app-listener values and two gateway diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index fb4a0c8d32..39d6452682 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1784,6 +1784,17 @@ membership. Direct app-listener authentication remains the ingress provider's responsibility. Disabling service auth retains gateway application routing; it does not disable the gateway data path. +For `openshell-sandbox-proxy`, check `/readyz` on port 9091 and discovery access +before inspecting application TLS. `/healthz` is liveness, while `/metrics` +reports accepted, rejected, and active connections. The proxy Service's port 443 +maps to a nonroot TLS-passthrough listener on 8443. It reads one explicit +workspace through the same snapshot/watch API; discovery identity files reload +on reconnect. Never mount application TLS private keys or grant the proxy +Kubernetes API access. Missing SNI, unknown/expired routes, and exhausted setup +capacity are rejected. Discovery authorization loss clears the cache; temporary +errors retain entries only until expiry. Preserve application SNI to the owning +supervisor and inspect NetworkPolicy on both proxy and supervisor paths. + For discovery-only readers, verify the TLS client CA and an `OU=discovery` certificate whose lowercase DER SHA-256 fingerprint appears in the administrator-managed `openshell.gateway.auth.discovery_grants_file` JSON. @@ -1793,3 +1804,9 @@ propagation precedes the gateway's one-second refresh and bounded collection rechecks. Do not replace a discovery identity with an operator certificate. Single, finite-set, and all-workspaces discovery require explicit selectors; workspace users retain membership checks and cannot request platform scope. + +For proxy workspace selection, configure exactly one of Helm `workspace`, +`workspaces`, or `allWorkspaces`. The CLI accepts repeatable `--workspace` flags +or `--all-workspaces`. Check the entire selected set against the certificate's +grant; rejection is never a partially authorized successful snapshot. Limits +apply to the combined view. Discovery credentials and bearer mode are exclusive. diff --git a/tasks/docker.toml b/tasks/docker.toml index ec585c2de9..b4476dbdff 100644 --- a/tasks/docker.toml +++ b/tasks/docker.toml @@ -69,3 +69,7 @@ run = "scripts/docker-cleanup.sh --force" ["docker:cleanup:dry-run"] description = "Preview what docker:cleanup would remove without deleting anything" run = "scripts/docker-cleanup.sh --dry-run" + +["docker:build:sandbox-proxy"] +description = "Build the optional sandbox TLS proxy image" +run = "tasks/scripts/docker-build-image.sh sandbox-proxy" diff --git a/tasks/helm.toml b/tasks/helm.toml index 047cdad602..d978a23e84 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -55,7 +55,7 @@ run = """ run_windows = "echo Skipping helm:lint: Helm validation is not part of the native Windows lane." ["helm:test"] -description = "Run gateway and workspace Helm chart unit tests" +description = "Run gateway, workspace, and sandbox proxy Helm chart unit tests" run = """ set -e if ! helm plugin list | grep -q unittest; then @@ -64,6 +64,7 @@ run = """ helm dependency build deploy/helm/openshell helm unittest deploy/helm/openshell helm unittest deploy/helm/openshell-workspace + helm unittest deploy/helm/openshell-sandbox-proxy deploy/helm/test-split-ownership.sh deploy/helm/test-gateway-config-parser.sh deploy/helm/test-gateway-resource-coherence.sh diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index 06102fc649..040ab5e2ad 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -40,6 +40,9 @@ prebuilt_arches() { required_prebuilt_binaries() { case "$1" in + sandbox-proxy) + echo "openshell-sandbox-proxy" + ;; gateway) echo "openshell-gateway" ;; @@ -95,7 +98,7 @@ ensure_prebuilt_binaries() { fi } -TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} +TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} shift IS_FINAL_IMAGE=0 @@ -103,6 +106,12 @@ IMAGE_NAME="" DOCKER_TARGET="" DOCKERFILE="" case "${TARGET}" in + sandbox-proxy) + IS_FINAL_IMAGE=1 + IMAGE_NAME="openshell/sandbox-proxy" + DOCKER_TARGET="sandbox-proxy" + DOCKERFILE="deploy/docker/Dockerfile.sandbox-proxy" + ;; gateway) IS_FINAL_IMAGE=1 IMAGE_NAME="openshell/gateway" diff --git a/tasks/scripts/docker-publish-multiarch.sh b/tasks/scripts/docker-publish-multiarch.sh index 430bfa2b7d..d101dfb9e4 100755 --- a/tasks/scripts/docker-publish-multiarch.sh +++ b/tasks/scripts/docker-publish-multiarch.sh @@ -54,13 +54,16 @@ _publish_multiarch_docker() { echo "Building multi-arch supervisor image..." tasks/scripts/docker-build-image.sh supervisor + echo "Building multi-arch sandbox proxy image..." + tasks/scripts/docker-build-image.sh sandbox-proxy + TAGS_TO_APPLY=("${EXTRA_TAGS[@]}") if [[ "${TAG_LATEST}" == "true" ]]; then TAGS_TO_APPLY+=("latest") fi if [[ ${#TAGS_TO_APPLY[@]} -gt 0 ]]; then - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do full_image="${REGISTRY}/${component}" for tag in "${TAGS_TO_APPLY[@]}"; do [[ "${tag}" == "${IMAGE_TAG}" ]] && continue @@ -83,7 +86,7 @@ _publish_multiarch_podman() { # Split comma-separated platforms into an array. IFS=',' read -ra PLATFORM_LIST <<< "${PLATFORMS}" - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do local full_image="${REGISTRY}/${component}" local manifest_name="${full_image}:${IMAGE_TAG}" @@ -151,6 +154,7 @@ echo "Done! Multi-arch images pushed to ${REGISTRY}:" echo " ${REGISTRY}/gateway:${IMAGE_TAG}" echo " ${REGISTRY}/sandbox:${IMAGE_TAG}" echo " ${REGISTRY}/supervisor:${IMAGE_TAG}" +echo " ${REGISTRY}/sandbox-proxy:${IMAGE_TAG}" if [[ "${TAG_LATEST}" == "true" ]]; then echo " (all also tagged :latest)" fi diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index 01d2bff12f..2a9284cf6a 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -12,7 +12,7 @@ ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" source "${SCRIPT_DIR}/build-env.sh" usage() { - echo "Usage: stage-prebuilt-binaries.sh " >&2 + echo "Usage: stage-prebuilt-binaries.sh " >&2 } normalize_arch() { @@ -84,8 +84,8 @@ detect_arches() { components_for_target() { case "$1" in - gateway) - echo "gateway" + gateway|sandbox-proxy) + echo "$1" ;; sandbox) echo "sandbox" @@ -94,7 +94,7 @@ components_for_target() { echo "supervisor" ;; all) - echo "gateway sandbox supervisor" + echo "gateway sandbox supervisor sandbox-proxy" ;; *) usage @@ -105,6 +105,11 @@ components_for_target() { resolve_component() { case "$1" in + sandbox-proxy) + crate=openshell-sandbox-proxy + binary=openshell-sandbox-proxy + target_libc=gnu + ;; gateway) crate=openshell-gateway binary=openshell-gateway From 90ed1076b665e4e9e5e4838347a3c3f4e745f799 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 9 Nov 2026 11:01:12 -0800 Subject: [PATCH 2/2] fix(proxy): consume the separate discovery service Signed-off-by: Taylor Mutch --- crates/openshell-sandbox-proxy/README.md | 3 +- .../openshell-sandbox-proxy/src/discovery.rs | 186 +++++++++++++++++- 2 files changed, 187 insertions(+), 2 deletions(-) diff --git a/crates/openshell-sandbox-proxy/README.md b/crates/openshell-sandbox-proxy/README.md index 8436019183..5a4287802c 100644 --- a/crates/openshell-sandbox-proxy/README.md +++ b/crates/openshell-sandbox-proxy/README.md @@ -6,7 +6,8 @@ SPDX-License-Identifier: Apache-2.0 # Sandbox application proxy `openshell-sandbox-proxy` is an optional consumer of the gateway service discovery -API. It selects the exact discovered hostname from TLS SNI and forwards TLS bytes +API through the separate `openshell.v1.ServiceDiscovery` gRPC service. +It selects the exact discovered hostname from TLS SNI and forwards TLS bytes to the owning supervisor's app listener. Application certificates and private keys remain at supervisors. HTTP, WebSockets, application authentication, and sandbox policy remain enforced by the existing listener and application. diff --git a/crates/openshell-sandbox-proxy/src/discovery.rs b/crates/openshell-sandbox-proxy/src/discovery.rs index 54709b6274..6348beb982 100644 --- a/crates/openshell-sandbox-proxy/src/discovery.rs +++ b/crates/openshell-sandbox-proxy/src/discovery.rs @@ -163,7 +163,7 @@ impl Args { async fn consume(args: &Args, routes: &cache::RouteCache, health: &Health) -> Result<(), Status> { let client = args.client().await?; - let mut grpc = client.raw_grpc(); + let mut grpc = client.raw_service_discovery(); let request = GetServiceRouteSnapshotRequest { workspace_scope: args.selector(), workspaces: args.selected_workspaces(), @@ -304,3 +304,187 @@ mod tests { assert!(too_many.validate().is_err()); } } + +#[cfg(test)] +mod service_tests { + use super::*; + use openshell_core::proto::{ + DiscoveredServiceRoute, GetServiceRouteSnapshotResponse, ServiceRouteEndpoint, + ServiceRouteSnapshot, WatchServiceRoutesResponse, WorkspaceSelector, + service_discovery_server::{ServiceDiscovery, ServiceDiscoveryServer}, + }; + use tonic::codegen::tokio_stream::wrappers::ReceiverStream; + use tonic::{Request, Response}; + + type Updates = Arc< + tokio::sync::Mutex< + Option>>, + >, + >; + + struct Discovery { + scope: Option, + workspaces: Vec, + snapshot: ServiceRouteSnapshot, + stream: Updates, + } + + #[tonic::async_trait] + impl ServiceDiscovery for Discovery { + async fn get_service_route_snapshot( + &self, + request: Request, + ) -> Result, Status> { + assert_eq!(request.get_ref().workspace_scope, self.scope); + assert_eq!(request.get_ref().workspaces, self.workspaces); + Ok(Response::new(GetServiceRouteSnapshotResponse { + snapshot: Some(self.snapshot.clone()), + })) + } + type WatchServiceRoutesStream = ReceiverStream>; + async fn watch_service_routes( + &self, + request: Request, + ) -> Result, Status> { + assert_eq!(request.get_ref().workspace_scope, self.scope); + assert_eq!(request.get_ref().workspaces, self.workspaces); + let (tx, rx) = tokio::sync::mpsc::channel(1); + tx.send(Ok(WatchServiceRoutesResponse { + snapshot: Some(self.snapshot.clone()), + })) + .await + .unwrap(); + *self.stream.lock().await = Some(tx); + Ok(Response::new(ReceiverStream::new(rx))) + } + } + + #[tokio::test] + async fn discovery_only_service_drives_proxy_routes_for_single_finite_and_all_workspaces() { + for mode in 0..3 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut flags = vec![ + "openshell-sandbox-proxy".to_string(), + "--gateway-endpoint".into(), + format!("http://{addr}"), + "--allow-insecure-gateway".into(), + ]; + if mode == 2 { + flags.push("--all-workspaces".into()); + } else { + flags.extend(["--workspace".into(), "default".into()]); + if mode == 1 { + flags.extend(["--workspace".into(), "team-b".into()]); + } + } + let args = Args::try_parse_from(flags).unwrap(); + args.validate().unwrap(); + let workspaces = if mode == 0 { + vec!["default"] + } else { + vec!["default", "team-b"] + }; + let snapshot = ServiceRouteSnapshot { + revision: "initial".into(), + routes: workspaces + .iter() + .map(|workspace| DiscoveredServiceRoute { + workspace: (*workspace).into(), + sandbox: "demo".into(), + name: "web".into(), + hostname: format!("{workspace}--demo--web.example.com"), + endpoint: Some(ServiceRouteEndpoint { + host: "127.0.0.1".into(), + port: 9443, + }), + installation_version: "installation-1".into(), + expires_at: Some( + openshell_core::time::timestamp_from_millis(crate::now_ms() + 12_000) + .unwrap(), + ), + ..Default::default() + }) + .collect(), + }; + let stream = Arc::new(tokio::sync::Mutex::new(None)); + let service = ServiceDiscoveryServer::new(Discovery { + scope: args.selector(), + workspaces: args.selected_workspaces(), + snapshot, + stream: stream.clone(), + }); + let (sockets, rx) = tokio::sync::mpsc::channel(4); + let accept = tokio::spawn(async move { + loop { + let (socket, _) = listener.accept().await.unwrap(); + if sockets.send(Ok::<_, std::io::Error>(socket)).await.is_err() { + break; + } + } + }); + let (stop, stopped) = tokio::sync::oneshot::channel(); + // No OpenShell management service is registered: old RPC paths fail. + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown(ReceiverStream::new(rx), async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let (routes, mut changes) = tokio::sync::watch::channel(cache::Routes::new()); + let health = Arc::new(Health::default()); + let task_routes = routes.clone(); + let task_health = health.clone(); + let consumer = + tokio::spawn(async move { consume(&args, &task_routes, &task_health).await }); + tokio::time::timeout(Duration::from_secs(3), async { + while stream.lock().await.is_none() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(routes.borrow().len(), workspaces.len()); + assert!(health.discovery_ready.load(Ordering::Acquire)); + for workspace in workspaces { + assert!( + routes + .borrow() + .contains_key(&format!("{workspace}--demo--web.example.com")) + ); + } + stream + .lock() + .await + .as_ref() + .unwrap() + .send(Ok(WatchServiceRoutesResponse { + snapshot: Some(ServiceRouteSnapshot { + routes: Vec::new(), + revision: "withdrawn".into(), + }), + })) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + while !changes.borrow_and_update().is_empty() { + changes.changed().await.unwrap(); + } + }) + .await + .unwrap(); + drop(stream.lock().await.take()); + tokio::time::timeout(Duration::from_secs(3), consumer) + .await + .unwrap() + .unwrap() + .unwrap(); + stop.send(()).unwrap(); + accept.abort(); + server.await.unwrap(); + } + } +}