test(e2e): port CLI exec, attach, and SSH host identity tests to testkit

.config/nextest.toml

@@ -10,10 +10,6 @@ failure-output = "immediate-final"
 slow-timeout = { period = "30s", terminate-after = 4 }
 
 [profile.e2e-kubernetes]
-# Temporary waiver: port-forward reconnect can attach read-only while the old
-# SSH session retains stdin ownership. Keep recovery coverage on other drivers.
-# Remove this exclusion after resolving https://github.com/NVIDIA/OpenShell/issues/4445.
-default-filter = "not test(/^canonical_main_connect_recovers_its_ssh_transport$/)"
 fail-fast = false
 status-level = "slow"
 final-status-level = "fail"

.github/workflows/e2e-suites.yml

@@ -82,6 +82,7 @@ jobs:
             {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"},
             {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"cli-attach","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"fedora-podman-rootful-userns-default","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},

TESTING.md

@@ -424,11 +424,14 @@ Suites:
 
 - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests.
 - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate:
-  lifecycle, the CLI, persistent workspaces, file transfer, and the policy
-  advisor. Driver E2E runs it with `openshell-conformance` against the lane's
-  gateway and writes a JUnit report under `results/`; set
-  `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does
-  with the static build it makes once per architecture. The
+  lifecycle, the CLI, persistent workspaces, file transfer, the policy
+  advisor, and credential gating. Driver E2E runs it with
+  `openshell-conformance` against the lane's gateway and writes a JUnit report
+  under `results/`; set `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program,
+  as Branch E2E does with the static build it makes once per architecture.
+  The Docker and Podman runners build `openshell/e2e-python:dev` for it; the
+  Kubernetes and VM runners set `OPENSHELL_TEST_PYTHON_IMAGE` to a pullable
+  Python image unless it is already set. The
   installed-artifact conformance lanes run the same crate through
   `openshell-test-runner` (see above).
 - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or
@@ -593,6 +596,11 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on
 the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to
 the node's address.
 
+The `cli-attach` feature suite (`tests/suites/features/cli-attach`) checks how
+`sandbox connect` supervises its `ssh` process: it reconnects after losing the
+SSH transport and forwards a SIGTERM to `ssh`. The tests find those processes
+in `/proc`, so they need a Linux test host. CI runs them on Docker rootful.
+
 Run the VM-backed Rust CLI e2e suite:
 
 ```shell
@@ -656,6 +664,10 @@ explains the isolation backend separation behind the capability-free runtime.
 On a **remote** cluster, drop the `e2e-host-gateway` feature. Those tests rely
 on the sandbox-side `host.openshell.internal` alias reaching the machine running
 the tests, which is unreachable from pods on a remote cluster, so they fail.
+The conformance suite's `credential_gating` tests need the alias to reach that
+machine too, and the feature selection doesn't skip them, so they fail on a
+remote cluster. [#5409](https://github.com/NVIDIA/OpenShell/issues/5409)
+tracks handling that.
 Left enabled, the `host_gateway_alias` suite fails because
 `host.openshell.internal` does not resolve inside the pod, so the gateway
 SSRF-denies the request (`DNS resolution failed` / `ssrf_denied`) — a networking
@@ -788,7 +800,7 @@ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync
 Run a single Docker-only test directly with cargo:
 
 ```shell
-cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_preflight
+cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_corporate_proxy
 ```
 
 The harness (`e2e/rust/src/harness/`) provides:

crates/openshell-cli/tests/doctor_integration.rs

@@ -0,0 +1,142 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! `openshell doctor check` against a fake `docker` on `PATH`, so the result
+//! doesn't depend on the host's Docker.
+
+#![cfg(unix)]
+
+use std::fs;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+struct Doctor {
+    combined: String,
+    code: i32,
+    elapsed: Duration,
+}
+
+/// Run `openshell doctor check` with only `bin` on `PATH`.
+fn doctor_check(bin: &Path) -> Doctor {
+    let home = tempfile::tempdir().expect("create isolated config dir");
+    let start = Instant::now();
+    let output = Command::new(env!("CARGO_BIN_EXE_openshell"))
+        .args(["doctor", "check"])
+        .env("XDG_CONFIG_HOME", home.path())
+        .env("HOME", home.path())
+        .env("PATH", bin)
+        .env("NO_COLOR", "1")
+        .env_remove("DOCKER_HOST")
+        .env_remove("OPENSHELL_GATEWAY")
+        .env_remove("OPENSHELL_GATEWAY_ENDPOINT")
+        .stdin(Stdio::null())
+        .output()
+        .expect("run openshell doctor check");
+    Doctor {
+        combined: format!(
+            "{}{}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        ),
+        code: output.status.code().unwrap_or(-1),
+        elapsed: start.elapsed(),
+    }
+}
+
+/// Write an executable `docker` script to `bin` that records its arguments
+/// in `bin/docker-args` and then runs `body`.
+fn fake_docker(bin: &Path, body: &str) {
+    let docker = bin.join("docker");
+    let args = bin.join("docker-args");
+    fs::write(
+        &docker,
+        format!(
+            "#!/bin/sh\nprintf '%s\\n' \"$*\" > '{}'\n{body}\n",
+            args.display()
+        ),
+    )
+    .expect("write fake docker");
+    fs::set_permissions(&docker, fs::Permissions::from_mode(0o755)).expect("chmod fake docker");
+}
+
+fn docker_args(bin: &Path) -> String {
+    fs::read_to_string(bin.join("docker-args")).unwrap_or_default()
+}
+
+#[test]
+fn doctor_check_passes_when_docker_info_succeeds() {
+    let bin = tempfile::tempdir().expect("create fake bin dir");
+    fake_docker(bin.path(), "echo 27.3.1");
+
+    let doctor = doctor_check(bin.path());
+    assert_eq!(doctor.code, 0, "doctor check failed:\n{}", doctor.combined);
+    assert_eq!(
+        docker_args(bin.path()).trim(),
+        "info --format {{.ServerVersion}}",
+        "doctor check ran docker with unexpected arguments"
+    );
+    for expected in ["Docker", "ok (version 27.3.1)", "All checks passed"] {
+        assert!(
+            doctor.combined.contains(expected),
+            "expected {expected:?} in doctor check output:\n{}",
+            doctor.combined
+        );
+    }
+}
+
+#[test]
+fn doctor_check_fails_fast_when_docker_info_fails() {
+    let bin = tempfile::tempdir().expect("create fake bin dir");
+    fake_docker(
+        bin.path(),
+        "echo 'Cannot connect to the Docker daemon' >&2\nexit 1",
+    );
+
+    let doctor = doctor_check(bin.path());
+    assert_ne!(
+        doctor.code, 0,
+        "doctor check passed although docker info failed:\n{}",
+        doctor.combined
+    );
+    assert!(
+        doctor.elapsed < Duration::from_secs(10),
+        "doctor check took {:?}",
+        doctor.elapsed
+    );
+    for expected in [
+        "Docker",
+        "FAILED",
+        "docker info failed",
+        "Cannot connect to the Docker daemon",
+    ] {
+        assert!(
+            doctor.combined.contains(expected),
+            "expected {expected:?} in doctor check output:\n{}",
+            doctor.combined
+        );
+    }
+    assert!(
+        !doctor.combined.contains("All checks passed"),
+        "doctor check reported success:\n{}",
+        doctor.combined
+    );
+}
+
+#[test]
+fn doctor_check_fails_without_docker_on_path() {
+    let bin = tempfile::tempdir().expect("create empty bin dir");
+
+    let doctor = doctor_check(bin.path());
+    assert_ne!(
+        doctor.code, 0,
+        "doctor check passed without docker:\n{}",
+        doctor.combined
+    );
+    assert!(
+        doctor.combined.contains("failed to execute docker info"),
+        "doctor check did not report that docker could not run:\n{}",
+        doctor.combined
+    );
+}

crates/openshell-gateway/tests/proxy_config_preflight.rs

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! An incoherent driver proxy table is fatal, rather than leaving every dial
+//! direct while looking configured. A bypass list without a proxy URL means
+//! the operator believed proxying was in effect.
+
+#![cfg(unix)]
+
+use std::fs;
+use std::path::PathBuf;
+use std::process::Output;
+use std::time::Duration;
+
+/// Drivers with an upstream proxy table, compiled into this gateway.
+const DRIVERS: &[&str] = &[
+    #[cfg(feature = "compute-driver-docker")]
+    "docker",
+    #[cfg(feature = "compute-driver-podman")]
+    "podman",
+    #[cfg(feature = "compute-driver-vm")]
+    "vm",
+];
+
+struct Fixture {
+    root: tempfile::TempDir,
+    config: PathBuf,
+}
+
+impl Fixture {
+    /// A gateway config that selects `driver` and sets only `no_proxy` in its
+    /// table.
+    fn incoherent(driver: &str) -> Self {
+        let root = tempfile::tempdir().expect("fixture root");
+        let config = root.path().join("gateway.toml");
+        fs::write(
+            &config,
+            format!(
+                "[openshell]
+version = 2
+[openshell.gateway]
+disable_tls = true
+bind_address = \"127.0.0.1:0\"
+compute_driver = {driver:?}
+[openshell.gateway.auth]
+allow_unauthenticated_users = true
+[openshell.drivers.{driver}]
+no_proxy = \"10.0.0.0/8\"
+"
+            ),
+        )
+        .expect("gateway configuration");
+        Self { root, config }
+    }
+
+    async fn run(&self, args: &[&str]) -> Output {
+        let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_openshell-gateway"));
+        command
+            .env_clear()
+            .env("HOME", self.root.path())
+            .env("PATH", "/usr/bin:/bin")
+            .env("NO_COLOR", "1")
+            .env("XDG_CONFIG_HOME", self.root.path().join("config-home"))
+            .env("XDG_STATE_HOME", self.root.path().join("state-home"))
+            .args(args)
+            .kill_on_drop(true);
+        tokio::time::timeout(Duration::from_secs(30), command.output())
+            .await
+            .expect("the gateway must exit instead of starting")
+            .expect("run openshell-gateway")
+    }
+
+    fn config_path(&self) -> &str {
+        self.config.to_str().expect("UTF-8 config path")
+    }
+}
+
+fn combined(output: &Output) -> String {
+    format!(
+        "{}{}",
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr)
+    )
+}
+
+#[test]
+fn proxy_drivers_are_compiled_in() {
+    assert!(
+        !DRIVERS.is_empty(),
+        "build with a compute driver that has a proxy table"
+    );
+}
+
+#[tokio::test]
+async fn preflight_rejects_no_proxy_without_proxy() {
+    for driver in DRIVERS {
+        let fixture = Fixture::incoherent(driver);
+        let original = fs::read(&fixture.config).expect("original config");
+        let output = fixture
+            .run(&["config", "preflight", "--path", fixture.config_path()])
+            .await;
+        let report = combined(&output);
+        assert!(
+            !output.status.success(),
+            "preflight accepted an incoherent [openshell.drivers.{driver}] proxy table:\n{report}"
+        );
+        assert!(
+            report.contains("category=malformed"),
+            "preflight did not classify the {driver} config as malformed:\n{report}"
+        );
+        assert_eq!(
+            fs::read(&fixture.config).expect("config after preflight"),
+            original,
+            "preflight changed the {driver} config"
+        );
+    }
+}
+
+#[tokio::test]
+async fn startup_rejects_no_proxy_without_proxy_and_names_the_keys() {
+    for driver in DRIVERS {
+        let fixture = Fixture::incoherent(driver);
+        let output = fixture.run(&["--config", fixture.config_path()]).await;
+        let report = combined(&output);
+        assert!(
+            !output.status.success(),
+            "the gateway started with an incoherent [openshell.drivers.{driver}] proxy table:\n{report}"
+        );
+        // Matched on the key names, not the wording, so error wrapping can
+        // change.
+        assert!(
+            report.contains("no_proxy") && report.contains("https_proxy"),
+            "the {driver} startup error does not name the offending keys:\n{report}"
+        );
+    }
+}

crates/openshell-server/Cargo.toml

@@ -142,6 +142,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["native
 rcgen = { workspace = true }
 rsa = { version = "0.9", features = ["pem"] }
 tokio-tungstenite = { workspace = true }
+openshell-sdk = { path = "../openshell-sdk" }
 futures-util = "0.3"
 wiremock = "0.6"
 openshell-otel-test-support = { path = "../openshell-otel-test-support" }