diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c691642904..e763deac08 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -32,6 +32,12 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; +/// Directory where the supervisor writes its rotated log files. +pub const LOG_DIR: &str = "OPENSHELL_LOG_DIR"; + +/// Supervisor log directory when [`LOG_DIR`] is not set. +pub const DEFAULT_LOG_DIR: &str = "/var/log"; + /// OTLP collector endpoint for supervisor trace export. pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT"; diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index cb9433d08e..25c73c6fca 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -284,7 +284,7 @@ RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \ ``` The VM guest's serial console is appended to `//console.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions and removes same-owner stale sockets. On clean shutdown, the gateway sends the managed driver `SIGTERM`, waits up to five seconds for it to flush telemetry and exit, then force-kills it if necessary and removes the socket. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. -The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. +The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Its rotated log files, including OCSF JSONL, go to `logs/` beside them. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. ## Network isolation diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index f7946c34c2..0ed2752945 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -214,6 +214,8 @@ const GUEST_BOUNDARY_CONFIG_DIR: &str = "/.openshell/state"; const GUEST_BOUNDARY_CONFIG_ENV: &str = "OPENSHELL_VM_SANDBOX_BOOTSTRAP"; const HOST_AUTH_BUNDLE_FILE: &str = "supervisor-auth.json"; const HOST_RUNTIME_DESCRIPTOR_FILE: &str = "runtime-descriptor.json"; +/// Private directory in the sandbox state directory for supervisor log files. +const HOST_SUPERVISOR_LOG_DIR: &str = "logs"; const HOST_BOUNDARY_GENERATION_FILE: &str = "boundary-generation"; /// Per-sandbox host-only copy of the gateway-normalized destination trust. /// @@ -1422,6 +1424,12 @@ impl VmDriver { .map_err(|error| Status::internal(format!("encode main process spec: {error}")))?; let upstream_proxy_args = upstream_proxy_cli_args(&self.config) .map_err(|error| Status::invalid_argument(format!("render upstream proxy: {error}")))?; + // The host supervisor cannot count on a writable `/var/log`, and + // sandboxes must not share log files there. + let log_dir = state_dir.join(HOST_SUPERVISOR_LOG_DIR); + create_private_dir_all(&log_dir) + .await + .map_err(|error| Status::internal(format!("create supervisor log dir: {error}")))?; let mut command = Command::new(&supervisor_binary); isolate_host_control_environment(&mut command); command @@ -1476,6 +1484,7 @@ impl VmDriver { openshell_core::sandbox_env::LOG_LEVEL, openshell_core::driver_utils::sandbox_log_level(sandbox, &self.config.log_level), ) + .env(openshell_core::sandbox_env::LOG_DIR, &log_dir) .env( openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::telemetry::enabled_env_value(), diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index bdf8898986..364c32d99c 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -76,7 +76,6 @@ const MAX_DENIALS_LIMIT: usize = 100; /// most recent two so a request just past midnight still has yesterday's /// denials. const DENIAL_LOG_FILES_TO_SCAN: usize = 2; -const LOG_DIR: &str = "/var/log"; /// `/v1/denials` reads `openshell.YYYY-MM-DD.log`, not the OCSF `.jsonl` files, /// which are opt-in via `ocsf_json_enabled`. const SHORTHAND_LOG_PREFIX: &str = "openshell."; @@ -100,7 +99,7 @@ pub enum DenialLogSource { impl Default for DenialLogSource { fn default() -> Self { - Self::ShorthandFiles(PathBuf::from(LOG_DIR)) + Self::ShorthandFiles(PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR)) } } @@ -441,7 +440,7 @@ async fn recent_denials_response( // `[]` in both cases and cannot tell the difference. The shorthand log is // always-on (no setting gates it), so the only way `log_available=false` // happens in practice is if the supervisor has not flushed any events to - // disk yet, or `/var/log` is not writable in this image. + // disk yet, or the log directory is not writable. let log_available = matches!( collect_shorthand_log_files(&log_dir, 1), Ok(files) if !files.is_empty() @@ -457,7 +456,7 @@ async fn recent_denials_response( }); if !log_available { payload["note"] = serde_json::json!( - "no shorthand log file is present yet at /var/log/openshell.YYYY-MM-DD.log; the supervisor may not have emitted any events to disk yet" + "no shorthand log file is present yet; the supervisor may not have emitted any events to disk yet" ); } @@ -1813,7 +1812,7 @@ mod tests { payload["note"] .as_str() .unwrap() - .contains("/var/log/openshell.") + .contains("no shorthand log file") ); } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 42626fbe7c..52820be05d 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -866,6 +866,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { &config, openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), openshell_ocsf::OcsfJsonSettings::default(), + PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); Box::pin(crate::run_sandbox_with_backend( &setup, @@ -1074,6 +1075,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { &config, openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), openshell_ocsf::OcsfJsonSettings::default(), + PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); Box::pin(crate::run_sandbox_with_backend( &setup, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index a9dd93b5c2..e6a916b699 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -323,6 +323,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) .unwrap(), openshell_ocsf::OcsfJsonSettings::default(), + std::path::PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); let startup = run_sandbox_with_backend(&setup, config, bindings, None, None); tokio::pin!(startup); diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index b9a5546bf2..adcfe05d41 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -977,19 +977,23 @@ pub struct SandboxInstanceBindings { impl SandboxInstanceBindings { /// Bindings of a process dedicated to the instance `config` launches. Its - /// process-wide log push and shorthand log files serve the instance. + /// process-wide log push and the shorthand log files in `log_dir` serve + /// the instance. #[must_use] pub fn dedicated( config: &SandboxInstanceConfig, gateway_credentials: openshell_core::grpc_client::GatewayCredentials, ocsf_json: openshell_ocsf::OcsfJsonSettings, + log_dir: std::path::PathBuf, ) -> Self { Self { gateway_credentials, event_context: Arc::new(config.event_context()), ocsf_json, log_sink: None, - denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::ShorthandFiles( + log_dir, + ), } } } @@ -4720,6 +4724,7 @@ mod tests { openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) .expect("credentials"), openshell_ocsf::OcsfJsonSettings::default(), + std::path::PathBuf::from(openshell_core::sandbox_env::DEFAULT_LOG_DIR), ); let image = bindings.event_context.container().unwrap().image.unwrap(); assert_eq!(image.uid, "sha256:workload"); diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 4c1bcac7a0..20f7545760 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -140,6 +140,14 @@ struct Args { #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] log_level: String, + /// Directory for rotated supervisor log files. + #[arg( + long, + default_value = openshell_core::sandbox_env::DEFAULT_LOG_DIR, + env = openshell_core::sandbox_env::LOG_DIR + )] + log_dir: PathBuf, + /// OTLP collector endpoint for supervisor trace export. #[arg(long, env = openshell_core::sandbox_env::OTLP_ENDPOINT)] otlp_endpoint: Option, @@ -554,7 +562,7 @@ fn main() -> Result<()> { let (logging_layers, _logging_guards, file_available) = logging::layers( std::io::stderr(), - Path::new("/var/log"), + &args.log_dir, console_filter, ocsf_enabled.clone(), ); @@ -570,7 +578,10 @@ fn main() -> Result<()> { ) .init(); if !file_available { - warn!("Could not open /var/log for log rotation; using stderr-only logging"); + warn!( + log_dir = %args.log_dir.display(), + "Could not open log directory for rotation; using stderr-only logging" + ); } if let Some(error) = otlp_setup_error { warn!(%error, "OTLP exporting could not be started; continuing without it"); @@ -680,6 +691,7 @@ fn main() -> Result<()> { openshell_ocsf::OcsfJsonSettings { enabled: ocsf_enabled, }, + args.log_dir.clone(), ); let result = Box::pin(openshell_supervisor::run_sandbox_with_backend( backend_setup.as_ref(), @@ -1272,6 +1284,18 @@ mod tests { ); } + #[test] + fn log_dir_defaults_to_var_log_and_accepts_an_override() { + let args = Args::try_parse_from(["openshell-supervisor"]).unwrap(); + assert_eq!( + args.log_dir, + Path::new(openshell_core::sandbox_env::DEFAULT_LOG_DIR) + ); + let args = + Args::try_parse_from(["openshell-supervisor", "--log-dir", "/tmp/logs"]).unwrap(); + assert_eq!(args.log_dir, Path::new("/tmp/logs")); + } + #[test] fn isolation_backend_inputs_are_mandatory() { let args = Args::try_parse_from(["openshell-supervisor"]).expect("parse defaults"); diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 243f181653..c127855303 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -91,7 +91,7 @@ from sandbox runtimes)", // settings apply to the events its threads emit. let (logging_layers, _logging_guards, _) = crate::logging::layers( std::io::stderr(), - std::path::Path::new("/var/log"), + &args.log_dir, console_filter, Arc::new(AtomicBool::new(false)), ); diff --git a/docs/observability/accessing-logs.mdx b/docs/observability/accessing-logs.mdx index a7ebc50bc7..deb67df6c5 100644 --- a/docs/observability/accessing-logs.mdx +++ b/docs/observability/accessing-logs.mdx @@ -78,13 +78,13 @@ The gateway merges the log and platform event sources before emitting, so events ## Direct Filesystem Access -The supervisor writes shorthand logs to `/var/log/openshell.YYYY-MM-DD.log` and, when enabled, OCSF JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` in its own filesystem. `openshell sandbox exec` runs in the workload and does not give access to the supervisor's files. +The supervisor writes shorthand logs to `openshell.YYYY-MM-DD.log` and, when enabled, OCSF JSONL to `openshell-ocsf.YYYY-MM-DD.jsonl` in its log directory, `/var/log` unless the driver sets another. `openshell sandbox exec` runs in the workload and does not give access to the supervisor's files. | Compute driver | Supervisor log location | Local storage lifetime | |---|---|---| | Kubernetes | `/var/log` in the separate supervisor pod's `supervisor` container | An `emptyDir` volume, removed with the pod. | | Docker / Podman | `/var/log` in the separate supervisor container | A tmpfs mount, lost when the container stops. | -| VM | `/var/log` on the host running the supervisor, if writable | Host-managed storage. The driver also captures stderr in `supervisor.err.log` in the sandbox's driver state directory. | +| VM | `logs/` in the sandbox's driver state directory on the host | Removed with the sandbox. The driver also captures stderr in `supervisor.err.log` beside it. | | Windows MXC | Gateway-local JSONL output | See [Windows MXC gateways](/observability/ocsf-json-export#windows-mxc-gateways). | Use platform tooling with access to the supervisor resources. For Podman, copy the current supervisor files to a local directory while the container is running: @@ -110,7 +110,7 @@ The default supervisor image has no shell or `tar`, so shell-based `kubectl exec The shorthand and JSONL files each rotate daily in UTC and keep the three most recent files, so retention is a file count rather than a number of days. The Docker supervisor's `/var/log` is a 64 MiB tmpfs, and the Docker driver removes the supervisor container when the sandbox stops or is deleted or when the supervisor exits. A Kubernetes `emptyDir` has no size limit and is removed with the supervisor pod, which the driver also deletes when it stops the sandbox. OpenShell sets no log options for container stdout and stderr, so their retention follows the Docker daemon or kubelet configuration. -Ship files before rotation or resource teardown and configure your collector to handle daily rotation. Local file writers are non-blocking and can also drop events under pressure. If the supervisor cannot open `/var/log`, it falls back to stderr-only logging and does not produce these files, even when `ocsf_json_enabled` is true. Neither the local files nor the gateway stream guarantees complete audit history. +Ship files before rotation or resource teardown and configure your collector to handle daily rotation. Local file writers are non-blocking and can also drop events under pressure. If the supervisor cannot open its log directory, it falls back to stderr-only logging and does not produce these files, even when `ocsf_json_enabled` is true. Neither the local files nor the gateway stream guarantees complete audit history. ## Filtering by Event Type diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index d7f068dca9..a95c1b248f 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -280,7 +280,7 @@ Set `landlock.compatibility` to `hard_requirement` in the policy to make these f ## Log File Location -The supervisor writes logs to `/var/log/` in its own filesystem, separate from the sandbox workload: +The supervisor writes logs to its log directory, `/var/log/` unless the driver sets another, in its own filesystem, separate from the sandbox workload: | File | Format | Rotation | |---|---|---| @@ -292,7 +292,7 @@ Each format rotates daily with an independent three-file limit. Configure supervisor OCSF collectors to match `openshell-ocsf.*.jsonl`. Shorthand filenames and Windows MXC gateway filenames are unchanged. -File output requires a writable `/var/log` and uses non-blocking writers that can drop events under pressure. If file logging cannot start, the supervisor falls back to stderr-only output. Enabling `ocsf_json_enabled` adds full `OCSF-JSON` records to stderr even without a file sink; see [OCSF JSON collection](/observability/ocsf-json-export#output-location). +File output requires a writable log directory and uses non-blocking writers that can drop events under pressure. If file logging cannot start, the supervisor falls back to stderr-only output. Enabling `ocsf_json_enabled` adds full `OCSF-JSON` records to stderr even without a file sink; see [OCSF JSON collection](/observability/ocsf-json-export#output-location). Docker and Podman use tmpfs for this directory; Kubernetes uses an `emptyDir` in the supervisor pod. Collect logs externally before rotation or teardown for durable retention. See [Accessing Logs](/observability/accessing-logs#direct-filesystem-access) for driver-specific access and storage lifetimes. diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 2f4b52c10f..3d664a015d 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -131,7 +131,7 @@ kubectl -n logs -c supervisor --follow JSON stderr output works independently of writable log storage and the diagnostic log filter. Disabling the setting stops new JSON records. The console writer uses a bounded queue of 1,024 complete lines shared by diagnostics, shorthand, and JSON; when the queue is full, new lines are dropped rather than blocking enforcement. Delivery is best-effort, and retention depends on your runtime and external collector. The gateway log API and `openshell logs` continue to return shorthand. -When file logging is available, supervisors also write plain JSONL to `/var/log/openshell-ocsf.YYYY-MM-DD.jsonl` in their own filesystem, separate from the workload. `openshell sandbox exec` cannot read those files. File output requires a writable `/var/log`; a missing file sink does not disable JSON stderr output. +When file logging is available, supervisors also write plain JSONL to `openshell-ocsf.YYYY-MM-DD.jsonl` in their log directory (`/var/log` unless the driver sets another), in their own filesystem, separate from the workload. `openshell sandbox exec` cannot read those files. File output requires a writable log directory; a missing file sink does not disable JSON stderr output. This file contains supervisor-originated events, not the [sandbox-runtime events](/observability/accessing-logs#main-process-output) emitted into workload container logs.