diff --git a/crates/openshell-cli/src/commands/provider.rs b/crates/openshell-cli/src/commands/provider.rs index efc3d1bb12..6cf2fa18ec 100644 --- a/crates/openshell-cli/src/commands/provider.rs +++ b/crates/openshell-cli/src/commands/provider.rs @@ -890,30 +890,7 @@ async fn discover_existing_provider_data( Ok(discovered) } -/// Canonical provider type string for Google Vertex AI. -const VERTEX_AI_PROVIDER_TYPE: &str = "google-vertex-ai"; - -/// Canonical provider type string for Google Cloud (GCP APIs). -const GOOGLE_CLOUD_PROVIDER_TYPE: &str = "google-cloud"; - fn missing_credentials_error(provider_type: &str) -> miette::Report { - if provider_type == VERTEX_AI_PROVIDER_TYPE { - return miette::miette!( - "no credentials resolved for provider type '{provider_type}'. \ - Set GOOGLE_VERTEX_AI_TOKEN, VERTEX_AI_TOKEN, \ - GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN, or VERTEX_AI_SERVICE_ACCOUNT_TOKEN; \ - or use --from-gcloud-adc or --from-existing with those env vars set." - ); - } - - if provider_type == GOOGLE_CLOUD_PROVIDER_TYPE { - return miette::miette!( - "no credentials resolved for provider type '{provider_type}'. \ - Set GCP_ADC_ACCESS_TOKEN or GCP_SA_ACCESS_TOKEN; \ - or use --from-gcloud-adc / --from-existing with those env vars set." - ); - } - miette::miette!( "no credentials resolved for provider type '{provider_type}'. \ Use --credential KEY[=VALUE], --runtime-credentials for runtime-resolved profile credentials, or --from-existing \ diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index 89247ba718..75a6f50dea 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -4639,6 +4639,7 @@ async fn provider_create_from_existing_uses_profile_discovery() { }], discovery: Some(ProviderProfileDiscovery { credentials: vec!["api_key".to_string()], + config_env_vars: Vec::new(), }), ..Default::default() }, @@ -4826,6 +4827,7 @@ async fn provider_create_from_existing_fails_when_profile_discovery_finds_nothin }], discovery: Some(ProviderProfileDiscovery { credentials: vec!["api_key".to_string()], + config_env_vars: Vec::new(), }), ..Default::default() }, @@ -4875,6 +4877,7 @@ async fn provider_update_from_existing_uses_profile_discovery() { }], discovery: Some(ProviderProfileDiscovery { credentials: vec!["api_key".to_string()], + config_env_vars: Vec::new(), }), ..Default::default() }, @@ -4949,6 +4952,7 @@ async fn provider_update_from_existing_preserves_global_profile_scope() { }], discovery: Some(ProviderProfileDiscovery { credentials: vec!["api_key".to_string()], + config_env_vars: Vec::new(), }), ..Default::default() }, @@ -6010,7 +6014,7 @@ async fn provider_create_from_gcloud_adc_rolls_back_provider_when_initial_rotate } #[tokio::test] -async fn provider_create_from_existing_vertex_config_only_reports_missing_vertex_credentials() { +async fn provider_create_from_existing_vertex_config_only_reports_missing_credentials() { let ts = run_server().await; Environment::from_pairs(&[ @@ -6034,8 +6038,7 @@ async fn provider_create_from_existing_vertex_config_only_reports_missing_vertex let msg = err.to_string(); assert!( - msg.contains("GOOGLE_VERTEX_AI_TOKEN") - && msg.contains("VERTEX_AI_SERVICE_ACCOUNT_TOKEN"), + msg.contains("no credentials resolved") && msg.contains("--credential KEY[=VALUE]"), "unexpected error: {msg}" ); assert!( diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 2779e78fb9..9481ef89ad 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -80,6 +80,9 @@ fn main() -> Result<(), Box> { .build_server(true) .build_client(true) .include_file("openshell.rs") + // Profile snapshots hash their protobuf encoding. Keep environment + // maps ordered across storage round trips so revisions remain stable. + .btree_map(".openshell.v1.ProviderProfileEnvironment") // Emit a binary FileDescriptorSet so the server can enumerate every // RPC at runtime (used by the per-handler auth exhaustiveness test). .file_descriptor_set_path(&descriptor_path) diff --git a/crates/openshell-core/src/google_cloud.rs b/crates/openshell-core/src/google_cloud.rs index efc15a6527..a71f03b7b9 100644 --- a/crates/openshell-core/src/google_cloud.rs +++ b/crates/openshell-core/src/google_cloud.rs @@ -1,13 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Shared GCP constants for the metadata emulator, provider env injection, -//! and credential resolution. +//! Shared destinations and environment keys used by the GCP metadata emulator. //! -//! This module is the single source of truth for GCP naming: env var aliases, -//! provider config keys, token search order, and Vertex-specific env vars. -//! `openshell-server`, `openshell-providers`, and `openshell-sandbox` -//! import from here. +//! Provider environment defaults and discovery keys belong to imported +//! profiles; this module holds only the names required by the supervisor's +//! metadata protocol. // ── Metadata emulator ─────────────────────────────────────────────────────── @@ -23,28 +21,14 @@ pub fn is_metadata_destination(destination: std::net::SocketAddr) -> bool { destination == std::net::SocketAddr::from(([127, 0, 0, 1], 8174)) } -// ── Env var alias arrays ──────────────────────────────────────────────────── +// ── Metadata environment keys ─────────────────────────────────────────────── /// Env vars that carry the GCP project ID inside sandboxes. pub const PROJECT_ID_ENV_VARS: &[&str] = &["GCP_PROJECT_ID", "GOOGLE_CLOUD_PROJECT"]; -/// Env vars that carry the GCP region/location inside sandboxes. -pub const REGION_ENV_VARS: &[&str] = &["CLOUD_ML_REGION", "GCP_LOCATION"]; - /// Env vars that carry the GCP service account email inside sandboxes. pub const SERVICE_ACCOUNT_EMAIL_ENV_VARS: &[&str] = &["GCP_SERVICE_ACCOUNT_EMAIL"]; -// ── Provider config keys ──────────────────────────────────────────────────── - -/// Config key for project ID in `gcp` providers. -pub const GCP_PROJECT_ID_CONFIG_KEY: &str = "project_id"; - -/// Config key for region in `gcp` providers. -pub const GCP_REGION_CONFIG_KEY: &str = "region"; - -/// Config key for service account email in `gcp` providers. -pub const GCP_SERVICE_ACCOUNT_EMAIL_CONFIG_KEY: &str = "service_account_email"; - // ── Token search order ────────────────────────────────────────────────────── /// GCP token env vars searched in priority order by the metadata emulator. @@ -52,43 +36,9 @@ pub const GCP_SERVICE_ACCOUNT_EMAIL_CONFIG_KEY: &str = "service_account_email"; /// credential precedence. pub const TOKEN_ENV_KEYS: &[&str] = &["GCP_SA_ACCESS_TOKEN", "GCP_ADC_ACCESS_TOKEN"]; -// ── Vertex-specific env vars ──────────────────────────────────────────────── - -/// Env var injected to signal Vertex AI usage to Goose. -pub const GOOSE_PROVIDER_ENV_VAR: &str = "GOOSE_PROVIDER"; - -/// Env var for Anthropic Vertex project ID (consumed by Claude Code SDK). -pub const ANTHROPIC_VERTEX_PROJECT_ID_ENV_VAR: &str = "ANTHROPIC_VERTEX_PROJECT_ID"; - -/// Env var for Vertex location (consumed by Claude Code SDK). -pub const VERTEX_LOCATION_ENV_VAR: &str = "VERTEX_LOCATION"; - -/// Non-secret GCP/Vertex config vars that must be resolved to real values -/// in the child environment. Everything else stays as placeholders for -/// proxy-time resolution. -/// -/// This list MUST be the union of all alias arrays above plus all -/// Vertex-specific env vars. If you add an alias to `PROJECT_ID_ENV_VARS`, -/// `REGION_ENV_VARS`, or a Vertex constant, add it here too. -pub const STATIC_CONFIG_KEYS: &[&str] = &[ - // project_id aliases - "GCP_PROJECT_ID", - "GOOGLE_CLOUD_PROJECT", - // region aliases - "CLOUD_ML_REGION", - "GCP_LOCATION", - // service account email - "GCP_SERVICE_ACCOUNT_EMAIL", - // Vertex-specific non-secret config - GOOSE_PROVIDER_ENV_VAR, - ANTHROPIC_VERTEX_PROJECT_ID_ENV_VAR, - VERTEX_LOCATION_ENV_VAR, -]; - #[cfg(test)] mod tests { use super::*; - use std::collections::HashSet; #[test] fn metadata_destination_matches_only_reserved_loopback_endpoint() { @@ -104,28 +54,4 @@ mod tests { assert!(!is_metadata_destination(address.parse().unwrap())); } } - - #[test] - fn static_config_keys_matches_alias_arrays_and_vertex_vars() { - let expected: HashSet<&str> = PROJECT_ID_ENV_VARS - .iter() - .chain(REGION_ENV_VARS) - .chain(SERVICE_ACCOUNT_EMAIL_ENV_VARS) - .copied() - .chain([ - GOOSE_PROVIDER_ENV_VAR, - ANTHROPIC_VERTEX_PROJECT_ID_ENV_VAR, - VERTEX_LOCATION_ENV_VAR, - ]) - .collect(); - let actual: HashSet<&str> = STATIC_CONFIG_KEYS.iter().copied().collect(); - assert_eq!( - expected, - actual, - "STATIC_CONFIG_KEYS must be the union of all alias arrays + Vertex vars. \ - Missing: {:?}, Extra: {:?}", - expected.difference(&actual).collect::>(), - actual.difference(&expected).collect::>(), - ); - } } diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index ab9bd26195..3eeab6e71e 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -521,21 +521,10 @@ impl ProviderCredentialState { inner.current = Arc::new(env); } - /// Return `child_env` with explicitly non-secret config vars resolved. - /// - /// The credential pipeline placeholderizes all env values. Workloads need - /// non-secret configuration, including provider file paths, at process - /// startup before any HTTP request flows through the proxy. Credential - /// values remain placeholders. - /// - /// Three layers of env var injection: - /// 1. **Synthetic vars** (`GCE_METADATA_IP`, `METADATA_SERVER_DETECTION`) - /// — sandbox-internal config not from user - /// input, inserted directly here with real values. - /// 2. **Classified non-secret keys** — user-provided config and provider - /// file paths un-placeholderized so workloads can read them at startup. - /// 3. Everything else stays as placeholders for proxy-time resolution. - pub fn child_env_with_gcp_resolved(&self) -> HashMap { + /// Resolve only environment keys explicitly classified as non-secret by + /// the gateway, including provider file paths needed at process startup. + /// Credentials remain placeholders for proxy-time resolution. + pub fn child_env_with_non_secret_resolved(&self) -> HashMap { let inner = self .inner .read() @@ -550,7 +539,7 @@ impl ProviderCredentialState { /// revision must describe the exact environment sent across the boundary, /// so callers must not obtain the two values through separate lock /// acquisitions. - pub fn child_env_snapshot_with_gcp_resolved( + pub fn child_env_snapshot_with_non_secret_resolved( &self, ) -> std::io::Result<(u64, HashMap)> { let inner = self @@ -586,47 +575,12 @@ impl ProviderCredentialState { fn resolve_child_env_snapshot( inner: &ProviderCredentialStateInner, ) -> (u64, HashMap) { - use crate::google_cloud; - let mut env = inner.current.child_env.clone(); - - let has_gcp_metadata = env.contains_key("GCE_METADATA_HOST") - && inner - .non_secret_environment_keys - .contains("GCE_METADATA_HOST"); - if !has_gcp_metadata && inner.non_secret_environment_keys.is_empty() { - return (inner.current.revision, env); - } - - if has_gcp_metadata { - // Synthetic vars: sandbox-internal config that doesn't originate - // from user input and was never placeholderized. - env.insert( - "GCE_METADATA_HOST".to_string(), - google_cloud::METADATA_LOOPBACK_ADDR.to_string(), - ); - // Python's google-auth builds its ping URL as http://{GCE_METADATA_IP} - // so the value must include the port. - env.insert( - "GCE_METADATA_IP".to_string(), - google_cloud::METADATA_LOOPBACK_ADDR.to_string(), - ); - // Node.js gcp-metadata uses METADATA_SERVER_DETECTION to skip the - // runtime ping that otherwise fails in sandboxed environments. - env.insert( - "METADATA_SERVER_DETECTION".to_string(), - "assume-present".to_string(), - ); - } - - // Only explicitly classified non-secret values may be unwrapped. - if let Some(ref resolver) = inner.combined_resolver { + if let Some(resolver) = &inner.current_resolver { for key in &inner.non_secret_environment_keys { - if !env.contains_key(key) || (has_gcp_metadata && key == "GCE_METADATA_HOST") { - continue; - } - let placeholder = crate::secrets::placeholder_for_env_key(key); - if let Some(value) = resolver.resolve_placeholder(&placeholder) { + if let Some(placeholder) = env.get(key) + && let Some(value) = resolver.resolve_placeholder(placeholder) + { env.insert(key.clone(), value.to_string()); } } @@ -1301,7 +1255,6 @@ fn merge_resolvers( #[cfg(test)] mod tests { use super::*; - use crate::google_cloud; fn jwt_alias(placeholder: &str) -> String { // Claims are irrelevant to injection authority: retain the exact issued @@ -3296,14 +3249,14 @@ mod tests { } #[test] - fn child_env_with_gcp_resolved_without_gcp_returns_unchanged() { + fn child_env_with_non_secret_resolved_without_classified_keys_returns_unchanged() { let state = ProviderCredentialState::from_environment( 1, HashMap::from([("GITHUB_TOKEN".to_string(), "ghp_abc".to_string())]), HashMap::new(), HashMap::new(), ); - let env = state.child_env_with_gcp_resolved(); + let env = state.child_env_with_non_secret_resolved(); assert_eq!( env.get("GITHUB_TOKEN").map(String::as_str), Some("openshell:resolve:env:v1_GITHUB_TOKEN"), @@ -3332,7 +3285,7 @@ mod tests { ) .expect("classified provider environment"); - let env = state.child_env_with_gcp_resolved(); + let env = state.child_env_with_non_secret_resolved(); assert_eq!(env.get("ACME_CONFIG_FILE").map(String::as_str), Some(path)); assert_eq!( env.get("ACME_TOKEN").map(String::as_str), @@ -3341,17 +3294,17 @@ mod tests { } #[test] - fn child_env_with_gcp_resolved_overrides_gcp_static_vars() { + fn child_env_with_non_secret_resolved_resolves_declared_custom_config() { let state = ProviderCredentialState::from_bound_environment( 1, HashMap::from([ - ("GCE_METADATA_HOST".to_string(), "marker".to_string()), + ("CUSTOM_ENDPOINT".to_string(), "marker".to_string()), ( "GCP_ADC_ACCESS_TOKEN".to_string(), "ya29.secret".to_string(), ), - ("GCP_PROJECT_ID".to_string(), "my-project".to_string()), - ("CLOUD_ML_REGION".to_string(), "us-central1".to_string()), + ("CUSTOM_PROJECT".to_string(), "my-project".to_string()), + ("CUSTOM_REGION".to_string(), "us-central1".to_string()), ]), HashMap::new(), HashMap::new(), @@ -3360,30 +3313,30 @@ mod tests { binding("oauth2.googleapis.com", 443, "/**"), )]), vec![ - "GCE_METADATA_HOST".to_string(), - "GCP_PROJECT_ID".to_string(), - "CLOUD_ML_REGION".to_string(), + "CUSTOM_ENDPOINT".to_string(), + "CUSTOM_PROJECT".to_string(), + "CUSTOM_REGION".to_string(), ], ) - .expect("classified GCP environment"); - let env = state.child_env_with_gcp_resolved(); + .expect("classified provider environment"); + let env = state.child_env_with_non_secret_resolved(); assert_eq!( - env.get("GCE_METADATA_HOST").map(String::as_str), - Some(google_cloud::METADATA_LOOPBACK_ADDR), - "GCE_METADATA_HOST should be the loopback address" + env.get("CUSTOM_ENDPOINT").map(String::as_str), + Some("marker"), + "non-secret values must be preserved without GCP-specific rewriting" ); assert!( !env.contains_key("CLAUDE_CODE_USE_VERTEX"), "inference-specific vars should not be injected" ); assert_eq!( - env.get("GCP_PROJECT_ID").map(String::as_str), + env.get("CUSTOM_PROJECT").map(String::as_str), Some("my-project"), "static config should be resolved to real value" ); assert_eq!( - env.get("CLOUD_ML_REGION").map(String::as_str), + env.get("CUSTOM_REGION").map(String::as_str), Some("us-central1"), ); @@ -3395,7 +3348,7 @@ mod tests { } #[test] - fn child_env_with_gcp_resolved_handles_missing_config_keys() { + fn child_env_with_non_secret_resolved_handles_missing_config_keys() { let state = ProviderCredentialState::from_bound_environment( 1, HashMap::from([ @@ -3411,11 +3364,11 @@ mod tests { vec!["GCE_METADATA_HOST".to_string()], ) .expect("classified GCP environment"); - let env = state.child_env_with_gcp_resolved(); + let env = state.child_env_with_non_secret_resolved(); assert_eq!( env.get("GCE_METADATA_HOST").map(String::as_str), - Some(google_cloud::METADATA_LOOPBACK_ADDR), + Some("marker"), ); assert!( !env.contains_key("GCP_PROJECT_ID") @@ -3530,7 +3483,7 @@ mod tests { } #[test] - fn child_env_with_gcp_resolved_resolves_vertex_vars_without_metadata_host() { + fn child_env_with_non_secret_resolved_resolves_vertex_vars_without_metadata_host() { let state = ProviderCredentialState::from_bound_environment( 1, HashMap::from([ @@ -3551,7 +3504,7 @@ mod tests { ], ) .expect("classified Vertex environment"); - let env = state.child_env_with_gcp_resolved(); + let env = state.child_env_with_non_secret_resolved(); assert_eq!( env.get("GOOSE_PROVIDER").map(String::as_str), Some("gcp_vertex_ai"), @@ -3572,7 +3525,7 @@ mod tests { } #[test] - fn child_env_with_gcp_resolved_only_unwraps_explicitly_non_secret_config() { + fn child_env_with_non_secret_resolved_only_unwraps_explicitly_non_secret_config() { let state = ProviderCredentialState::from_bound_environment( 1, HashMap::from([ @@ -3618,7 +3571,7 @@ mod tests { ) .expect("refreshed GCP environment"); - let env = state.child_env_with_gcp_resolved(); + let env = state.child_env_with_non_secret_resolved(); assert_eq!( env.get("GCP_PROJECT_ID").map(String::as_str), Some("visible-project-config"), @@ -3826,7 +3779,7 @@ mod tests { "opaque revisions may move numerically backwards" ); - let (revision, env) = state.child_env_snapshot_with_gcp_resolved().unwrap(); + let (revision, env) = state.child_env_snapshot_with_non_secret_resolved().unwrap(); assert_eq!(revision, 2); assert!(env.is_empty(), "an empty snapshot must revoke the old env"); } @@ -3868,7 +3821,7 @@ mod tests { .compare_and_install_child_env_snapshot(4, 5, HashMap::new()) .is_err() ); - assert!(state.child_env_snapshot_with_gcp_resolved().is_err()); + assert!(state.child_env_snapshot_with_non_secret_resolved().is_err()); } #[test] diff --git a/crates/openshell-providers/README.md b/crates/openshell-providers/README.md index 2a9b686019..10c965c468 100644 --- a/crates/openshell-providers/README.md +++ b/crates/openshell-providers/README.md @@ -1,18 +1,20 @@ # openshell-providers -Provider discovery and normalization for credentials that sandboxes need at -runtime. +Imported provider profile parsing, validation, discovery, and environment +defaults. The gateway persists provider records. The sandbox supervisor fetches resolved provider environment from the gateway and injects credentials into agent child -processes. This crate keeps provider-specific discovery and normalization logic -out of the CLI and gateway control flow. +processes. Profile declarations determine the behavior; profile IDs do not +select compiled provider plugins. ## Responsibilities -- Discover local credentials from environment variables and known config files. -- Normalize discovered data into provider records. -- Keep provider-specific parsing rules in provider modules. +- Parse and round-trip the YAML, JSON, and protobuf profile schema. +- Discover credentials and non-secret config from declared environment keys. +- Apply bounded, literal non-secret environment defaults without overwriting + existing values. +- Validate credential collisions and environment declarations. - Use the shared authored-YAML compatibility boundary for profile imports and exports: reject duplicate keys and null objects without coercion, retain optional nulls, and preserve string types for YAML 1.1 readers. @@ -25,7 +27,29 @@ out of the CLI and gateway control flow. - Injecting credentials into sandbox child processes. - Routing inference requests. -Those are owned by the gateway, sandbox supervisor, and router. +Those are owned by the gateway and sandbox supervisor. + +## Provider Behavior Inventory + +| Behavior | Declaration or disposition | +|---|---| +| GCP and Vertex project, region, and SDK aliases | `environment.config` | +| Metadata SDK variables and Goose provider default | `environment.fixed` | +| Credential discovery | `discovery.credentials` and credential `env_vars` | +| Vertex config discovery | `discovery.config_env_vars` | +| Workload resolution of non-secret values | Gateway key classification, independent of provider names | +| GCP metadata runtime requirement | Documented in the example profile header; the supervisor provides metadata for Linux sandboxes | +| Vertex service account JSON credential | Removed from the example; configure gateway refresh material directly | +| ID-selected provider plugins | Removed | +| ID-selected CLI credential hints | Removed; setup guidance applies to the imported profile | +| `ProviderDiscoverySpec` and `discover_with_spec` | Removed; no supported caller | +| GCP token response helper | Retained in `openshell-core` for the supervisor metadata service | +| GCP project and region config-key catalog | Removed; profile data supplies these declarations | + +Credential refresh strategies remain explicitly declared in credential metadata. +The supervisor provides the GCP metadata service for Linux sandboxes; the +Windows/MXC runtime does not provide it. Profile import and attachment do not +validate runtime support. ## Security Notes diff --git a/crates/openshell-providers/src/discovery.rs b/crates/openshell-providers/src/discovery.rs index 7a4fcdfd69..645e3a705a 100644 --- a/crates/openshell-providers/src/discovery.rs +++ b/crates/openshell-providers/src/discovery.rs @@ -1,35 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::{ - DiscoveredProvider, DiscoveryContext, ProviderDiscoverySpec, ProviderError, ProviderTypeProfile, -}; +use crate::{DiscoveredProvider, DiscoveryContext, ProviderError, ProviderTypeProfile}; use std::collections::HashSet; -pub fn discover_with_spec( - spec: &ProviderDiscoverySpec, - context: &dyn DiscoveryContext, -) -> Result, ProviderError> { - let mut discovered = DiscoveredProvider::default(); - - for key in spec.credential_env_vars { - if let Some(value) = context.env_var(key) - && !value.trim().is_empty() - { - discovered - .credentials - .entry((*key).to_string()) - .or_insert(value); - } - } - - if discovered.is_empty() { - Ok(None) - } else { - Ok(Some(discovered)) - } -} - pub fn discover_from_profile( profile: &ProviderTypeProfile, context: &dyn DiscoveryContext, @@ -66,13 +40,11 @@ pub fn discover_from_profile( } } - if profile.id == "google-vertex-ai" { - for key in crate::VERTEX_AI_CONFIG_KEY_NAMES { - if let Some(value) = context.env_var(key) - && !value.trim().is_empty() - { - discovered.config.entry((*key).to_string()).or_insert(value); - } + for key in &profile.discovery.config_env_vars { + if let Some(value) = context.env_var(key) + && !value.trim().is_empty() + { + discovered.config.entry(key.clone()).or_insert(value); } } @@ -88,7 +60,7 @@ mod tests { use super::discover_from_profile; use crate::profiles::{CredentialProfile, DiscoveryProfile}; use crate::test_helpers::MockDiscoveryContext; - use crate::{ProviderError, ProviderTypeProfile}; + use crate::{EnvironmentProfile, ProviderError, ProviderTypeProfile}; fn profile() -> ProviderTypeProfile { ProviderTypeProfile { @@ -132,7 +104,9 @@ mod tests { inference_capable: false, discovery: DiscoveryProfile { credentials: vec!["api_key".to_string(), "secondary".to_string()], + config_env_vars: Vec::new(), }, + environment: EnvironmentProfile::default(), source: String::new(), scope: String::new(), } @@ -180,9 +154,10 @@ mod tests { } #[test] - fn vertex_profile_discovery_includes_supported_configuration() { + fn profile_discovery_includes_declared_configuration() { let mut profile = profile(); - profile.id = "google-vertex-ai".to_string(); + profile.discovery.config_env_vars = + vec!["VERTEX_AI_PROJECT_ID".into(), "VERTEX_AI_REGION".into()]; let ctx = MockDiscoveryContext::new() .with_env("CUSTOM_API_KEY", "vertex-token") .with_env("VERTEX_AI_PROJECT_ID", "project-a") diff --git a/crates/openshell-providers/src/environment_tests.rs b/crates/openshell-providers/src/environment_tests.rs new file mode 100644 index 0000000000..df25f0f725 --- /dev/null +++ b/crates/openshell-providers/src/environment_tests.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use crate::test_helpers::MockDiscoveryContext; +use crate::{ + EnvironmentProfile, ProviderTypeProfile, discover_from_profile, parse_profile_yaml, + profile_to_yaml, validate_profile_set, +}; + +fn profile() -> ProviderTypeProfile { + parse_profile_yaml( + r" +id: custom +display_name: Custom +credentials: + - name: token + env_vars: [CUSTOM_TOKEN] +environment: + config: + CUSTOM_PROJECT: project + fixed: + CUSTOM_MODE: native +discovery: + credentials: [token] + config_env_vars: [CUSTOM_PROJECT] +", + ) + .unwrap() +} + +#[test] +fn environment_and_discovery_declarations_round_trip() { + let profile = profile(); + let proto = profile.to_proto(); + let restored = ProviderTypeProfile::from_proto(&proto); + assert_eq!(restored.to_proto(), proto); + let yaml = profile_to_yaml(&restored).unwrap(); + assert_eq!(parse_profile_yaml(&yaml).unwrap().to_proto(), proto); + assert!(validate_profile_set(&[("custom.yaml".into(), profile)]).is_empty()); +} + +#[test] +fn projection_preserves_existing_values_and_skips_blank_config() { + let profile = profile(); + let config = HashMap::from([("project".into(), " project-a ".into())]); + let mut env = HashMap::new(); + profile.environment.inject(&config, &mut env); + assert_eq!( + env, + HashMap::from([ + ("CUSTOM_PROJECT".into(), "project-a".into()), + ("CUSTOM_MODE".into(), "native".into()) + ]) + ); + + let mut existing = HashMap::from([ + ("CUSTOM_PROJECT".into(), String::new()), + ("CUSTOM_MODE".into(), "caller".into()), + ]); + let before = existing.clone(); + profile.environment.inject(&config, &mut existing); + assert_eq!(existing, before); + + for config in [ + HashMap::new(), + HashMap::from([("project".into(), " \t ".into())]), + ] { + let mut env = HashMap::new(); + profile.environment.inject(&config, &mut env); + assert!(!env.contains_key("CUSTOM_PROJECT")); + assert_eq!(env["CUSTOM_MODE"], "native"); + } +} + +#[test] +fn projection_skips_legacy_private_key_defaults_in_stored_profiles() { + let mut profile = profile(); + profile.environment.config.insert( + crate::LEGACY_VERTEX_PRIVATE_KEY_ENV.into(), + "private_key".into(), + ); + let config = HashMap::from([("private_key".into(), "secret".into())]); + let mut env = HashMap::new(); + profile.environment.inject(&config, &mut env); + assert!(!env.contains_key(crate::LEGACY_VERTEX_PRIVATE_KEY_ENV)); + + profile.environment.config.clear(); + profile + .environment + .fixed + .insert(crate::LEGACY_VERTEX_PRIVATE_KEY_ENV.into(), "secret".into()); + profile.environment.inject(&config, &mut env); + assert!(!env.contains_key(crate::LEGACY_VERTEX_PRIVATE_KEY_ENV)); +} + +#[test] +fn renamed_google_examples_keep_environment_and_discovery_behavior() { + let config = HashMap::from([ + ("project_id".into(), "cloud-project".into()), + ("region".into(), "us-east1".into()), + ("service_account_email".into(), "sa@example.com".into()), + ("VERTEX_AI_PROJECT_ID".into(), "vertex-project".into()), + ("VERTEX_AI_REGION".into(), "us-central1".into()), + ]); + let context = MockDiscoveryContext::new() + .with_env("GOOGLE_VERTEX_AI_TOKEN", "test-token") + .with_env("GCP_ADC_ACCESS_TOKEN", "test-adc-token") + .with_env("VERTEX_AI_PROJECT_ID", "vertex-project") + .with_env("VERTEX_AI_REGION", "us-central1"); + for id in ["google-cloud", "google-vertex-ai"] { + let canonical = crate::example_profiles::load(id); + let mut renamed = canonical.clone(); + renamed.id = "acme-fork".into(); + let mut canonical_env = HashMap::new(); + let mut renamed_env = HashMap::new(); + canonical.environment.inject(&config, &mut canonical_env); + renamed.environment.inject(&config, &mut renamed_env); + assert!(!canonical_env.is_empty()); + assert_eq!(canonical_env, renamed_env); + assert_eq!( + discover_from_profile(&canonical, &context).unwrap(), + discover_from_profile(&renamed, &context).unwrap() + ); + } +} + +#[test] +fn canonical_id_without_declarations_has_no_special_behavior() { + let mut profile = profile(); + profile.id = "google-vertex-ai".into(); + profile.environment = EnvironmentProfile::default(); + profile.discovery.config_env_vars.clear(); + let mut env = HashMap::new(); + profile.environment.inject( + &HashMap::from([("VERTEX_AI_PROJECT_ID".into(), "project".into())]), + &mut env, + ); + assert!(env.is_empty()); + assert!( + discover_from_profile( + &profile, + &MockDiscoveryContext::new().with_env("VERTEX_AI_PROJECT_ID", "project") + ) + .unwrap() + .is_none() + ); +} + +#[test] +fn lint_rejects_non_secret_credential_collisions_and_duplicate_targets() { + let mut profile = profile(); + profile + .environment + .fixed + .insert("CUSTOM_TOKEN".into(), "value".into()); + profile + .environment + .fixed + .insert("CUSTOM_PROJECT".into(), "other".into()); + profile + .discovery + .config_env_vars + .push("CUSTOM_TOKEN".into()); + let diagnostics = validate_profile_set(&[("custom.yaml".into(), profile)]); + assert!( + diagnostics + .iter() + .any(|d| d.field == "environment" && d.message.contains("credential env_vars")) + ); + assert!( + diagnostics + .iter() + .any(|d| d.message.contains("both config and fixed")) + ); + assert!(diagnostics.iter().any( + |d| d.field == "discovery.config_env_vars" && d.message.contains("credential env_vars") + )); +} + +#[test] +fn lint_bounds_environment_and_discovery_declarations() { + let mut profile = profile(); + profile + .environment + .fixed + .insert("INVALID=NAME".into(), "value".into()); + profile + .environment + .fixed + .insert("LONG_VALUE".into(), "x".repeat(4097)); + profile + .discovery + .config_env_vars + .push("CUSTOM_PROJECT".into()); + profile.discovery.config_env_vars.push(" padded ".into()); + let diagnostics = validate_profile_set(&[("custom.yaml".into(), profile)]); + assert!(diagnostics.iter().any(|d| d.field == "environment")); + assert!(diagnostics.iter().any(|d| d.field == "environment.fixed")); + assert!( + diagnostics + .iter() + .any(|d| d.message == "duplicate discovery config key") + ); + assert!( + diagnostics + .iter() + .any(|d| d.message.contains("discovery config keys must")) + ); +} + +#[test] +fn lint_enforces_declaration_count_limits() { + let mut profile = profile(); + profile.environment.config.clear(); + profile.environment.fixed = (0..64) + .map(|index| (format!("CONFIG_{index}"), String::new())) + .collect(); + profile.discovery.config_env_vars = profile.environment.fixed.keys().cloned().collect(); + assert!(validate_profile_set(&[("custom.yaml".into(), profile.clone())]).is_empty()); + + profile + .environment + .config + .insert("EXTRA".into(), "extra".into()); + profile.discovery.config_env_vars.push("EXTRA".into()); + let diagnostics = validate_profile_set(&[("custom.yaml".into(), profile)]); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic.message == "at most 64 environment defaults are allowed") + ); + assert!( + diagnostics + .iter() + .any(|diagnostic| diagnostic.message == "at most 64 discovery config keys are allowed") + ); +} diff --git a/crates/openshell-providers/src/lib.rs b/crates/openshell-providers/src/lib.rs index 87ac5d8007..7e7148e2e6 100644 --- a/crates/openshell-providers/src/lib.rs +++ b/crates/openshell-providers/src/lib.rs @@ -5,10 +5,11 @@ mod context; mod discovery; +#[cfg(test)] +mod environment_tests; #[cfg(any(test, feature = "example-profiles"))] pub mod example_profiles; mod profiles; -mod providers; #[cfg(test)] mod test_helpers; @@ -16,25 +17,19 @@ use std::collections::HashMap; pub use openshell_core::proto::Provider; +/// Legacy Vertex bootstrap material must never become a workload credential. +/// Older stored profiles may still declare this key as an injectable env var. +pub const LEGACY_VERTEX_PRIVATE_KEY_ENV: &str = "GOOGLE_SERVICE_ACCOUNT_KEY"; + pub use context::{DiscoveryContext, RealDiscoveryContext}; -pub use discovery::{discover_from_profile, discover_with_spec}; +pub use discovery::discover_from_profile; pub use profiles::{ - CredentialRefreshProfile, ProfileError, ProfileValidationDiagnostic, ProviderTypeProfile, - is_gateway_mintable_strategy, normalize_profile_id, parse_profile_json, parse_profile_yaml, - profile_to_json, profile_to_yaml, profiles_to_json, profiles_to_yaml, strategy_output_env_key, - strategy_output_spec, strategy_primary_env_key, validate_profile_set, + CredentialRefreshProfile, EnvironmentProfile, ProfileError, ProfileValidationDiagnostic, + ProviderTypeProfile, is_gateway_mintable_strategy, normalize_profile_id, parse_profile_json, + parse_profile_yaml, profile_to_json, profile_to_yaml, profiles_to_json, profiles_to_yaml, + strategy_output_env_key, strategy_output_spec, strategy_primary_env_key, validate_profile_set, }; -pub const VERTEX_AI_PROJECT_ID_KEY: &str = "VERTEX_AI_PROJECT_ID"; -pub const VERTEX_AI_REGION_KEY: &str = "VERTEX_AI_REGION"; -pub const VERTEX_AI_CONFIG_KEY_NAMES: &[&str] = &[ - VERTEX_AI_PROJECT_ID_KEY, - VERTEX_AI_REGION_KEY, - "GOOGLE_VERTEX_AI_BASE_URL", - "VERTEX_AI_BASE_URL", - "VERTEX_AI_PUBLISHER", -]; - #[derive(Debug, thiserror::Error)] pub enum ProviderError { #[error("unsupported provider type: {0}")] @@ -60,67 +55,3 @@ impl DiscoveredProvider { self.credentials.is_empty() && self.config.is_empty() } } - -#[derive(Debug, Clone, Copy)] -pub struct ProviderDiscoverySpec { - pub id: &'static str, - pub credential_env_vars: &'static [&'static str], -} - -trait ProviderPlugin: Send + Sync { - /// Canonical provider id. - fn id(&self) -> &'static str; - - /// Inject provider-specific environment variables into the sandbox env. - /// - /// Called during sandbox creation to project provider config (project IDs, - /// regions, SDK flags) into env vars the sandbox process will inherit. - /// Default is a no-op; GCP and Vertex providers override this. - fn inject_env(&self, _provider: &Provider, _env: &mut HashMap) {} -} - -#[derive(Default)] -pub struct ProviderRegistry { - plugins: HashMap<&'static str, Box>, -} - -impl ProviderRegistry { - #[must_use] - pub fn new() -> Self { - let mut registry = Self::default(); - // Keep only the legacy config projectors required to run existing - // Google Cloud and Vertex records. Public provider discovery is - // profile-driven; this registry is an internal compatibility adapter. - registry.register(providers::google_cloud::GoogleCloudProvider); - registry.register(providers::vertex::VertexProvider); - registry - } - - fn register

(&mut self, plugin: P) - where - P: ProviderPlugin + 'static, - { - self.plugins.insert(plugin.id(), Box::new(plugin)); - } - - #[must_use] - fn get(&self, id: &str) -> Option<&dyn ProviderPlugin> { - self.plugins.get(id).map(Box::as_ref) - } - - /// Inject provider-specific config for a resolved profile ID. - /// - /// Plugins are selected by the exact ID of the profile the gateway - /// resolved. There is no alias table: a profile activates the plugin whose - /// ID it matches, and nothing else does. - pub fn inject_env_for_profile_id( - &self, - provider: &Provider, - profile_id: &str, - env: &mut HashMap, - ) { - if let Some(plugin) = self.get(profile_id) { - plugin.inject_env(provider, env); - } - } -} diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index b56dc2c7e4..654354ed54 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -13,7 +13,8 @@ use openshell_core::proto::{ ProviderCredentialRefreshMaterial, ProviderCredentialRefreshOutput, ProviderCredentialRefreshStrategy, ProviderCredentialTokenGrantSubjectToken, ProviderCredentialTokenGrantType, ProviderProfile, ProviderProfileCategory, - ProviderProfileCredential, ProviderProfileDiscovery, ProviderProfileFile, + ProviderProfileCredential, ProviderProfileDiscovery, ProviderProfileEnvironment, + ProviderProfileFile, }; use openshell_core::secrets::uses_reserved_revision_namespace; use openshell_policy::{ @@ -23,7 +24,7 @@ use openshell_policy::{ validate_l7_endpoint_semantics, }; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::net::IpAddr; const PATH_TEMPLATE_CREDENTIAL_PLACEHOLDER: &str = "{credential}"; @@ -322,6 +323,47 @@ pub struct CredentialRefreshOutputProfile { pub struct DiscoveryProfile { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub credentials: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub config_env_vars: Vec, +} + +/// Literal, non-secret workload environment defaults. +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentProfile { + /// Destination environment key -> provider config key. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub config: BTreeMap, + /// Destination environment key -> literal value. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub fixed: BTreeMap, +} + +impl EnvironmentProfile { + fn is_empty(&self) -> bool { + self.config.is_empty() && self.fixed.is_empty() + } + + /// Fill missing values only. Projection never changes a caller's value, + /// including an explicitly empty value. + pub fn inject(&self, config: &HashMap, env: &mut HashMap) { + for (key, config_key) in &self.config { + if key == crate::LEGACY_VERTEX_PRIVATE_KEY_ENV { + continue; + } + if let Some(value) = config.get(config_key).map(|value| value.trim()) + && !value.is_empty() + { + env.entry(key.clone()).or_insert_with(|| value.to_string()); + } + } + for (key, value) in &self.fixed { + if key == crate::LEGACY_VERTEX_PRIVATE_KEY_ENV { + continue; + } + env.entry(key.clone()).or_insert_with(|| value.clone()); + } + } } // These YAML/JSON DTOs mirror the network policy protos intentionally. Keep @@ -713,6 +755,8 @@ pub struct ProviderTypeProfile { pub inference_capable: bool, #[serde(default, skip_serializing_if = "discovery_is_empty")] pub discovery: DiscoveryProfile, + #[serde(default, skip_serializing_if = "EnvironmentProfile::is_empty")] + pub environment: EnvironmentProfile, #[serde(default, skip_serializing_if = "String::is_empty")] pub source: String, #[serde(default, skip_serializing_if = "String::is_empty")] @@ -848,6 +892,13 @@ impl ProviderTypeProfile { .unwrap_or_default(), source: profile.source.clone(), scope: profile.scope.clone(), + environment: profile.environment.as_ref().map_or_else( + EnvironmentProfile::default, + |environment| EnvironmentProfile { + config: environment.config.clone(), + fixed: environment.fixed.clone(), + }, + ), } } @@ -1022,6 +1073,10 @@ impl ProviderTypeProfile { .then(|| discovery_to_proto(&self.discovery)), source: self.source.clone(), scope: self.scope.clone(), + environment: (!self.environment.is_empty()).then(|| ProviderProfileEnvironment { + config: self.environment.config.clone(), + fixed: self.environment.fixed.clone(), + }), } } @@ -1213,7 +1268,7 @@ pub fn strategy_output_env_key( } fn discovery_is_empty(discovery: &DiscoveryProfile) -> bool { - discovery.credentials.is_empty() + discovery.credentials.is_empty() && discovery.config_env_vars.is_empty() } impl Serialize for BinaryProfile { @@ -1628,12 +1683,14 @@ fn token_grant_audience_override_to_proto( fn discovery_from_proto(discovery: &ProviderProfileDiscovery) -> DiscoveryProfile { DiscoveryProfile { credentials: discovery.credentials.clone(), + config_env_vars: discovery.config_env_vars.clone(), } } fn discovery_to_proto(discovery: &DiscoveryProfile) -> ProviderProfileDiscovery { ProviderProfileDiscovery { credentials: discovery.credentials.clone(), + config_env_vars: discovery.config_env_vars.clone(), } } @@ -2141,6 +2198,7 @@ pub fn validate_profile_set( let mut diagnostics = Vec::new(); let mut ids = HashSet::new(); for (source, profile) in profiles { + collect_environment_diagnostics(source, profile, &mut diagnostics); let raw_profile_id = profile.id.as_str(); let profile_id = raw_profile_id.trim(); if profile_id.is_empty() { @@ -3155,6 +3213,125 @@ pub fn validate_profile_set( diagnostics } +// Keep this declaration surface bounded and data-only. These limits apply to +// both YAML and protobuf imports through the shared profile validator. +const MAX_ENVIRONMENT_ENTRIES: usize = 64; +const MAX_ENVIRONMENT_KEY_BYTES: usize = 128; +const MAX_FIXED_ENVIRONMENT_VALUE_BYTES: usize = 4096; + +fn valid_environment_key(key: &str) -> bool { + let mut bytes = key.bytes(); + key.len() <= MAX_ENVIRONMENT_KEY_BYTES + && bytes + .next() + .is_some_and(|byte| byte == b'_' || byte.is_ascii_alphabetic()) + && bytes.all(|byte| byte == b'_' || byte.is_ascii_alphanumeric()) + && !uses_reserved_revision_namespace(key) +} + +fn collect_environment_diagnostics( + source: &str, + profile: &ProviderTypeProfile, + diagnostics: &mut Vec, +) { + let mut error = |field: &str, message: &str| { + diagnostics.push(ProfileValidationDiagnostic::error( + source, + &profile.id, + field, + message, + )); + }; + let environment = &profile.environment; + if environment.config.len() + environment.fixed.len() > MAX_ENVIRONMENT_ENTRIES { + error("environment", "at most 64 environment defaults are allowed"); + } + let credential_keys = profile.credential_env_vars(); + for key in environment + .config + .keys() + .chain(environment.fixed.keys()) + .take(MAX_ENVIRONMENT_ENTRIES) + { + if key == crate::LEGACY_VERTEX_PRIVATE_KEY_ENV { + error( + "environment", + "GOOGLE_SERVICE_ACCOUNT_KEY contains private key material and cannot be a non-secret environment default", + ); + } + if !valid_environment_key(key) { + error( + "environment", + "environment keys must be valid non-reserved variable names of at most 128 bytes", + ); + } + if credential_keys.contains(&key.as_str()) { + error( + "environment", + "non-secret environment defaults must not collide with credential env_vars", + ); + } + } + for (key, config_key) in environment.config.iter().take(MAX_ENVIRONMENT_ENTRIES) { + if config_key.is_empty() + || config_key.trim() != config_key + || config_key.len() > MAX_ENVIRONMENT_KEY_BYTES + || config_key.contains('\0') + { + error( + "environment.config", + "config keys must be nonempty, unpadded strings of at most 128 bytes without NUL", + ); + } + if environment.fixed.contains_key(key) { + error( + "environment", + "an environment key cannot have both config and fixed defaults", + ); + } + } + for value in environment.fixed.values().take(MAX_ENVIRONMENT_ENTRIES) { + if value.len() > MAX_FIXED_ENVIRONMENT_VALUE_BYTES || value.contains('\0') { + error( + "environment.fixed", + "fixed values must contain at most 4096 bytes and no NUL", + ); + } + } + if profile.discovery.config_env_vars.len() > MAX_ENVIRONMENT_ENTRIES { + error( + "discovery.config_env_vars", + "at most 64 discovery config keys are allowed", + ); + } + let mut discovered = HashSet::new(); + for key in profile + .discovery + .config_env_vars + .iter() + .take(MAX_ENVIRONMENT_ENTRIES) + { + if !valid_environment_key(key) { + error( + "discovery.config_env_vars", + "discovery config keys must be valid non-reserved variable names of at most 128 bytes", + ); + } + if !discovered.insert(key) { + error( + "discovery.config_env_vars", + "duplicate discovery config key", + ); + } + if credential_keys.contains(&key.as_str()) { + error( + "discovery.config_env_vars", + "non-secret discovery config must not collide with credential env_vars", + ); + } + } +} + fn collect_mcp_profile_diagnostics( source: &str, profile_id: &str, @@ -3741,12 +3918,12 @@ mod tests { use openshell_core::proto::{ProviderCredentialTokenGrantType, ProviderProfileCategory}; use super::{ - DiscoveryProfile, EndpointProfile, L7AllowProfile, L7QueryMatcherProfile, - ProfileDurationWkt, ProfileError, ProviderTypeProfile, is_mcp_diagnostic_field, - normalize_profile_id, parse_profile_catalog_yamls, parse_profile_json, parse_profile_yaml, - profile_duration_to_proto, profile_to_json, profile_to_yaml, profiles_to_json, - profiles_to_yaml, token_grant_from_proto, token_grant_to_proto, validate_profile_duration, - validate_profile_set, + DiscoveryProfile, EndpointProfile, EnvironmentProfile, L7AllowProfile, + L7QueryMatcherProfile, ProfileDurationWkt, ProfileError, ProviderTypeProfile, + is_mcp_diagnostic_field, normalize_profile_id, parse_profile_catalog_yamls, + parse_profile_json, parse_profile_yaml, profile_duration_to_proto, profile_to_json, + profile_to_yaml, profiles_to_json, profiles_to_yaml, token_grant_from_proto, + token_grant_to_proto, validate_profile_duration, validate_profile_set, }; /// The example profiles in `providers/`, parsed once per test binary. @@ -6385,6 +6562,7 @@ binaries: binaries: Vec::new(), inference_capable: false, discovery: DiscoveryProfile::default(), + environment: EnvironmentProfile::default(), source: String::new(), scope: String::new(), }, @@ -6404,6 +6582,7 @@ binaries: binaries: Vec::new(), inference_capable: false, discovery: DiscoveryProfile::default(), + environment: EnvironmentProfile::default(), source: String::new(), scope: String::new(), }, @@ -6423,6 +6602,7 @@ binaries: binaries: Vec::new(), inference_capable: false, discovery: DiscoveryProfile::default(), + environment: EnvironmentProfile::default(), source: String::new(), scope: String::new(), }, diff --git a/crates/openshell-providers/src/providers/google_cloud.rs b/crates/openshell-providers/src/providers/google_cloud.rs deleted file mode 100644 index 3bea53d825..0000000000 --- a/crates/openshell-providers/src/providers/google_cloud.rs +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; - -use openshell_core::google_cloud; - -use crate::{Provider, ProviderPlugin}; - -pub struct GoogleCloudProvider; - -impl ProviderPlugin for GoogleCloudProvider { - fn id(&self) -> &'static str { - "google-cloud" - } - - fn inject_env(&self, provider: &Provider, env: &mut HashMap) { - if let Some(project) = provider - .config - .get(google_cloud::GCP_PROJECT_ID_CONFIG_KEY) - .filter(|v| !v.trim().is_empty()) - { - for var in google_cloud::PROJECT_ID_ENV_VARS { - env.entry((*var).to_string()) - .or_insert_with(|| project.trim().to_string()); - } - } - - if let Some(region) = provider - .config - .get(google_cloud::GCP_REGION_CONFIG_KEY) - .filter(|v| !v.trim().is_empty()) - { - for var in google_cloud::REGION_ENV_VARS { - env.entry((*var).to_string()) - .or_insert_with(|| region.trim().to_string()); - } - } - - env.entry("GCE_METADATA_HOST".to_string()) - .or_insert_with(|| google_cloud::METADATA_HOST.to_string()); - - if let Some(email) = provider - .config - .get(google_cloud::GCP_SERVICE_ACCOUNT_EMAIL_CONFIG_KEY) - .filter(|v| !v.trim().is_empty()) - { - for var in google_cloud::SERVICE_ACCOUNT_EMAIL_ENV_VARS { - env.entry((*var).to_string()) - .or_insert_with(|| email.trim().to_string()); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_provider(config: HashMap) -> Provider { - Provider { - config, - r#type: "google-cloud".to_string(), - ..Default::default() - } - } - - #[test] - fn injects_project_id_aliases() { - let provider = make_provider(HashMap::from([( - "project_id".to_string(), - "my-project".to_string(), - )])); - let mut env = HashMap::new(); - GoogleCloudProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("GCP_PROJECT_ID").map(String::as_str), - Some("my-project") - ); - assert_eq!( - env.get("GOOGLE_CLOUD_PROJECT").map(String::as_str), - Some("my-project") - ); - } - - #[test] - fn injects_region_aliases() { - let provider = make_provider(HashMap::from([( - "region".to_string(), - "us-central1".to_string(), - )])); - let mut env = HashMap::new(); - GoogleCloudProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("CLOUD_ML_REGION").map(String::as_str), - Some("us-central1") - ); - assert_eq!( - env.get("GCP_LOCATION").map(String::as_str), - Some("us-central1") - ); - } - - #[test] - fn injects_metadata_host() { - let provider = make_provider(HashMap::new()); - let mut env = HashMap::new(); - GoogleCloudProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("GCE_METADATA_HOST").map(String::as_str), - Some(google_cloud::METADATA_HOST) - ); - } - - #[test] - fn injects_service_account_email() { - let provider = make_provider(HashMap::from([( - "service_account_email".to_string(), - "sa@project.iam.gserviceaccount.com".to_string(), - )])); - let mut env = HashMap::new(); - GoogleCloudProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("GCP_SERVICE_ACCOUNT_EMAIL").map(String::as_str), - Some("sa@project.iam.gserviceaccount.com") - ); - } - - #[test] - fn does_not_overwrite_existing_env() { - let provider = make_provider(HashMap::from([( - "project_id".to_string(), - "new-project".to_string(), - )])); - let mut env = - HashMap::from([("GCP_PROJECT_ID".to_string(), "existing-project".to_string())]); - GoogleCloudProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("GCP_PROJECT_ID").map(String::as_str), - Some("existing-project"), - "should not overwrite existing env" - ); - } - - #[test] - fn skips_empty_config_values() { - let provider = make_provider(HashMap::from([( - "project_id".to_string(), - " ".to_string(), - )])); - let mut env = HashMap::new(); - GoogleCloudProvider.inject_env(&provider, &mut env); - - assert!(!env.contains_key("GCP_PROJECT_ID")); - } -} diff --git a/crates/openshell-providers/src/providers/mod.rs b/crates/openshell-providers/src/providers/mod.rs deleted file mode 100644 index b6450e170e..0000000000 --- a/crates/openshell-providers/src/providers/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -pub mod google_cloud; -pub mod vertex; diff --git a/crates/openshell-providers/src/providers/vertex.rs b/crates/openshell-providers/src/providers/vertex.rs deleted file mode 100644 index e8e1857a91..0000000000 --- a/crates/openshell-providers/src/providers/vertex.rs +++ /dev/null @@ -1,149 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; - -use openshell_core::google_cloud; - -use crate::{Provider, ProviderPlugin, VERTEX_AI_PROJECT_ID_KEY, VERTEX_AI_REGION_KEY}; - -pub struct VertexProvider; - -impl ProviderPlugin for VertexProvider { - fn id(&self) -> &'static str { - "google-vertex-ai" - } - - fn inject_env(&self, provider: &Provider, env: &mut HashMap) { - if let Some(project) = provider - .config - .get(VERTEX_AI_PROJECT_ID_KEY) - .filter(|v| !v.trim().is_empty()) - { - let trimmed = project.trim().to_string(); - for var in google_cloud::PROJECT_ID_ENV_VARS { - env.entry((*var).to_string()) - .or_insert_with(|| trimmed.clone()); - } - env.entry(google_cloud::ANTHROPIC_VERTEX_PROJECT_ID_ENV_VAR.to_string()) - .or_insert_with(|| trimmed.clone()); - } - - if let Some(region) = provider - .config - .get(VERTEX_AI_REGION_KEY) - .filter(|v| !v.trim().is_empty()) - { - let trimmed = region.trim().to_string(); - for var in google_cloud::REGION_ENV_VARS { - env.entry((*var).to_string()) - .or_insert_with(|| trimmed.clone()); - } - env.entry(google_cloud::VERTEX_LOCATION_ENV_VAR.to_string()) - .or_insert_with(|| trimmed.clone()); - } - - env.entry(google_cloud::GOOSE_PROVIDER_ENV_VAR.to_string()) - .or_insert_with(|| "gcp_vertex_ai".to_string()); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_provider(config: HashMap) -> Provider { - Provider { - config, - r#type: "google-vertex-ai".to_string(), - ..Default::default() - } - } - - #[test] - fn injects_project_id_and_anthropic_alias() { - let provider = make_provider(HashMap::from([( - "VERTEX_AI_PROJECT_ID".to_string(), - "my-vertex-project".to_string(), - )])); - let mut env = HashMap::new(); - VertexProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("GCP_PROJECT_ID").map(String::as_str), - Some("my-vertex-project") - ); - assert_eq!( - env.get("GOOGLE_CLOUD_PROJECT").map(String::as_str), - Some("my-vertex-project") - ); - assert_eq!( - env.get("ANTHROPIC_VERTEX_PROJECT_ID").map(String::as_str), - Some("my-vertex-project") - ); - } - - #[test] - fn injects_region_and_vertex_location() { - let provider = make_provider(HashMap::from([( - "VERTEX_AI_REGION".to_string(), - "us-east4".to_string(), - )])); - let mut env = HashMap::new(); - VertexProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("CLOUD_ML_REGION").map(String::as_str), - Some("us-east4") - ); - assert_eq!( - env.get("GCP_LOCATION").map(String::as_str), - Some("us-east4") - ); - assert_eq!( - env.get("VERTEX_LOCATION").map(String::as_str), - Some("us-east4") - ); - } - - #[test] - fn injects_inference_flags() { - let provider = make_provider(HashMap::new()); - let mut env = HashMap::new(); - VertexProvider.inject_env(&provider, &mut env); - - assert!(!env.contains_key("CLAUDE_CODE_USE_VERTEX")); - assert_eq!( - env.get("GOOSE_PROVIDER").map(String::as_str), - Some("gcp_vertex_ai") - ); - } - - #[test] - fn does_not_overwrite_existing_env() { - let provider = make_provider(HashMap::from([( - "VERTEX_AI_PROJECT_ID".to_string(), - "new".to_string(), - )])); - let mut env = HashMap::from([("GCP_PROJECT_ID".to_string(), "existing".to_string())]); - VertexProvider.inject_env(&provider, &mut env); - - assert_eq!( - env.get("GCP_PROJECT_ID").map(String::as_str), - Some("existing") - ); - } - - #[test] - fn skips_empty_config_values() { - let provider = make_provider(HashMap::from([( - "VERTEX_AI_PROJECT_ID".to_string(), - " ".to_string(), - )])); - let mut env = HashMap::new(); - VertexProvider.inject_env(&provider, &mut env); - - assert!(!env.contains_key("GCP_PROJECT_ID")); - assert!(!env.contains_key("ANTHROPIC_VERTEX_PROJECT_ID")); - } -} diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index d823fef977..7f688eff2d 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -180,7 +180,10 @@ impl LocalBoundaryExec { command.env(key, value); } } - for (key, value) in self.provider_credentials.child_env_with_gcp_resolved() { + for (key, value) in self + .provider_credentials + .child_env_with_non_secret_resolved() + { if !crate::process::is_supervisor_only_env_var(&key) { command.env(key, value); } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index cc83789417..5c58d68863 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -3075,6 +3075,11 @@ pub async fn build_sandbox_config_snapshot_from_inputs( &policy_credential_bindings, )?; } + let environment_overrides = super::provider::non_secret_provider_environment_overrides( + provider_profile_catalog, + provider_records, + sandbox, + ); let provider_env_revision = compute_provider_env_revision_from_records_and_policy_bindings( provider_profile_catalog, provider_records, @@ -3082,6 +3087,7 @@ pub async fn build_sandbox_config_snapshot_from_inputs( policy .as_ref() .filter(|_| matches!(policy_source, PolicySource::Global)), + &environment_overrides, )?; let service_routes = super::service::configured_app_routes(state, sandbox).await?; @@ -3381,6 +3387,7 @@ fn compute_provider_env_revision_from_records( records, &HashMap::new(), None, + &BTreeMap::new(), ) } @@ -3389,6 +3396,7 @@ fn compute_provider_env_revision_from_records_and_policy_bindings( records: &[super::provider::ProviderEnvironmentRecord], policy_bindings: &HashMap>, global_policy: Option<&ProtoSandboxPolicy>, + environment_overrides: &BTreeMap, ) -> Result { let mut hasher = Sha256::new(); hasher.update(b"openshell-provider-env-revision-v4"); @@ -3432,6 +3440,18 @@ fn compute_provider_env_revision_from_records_and_policy_bindings( hasher.update(deterministic_policy_hash(policy).as_bytes()); } + if !environment_overrides.is_empty() { + // Hash the same effective values that the environment snapshot installs. + // Length prefixes preserve boundaries, including explicitly empty values. + hasher.update(b"caller-non-secret-environment"); + for (key, value) in environment_overrides { + hasher.update((key.len() as u64).to_le_bytes()); + hasher.update(key.as_bytes()); + hasher.update((value.len() as u64).to_le_bytes()); + hasher.update(value.as_bytes()); + } + } + let digest = hasher.finalize(); Ok(u64::from_le_bytes(digest[..8].try_into().map_err( |_| Status::internal("provider env revision digest too short"), @@ -3898,6 +3918,7 @@ pub async fn build_provider_environment_snapshot( /// is resolved. struct ProviderEnvironmentTarget<'a> { spec: &'a openshell_core::proto::SandboxSpec, + environment_overrides: BTreeMap, effective_policy: ProtoSandboxPolicy, policy_source: PolicySource, policy_credential_bindings: HashMap>, @@ -3929,14 +3950,21 @@ fn provider_environment_target( &effective_policy, &policy_credential_bindings, )?; + let environment_overrides = super::provider::non_secret_provider_environment_overrides( + provider_profile_catalog, + provider_records, + sandbox, + ); let provider_env_revision = compute_provider_env_revision_from_records_and_policy_bindings( provider_profile_catalog, provider_records, &policy_credential_bindings, matches!(policy_source, PolicySource::Global).then_some(&effective_policy), + &environment_overrides, )?; Ok(ProviderEnvironmentTarget { spec, + environment_overrides, effective_policy, policy_source, policy_credential_bindings, @@ -3955,6 +3983,7 @@ pub async fn build_provider_environment_snapshot_from_inputs( let sandbox_id = inputs.sandbox.object_id().to_string(); let ProviderEnvironmentTarget { spec, + environment_overrides, effective_policy, policy_source, policy_credential_bindings, @@ -3992,6 +4021,12 @@ pub async fn build_provider_environment_snapshot_from_inputs( credential.token_grant_owners.clone_from(&owners); } } + // Retain effective non-secret values for both process environments and + // supervisor consumers such as metadata. Credential keys are excluded + // from overrides and keep their binding and placeholder precedence. + provider_environment + .environment + .extend(environment_overrides); let mut readiness_reason = provider_environment.readiness_reason; @@ -12493,6 +12528,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), }) .await @@ -12600,6 +12636,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), }) .await @@ -12659,6 +12696,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), }) .await @@ -14151,6 +14189,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), } } @@ -14433,6 +14472,222 @@ mod tests { ); } + #[tokio::test] + async fn caller_environment_wins_over_non_secret_profile_defaults_only() { + let state = test_server_state().await; + let mut profile = openshell_providers::example_profiles::load("github"); + profile.id = "declared-env".to_string(); + profile + .environment + .config + .insert("CUSTOM_PROJECT".to_string(), "project".to_string()); + profile + .environment + .config + .insert("CUSTOM_MISSING".to_string(), "missing".to_string()); + profile + .environment + .fixed + .insert("CUSTOM_MODE".to_string(), "native".to_string()); + profile + .environment + .fixed + .insert("CUSTOM_UNSET".to_string(), "default".to_string()); + state + .store + .put_message(&crate::provider_profile_sources::stored_provider_profile( + profile.to_proto(), + )) + .await + .unwrap(); + + let mut provider = test_provider("work-env", "declared-env"); + provider + .config + .insert("project".to_string(), "profile-project".to_string()); + state.store.put_message(&provider).await.unwrap(); + let mut sandbox = test_sandbox( + "sb-caller-env", + "caller-env", + ProtoSandboxPolicy::default(), + vec!["work-env".to_string()], + ); + let spec = sandbox.spec.as_mut().unwrap(); + spec.environment + .insert("CUSTOM_PROJECT".to_string(), "caller-project".to_string()); + spec.environment + .insert("GITHUB_TOKEN".to_string(), "caller-token".to_string()); + spec.environment + .insert("CUSTOM_MISSING".to_string(), " caller value ".to_string()); + spec.environment + .insert("UNRELATED".to_string(), "caller-only".to_string()); + let template = spec.template.get_or_insert_with(Default::default); + template.environment = HashMap::from([ + ("CUSTOM_PROJECT".to_string(), "template-project".to_string()), + ("CUSTOM_MODE".to_string(), String::new()), + ]); + state.store.put_message(&sandbox).await.unwrap(); + + let snapshot = load_sandbox_provider_environment(&state, &sandbox, true, true) + .await + .unwrap(); + assert_eq!( + snapshot + .environment + .get("CUSTOM_PROJECT") + .map(String::as_str), + Some("caller-project") + ); + assert_eq!( + snapshot.environment.get("CUSTOM_MODE").map(String::as_str), + Some("") + ); + assert_eq!( + snapshot + .environment + .get("CUSTOM_MISSING") + .map(String::as_str), + Some(" caller value ") + ); + assert!(!snapshot.environment.contains_key("UNRELATED")); + assert_eq!( + snapshot.environment.get("CUSTOM_UNSET").map(String::as_str), + Some("default") + ); + assert_eq!( + snapshot.environment.get("GITHUB_TOKEN").map(String::as_str), + Some("ghp-test") + ); + assert!( + snapshot + .static_credential_bindings + .contains_key("GITHUB_TOKEN") + ); + assert_eq!( + snapshot.non_secret_environment_keys, + [ + "CUSTOM_MISSING", + "CUSTOM_MODE", + "CUSTOM_PROJECT", + "CUSTOM_UNSET" + ] + ); + } + + #[tokio::test] + async fn caller_environment_revision_tracks_effective_non_secret_values() { + let state = test_server_state().await; + let mut provider = test_provider("work-gcp", "google-cloud"); + provider.credentials.clear(); + state.store.put_message(&provider).await.unwrap(); + let sandbox = test_sandbox( + "sb-caller-revision", + "caller-revision", + ProtoSandboxPolicy::default(), + vec!["work-gcp".to_string()], + ); + let mut inputs = load_sandbox_config_inputs(&state, sandbox).await.unwrap(); + let mut revisions = Vec::new(); + for (spec_value, template_value, unrelated, expected) in [ + (None, None, "one", None), + ( + None, + Some("template-project"), + "one", + Some("template-project"), + ), + ( + Some("caller-project"), + Some("template-project"), + "one", + Some("caller-project"), + ), + ( + Some("caller-project"), + Some("shadowed-project"), + "two", + Some("caller-project"), + ), + (Some(""), Some("template-project"), "one", Some("")), + ] { + let spec = inputs.sandbox.spec.as_mut().unwrap(); + spec.environment = HashMap::from([("UNRELATED".to_string(), unrelated.to_string())]); + if let Some(value) = spec_value { + spec.environment + .insert("GCP_PROJECT_ID".to_string(), value.to_string()); + } + spec.template + .get_or_insert_with(Default::default) + .environment = template_value + .map(|value| HashMap::from([("GCP_PROJECT_ID".to_string(), value.to_string())])) + .unwrap_or_default(); + let config = build_sandbox_config_snapshot_from_inputs(&state, &inputs) + .await + .unwrap(); + let snapshot = + build_provider_environment_snapshot_from_inputs(&state, &inputs, true, true) + .await + .unwrap(); + assert_eq!(config.provider_env_revision, snapshot.provider_env_revision); + let value = snapshot + .values + .iter() + .find(|value| value.name == "GCP_PROJECT_ID"); + assert_eq!(value.map(|value| value.value.as_str()), expected); + if let Some(value) = value { + assert_eq!( + value.classification, + ProviderEnvironmentValueClassification::NonSecret as i32 + ); + } + revisions.push(snapshot.provider_env_revision); + } + assert_ne!(revisions[0], revisions[1]); + assert_ne!(revisions[1], revisions[2]); + assert_eq!(revisions[2], revisions[3]); + assert_ne!(revisions[2], revisions[4]); + assert_ne!(revisions[0], revisions[4]); + } + + #[tokio::test] + async fn caller_environment_does_not_reclassify_expired_credentials_as_non_secret() { + let state = test_server_state().await; + let mut provider = test_provider("work-gcp", "google-cloud"); + provider.credentials.clear(); + state.store.put_message(&provider).await.unwrap(); + let mut legacy_provider = test_provider("legacy-project", "legacy-custom"); + legacy_provider.credentials = + HashMap::from([("GCP_PROJECT_ID".to_string(), "expired-secret".to_string())]); + legacy_provider.credential_expiration_times.insert( + "GCP_PROJECT_ID".to_string(), + openshell_core::time::timestamp_from_millis(1).unwrap(), + ); + state.store.put_message(&legacy_provider).await.unwrap(); + let mut sandbox = test_sandbox( + "sb-caller-expired", + "caller-expired", + ProtoSandboxPolicy::default(), + vec!["work-gcp".to_string(), "legacy-project".to_string()], + ); + sandbox + .spec + .as_mut() + .unwrap() + .environment + .insert("GCP_PROJECT_ID".to_string(), "caller-project".to_string()); + + let snapshot = load_sandbox_provider_environment(&state, &sandbox, true, true) + .await + .unwrap(); + assert!(!snapshot.environment.contains_key("GCP_PROJECT_ID")); + assert!( + !snapshot + .non_secret_environment_keys + .iter() + .any(|key| key == "GCP_PROJECT_ID") + ); + } + #[tokio::test] async fn provider_readiness_snapshot_uses_baseline_static_binding_contract() { let state = test_server_state().await; @@ -15610,6 +15865,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), } } @@ -15949,6 +16205,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), }], workspace_scope: Some(openshell_core::proto::workspace_selector( @@ -20321,6 +20578,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), }) .await diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 6ef3c3b15d..c318b3bf82 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -35,9 +35,10 @@ use openshell_core::telemetry::{ }; use openshell_credential_issuer::refresh::CredentialRefresh; use openshell_policy::ProviderPolicyLayer; +use openshell_providers::LEGACY_VERTEX_PRIVATE_KEY_ENV; use prost::Message; use sha2::{Digest, Sha256}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use tonic::Status; use tracing::warn; @@ -1219,7 +1220,6 @@ pub(super) async fn resolve_provider_environment_from_records_with_policy_bindin let mut expired_static_keys = HashSet::new(); let now_ms = crate::persistence::current_time_ms(); validate_provider_environment_records_unique_at(catalog, records, now_ms)?; - let registry = openshell_providers::ProviderRegistry::new(); for record in records { let name = &record.name; @@ -1253,10 +1253,10 @@ pub(super) async fn resolve_provider_environment_from_records_with_policy_bindin .as_ref() .is_none_or(|profile| provider_profile_endpoints_are_active(profile, provider)); let profile_proto = profile.as_ref().map(ProviderTypeProfile::to_proto); - let broker_only_credential_keys = profile_proto - .as_ref() - .map(broker_only_provider_credential_keys) - .unwrap_or_default(); + let broker_only_credential_keys = profile_proto.as_ref().map_or_else( + reserved_bootstrap_credential_keys, + broker_only_provider_credential_keys, + ); let profile_endpoints = profile_proto.as_ref().map(|profile| { if !profile_endpoints_are_active { return Vec::new(); @@ -1300,27 +1300,25 @@ pub(super) async fn resolve_provider_environment_from_records_with_policy_bindin let refresh_epochs = refresh_authorization_epochs_by_key(record)?; for (key, value) in &provider.credentials { - if accepted_stored_credential_keys - .as_ref() - .is_some_and(|accepted| !accepted.contains(key)) - { + if broker_only_credential_keys.contains(key) { warn!( provider_name = %name, key = %key, - "withholding provider credential not declared by resolved profile" + "skipping non-injectable provider credential" ); - readiness_reason = - openshell_core::proto::ProviderReadinessReason::CredentialsWithheld; continue; } - if is_non_injectable_provider_credential(provider, key) - || broker_only_credential_keys.contains(key) + if accepted_stored_credential_keys + .as_ref() + .is_some_and(|accepted| !accepted.contains(key)) { warn!( provider_name = %name, key = %key, - "skipping non-injectable provider credential" + "withholding provider credential not declared by resolved profile" ); + readiness_reason = + openshell_core::proto::ProviderReadinessReason::CredentialsWithheld; continue; } if is_valid_env_key(key) { @@ -1394,34 +1392,30 @@ pub(super) async fn resolve_provider_environment_from_records_with_policy_bindin // Expired handles are removed by the credential runtime before values // reach this loop. Preserve omission evidence without exposing handles. if provider.credential_handles.keys().any(|key| { - !is_non_injectable_provider_credential(provider, key) - && !broker_only_credential_keys.contains(key) - && !resolved_refs.values.contains_key(key) + !broker_only_credential_keys.contains(key) && !resolved_refs.values.contains_key(key) }) { readiness_reason = openshell_core::proto::ProviderReadinessReason::CredentialExpired; } for (key, value) in resolved_refs.values { - if accepted_stored_credential_keys - .as_ref() - .is_some_and(|accepted| !accepted.contains(&key)) - { + if broker_only_credential_keys.contains(&key) { warn!( provider_name = %name, key = %key, - "withholding provider credential handle not declared by resolved profile" + "skipping non-injectable provider credential handle" ); - readiness_reason = - openshell_core::proto::ProviderReadinessReason::CredentialsWithheld; continue; } - if is_non_injectable_provider_credential(provider, &key) - || broker_only_credential_keys.contains(&key) + if accepted_stored_credential_keys + .as_ref() + .is_some_and(|accepted| !accepted.contains(&key)) { warn!( provider_name = %name, key = %key, - "skipping non-injectable provider credential handle" + "withholding provider credential handle not declared by resolved profile" ); + readiness_reason = + openshell_core::proto::ProviderReadinessReason::CredentialsWithheld; continue; } if is_valid_env_key(&key) { @@ -1487,7 +1481,7 @@ pub(super) async fn resolve_provider_environment_from_records_with_policy_bindin // provider's earlier output cannot change how this provider classifies // or populates its own keys. Cross-provider credential/config // collisions have already been rejected by the validation above. - inject_provider_plugin_environment(catalog, provider, ®istry, &mut provider_env); + inject_provider_profile_environment(catalog, provider, &mut provider_env); for key in profile_stable_placeholder_keys { // A stable placeholder is emitted only when the binding carries // the opaque handle that binds it to the sandbox, provider, @@ -2179,7 +2173,7 @@ pub async fn validate_provider_update_against_attached_sandboxes_with_catalog( store, refresh, catalog, - workspace, + workspace, &spec.providers, Some(provider), crate::persistence::current_time_ms(), @@ -2205,7 +2199,7 @@ async fn validate_provider_environment_keys_unique_at( now_ms: i64, ) -> Result<(), Status> { let mut seen_credentials = HashMap::::new(); - let mut seen_plugin_config = HashMap::::new(); + let mut seen_profile_config = HashMap::::new(); let mut dynamic_bindings = Vec::new(); for name in provider_names { let provider = match candidate_provider { @@ -2221,10 +2215,10 @@ async fn validate_provider_environment_keys_unique_at( let provider_name = provider.object_name().to_string(); validate_provider_environment_key_ownership( &mut seen_credentials, - &mut seen_plugin_config, + &mut seen_profile_config, &provider_name, active_provider_environment_keys(refresh, catalog, &provider, now_ms).await?, - provider_plugin_environment_keys(catalog, &provider), + provider_profile_environment_values(catalog, &provider), )?; dynamic_bindings.extend(dynamic_token_grant_bindings_for_provider_with_catalog( catalog, &provider, @@ -2240,16 +2234,16 @@ fn validate_provider_environment_records_unique_at( now_ms: i64, ) -> Result<(), Status> { let mut seen_credentials = HashMap::::new(); - let mut seen_plugin_config = HashMap::::new(); + let mut seen_profile_config = HashMap::::new(); let mut dynamic_bindings = Vec::new(); for record in records { let provider = &record.provider; validate_provider_environment_key_ownership( &mut seen_credentials, - &mut seen_plugin_config, + &mut seen_profile_config, &record.name, active_keys_with_refresh_states(catalog, provider, &record.refresh_states, now_ms), - provider_plugin_environment_keys(catalog, provider), + provider_profile_environment_values(catalog, provider), )?; dynamic_bindings.extend(dynamic_token_grant_bindings_for_provider_with_catalog( catalog, provider, @@ -2259,37 +2253,87 @@ fn validate_provider_environment_records_unique_at( Ok(()) } -fn provider_plugin_environment_keys( +/// Effective caller values for declared non-secret destinations only. Reserve +/// credential keys even when their values are missing, expired, or withheld. +pub(super) fn non_secret_provider_environment_overrides( + catalog: &EffectiveProviderProfileCatalog, + records: &[ProviderEnvironmentRecord], + sandbox: &Sandbox, +) -> BTreeMap { + let Some(spec) = sandbox.spec.as_ref() else { + return BTreeMap::new(); + }; + let mut destinations = HashSet::new(); + let mut credential_keys = reserved_bootstrap_credential_keys(); + for record in records { + let provider = &record.provider; + credential_keys.extend(provider.credentials.keys().cloned()); + credential_keys.extend(provider.credential_handles.keys().cloned()); + for state in &record.refresh_states { + credential_keys.insert(state.credential_key.clone()); + credential_keys.extend(state.additional_output_keys.values().cloned()); + } + if let Some(profile) = get_provider_type_profile_for_scope( + catalog, + &provider.r#type, + &provider.profile_workspace, + ) { + destinations.extend(profile.environment.config.keys().cloned()); + destinations.extend(profile.environment.fixed.keys().cloned()); + credential_keys.extend( + profile + .credentials + .iter() + .flat_map(|credential| credential.accepted_stored_keys()) + .map(str::to_string), + ); + credential_keys.extend(broker_only_provider_credential_keys(&profile.to_proto())); + } + } + destinations + .into_iter() + .filter(|key| is_valid_env_key(key) && !credential_keys.contains(key)) + .filter_map(|key| { + let value = spec.environment.get(&key).or_else(|| { + spec.template + .as_ref() + .and_then(|template| template.environment.get(&key)) + })?; + Some((key, value.clone())) + }) + .collect() +} + +fn provider_profile_environment_values( catalog: &EffectiveProviderProfileCatalog, provider: &Provider, -) -> Vec { - let mut plugin_environment = HashMap::new(); - let registry = openshell_providers::ProviderRegistry::new(); - inject_provider_plugin_environment(catalog, provider, ®istry, &mut plugin_environment); - plugin_environment.into_keys().collect() +) -> Vec<(String, String)> { + let mut profile_environment = HashMap::new(); + inject_provider_profile_environment(catalog, provider, &mut profile_environment); + let mut values: Vec<_> = profile_environment.into_iter().collect(); + values.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + values } -fn inject_provider_plugin_environment( +fn inject_provider_profile_environment( catalog: &EffectiveProviderProfileCatalog, provider: &Provider, - registry: &openshell_providers::ProviderRegistry, environment: &mut HashMap, ) { - // A plugin activates only for a profile the gateway actually resolved. With - // no profile there is nothing to project. + // Only the resolved profile declares non-secret environment defaults. if let Some(profile) = get_provider_type_profile_for_scope(catalog, &provider.r#type, &provider.profile_workspace) { - registry.inject_env_for_profile_id(provider, &profile.id, environment); + profile.environment.inject(&provider.config, environment); } } fn validate_provider_environment_key_ownership( seen_credentials: &mut HashMap, - seen_plugin_config: &mut HashMap, + seen_profile_config: &mut HashMap, provider_name: &str, credential_keys: Vec, - plugin_config_keys: Vec, + profile_config_values: Vec<(String, String)>, ) -> Result<(), Status> { for key in credential_keys { if let Some(first_provider) = seen_credentials.get(&key) { @@ -2301,7 +2345,7 @@ fn validate_provider_environment_key_ownership( } else { seen_credentials.insert(key.clone(), provider_name.to_string()); } - if let Some(config_provider) = seen_plugin_config.get(&key) + if let Some((config_provider, _)) = seen_profile_config.get(&key) && config_provider != provider_name { return Err(provider_credential_config_key_collision( @@ -2312,7 +2356,18 @@ fn validate_provider_environment_key_ownership( } } - for key in plugin_config_keys { + for (key, value) in profile_config_values { + if let Some((config_provider, config_value)) = seen_profile_config.get(&key) + && config_provider != provider_name + && config_value != &value + { + let mut providers = [config_provider.as_str(), provider_name]; + providers.sort_unstable(); + return Err(Status::failed_precondition(format!( + "non-secret env key '{key}' is provided by both provider '{}' and provider '{}'; use provider-specific env names", + providers[0], providers[1] + ))); + } if let Some(credential_provider) = seen_credentials.get(&key) && credential_provider != provider_name { @@ -2322,9 +2377,9 @@ fn validate_provider_environment_key_ownership( provider_name, )); } - seen_plugin_config + seen_profile_config .entry(key) - .or_insert_with(|| provider_name.to_string()); + .or_insert_with(|| (provider_name.to_string(), value)); } Ok(()) } @@ -2568,7 +2623,6 @@ fn active_provider_credential_keys( let mut keys: Vec = provider .credentials .keys() - .filter(|key| !is_non_injectable_provider_credential(provider, key)) .filter(|key| !broker_only_credential_keys.contains(*key)) .filter(|key| is_valid_env_key(key)) .filter(|key| provider_credential_not_expired(provider, key, now_ms)) @@ -2578,7 +2632,6 @@ fn active_provider_credential_keys( provider .credential_handles .keys() - .filter(|key| !is_non_injectable_provider_credential(provider, key)) .filter(|key| !broker_only_credential_keys.contains(*key)) .filter(|key| is_valid_env_key(key)) .filter(|key| provider_credential_not_expired(provider, key, now_ms)) @@ -2594,12 +2647,13 @@ fn broker_only_provider_credential_keys_for_provider( get_provider_type_profile_for_scope(catalog, &provider.r#type, &provider.profile_workspace) .as_ref() .map(ProviderTypeProfile::to_proto) - .map(|profile| broker_only_provider_credential_keys(&profile)) - .unwrap_or_default() + .map_or_else(reserved_bootstrap_credential_keys, |profile| { + broker_only_provider_credential_keys(&profile) + }) } pub(super) fn broker_only_provider_credential_keys(profile: &ProviderProfile) -> HashSet { - profile + let mut keys: HashSet<_> = profile .credentials .iter() .filter_map(|credential| credential.token_grant.as_ref()) @@ -2614,7 +2668,13 @@ pub(super) fn broker_only_provider_credential_keys(profile: &ProviderProfile) -> let key = subject_token.credential.trim(); (!key.is_empty()).then(|| key.to_string()) }) - .collect() + .collect(); + keys.insert(LEGACY_VERTEX_PRIVATE_KEY_ENV.to_string()); + keys +} + +fn reserved_bootstrap_credential_keys() -> HashSet { + HashSet::from([LEGACY_VERTEX_PRIVATE_KEY_ENV.to_string()]) } fn provider_credential_not_expired(provider: &Provider, key: &str, now_ms: i64) -> bool { @@ -3100,6 +3160,7 @@ pub(super) async fn handle_import_provider_profiles( profile_conflict_diagnostics(state.store.as_ref(), &catalog, &workspace, &profiles).await?, ); diagnostics.extend(validate_profile_set(&profiles)); + diagnostics.extend(profile_legacy_credential_diagnostics(&profiles)); if !has_errors(&diagnostics) { diagnostics.extend( profile_attached_sandbox_diagnostics( @@ -3204,6 +3265,7 @@ pub(super) async fn handle_update_provider_profiles( .await?, ); diagnostics.extend(validate_profile_set(&profiles)); + diagnostics.extend(profile_legacy_credential_diagnostics(&profiles)); let expected_resource_version = if request.expected_resource_version != 0 { Some(request.expected_resource_version) } else { @@ -3325,6 +3387,7 @@ pub(super) async fn handle_lint_provider_profiles( profile_conflict_diagnostics(state.store.as_ref(), &catalog, &workspace, &profiles).await?, ); diagnostics.extend(validate_profile_set(&profiles)); + diagnostics.extend(profile_legacy_credential_diagnostics(&profiles)); let valid = !has_errors(&diagnostics); Ok(Response::new(LintProviderProfilesResponse { @@ -3742,6 +3805,28 @@ fn normalize_profile_id_request(id: &str) -> Result { }) } +fn profile_legacy_credential_diagnostics( + profiles: &[(String, ProviderTypeProfile)], +) -> Vec { + profiles + .iter() + .filter(|(_, profile)| { + profile.credentials.iter().any(|credential| { + credential + .accepted_stored_keys() + .contains(&LEGACY_VERTEX_PRIVATE_KEY_ENV) + }) + }) + .map(|(source, profile)| ProfileValidationDiagnostic { + source: source.clone(), + profile_id: profile.id.clone(), + field: "credentials.env_vars".to_string(), + message: "GOOGLE_SERVICE_ACCOUNT_KEY contains legacy service-account private key material and cannot be declared as a provider credential; configure refresh material instead".to_string(), + severity: "error".to_string(), + }) + .collect() +} + fn profiles_from_import_items( items: &[ProviderProfileImportItem], ) -> ( @@ -4010,7 +4095,7 @@ async fn profile_attached_sandbox_diagnostics( let has_static_credentials = provider .credentials .keys() - .any(|key| !is_non_injectable_provider_credential(&provider, key)); + .any(|key| profile.credential_env_vars().contains(&key.as_str())); let has_usable_endpoint = profile.to_proto().endpoints.iter().any(|endpoint| { !endpoint_ports(endpoint.port, &endpoint.ports).is_empty() && !endpoint.host.trim().is_empty() @@ -5646,6 +5731,14 @@ mod tests { .await .expect("store example provider profile"); } + // Binding tests need generic non-secret config, without a metadata + // service. Use an explicit data-only fork for those scenarios. + let mut cloud_config = openshell_providers::example_profiles::load("google-cloud"); + cloud_config.id = "cloud-env-fixture".to_string(); + store + .put_message(&stored_provider_profile(cloud_config.to_proto())) + .await + .unwrap(); store } use openshell_core::proto::{ @@ -6167,6 +6260,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }; let mut dynamic_creds = HashMap::new(); @@ -6309,6 +6403,74 @@ mod tests { })); } + #[tokio::test] + async fn import_rejects_legacy_vertex_private_key_declaration() { + let state = test_server_state_without_provider_profiles().await; + let mut profile = openshell_providers::example_profiles::load("google-vertex-ai"); + profile.id = "legacy-vertex-export".into(); + let mut legacy_credential = profile.credentials[0].clone(); + legacy_credential.name = "legacy_service_account_key".into(); + legacy_credential.env_vars = vec![LEGACY_VERTEX_PRIVATE_KEY_ENV.into()]; + legacy_credential.refresh = None; + profile.credentials.push(legacy_credential); + + let response = handle_import_provider_profiles( + &state, + authed_request(ImportProviderProfilesRequest { + profiles: vec![ProviderProfileImportItem { + profile: Some(profile.to_proto()), + source: "legacy.yaml".into(), + }], + ..Default::default() + }), + ) + .await + .unwrap() + .into_inner(); + assert!(!response.imported); + assert!(response.diagnostics.iter().any(|diagnostic| { + diagnostic.field == "credentials.env_vars" + && diagnostic.message.contains(LEGACY_VERTEX_PRIVATE_KEY_ENV) + })); + } + + #[tokio::test] + async fn import_rejects_legacy_vertex_private_key_environment_defaults() { + for config_default in [false, true] { + let state = test_server_state_without_provider_profiles().await; + let mut profile = openshell_providers::example_profiles::load("google-vertex-ai"); + if config_default { + profile + .environment + .config + .insert(LEGACY_VERTEX_PRIVATE_KEY_ENV.into(), "private_key".into()); + } else { + profile + .environment + .fixed + .insert(LEGACY_VERTEX_PRIVATE_KEY_ENV.into(), "private-key".into()); + } + let response = handle_import_provider_profiles( + &state, + authed_request(ImportProviderProfilesRequest { + profiles: vec![ProviderProfileImportItem { + profile: Some(profile.to_proto()), + source: "legacy.yaml".into(), + }], + ..Default::default() + }), + ) + .await + .unwrap() + .into_inner(); + assert!(!response.imported); + assert!(response.diagnostics.iter().any(|diagnostic| { + diagnostic.field == "environment" + && diagnostic.message.contains(LEGACY_VERTEX_PRIVATE_KEY_ENV) + })); + } + } + async fn import_token_grant_profile( state: &Arc, id: &str, @@ -7243,6 +7405,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() } } @@ -8095,6 +8258,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), source: "advanced-api.yaml".to_string(), }], @@ -11759,6 +11923,7 @@ mod tests { discovery: None, source: String::new(), scope: String::new(), + ..Default::default() }), source: "delegated-refresh-api.yaml".to_string(), }], @@ -13814,7 +13979,7 @@ mod tests { } #[tokio::test] - async fn provider_environment_rejects_plugin_config_credential_collision_in_both_orders() { + async fn provider_environment_rejects_profile_config_credential_collision_in_both_orders() { let store = test_store().await; create_provider_record( &store, @@ -13830,7 +13995,7 @@ mod tests { workspace: "default".to_string(), deletion_time: None, }), - r#type: "google-cloud".to_string(), + r#type: "cloud-env-fixture".to_string(), credentials: std::iter::once(( "GCP_ACCESS_TOKEN".to_string(), "google-token".to_string(), @@ -13903,134 +14068,298 @@ mod tests { } #[tokio::test] - async fn resolve_provider_env_injects_vertex_agent_config() { + async fn provider_environment_rejects_non_secret_overlap_in_both_orders() { let store = test_store().await; - create_provider_record( - &store, - "default", - Provider { - metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { - id: String::new(), - name: "vertex-local".to_string(), - created_time: None, - labels: HashMap::new(), - resource_version: 0, - annotations: HashMap::new(), - workspace: "default".to_string(), - deletion_time: None, - }), - r#type: "google-vertex-ai".to_string(), - credentials: std::iter::once(( - "GOOGLE_VERTEX_AI_TOKEN".to_string(), - "ya29.token".to_string(), - )) - .collect(), - config: [ - ( - "VERTEX_AI_PROJECT_ID".to_string(), - "my-gcp-project".to_string(), - ), - ("VERTEX_AI_REGION".to_string(), "us-central1".to_string()), - ] - .into_iter() - .collect(), - credential_expiration_times: HashMap::new(), - profile_workspace: "default".to_string(), - credential_handles: HashMap::new(), - }, - ) - .await - .unwrap(); - - let result = resolve_provider_environment(&store, "default", &["vertex-local".to_string()]) + let mut project_profile = openshell_providers::example_profiles::load("google-cloud"); + project_profile.id = "project-env-fixture".into(); + project_profile.environment.fixed.clear(); + project_profile + .environment + .config + .retain(|key, _| key == "GCP_PROJECT_ID"); + store + .put_message(&stored_provider_profile(project_profile.to_proto())) + .await + .unwrap(); + for (name, project) in [("project-a", "first"), ("project-b", "second")] { + create_provider_record( + &store, + "default", + Provider { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: String::new(), + name: name.to_string(), + created_time: None, + labels: HashMap::new(), + resource_version: 0, + annotations: HashMap::new(), + workspace: "default".to_string(), + deletion_time: None, + }), + r#type: "project-env-fixture".to_string(), + credentials: HashMap::new(), + config: HashMap::from([("project_id".to_string(), project.to_string())]), + credential_expiration_times: HashMap::new(), + profile_workspace: "default".to_string(), + credential_handles: HashMap::new(), + }, + ) .await .unwrap(); + } - // Credential still injected. - assert_eq!( - result.get("GOOGLE_VERTEX_AI_TOKEN"), - Some(&"ya29.token".to_string()) - ); - // Static flags. - assert!(!result.contains_key("CLAUDE_CODE_USE_VERTEX")); - assert_eq!( - result.get("GOOSE_PROVIDER"), - Some(&"gcp_vertex_ai".to_string()) - ); - // Project ID derived vars. - assert_eq!( - result.get("ANTHROPIC_VERTEX_PROJECT_ID"), - Some(&"my-gcp-project".to_string()) - ); - assert_eq!( - result.get("GCP_PROJECT_ID"), - Some(&"my-gcp-project".to_string()) - ); - assert_eq!( - result.get("GOOGLE_CLOUD_PROJECT"), - Some(&"my-gcp-project".to_string()) - ); - // Region derived vars. - assert_eq!( - result.get("CLOUD_ML_REGION"), - Some(&"us-central1".to_string()) - ); - assert_eq!(result.get("GCP_LOCATION"), Some(&"us-central1".to_string())); - assert_eq!( - result.get("VERTEX_LOCATION"), - Some(&"us-central1".to_string()) - ); + let mut messages = Vec::new(); + for providers in [ + vec!["project-a".to_string(), "project-b".to_string()], + vec!["project-b".to_string(), "project-a".to_string()], + ] { + let validation_error = + validate_provider_environment_keys_unique(&store, "default", &providers) + .await + .unwrap_err(); + let resolution_error = resolve_provider_environment(&store, "default", &providers) + .await + .unwrap_err(); + assert_eq!(validation_error.code(), Code::FailedPrecondition); + assert_eq!(validation_error.message(), resolution_error.message()); + assert!(validation_error.message().contains("GCP_PROJECT_ID")); + assert!(validation_error.message().contains("project-a")); + assert!(validation_error.message().contains("project-b")); + messages.push(validation_error.message().to_string()); + } + assert_eq!(messages[0], messages[1]); } #[tokio::test] - async fn resolve_provider_env_vertex_never_injects_service_account_key() { + async fn provider_environment_allows_identical_vertex_defaults_from_two_providers() { let store = test_store().await; - create_provider_record( - &store, - "default", - Provider { - metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { - id: String::new(), - name: "vertex-bootstrap".to_string(), - created_time: None, - labels: HashMap::new(), - resource_version: 0, - annotations: HashMap::new(), - workspace: "default".to_string(), - deletion_time: None, - }), - r#type: "google-vertex-ai".to_string(), - credentials: [ - ( - "GOOGLE_SERVICE_ACCOUNT_KEY".to_string(), - r#"{"type":"service_account","private_key":"secret"}"#.to_string(), - ), - ( - "GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN".to_string(), - "ya29.short-lived".to_string(), - ), - ] - .into_iter() - .collect(), - config: HashMap::new(), - credential_expiration_times: HashMap::new(), - profile_workspace: "default".to_string(), - credential_handles: HashMap::new(), - }, - ) - .await - .unwrap(); + let profile = openshell_providers::example_profiles::load("google-vertex-ai"); + store + .put_message(&stored_provider_profile(profile.to_proto())) + .await + .unwrap(); + for (name, credential_key, token) in [ + ("vertex-a", "GOOGLE_VERTEX_AI_TOKEN", "token-a"), + ( + "vertex-b", + "GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN", + "token-b", + ), + ] { + let mut provider = + provider_with_credential_value(name, "google-vertex-ai", credential_key, token); + provider.config = HashMap::from([ + ("VERTEX_AI_PROJECT_ID".into(), "shared-project".into()), + ("VERTEX_AI_REGION".into(), "us-central1".into()), + ]); + create_provider_record(&store, "default", provider) + .await + .unwrap(); + } - let result = - resolve_provider_environment(&store, "default", &["vertex-bootstrap".to_string()]) + for names in [ + ["vertex-a".to_string(), "vertex-b".to_string()], + ["vertex-b".to_string(), "vertex-a".to_string()], + ] { + validate_provider_environment_keys_unique(&store, "default", &names) + .await + .unwrap(); + let result = resolve_provider_environment(&store, "default", &names) .await .unwrap(); + assert_eq!(result.get("GOOSE_PROVIDER"), Some(&"gcp_vertex_ai".into())); + assert_eq!(result.get("GCP_PROJECT_ID"), Some(&"shared-project".into())); + assert_eq!( + result.get("GOOGLE_VERTEX_AI_TOKEN"), + Some(&"token-a".into()) + ); + assert_eq!( + result.get("GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN"), + Some(&"token-b".into()) + ); + } + } - assert!(!result.contains_key("GOOGLE_SERVICE_ACCOUNT_KEY")); - assert_eq!( - result.get("GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN"), - Some(&"ya29.short-lived".to_string()) - ); + #[tokio::test] + async fn resolve_provider_env_injects_vertex_agent_config() { + for profile_id in ["google-vertex-ai", "acme-vertex"] { + let store = test_store().await; + let mut fork = openshell_providers::example_profiles::load("google-vertex-ai"); + fork.id = "acme-vertex".to_string(); + store + .put_message(&stored_provider_profile(fork.to_proto())) + .await + .unwrap(); + create_provider_record( + &store, + "default", + Provider { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: String::new(), + name: "vertex-local".to_string(), + created_time: None, + labels: HashMap::new(), + resource_version: 0, + annotations: HashMap::new(), + workspace: "default".to_string(), + deletion_time: None, + }), + r#type: profile_id.to_string(), + credentials: std::iter::once(( + "GOOGLE_VERTEX_AI_TOKEN".to_string(), + "ya29.token".to_string(), + )) + .collect(), + config: [ + ( + "VERTEX_AI_PROJECT_ID".to_string(), + "my-gcp-project".to_string(), + ), + ("VERTEX_AI_REGION".to_string(), "us-central1".to_string()), + ] + .into_iter() + .collect(), + credential_expiration_times: HashMap::new(), + profile_workspace: "default".to_string(), + credential_handles: HashMap::new(), + }, + ) + .await + .unwrap(); + + let result = + resolve_provider_environment(&store, "default", &["vertex-local".to_string()]) + .await + .unwrap(); + + // Credential still injected. + assert_eq!( + result.get("GOOGLE_VERTEX_AI_TOKEN"), + Some(&"ya29.token".to_string()) + ); + // Static flags. + assert!(!result.contains_key("CLAUDE_CODE_USE_VERTEX")); + assert_eq!( + result.get("GOOSE_PROVIDER"), + Some(&"gcp_vertex_ai".to_string()) + ); + // Project ID derived vars. + assert_eq!( + result.get("ANTHROPIC_VERTEX_PROJECT_ID"), + Some(&"my-gcp-project".to_string()) + ); + assert_eq!( + result.get("GCP_PROJECT_ID"), + Some(&"my-gcp-project".to_string()) + ); + assert_eq!( + result.get("GOOGLE_CLOUD_PROJECT"), + Some(&"my-gcp-project".to_string()) + ); + // Region derived vars. + assert_eq!( + result.get("CLOUD_ML_REGION"), + Some(&"us-central1".to_string()) + ); + assert_eq!(result.get("GCP_LOCATION"), Some(&"us-central1".to_string())); + assert_eq!( + result.get("VERTEX_LOCATION"), + Some(&"us-central1".to_string()) + ); + } + } + + #[tokio::test] + async fn resolve_provider_env_vertex_never_injects_service_account_key() { + for (profile_id, declares_legacy_key) in + [("google-vertex-ai", false), ("acme-vertex", true)] + { + let store = test_store().await; + let mut fork = openshell_providers::example_profiles::load("google-vertex-ai"); + fork.id = profile_id.to_string(); + if declares_legacy_key { + let mut legacy_credential = fork.credentials[0].clone(); + legacy_credential.name = "legacy_service_account_key".into(); + legacy_credential.env_vars = vec![LEGACY_VERTEX_PRIVATE_KEY_ENV.into()]; + legacy_credential.refresh = None; + fork.credentials.push(legacy_credential); + } + store + .put_message(&stored_provider_profile(fork.to_proto())) + .await + .unwrap(); + create_provider_record( + &store, + "default", + Provider { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: String::new(), + name: "vertex-bootstrap".to_string(), + created_time: None, + labels: HashMap::new(), + resource_version: 0, + annotations: HashMap::new(), + workspace: "default".to_string(), + deletion_time: None, + }), + r#type: profile_id.to_string(), + credentials: [ + ( + "GOOGLE_SERVICE_ACCOUNT_KEY".to_string(), + r#"{"type":"service_account","private_key":"secret"}"#.to_string(), + ), + ( + "GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN".to_string(), + "ya29.short-lived".to_string(), + ), + ] + .into_iter() + .collect(), + config: HashMap::from([ + ("VERTEX_AI_PROJECT_ID".to_string(), "my-project".to_string()), + ("VERTEX_AI_REGION".to_string(), "us-central1".to_string()), + ]), + credential_expiration_times: HashMap::new(), + profile_workspace: "default".to_string(), + credential_handles: HashMap::new(), + }, + ) + .await + .unwrap(); + + let result = + resolve_provider_environment(&store, "default", &["vertex-bootstrap".to_string()]) + .await + .unwrap(); + + assert!(!result.contains_key("GOOGLE_SERVICE_ACCOUNT_KEY")); + assert!( + !result + .static_credential_bindings + .contains_key("GOOGLE_SERVICE_ACCOUNT_KEY") + ); + assert!( + !result + .static_credential_keys + .contains("GOOGLE_SERVICE_ACCOUNT_KEY") + ); + assert_eq!( + result.get("GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN"), + Some(&"ya29.short-lived".to_string()) + ); + assert_eq!( + result.readiness_reason, + openshell_core::proto::ProviderReadinessReason::Unspecified + ); + assert_eq!( + result.get("GCP_PROJECT_ID"), + Some(&"my-project".to_string()) + ); + assert_eq!( + result.get("CLOUD_ML_REGION"), + Some(&"us-central1".to_string()) + ); + } } #[tokio::test] @@ -14300,7 +14629,7 @@ mod tests { } #[tokio::test] - async fn update_provider_rejects_plugin_config_credential_collision() { + async fn update_provider_rejects_profile_config_credential_collision() { let store = test_store().await; create_provider_record( &store, @@ -14316,7 +14645,7 @@ mod tests { workspace: "default".to_string(), deletion_time: None, }), - r#type: "google-cloud".to_string(), + r#type: "cloud-env-fixture".to_string(), credentials: std::iter::once(( "GCP_ACCESS_TOKEN".to_string(), "google-token".to_string(), @@ -15954,7 +16283,7 @@ mod tests { workspace: "default".to_string(), deletion_time: None, }), - r#type: "google-cloud".to_string(), + r#type: "cloud-env-fixture".to_string(), credentials: HashMap::new(), config, credential_expiration_times: HashMap::new(), @@ -15963,143 +16292,6 @@ mod tests { } } - #[test] - fn inject_gcp_env_sets_metadata_host() { - use openshell_core::google_cloud; - let provider = google_cloud_provider(HashMap::new()); - let mut env = HashMap::new(); - openshell_providers::ProviderRegistry::new().inject_env_for_profile_id( - &provider, - &provider.r#type, - &mut env, - ); - assert_eq!( - env.get("GCE_METADATA_HOST").map(String::as_str), - Some(google_cloud::METADATA_HOST), - ); - assert!( - !env.contains_key("CLAUDE_CODE_USE_VERTEX"), - "CLAUDE_CODE_USE_VERTEX is synthetic, should not be injected here" - ); - } - - #[test] - fn inject_gcp_env_propagates_project_id() { - use openshell_core::google_cloud; - let provider = google_cloud_provider(HashMap::from([( - "project_id".to_string(), - "my-project".to_string(), - )])); - let mut env = HashMap::new(); - openshell_providers::ProviderRegistry::new().inject_env_for_profile_id( - &provider, - &provider.r#type, - &mut env, - ); - for var in google_cloud::PROJECT_ID_ENV_VARS { - assert_eq!( - env.get(*var).map(String::as_str), - Some("my-project"), - "{var} should be set to project_id config value" - ); - } - } - - #[test] - fn inject_gcp_env_propagates_region() { - use openshell_core::google_cloud; - let provider = google_cloud_provider(HashMap::from([( - "region".to_string(), - "us-central1".to_string(), - )])); - let mut env = HashMap::new(); - openshell_providers::ProviderRegistry::new().inject_env_for_profile_id( - &provider, - &provider.r#type, - &mut env, - ); - for var in google_cloud::REGION_ENV_VARS { - assert_eq!( - env.get(*var).map(String::as_str), - Some("us-central1"), - "{var} should be set to region config value" - ); - } - } - - #[test] - fn inject_gcp_env_propagates_service_account_email() { - use openshell_core::google_cloud; - let provider = google_cloud_provider(HashMap::from([( - "service_account_email".to_string(), - "sa@proj.iam.gserviceaccount.com".to_string(), - )])); - let mut env = HashMap::new(); - openshell_providers::ProviderRegistry::new().inject_env_for_profile_id( - &provider, - &provider.r#type, - &mut env, - ); - for var in google_cloud::SERVICE_ACCOUNT_EMAIL_ENV_VARS { - assert_eq!( - env.get(*var).map(String::as_str), - Some("sa@proj.iam.gserviceaccount.com"), - "{var} should be set to service_account_email config value" - ); - } - } - - #[test] - fn inject_gcp_env_does_not_overwrite_existing_values() { - let provider = google_cloud_provider(HashMap::from([( - "project_id".to_string(), - "from-config".to_string(), - )])); - let mut env = HashMap::from([("GCP_PROJECT_ID".to_string(), "user-override".to_string())]); - openshell_providers::ProviderRegistry::new().inject_env_for_profile_id( - &provider, - &provider.r#type, - &mut env, - ); - assert_eq!( - env.get("GCP_PROJECT_ID").map(String::as_str), - Some("user-override"), - "user-provided value should not be overwritten" - ); - } - - #[test] - fn inject_non_gcp_provider_does_nothing() { - let provider = Provider { - metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { - id: String::new(), - name: "github".to_string(), - created_time: None, - labels: HashMap::new(), - resource_version: 0, - annotations: HashMap::new(), - workspace: "default".to_string(), - deletion_time: None, - }), - r#type: "github".to_string(), - credentials: HashMap::new(), - config: HashMap::from([("project_id".to_string(), "should-be-ignored".to_string())]), - credential_expiration_times: HashMap::new(), - profile_workspace: "default".to_string(), - credential_handles: HashMap::new(), - }; - let mut env = HashMap::new(); - openshell_providers::ProviderRegistry::new().inject_env_for_profile_id( - &provider, - &provider.r#type, - &mut env, - ); - assert!( - env.is_empty(), - "non-GCP provider should not inject any env vars" - ); - } - #[tokio::test] async fn provider_crud_is_workspace_isolated() { use openshell_core::proto::{ diff --git a/crates/openshell-server/src/provider_profile_sources.rs b/crates/openshell-server/src/provider_profile_sources.rs index 77230e5cce..fef2dc9887 100644 --- a/crates/openshell-server/src/provider_profile_sources.rs +++ b/crates/openshell-server/src/provider_profile_sources.rs @@ -1191,6 +1191,51 @@ mod tests { .expect("canonical provider profile MCP options") } + #[test] + fn environment_profile_fingerprints_survive_storage_round_trips() { + let profile = openshell_providers::example_profiles::load("google-vertex-ai").to_proto(); + let encoded = profile.encode_to_vec(); + let revision = profile_snapshot_revision(std::slice::from_ref(&profile)); + let fingerprint = |profile| { + let catalog = build_effective_profiles(vec![CollectedProviderProfileSnapshot { + source_id: "external/test".to_string(), + revision: "same-revision".to_string(), + profiles: vec![ScopedSnapshotProfile { + scope: ProfileScope::Static, + profile, + }], + user_managed: false, + allow_empty: false, + }]) + .expect("valid environment profile"); + let mut hash = Sha256::new(); + catalog.hash_type_profile_revision_for_scope("google-vertex-ai", "", &mut hash); + hash.finalize() + }; + let original_fingerprint = fingerprint(profile.clone()); + for _ in 0..16 { + let decoded = ProviderProfile::decode(encoded.as_slice()).unwrap(); + assert_eq!( + profile_snapshot_revision(std::slice::from_ref(&decoded)), + revision + ); + assert_eq!(fingerprint(decoded), original_fingerprint); + } + + let mut changed = profile; + changed + .environment + .as_mut() + .unwrap() + .fixed + .insert("GOOSE_PROVIDER".to_string(), "custom_vertex".to_string()); + assert_ne!( + profile_snapshot_revision(std::slice::from_ref(&changed)), + revision + ); + assert_ne!(fingerprint(changed), original_fingerprint); + } + #[test] fn equivalent_mcp_version_order_produces_identical_source_profile_fingerprints() { let catalog = |versions: &[&str]| { diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index d8da420871..3419a8961e 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -126,12 +126,15 @@ mod tests { // Restart policy is stored in SandboxSpec, and the count and well-known // timestamps are stored in SandboxStatus. Legacy payloads decode with // Unspecified (treated as Never), zero count, and absent timestamps. - // ProviderProfileFile is reachable from stored provider profiles. Its - // additive declaration changes the durable and public/durable overlap - // inventories; the provider-environment file map and peer configuration - // notification are public-only. The - // request has no provider-file capability field: older supervisors ignore - // the additive file map while retaining the rest of the response. + // ProviderProfileFile and ProviderProfileEnvironment are reachable from + // stored provider profiles. Their additive declarations change the public, + // durable, and overlap inventories while preserving existing field tags and + // legacy payload decoding. The provider-environment file map and peer + // configuration notification are public-only. Older supervisors ignore the + // additive file map while retaining the rest of the response. + // ProviderProfile's adapter requirement is removed with its field number + // and name reserved. Prior payloads ignore that field while preserving + // environment declarations at field 15; storage.v1 remains unchanged. // Preparation timing adds two optional timestamps to SandboxProvisioning. // Existing rows decode with both absent and retain their active deadline; // no stored attempt gains another phase or time budget on upgrade. @@ -178,11 +181,11 @@ mod tests { // image wire identity, now with explicit presence. Compute configuration // updates use a separate capability/service and add one classified RPC. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "3b4c2af6cf22e7ed1d84279ff865acb0db7199717bee9bc041caef791334104d"; + "82bafb2d6761f931bd61d779c138bd87029d1616a952671eccce92a15e99d21b"; const DURABLE_SCHEMA_SHA256: &str = - "cb15f8f79a855064a43fe8c6cd908bf07eca6604e2bf8ab7d2de0ef33e97da4e"; + "0428468afb6a08ed0fb2d8c66dbc3c601771c3744f1fd4016b055d5c1a546a6f"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = - "c0c3ea17f105aaa6d843c466fd4240451c2105dce7675384935d4aaecdac8abf"; + "324526d75a8d523328a4c902f867400105826e6f540123f88a3ba1bbe8925958"; // Encoded with the schema that omitted stable_placeholder. Keep these // bytes fixed so the current encoder cannot hide a compatibility change. const LEGACY_STATIC_PROFILE: &str = "0a240a116c65676163792d70726f66696c652d6964120d6c65676163792d737461746963280712510a0d6c65676163792d73746174696312184c6567616379207374617469632063726564656e7469616c2a260a076170695f6b65791a1153594e5448455449435f4150495f4b455920012a06626561726572"; @@ -723,9 +726,9 @@ mod tests { overlap_hash.as_str(), ), ( - (337, 31), - (100, 22), - (86, 22), + (340, 31), + (103, 22), + (89, 22), PUBLIC_RPC_SCHEMA_SHA256, DURABLE_SCHEMA_SHA256, PUBLIC_DURABLE_OVERLAP_SHA256 @@ -867,6 +870,7 @@ mod tests { let profile = profile.profile.expect("profile"); assert_eq!(profile.id, "profile"); assert_eq!(profile.display_name, "Legacy"); + assert!(profile.environment.is_none()); let policy_payload = PolicyRevisionPayload::decode(legacy_bytes(V0_0_116_POLICY_PAYLOAD).as_slice()) diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 02736ad184..157c60b359 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -1997,7 +1997,7 @@ mod tests { static_provider_environment(1, Some("initial")), &readiness, ); - let (revision, child_env) = state.child_env_snapshot_with_gcp_resolved().unwrap(); + let (revision, child_env) = state.child_env_snapshot_with_non_secret_resolved().unwrap(); let reference = &child_env["EXTERNAL_TOKEN"]; assert_eq!(revision, 1); assert_eq!(reference, "openshell:resolve:env:v1_EXTERNAL_TOKEN"); @@ -2067,7 +2067,7 @@ mod tests { ); assert!(state.requires_stable_placeholders()); assert!(readiness.observation(&state).credentials_installed); - let (revision, child_env) = state.child_env_snapshot_with_gcp_resolved().unwrap(); + let (revision, child_env) = state.child_env_snapshot_with_non_secret_resolved().unwrap(); assert_eq!(prepared.snapshot().child_env, child_env); let reference = &child_env["EXTERNAL_TOKEN"]; assert_eq!(revision, 1); diff --git a/docs/how-it-works/providers/google.mdx b/docs/how-it-works/providers/google.mdx index 6d5b0c1256..6c937e9fef 100644 --- a/docs/how-it-works/providers/google.mdx +++ b/docs/how-it-works/providers/google.mdx @@ -7,37 +7,33 @@ description: "Authenticate with Google Cloud APIs and Vertex AI inside OpenShell keywords: "Generative AI, Google Cloud, Vertex AI, GCP, OAuth2, Credentials, Sandbox" --- -The `google-cloud` provider gives sandboxes native GCP credentials so -any Google Cloud SDK works out of the box — Cloud Storage, BigQuery, Drive, -Maps, Discovery Engine, or any other GCP API. A GCE metadata server emulator -on loopback provides credential placeholders that the -sandbox proxy resolves to real tokens at request time. The sandbox process -never holds a real GCP credential. +Google provider profiles declare credential refresh, environment defaults, and +local discovery behavior. The Google Cloud profile supports SDK metadata +discovery; the Vertex AI profile supports native API requests with +gateway-refreshed bearer tokens. -## Quick Start +## Google Cloud Metadata -Import the `google-cloud` profile first; a gateway serves only the profiles you -imported: +The `google-cloud` example requires a Linux sandbox using Docker, Podman, +Kubernetes, or VM. The sandbox relays metadata requests to the supervisor. +Windows/MXC does not provide this metadata service. Verify the sandbox runtime +before using the profile. Import and attachment do not check runtime support; +on an unsupported runtime, SDK metadata discovery fails when the client tries +to use it. Native Vertex bearer-token authentication does not require this +service. -```shell -openshell profile import --url https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/google-cloud.yaml --global -``` - -If you already have `gcloud` configured with Application Default Credentials, -create a provider with automatic credential refresh in one command: +Import the profile before creating a provider. This example has no API +endpoints, so bind the provider to each Google API host in sandbox policy as +shown below. ```shell -openshell provider create \ - --name my-gcp \ - --type google-cloud \ - --from-gcloud-adc \ - --config project_id="$(gcloud config get-value project)" \ - --config region=global +curl -fsSLo google-cloud.yaml https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/google-cloud.yaml +openshell profile import -f google-cloud.yaml --global ``` -`--from-gcloud-adc` reads your ADC file, configures OAuth2 refresh on the -gateway, and mints the first access token before the command returns. The -gateway rotates the token automatically — no manual refresh needed. +The profile's `environment` declarations provide the non-secret metadata and +project values listed below. A copy imported under another ID keeps those +declarations and still requires a supported sandbox runtime. ## Authentication Flows @@ -52,6 +48,7 @@ exchanges the refresh token for short-lived access tokens automatically. openshell provider create \ --name my-gcp \ --type google-cloud \ + --global-profile \ --config project_id=my-project \ --config region=us-central1 \ --credential GCP_ADC_ACCESS_TOKEN=placeholder @@ -90,6 +87,7 @@ strategy. openshell provider create \ --name my-gcp \ --type google-cloud \ + --global-profile \ --config project_id=my-project \ --config region=us-central1 \ --credential GCP_SA_ACCESS_TOKEN=placeholder @@ -174,6 +172,13 @@ as placeholders for proxy-time resolution. | `CLOUD_ML_REGION` | from `region` config | GCP region | | `GCP_LOCATION` | from `region` config | Alternative region var | +For each non-secret variable, a sandbox create-time value overrides its template +value, which overrides the provider default. The metadata project-ID endpoint +uses the effective `GCP_PROJECT_ID`, including an explicitly empty value. A caller +can supply it even when the provider has no `project_id` config. OpenShell resolves +aliases independently; set both `GCP_PROJECT_ID` and `GOOGLE_CLOUD_PROJECT` when +overriding the project so SDKs that read different variables agree. + ## Using with GCP APIs The metadata emulator serves tokens with the `cloud-platform` OAuth2 scope, @@ -223,6 +228,14 @@ endpoints by default. You must add endpoint rules to your sandbox policy for each GCP API the sandbox needs to reach. See "Using with GCP APIs" above for an example. +For a client that accepts bearer-token authentication directly, you can author +a [provider profile](/how-it-works/providers/profiles) with the needed Google +API endpoints, client `binaries`, project and region defaults, and an ADC or +service-account refresh strategy. Configure refresh on the provider and send +the token placeholder in an `Authorization: Bearer` header. If the profile has +no endpoints, bind the attached provider to each allowed host in the +[sandbox policy](/how-it-works/providers/profiles#understand-static-credential-endpoint-binding). + ## Vertex AI The `google-vertex-ai` provider gives selected sandboxes access to native @@ -242,21 +255,42 @@ native Vertex endpoint and request format for its selected model. ### Create a Vertex AI Provider -Import the `google-vertex-ai` profile first: +Copy the example profile and set its top-level `binaries` to the absolute paths +of the clients that will call Vertex AI from your sandbox image. The example +declares no binaries, so importing it unchanged does not authorize a client. +For a native Claude Code installation, use the resolved executable path inside +the image; `command -v claude` may return a symlink to a versioned binary. For +an npm or other script installation, use the resolved Node.js interpreter path +instead. OpenShell authorizes the executable and its trusted executable +ancestors, not the script path from the process command line. Include the path +to `curl` if you plan to use the native API example below. ```shell -openshell profile import --url https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/google-vertex-ai.yaml --global +curl -fsSLo google-vertex-ai.yaml https://raw.githubusercontent.com/NVIDIA/OpenShell/main/providers/google-vertex-ai.yaml +# Edit google-vertex-ai.yaml and add, for example: +# Native Claude Code: +# binaries: [/usr/bin/curl, /path/to/resolved/claude] +# npm-installed Claude Code: +# binaries: [/usr/bin/curl, /path/to/resolved/node] +openshell profile lint -f google-vertex-ai.yaml +openshell profile import -f google-vertex-ai.yaml --global ``` +For a native installation, find the resolved path with +`readlink -f "$(command -v claude)"`. For an npm installation, find the Node.js +interpreter with `readlink -f "$(command -v node)"`. Run the applicable command +inside the image. + #### Service Account Key -Create the provider with the JSON key as gateway-only bootstrap material: +Create the provider for runtime credentials: ```shell openshell provider create \ --name vertex-prod \ --type google-vertex-ai \ - --credential GOOGLE_SERVICE_ACCOUNT_KEY="$(cat /path/to/key.json)" \ + --global-profile \ + --runtime-credentials \ --config VERTEX_AI_PROJECT_ID=my-gcp-project \ --config VERTEX_AI_REGION=us-central1 ``` @@ -275,6 +309,16 @@ openshell provider refresh configure vertex-prod \ The private key remains in the gateway credential store. Sandboxes receive only an opaque placeholder for the short-lived access token. +The profile's `environment` and `discovery.config_env_vars` declarations supply +SDK configuration and local discovery behavior. A copy imported under another +ID keeps those effects. Do not store `GOOGLE_SERVICE_ACCOUNT_KEY` as a provider +credential or non-secret environment default; supply the private key through +refresh configuration. If an older provider record still holds that key after +the profile declaration is removed, the gateway omits it and continues to +provide the access token and SDK configuration. Two Vertex providers can share +the same non-secret SDK defaults when their values match and their credential +environment keys differ. + #### gcloud Application Default Credentials For local development: @@ -285,6 +329,7 @@ gcloud auth application-default login openshell provider create \ --name vertex-local \ --type google-vertex-ai \ + --global-profile \ --from-gcloud-adc \ --config VERTEX_AI_PROJECT_ID=my-gcp-project \ --config VERTEX_AI_REGION=us-central1 @@ -299,7 +344,7 @@ file and refresh token do not enter the sandbox. | Key | Required | Default | Description | |---|---|---|---| | `VERTEX_AI_PROJECT_ID` | Yes | — | GCP project ID exposed as non-secret workload configuration. | -| `VERTEX_AI_REGION` | No | `us-central1` | Vertex location exposed as non-secret workload configuration. | +| `VERTEX_AI_REGION` | Client dependent | — | Vertex location exposed as non-secret workload configuration. Set it explicitly for regional endpoints. | When the provider is attached, OpenShell also projects standard project and location aliases such as `GOOGLE_CLOUD_PROJECT`, `ANTHROPIC_VERTEX_PROJECT_ID`, @@ -326,6 +371,36 @@ Launch a new process after runtime attachment so it receives the provider environment. Existing processes do not gain newly attached environment variables. +### Run Claude Code with Vertex AI + +Use an image with Claude Code installed. In the imported profile's `binaries`, +include the resolved Claude executable for a native installation or the +resolved Node.js interpreter for an npm installation. After attaching the +provider, start an interactive shell: + +```shell +openshell sandbox connect vertex-agent +``` + +Then start Claude Code inside the sandbox: + +```shell +vertex_token=${GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN:-${GOOGLE_VERTEX_AI_TOKEN:-}} +test -n "$vertex_token" || { echo "Vertex token unavailable" >&2; exit 1; } +ANTHROPIC_AUTH_TOKEN="$vertex_token" \ + CLAUDE_CODE_USE_VERTEX=1 \ + CLAUDE_CODE_SKIP_VERTEX_AUTH=1 \ + claude +``` + +Run these commands in the sandbox shell so the token comes from the attached +provider. The project and region variables come from the profile. Claude Code +uses `ANTHROPIC_AUTH_TOKEN` for the bearer Authorization header and +`CLAUDE_CODE_SKIP_VERTEX_AUTH=1` to skip its own Google credential lookup. +OpenShell replaces the token placeholder on authorized Vertex requests. See +[Claude Code's gateway authentication settings](https://docs.anthropic.com/en/docs/claude-code/llm-gateway) +for the client environment variables. + ### Call the Native Vertex API Claude models use Vertex's publisher-model endpoint. Run a request from a new @@ -375,8 +450,13 @@ Common failures: provider profile's endpoint binding. - A Vertex 400 or 404 usually means the model, location, publisher path, or request body does not match the native API. -- A Vertex 401 or 403 can indicate an expired refresh grant or missing GCP IAM - permission. Check `provider refresh status` and the Vertex AI User role. +- A Vertex 401 with `CREDENTIALS_MISSING` means the request reached Google + without a usable Authorization header. For Claude Code, pass the provider + token through `ANTHROPIC_AUTH_TOKEN` as shown above; setting only + `CLAUDE_CODE_SKIP_VERTEX_AUTH=1` leaves the request unauthenticated. +- Other 401 responses can indicate an expired refresh grant. Check + `provider refresh status`. A 403 can indicate missing GCP IAM permission or + model access; check the Vertex AI User role and access to the selected model. Provider creation does not verify model access. The native request is the end-to-end check. diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index 359b59c486..91030e12bc 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -38,7 +38,7 @@ Provider profiles include these user-facing features: - `openshell profile export`, `import`, `update`, `lint`, and `delete` for custom profiles. - Provider instances created from imported profile IDs with `openshell provider create --type `. - Provider instances whose submitted credentials can be stored by a configured gateway credential driver. -- Profile-backed credential discovery for explicit `openshell provider create --from-existing` and `openshell provider update --from-existing` flows. The `google-vertex-ai` profile also supplements discovery with Vertex config env vars such as `VERTEX_AI_PROJECT_ID` and `VERTEX_AI_REGION`. +- Profile-backed discovery for explicit `openshell provider create --from-existing` and `openshell provider update --from-existing` flows. `discovery.credentials` selects credentials, and `discovery.config_env_vars` selects non-secret configuration variables. - Just-in-time effective policy composition from sandbox policy plus attached provider profiles. - Runtime sandbox provider commands under `openshell sandbox provider list|attach|detach|status`, with an option to wait until the sandbox applies a change. - Credential refresh configuration with `openshell provider refresh status|configure|rotate|delete`. @@ -46,6 +46,55 @@ Provider profiles include these user-facing features: - Dynamic token grants that use the sandbox's SPIFFE JWT-SVID as an OAuth2 client assertion and inject short-lived tokens into supported headers for matching profile endpoints. - Endpoint-bound static credential placeholders. The sandbox proxy resolves a static credential only for request hosts, ports, and paths declared by its provider profile or explicitly bound in sandbox policy for an endpointless profile. +## Declare Environment and Discovery Behavior + +The profile ID identifies a definition. Renaming a profile does not activate or +disable SDK configuration. Declare every non-secret environment default in the +profile: + +```yaml +environment: + config: + CUSTOM_PROJECT: CUSTOM_PROJECT + CUSTOM_REGION: CUSTOM_REGION + fixed: + CUSTOM_MODE: native +discovery: + credentials: [api_key] + config_env_vars: [CUSTOM_PROJECT, CUSTOM_REGION] +``` + +`environment.config` maps each destination environment variable to a provider +config key. OpenShell trims the config value and skips missing or blank values. +`environment.fixed` supplies literal values, including an explicitly empty +string. Create-time environment values take precedence over sandbox template +values, which take precedence over these non-secret defaults. Overrides preserve +explicitly empty values. Supervisor metadata services read the same effective +non-secret values as the workload. +Provider credential keys remain credential-owned and reach the workload as +opaque placeholders. + +`discovery.config_env_vars` copies nonempty local environment values into +provider config under the same key. Use that key in `environment.config` if the +discovered value should become a workload environment default. Discovery does +not read undeclared keys or infer behavior from the profile ID. + +Profiles allow at most 64 environment defaults and 64 discovery config keys. +Environment names and config keys are limited to 128 bytes; fixed values are +limited to 4096 bytes. Values do not support interpolation or scripts. Lint +rejects invalid environment names, duplicate destinations across `config` and +`fixed`, and collisions with credential `env_vars`. The legacy +`GOOGLE_SERVICE_ACCOUNT_KEY` cannot be a non-secret default. Attachment accepts +the same non-secret destination from multiple providers when their values +match. It rejects differing values and overlapping credential keys. Use +distinct destination names when providers need different values. + +Check each example profile's header for runtime requirements before using it. +Import and attachment validate the profile definition but do not check whether +the sandbox runtime provides services its SDK settings depend on. For example, +[Google Cloud metadata discovery](/how-it-works/providers/google#google-cloud-metadata) +requires a supported Linux sandbox runtime. + ## Understand Static Credential Endpoint Binding Static credential endpoint binding prevents a placeholder for one service from @@ -339,12 +388,19 @@ Import one profile file at platform scope: openshell profile import -f providers/github.yaml --global ``` -Import all non-recursive `*.yaml`, `*.yml`, and `*.json` files from a directory: +Import all non-recursive `*.yaml`, `*.yml`, and `*.json` files from a directory +of profiles you reviewed and selected: ```shell -openshell profile import --from ./providers --global +mkdir -p selected-profiles +cp providers/github.yaml providers/openai.yaml selected-profiles/ +openshell profile import --from ./selected-profiles --global ``` +Imports are atomic: an invalid or conflicting profile rejects the whole batch. +A successful import does not confirm runtime compatibility. Check the selected +profiles' documented runtime requirements before attaching their providers. + Omit `--global` to import into the current workspace instead. Export a profile as YAML, to edit or to keep before an upgrade: diff --git a/docs/index.yml b/docs/index.yml index 596ac87172..7baee220d7 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -134,6 +134,8 @@ navigation: - section: "Upgrade Guides" slug: upgrade contents: + - page: "0.2.0" + path: upgrade/0-2-0.mdx - page: "0.1.0" path: upgrade/0-1-0.mdx - folder: resources diff --git a/docs/upgrade/0-2-0.mdx b/docs/upgrade/0-2-0.mdx new file mode 100644 index 0000000000..1bb7336ee4 --- /dev/null +++ b/docs/upgrade/0-2-0.mdx @@ -0,0 +1,242 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Upgrade to NVIDIA OpenShell 0.2.0" +sidebar-title: "0.2.0" +description: "Prepare OpenShell operators and users for the breaking changes in version 0.2.0." +keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Upgrade, Migration" +--- + +OpenShell 0.2.0 introduces breaking changes. Review the changes that apply to +your role before upgrading from OpenShell 0.1.x to 0.2.0. + +## Operators + +If you run OpenShell for a team, start here. + +### Migrate Google Cloud and Vertex AI profiles + +In OpenShell 0.1.x, the gateway recognized the Google profile IDs and added SDK +settings that were absent from the imported YAML. Exporting a profile did not +show all of its effects, and copying it under a new ID could lose the automatic +project, region, and client settings. + +Google Cloud and Vertex AI profiles must declare their environment defaults and +local discovery behavior. The `google-cloud` and `google-vertex-ai` IDs no longer +activate SDK configuration or discovery keys. These declarations let you review +and customize the SDK settings in the profile itself. A renamed copy keeps the +same behavior because it carries those settings with it. + +The discovery declarations also make the local setup inputs explicit. You can +review which variables `--from-existing` uses to discover credentials and +non-secret project or region configuration before creating or updating a +provider. + + +An existing profile without an `environment` block still attaches successfully, +without a migration warning. Its sandboxes receive credential placeholders but +no profile-provided project, region, client-selection, or metadata-discovery +settings, including `GOOSE_PROVIDER`, `ANTHROPIC_VERTEX_PROJECT_ID`, and +`GCE_METADATA_HOST`. + +Without the metadata host setting, Google SDKs can fall back to the cloud +metadata address `169.254.169.254`, which OpenShell blocks. Workloads can report +metadata discovery, missing project, or authentication errors even though the +provider attached successfully. + + +Export each existing Google profile before upgrading. For example, export a +platform-scoped Google Cloud profile: + +```shell +openshell profile export google-cloud -o yaml --global > google-cloud.yaml +``` + +Merge `environment.config`, `environment.fixed`, and `discovery.config_env_vars` +from the current [Google Cloud](https://github.com/NVIDIA/OpenShell/blob/main/providers/google-cloud.yaml) +or [Vertex AI](https://github.com/NVIDIA/OpenShell/blob/main/providers/google-vertex-ai.yaml) +example into the exported profile. Preserve its `resource_version`, binary +restrictions, and endpoints. Keep credentials out of these non-secret fields; +lint rejects collisions with credential `env_vars`. See +[profile environment declarations](/how-it-works/providers/profiles#declare-environment-and-discovery-behavior). + +For example, merge these declarations into a Vertex profile whose provider +config uses `VERTEX_AI_PROJECT_ID` and `VERTEX_AI_REGION`. Entries under +`discovery.credentials` refer to names in the profile's top-level `credentials` +list. This example selects `service_account_token`; keep any other valid +selections from your exported profile: + +```yaml +environment: + config: + GCP_PROJECT_ID: VERTEX_AI_PROJECT_ID + GOOGLE_CLOUD_PROJECT: VERTEX_AI_PROJECT_ID + ANTHROPIC_VERTEX_PROJECT_ID: VERTEX_AI_PROJECT_ID + CLOUD_ML_REGION: VERTEX_AI_REGION + GCP_LOCATION: VERTEX_AI_REGION + VERTEX_LOCATION: VERTEX_AI_REGION + fixed: + GOOSE_PROVIDER: gcp_vertex_ai +discovery: + credentials: [service_account_token] + config_env_vars: + - VERTEX_AI_PROJECT_ID + - VERTEX_AI_REGION +``` + +If a profile declares `GOOGLE_SERVICE_ACCOUNT_KEY`, +[migrate its private key](#migrate-service-account-private-keys) before updating +the profile. + +On the upgraded gateway, lint the edited profile and update the existing +definition: + +```shell +openshell profile lint -f google-cloud.yaml --global +openshell profile update google-cloud -f google-cloud.yaml --global +``` + +Repeat for `google-vertex-ai` and any copies imported under other IDs. Use the +same scope as the existing profile; omit `--global` for workspace-scoped +profiles. Import is create-only, so use `profile import` only when the profile +does not already exist. See [profile management](/how-it-works/providers/profiles#provider-profiles). + +The Google Cloud metadata example requires a Linux sandbox using Docker, +Podman, Kubernetes, or VM. Windows/MXC does not provide the metadata endpoint. +Import and attachment do not check runtime support, so they can succeed even +when SDK metadata discovery will fail. Verify the sandbox runtime before using +this profile. Native Vertex bearer-token authentication does not require the +metadata service. See [Google Cloud metadata discovery](/how-it-works/providers/google#google-cloud-metadata). + +When importing several new profiles with `--from`, put reviewed profiles in a +separate directory and check their runtime requirements first. Import is atomic: +an invalid or conflicting definition rejects the whole batch. A successful +import does not confirm that the selected profiles work with your runtime. + +### Migrate service-account private keys + +OpenShell 0.2.0 stops injecting `GOOGLE_SERVICE_ACCOUNT_KEY` for every provider +profile, including custom profiles and renamed forks. Previously, this +restriction applied only to `google-vertex-ai`. + +Service-account JSON contains a long-lived signing key used to mint access +tokens. Gateway-managed refresh keeps that key in the configured credential +store and renews short-lived access tokens automatically. Workloads receive an +opaque token placeholder that OpenShell resolves on authorized outbound +requests. Applying the restriction to every profile ID keeps the signing key +out of this legacy credential path even when a profile is renamed or customized. + +An older stored provider record can retain this key, but the gateway omits it +from sandbox workloads. Attachment can still succeed, while a client that +depends on the key fails to authenticate. In gateway logs, look for +`skipping non-injectable provider credential` or +`skipping non-injectable provider credential handle` with the key +`GOOGLE_SERVICE_ACCOUNT_KEY`. The gateway continues to supply the provider's +other valid access tokens and declared non-secret SDK settings. + +Before upgrading, remove `GOOGLE_SERVICE_ACCOUNT_KEY` from the profile's +credential declarations and configure its private key as gateway-managed +refresh material. Use refreshed access tokens in workloads. For new Vertex +service-account providers, create with `--runtime-credentials` and configure +refresh directly. Follow the [Vertex service-account workflow](/how-it-works/providers/google#create-a-vertex-ai-provider). + +New profile imports reject `GOOGLE_SERVICE_ACCOUNT_KEY` in credential +declarations and in `environment.config` or `environment.fixed`. Keep the +private key in refresh configuration when migrating custom profiles. + +#### Example custom profile migration + +This example migrates an existing platform-scoped profile named `acme-vertex`, +used by a provider named `acme-vertex-prod`. The profile already declares its +client's Vertex endpoints and binaries, and uses the environment declarations +above. Export its current definition: + +```shell +openshell profile export acme-vertex -o yaml --global > acme-vertex.yaml +``` + +The legacy entry in its `credentials` list might look like this: + +```yaml +- name: service_account_key + env_vars: [GOOGLE_SERVICE_ACCOUNT_KEY] + required: true +``` + +Replace that entry with a refreshable access-token declaration. These snippets +show individual credential entries; preserve the rest of the exported profile, +including its `resource_version`, binary restrictions, and endpoints: + +```yaml +- name: service_account_token + env_vars: [ACME_VERTEX_ACCESS_TOKEN] + required: false + auth_style: bearer + header_name: authorization + refresh: + strategy: google_service_account_jwt + token_url: https://oauth2.googleapis.com/token + scopes: [https://www.googleapis.com/auth/cloud-platform] + refresh_before_seconds: 300 + max_lifetime_seconds: 3600 + material: + - name: client_email + required: true + - name: private_key + required: true + secret: true +``` + +If `discovery.credentials` includes `service_account_key`, replace that selection +with `service_account_token`. Keep any other credential entries and discovery +selections that still apply. On the upgraded gateway, validate and update the +existing profile: + +```shell +openshell profile lint -f acme-vertex.yaml --global +openshell profile update acme-vertex -f acme-vertex.yaml --global +``` + +Configure refresh for the existing provider using your service-account JSON +file. This example requires `jq`; replace `/path/to/key.json` with your file's +path. `--secret-material-env` reads the private key from the local environment +and stores it as secret refresh material: + +```shell +export GCP_PRIVATE_KEY="$(jq -er .private_key /path/to/key.json)" +openshell provider refresh configure acme-vertex-prod \ + --credential-key ACME_VERTEX_ACCESS_TOKEN \ + --strategy google-service-account-jwt \ + --material client_email="$(jq -er .client_email /path/to/key.json)" \ + --secret-material-env private_key=GCP_PRIVATE_KEY +unset GCP_PRIVATE_KEY + +openshell provider refresh rotate acme-vertex-prod \ + --credential-key ACME_VERTEX_ACCESS_TOKEN +openshell provider refresh status acme-vertex-prod +``` + +Wait for refresh to complete, then restart clients so they receive the new token +placeholder. Update clients to send `ACME_VERTEX_ACCESS_TOKEN` as the bearer +token instead of reading service-account JSON from `GOOGLE_SERVICE_ACCOUNT_KEY`. + +#### Verify the migrated credential + +For a sandbox named `vertex-agent` with `acme-vertex-prod` attached, launch a new +process to verify the legacy key is absent and the token authorizes a Vertex +API request. The image must include `curl`, its executable must be authorized +by the profile's `binaries`, and the profile must allow the regional Vertex +endpoint. Set the provider's project and region config to values your service +account can access. The following request lists models without printing the +token or response body: + +```shell +openshell sandbox exec -n vertex-agent -- sh -c ' + test -z "${GOOGLE_SERVICE_ACCOUNT_KEY+x}" && + test -n "${ACME_VERTEX_ACCESS_TOKEN:-}" && + curl --fail --silent --show-error --output /dev/null \ + -H "Authorization: Bearer $ACME_VERTEX_ACCESS_TOKEN" \ + "https://${CLOUD_ML_REGION}-aiplatform.googleapis.com/v1/projects/${GOOGLE_CLOUD_PROJECT}/locations/${CLOUD_ML_REGION}/models" && + printf "%s\n" "Vertex authentication succeeded; legacy key absent." +' +``` diff --git a/e2e/python/test_sandbox_providers.py b/e2e/python/test_sandbox_providers.py index 6619c4b465..3041cb3cce 100644 --- a/e2e/python/test_sandbox_providers.py +++ b/e2e/python/test_sandbox_providers.py @@ -157,6 +157,25 @@ def imported_provider_profile( _delete_provider_profile(stub, profile.id) +def _endpointless_credential_profile(profile_id: str) -> openshell_pb2.ProviderProfile: + """Build an endpointless credential profile for placeholder delivery tests.""" + return openshell_pb2.ProviderProfile( + id=profile_id, + display_name=f"{profile_id} display", + category=openshell_pb2.PROVIDER_PROFILE_CATEGORY_OTHER, + credentials=[ + openshell_pb2.ProviderProfileCredential( + name="api_token", + description="E2E endpointless credential", + env_vars=["E2E_ENDPOINTLESS_TOKEN"], + required=True, + auth_style="bearer", + header_name="authorization", + ) + ], + ) + + def _native_inference_profile( *, profile_id: str, @@ -348,66 +367,64 @@ def test_endpointless_profile_credentials_fail_closed_without_policy_binding( sandbox_client: SandboxClient, ) -> None: """Endpointless profile credentials are withheld without an explicit binding.""" - with provider( - sandbox_client._stub, - name="e2e-test-google-cloud-without-policy-binding", - provider_type="google-cloud", - credentials={"GCP_ADC_ACCESS_TOKEN": "gcp-e2e-token"}, - ) as provider_name: + profile_id = "e2e-endpointless-without-policy-binding" + with ( + imported_provider_profile( + sandbox_client._stub, + profile=_endpointless_credential_profile(profile_id), + source=f"{profile_id}.yaml", + ), + provider( + sandbox_client._stub, + name="e2e-test-endpointless-without-policy-binding", + provider_type=profile_id, + credentials={"E2E_ENDPOINTLESS_TOKEN": "e2e-token"}, + profile_workspace="default", + ) as provider_name, + ): spec = datamodel_pb2.SandboxSpec( policy=_default_policy(), providers=[provider_name], ) - def read_gcp_token() -> str: + def read_token() -> str: import os - return os.environ.get("GCP_ADC_ACCESS_TOKEN", "NOT_SET") + return os.environ.get("E2E_ENDPOINTLESS_TOKEN", "NOT_SET") with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(read_gcp_token) + result = sb.exec_python(read_token) assert result.exit_code == 0, result.stderr assert result.stdout.strip() == "NOT_SET" - def read_metadata_token_status() -> int: - import os - import urllib.error - import urllib.request - - request = urllib.request.Request( - f"http://{os.environ['GCE_METADATA_HOST']}/computeMetadata/v1/instance/service-accounts/default/token", - headers={"Metadata-Flavor": "Google"}, - ) - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - try: - with opener.open(request, timeout=5) as response: - return response.status - except urllib.error.HTTPError as error: - return error.code - - result = sb.exec_python(read_metadata_token_status) - assert result.exit_code == 0, result.stderr - assert result.stdout.strip() == "503" - def test_endpointless_profile_credentials_use_explicit_policy_binding( sandbox: Callable[..., Sandbox], sandbox_client: SandboxClient, ) -> None: """An endpointless profile emits credentials only with an explicit binding.""" - with provider( - sandbox_client._stub, - name="e2e-test-google-cloud-policy-binding", - provider_type="google-cloud", - credentials={"GCP_ADC_ACCESS_TOKEN": "gcp-e2e-token"}, - ) as provider_name: + profile_id = "e2e-endpointless-policy-binding" + with ( + imported_provider_profile( + sandbox_client._stub, + profile=_endpointless_credential_profile(profile_id), + source=f"{profile_id}.yaml", + ), + provider( + sandbox_client._stub, + name="e2e-test-endpointless-policy-binding", + provider_type=profile_id, + credentials={"E2E_ENDPOINTLESS_TOKEN": "e2e-token"}, + profile_workspace="default", + ) as provider_name, + ): policy = _default_policy() - policy.network_policies["gcp_storage"].CopyFrom( + policy.network_policies["endpointless_api"].CopyFrom( sandbox_pb2.NetworkPolicyRule( - name="gcp_storage", + name="endpointless_api", endpoints=[ sandbox_pb2.NetworkEndpoint( - host="storage.googleapis.com", + host="api.example.com", port=443, protocol="rest", access=sandbox_pb2.NETWORK_ACCESS_PRESET_FULL, @@ -423,36 +440,67 @@ def test_endpointless_profile_credentials_use_explicit_policy_binding( providers=[provider_name], ) - def read_gcp_token() -> str: + def read_token() -> str: import os - return os.environ.get("GCP_ADC_ACCESS_TOKEN", "NOT_SET") + return os.environ.get("E2E_ENDPOINTLESS_TOKEN", "NOT_SET") with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(read_gcp_token) + result = sb.exec_python(read_token) assert result.exit_code == 0, result.stderr assert _is_placeholder_for_env_key( - result.stdout.strip(), "GCP_ADC_ACCESS_TOKEN" + result.stdout.strip(), "E2E_ENDPOINTLESS_TOKEN" ) @pytest.mark.parametrize( - ("service_account_email", "provider_suffix"), + ( + "service_account_email", + "provider_project", + "spec_project", + "template_project", + "provider_suffix", + ), [ - (None, "no-email"), - ("", "empty-email"), - ("sdk@metadata-test-project.iam.gserviceaccount.com", "configured-email"), + (None, "metadata-test-project", None, None, "no-email"), + ("", "metadata-test-project", None, None, "empty-email"), + ( + "sdk@metadata-test-project.iam.gserviceaccount.com", + "metadata-test-project", + None, + None, + "configured-email", + ), + (None, "provider-project", None, "template-project", "template-project"), + ( + None, + "provider-project", + "caller-project", + "template-project", + "spec-project", + ), + (None, None, "caller-project", None, "missing-provider-project"), + ], + ids=[ + "no-email", + "empty-email", + "configured-email", + "template-project", + "spec-project", + "missing-provider-project", ], - ids=["no-email", "empty-email", "configured-email"], ) def test_google_metadata_sdk_discovery( sandbox: Callable[..., Sandbox], sandbox_client: SandboxClient, service_account_email: str | None, + provider_project: str | None, + spec_project: str | None, + template_project: str | None, provider_suffix: str, ) -> None: """Google's SDK discovers project/account metadata and refreshes a placeholder.""" - config = {"project_id": "metadata-test-project"} + config = {"project_id": provider_project} if provider_project is not None else {} if service_account_email is not None: config["service_account_email"] = service_account_email with provider( @@ -480,6 +528,14 @@ def test_google_metadata_sdk_discovery( ) ) spec = datamodel_pb2.SandboxSpec(policy=policy, providers=[provider_name]) + if spec_project is not None: + spec.environment["GCP_PROJECT_ID"] = spec_project + if template_project is not None: + spec.template.CopyFrom( + openshell_pb2.SandboxTemplate( + environment={"GCP_PROJECT_ID": template_project} + ) + ) def discover_metadata() -> str: import json @@ -509,6 +565,7 @@ def discover_metadata() -> str: return json.dumps( { "project": project, + "environment_project": os.environ.get("GCP_PROJECT_ID"), "account": credentials.service_account_email, "token": credentials.token, "expiry_present": credentials.expiry is not None, @@ -521,7 +578,9 @@ def discover_metadata() -> str: result = sb.exec_python(discover_metadata) assert result.exit_code == 0, result.stderr data = json.loads(result.stdout) - assert data["project"] == "metadata-test-project" + expected_project = spec_project or template_project or provider_project + assert data["project"] == expected_project + assert data["environment_project"] == expected_project assert data["account"] == (service_account_email or "default") assert data["metadata_host"] == data["metadata_ip"] == "127.0.0.1:8174" assert data["expiry_present"] diff --git a/e2e/rust/tests/provider_refresh_handles.rs b/e2e/rust/tests/provider_refresh_handles.rs index 1de651415d..4204494961 100644 --- a/e2e/rust/tests/provider_refresh_handles.rs +++ b/e2e/rust/tests/provider_refresh_handles.rs @@ -9,6 +9,8 @@ //! retains its original environment while the gateway rotates the provider 12 //! times. The shell must continue reaching the resource with the newest token, //! and explicit refresh reconfiguration must revoke its old handle. +//! Declared non-secret defaults must reach the shell as literal values, while +//! caller environment takes precedence in both main and exec processes. use std::io::Write; use std::process::Stdio; @@ -113,6 +115,13 @@ fn write_profile(resource_port: u16, token_port: u16) -> Result Result<(), String> { PROFILE_ID, "--credential", TOKEN_ENV, + "--config", + "project=custom-profile-project", ], &[(TOKEN_ENV, "bootstrap-token")], ) @@ -301,8 +312,9 @@ async fn long_running_process_survives_rotations_and_reconfigure_revokes() -> Re delete_provider_resources().await; let fixture_port = find_free_port(); let fixture_script = FIXTURE_SCRIPT.replace("__PORT__", &fixture_port.to_string()); - let fixture = - HostSupportContainer::start_python_on_host_network(&fixture_script, fixture_port).await?; + // Publish the fixture for both the native gateway and the supervisor. On + // macOS, Podman's host network is inside its VM, not the gateway's host. + let fixture = HostSupportContainer::start_python(&fixture_script, fixture_port).await?; let profile = write_profile(fixture.port, fixture.port)?; let policy = write_policy(fixture.port)?; configure_refresh(&profile).await?; @@ -314,6 +326,10 @@ async fn long_running_process_survives_rotations_and_reconfigure_revokes() -> Re openshell:resolve:env:s*_REFRESH_E2E_ACCESS_TOKEN) ;; *) exit 64 ;; esac +test "$REFRESH_E2E_PROJECT" = caller-project || exit 65 +test "${{REFRESH_E2E_MODE+x}}" = x && test -z "$REFRESH_E2E_MODE" || exit 66 +test "$REFRESH_E2E_DEFAULT_PROJECT" = custom-profile-project || exit 67 +test "$REFRESH_E2E_DEFAULT_MODE" = native || exit 68 echo {READY_MARKER} while true; do if [ -f /sandbox/probe-trigger ]; then @@ -328,13 +344,36 @@ while true; do done"# ); let mut sandbox = SandboxGuard::create_keep_with_args( - &["--provider", PROVIDER_NAME, "--policy", &policy_path], + &[ + "--provider", + PROVIDER_NAME, + "--policy", + &policy_path, + "--env", + "REFRESH_E2E_PROJECT=caller-project", + "--env", + "REFRESH_E2E_MODE=", + "--env", + "REFRESH_E2E_ACCESS_TOKEN=caller-token", + ], &["sh", "-c", &parent_script], READY_MARKER, ) .await?; let result = async { + sandbox + .exec(&[ + "sh", + "-c", + r#"test "$REFRESH_E2E_PROJECT" = caller-project && +test "${REFRESH_E2E_MODE+x}" = x && test -z "$REFRESH_E2E_MODE" && +test "$REFRESH_E2E_DEFAULT_PROJECT" = custom-profile-project && +test "$REFRESH_E2E_DEFAULT_MODE" = native && +case "$REFRESH_E2E_ACCESS_TOKEN" in openshell:resolve:env:s*_REFRESH_E2E_ACCESS_TOKEN) true;; *) false;; esac"#, + ]) + .await?; + if trigger_probe(&sandbox).await? != "ok" { return Err(format!( "initial long-running credential probe failed; fixture logs:\n{}", diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index f69518f0bc..954ec0ce5c 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -329,12 +329,15 @@ EOF e2e_import_example_provider_profiles() { local cli_bin=$1 local root=$2 + local profile echo "Importing example provider profiles from ${root}/providers..." - if ! "${cli_bin}" provider profile import --from "${root}/providers" --global; then - echo "ERROR: failed to import example provider profiles" >&2 - return 1 - fi + for profile in "${root}"/providers/*.yaml; do + if ! "${cli_bin}" provider profile import --file "${profile}" --global; then + echo "ERROR: failed to import example provider profile ${profile}" >&2 + return 1 + fi + done } # Register an administrator OIDC session for a gateway, non-interactively. diff --git a/proto/openshell.proto b/proto/openshell.proto index b40ec68263..d404a659db 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -2545,6 +2545,17 @@ message ProviderCredentialRefreshStatus { message ProviderProfileDiscovery { // Credential names from ProviderProfile.credentials eligible for local discovery. repeated string credentials = 1; + // Non-secret environment keys copied into provider config under the same key. + repeated string config_env_vars = 2; +} + +// Bounded, non-secret environment defaults. Existing workload values win. +// Values are literal strings; interpolation and scripts are not supported. +message ProviderProfileEnvironment { + // Destination environment key -> provider config key. Blank config is skipped. + map config = 1; + // Destination environment key -> literal non-secret value. + map fixed = 2; } message GetProviderRefreshStatusRequest { @@ -2626,6 +2637,9 @@ enum ProviderProfileCategory { // Provider type profile metadata exposed to clients. message ProviderProfile { + reserved 16; + reserved "required_platform_adapter"; + string id = 1; string display_name = 2; string description = 3; @@ -2635,13 +2649,13 @@ message ProviderProfile { repeated openshell.sandbox.v1.NetworkBinary binaries = 7; bool inference_capable = 8; ProviderProfileDiscovery discovery = 9; - // Storage resource version for custom profiles. Built-in profiles and new - // profile files use 0. Gateway responses set this for stored custom profiles. + // Storage resource version. New profile files use 0; gateway responses set + // this for stored profiles. // Update calls use this for optimistic concurrency. uint64 resource_version = 10; // Optional non-secret annotations attached by profile sources or importers. map annotations = 11; - // Server-set provenance: "builtin", "user", or "interceptor/{name}". + // Server-set provenance: "user" or "interceptor/{name}". // Ignored on import/update payloads. string source = 12; // Server-set visibility: "platform", "workspace", or empty for @@ -2650,6 +2664,8 @@ message ProviderProfile { // EXPERIMENTAL: Non-secret files rendered from provider configuration for the // workload. This API and its behavior may change or be removed. repeated ProviderProfileFile files = 14; + // Non-secret environment defaults declared by this profile. + ProviderProfileEnvironment environment = 15; } // EXPERIMENTAL: Provider file templates may change or be removed. diff --git a/providers/README.md b/providers/README.md index 9a721af9a8..a102081670 100644 --- a/providers/README.md +++ b/providers/README.md @@ -16,14 +16,20 @@ openshell provider profile lint -f providers/github.yaml openshell provider profile import -f providers/github.yaml --global ``` -Or import the whole directory: +To import several profiles, put compatible copies in a separate directory: ```shell -openshell provider profile import --from providers --global +openshell provider profile import --from ./selected-profiles --global ``` Drop `--global` to import into the current workspace instead. +The `google-cloud.yaml` example requires the supervisor's metadata service in +a Linux sandbox using Docker, Podman, Kubernetes, or VM. Windows/MXC does not +provide this service. Import and attachment do not check runtime support, so +verify it before using the profile. The Vertex example uses bearer-token +authentication and does not require the metadata service. + ## Read the header before importing Every file opens with a comment block naming its expected client binaries, the @@ -48,6 +54,9 @@ workload, and import your copy. injection safe. - Keep `endpoints` limited to the hosts the credential should reach. A credential is only sent to the endpoints its profile declares. +- Review non-secret `environment` defaults and `discovery.config_env_vars`. + Renaming the profile preserves these declarations. Check the header for + runtime requirements before using your copy. - Run `openshell provider profile lint` before importing. See [Provider profiles](https://docs.nvidia.com/openshell/latest/how-it-works/providers/profiles) for the diff --git a/providers/google-cloud.yaml b/providers/google-cloud.yaml index 17fa04fa14..764248ce67 100644 --- a/providers/google-cloud.yaml +++ b/providers/google-cloud.yaml @@ -10,10 +10,10 @@ # below, so it has to name the paths in *your* image. # # Client binaries: none declared — this profile grants no egress. -# Reference layout: any image whose GCP SDKs honour the metadata server. The -# sandbox reaches credentials through OpenShell's GCE metadata -# emulator, so gcloud, google-cloud-* libraries and anything -# else using Application Default Credentials work unmodified. +# Runtime required: a Linux sandbox using Docker, Podman, Kubernetes, or VM. +# The supervisor provides the metadata service there; +# Windows/MXC does not. Import and attachment do not check +# runtime support, so verify it before using this profile. # Credential scope: a gateway-refreshed GCP access token, as GCP_SA_ACCESS_TOKEN # (service-account JWT flow) or GCP_ADC_ACCESS_TOKEN (gcloud # ADC flow). Configure the refresh material with @@ -32,6 +32,18 @@ description: Native GCP SDK credentials for sandboxes via metadata emulator category: other inference_capable: false +environment: + config: + GCP_PROJECT_ID: project_id + GOOGLE_CLOUD_PROJECT: project_id + CLOUD_ML_REGION: region + GCP_LOCATION: region + GCP_SERVICE_ACCOUNT_EMAIL: service_account_email + fixed: + GCE_METADATA_HOST: "127.0.0.1:8174" + GCE_METADATA_IP: "127.0.0.1:8174" + METADATA_SERVER_DETECTION: assume-present + credentials: # Service account JWT flow: gateway signs a JWT and exchanges it for an # access token. Configure with `openshell provider refresh configure`. diff --git a/providers/google-vertex-ai.yaml b/providers/google-vertex-ai.yaml index a0994e413e..15d2576169 100644 --- a/providers/google-vertex-ai.yaml +++ b/providers/google-vertex-ai.yaml @@ -18,8 +18,8 @@ # Credential scope: a gateway-refreshed Google access token, as # GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN (service-account JWT) # or GOOGLE_VERTEX_AI_TOKEN (gcloud ADC), sent as a bearer -# authorization header. GOOGLE_SERVICE_ACCOUNT_KEY is refresh -# bootstrap material and is never injected into the sandbox. +# authorization header. Supply service account private keys +# through refresh configuration, not provider credentials. # Endpoint access: the aiplatform.googleapis.com regional and replicated hosts, # read-write, L7 enforced. # Smoke test: openshell sandbox create --provider -- \ @@ -31,11 +31,17 @@ display_name: Google Vertex AI description: Google Vertex AI inference provider (Anthropic Claude, Gemini, and third-party models) category: inference inference_capable: true +environment: + config: + GCP_PROJECT_ID: VERTEX_AI_PROJECT_ID + GOOGLE_CLOUD_PROJECT: VERTEX_AI_PROJECT_ID + ANTHROPIC_VERTEX_PROJECT_ID: VERTEX_AI_PROJECT_ID + CLOUD_ML_REGION: VERTEX_AI_REGION + GCP_LOCATION: VERTEX_AI_REGION + VERTEX_LOCATION: VERTEX_AI_REGION + fixed: + GOOSE_PROVIDER: gcp_vertex_ai credentials: - - name: service_account_key - description: Google service account JSON refresh bootstrap material; not injected into sandboxes - env_vars: [GOOGLE_SERVICE_ACCOUNT_KEY] - required: false - name: service_account_token description: Google Cloud access token refreshed from service account JWT material env_vars: [GOOGLE_VERTEX_AI_SERVICE_ACCOUNT_TOKEN, VERTEX_AI_SERVICE_ACCOUNT_TOKEN] @@ -84,6 +90,12 @@ credentials: secret: true discovery: credentials: [service_account_token, gcloud_adc_token] + config_env_vars: + - VERTEX_AI_PROJECT_ID + - VERTEX_AI_REGION + - GOOGLE_VERTEX_AI_BASE_URL + - VERTEX_AI_BASE_URL + - VERTEX_AI_PUBLISHER endpoints: - host: "*-aiplatform.googleapis.com" port: 443 diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 23d58efa25..8dd6eb8caf 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1622,7 +1622,7 @@ credential failures. | Gateway fails before serving health after enabling an interceptor | Interceptor endpoint unavailable or manifest/binding validation failed | Gateway and interceptor logs; interceptor socket; `binding_policy`, phases, and failure policy | | Authenticated interceptor or middleware rejects gateway calls | Private CA or hostname mismatch, expected audience or issuer mismatch, stale/unknown `kid`, or malformed extension token | `tls_ca_cert_path`, registration `audience`, service verifier config and logs; fetch well-known metadata only through the already-trusted gateway TLS endpoint | | Provider profiles disappear after enabling an interceptor catalog | `provider_profile_sources` selected only an authoritative interceptor or returned invalid/duplicate IDs | Inspect source list and interceptor `Describe`/catalog logs; include `user` when composition with imported profiles is intended | -| `provider list-profiles` is empty on a new gateway | Profiles are import-only and nothing has been imported | Import with `openshell provider profile import --from providers --global`; an empty catalog is a valid ready state, not a failure | +| `provider list-profiles` is empty on a new gateway | Profiles are import-only and nothing has been imported | Import a reviewed profile with `openshell profile import -f --global`; an empty catalog is a valid ready state, not a failure. A directory import containing a profile that requires an unavailable adapter fails as a batch. | | Sandbox create or provider attach fails naming a missing profile | The provider's profile was never imported, was deleted, or lives at another scope | Import it at the scope the provider uses; the error names the profile ID and the command | | Gateway fails after registering supervisor middleware | Service unavailable, invalid manifest, duplicate binding, reserved name, or invalid payload/timeout limit | Middleware service and gateway logs; `[[openshell.supervisor.middleware]]`; `Describe` response | | Policy update rejects `network_middlewares` | Unknown middleware name, implementation-owned config invalid, duplicate order, broad/invalid host selector, or fail-closed coverage of `tls: skip` | Policy error, gateway logs, middleware `ValidateConfig`, selector and order fields | diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index b757172b7a..d7fcaccd8e 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -207,7 +207,7 @@ openshell profile import --url https://example.com/profiles/my-profile.yaml binary grants before importing it. The URL path must end in `.yaml`, `.yml`, or `.json`; downloads are limited to 1 MiB and 15 seconds. -Use `profile describe` to inspect a definition's credential metadata, endpoints, TLS handling, MCP access settings, rule counts, binaries, source, and scope before creating a provider. Check for `tls: skip` and the uninspected-credential opt-in before relying on displayed L7 rules. List and describe accept table, JSON, and YAML output; use structured output for complete rule definitions, `--workspace` for a workspace catalog, or `--global` for platform scope. Use `profile export` when preparing an editable definition, `profile update --file ` to replace an existing custom profile with its current resource version, and `profile delete ...` to remove custom profiles. Provider instances remain under `provider`. +Use `profile describe` to inspect a definition's credential metadata, endpoints, TLS handling, MCP access settings, rule counts, binaries, source, and scope before creating a provider. Inspect structured output for declared environment defaults and discovery config keys. Renaming a profile preserves its declared behavior; the ID does not activate SDK defaults. Check documented runtime requirements before attaching providers; import and attachment do not validate runtime support. Google Cloud metadata discovery requires a supported Linux sandbox runtime. Check for `tls: skip` and the uninspected-credential opt-in before relying on displayed L7 rules. List and describe accept table, JSON, and YAML output; use structured output for complete rule definitions, `--workspace` for a workspace catalog, or `--global` for platform scope. Use `profile export` when preparing an editable definition, `profile update --file ` to replace an existing custom profile with its current resource version, and `profile delete ...` to remove custom profiles. Provider instances remain under `provider`. Existing scripts can continue using `provider list-profiles` and `provider profile export/import/update/lint/delete`. These commands share the top-level handlers and preserve their arguments, output options, and workspace/global flags. Prefer `profile` when writing new commands. diff --git a/tasks/scripts/test-e2e-provider-profiles.sh b/tasks/scripts/test-e2e-provider-profiles.sh new file mode 100644 index 0000000000..36b48a5f05 --- /dev/null +++ b/tasks/scripts/test-e2e-provider-profiles.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=e2e/support/gateway-common.sh +source "${ROOT}/e2e/support/gateway-common.sh" + +PROFILE_WORK="$(mktemp -d)" +trap 'rm -rf "${PROFILE_WORK}"' EXIT +mkdir -p "${PROFILE_WORK}/providers" +printf 'id: google-cloud\n' > "${PROFILE_WORK}/providers/google-cloud.yaml" +printf 'id: acme\n' > "${PROFILE_WORK}/providers/acme.yaml" +printf 'id: future\n' > "${PROFILE_WORK}/providers/future.yaml" + +capture_profile_import() { + printf '%s\n' "$*" >> "${PROFILE_WORK}/imports" +} + +e2e_import_example_provider_profiles capture_profile_import "${PROFILE_WORK}" +expected="provider profile import --file ${PROFILE_WORK}/providers/acme.yaml --global +provider profile import --file ${PROFILE_WORK}/providers/future.yaml --global +provider profile import --file ${PROFILE_WORK}/providers/google-cloud.yaml --global" +if [ "$(cat "${PROFILE_WORK}/imports")" != "${expected}" ]; then + echo "FAIL: gateway setup must import every example profile" >&2 + exit 1 +fi + +fail_profile_import() { + return 1 +} + +if e2e_import_example_provider_profiles fail_profile_import "${PROFILE_WORK}" > "${PROFILE_WORK}/failure.log" 2>&1; then + echo "FAIL: an import error must fail gateway setup" >&2 + exit 1 +fi + +echo "E2E provider profile setup tests passed." diff --git a/tasks/test.toml b/tasks/test.toml index 2839c1cf0c..24f1da2a2f 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -15,6 +15,7 @@ depends = [ "test:postgres-runner", "test:gateway-pull-policy", "test:e2e-image-overrides", + "test:e2e-provider-profiles", "test:gateway-config", "test:packaging-assets", "test:qualification-summary", @@ -70,6 +71,12 @@ run = "tasks/scripts/test-e2e-image-overrides.sh" run_windows = "echo Skipping test:e2e-image-overrides: Unix E2E wrappers do not apply on Windows." hide = true +["test:e2e-provider-profiles"] +description = "Test E2E provider profile selection and import failures" +run = "bash tasks/scripts/test-e2e-provider-profiles.sh" +run_windows = "echo Skipping test:e2e-provider-profiles: Unix E2E wrappers do not apply on Windows." +hide = true + ["test:packaging-assets"] description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh"