diff --git a/Cargo.lock b/Cargo.lock index 7c5124e9..ac8aa009 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -915,6 +930,43 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1759,8 +1811,11 @@ dependencies = [ "bytes", "futures", "http-body-util", + "httpdate", + "openssl", "rcgen", "reqwest", + "ring", "serde", "serde_json", "sha2", diff --git a/Cargo.toml b/Cargo.toml index 016a3bb3..46be7e7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,8 @@ httpdate = "1" futures = "0.3" # Hosted provider and embedding transports. reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } +# RSA PKCS#1 v1.5 signing for OCI HTTP Signature auth (already in the TLS tree). +ring = "0.17" once_cell = "1" regex = "1" # Public request, response, message, tool, and usage shapes are serializable. diff --git a/crates/tinyinference-llm/Cargo.toml b/crates/tinyinference-llm/Cargo.toml index f1e2acb7..5419c791 100644 --- a/crates/tinyinference-llm/Cargo.toml +++ b/crates/tinyinference-llm/Cargo.toml @@ -17,6 +17,9 @@ default = [] # host `ChatBackend` (lives here, not in tinyinference-providers: this crate # already depends on providers, so the reverse edge would be a cycle). server-http = ["dep:axum", "dep:uuid"] +# OCI instance-principal and OKE workload-identity auth. Needs OpenSSL for +# RSA session-key generation and X.509 parsing, so it is off by default. +oci-principals = ["dep:openssl"] [dependencies] axum = { workspace = true, optional = true } @@ -25,7 +28,10 @@ base64 = "0.23" async-trait = { workspace = true } bytes = { workspace = true } futures = { workspace = true } +httpdate = { workspace = true } +openssl = { version = "0.10", optional = true } reqwest = { workspace = true } +ring = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2.workspace = true @@ -42,6 +48,9 @@ tracing = { workspace = true } [dev-dependencies] axum = { workspace = true } +# Generates the throwaway RSA key and certificate the OCI tests sign with, so +# no key material is committed. +openssl = "0.10" uuid = { workspace = true } tower = { version = "0.5", features = ["util"] } http-body-util = "0.1" diff --git a/crates/tinyinference-llm/examples/live_oci.rs b/crates/tinyinference-llm/examples/live_oci.rs new file mode 100644 index 00000000..ace17766 --- /dev/null +++ b/crates/tinyinference-llm/examples/live_oci.rs @@ -0,0 +1,251 @@ +//! Live smoke test: the OCI Generative AI provider against the real service. +//! +//! Network- and credential-gated. Runs a completion, a streamed completion, +//! and a streamed tool round trip for each case; exits cleanly (status 0, +//! "skipped") unless `OCI_LIVE_PROFILE` or `OCI_LIVE_API_KEY` is set, and +//! with status 1 when any check fails. +//! +//! ```bash +//! OCI_LIVE_PROFILE=MY_PROFILE OCI_LIVE_REGION=us-chicago-1 \ +//! OCI_LIVE_COMPARTMENT=ocid1.compartment... \ +//! OCI_LIVE_CASES='native_generic=openai.gpt-5.5,native_cohere=cohere.command-a-03-2025' \ +//! cargo run -p tinyinference-llm --example live_oci +//! ``` +//! +//! | Variable | Meaning | +//! | ---------------------- | --------------------------------------------------------------- | +//! | `OCI_LIVE_PROFILE` | `~/.oci/config` profile (API key or session token) | +//! | `OCI_LIVE_CONFIG` | config file path, default `~/.oci/config` | +//! | `OCI_LIVE_API_KEY` | GenAI API key; used instead of the profile when set | +//! | `OCI_LIVE_REGION` | region, default the profile's region | +//! | `OCI_LIVE_COMPARTMENT` | compartment OCID, default the profile's tenancy | +//! | `OCI_LIVE_PROJECT` | GenAI project OCID, required by the `openai` (`/openai/v1`) base | +//! | `OCI_LIVE_CASES` | comma list of `=` | +//! +//! Transports: `native_generic`, `native_cohere`, `actions` (`/20231130/actions/v1`), +//! `openai` (`/openai/v1`). + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Instant; + +use serde_json::json; +use tinyinference_llm::model::collect_model_stream; +use tinyinference_llm::providers::oci::{ + OciApiFormat, OciAuth, OciConfig, OciOpenAiBase, OciTransport, build_oci_model, load_profile, +}; +use tinyinference_llm::{ChatModel, Message, ModelRequest, ToolSchema}; + +const DEFAULT_CASES: &str = "native_generic=meta.llama-3.3-70b-instruct,\ +native_cohere=cohere.command-a-03-2025"; + +struct Env { + auth: OciAuth, + region: Option, + compartment: Option, + project: Option, + cases: Vec<(String, String)>, +} + +fn env() -> Option { + let var = |name: &str| std::env::var(name).ok().filter(|v| !v.trim().is_empty()); + let (auth, profile_region, tenancy) = if let Some(bearer) = var("OCI_LIVE_API_KEY") { + (OciAuth::ApiKey { bearer }, None, None) + } else { + let profile = var("OCI_LIVE_PROFILE")?; + let path = var("OCI_LIVE_CONFIG") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(std::env::var("HOME").expect("HOME")).join(".oci/config") + }); + let loaded = load_profile(&path, &profile).expect("load OCI profile"); + (loaded.auth, loaded.region, loaded.tenancy) + }; + let cases = var("OCI_LIVE_CASES") + .unwrap_or_else(|| DEFAULT_CASES.to_string()) + .split(',') + .filter_map(|pair| { + let (transport, model) = pair.trim().split_once('=')?; + Some((transport.trim().to_string(), model.trim().to_string())) + }) + .collect(); + Some(Env { + auth, + region: var("OCI_LIVE_REGION").or(profile_region), + compartment: var("OCI_LIVE_COMPARTMENT").or(tenancy), + project: var("OCI_LIVE_PROJECT"), + cases, + }) +} + +fn transport(name: &str) -> OciTransport { + match name { + "native_generic" => OciTransport::Native { + api_format: OciApiFormat::Generic, + }, + "native_cohere" => OciTransport::Native { + api_format: OciApiFormat::Cohere, + }, + "actions" => OciTransport::OpenAiCompat { + base: OciOpenAiBase::Actions, + }, + "openai" => OciTransport::OpenAiCompat { + base: OciOpenAiBase::OpenAi, + }, + other => panic!("unknown OCI_LIVE_CASES transport `{other}`"), + } +} + +fn model_for(env: &Env, transport_name: &str, model: &str) -> Arc> { + let mut config = OciConfig::new(env.auth.clone(), model); + config.region = env.region.clone(); + config.compartment_id = env.compartment.clone(); + config.project_id = env.project.clone(); + config.transport = transport(transport_name); + build_oci_model(config).expect("build OCI model") +} + +fn weather_tool() -> ToolSchema { + ToolSchema::new( + "get_weather", + "Get the current weather for a city.", + json!({ + "type": "object", + "properties": { "city": { "type": "string", "description": "City name" } }, + "required": ["city"] + }), + ) +} + +/// Sends a temperature the way a host default does, so the provider's +/// per-model sampling rules are exercised against the live service. +fn request(messages: Vec) -> ModelRequest { + ModelRequest { + messages, + max_tokens: Some(512), + temperature: Some(0.7), + ..ModelRequest::default() + } +} + +/// One plain completion: non-empty text and reported usage. +async fn check_invoke(model: &Arc>) -> Result { + let response = model + .invoke( + &(), + request(vec![Message::user("Reply with exactly the word: pong")]), + ) + .await + .map_err(|e| e.to_string())?; + let text = Message::Assistant(response.message.clone()).text(); + if !text.to_lowercase().contains("pong") { + return Err(format!("unexpected text {text:?}")); + } + let usage = response.usage.ok_or("no usage reported")?; + Ok(format!("text={text:?} usage={usage:?}")) +} + +/// Streaming: the accumulated stream yields text and a finish reason. +async fn check_stream(model: &Arc>) -> Result { + let stream = model + .stream( + &(), + request(vec![Message::user("Count from 1 to 5, comma separated.")]), + ) + .await + .map_err(|e| e.to_string())?; + let response = collect_model_stream(stream) + .await + .map_err(|e| e.to_string())?; + let text = Message::Assistant(response.message.clone()).text(); + if !text.contains('3') { + return Err(format!("stream text missing content: {text:?}")); + } + Ok(format!( + "text={text:?} finish={:?} usage={}", + response.finish_reason, + response.usage.is_some() + )) +} + +/// Tool round trip, streamed: the model calls the tool, we answer, it uses it. +async fn check_tools(model: &Arc>) -> Result { + let ask = Message::user("What is the weather in Paris? Use the get_weather tool."); + let mut first = request(vec![ask.clone()]); + first.tools = vec![weather_tool()]; + let stream = model.stream(&(), first).await.map_err(|e| e.to_string())?; + let response = collect_model_stream(stream) + .await + .map_err(|e| e.to_string())?; + let call = response + .message + .tool_calls + .first() + .cloned() + .ok_or("model made no tool call")?; + if call.name != "get_weather" || call.invalid.is_some() { + return Err(format!("bad tool call {call:?}")); + } + let city = call.arguments["city"] + .as_str() + .unwrap_or_default() + .to_string(); + if !city.to_lowercase().contains("paris") { + return Err(format!("tool arguments missing city: {}", call.arguments)); + } + + let mut second = request(vec![ + ask, + Message::Assistant(response.message.clone()), + Message::tool(call.id.clone(), r#"{"temperature_c": 17, "sky": "cloudy"}"#), + ]); + second.tools = vec![weather_tool()]; + let answer = model.invoke(&(), second).await.map_err(|e| e.to_string())?; + let text = Message::Assistant(answer.message).text(); + if !text.contains("17") { + return Err(format!("final answer ignored the tool result: {text:?}")); + } + Ok(format!("args={} answer={text:?}", call.arguments)) +} + +#[tokio::main] +async fn main() { + let Some(env) = env() else { + eprintln!("[oci_live] skipped: set OCI_LIVE_PROFILE or OCI_LIVE_API_KEY"); + return; + }; + tinyinference_llm::allow_network_models(); + let mut failures = Vec::new(); + for (transport_name, model_id) in &env.cases { + let model = model_for(&env, transport_name, model_id); + for (check, outcome) in [ + ("invoke", timed(check_invoke(&model)).await), + ("stream", timed(check_stream(&model)).await), + ("tools", timed(check_tools(&model)).await), + ] { + let (result, ms) = outcome; + match result { + Ok(detail) => { + println!("[oci_live] PASS {transport_name} {model_id} {check} {ms}ms {detail}") + } + Err(error) => { + println!("[oci_live] FAIL {transport_name} {model_id} {check} {ms}ms {error}"); + failures.push(format!("{transport_name} {model_id} {check}: {error}")); + } + } + } + } + if !failures.is_empty() { + eprintln!("OCI live failures:\n{}", failures.join("\n")); + std::process::exit(1); + } + println!("[oci_live] all checks passed"); +} + +async fn timed>>( + future: F, +) -> (Result, u128) { + let start = Instant::now(); + let result = future.await; + (result, start.elapsed().as_millis()) +} diff --git a/crates/tinyinference-llm/src/providers/mod.rs b/crates/tinyinference-llm/src/providers/mod.rs index c4134597..87ffef2a 100644 --- a/crates/tinyinference-llm/src/providers/mod.rs +++ b/crates/tinyinference-llm/src/providers/mod.rs @@ -15,6 +15,7 @@ //! | [`MockModel`] | Implemented — deterministic, no network | //! | [`openai`] (and OpenAI-compatible endpoints) | Implemented | //! | [`anthropic`] (Messages API, including prompt caching) | Implemented | +//! | [`oci`] (OCI Generative AI: OpenAI-compatible and native chat) | Implemented | //! //! [`MockModel`] is always compiled and needs no network, keeping the default //! build offline and deterministic. The [`openai`] module is always compiled @@ -45,6 +46,7 @@ mod types; // added behind their own Cargo feature. pub mod anthropic; pub mod http; +pub mod oci; pub mod omission; pub mod openai; pub mod overrides; diff --git a/crates/tinyinference-llm/src/providers/oci/auth.rs b/crates/tinyinference-llm/src/providers/oci/auth.rs new file mode 100644 index 00000000..5348ac7c --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/auth.rs @@ -0,0 +1,350 @@ +//! OCI credential models and the signing-key sources behind them. +//! +//! [`OciAuth`] is what a host configures. Resolving it yields either a bearer +//! token (OCI Generative AI API keys) or a signing key source used by the +//! [`OciRequestSigner`](super::signer::OciRequestSigner). + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use super::signer::SigningKey; +use crate::{Error, Result}; + +/// Environment variable naming the resource-principal version. +const RP_VERSION_ENV: &str = "OCI_RESOURCE_PRINCIPAL_VERSION"; +/// Environment variable holding the RPST token (or an absolute path to it). +const RP_RPST_ENV: &str = "OCI_RESOURCE_PRINCIPAL_RPST"; +/// Environment variable holding the session private PEM (or an absolute path). +const RP_PRIVATE_PEM_ENV: &str = "OCI_RESOURCE_PRINCIPAL_PRIVATE_PEM"; +/// Environment variable holding the optional PEM passphrase (unsupported). +const RP_PASSPHRASE_ENV: &str = "OCI_RESOURCE_PRINCIPAL_PRIVATE_PEM_PASSPHRASE"; +/// Environment variable holding the resource-principal region. +const RP_REGION_ENV: &str = "OCI_RESOURCE_PRINCIPAL_REGION"; + +/// A session token plus the private key whose public half it is bound to. +#[derive(Clone)] +pub struct OciSessionMaterial { + /// The security token (the `keyId` becomes `ST$`). + pub token: String, + /// Unencrypted PEM private key matching the token. + pub private_key_pem: String, +} + +impl std::fmt::Debug for OciSessionMaterial { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("OciSessionMaterial([redacted])") + } +} + +/// Supplies fresh session material on demand (token refresh, rotated files, +/// federation). Called once per signed request, so implementations should be +/// cheap or cache internally. +pub trait OciTokenSource: Send + Sync + std::fmt::Debug { + /// Returns the current token and key. + /// + /// # Errors + /// + /// Returns an error when the material cannot be obtained. + fn fetch(&self) -> Result; +} + +/// How requests to OCI are authenticated. +#[derive(Clone)] +pub enum OciAuth { + /// An OCI Generative AI API key sent as `Authorization: Bearer`. + ApiKey { + /// The API key. + bearer: String, + }, + /// Classic user API-key signing. + UserPrincipal { + /// Tenancy OCID. + tenancy: String, + /// User OCID. + user: String, + /// Public key fingerprint (`aa:bb:...`). + fingerprint: String, + /// Unencrypted PEM private key. + private_key_pem: String, + }, + /// A session token and its key, fixed at construction. + SessionToken { + /// Security token. + token: String, + /// Unencrypted PEM private key. + private_key_pem: String, + }, + /// A session token profile whose files are re-read on every request so + /// `oci session refresh` is picked up without a restart. + SessionTokenFile { + /// Path to the security token file. + token_file: PathBuf, + /// Path to the PEM private key. + key_file: PathBuf, + }, + /// Resource principal v2.2 from `OCI_RESOURCE_PRINCIPAL_*` environment + /// variables (Functions, Container Instances, and similar). + ResourcePrincipal, + /// Instance principal via IMDS and the X.509 federation endpoint. Needs + /// the `oci-principals` cargo feature, else [`Error::Unsupported`]. + InstancePrincipal, + /// OKE workload identity via the cluster proxymux. Needs the + /// `oci-principals` cargo feature, else [`Error::Unsupported`]. + OkeWorkloadIdentity, + /// A host-supplied token source. + TokenSource(Arc), +} + +impl std::fmt::Debug for OciAuth { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ApiKey { .. } => formatter.write_str("OciAuth::ApiKey([redacted])"), + Self::UserPrincipal { + tenancy, + user, + fingerprint, + .. + } => formatter + .debug_struct("OciAuth::UserPrincipal") + .field("tenancy", tenancy) + .field("user", user) + .field("fingerprint", fingerprint) + .field("private_key_pem", &"[redacted]") + .finish(), + Self::SessionToken { .. } => formatter.write_str("OciAuth::SessionToken([redacted])"), + Self::SessionTokenFile { + token_file, + key_file, + } => formatter + .debug_struct("OciAuth::SessionTokenFile") + .field("token_file", token_file) + .field("key_file", key_file) + .finish(), + Self::ResourcePrincipal => formatter.write_str("OciAuth::ResourcePrincipal"), + Self::InstancePrincipal => formatter.write_str("OciAuth::InstancePrincipal"), + Self::OkeWorkloadIdentity => formatter.write_str("OciAuth::OkeWorkloadIdentity"), + Self::TokenSource(source) => formatter + .debug_tuple("OciAuth::TokenSource") + .field(source) + .finish(), + } + } +} + +/// Resolved auth: a bearer header or a signing key source. +#[derive(Debug)] +pub(crate) enum ResolvedAuth { + Bearer(String), + Signed(KeySource), +} + +impl OciAuth { + /// Resolves the configuration into something that can authenticate. + /// + /// Static keys are parsed (and validated) here so configuration mistakes + /// surface at build time rather than on the first request. + pub(crate) fn resolve(&self) -> Result { + match self { + Self::ApiKey { bearer } => { + if bearer.trim().is_empty() { + return Err(Error::Validation("OCI API key is empty".into())); + } + Ok(ResolvedAuth::Bearer(bearer.clone())) + } + Self::UserPrincipal { + tenancy, + user, + fingerprint, + private_key_pem, + } => Ok(ResolvedAuth::Signed(KeySource::Static { + key_id: format!("{tenancy}/{user}/{fingerprint}"), + key: Arc::new(SigningKey::from_pem(private_key_pem)?), + })), + Self::SessionToken { + token, + private_key_pem, + } => Ok(ResolvedAuth::Signed(KeySource::Static { + key_id: format!("ST${token}"), + key: Arc::new(SigningKey::from_pem(private_key_pem)?), + })), + Self::SessionTokenFile { + token_file, + key_file, + } => Ok(dynamic(Arc::new(FileSessionSource { + token_file: token_file.clone(), + key_file: key_file.clone(), + }))), + Self::ResourcePrincipal => { + ResourcePrincipalSource::validate_env()?; + Ok(dynamic(Arc::new(ResourcePrincipalSource))) + } + Self::InstancePrincipal => Ok(dynamic(super::principals::instance_source()?)), + Self::OkeWorkloadIdentity => Ok(dynamic(super::principals::oke_source()?)), + Self::TokenSource(source) => Ok(dynamic(Arc::clone(source))), + } + } + + /// Region advertised by the credential itself, when it carries one + /// (resource principal). + pub fn implied_region(&self) -> Option { + match self { + Self::ResourcePrincipal => std::env::var(RP_REGION_ENV) + .ok() + .filter(|value| !value.trim().is_empty()), + Self::InstancePrincipal => super::principals::instance_region(), + Self::OkeWorkloadIdentity => super::principals::oke_region(), + _ => None, + } + } +} + +fn dynamic(source: Arc) -> ResolvedAuth { + ResolvedAuth::Signed(KeySource::Dynamic { + source, + cache: Mutex::new(None), + }) +} + +/// Where the signing key and `keyId` come from for each request. +pub(crate) enum KeySource { + /// Fixed key and key id. + Static { + key_id: String, + key: Arc, + }, + /// Token source consulted on every request; the parsed key is cached + /// until the PEM changes. + Dynamic { + source: Arc, + cache: Mutex)>>, + }, +} + +impl std::fmt::Debug for KeySource { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Static { .. } => formatter.write_str("KeySource::Static([redacted])"), + Self::Dynamic { source, .. } => formatter + .debug_tuple("KeySource::Dynamic") + .field(source) + .finish(), + } + } +} + +impl KeySource { + /// The `keyId` and key to sign the next request with. + pub(crate) fn current(&self) -> Result<(String, Arc)> { + match self { + Self::Static { key_id, key } => Ok((key_id.clone(), Arc::clone(key))), + Self::Dynamic { source, cache } => { + let material = source.fetch()?; + let mut guard = cache + .lock() + .map_err(|_| Error::Model("OCI key cache lock poisoned".into()))?; + let key = match guard.as_ref() { + Some((pem, key)) if *pem == material.private_key_pem => Arc::clone(key), + _ => { + let key = Arc::new(SigningKey::from_pem(&material.private_key_pem)?); + *guard = Some((material.private_key_pem.clone(), Arc::clone(&key))); + key + } + }; + Ok((format!("ST${}", material.token.trim()), key)) + } + } + } +} + +fn read_trimmed(path: &std::path::Path, what: &str) -> Result { + std::fs::read_to_string(path) + .map(|text| text.trim().to_string()) + .map_err(|error| { + Error::Validation(format!( + "could not read OCI {what} file {}: {}", + path.display(), + error.kind() + )) + }) +} + +/// Session token and key files re-read on every use. +#[derive(Debug)] +struct FileSessionSource { + token_file: PathBuf, + key_file: PathBuf, +} + +impl OciTokenSource for FileSessionSource { + fn fetch(&self) -> Result { + Ok(OciSessionMaterial { + token: read_trimmed(&self.token_file, "session token")?, + private_key_pem: std::fs::read_to_string(&self.key_file).map_err(|error| { + Error::Validation(format!( + "could not read OCI key file {}: {}", + self.key_file.display(), + error.kind() + )) + })?, + }) + } +} + +/// Resource principal v2.2 read from the environment, following the OCI Go +/// SDK `common/auth` package: each value is a file path when it is an +/// absolute path, otherwise the literal material. Files are re-read per use +/// because the platform rotates them. +#[derive(Debug)] +struct ResourcePrincipalSource; + +impl ResourcePrincipalSource { + fn validate_env() -> Result<()> { + match std::env::var(RP_VERSION_ENV).as_deref() { + Ok("2.2") => {} + Ok(other) => { + return Err(Error::Unsupported(format!( + "OCI resource principal version `{other}` is not supported (only 2.2)" + ))); + } + Err(_) => { + return Err(Error::Validation(format!("{RP_VERSION_ENV} is not set"))); + } + } + if std::env::var(RP_PASSPHRASE_ENV).is_ok_and(|value| !value.is_empty()) { + return Err(Error::Unsupported( + "encrypted resource principal private keys are not supported".into(), + )); + } + for name in [RP_RPST_ENV, RP_PRIVATE_PEM_ENV, RP_REGION_ENV] { + if std::env::var(name).is_err() { + return Err(Error::Validation(format!("{name} is not set"))); + } + } + Ok(()) + } + + fn env_material(name: &str) -> Result { + let value = + std::env::var(name).map_err(|_| Error::Validation(format!("{name} is not set")))?; + if std::path::Path::new(&value).is_absolute() { + std::fs::read_to_string(&value).map_err(|error| { + Error::Validation(format!("could not read {name} file: {}", error.kind())) + }) + } else { + Ok(value) + } + } +} + +impl OciTokenSource for ResourcePrincipalSource { + fn fetch(&self) -> Result { + Ok(OciSessionMaterial { + token: Self::env_material(RP_RPST_ENV)?.trim().to_string(), + private_key_pem: Self::env_material(RP_PRIVATE_PEM_ENV)?, + }) + } +} + +#[cfg(test)] +#[path = "auth_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/auth_tests.rs b/crates/tinyinference-llm/src/providers/oci/auth_tests.rs new file mode 100644 index 00000000..338a7c63 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/auth_tests.rs @@ -0,0 +1,107 @@ +use super::*; +use crate::providers::oci::fixtures::test_pkcs8_pem; + +#[derive(Debug)] +struct FixedSource(OciSessionMaterial); + +impl OciTokenSource for FixedSource { + fn fetch(&self) -> Result { + Ok(self.0.clone()) + } +} + +fn key_id(auth: &OciAuth) -> String { + match auth.resolve().unwrap() { + ResolvedAuth::Signed(source) => source.current().unwrap().0, + ResolvedAuth::Bearer(_) => panic!("expected signed auth"), + } +} + +#[test] +fn api_key_resolves_to_bearer() { + let auth = OciAuth::ApiKey { + bearer: "sk-test".into(), + }; + assert!(matches!(auth.resolve().unwrap(), ResolvedAuth::Bearer(key) if key == "sk-test")); + assert!(OciAuth::ApiKey { bearer: " ".into() }.resolve().is_err()); +} + +#[test] +fn user_principal_key_id_is_tenancy_user_fingerprint() { + let auth = OciAuth::UserPrincipal { + tenancy: "ocid1.tenancy.x".into(), + user: "ocid1.user.y".into(), + fingerprint: "aa:bb".into(), + private_key_pem: test_pkcs8_pem().into(), + }; + assert_eq!(key_id(&auth), "ocid1.tenancy.x/ocid1.user.y/aa:bb"); +} + +#[test] +fn session_token_key_id_has_st_prefix() { + let auth = OciAuth::SessionToken { + token: "abc".into(), + private_key_pem: test_pkcs8_pem().into(), + }; + assert_eq!(key_id(&auth), "ST$abc"); +} + +#[test] +fn token_source_is_consulted_and_trimmed() { + let auth = OciAuth::TokenSource(Arc::new(FixedSource(OciSessionMaterial { + token: " rpst \n".into(), + private_key_pem: test_pkcs8_pem().into(), + }))); + assert_eq!(key_id(&auth), "ST$rpst"); +} + +#[test] +fn session_token_file_is_reread_on_each_use() { + let dir = tempfile::tempdir().unwrap(); + let token = dir.path().join("token"); + let key = dir.path().join("key.pem"); + std::fs::write(&key, test_pkcs8_pem()).unwrap(); + std::fs::write(&token, "first\n").unwrap(); + let auth = OciAuth::SessionTokenFile { + token_file: token.clone(), + key_file: key, + }; + let ResolvedAuth::Signed(source) = auth.resolve().unwrap() else { + panic!("expected signed auth"); + }; + assert_eq!(source.current().unwrap().0, "ST$first"); + std::fs::write(&token, "second").unwrap(); + assert_eq!(source.current().unwrap().0, "ST$second"); +} + +#[cfg(not(feature = "oci-principals"))] +#[test] +fn unimplemented_flows_report_unsupported() { + for auth in [OciAuth::InstancePrincipal, OciAuth::OkeWorkloadIdentity] { + assert!(matches!(auth.resolve(), Err(Error::Unsupported(_)))); + } +} + +#[test] +fn bad_static_key_fails_at_resolve_time() { + let auth = OciAuth::SessionToken { + token: "t".into(), + private_key_pem: "garbage".into(), + }; + assert!(auth.resolve().is_err()); +} + +#[test] +fn debug_redacts_secrets() { + let rendered = format!( + "{:?} {:?}", + OciAuth::ApiKey { + bearer: "sk-secret".into() + }, + OciAuth::SessionToken { + token: "tok-secret".into(), + private_key_pem: "pem-secret".into() + } + ); + assert!(!rendered.contains("secret"), "{rendered}"); +} diff --git a/crates/tinyinference-llm/src/providers/oci/config.rs b/crates/tinyinference-llm/src/providers/oci/config.rs new file mode 100644 index 00000000..bbd22e9f --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/config.rs @@ -0,0 +1,246 @@ +//! Construction policy for OCI Generative AI chat models. + +use std::sync::Arc; + +use super::OciNativeModel; +use super::auth::{OciAuth, ResolvedAuth}; +use super::signer::OciRequestSigner; +use crate::model::ChatModel; +use crate::providers::openai::{AuthStyle, OpenAiModel}; +use crate::{Error, Result}; + +/// Which OpenAI-compatible surface to call. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum OciOpenAiBase { + /// `https://inference.generativeai..oci.oraclecloud.com/openai/v1` + #[default] + OpenAi, + /// `.../20231130/actions/v1` + Actions, +} + +/// Native chat API format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum OciApiFormat { + /// `apiFormat: GENERIC` (Meta, xAI, Google, OpenAI-family models, ...). + #[default] + Generic, + /// `apiFormat: COHERE` (Cohere command models). + Cohere, +} + +/// Which wire protocol to speak. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OciTransport { + /// Reuse the OpenAI Chat Completions adapter against an OCI base path. + OpenAiCompat { + /// Which OpenAI-compatible base path to use. + base: OciOpenAiBase, + }, + /// `POST /20231130/actions/chat`. + Native { + /// GENERIC or COHERE request/response shape. + api_format: OciApiFormat, + }, +} + +/// How the model is addressed. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum OciServing { + /// On-demand serving by model id ([`OciConfig::model`]). + #[default] + OnDemand, + /// A dedicated AI cluster endpoint. + Dedicated { + /// The endpoint OCID. + endpoint_id: String, + }, +} + +/// Fully resolved configuration for an OCI Generative AI chat model. +#[derive(Debug, Clone)] +pub struct OciConfig { + /// OCI region identifier (for example `us-chicago-1`). Ignored when + /// [`Self::endpoint`] is set; falls back to the credential's region. + pub region: Option, + /// Full service endpoint override (`https://...`, no path). + pub endpoint: Option, + /// How requests authenticate. + pub auth: OciAuth, + /// Wire protocol. + pub transport: OciTransport, + /// Compartment OCID. Required for the native transport and for IAM-signed + /// calls on either OpenAI-compatible base, where it is sent as + /// `opc-compartment-id`. + pub compartment_id: Option, + /// Generative AI project OCID, sent as `OpenAI-Project` on `/openai/v1`. + pub project_id: Option, + /// Model addressing. + pub serving: OciServing, + /// Default model id. + pub model: String, + /// Static headers attached to every request. + pub extra_headers: Vec<(String, String)>, + /// Sampling temperature that replaces the request's own, like + /// [`OpenAiConfig::temperature_override`](crate::providers::openai::OpenAiConfig::temperature_override). + /// OpenAI reasoning models still receive none. + pub temperature_override: Option, +} + +impl OciConfig { + /// Starts a configuration with the OpenAI-compatible `/openai/v1` + /// transport and on-demand serving. + pub fn new(auth: OciAuth, model: impl Into) -> Self { + Self { + region: None, + endpoint: None, + auth, + transport: OciTransport::OpenAiCompat { + base: OciOpenAiBase::OpenAi, + }, + compartment_id: None, + project_id: None, + serving: OciServing::OnDemand, + model: model.into(), + extra_headers: Vec::new(), + temperature_override: None, + } + } + + /// Resolves the service endpoint (no trailing slash, no path). + /// + /// # Errors + /// + /// [`Error::Validation`] when neither an endpoint nor a region is known, + /// or the endpoint is not `https://`. + pub fn service_endpoint(&self) -> Result { + if let Some(endpoint) = self.endpoint.as_deref().filter(|e| !e.trim().is_empty()) { + let endpoint = endpoint.trim().trim_end_matches('/'); + if !endpoint.starts_with("https://") && !is_loopback_http(endpoint) { + return Err(Error::Validation( + "OCI endpoint must start with https:// (plain http is allowed only for loopback hosts)".into(), + )); + } + return Ok(endpoint.to_string()); + } + let region = self + .region + .clone() + .or_else(|| self.auth.implied_region()) + .filter(|region| !region.trim().is_empty()) + .ok_or_else(|| Error::Validation("OCI region or endpoint is required".into()))?; + let region = region.trim(); + if !region + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-') + { + return Err(Error::Validation(format!("invalid OCI region `{region}`"))); + } + Ok(format!( + "https://inference.generativeai.{region}.oci.oraclecloud.com" + )) + } +} + +fn is_loopback_http(endpoint: &str) -> bool { + reqwest::Url::parse(endpoint).is_ok_and(|url| { + url.scheme() == "http" + && matches!(url.host_str(), Some("localhost" | "127.0.0.1" | "[::1]")) + }) +} + +/// Builds an OCI Generative AI chat model from fully resolved configuration. +/// +/// # Errors +/// +/// [`Error::Validation`] for missing region/compartment or unusable keys, +/// [`Error::Unsupported`] for auth flows that are not implemented. +pub fn build_oci_model(config: OciConfig) -> Result>> { + let endpoint = config.service_endpoint()?; + let resolved = config.auth.resolve()?; + tracing::debug!( + target: "tinyinference::oci", + transport = ?config.transport, + serving = ?config.serving, + "building oci model" + ); + match config.transport { + OciTransport::OpenAiCompat { base } => Ok(Arc::new(openai_compat_model( + &config, &endpoint, base, resolved, + ))), + OciTransport::Native { api_format } => { + let compartment_id = config + .compartment_id + .clone() + .filter(|c| !c.trim().is_empty()) + .ok_or_else(|| { + Error::Validation("OCI native chat requires a compartment_id".into()) + })?; + Ok(Arc::new( + OciNativeModel::new( + endpoint, + resolved, + compartment_id, + config.serving, + api_format, + config.model, + config.extra_headers, + ) + .with_temperature_override(config.temperature_override), + )) + } + } +} + +fn openai_compat_model( + config: &OciConfig, + endpoint: &str, + base: OciOpenAiBase, + resolved: ResolvedAuth, +) -> OpenAiModel { + let path = match base { + OciOpenAiBase::OpenAi => "/openai/v1", + OciOpenAiBase::Actions => "/20231130/actions/v1", + }; + let model_id = match &config.serving { + OciServing::OnDemand => config.model.clone(), + OciServing::Dedicated { endpoint_id } => endpoint_id.clone(), + }; + let base_url = format!("{endpoint}{path}"); + let mut model = match resolved { + ResolvedAuth::Bearer(key) => OpenAiModel::new(key), + ResolvedAuth::Signed(source) => OpenAiModel::new("") + .with_auth_style(AuthStyle::None) + .with_request_signer(Arc::new(OciRequestSigner::new(source))), + } + .with_provider("oci") + .with_base_url(base_url) + .with_model(model_id.clone()); + // OpenAI reasoning models reject a non-default temperature on OCI, the + // same as on the native API (see `request.rs`). + if crate::providers::openai::is_reasoning_model(&config.model) { + model = model.with_temperature_unsupported_models([model_id]); + } + if config.temperature_override.is_some() { + model = model.with_temperature_override(config.temperature_override); + } + if base == OciOpenAiBase::OpenAi + && let Some(project) = config.project_id.as_deref().filter(|p| !p.is_empty()) + { + model = model.with_header("OpenAI-Project", project); + } + // IAM-signed calls name the billed compartment on both bases (`/openai/v1` + // answers "Compartment ID must be provided" without it); a bearer key + // already carries its own, and the header is harmless alongside it. + if let Some(compartment) = config.compartment_id.as_deref().filter(|c| !c.is_empty()) { + model = model.with_header("opc-compartment-id", compartment); + } + for (name, value) in &config.extra_headers { + model = model.with_header(name.clone(), value.clone()); + } + model +} + +#[cfg(test)] +#[path = "config_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/config_file.rs b/crates/tinyinference-llm/src/providers/oci/config_file.rs new file mode 100644 index 00000000..5e1bb53c --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/config_file.rs @@ -0,0 +1,140 @@ +//! Parser for the OCI CLI/SDK config file (`~/.oci/config`). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use super::auth::OciAuth; +use crate::{Error, Result}; + +/// What a config-file profile resolves to. +#[derive(Debug, Clone)] +pub struct OciProfile { + /// Credentials for the profile. + pub auth: OciAuth, + /// The profile's `region`, when set. + pub region: Option, + /// The profile's `tenancy`, when set (useful as a compartment default). + pub tenancy: Option, +} + +/// Default config location: `$OCI_CLI_CONFIG_FILE`, else `~/.oci/config`. +pub fn default_config_path() -> Option { + if let Ok(path) = std::env::var("OCI_CLI_CONFIG_FILE") + && !path.trim().is_empty() + { + return Some(PathBuf::from(path)); + } + std::env::var_os("HOME").map(|home| Path::new(&home).join(".oci").join("config")) +} + +/// Reads `path` and resolves `profile` (usually `DEFAULT`). +/// +/// # Errors +/// +/// [`Error::Validation`] when the file or profile is missing or incomplete. +pub fn load_profile(path: &Path, profile: &str) -> Result { + let contents = std::fs::read_to_string(path).map_err(|error| { + Error::Validation(format!( + "could not read OCI config {}: {}", + path.display(), + error.kind() + )) + })?; + let home = std::env::var_os("HOME").map(PathBuf::from); + parse_profile(&contents, profile, home.as_deref()) +} + +/// Parses config `contents` and resolves `profile`; `home` expands a leading +/// `~` in file paths. Keys from `[DEFAULT]` are inherited unless the profile +/// overrides them. The private key file is read here for API-key profiles; a +/// session-token profile keeps its file paths so refreshes are honoured. +/// +/// # Errors +/// +/// [`Error::Validation`] for a missing profile, missing required keys, or a +/// passphrase-protected key. +pub fn parse_profile(contents: &str, profile: &str, home: Option<&Path>) -> Result { + let sections = parse_sections(contents); + let selected = sections.get(profile); + if selected.is_none() && profile != "DEFAULT" { + return Err(Error::Validation(format!( + "OCI config has no profile named `{profile}`" + ))); + } + let mut values: HashMap = sections.get("DEFAULT").cloned().unwrap_or_default(); + if let Some(selected) = selected { + values.extend(selected.clone()); + } + let get = |key: &str| values.get(key).filter(|value| !value.is_empty()).cloned(); + let need = |key: &str| { + get(key) + .ok_or_else(|| Error::Validation(format!("OCI profile `{profile}` is missing `{key}`"))) + }; + if get("pass_phrase").is_some() { + return Err(Error::Validation( + "OCI profiles with a key pass_phrase are not supported; use an unencrypted key".into(), + )); + } + let key_file = expand_home(&need("key_file")?, home); + let auth = if let Some(token_file) = get("security_token_file") { + OciAuth::SessionTokenFile { + token_file: expand_home(&token_file, home), + key_file, + } + } else { + let private_key_pem = std::fs::read_to_string(&key_file).map_err(|error| { + Error::Validation(format!( + "could not read OCI key file {}: {}", + key_file.display(), + error.kind() + )) + })?; + OciAuth::UserPrincipal { + tenancy: need("tenancy")?, + user: need("user")?, + fingerprint: need("fingerprint")?, + private_key_pem, + } + }; + Ok(OciProfile { + auth, + region: get("region"), + tenancy: get("tenancy"), + }) +} + +fn expand_home(path: &str, home: Option<&Path>) -> PathBuf { + match (path.strip_prefix("~/"), home) { + (Some(rest), Some(home)) => home.join(rest), + _ if path == "~" => home.map_or_else(|| PathBuf::from(path), Path::to_path_buf), + _ => PathBuf::from(path), + } +} + +fn parse_sections(contents: &str) -> HashMap> { + let mut sections: HashMap> = HashMap::new(); + let mut current: Option = None; + for line in contents.lines().map(str::trim) { + if line.is_empty() || line.starts_with('#') || line.starts_with(';') { + continue; + } + if let Some(name) = line + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + { + let name = name.trim().to_string(); + sections.entry(name.clone()).or_default(); + current = Some(name); + } else if let (Some(section), Some((key, value))) = (¤t, line.split_once('=')) { + sections + .entry(section.clone()) + .or_default() + .insert(key.trim().to_string(), value.trim().to_string()); + } + } + sections +} + +#[cfg(test)] +#[path = "config_file_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/config_file_tests.rs b/crates/tinyinference-llm/src/providers/oci/config_file_tests.rs new file mode 100644 index 00000000..c816d731 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/config_file_tests.rs @@ -0,0 +1,123 @@ +use super::*; +use crate::providers::oci::fixtures::test_pkcs8_pem; + +fn setup() -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join(".oci")).unwrap(); + std::fs::write(dir.path().join(".oci/key.pem"), test_pkcs8_pem()).unwrap(); + let config = "\ +# comment +[DEFAULT] +user=ocid1.user.default +fingerprint=aa:bb:cc +tenancy=ocid1.tenancy.default +region=us-ashburn-1 +key_file=~/.oci/key.pem + +[OTHER] +region = us-chicago-1 +user = ocid1.user.other + +[SESSION] +fingerprint=dd +key_file=~/.oci/key.pem +security_token_file=~/.oci/token +region=eu-frankfurt-1 + +[ENCRYPTED] +pass_phrase=hunter2 +" + .to_string(); + (dir, config) +} + +#[test] +fn default_profile_resolves_to_user_principal() { + let (dir, config) = setup(); + let profile = parse_profile(&config, "DEFAULT", Some(dir.path())).unwrap(); + assert_eq!(profile.region.as_deref(), Some("us-ashburn-1")); + assert_eq!(profile.tenancy.as_deref(), Some("ocid1.tenancy.default")); + match profile.auth { + OciAuth::UserPrincipal { + tenancy, + user, + fingerprint, + private_key_pem, + } => { + assert_eq!(tenancy, "ocid1.tenancy.default"); + assert_eq!(user, "ocid1.user.default"); + assert_eq!(fingerprint, "aa:bb:cc"); + assert_eq!(private_key_pem, test_pkcs8_pem()); + } + other => panic!("unexpected auth {other:?}"), + } +} + +#[test] +fn named_profile_inherits_from_default_and_overrides() { + let (dir, config) = setup(); + let profile = parse_profile(&config, "OTHER", Some(dir.path())).unwrap(); + assert_eq!(profile.region.as_deref(), Some("us-chicago-1")); + let OciAuth::UserPrincipal { + user, + fingerprint, + tenancy, + .. + } = profile.auth + else { + panic!("expected user principal"); + }; + assert_eq!(user, "ocid1.user.other"); + assert_eq!(fingerprint, "aa:bb:cc"); + assert_eq!(tenancy, "ocid1.tenancy.default"); +} + +#[test] +fn security_token_profile_becomes_a_session_token_file_auth() { + let (dir, config) = setup(); + let profile = parse_profile(&config, "SESSION", Some(dir.path())).unwrap(); + assert_eq!(profile.region.as_deref(), Some("eu-frankfurt-1")); + match profile.auth { + OciAuth::SessionTokenFile { + token_file, + key_file, + } => { + assert_eq!(token_file, dir.path().join(".oci/token")); + assert_eq!(key_file, dir.path().join(".oci/key.pem")); + } + other => panic!("unexpected auth {other:?}"), + } +} + +#[test] +fn missing_profile_and_passphrase_are_errors() { + let (dir, config) = setup(); + assert!(parse_profile(&config, "NOPE", Some(dir.path())).is_err()); + let error = parse_profile(&config, "ENCRYPTED", Some(dir.path())).unwrap_err(); + assert!(error.to_string().contains("pass_phrase")); +} + +#[test] +fn missing_required_key_is_named_in_the_error() { + let config = "[DEFAULT]\nkey_file=/nonexistent\n"; + let error = parse_profile(config, "DEFAULT", None) + .unwrap_err() + .to_string(); + assert!( + error.contains("key file") || error.contains("missing"), + "{error}" + ); +} + +#[test] +fn tilde_expansion_only_applies_with_a_home() { + assert_eq!( + expand_home("~/a", Some(Path::new("/h"))), + PathBuf::from("/h/a") + ); + assert_eq!(expand_home("~/a", None), PathBuf::from("~/a")); + assert_eq!( + expand_home("/abs", Some(Path::new("/h"))), + PathBuf::from("/abs") + ); +} diff --git a/crates/tinyinference-llm/src/providers/oci/config_tests.rs b/crates/tinyinference-llm/src/providers/oci/config_tests.rs new file mode 100644 index 00000000..e50de544 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/config_tests.rs @@ -0,0 +1,261 @@ +use super::*; +use crate::message::Message; +use crate::model::ModelRequest; +use crate::providers::oci::fixtures::test_pkcs8_pem; + +fn api_key() -> OciAuth { + OciAuth::ApiKey { + bearer: "sk-test".into(), + } +} + +fn signed() -> OciAuth { + OciAuth::SessionToken { + token: "tok".into(), + private_key_pem: test_pkcs8_pem().into(), + } +} + +fn config(auth: OciAuth) -> OciConfig { + let mut config = OciConfig::new(auth, "openai.gpt-4o"); + config.region = Some("us-chicago-1".into()); + config +} + +#[test] +fn endpoint_derives_from_region_or_override() { + assert_eq!( + config(api_key()).service_endpoint().unwrap(), + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + ); + let mut overridden = config(api_key()); + overridden.endpoint = Some("https://proxy.example/".into()); + assert_eq!( + overridden.service_endpoint().unwrap(), + "https://proxy.example" + ); + overridden.endpoint = Some("http://example.com".into()); + assert!(overridden.service_endpoint().is_err()); + overridden.endpoint = Some("http://127.0.0.1:9999".into()); + assert!(overridden.service_endpoint().is_ok()); + let mut missing = config(api_key()); + missing.region = None; + assert!(missing.service_endpoint().is_err()); + missing.region = Some("bad/region".into()); + assert!(missing.service_endpoint().is_err()); +} + +#[test] +fn openai_base_uses_bearer_and_project_header() { + let mut cfg = config(api_key()); + cfg.project_id = Some("ocid1.generativeaiproject.p".into()); + cfg.compartment_id = Some("ocid1.compartment.c".into()); + let endpoint = cfg.service_endpoint().unwrap(); + let ResolvedAuth::Bearer(_) = cfg.auth.resolve().unwrap() else { + panic!("bearer expected"); + }; + let model = openai_compat_model( + &cfg, + &endpoint, + OciOpenAiBase::OpenAi, + cfg.auth.resolve().unwrap(), + ); + assert_eq!( + model.base_url(), + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/openai/v1" + ); + assert_eq!(model.provider(), "oci"); + assert_eq!(model.model(), "openai.gpt-4o"); + assert_eq!(*model.auth_style(), AuthStyle::Bearer); + assert!(!model.has_request_signer()); + assert_eq!( + model.extra_headers(), + [ + ( + "OpenAI-Project".to_string(), + "ocid1.generativeaiproject.p".to_string() + ), + ( + "opc-compartment-id".to_string(), + "ocid1.compartment.c".to_string() + ) + ] + ); +} + +fn compat_body(model_id: &str) -> serde_json::Value { + let mut cfg = config(api_key()); + cfg.model = model_id.into(); + let endpoint = cfg.service_endpoint().unwrap(); + let model = openai_compat_model( + &cfg, + &endpoint, + OciOpenAiBase::OpenAi, + cfg.auth.resolve().unwrap(), + ); + let mut request = ModelRequest::new(vec![Message::user("hi")]).with_max_tokens(64); + request.temperature = Some(0.7); + serde_json::to_value(model.translate_request(&request).unwrap()).unwrap() +} + +#[test] +fn compat_path_drops_temperature_and_max_tokens_for_openai_reasoning_models() { + let body = compat_body("openai.gpt-5.5"); + assert!(body.get("temperature").is_none(), "{body}"); + assert!(body.get("max_tokens").is_none(), "{body}"); + assert_eq!(body["max_completion_tokens"], 64); + + let body = compat_body("openai.gpt-oss-120b"); + assert_eq!(body["temperature"], 0.7); + assert_eq!(body["max_tokens"], 64); + assert!(body.get("max_completion_tokens").is_none(), "{body}"); +} + +#[test] +fn compat_path_applies_the_temperature_override_except_on_reasoning_models() { + for (model_id, expected) in [("openai.gpt-oss-120b", Some(0.2)), ("openai.gpt-5.5", None)] { + let mut cfg = config(api_key()); + cfg.model = model_id.into(); + cfg.temperature_override = Some(0.2); + let endpoint = cfg.service_endpoint().unwrap(); + let model = openai_compat_model( + &cfg, + &endpoint, + OciOpenAiBase::OpenAi, + cfg.auth.resolve().unwrap(), + ); + let mut request = ModelRequest::new(vec![Message::user("hi")]); + request.temperature = Some(0.7); + let body = serde_json::to_value(model.translate_request(&request).unwrap()).unwrap(); + assert_eq!( + body.get("temperature").and_then(|t| t.as_f64()), + expected, + "{body}" + ); + } +} + +#[test] +fn native_path_applies_the_temperature_override() { + let body = |model: &OciNativeModel| { + let mut request = ModelRequest::new(vec![Message::user("hi")]); + request.temperature = Some(0.7); + let request = model.prepare(&request, false).unwrap(); + serde_json::from_slice::(request.body().unwrap().as_bytes().unwrap()) + .unwrap() + }; + let plain = native(api_key(), OciApiFormat::Generic); + assert_eq!(body(&plain)["chatRequest"]["temperature"], 0.7); + let overridden = native(api_key(), OciApiFormat::Generic).with_temperature_override(Some(0.2)); + assert_eq!(body(&overridden)["chatRequest"]["temperature"], 0.2); +} + +#[test] +fn actions_base_with_signature_auth_has_no_header_credential() { + let mut cfg = config(signed()); + cfg.compartment_id = Some("ocid1.compartment.c".into()); + cfg.project_id = Some("ignored-on-actions".into()); + let endpoint = cfg.service_endpoint().unwrap(); + let model = openai_compat_model( + &cfg, + &endpoint, + OciOpenAiBase::Actions, + cfg.auth.resolve().unwrap(), + ); + assert!(model.base_url().ends_with("/20231130/actions/v1")); + assert_eq!(*model.auth_style(), AuthStyle::None); + assert!(model.has_request_signer()); + assert_eq!( + model.extra_headers(), + [( + "opc-compartment-id".to_string(), + "ocid1.compartment.c".to_string() + )] + ); +} + +#[test] +fn dedicated_serving_addresses_the_endpoint_on_the_compat_path() { + let mut cfg = config(api_key()); + cfg.serving = OciServing::Dedicated { + endpoint_id: "ocid1.generativeaiendpoint.e".into(), + }; + let endpoint = cfg.service_endpoint().unwrap(); + let model = openai_compat_model( + &cfg, + &endpoint, + OciOpenAiBase::OpenAi, + cfg.auth.resolve().unwrap(), + ); + assert_eq!(model.model(), "ocid1.generativeaiendpoint.e"); +} + +fn native(auth: OciAuth, format: OciApiFormat) -> OciNativeModel { + let mut cfg = config(auth); + cfg.endpoint = Some("http://127.0.0.1:1".into()); + cfg.compartment_id = Some("ocid1.compartment.c".into()); + cfg.transport = OciTransport::Native { api_format: format }; + let resolved = cfg.auth.resolve().unwrap(); + OciNativeModel::new( + cfg.service_endpoint().unwrap(), + resolved, + "ocid1.compartment.c".into(), + OciServing::OnDemand, + format, + cfg.model, + vec![], + ) +} + +#[test] +fn native_bearer_request_carries_the_key_and_no_signature() { + let model = native(api_key(), OciApiFormat::Generic); + let request = model + .prepare(&ModelRequest::new(vec![Message::user("hi")]), false) + .unwrap(); + assert_eq!(request.url().path(), "/20231130/actions/chat"); + assert_eq!(request.headers()["authorization"], "Bearer sk-test"); + assert!(request.headers().get("x-content-sha256").is_none()); + assert_eq!(request.headers()["accept"], "application/json"); +} + +#[test] +fn native_signed_request_uses_signature_auth_and_digest() { + let model = native(signed(), OciApiFormat::Cohere); + let request = model + .prepare(&ModelRequest::new(vec![Message::user("hi")]), true) + .unwrap(); + let authorization = request.headers()["authorization"].to_str().unwrap(); + assert!(authorization.starts_with("Signature version=\"1\",keyId=\"ST$tok\"")); + assert!(authorization.contains("x-content-sha256 content-type content-length")); + assert!(request.headers().get("x-content-sha256").is_some()); + assert_eq!(request.headers()["accept"], "text/event-stream"); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!(body["chatRequest"]["apiFormat"], "COHERE"); +} + +#[test] +fn native_requires_a_compartment_and_unsupported_auth_fails_to_build() { + let mut cfg = config(api_key()); + cfg.transport = OciTransport::Native { + api_format: OciApiFormat::Generic, + }; + assert!(build_oci_model(cfg.clone()).is_err()); + cfg.compartment_id = Some("ocid1.compartment.c".into()); + assert!(build_oci_model(cfg).is_ok()); + #[cfg(not(feature = "oci-principals"))] + { + let unsupported = config(OciAuth::InstancePrincipal); + assert!(matches!( + build_oci_model(unsupported), + Err(Error::Unsupported(_)) + )); + } +} + +#[test] +fn native_debug_does_not_leak_the_key() { + let model = native(api_key(), OciApiFormat::Generic); + assert!(!format!("{model:?}").contains("sk-test")); +} diff --git a/crates/tinyinference-llm/src/providers/oci/fixtures.rs b/crates/tinyinference-llm/src/providers/oci/fixtures.rs new file mode 100644 index 00000000..55817354 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/fixtures.rs @@ -0,0 +1,120 @@ +//! Test key material, generated with OpenSSL once per test process. No key +//! or certificate is committed to the repository. + +use std::sync::LazyLock; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as B64; +use openssl::hash::MessageDigest; +use openssl::pkey::{PKey, Private}; +use openssl::rsa::Rsa; +use openssl::sign::Signer; + +/// Tenancy OCID embedded in the test certificate. +#[cfg(feature = "oci-principals")] +pub(super) const TEST_TENANCY: &str = "ocid1.tenancy.oc1..testtenant"; + +struct Material { + key: PKey, + pkcs8_pem: String, + pkcs1_pem: String, + #[cfg(feature = "oci-principals")] + leaf_cert_pem: String, + #[cfg(feature = "oci-principals")] + leaf_fingerprint: String, +} + +static MATERIAL: LazyLock = LazyLock::new(|| { + let rsa = Rsa::generate(2048).expect("generate test key"); + let pkcs1_pem = String::from_utf8(rsa.private_key_to_pem().unwrap()).unwrap(); + let key = PKey::from_rsa(rsa).unwrap(); + let pkcs8_pem = String::from_utf8(key.private_key_to_pem_pkcs8().unwrap()).unwrap(); + #[cfg(feature = "oci-principals")] + let (leaf_cert_pem, leaf_fingerprint) = leaf_cert(&key); + Material { + key, + pkcs8_pem, + pkcs1_pem, + #[cfg(feature = "oci-principals")] + leaf_cert_pem, + #[cfg(feature = "oci-principals")] + leaf_fingerprint, + } +}); + +/// Self-signed leaf certificate for the test key, with subject attributes +/// `opc-tenant:` and `opc-identity:...`, plus the lowercase +/// colon-separated SHA-256 fingerprint of its DER form. +#[cfg(feature = "oci-principals")] +fn leaf_cert(key: &PKey) -> (String, String) { + use openssl::asn1::Asn1Time; + use openssl::x509::{X509, X509NameBuilder}; + + let mut name = X509NameBuilder::new().unwrap(); + name.append_entry_by_text("O", "Oracle").unwrap(); + name.append_entry_by_text("OU", &format!("opc-tenant:{TEST_TENANCY}")) + .unwrap(); + name.append_entry_by_text("OU", "opc-identity:ocid1.instance.oc1..testinst") + .unwrap(); + name.append_entry_by_text("CN", "test-leaf").unwrap(); + let name = name.build(); + + let mut cert = X509::builder().unwrap(); + cert.set_version(2).unwrap(); + cert.set_subject_name(&name).unwrap(); + cert.set_issuer_name(&name).unwrap(); + cert.set_pubkey(key).unwrap(); + cert.set_not_before(&Asn1Time::days_from_now(0).unwrap()) + .unwrap(); + cert.set_not_after(&Asn1Time::days_from_now(1).unwrap()) + .unwrap(); + cert.sign(key, MessageDigest::sha256()).unwrap(); + let cert = cert.build(); + + let fingerprint = cert + .digest(MessageDigest::sha256()) + .unwrap() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(":"); + ( + String::from_utf8(cert.to_pem().unwrap()).unwrap(), + fingerprint, + ) +} + +/// PKCS#8 (`PRIVATE KEY`) PEM of the test key. +pub(super) fn test_pkcs8_pem() -> &'static str { + &MATERIAL.pkcs8_pem +} + +/// PKCS#1 (`RSA PRIVATE KEY`) PEM of the same key. +pub(super) fn test_pkcs1_pem() -> &'static str { + &MATERIAL.pkcs1_pem +} + +/// Base64 RSA-SHA256 (PKCS#1 v1.5) signature of `message` under the test key, +/// computed by OpenSSL, independently of the signer under test. +pub(super) fn openssl_signature_b64(message: &[u8]) -> String { + let mut signer = Signer::new(MessageDigest::sha256(), &MATERIAL.key).unwrap(); + signer.update(message).unwrap(); + B64.encode(signer.sign_to_vec().unwrap()) +} + +#[cfg(feature = "oci-principals")] +pub(super) fn test_leaf_cert_pem() -> &'static str { + &MATERIAL.leaf_cert_pem +} + +#[cfg(feature = "oci-principals")] +pub(super) fn test_leaf_fingerprint() -> &'static str { + &MATERIAL.leaf_fingerprint +} + +/// A PEM document with `label` armor around `body`. Tests build armor this +/// way so the source never contains a literal private-key header. +pub(super) fn pem(label: &str, body: &str) -> String { + let dashes = "-".repeat(5); + format!("{dashes}BEGIN {label}{dashes}\n{body}\n{dashes}END {label}{dashes}\n") +} diff --git a/crates/tinyinference-llm/src/providers/oci/mod.rs b/crates/tinyinference-llm/src/providers/oci/mod.rs new file mode 100644 index 00000000..07ccec7e --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/mod.rs @@ -0,0 +1,346 @@ +//! Oracle Cloud Infrastructure (OCI) Generative AI chat provider. +//! +//! Two transports share one configuration ([`OciConfig`]): +//! +//! * [`OciTransport::OpenAiCompat`] reuses the OpenAI Chat Completions +//! adapter against `/openai/v1` or `/20231130/actions/v1`, adding OCI auth +//! (a bearer API key, or an HTTP Signature applied through +//! [`RequestSigner`]). +//! * [`OciTransport::Native`] speaks `POST /20231130/actions/chat` with the +//! GENERIC or COHERE `chatRequest` shapes, including SSE streaming. +//! +//! # Authentication +//! +//! | [`OciAuth`] | Status | +//! |---|---| +//! | `ApiKey` | Implemented (`Authorization: Bearer`) | +//! | `UserPrincipal`, `SessionToken`, `SessionTokenFile` | Implemented (HTTP Signature) | +//! | `ResourcePrincipal` (v2.2, env) | Implemented | +//! | `TokenSource` | Implemented (host-supplied) | +//! | `InstancePrincipal`, `OkeWorkloadIdentity` | Implemented behind the `oci-principals` cargo feature (OpenSSL); otherwise [`Error::Unsupported`] | + +mod auth; +mod config; +mod config_file; +mod principals; +mod request; +mod response; +mod signer; +mod stream; + +#[cfg(test)] +#[path = "fixtures.rs"] +mod fixtures; + +pub use auth::{OciAuth, OciSessionMaterial, OciTokenSource}; +pub use config::{ + OciApiFormat, OciConfig, OciOpenAiBase, OciServing, OciTransport, build_oci_model, +}; +pub use config_file::{OciProfile, default_config_path, load_profile, parse_profile}; +#[cfg(feature = "oci-principals")] +pub use principals::{ + InstancePrincipalSettings, InstancePrincipalSource, OkeWorkloadIdentitySettings, + OkeWorkloadIdentitySource, +}; + +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::Value; + +use crate::model::{ + ChatModel, Modalities, ModelProfile, ModelRequest, ModelResponse, ModelStream, ProviderError, +}; +use crate::providers::RequestSigner; +use crate::{Error, Result}; + +use auth::ResolvedAuth; +use signer::OciRequestSigner; + +pub(super) const PROVIDER: &str = "oci"; +/// [`crate::message::MessageOrigin::api`] stamped on native responses. +pub(super) const API_NATIVE: &str = "actions-chat"; +const CONNECT_TIMEOUT_SECS: u64 = 30; + +enum NativeAuth { + Bearer(String), + Signed(OciRequestSigner), +} + +/// A chat model backed by OCI Generative AI's native chat API. +/// +/// Build one through [`build_oci_model`]. +pub struct OciNativeModel { + client: reqwest::Client, + url: String, + auth: NativeAuth, + compartment_id: String, + serving: OciServing, + api_format: OciApiFormat, + model: String, + profile: ModelProfile, + extra_headers: Vec<(String, String)>, + temperature_override: Option, +} + +impl std::fmt::Debug for OciNativeModel { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OciNativeModel") + .field("url", &self.url) + .field("serving", &self.serving) + .field("api_format", &self.api_format) + .field("model", &self.model) + .field("auth", &"[redacted]") + .finish() + } +} + +impl OciNativeModel { + pub(super) fn new( + endpoint: String, + auth: ResolvedAuth, + compartment_id: String, + serving: OciServing, + api_format: OciApiFormat, + model: String, + extra_headers: Vec<(String, String)>, + ) -> Self { + let auth = match auth { + ResolvedAuth::Bearer(key) => NativeAuth::Bearer(key), + ResolvedAuth::Signed(source) => NativeAuth::Signed(OciRequestSigner::new(source)), + }; + let profile = ModelProfile { + provider: Some(PROVIDER.to_string()), + model: Some(model.clone()), + modalities: Modalities { + text_in: true, + text_out: true, + image_in: api_format == OciApiFormat::Generic, + ..Modalities::default() + }, + tool_calling: true, + parallel_tool_calls: api_format == OciApiFormat::Generic, + streaming: true, + streaming_tool_chunks: true, + ..ModelProfile::default() + }; + Self { + client: reqwest::Client::builder() + .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("OCI default HTTP client configuration is valid"), + url: format!("{endpoint}/20231130/actions/chat"), + auth, + compartment_id, + serving, + api_format, + model, + profile, + extra_headers, + temperature_override: None, + } + } + + /// Replaces each request's temperature with `temperature` when set. + pub(super) fn with_temperature_override(mut self, temperature: Option) -> Self { + self.temperature_override = temperature; + self + } + + fn request_model<'a>(&'a self, request: &'a ModelRequest) -> &'a str { + request.model.as_deref().unwrap_or(&self.model) + } + + fn provider_error( + &self, + request: &ModelRequest, + message: String, + status: Option, + code: Option, + raw: Option, + retry_after: Option<&str>, + ) -> ProviderError { + let retryable = + crate::failure::classify_provider_failure(status, code.as_deref(), &message) + .is_retryable(); + ProviderError { + provider: PROVIDER.to_string(), + model: Some(self.request_model(request).to_string()), + status, + code, + message, + retryable, + retry_after_ms: tinyinference_core::parse_retry_after_ms(retry_after), + raw, + partial_message: None, + stop_reason: None, + } + } + + /// Builds the fully authenticated (bearer or signed) HTTP request. + fn prepare(&self, request: &ModelRequest, streaming: bool) -> Result { + let overridden; + let request = match self.temperature_override { + Some(temperature) => { + overridden = ModelRequest { + temperature: Some(temperature), + ..request.clone() + }; + &overridden + } + None => request, + }; + let body = request::chat_details( + request, + &self.model, + &self.serving, + &self.compartment_id, + self.api_format, + streaming, + ); + // Serialize first: the signature covers the exact bytes sent. + let bytes = serde_json::to_vec(&body)?; + let mut builder = self + .client + .post(&self.url) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header( + reqwest::header::ACCEPT, + if streaming { + "text/event-stream" + } else { + "application/json" + }, + ); + for (name, value) in &self.extra_headers { + builder = builder.header(name.as_str(), value.as_str()); + } + if let NativeAuth::Bearer(key) = &self.auth { + let mut value = reqwest::header::HeaderValue::from_str(&format!("Bearer {key}")) + .map_err(|_| Error::Validation("OCI API key is not a valid header value".into()))?; + value.set_sensitive(true); + builder = builder.header(reqwest::header::AUTHORIZATION, value); + } + builder = builder.body(bytes); + if let (false, Some(timeout_ms)) = (streaming, request.timeout_ms) { + builder = builder.timeout(Duration::from_millis(timeout_ms)); + } + let mut built = builder.build().map_err(|error| { + Error::Validation(format!("OCI request could not be built: {error}")) + })?; + if let NativeAuth::Signed(signer) = &self.auth { + signer.sign(&mut built)?; + } + Ok(built) + } + + async fn post(&self, request: &ModelRequest, streaming: bool) -> Result { + crate::network_guard::ensure_network_models_allowed()?; + let built = self.prepare(request, streaming)?; + tracing::debug!( + target: "tinyinference::oci", + streaming, + model = self.request_model(request), + "oci native chat request" + ); + let response = self.client.execute(built).await.map_err(|error| { + Error::Provider(Box::new(self.provider_error( + request, + format!("oci request failed: {error}"), + None, + None, + None, + None, + ))) + })?; + let status = response.status(); + if status.is_success() { + return Ok(response); + } + let retry_after = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let text = response.text().await.unwrap_or_default(); + Err(Error::Provider(Box::new(self.parse_error_body( + request, + status.as_u16(), + &text, + retry_after.as_deref(), + )))) + } + + /// OCI errors are `{"code":"...","message":"..."}`. + fn parse_error_body( + &self, + request: &ModelRequest, + status: u16, + text: &str, + retry_after: Option<&str>, + ) -> ProviderError { + let raw = serde_json::from_str::(text).ok(); + let message = raw + .as_ref() + .and_then(|value| value["message"].as_str()) + .filter(|message| !message.trim().is_empty()) + .unwrap_or(text) + .to_string(); + let code = raw + .as_ref() + .and_then(|value| value["code"].as_str()) + .map(str::to_string); + self.provider_error( + request, + format!("oci returned HTTP {status}: {message}"), + Some(status), + code, + raw, + retry_after, + ) + } + + fn origin_for(&self, request: &ModelRequest) -> crate::message::MessageOrigin { + crate::message::MessageOrigin { + provider: PROVIDER.to_string(), + api: API_NATIVE.to_string(), + model: self.request_model(request).to_string(), + } + } +} + +#[async_trait] +impl ChatModel for OciNativeModel { + fn profile(&self) -> Option<&ModelProfile> { + Some(&self.profile) + } + + fn cache_identity(&self) -> Option { + Some(format!("oci:{}:{}", self.url, self.model)) + } + + async fn invoke(&self, _state: &State, request: ModelRequest) -> Result { + let response = self.post(&request, false).await?; + let body: Value = response + .json() + .await + .map_err(|error| Error::Model(format!("oci response was not JSON: {error}")))?; + let origin = self.origin_for(&request); + response::parse_response(body).map(|mut response| { + response.message.origin = Some(origin); + response.inherit_correlation(request.correlation) + }) + } + + async fn stream(&self, _state: &State, request: ModelRequest) -> Result { + let response = self.post(&request, true).await?; + let model = self.request_model(&request).to_string(); + let stream = stream::into_model_stream(response, model); + Ok(match request.correlation { + Some(correlation) => stream.with_correlation(correlation), + None => stream, + }) + } +} diff --git a/crates/tinyinference-llm/src/providers/oci/principals/disabled_tests.rs b/crates/tinyinference-llm/src/providers/oci/principals/disabled_tests.rs new file mode 100644 index 00000000..b7c68504 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/disabled_tests.rs @@ -0,0 +1,11 @@ +use super::*; + +#[test] +fn flows_report_the_feature_name_when_disabled() { + for result in [instance_source(), oke_source()] { + match result { + Err(error) => assert!(error.to_string().contains("oci-principals"), "{error}"), + Ok(_) => panic!("expected the flow to be unsupported"), + } + } +} diff --git a/crates/tinyinference-llm/src/providers/oci/principals/instance.rs b/crates/tinyinference-llm/src/providers/oci/principals/instance.rs new file mode 100644 index 00000000..650c1fcd --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/instance.rs @@ -0,0 +1,262 @@ +//! Instance principal: IMDS certificates exchanged at the X.509 federation +//! endpoint for a session token (oci-go-sdk `instance_principal_key_provider.go`, +//! `federation_client.go`, `certificate_retriever.go`, `utils.go`). + +use std::time::Duration; + +use serde_json::{Value, json}; + +use super::keys::{generate_session_keypair, parse_leaf_cert, sanitize_pem}; +use super::token::{TokenCache, block_on, now_secs}; +use crate::providers::oci::auth::{OciSessionMaterial, OciTokenSource}; +use crate::providers::oci::signer::{SigningKey, sign_request_with}; +use crate::{Error, Result}; + +const DEFAULT_METADATA_BASE_URL: &str = "http://169.254.169.254/opc/v2"; +const METADATA_BASE_URL_ENV: &str = "OCI_METADATA_BASE_URL"; +const REGION_PATH: &str = "/instance/region"; +const LEAF_CERT_PATH: &str = "/identity/cert.pem"; +const LEAF_KEY_PATH: &str = "/identity/key.pem"; +const INTERMEDIATE_PATH: &str = "/identity/intermediate.pem"; +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Endpoints for the instance-principal exchange; all injectable for tests +/// and non-standard realms. +#[derive(Debug, Clone)] +pub struct InstancePrincipalSettings { + /// IMDS v2 base URL (default `http://169.254.169.254/opc/v2`, or + /// `OCI_METADATA_BASE_URL`). + pub metadata_base_url: String, + /// Federation service origin, e.g. `https://auth.us-phoenix-1.oraclecloud.com`. + /// Derived from the region (commercial realm) when `None`. + pub federation_endpoint: Option, + /// Region identifier; read from IMDS when `None`. + pub region: Option, +} + +impl InstancePrincipalSettings { + /// Defaults, honouring `OCI_METADATA_BASE_URL`. + pub fn from_env() -> Self { + let base = std::env::var(METADATA_BASE_URL_ENV) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_METADATA_BASE_URL.to_string()); + Self { + metadata_base_url: base, + federation_endpoint: None, + region: None, + } + } +} + +/// An [`OciTokenSource`] for instance principals. Caches the session token +/// and renews it five minutes before expiry; safe to share across threads. +#[derive(Debug)] +pub struct InstancePrincipalSource { + settings: InstancePrincipalSettings, + cache: TokenCache, +} + +impl InstancePrincipalSource { + /// Creates a source; nothing is fetched until the first request. + pub fn new(settings: InstancePrincipalSettings) -> Self { + Self { + settings, + cache: TokenCache::default(), + } + } +} + +impl OciTokenSource for InstancePrincipalSource { + fn fetch(&self) -> Result { + self.cache.get_or_renew(now_secs(), false, || { + block_on(renew(&self.settings, now_secs())) + }) + } +} + +/// Maps IMDS short region codes (`phx`) to full identifiers. Values that +/// already look like full identifiers pass through; this table is partial. +pub(super) fn normalize_region(raw: &str) -> String { + let raw = raw.trim().to_ascii_lowercase(); + let full = match raw.as_str() { + "phx" => "us-phoenix-1", + "iad" => "us-ashburn-1", + "ord" => "us-chicago-1", + "fra" => "eu-frankfurt-1", + "lhr" => "uk-london-1", + "ams" => "eu-amsterdam-1", + "zrh" => "eu-zurich-1", + "yyz" => "ca-toronto-1", + "yul" => "ca-montreal-1", + "nrt" => "ap-tokyo-1", + "kix" => "ap-osaka-1", + "icn" => "ap-seoul-1", + "bom" => "ap-mumbai-1", + "syd" => "ap-sydney-1", + "sin" => "ap-singapore-1", + "gru" => "sa-saopaulo-1", + "dxb" => "me-dubai-1", + "jed" => "me-jeddah-1", + other => other, + }; + full.to_string() +} + +/// Federation endpoint for a region (commercial realm `oraclecloud.com`). +pub(super) fn federation_origin(settings: &InstancePrincipalSettings, region: &str) -> String { + match &settings.federation_endpoint { + Some(endpoint) => endpoint.trim_end_matches('/').to_string(), + None => format!("https://auth.{region}.oraclecloud.com"), + } +} + +fn client() -> Result { + reqwest::Client::builder() + .timeout(HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| Error::Model(format!("could not build HTTP client: {error}"))) +} + +/// IMDS GET with the required `Authorization: Bearer Oracle` header. +async fn imds_get(client: &reqwest::Client, url: &str) -> Result { + let response = client + .get(url) + .header(reqwest::header::AUTHORIZATION, "Bearer Oracle") + .send() + .await + .map_err(|error| Error::Model(format!("IMDS request to {url} failed: {error}")))?; + let status = response.status(); + if !status.is_success() { + return Err(Error::Model(format!("IMDS {url} returned HTTP {status}"))); + } + response + .text() + .await + .map_err(|error| Error::Model(format!("IMDS response from {url} unreadable: {error}"))) +} + +/// Reads the region from IMDS (blocking). +pub(super) fn fetch_region(settings: &InstancePrincipalSettings) -> Result { + block_on(async { + let url = format!( + "{}{REGION_PATH}", + settings.metadata_base_url.trim_end_matches('/') + ); + Ok(normalize_region(&imds_get(&client()?, &url).await?)) + }) +} + +/// The JSON body of the X.509 federation request (`X509FederationDetails`). +pub(super) fn federation_body( + leaf_pem: &str, + intermediate_pems: &[String], + public_pem: &str, +) -> Value { + json!({ + "certificate": sanitize_pem(leaf_pem), + "publicKey": sanitize_pem(public_pem), + "intermediateCertificates": intermediate_pems.iter().map(|p| sanitize_pem(p)).collect::>(), + "fingerprintAlgorithm": "SHA256", + }) +} + +/// Builds the federation request signed with the leaf key. The keyId is +/// `/fed-x509-sha256/` and `host` is not signed. +pub(super) fn signed_federation_request( + client: &reqwest::Client, + origin: &str, + body: &Value, + tenancy: &str, + fingerprint: &str, + leaf_key: &SigningKey, + date: &str, +) -> Result { + let mut request = client + .post(format!("{origin}/v1/x509")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(serde_json::to_vec(body)?) + .build() + .map_err(|error| Error::Model(format!("federation request build failed: {error}")))?; + let key_id = format!("{tenancy}/fed-x509-sha256/{fingerprint}"); + sign_request_with(&mut request, &key_id, leaf_key, date, false)?; + Ok(request) +} + +/// Extracts `token` from the federation response body. +pub(super) fn parse_federation_response(body: &str) -> Result { + let value: Value = serde_json::from_str(body) + .map_err(|_| Error::Model("federation response was not JSON".into()))?; + value["token"] + .as_str() + .filter(|token| !token.is_empty()) + .map(str::to_string) + .ok_or_else(|| Error::Model("federation response carried no token".into())) +} + +async fn renew(settings: &InstancePrincipalSettings, now: i64) -> Result { + let client = client()?; + let base = settings.metadata_base_url.trim_end_matches('/'); + let leaf_pem = imds_get(&client, &format!("{base}{LEAF_CERT_PATH}")).await?; + let leaf_key_pem = imds_get(&client, &format!("{base}{LEAF_KEY_PATH}")).await?; + let intermediate = imds_get(&client, &format!("{base}{INTERMEDIATE_PATH}")).await?; + let region = match &settings.region { + Some(region) => region.clone(), + None => normalize_region(&imds_get(&client, &format!("{base}{REGION_PATH}")).await?), + }; + let leaf = parse_leaf_cert(&leaf_pem)?; + let leaf_key = SigningKey::from_pem(&leaf_key_pem)?; + let session = generate_session_keypair()?; + let body = federation_body(&leaf_pem, &[intermediate], &session.public_pem); + let origin = federation_origin(settings, ®ion); + let date = httpdate::fmt_http_date( + std::time::UNIX_EPOCH + Duration::from_secs(u64::try_from(now).unwrap_or(0)), + ); + let mut last_error = Error::Model("federation exchange not attempted".into()); + for attempt in 0..3u32 { + let request = signed_federation_request( + &client, + &origin, + &body, + &leaf.tenancy, + &leaf.fingerprint, + &leaf_key, + &date, + )?; + tracing::debug!(target: "tinyinference::oci", attempt, "instance principal federation request"); + match client.execute(request).await { + Ok(response) if response.status().is_success() => { + let text = response.text().await.map_err(|error| { + Error::Model(format!("federation response unreadable: {error}")) + })?; + return Ok(OciSessionMaterial { + token: parse_federation_response(&text)?, + private_key_pem: session.private_pem, + }); + } + Ok(response) if response.status().is_client_error() => { + // The SDK does not retry 4xx from the auth service. + return Err(Error::Model(format!( + "federation endpoint rejected the request with HTTP {}", + response.status() + ))); + } + Ok(response) => { + last_error = Error::Model(format!( + "federation endpoint returned HTTP {}", + response.status() + )); + } + Err(error) => { + last_error = Error::Model(format!("federation request failed: {error}")); + } + } + tokio::time::sleep(Duration::from_millis(1000 << attempt)).await; + } + Err(last_error) +} + +#[cfg(test)] +#[path = "instance_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/principals/instance_tests.rs b/crates/tinyinference-llm/src/providers/oci/principals/instance_tests.rs new file mode 100644 index 00000000..26071a91 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/instance_tests.rs @@ -0,0 +1,156 @@ +use super::*; +use crate::providers::oci::fixtures::{ + TEST_TENANCY, test_leaf_cert_pem, test_leaf_fingerprint, test_pkcs8_pem, +}; +use crate::providers::oci::principals::mock_http::{self, jwt}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as B64; + +const DATE: &str = "Thu, 05 Jan 2014 21:31:40 GMT"; + +#[test] +fn region_codes_are_normalized() { + assert_eq!(normalize_region("phx\n"), "us-phoenix-1"); + assert_eq!(normalize_region("us-chicago-1"), "us-chicago-1"); +} + +#[test] +fn federation_origin_prefers_the_override() { + let mut settings = InstancePrincipalSettings::from_env(); + assert_eq!( + federation_origin(&settings, "us-phoenix-1"), + "https://auth.us-phoenix-1.oraclecloud.com" + ); + settings.federation_endpoint = Some("http://127.0.0.1:1/".into()); + assert_eq!(federation_origin(&settings, "x"), "http://127.0.0.1:1"); +} + +#[test] +fn federation_body_matches_the_sdk_shape() { + let body = federation_body( + "-----BEGIN CERTIFICATE-----\nAA\n-----END CERTIFICATE-----\n", + &["-----BEGIN CERTIFICATE-----\nBB\n-----END CERTIFICATE-----\n".to_string()], + "-----BEGIN PUBLIC KEY-----\nCC\n-----END PUBLIC KEY-----\n", + ); + assert_eq!( + body, + serde_json::json!({ + "certificate": "AA", + "publicKey": "CC", + "intermediateCertificates": ["BB"], + "fingerprintAlgorithm": "SHA256", + }) + ); +} + +#[test] +fn federation_request_uses_fed_x509_sha256_key_id_and_no_host() { + let key = SigningKey::from_pem(test_pkcs8_pem()).unwrap(); + let request = signed_federation_request( + &reqwest::Client::new(), + "https://auth.us-phoenix-1.oraclecloud.com", + &serde_json::json!({"a": 1}), + TEST_TENANCY, + test_leaf_fingerprint(), + &key, + DATE, + ) + .unwrap(); + assert_eq!(request.url().path(), "/v1/x509"); + let authorization = request.headers()["authorization"].to_str().unwrap(); + assert!(authorization.contains(&format!( + "keyId=\"{TEST_TENANCY}/fed-x509-sha256/{}\"", + test_leaf_fingerprint() + ))); + assert!(authorization.contains( + "headers=\"(request-target) date x-content-sha256 content-type content-length\"" + )); +} + +#[test] +fn federation_response_requires_a_token() { + assert_eq!( + parse_federation_response(r#"{"token":"abc"}"#).unwrap(), + "abc" + ); + assert!(parse_federation_response(r#"{"token":""}"#).is_err()); + assert!(parse_federation_response("nope").is_err()); +} + +#[test] +fn full_exchange_against_a_local_mock_and_caching() { + let token = jwt(now_secs(), now_secs() + 3600); + let reply = serde_json::json!({ "token": token }).to_string(); + let (url, log) = mock_http::start(move |req| match req.path.as_str() { + "/opc/v2/identity/cert.pem" => (200, test_leaf_cert_pem().to_string()), + "/opc/v2/identity/key.pem" => (200, test_pkcs8_pem().to_string()), + "/opc/v2/identity/intermediate.pem" => (200, test_leaf_cert_pem().to_string()), + "/opc/v2/instance/region" => (200, "phx".to_string()), + "/v1/x509" => (200, reply.clone()), + _ => (404, String::new()), + }); + let source = InstancePrincipalSource::new(InstancePrincipalSettings { + metadata_base_url: format!("{url}/opc/v2"), + federation_endpoint: Some(url.clone()), + region: None, + }); + let first = source.fetch().unwrap(); + assert_eq!(first.token, token); + assert!(first.private_key_pem.contains("RSA PRIVATE KEY")); + let second = source.fetch().unwrap(); + assert_eq!(second.private_key_pem, first.private_key_pem); + + let requests = log.lock().unwrap().clone(); + let federation: Vec<_> = requests.iter().filter(|r| r.path == "/v1/x509").collect(); + assert_eq!(federation.len(), 1, "second fetch must hit the cache"); + for imds in requests.iter().filter(|r| r.path.starts_with("/opc/")) { + assert_eq!(imds.header("authorization"), Some("Bearer Oracle")); + } + let body: serde_json::Value = serde_json::from_str(&federation[0].body).unwrap(); + assert_eq!(body["fingerprintAlgorithm"], "SHA256"); + assert!(body["publicKey"].as_str().unwrap().len() > 100); + assert!( + federation[0] + .header("authorization") + .unwrap() + .contains("/fed-x509-sha256/") + ); + assert!(B64.decode(body["certificate"].as_str().unwrap()).is_ok()); +} + +#[test] +fn client_errors_from_the_federation_endpoint_are_not_retried() { + let (url, log) = mock_http::start(|req| match req.path.as_str() { + "/opc/v2/identity/cert.pem" | "/opc/v2/identity/intermediate.pem" => { + (200, test_leaf_cert_pem().to_string()) + } + "/opc/v2/identity/key.pem" => (200, test_pkcs8_pem().to_string()), + "/v1/x509" => (401, "{}".into()), + _ => (404, String::new()), + }); + let source = InstancePrincipalSource::new(InstancePrincipalSettings { + metadata_base_url: format!("{url}/opc/v2"), + federation_endpoint: Some(url), + region: Some("us-phoenix-1".into()), + }); + let error = source.fetch().unwrap_err().to_string(); + assert!(error.contains("401"), "{error}"); + let calls = log + .lock() + .unwrap() + .iter() + .filter(|r| r.path == "/v1/x509") + .count(); + assert_eq!(calls, 1); +} + +#[test] +fn imds_failure_is_reported() { + let (url, _log) = mock_http::start(|_| (500, "boom".into())); + let source = InstancePrincipalSource::new(InstancePrincipalSettings { + metadata_base_url: format!("{url}/opc/v2"), + federation_endpoint: None, + region: Some("us-phoenix-1".into()), + }); + assert!(source.fetch().unwrap_err().to_string().contains("500")); +} diff --git a/crates/tinyinference-llm/src/providers/oci/principals/keys.rs b/crates/tinyinference-llm/src/providers/oci/principals/keys.rs new file mode 100644 index 00000000..a2994091 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/keys.rs @@ -0,0 +1,80 @@ +//! OpenSSL-backed helpers: RSA session keypairs and leaf-certificate parsing. + +use openssl::hash::{MessageDigest, hash}; +use openssl::pkey::PKey; +use openssl::rsa::Rsa; +use openssl::x509::X509; + +use crate::{Error, Result}; + +/// A fresh RSA-2048 session keypair. +pub(super) struct SessionKeypair { + /// PKCS#1 `RSA PRIVATE KEY` PEM (consumed by the signer). + pub(super) private_pem: String, + /// SPKI `PUBLIC KEY` PEM (what the exchange endpoints bind the token to). + pub(super) public_pem: String, +} + +pub(super) fn generate_session_keypair() -> Result { + let fail = |what: &str| Error::Model(format!("session key generation failed ({what})")); + let rsa = Rsa::generate(2048).map_err(|_| fail("generate"))?; + let private_pem = String::from_utf8(rsa.private_key_to_pem().map_err(|_| fail("private pem"))?) + .map_err(|_| fail("private utf8"))?; + let public = PKey::from_rsa(rsa).map_err(|_| fail("pkey"))?; + let public_pem = String::from_utf8(public.public_key_to_pem().map_err(|_| fail("public pem"))?) + .map_err(|_| fail("public utf8"))?; + Ok(SessionKeypair { + private_pem, + public_pem, + }) +} + +/// Facts extracted from the instance leaf certificate. +#[derive(Debug, PartialEq, Eq)] +pub(super) struct LeafCert { + /// Tenancy OCID from the `opc-tenant:` subject attribute. + pub(super) tenancy: String, + /// Lowercase colon-separated SHA-256 of the DER certificate. + pub(super) fingerprint: String, +} + +pub(super) fn parse_leaf_cert(pem: &str) -> Result { + let cert = X509::from_pem(pem.as_bytes()) + .map_err(|_| Error::Model("instance leaf certificate is not valid PEM X.509".into()))?; + let tenancy = cert + .subject_name() + .entries() + .filter_map(|entry| entry.data().to_string().ok()) + .find_map(|value| value.strip_prefix("opc-tenant:").map(str::to_string)) + .ok_or_else(|| { + Error::Model("instance leaf certificate has no opc-tenant subject attribute".into()) + })?; + let der = cert + .to_der() + .map_err(|_| Error::Model("could not encode leaf certificate".into()))?; + let digest = hash(MessageDigest::sha256(), &der) + .map_err(|_| Error::Model("could not hash leaf certificate".into()))?; + let fingerprint = digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(":"); + Ok(LeafCert { + tenancy, + fingerprint, + }) +} + +/// Strips PEM armor and newlines, as the federation request expects +/// (`sanitizeCertificateString` in the SDK). +pub(super) fn sanitize_pem(pem: &str) -> String { + pem.replace("-----BEGIN CERTIFICATE-----", "") + .replace("-----END CERTIFICATE-----", "") + .replace("-----BEGIN PUBLIC KEY-----", "") + .replace("-----END PUBLIC KEY-----", "") + .replace('\n', "") +} + +#[cfg(test)] +#[path = "keys_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/principals/keys_tests.rs b/crates/tinyinference-llm/src/providers/oci/principals/keys_tests.rs new file mode 100644 index 00000000..4e150913 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/keys_tests.rs @@ -0,0 +1,39 @@ +use super::*; +use crate::providers::oci::fixtures::{ + TEST_TENANCY, pem, test_leaf_cert_pem, test_leaf_fingerprint, +}; +use crate::providers::oci::signer::SigningKey; + +#[test] +fn session_keypair_is_usable_by_the_signer() { + let pair = generate_session_keypair().unwrap(); + assert!(pair.public_pem.starts_with("-----BEGIN PUBLIC KEY-----")); + let header = pem("RSA PRIVATE KEY", ""); + assert!(pair.private_pem.starts_with(header.lines().next().unwrap())); + let key = SigningKey::from_pem(&pair.private_pem).unwrap(); + assert!(!key.sign_b64(b"x").unwrap().is_empty()); +} + +#[test] +fn leaf_cert_yields_tenancy_and_sha256_fingerprint() { + let leaf = parse_leaf_cert(test_leaf_cert_pem()).unwrap(); + assert_eq!(leaf.tenancy, TEST_TENANCY); + assert_eq!(leaf.fingerprint, test_leaf_fingerprint()); +} + +#[test] +fn bad_certificates_are_rejected() { + assert!(parse_leaf_cert("nope").is_err()); +} + +#[test] +fn sanitize_strips_armor_and_newlines() { + assert_eq!( + sanitize_pem("-----BEGIN CERTIFICATE-----\nAAA\nBBB\n-----END CERTIFICATE-----\n"), + "AAABBB" + ); + assert_eq!( + sanitize_pem("-----BEGIN PUBLIC KEY-----\nCC\n-----END PUBLIC KEY-----"), + "CC" + ); +} diff --git a/crates/tinyinference-llm/src/providers/oci/principals/mock_http.rs b/crates/tinyinference-llm/src/providers/oci/principals/mock_http.rs new file mode 100644 index 00000000..d41acb9c --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/mock_http.rs @@ -0,0 +1,102 @@ +//! A tiny loopback HTTP server for exchange tests (no external network). + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; + +/// A request captured by the mock server. +#[derive(Debug, Clone)] +pub(super) struct Recorded { + pub(super) method: String, + pub(super) path: String, + pub(super) headers: Vec<(String, String)>, + pub(super) body: String, +} + +impl Recorded { + pub(super) fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} + +/// Starts a server answering each request with `handler(&recorded) -> +/// (status, body)`. Returns the base URL and the shared request log. +pub(super) fn start( + handler: impl Fn(&Recorded) -> (u16, String) + Send + 'static, +) -> (String, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let url = format!("http://{}", listener.local_addr().unwrap()); + let log = Arc::new(Mutex::new(Vec::new())); + let thread_log = Arc::clone(&log); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let mut data = Vec::new(); + let mut chunk = [0u8; 4096]; + let header_end = loop { + let read = stream.read(&mut chunk).unwrap_or(0); + if read == 0 { + break None; + } + data.extend_from_slice(&chunk[..read]); + if let Some(pos) = data.windows(4).position(|w| w == b"\r\n\r\n") { + break Some(pos + 4); + } + }; + let Some(header_end) = header_end else { + continue; + }; + let head = String::from_utf8_lossy(&data[..header_end]).to_string(); + let mut lines = head.lines(); + let request_line = lines.next().unwrap_or_default().to_string(); + let mut parts = request_line.split_whitespace(); + let method = parts.next().unwrap_or_default().to_string(); + let path = parts.next().unwrap_or_default().to_string(); + let headers: Vec<(String, String)> = lines + .filter_map(|line| line.split_once(':')) + .map(|(k, v)| (k.trim().to_string(), v.trim().to_string())) + .collect(); + let length = headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case("content-length")) + .and_then(|(_, v)| v.parse::().ok()) + .unwrap_or(0); + while data.len() < header_end + length { + let read = stream.read(&mut chunk).unwrap_or(0); + if read == 0 { + break; + } + data.extend_from_slice(&chunk[..read]); + } + let body = String::from_utf8_lossy(&data[header_end..]).to_string(); + let recorded = Recorded { + method, + path, + headers, + body, + }; + let (status, reply) = handler(&recorded); + thread_log.lock().unwrap().push(recorded); + let response = format!( + "HTTP/1.1 {status} X\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{reply}", + reply.len() + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + (url, log) +} + +/// Builds an unsigned JWT-shaped token with the given claims. +pub(super) fn jwt(iat: i64, exp: i64) -> String { + use base64::Engine as _; + use base64::engine::general_purpose::URL_SAFE_NO_PAD as B; + format!( + "{}.{}.sig", + B.encode(r#"{"alg":"RS256"}"#), + B.encode(format!(r#"{{"iat":{iat},"exp":{exp}}}"#)) + ) +} diff --git a/crates/tinyinference-llm/src/providers/oci/principals/mod.rs b/crates/tinyinference-llm/src/providers/oci/principals/mod.rs new file mode 100644 index 00000000..a3d6e4ef --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/mod.rs @@ -0,0 +1,89 @@ +//! Instance principal and OKE workload identity auth. +//! +//! Both flows exchange node/pod identity for a short-lived OCI session token +//! bound to a freshly generated RSA session keypair, then sign requests with +//! `ST$` (oci-go-sdk `common/auth`). They need OpenSSL for RSA keygen +//! and X.509 parsing, so they sit behind the `oci-principals` cargo feature; +//! without it [`instance_source`] and [`oke_source`] return +//! [`Error::Unsupported`](crate::Error::Unsupported) naming the feature. + +#[cfg(feature = "oci-principals")] +mod instance; +#[cfg(feature = "oci-principals")] +mod keys; +#[cfg(all(test, feature = "oci-principals"))] +mod mock_http; +#[cfg(feature = "oci-principals")] +mod oke; +#[cfg(feature = "oci-principals")] +mod token; + +#[cfg(feature = "oci-principals")] +pub use instance::{InstancePrincipalSettings, InstancePrincipalSource}; +#[cfg(feature = "oci-principals")] +pub use oke::{OkeWorkloadIdentitySettings, OkeWorkloadIdentitySource}; + +use std::sync::Arc; + +use super::auth::OciTokenSource; +use crate::Result; + +/// Instance-principal token source using IMDS defaults (honours +/// `OCI_METADATA_BASE_URL`). +#[cfg(feature = "oci-principals")] +pub(super) fn instance_source() -> Result> { + Ok(Arc::new(InstancePrincipalSource::new( + InstancePrincipalSettings::from_env(), + ))) +} + +/// OKE workload-identity token source from the pod environment. +#[cfg(feature = "oci-principals")] +pub(super) fn oke_source() -> Result> { + Ok(Arc::new(OkeWorkloadIdentitySource::new( + OkeWorkloadIdentitySettings::from_env()?, + ))) +} + +/// Region implied by OKE workload identity (`OCI_RESOURCE_PRINCIPAL_REGION`). +#[cfg(feature = "oci-principals")] +pub(super) fn oke_region() -> Option { + oke::region_from_env() +} + +/// Region implied by instance principal (read from IMDS). +#[cfg(feature = "oci-principals")] +pub(super) fn instance_region() -> Option { + instance::fetch_region(&InstancePrincipalSettings::from_env()).ok() +} + +#[cfg(not(feature = "oci-principals"))] +fn disabled(flow: &str) -> crate::Error { + crate::Error::Unsupported(format!( + "OCI {flow} auth requires the `oci-principals` cargo feature of tinyinference-llm (it needs OpenSSL for RSA keygen and X.509 parsing)" + )) +} + +#[cfg(not(feature = "oci-principals"))] +pub(super) fn instance_source() -> Result> { + Err(disabled("instance principal")) +} + +#[cfg(not(feature = "oci-principals"))] +pub(super) fn oke_source() -> Result> { + Err(disabled("OKE workload identity")) +} + +#[cfg(not(feature = "oci-principals"))] +pub(super) fn oke_region() -> Option { + None +} + +#[cfg(not(feature = "oci-principals"))] +pub(super) fn instance_region() -> Option { + None +} + +#[cfg(all(test, not(feature = "oci-principals")))] +#[path = "disabled_tests.rs"] +mod disabled_tests; diff --git a/crates/tinyinference-llm/src/providers/oci/principals/oke.rs b/crates/tinyinference-llm/src/providers/oci/principals/oke.rs new file mode 100644 index 00000000..33751380 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/oke.rs @@ -0,0 +1,204 @@ +//! OKE workload identity: a Kubernetes service-account token exchanged at the +//! cluster's proxymux for a session token (oci-go-sdk +//! `federation_client_oke_workload_identity.go`, `resource_principal_key_provider.go`). + +use std::path::PathBuf; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as B64; +use serde_json::{Value, json}; + +use super::keys::generate_session_keypair; +use super::token::{TokenCache, block_on, now_secs}; +use crate::providers::oci::auth::{OciSessionMaterial, OciTokenSource}; +use crate::{Error, Result}; + +const SA_TOKEN_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token"; +const SA_CA_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; +const CA_PATH_ENV: &str = "OCI_KUBERNETES_SERVICE_ACCOUNT_CERT_PATH"; +const SERVICE_HOST_ENV: &str = "KUBERNETES_SERVICE_HOST"; +const REGION_ENV: &str = "OCI_RESOURCE_PRINCIPAL_REGION"; +const PROXYMUX_PORT: &str = "12250"; +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Locations for the workload-identity exchange; all injectable. +#[derive(Debug, Clone)] +pub struct OkeWorkloadIdentitySettings { + /// Full proxymux URL, normally + /// `https://$KUBERNETES_SERVICE_HOST:12250/resourcePrincipalSessionTokens`. + pub proxymux_url: String, + /// Service-account token file (re-read on every renewal; it rotates). + pub service_account_token_path: PathBuf, + /// CA bundle trusted for the proxymux (the only roots used, as in the SDK). + pub ca_cert_path: PathBuf, +} + +impl OkeWorkloadIdentitySettings { + /// Settings from the pod environment (`KUBERNETES_SERVICE_HOST`, + /// `OCI_KUBERNETES_SERVICE_ACCOUNT_CERT_PATH`) and standard paths. + /// + /// # Errors + /// + /// [`Error::Validation`] when `KUBERNETES_SERVICE_HOST` is unset. + pub fn from_env() -> Result { + let host = std::env::var(SERVICE_HOST_ENV) + .ok() + .filter(|value| !value.is_empty()) + .ok_or_else(|| Error::Validation(format!("{SERVICE_HOST_ENV} is not set")))?; + let ca = std::env::var(CA_PATH_ENV) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| SA_CA_PATH.to_string()); + Ok(Self { + proxymux_url: format!("https://{host}:{PROXYMUX_PORT}/resourcePrincipalSessionTokens"), + service_account_token_path: PathBuf::from(SA_TOKEN_PATH), + ca_cert_path: PathBuf::from(ca), + }) + } +} + +/// An [`OciTokenSource`] for OKE workload identity. Caches the session token +/// and renews it at half its lifetime (never later than five minutes before +/// expiry); safe to share across threads. +#[derive(Debug)] +pub struct OkeWorkloadIdentitySource { + settings: OkeWorkloadIdentitySettings, + cache: TokenCache, +} + +impl OkeWorkloadIdentitySource { + /// Creates a source; nothing is fetched until the first request. + pub fn new(settings: OkeWorkloadIdentitySettings) -> Self { + Self { + settings, + cache: TokenCache::default(), + } + } +} + +impl OciTokenSource for OkeWorkloadIdentitySource { + fn fetch(&self) -> Result { + self.cache + .get_or_renew(now_secs(), true, || block_on(renew(&self.settings))) + } +} + +/// Region from `OCI_RESOURCE_PRINCIPAL_REGION` (required by the SDK flow). +pub(super) fn region_from_env() -> Option { + std::env::var(REGION_ENV) + .ok() + .filter(|v| !v.trim().is_empty()) +} + +/// Request body: `{"podKey": ""}` (unsanitized PEM). +pub(super) fn proxymux_body(public_pem: &str) -> Value { + json!({ "podKey": public_pem }) +} + +/// Decodes the proxymux reply: a JSON string holding base64 of +/// `{"token": "<3-char prefix>"}`; the first three characters of the +/// token are dropped (the SDK does `token[3:]`). +pub(super) fn parse_proxymux_response(body: &str) -> Result { + let invalid = |what: &str| Error::Model(format!("invalid workload identity response: {what}")); + let inner: String = serde_json::from_str(body.trim()) + .or_else(|_| { + // The SDK strips the first and last byte instead of JSON-decoding. + let trimmed = body.trim(); + trimmed + .get(1..trimmed.len().saturating_sub(1)) + .map(str::to_string) + .ok_or(()) + }) + .map_err(|_| invalid("body"))?; + let decoded = B64 + .decode(inner.as_bytes()) + .map_err(|_| invalid("base64"))?; + let value: Value = serde_json::from_slice(&decoded).map_err(|_| invalid("json"))?; + let token = value + .get("token") + .or_else(|| value.get("Token")) + .and_then(Value::as_str) + .ok_or_else(|| invalid("missing token"))?; + if token.is_empty() { + return Err(invalid("empty token")); + } + token + .get(3..) + .filter(|rest| !rest.is_empty()) + .map(str::to_string) + .ok_or_else(|| invalid("token too short")) +} + +fn http_client(ca_path: &std::path::Path) -> Result { + let mut builder = reqwest::Client::builder() + .timeout(HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()); + let pem = std::fs::read(ca_path).map_err(|error| { + Error::Validation(format!( + "could not read Kubernetes CA {}: {}", + ca_path.display(), + error.kind() + )) + })?; + let certs = reqwest::Certificate::from_pem_bundle(&pem) + .map_err(|_| Error::Validation("Kubernetes CA bundle is not valid PEM".into()))?; + builder = builder.tls_built_in_root_certs(false); + for cert in certs { + builder = builder.add_root_certificate(cert); + } + builder + .build() + .map_err(|error| Error::Model(format!("could not build HTTP client: {error}"))) +} + +async fn renew(settings: &OkeWorkloadIdentitySettings) -> Result { + let client = http_client(&settings.ca_cert_path)?; + let sa_token = std::fs::read_to_string(&settings.service_account_token_path) + .map_err(|error| { + Error::Validation(format!( + "could not read service account token {}: {}", + settings.service_account_token_path.display(), + error.kind() + )) + })? + .trim() + .to_string(); + let session = generate_session_keypair()?; + let mut authorization = + reqwest::header::HeaderValue::from_str(&format!("Bearer {sa_token}")) + .map_err(|_| Error::Validation("service account token is not a valid header".into()))?; + authorization.set_sensitive(true); + tracing::debug!(target: "tinyinference::oci", "oke workload identity exchange"); + let response = client + .post(&settings.proxymux_url) + .header(reqwest::header::AUTHORIZATION, authorization) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(serde_json::to_vec(&proxymux_body(&session.public_pem))?) + .send() + .await + .map_err(|error| Error::Model(format!("workload identity request failed: {error}")))?; + let status = response.status(); + let text = response + .text() + .await + .map_err(|error| Error::Model(format!("workload identity response unreadable: {error}")))?; + if status == reqwest::StatusCode::FORBIDDEN { + return Err(Error::Model( + "workload identity request was forbidden; ensure the cluster type is enhanced".into(), + )); + } + if !status.is_success() { + return Err(Error::Model(format!( + "workload identity exchange failed with HTTP {status}" + ))); + } + Ok(OciSessionMaterial { + token: parse_proxymux_response(&text)?, + private_key_pem: session.private_pem, + }) +} + +#[cfg(test)] +#[path = "oke_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/principals/oke_tests.rs b/crates/tinyinference-llm/src/providers/oci/principals/oke_tests.rs new file mode 100644 index 00000000..f53e747c --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/oke_tests.rs @@ -0,0 +1,122 @@ +use super::*; +use crate::providers::oci::fixtures::test_leaf_cert_pem; +use crate::providers::oci::principals::mock_http::{self, jwt}; + +fn wrap(token: &str) -> String { + let inner = B64.encode(json!({ "token": token }).to_string()); + format!("\"{inner}\"") +} + +#[test] +fn body_carries_the_pem_public_key_unsanitized() { + let pem = "-----BEGIN PUBLIC KEY-----\nAA\n-----END PUBLIC KEY-----\n"; + assert_eq!(proxymux_body(pem), json!({ "podKey": pem })); +} + +#[test] +fn response_is_unwrapped_decoded_and_prefix_stripped() { + assert_eq!(parse_proxymux_response(&wrap("ST$a.b.c")).unwrap(), "a.b.c"); +} + +#[test] +fn legacy_capitalised_token_field_is_accepted() { + let inner = B64.encode(r#"{"Token":"ST$x.y.z"}"#); + assert_eq!( + parse_proxymux_response(&format!("\"{inner}\"")).unwrap(), + "x.y.z" + ); +} + +#[test] +fn bad_responses_are_errors() { + let empty = B64.encode(r#"{"token":""}"#); + let short = B64.encode(r#"{"token":"ST"}"#); + for body in [ + "".to_string(), + "\"!!!\"".to_string(), + format!("\"{}\"", B64.encode("not json")), + format!("\"{empty}\""), + format!("\"{short}\""), + format!("\"{}\"", B64.encode("{}")), + ] { + assert!(parse_proxymux_response(&body).is_err(), "{body}"); + } +} + +fn settings(dir: &std::path::Path, url: &str) -> OkeWorkloadIdentitySettings { + std::fs::write(dir.join("token"), "sa-token-value\n").unwrap(); + std::fs::write(dir.join("ca.crt"), test_leaf_cert_pem()).unwrap(); + OkeWorkloadIdentitySettings { + proxymux_url: format!("{url}/resourcePrincipalSessionTokens"), + service_account_token_path: dir.join("token"), + ca_cert_path: dir.join("ca.crt"), + } +} + +#[test] +fn full_exchange_against_a_local_mock_and_caching() { + let dir = tempfile::tempdir().unwrap(); + let now = now_secs(); + let jwt = jwt(now, now + 7200); + let reply = wrap(&format!("ST${jwt}")); + let (url, log) = mock_http::start(move |_| (200, reply.clone())); + let source = OkeWorkloadIdentitySource::new(settings(dir.path(), &url)); + + let first = source.fetch().unwrap(); + assert_eq!(first.token, jwt); + assert!(first.private_key_pem.contains("RSA PRIVATE KEY")); + source.fetch().unwrap(); + + let requests = log.lock().unwrap().clone(); + assert_eq!(requests.len(), 1, "second fetch must hit the cache"); + let request = &requests[0]; + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/resourcePrincipalSessionTokens"); + assert_eq!( + request.header("authorization"), + Some("Bearer sa-token-value") + ); + assert_eq!(request.header("content-type"), Some("application/json")); + let body: Value = serde_json::from_str(&request.body).unwrap(); + assert!( + body["podKey"] + .as_str() + .unwrap() + .starts_with("-----BEGIN PUBLIC KEY-----") + ); +} + +#[test] +fn tokens_past_half_life_are_renewed() { + let dir = tempfile::tempdir().unwrap(); + let now = now_secs(); + // Issued long ago and expiring soon enough that the soft expiry passed. + let jwt = jwt(now - 3000, now + 1200); + let reply = wrap(&format!("ST${jwt}")); + let (url, log) = mock_http::start(move |_| (200, reply.clone())); + let source = OkeWorkloadIdentitySource::new(settings(dir.path(), &url)); + source.fetch().unwrap(); + source.fetch().unwrap(); + assert_eq!(log.lock().unwrap().len(), 2); +} + +#[test] +fn forbidden_and_missing_files_have_clear_errors() { + let dir = tempfile::tempdir().unwrap(); + let (url, _log) = mock_http::start(|_| (403, "no".into())); + let source = OkeWorkloadIdentitySource::new(settings(dir.path(), &url)); + assert!(source.fetch().unwrap_err().to_string().contains("enhanced")); + + let missing = OkeWorkloadIdentitySource::new(OkeWorkloadIdentitySettings { + proxymux_url: "http://127.0.0.1:1/x".into(), + service_account_token_path: dir.path().join("none"), + ca_cert_path: dir.path().join("ca.crt"), + }); + assert!( + missing + .fetch() + .unwrap_err() + .to_string() + .contains("service account token") + ); +} diff --git a/crates/tinyinference-llm/src/providers/oci/principals/token.rs b/crates/tinyinference-llm/src/providers/oci/principals/token.rs new file mode 100644 index 00000000..e13ba1c9 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/token.rs @@ -0,0 +1,146 @@ +//! Shared session-token plumbing: JWT expiry parsing, a thread-safe renewing +//! cache, and a blocking bridge for the async HTTP client. + +use std::future::Future; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde_json::Value; + +use crate::providers::oci::auth::OciSessionMaterial; +use crate::{Error, Result}; + +/// Tokens are renewed this long before they expire (oci-go-sdk `jwt.go`, +/// `bufferTimeBeforeTokenExpiration`). +pub(super) const REFRESH_BUFFER_SECS: i64 = 300; + +/// The `iat`/`exp` claims of a session token. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct TokenTimes { + pub(super) iat: Option, + pub(super) exp: i64, +} + +impl TokenTimes { + /// Whether the token may still be used at `now` (unix seconds). With + /// `soft`, the token is also retired at the midpoint of its lifetime + /// (`rpstValidForRatio` 0.5 in the SDK's OKE flow). + pub(super) fn valid_at(&self, now: i64, soft: bool) -> bool { + let horizon = now + REFRESH_BUFFER_SECS; + if self.exp <= horizon { + return false; + } + if soft && let Some(iat) = self.iat { + let soft_exp = iat + (self.exp - iat) / 2; + return soft_exp > horizon; + } + true + } +} + +/// Reads the (unverified) claims needed for caching from a JWT. +pub(super) fn parse_token_times(token: &str) -> Result { + let mut parts = token.split('.'); + let (Some(_), Some(payload), Some(_), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(Error::Model( + "OCI session token is not a three-part JWT".into(), + )); + }; + let bytes = URL_SAFE_NO_PAD + .decode(payload.trim_end_matches('=')) + .map_err(|_| Error::Model("OCI session token payload is not base64url".into()))?; + let claims: Value = serde_json::from_slice(&bytes) + .map_err(|_| Error::Model("OCI session token payload is not JSON".into()))?; + let exp = claims["exp"] + .as_f64() + .ok_or_else(|| Error::Model("OCI session token has no exp claim".into()))?; + Ok(TokenTimes { + iat: claims["iat"].as_f64().map(|v| v as i64), + exp: exp as i64, + }) +} + +pub(super) fn now_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs() as i64) +} + +struct Cached { + material: OciSessionMaterial, + times: TokenTimes, +} + +/// A mutex-guarded token that renews itself when it nears expiry. The lock is +/// held during renewal so concurrent callers share one exchange. +#[derive(Default)] +pub(super) struct TokenCache { + inner: Mutex>, +} + +impl std::fmt::Debug for TokenCache { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("TokenCache([redacted])") + } +} + +impl TokenCache { + /// Returns the cached material, renewing through `renew` when absent or + /// no longer valid at `now`. + pub(super) fn get_or_renew( + &self, + now: i64, + soft: bool, + renew: impl FnOnce() -> Result, + ) -> Result { + let mut guard = self + .inner + .lock() + .map_err(|_| Error::Model("OCI token cache lock poisoned".into()))?; + if let Some(cached) = guard.as_ref() + && cached.times.valid_at(now, soft) + { + return Ok(cached.material.clone()); + } + tracing::debug!(target: "tinyinference::oci", "renewing oci principal session token"); + let material = renew()?; + let times = parse_token_times(&material.token)?; + *guard = Some(Cached { + material: material.clone(), + times, + }); + Ok(material) + } +} + +/// Runs an async HTTP exchange to completion from synchronous code. +/// +/// The future runs on a scoped helper thread with its own current-thread +/// runtime, so this is safe to call from inside a Tokio worker (it blocks the +/// caller only during a renewal). +pub(super) fn block_on(future: F) -> Result +where + F: Future> + Send, + T: Send, +{ + std::thread::scope(|scope| { + scope + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| Error::Model(format!("could not start runtime: {error}")))? + .block_on(future) + }) + .join() + .map_err(|_| Error::Model("OCI principal exchange thread panicked".into()))? + }) +} + +#[cfg(test)] +#[path = "token_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/principals/token_tests.rs b/crates/tinyinference-llm/src/providers/oci/principals/token_tests.rs new file mode 100644 index 00000000..4948ad33 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/principals/token_tests.rs @@ -0,0 +1,89 @@ +use super::*; +use crate::providers::oci::principals::mock_http::jwt; +use std::cell::Cell; + +fn material(token: String) -> OciSessionMaterial { + OciSessionMaterial { + token, + private_key_pem: "pem".into(), + } +} + +#[test] +fn parses_exp_and_iat_from_a_jwt() { + let times = parse_token_times(&jwt(100, 4000)).unwrap(); + assert_eq!( + times, + TokenTimes { + iat: Some(100), + exp: 4000 + } + ); +} + +#[test] +fn rejects_malformed_tokens() { + for token in ["", "a.b", "a.b.c.d", "a.!!!.c", "a.e30.c"] { + assert!(parse_token_times(token).is_err(), "{token}"); + } +} + +#[test] +fn validity_honours_the_five_minute_buffer() { + let times = TokenTimes { + iat: None, + exp: 10_000, + }; + assert!(times.valid_at(10_000 - REFRESH_BUFFER_SECS - 1, false)); + assert!(!times.valid_at(10_000 - REFRESH_BUFFER_SECS, false)); + assert!(!times.valid_at(20_000, false)); +} + +#[test] +fn soft_validity_retires_tokens_at_half_life() { + let times = TokenTimes { + iat: Some(0), + exp: 7_200, + }; + assert!(times.valid_at(3_000, true)); + assert!(!times.valid_at(3_400, true)); + assert!(times.valid_at(3_400, false)); +} + +#[test] +fn cache_reuses_until_expiry_then_renews() { + let cache = TokenCache::default(); + let calls = Cell::new(0); + let renew = |exp: i64| { + calls.set(calls.get() + 1); + Ok(material(jwt(0, exp))) + }; + cache.get_or_renew(1_000, false, || renew(10_000)).unwrap(); + cache.get_or_renew(2_000, false, || renew(10_000)).unwrap(); + assert_eq!(calls.get(), 1); + cache.get_or_renew(9_800, false, || renew(20_000)).unwrap(); + assert_eq!(calls.get(), 2); + cache.get_or_renew(10_000, false, || renew(20_000)).unwrap(); + assert_eq!(calls.get(), 2); +} + +#[test] +fn failed_renewal_is_not_cached() { + let cache = TokenCache::default(); + assert!( + cache + .get_or_renew(0, false, || Err(Error::Model("boom".into()))) + .is_err() + ); + assert!( + cache + .get_or_renew(0, false, || Ok(material(jwt(0, 100_000)))) + .is_ok() + ); +} + +#[tokio::test] +async fn block_on_works_inside_a_runtime() { + let value = block_on(async { Ok::<_, Error>(7) }).unwrap(); + assert_eq!(value, 7); +} diff --git a/crates/tinyinference-llm/src/providers/oci/request.rs b/crates/tinyinference-llm/src/providers/oci/request.rs new file mode 100644 index 00000000..8aa52e41 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/request.rs @@ -0,0 +1,383 @@ +//! Native `POST /20231130/actions/chat` request construction (GENERIC and +//! COHERE API formats). Pure, so every mapping is testable without a network. +//! +//! Field names follow the OCI Go SDK `generativeaiinference` package +//! (`ChatDetails`, `GenericChatRequest`, `CohereChatRequest`, ...). + +use serde_json::{Map, Value, json}; + +use super::config::{OciApiFormat, OciServing}; +use crate::message::{ContentBlock, Message}; +use crate::model::{ModelRequest, ReasoningEffort, ToolChoice}; + +/// Body keys the adapter owns; the same key in `provider_options` is dropped. +const RESERVED_OPTIONS: &[&str] = &[ + "compartmentId", + "servingMode", + "chatRequest", + "apiFormat", + "messages", + "message", + "chatHistory", + "isStream", + "tools", + "toolChoice", + "toolResults", + "preambleOverride", + "maxTokens", + "maxCompletionTokens", + "temperature", + "topP", + "stop", + "stopSequences", +]; + +/// Whether `key` can be a native `chatRequest` field. +/// +/// Native OCI field names are camelCase (Go SDK `GenericChatRequest`, +/// `CohereChatRequest`). A snake_case key such as `prompt_cache_key` is an +/// OpenAI-wire option a host set for OpenAI-compatible providers; the native +/// endpoint rejects any unknown field with "Please pass in correct format of +/// request", so those keys are dropped rather than forwarded. +fn is_native_field_name(key: &str) -> bool { + !key.is_empty() && !key.contains('_') +} + +/// Builds the full `ChatDetails` body. +pub(super) fn chat_details( + request: &ModelRequest, + default_model: &str, + serving: &OciServing, + compartment_id: &str, + format: OciApiFormat, + streaming: bool, +) -> Value { + let model_id = request.model.as_deref().unwrap_or(default_model); + let serving_mode = match serving { + OciServing::OnDemand => json!({ "servingType": "ON_DEMAND", "modelId": model_id }), + OciServing::Dedicated { endpoint_id } => { + json!({ "servingType": "DEDICATED", "endpointId": endpoint_id }) + } + }; + let mut chat_request = match format { + OciApiFormat::Generic => generic_request(request, model_id), + OciApiFormat::Cohere => cohere_request(request), + }; + if streaming { + chat_request["isStream"] = Value::Bool(true); + chat_request["streamOptions"] = json!({ "isIncludeUsage": true }); + } + if let (Some(extra), Some(object)) = ( + request.provider_options.as_object(), + chat_request.as_object_mut(), + ) { + for (key, value) in extra { + if RESERVED_OPTIONS.contains(&key.as_str()) { + continue; + } + if !is_native_field_name(key) { + tracing::debug!(key = %key, "[oci] dropping non-native provider option"); + continue; + } + object.insert(key.clone(), value.clone()); + } + } + json!({ + "compartmentId": compartment_id, + "servingMode": serving_mode, + "chatRequest": chat_request, + }) +} + +fn text_part(text: &str) -> Option { + (!text.is_empty()).then(|| json!({ "type": "TEXT", "text": text })) +} + +fn generic_parts(content: &[ContentBlock], allow_images: bool) -> Vec { + content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => text_part(text), + ContentBlock::Json(value) => text_part(&value.to_string()), + ContentBlock::Image(image) if allow_images => { + Some(json!({ "type": "IMAGE", "imageUrl": { "url": image.url } })) + } + _ => None, + }) + .collect() +} + +fn plain_text(content: &[ContentBlock]) -> String { + content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text.clone()), + ContentBlock::Json(value) => Some(value.to_string()), + _ => None, + }) + .collect::>() + .join("\n") +} + +/// Whether the GENERIC token cap must be sent as `maxCompletionTokens`. +/// +/// OCI forwards OpenAI models to the OpenAI API, where the reasoning models +/// (gpt-5 family, o-series) reject `max_tokens` with `unsupported_parameter`. +/// Every current OpenAI chat model accepts `max_completion_tokens`, so the +/// whole `openai.` family uses it; other vendors keep `maxTokens`. +fn uses_max_completion_tokens(model_id: &str) -> bool { + model_id.starts_with("openai.") +} + +/// Whether the model accepts only its default sampling parameters. +/// +/// OpenAI reasoning models reject any non-default `temperature` (and +/// `top_p`), so the adapter omits both for them rather than failing the call. +fn rejects_sampling_parameters(model_id: &str) -> bool { + model_id + .strip_prefix("openai.") + .is_some_and(crate::providers::openai::is_reasoning_model) +} + +fn generic_request(request: &ModelRequest, model_id: &str) -> Value { + let mut messages: Vec = Vec::new(); + for message in &request.messages { + match message { + Message::System(m) => { + messages + .push(json!({ "role": "SYSTEM", "content": generic_parts(&m.content, false) })); + } + Message::User(m) => { + messages + .push(json!({ "role": "USER", "content": generic_parts(&m.content, true) })); + } + Message::Assistant(m) => { + let mut value = json!({ "role": "ASSISTANT" }); + let parts = generic_parts(&m.content, false); + if !parts.is_empty() { + value["content"] = Value::Array(parts); + } + if !m.tool_calls.is_empty() { + value["toolCalls"] = Value::Array( + m.tool_calls + .iter() + .map(|call| { + json!({ + "type": "FUNCTION", + "id": call.id, + "name": call.name, + "arguments": call.arguments.to_string(), + }) + }) + .collect(), + ); + } + messages.push(value); + } + Message::Tool(m) => { + let mut parts = generic_parts(&m.content, false); + if parts.is_empty() { + parts.push(json!({ "type": "TEXT", "text": "" })); + } + messages.push(json!({ + "role": "TOOL", + "toolCallId": m.tool_call_id, + "content": parts, + })); + } + Message::Custom(_) => {} + } + } + let mut body = Map::new(); + body.insert("apiFormat".into(), json!("GENERIC")); + body.insert("messages".into(), Value::Array(messages)); + if let Some(max_tokens) = request.max_tokens { + let key = if uses_max_completion_tokens(model_id) { + "maxCompletionTokens" + } else { + "maxTokens" + }; + body.insert(key.into(), json!(max_tokens)); + } + if rejects_sampling_parameters(model_id) { + if request.temperature.is_some() || request.top_p.is_some() { + tracing::debug!( + model = model_id, + "[oci] omitting temperature/top_p for reasoning model" + ); + } + } else { + if let Some(temperature) = request.temperature { + body.insert("temperature".into(), json!(temperature)); + } + if let Some(top_p) = request.top_p { + body.insert("topP".into(), json!(top_p)); + } + } + if !request.stop_sequences.is_empty() { + body.insert("stop".into(), json!(request.stop_sequences)); + } + if !request.tools.is_empty() { + let tools: Vec = request + .tools + .iter() + .map(|tool| { + json!({ + "type": "FUNCTION", + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters, + }) + }) + .collect(); + body.insert("tools".into(), Value::Array(tools)); + body.insert( + "toolChoice".into(), + match &request.tool_choice { + ToolChoice::Auto => json!({ "type": "AUTO" }), + ToolChoice::None => json!({ "type": "NONE" }), + ToolChoice::Required => json!({ "type": "REQUIRED" }), + ToolChoice::Tool(name) => json!({ "type": "FUNCTION", "name": name }), + }, + ); + } + if let Some(effort) = request.reasoning.as_ref().and_then(|r| r.effort) { + body.insert( + "reasoningEffort".into(), + json!(match effort { + ReasoningEffort::None => "NONE", + ReasoningEffort::Minimal => "MINIMAL", + ReasoningEffort::Low => "LOW", + ReasoningEffort::Medium => "MEDIUM", + ReasoningEffort::High | ReasoningEffort::XHigh => "HIGH", + }), + ); + } + Value::Object(body) +} + +/// Maps a JSON-schema `properties` object to Cohere `parameterDefinitions`. +fn cohere_tool(tool: &crate::tool::ToolSchema) -> Value { + let required: Vec<&str> = tool + .parameters + .get("required") + .and_then(Value::as_array) + .map(|names| names.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let mut definitions = Map::new(); + if let Some(properties) = tool.parameters.get("properties").and_then(Value::as_object) { + for (name, schema) in properties { + let mut definition = json!({ + "type": schema.get("type").and_then(Value::as_str).unwrap_or("string"), + "isRequired": required.contains(&name.as_str()), + }); + if let Some(description) = schema.get("description").and_then(Value::as_str) { + definition["description"] = json!(description); + } + definitions.insert(name.clone(), definition); + } + } + json!({ + "name": tool.name, + "description": tool.description, + "parameterDefinitions": definitions, + }) +} + +fn cohere_request(request: &ModelRequest) -> Value { + let mut preamble: Vec = Vec::new(); + let mut history: Vec = Vec::new(); + let mut tool_results: Vec = Vec::new(); + let mut known_calls: Vec<(String, String, Value)> = Vec::new(); + let mut message = String::new(); + + let count = request.messages.len(); + // The trailing run of Tool messages become `toolResults`; the final user + // message (when last) becomes `message`; everything else is history. + let mut trailing_tools_start = count; + while trailing_tools_start > 0 + && matches!(request.messages[trailing_tools_start - 1], Message::Tool(_)) + { + trailing_tools_start -= 1; + } + for (index, entry) in request.messages.iter().enumerate() { + match entry { + Message::System(m) => preamble.push(plain_text(&m.content)), + Message::User(m) => { + let text = plain_text(&m.content); + if index + 1 == count { + message = text; + } else { + history.push(json!({ "role": "USER", "message": text })); + } + } + Message::Assistant(m) => { + let mut value = json!({ "role": "CHATBOT", "message": plain_text(&m.content) }); + if !m.tool_calls.is_empty() { + value["toolCalls"] = Value::Array( + m.tool_calls + .iter() + .map(|c| json!({ "name": c.name, "parameters": c.arguments })) + .collect(), + ); + } + for call in &m.tool_calls { + known_calls.push((call.id.clone(), call.name.clone(), call.arguments.clone())); + } + history.push(value); + } + Message::Tool(m) => { + let (name, parameters) = known_calls + .iter() + .find(|(id, _, _)| *id == m.tool_call_id) + .map(|(_, name, args)| (name.clone(), args.clone())) + .unwrap_or_else(|| (m.tool_call_id.clone(), json!({}))); + let result = json!({ + "call": { "name": name, "parameters": parameters }, + "outputs": [{ "output": plain_text(&m.content) }], + }); + if index >= trailing_tools_start { + tool_results.push(result); + } else { + history.push(json!({ "role": "TOOL", "toolResults": [result] })); + } + } + Message::Custom(_) => {} + } + } + let mut body = Map::new(); + body.insert("apiFormat".into(), json!("COHERE")); + body.insert("message".into(), json!(message)); + if !history.is_empty() { + body.insert("chatHistory".into(), Value::Array(history)); + } + if !preamble.is_empty() { + body.insert("preambleOverride".into(), json!(preamble.join("\n\n"))); + } + if !tool_results.is_empty() { + body.insert("toolResults".into(), Value::Array(tool_results)); + } + if !request.tools.is_empty() { + body.insert( + "tools".into(), + Value::Array(request.tools.iter().map(cohere_tool).collect()), + ); + } + if let Some(max_tokens) = request.max_tokens { + body.insert("maxTokens".into(), json!(max_tokens)); + } + if let Some(temperature) = request.temperature { + body.insert("temperature".into(), json!(temperature)); + } + if let Some(top_p) = request.top_p { + body.insert("topP".into(), json!(top_p)); + } + if !request.stop_sequences.is_empty() { + body.insert("stopSequences".into(), json!(request.stop_sequences)); + } + Value::Object(body) +} + +#[cfg(test)] +#[path = "request_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/request_tests.rs b/crates/tinyinference-llm/src/providers/oci/request_tests.rs new file mode 100644 index 00000000..9cfbecb3 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/request_tests.rs @@ -0,0 +1,353 @@ +use super::*; +use crate::message::{AssistantMessage, ImageRef}; +use crate::tool::{ToolCall, ToolSchema}; + +fn weather_tool() -> ToolSchema { + ToolSchema::new( + "get_weather", + "Look up weather", + json!({ + "type": "object", + "properties": { + "city": { "type": "string", "description": "City name" }, + "days": { "type": "integer" } + }, + "required": ["city"] + }), + ) +} + +fn assistant_with_call() -> Message { + Message::Assistant(AssistantMessage { + id: None, + content: vec![], + tool_calls: vec![ToolCall::new( + "call_1", + "get_weather", + json!({"city": "Rome"}), + )], + usage: None, + origin: None, + }) +} + +fn details( + request: &ModelRequest, + serving: &OciServing, + format: OciApiFormat, + stream: bool, +) -> Value { + chat_details( + request, + "meta.llama-3.3-70b-instruct", + serving, + "ocid1.compartment.x", + format, + stream, + ) +} + +#[test] +fn generic_on_demand_envelope_and_messages() { + let mut request = ModelRequest::new(vec![ + Message::system("be brief"), + Message::user("hi"), + assistant_with_call(), + Message::tool("call_1", "sunny"), + ]); + request.max_tokens = Some(64); + request.temperature = Some(0.3); + request.top_p = Some(0.9); + request.stop_sequences = vec!["END".into()]; + request.tools = vec![weather_tool()]; + let body = details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + ); + + assert_eq!(body["compartmentId"], "ocid1.compartment.x"); + assert_eq!( + body["servingMode"], + json!({"servingType": "ON_DEMAND", "modelId": "meta.llama-3.3-70b-instruct"}) + ); + let chat = &body["chatRequest"]; + assert_eq!(chat["apiFormat"], "GENERIC"); + assert_eq!(chat["maxTokens"], 64); + assert_eq!(chat["temperature"], 0.3); + assert_eq!(chat["topP"], 0.9); + assert_eq!(chat["stop"], json!(["END"])); + assert!(chat.get("isStream").is_none()); + assert_eq!( + chat["messages"], + json!([ + {"role": "SYSTEM", "content": [{"type": "TEXT", "text": "be brief"}]}, + {"role": "USER", "content": [{"type": "TEXT", "text": "hi"}]}, + {"role": "ASSISTANT", "toolCalls": [ + {"type": "FUNCTION", "id": "call_1", "name": "get_weather", "arguments": "{\"city\":\"Rome\"}"} + ]}, + {"role": "TOOL", "toolCallId": "call_1", "content": [{"type": "TEXT", "text": "sunny"}]} + ]) + ); + assert_eq!(chat["tools"][0]["type"], "FUNCTION"); + assert_eq!(chat["tools"][0]["name"], "get_weather"); + assert_eq!(chat["tools"][0]["parameters"]["required"], json!(["city"])); + assert_eq!(chat["toolChoice"], json!({"type": "AUTO"})); +} + +#[test] +fn model_override_and_tool_choice_variants() { + let mut request = ModelRequest::new(vec![Message::user("x")]); + request.model = Some("xai.grok-4".into()); + request.tools = vec![weather_tool()]; + request.tool_choice = ToolChoice::Tool("get_weather".into()); + let body = details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + ); + assert_eq!(body["servingMode"]["modelId"], "xai.grok-4"); + assert_eq!( + body["chatRequest"]["toolChoice"], + json!({"type": "FUNCTION", "name": "get_weather"}) + ); + request.tool_choice = ToolChoice::Required; + let body = details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + ); + assert_eq!( + body["chatRequest"]["toolChoice"], + json!({"type": "REQUIRED"}) + ); +} + +#[test] +fn dedicated_serving_uses_the_endpoint_id() { + let request = ModelRequest::new(vec![Message::user("x")]); + let serving = OciServing::Dedicated { + endpoint_id: "ocid1.generativeaiendpoint.y".into(), + }; + let body = details(&request, &serving, OciApiFormat::Generic, false); + assert_eq!( + body["servingMode"], + json!({"servingType": "DEDICATED", "endpointId": "ocid1.generativeaiendpoint.y"}) + ); +} + +#[test] +fn streaming_sets_is_stream_and_include_usage() { + let request = ModelRequest::new(vec![Message::user("x")]); + for format in [OciApiFormat::Generic, OciApiFormat::Cohere] { + let body = details(&request, &OciServing::OnDemand, format, true); + assert_eq!(body["chatRequest"]["isStream"], true); + assert_eq!( + body["chatRequest"]["streamOptions"], + json!({"isIncludeUsage": true}) + ); + } +} + +#[test] +fn images_become_image_parts_and_reasoning_effort_maps() { + let mut request = ModelRequest::new(vec![Message::User(crate::message::UserMessage { + content: vec![ + ContentBlock::Text("look".into()), + ContentBlock::Image(ImageRef { + url: "https://img.example/a.png".into(), + mime_type: None, + }), + ], + })]); + request.reasoning = Some(crate::model::ReasoningConfig { + effort: Some(ReasoningEffort::XHigh), + budget_tokens: None, + summary: None, + }); + let body = details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + ); + let content = &body["chatRequest"]["messages"][0]["content"]; + assert_eq!( + content[1], + json!({"type": "IMAGE", "imageUrl": {"url": "https://img.example/a.png"}}) + ); + assert_eq!(body["chatRequest"]["reasoningEffort"], "HIGH"); +} + +#[test] +fn provider_options_merge_but_reserved_keys_are_dropped() { + let mut request = ModelRequest::new(vec![Message::user("x")]); + request.provider_options = json!({"seed": 7, "servingMode": "evil", "messages": []}); + let body = details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + ); + assert_eq!(body["chatRequest"]["seed"], 7); + assert_eq!(body["servingMode"]["servingType"], "ON_DEMAND"); + assert_eq!(body["chatRequest"]["messages"].as_array().unwrap().len(), 1); +} + +#[test] +fn cohere_maps_message_history_preamble_and_tools() { + let mut request = ModelRequest::new(vec![ + Message::system("You are terse."), + Message::user("first"), + Message::assistant("reply"), + Message::user("second"), + ]); + request.max_tokens = Some(10); + request.stop_sequences = vec!["X".into()]; + request.tools = vec![weather_tool()]; + let body = details(&request, &OciServing::OnDemand, OciApiFormat::Cohere, false); + let chat = &body["chatRequest"]; + assert_eq!(chat["apiFormat"], "COHERE"); + assert_eq!(chat["message"], "second"); + assert_eq!(chat["preambleOverride"], "You are terse."); + assert_eq!( + chat["chatHistory"], + json!([ + {"role": "USER", "message": "first"}, + {"role": "CHATBOT", "message": "reply"} + ]) + ); + assert_eq!(chat["maxTokens"], 10); + assert_eq!(chat["stopSequences"], json!(["X"])); + assert_eq!( + chat["tools"][0]["parameterDefinitions"], + json!({ + "city": {"type": "string", "isRequired": true, "description": "City name"}, + "days": {"type": "integer", "isRequired": false} + }) + ); +} + +#[test] +fn cohere_trailing_tool_messages_become_tool_results() { + let request = ModelRequest::new(vec![ + Message::user("weather?"), + assistant_with_call(), + Message::tool("call_1", "sunny"), + ]); + let body = details(&request, &OciServing::OnDemand, OciApiFormat::Cohere, false); + let chat = &body["chatRequest"]; + assert_eq!(chat["message"], ""); + assert_eq!( + chat["toolResults"], + json!([{ + "call": {"name": "get_weather", "parameters": {"city": "Rome"}}, + "outputs": [{"output": "sunny"}] + }]) + ); + assert_eq!( + chat["chatHistory"][1]["toolCalls"][0]["name"], + "get_weather" + ); +} + +#[test] +fn generic_openai_models_send_max_completion_tokens() { + // OCI forwards `openai.*` to OpenAI, whose reasoning models reject + // `max_tokens` (`unsupported_parameter`); seen live with openai.gpt-5.5. + let mut request = ModelRequest::new(vec![Message::user("hi")]); + request.max_tokens = Some(64); + let body = chat_details( + &request, + "openai.gpt-5.5", + &OciServing::OnDemand, + "ocid1.compartment.x", + OciApiFormat::Generic, + false, + ); + let chat = &body["chatRequest"]; + assert_eq!(chat["maxCompletionTokens"], 64); + assert!(chat.get("maxTokens").is_none()); +} + +#[test] +fn generic_request_model_override_selects_the_token_field() { + let mut request = ModelRequest::new(vec![Message::user("hi")]); + request.max_tokens = Some(8); + request.model = Some("openai.o4-mini".into()); + let chat = &details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + )["chatRequest"]; + assert_eq!(chat["maxCompletionTokens"], 8); + + request.model = Some("meta.llama-4-scout-17b-16e-instruct".into()); + let chat = &details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + )["chatRequest"]; + assert_eq!(chat["maxTokens"], 8); + assert!(chat.get("maxCompletionTokens").is_none()); +} + +#[test] +fn generic_reasoning_models_omit_sampling_parameters() { + // Live: openai.gpt-5.5 rejects temperature 0.7 ("only the default (1)"). + let mut request = ModelRequest::new(vec![Message::user("hi")]); + request.temperature = Some(0.7); + request.top_p = Some(0.9); + for model in ["openai.gpt-5.5", "openai.gpt-6-sol", "openai.o4-mini"] { + request.model = Some(model.into()); + let chat = &details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + )["chatRequest"]; + assert!(chat.get("temperature").is_none(), "{model}"); + assert!(chat.get("topP").is_none(), "{model}"); + } + for model in [ + "openai.gpt-4.1", + "meta.llama-3.3-70b-instruct", + "xai.grok-4.7", + ] { + request.model = Some(model.into()); + let chat = &details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + )["chatRequest"]; + assert_eq!(chat["temperature"], 0.7, "{model}"); + assert_eq!(chat["topP"], 0.9, "{model}"); + } +} + +#[test] +fn native_drops_openai_wire_provider_options() { + // Live: the OpenHuman harness sets `prompt_cache_key`; the native endpoint + // answered 400 "Please pass in correct format of request" until it was dropped. + let mut request = ModelRequest::new(vec![Message::user("hi")]); + request.provider_options = json!({ + "prompt_cache_key": "tap-1", + "reasoningEffort": "LOW", + "seed": 7 + }); + let chat = &details( + &request, + &OciServing::OnDemand, + OciApiFormat::Generic, + false, + )["chatRequest"]; + assert!(chat.get("prompt_cache_key").is_none()); + assert_eq!(chat["reasoningEffort"], "LOW"); + assert_eq!(chat["seed"], 7); +} diff --git a/crates/tinyinference-llm/src/providers/oci/response.rs b/crates/tinyinference-llm/src/providers/oci/response.rs new file mode 100644 index 00000000..15cc6929 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/response.rs @@ -0,0 +1,170 @@ +//! Native chat response parsing (`ChatResult` with a GENERIC or COHERE +//! `chatResponse`), shared by the unary and streaming paths. + +use serde_json::Value; + +use crate::message::{AssistantMessage, ContentBlock}; +use crate::model::ModelResponse; +use crate::tool::ToolCall; +use crate::usage::Usage; +use crate::{Error, Result}; + +fn malformed(message: &str) -> Error { + Error::Model(format!("invalid OCI GenAI response: {message}")) +} + +/// Maps an OCI `usage` object (`promptTokens`, `completionTokens`, +/// `totalTokens`, `promptTokensDetails.cachedTokens`, +/// `completionTokensDetails.reasoningTokens`) onto [`Usage`]. +pub(super) fn parse_usage(usage: &Value) -> Usage { + let count = |value: &Value| value.as_u64().unwrap_or(0); + let input_tokens = count(&usage["promptTokens"]); + let output_tokens = count(&usage["completionTokens"]); + let total = count(&usage["totalTokens"]); + Usage { + input_tokens, + output_tokens, + total_tokens: if total > 0 { + total + } else { + input_tokens + output_tokens + }, + cache_read_tokens: count(&usage["promptTokensDetails"]["cachedTokens"]), + reasoning_tokens: count(&usage["completionTokensDetails"]["reasoningTokens"]), + ..Usage::default() + } +} + +/// Builds a [`ToolCall`], keeping malformed argument JSON as an invalid call +/// the agent loop can feed back instead of failing the turn. +pub(super) fn tool_call(id: &str, name: &str, arguments: &str) -> ToolCall { + if arguments.trim().is_empty() { + return ToolCall::new(id, name, Value::Object(Default::default())); + } + match serde_json::from_str::(arguments) { + Ok(value) => ToolCall::new(id, name, value), + Err(error) => ToolCall::invalid(id, name, arguments.to_string(), error.to_string()), + } +} + +/// Parses a unary `ChatResult` body. +pub(crate) fn parse_response(body: Value) -> Result { + let chat = body + .get("chatResponse") + .filter(|value| value.is_object()) + .ok_or_else(|| malformed("chatResponse is required"))?; + let (content, tool_calls, finish_reason, usage) = match chat["apiFormat"].as_str() { + Some("COHERE") => parse_cohere(chat), + Some("GENERIC") | None => parse_generic(chat)?, + Some(other) => return Err(malformed(&format!("unsupported apiFormat `{other}`"))), + }; + Ok(ModelResponse { + message: AssistantMessage { + id: None, + content, + tool_calls, + usage, + origin: None, + }, + usage, + finish_reason, + raw: Some(body), + resolved_model: None, + continue_turn: None, + served_from_cache: false, + correlation: None, + resolved_route: None, + }) +} + +type Parsed = ( + Vec, + Vec, + Option, + Option, +); + +fn parse_generic(chat: &Value) -> Result { + let choice = chat["choices"] + .as_array() + .and_then(|choices| choices.first()) + .ok_or_else(|| malformed("choices must be a non-empty array"))?; + let message = &choice["message"]; + let mut content = Vec::new(); + if let Some(reasoning) = message["reasoningContent"] + .as_str() + .filter(|t| !t.is_empty()) + { + content.push(ContentBlock::Thinking { + text: reasoning.to_string(), + signature: None, + }); + } + for part in message["content"].as_array().into_iter().flatten() { + if part["type"].as_str() == Some("TEXT") + && let Some(text) = part["text"].as_str().filter(|t| !t.is_empty()) + { + content.push(ContentBlock::Text(text.to_string())); + } + } + let tool_calls = message["toolCalls"] + .as_array() + .into_iter() + .flatten() + .map(|call| { + tool_call( + call["id"].as_str().unwrap_or_default(), + call["name"].as_str().unwrap_or_default(), + call["arguments"].as_str().unwrap_or_default(), + ) + }) + .collect(); + // Usage is on the response; some deployments put it on the choice. + let usage = chat + .get("usage") + .or_else(|| choice.get("usage")) + .filter(|value| value.is_object()) + .map(parse_usage); + Ok(( + content, + tool_calls, + choice["finishReason"].as_str().map(str::to_string), + usage, + )) +} + +fn parse_cohere(chat: &Value) -> Parsed { + let mut content = Vec::new(); + if let Some(text) = chat["text"].as_str().filter(|t| !t.is_empty()) { + content.push(ContentBlock::Text(text.to_string())); + } + // Cohere v1 tool calls carry no id; synthesize stable ones. + let tool_calls = chat["toolCalls"] + .as_array() + .into_iter() + .flatten() + .enumerate() + .map(|(index, call)| { + ToolCall::new( + format!("call_{index}"), + call["name"].as_str().unwrap_or_default(), + call.get("parameters") + .filter(|value| value.is_object()) + .cloned() + .unwrap_or_else(|| Value::Object(Default::default())), + ) + }) + .collect(); + ( + content, + tool_calls, + chat["finishReason"].as_str().map(str::to_string), + chat.get("usage") + .filter(|value| value.is_object()) + .map(parse_usage), + ) +} + +#[cfg(test)] +#[path = "response_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/response_tests.rs b/crates/tinyinference-llm/src/providers/oci/response_tests.rs new file mode 100644 index 00000000..fac035b5 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/response_tests.rs @@ -0,0 +1,103 @@ +use super::*; +use serde_json::json; + +#[test] +fn generic_text_response_with_usage() { + let body = json!({ + "modelId": "meta.llama", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "timeCreated": "2025-01-01T00:00:00Z", + "choices": [{ + "index": 0, + "message": {"role": "ASSISTANT", "content": [{"type": "TEXT", "text": "Hello!"}]}, + "finishReason": "stop" + }], + "usage": { + "promptTokens": 10, "completionTokens": 4, "totalTokens": 14, + "promptTokensDetails": {"cachedTokens": 6}, + "completionTokensDetails": {"reasoningTokens": 2} + } + } + }); + let response = parse_response(body).unwrap(); + assert_eq!( + response.message.content, + vec![ContentBlock::Text("Hello!".into())] + ); + assert_eq!(response.finish_reason.as_deref(), Some("stop")); + let usage = response.usage.unwrap(); + assert_eq!( + (usage.input_tokens, usage.output_tokens, usage.total_tokens), + (10, 4, 14) + ); + assert_eq!((usage.cache_read_tokens, usage.reasoning_tokens), (6, 2)); +} + +#[test] +fn generic_tool_calls_reasoning_and_malformed_arguments() { + let body = json!({"chatResponse": {"apiFormat": "GENERIC", "choices": [{ + "index": 0, + "message": { + "role": "ASSISTANT", + "content": null, + "reasoningContent": "thinking...", + "toolCalls": [ + {"type": "FUNCTION", "id": "c1", "name": "get_weather", "arguments": "{\"city\":\"Rome\"}"}, + {"type": "FUNCTION", "id": "c2", "name": "broken", "arguments": "{oops"} + ] + }, + "finishReason": "tool_calls" + }]}}); + let response = parse_response(body).unwrap(); + assert!(matches!( + &response.message.content[0], + ContentBlock::Thinking { text, signature: None } if text == "thinking..." + )); + let calls = &response.message.tool_calls; + assert_eq!(calls[0].arguments, json!({"city": "Rome"})); + assert!(calls[0].invalid.is_none()); + assert!(calls[1].invalid.is_some()); + assert_eq!(response.finish_reason.as_deref(), Some("tool_calls")); +} + +#[test] +fn cohere_response_text_tool_calls_and_usage() { + let body = json!({"chatResponse": { + "apiFormat": "COHERE", + "text": "Sure", + "finishReason": "COMPLETE", + "toolCalls": [{"name": "get_weather", "parameters": {"city": "Oslo"}}], + "usage": {"promptTokens": 3, "completionTokens": 1, "totalTokens": 4} + }}); + let response = parse_response(body).unwrap(); + assert_eq!( + response.message.content, + vec![ContentBlock::Text("Sure".into())] + ); + assert_eq!(response.message.tool_calls[0].id, "call_0"); + assert_eq!( + response.message.tool_calls[0].arguments, + json!({"city": "Oslo"}) + ); + assert_eq!(response.finish_reason.as_deref(), Some("COMPLETE")); + assert_eq!(response.usage.unwrap().total_tokens, 4); +} + +#[test] +fn malformed_bodies_are_errors() { + for body in [ + json!({}), + json!({"chatResponse": {"apiFormat": "GENERIC", "choices": []}}), + json!({"chatResponse": {"apiFormat": "WAT"}}), + ] { + assert!(parse_response(body).is_err()); + } +} + +#[test] +fn total_tokens_falls_back_to_the_sum() { + let usage = parse_usage(&json!({"promptTokens": 2, "completionTokens": 3})); + assert_eq!(usage.total_tokens, 5); +} diff --git a/crates/tinyinference-llm/src/providers/oci/signer.rs b/crates/tinyinference-llm/src/providers/oci/signer.rs new file mode 100644 index 00000000..a63b39d4 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/signer.rs @@ -0,0 +1,291 @@ +//! OCI HTTP Signature v1 (`Authorization: Signature version="1",...`). +//! +//! The signature is RSA-SHA256 (PKCS#1 v1.5) over a signing string made of +//! `name: value` lines joined by `\n`. GET/DELETE/HEAD sign +//! `(request-target) date host`; requests with a body (POST/PUT/PATCH) also +//! sign `x-content-sha256 content-type content-length`, where +//! `x-content-sha256` is the base64 SHA-256 of the body (required even when +//! the body is empty). The `keyId` is `//` for API +//! keys and `ST$` for session / principal tokens. + +use std::sync::Arc; +use std::time::SystemTime; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as B64; +use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, HeaderName, HeaderValue}; +use ring::rand::SystemRandom; +use ring::signature::{RSA_PKCS1_SHA256, RsaKeyPair}; +use sha2::{Digest, Sha256}; + +use super::auth::KeySource; +use crate::providers::RequestSigner; +use crate::{Error, Result}; + +/// An RSA private key able to sign OCI requests. +pub(crate) struct SigningKey { + pair: RsaKeyPair, +} + +impl std::fmt::Debug for SigningKey { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("SigningKey([redacted])") + } +} + +impl SigningKey { + /// Parses an unencrypted PKCS#1 (`RSA PRIVATE KEY`) or PKCS#8 + /// (`PRIVATE KEY`) PEM document. + /// + /// # Errors + /// + /// [`Error::Validation`] for encrypted, malformed, or unsupported keys. + pub(crate) fn from_pem(pem: &str) -> Result { + let (label, der) = decode_pem(pem)?; + let pair = match label.as_str() { + "PRIVATE KEY" => RsaKeyPair::from_pkcs8(&der), + "RSA PRIVATE KEY" => RsaKeyPair::from_der(&der), + other => { + return Err(Error::Validation(format!( + "unsupported OCI private key PEM type `{other}` (expected PRIVATE KEY or RSA PRIVATE KEY)" + ))); + } + } + .map_err(|_| { + Error::Validation( + "OCI private key is not a valid RSA key (2048 to 8192 bits required)".to_string(), + ) + })?; + Ok(Self { pair }) + } + + /// Signs `message` and returns the base64 signature. + pub(crate) fn sign_b64(&self, message: &[u8]) -> Result { + let mut signature = vec![0u8; self.pair.public().modulus_len()]; + self.pair + .sign( + &RSA_PKCS1_SHA256, + &SystemRandom::new(), + message, + &mut signature, + ) + .map_err(|_| Error::Model("OCI request signing failed".to_string()))?; + Ok(B64.encode(signature)) + } + + /// DER `RSAPublicKey` for the key (used by tests to verify signatures). + #[cfg(test)] + pub(crate) fn public_der(&self) -> Vec { + use ring::signature::KeyPair; + self.pair.public_key().as_ref().to_vec() + } +} + +/// Decodes the first PEM block, rejecting encrypted keys with a clear error. +fn decode_pem(pem: &str) -> Result<(String, Vec)> { + let mut label: Option = None; + let mut body = String::new(); + for line in pem.lines().map(str::trim) { + if let Some(rest) = line.strip_prefix("-----BEGIN ") { + let name = rest.trim_end_matches('-').trim().to_string(); + if name == "ENCRYPTED PRIVATE KEY" { + return Err(encrypted_key_error()); + } + label = Some(name); + } else if line.starts_with("-----END ") { + break; + } else if label.is_some() && !line.is_empty() { + if line.starts_with("Proc-Type:") && line.contains("ENCRYPTED") { + return Err(encrypted_key_error()); + } + if line.contains(':') { + // Other RFC 1421 header (e.g. DEK-Info); not key material. + continue; + } + body.push_str(line); + } + } + let label = + label.ok_or_else(|| Error::Validation("OCI private key is not PEM encoded".to_string()))?; + let der = B64 + .decode(body.as_bytes()) + .map_err(|_| Error::Validation("OCI private key PEM body is not valid base64".into()))?; + Ok((label, der)) +} + +fn encrypted_key_error() -> Error { + Error::Validation( + "encrypted OCI private keys are not supported; decrypt the key (openssl rsa -in key.pem -out key_plain.pem) or use an unencrypted key".to_string(), + ) +} + +/// The headers and signing string for one request, before signing. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct SigningMaterial { + /// Space-separated `headers="..."` list. + pub(crate) header_names: String, + /// The exact string that is signed. + pub(crate) signing_string: String, +} + +/// Builds the signing string. `body` is `Some((x_content_sha256, content_type, +/// content_length))` for body-bearing methods. +pub(crate) fn signing_material( + method: &str, + target: &str, + date: &str, + host: Option<&str>, + body: Option<(&str, &str, usize)>, +) -> SigningMaterial { + let mut names = vec!["(request-target)", "date"]; + let mut lines = vec![ + format!( + "(request-target): {} {}", + method.to_ascii_lowercase(), + target + ), + format!("date: {date}"), + ]; + if let Some(host) = host { + names.push("host"); + lines.push(format!("host: {host}")); + } + if let Some((sha, content_type, length)) = body { + names.extend(["x-content-sha256", "content-type", "content-length"]); + lines.push(format!("x-content-sha256: {sha}")); + lines.push(format!("content-type: {content_type}")); + lines.push(format!("content-length: {length}")); + } + SigningMaterial { + header_names: names.join(" "), + signing_string: lines.join("\n"), + } +} + +/// Formats the `Authorization` header value. +pub(crate) fn authorization_header(key_id: &str, header_names: &str, signature: &str) -> String { + format!( + "Signature version=\"1\",keyId=\"{key_id}\",algorithm=\"rsa-sha256\",headers=\"{header_names}\",signature=\"{signature}\"" + ) +} + +fn has_body(method: &reqwest::Method) -> bool { + matches!( + *method, + reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH + ) +} + +/// Signs `request` in place with an explicit `date` (RFC 7231 GMT), signing +/// the `host` header as OCI services expect. +pub(crate) fn sign_request_at( + request: &mut reqwest::Request, + key_id: &str, + key: &SigningKey, + date: &str, +) -> Result<()> { + sign_request_with(request, key_id, key, date, true) +} + +/// Like [`sign_request_at`], with control over whether `host` is signed. The +/// X.509 federation endpoint signs `date (request-target)` plus the body +/// headers and no `host` (oci-go-sdk `federation_client.go`: `genericHeaders`). +pub(crate) fn sign_request_with( + request: &mut reqwest::Request, + key_id: &str, + key: &SigningKey, + date: &str, + include_host: bool, +) -> Result<()> { + let url = request.url().clone(); + let host = match (url.host_str(), url.port()) { + (Some(host), Some(port)) => format!("{host}:{port}"), + (Some(host), None) => host.to_string(), + (None, _) => return Err(Error::Validation("OCI request URL has no host".into())), + }; + let target = match url.query() { + Some(query) => format!("{}?{query}", url.path()), + None => url.path().to_string(), + }; + let method = request.method().clone(); + let body_parts = if has_body(&method) { + let bytes: &[u8] = match request.body() { + None => &[], + Some(body) => body.as_bytes().ok_or_else(|| { + Error::Validation("OCI signing requires a buffered request body".into()) + })?, + }; + let sha = B64.encode(Sha256::digest(bytes)); + let content_type = request + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("application/json") + .to_string(); + Some((sha, content_type, bytes.len())) + } else { + None + }; + let material = signing_material( + method.as_str(), + &target, + date, + include_host.then_some(host.as_str()), + body_parts + .as_ref() + .map(|(sha, ct, len)| (sha.as_str(), ct.as_str(), *len)), + ); + let signature = key.sign_b64(material.signing_string.as_bytes())?; + let authorization = authorization_header(key_id, &material.header_names, &signature); + + let headers = request.headers_mut(); + let set = |headers: &mut reqwest::header::HeaderMap, name: HeaderName, value: &str| { + let value = HeaderValue::from_str(value) + .map_err(|_| Error::Validation("OCI signature header was not valid".into()))?; + headers.insert(name, value); + Ok::<(), Error>(()) + }; + set(headers, HeaderName::from_static("date"), date)?; + if let Some((sha, content_type, length)) = &body_parts { + set(headers, HeaderName::from_static("x-content-sha256"), sha)?; + set(headers, CONTENT_TYPE, content_type)?; + set(headers, CONTENT_LENGTH, &length.to_string())?; + } + let mut value = HeaderValue::from_str(&authorization) + .map_err(|_| Error::Validation("OCI authorization header was not valid".into()))?; + value.set_sensitive(true); + headers.insert(reqwest::header::AUTHORIZATION, value); + tracing::debug!( + target: "tinyinference::oci", + method = method.as_str(), + headers = material.header_names.as_str(), + "oci request signed" + ); + Ok(()) +} + +/// A [`RequestSigner`] that signs with OCI credentials resolved per request. +#[derive(Debug)] +pub(crate) struct OciRequestSigner { + source: Arc, +} + +impl OciRequestSigner { + pub(crate) fn new(source: KeySource) -> Self { + Self { + source: Arc::new(source), + } + } +} + +impl RequestSigner for OciRequestSigner { + fn sign(&self, request: &mut reqwest::Request) -> Result<()> { + let (key_id, key) = self.source.current()?; + let date = httpdate::fmt_http_date(SystemTime::now()); + sign_request_at(request, &key_id, &key, &date) + } +} + +#[cfg(test)] +#[path = "signer_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/signer_tests.rs b/crates/tinyinference-llm/src/providers/oci/signer_tests.rs new file mode 100644 index 00000000..cfe72ce5 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/signer_tests.rs @@ -0,0 +1,187 @@ +use super::*; +use crate::providers::oci::fixtures::{openssl_signature_b64, pem, test_pkcs1_pem, test_pkcs8_pem}; +use ring::signature::{RSA_PKCS1_2048_8192_SHA256, UnparsedPublicKey}; + +const DATE: &str = "Thu, 05 Jan 2014 21:31:40 GMT"; + +fn verify(key: &SigningKey, message: &str, signature_b64: &str) -> bool { + let signature = B64.decode(signature_b64).unwrap(); + UnparsedPublicKey::new(&RSA_PKCS1_2048_8192_SHA256, key.public_der()) + .verify(message.as_bytes(), &signature) + .is_ok() +} + +#[test] +fn pkcs8_and_pkcs1_keys_produce_the_openssl_signature() { + for pem in [test_pkcs8_pem(), test_pkcs1_pem()] { + let key = SigningKey::from_pem(pem).expect("key parses"); + assert_eq!( + key.sign_b64(b"line1\nline2").unwrap(), + openssl_signature_b64(b"line1\nline2") + ); + } +} + +#[test] +fn encrypted_keys_are_rejected_with_a_clear_error() { + let pkcs8 = pem("ENCRYPTED PRIVATE KEY", "AAAA"); + let pkcs1 = pem( + "RSA PRIVATE KEY", + "Proc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,00\n\nAAAA", + ); + for pem in [pkcs8, pkcs1] { + let error = SigningKey::from_pem(&pem).unwrap_err().to_string(); + assert!(error.contains("encrypted"), "{error}"); + } +} + +#[test] +fn malformed_pem_is_a_validation_error() { + assert!(SigningKey::from_pem("not pem").is_err()); + assert!(SigningKey::from_pem(&pem("PRIVATE KEY", "!!!")).is_err()); + assert!( + SigningKey::from_pem("-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----").is_err() + ); +} + +#[test] +fn get_signs_request_target_date_and_host() { + let material = signing_material( + "GET", + "/20231130/models?x=1", + DATE, + Some("h.example.com"), + None, + ); + assert_eq!(material.header_names, "(request-target) date host"); + assert_eq!( + material.signing_string, + "(request-target): get /20231130/models?x=1\ndate: Thu, 05 Jan 2014 21:31:40 GMT\nhost: h.example.com" + ); +} + +#[test] +fn post_additionally_signs_body_headers() { + let material = signing_material( + "POST", + "/20231130/actions/chat", + DATE, + Some("h.example.com"), + Some(("SHA=", "application/json", 12)), + ); + assert_eq!( + material.header_names, + "(request-target) date host x-content-sha256 content-type content-length" + ); + assert_eq!( + material.signing_string, + "(request-target): post /20231130/actions/chat\n\ + date: Thu, 05 Jan 2014 21:31:40 GMT\n\ + host: h.example.com\n\ + x-content-sha256: SHA=\n\ + content-type: application/json\n\ + content-length: 12" + ); +} + +#[test] +fn authorization_header_has_the_documented_shape() { + assert_eq!( + authorization_header("t/u/fp", "(request-target) date host", "SIG"), + "Signature version=\"1\",keyId=\"t/u/fp\",algorithm=\"rsa-sha256\",headers=\"(request-target) date host\",signature=\"SIG\"" + ); +} + +fn header<'a>(request: &'a reqwest::Request, name: &str) -> &'a str { + request.headers().get(name).unwrap().to_str().unwrap() +} + +fn signature_of(authorization: &str) -> &str { + authorization + .split("signature=\"") + .nth(1) + .unwrap() + .trim_end_matches('"') +} + +#[test] +fn post_request_is_signed_with_a_verifiable_signature() { + let key = SigningKey::from_pem(test_pkcs8_pem()).unwrap(); + let body = br#"{"hello":"world"}"#.to_vec(); + let mut request = reqwest::Client::new() + .post( + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/chat", + ) + .body(body.clone()) + .build() + .unwrap(); + sign_request_at(&mut request, "ST$tok", &key, DATE).unwrap(); + + let sha = B64.encode(Sha256::digest(&body)); + assert_eq!(header(&request, "x-content-sha256"), sha); + assert_eq!(header(&request, "content-length"), body.len().to_string()); + assert_eq!(header(&request, "content-type"), "application/json"); + assert_eq!(header(&request, "date"), DATE); + + let authorization = header(&request, "authorization"); + assert!( + authorization + .starts_with("Signature version=\"1\",keyId=\"ST$tok\",algorithm=\"rsa-sha256\"") + ); + let expected = format!( + "(request-target): post /20231130/actions/chat\ndate: {DATE}\nhost: inference.generativeai.us-chicago-1.oci.oraclecloud.com\nx-content-sha256: {sha}\ncontent-type: application/json\ncontent-length: {}", + body.len() + ); + assert!(verify(&key, &expected, signature_of(authorization))); +} + +#[test] +fn empty_post_body_still_carries_the_content_digest() { + let key = SigningKey::from_pem(test_pkcs1_pem()).unwrap(); + let mut request = reqwest::Client::new() + .post("https://example.com/x") + .build() + .unwrap(); + sign_request_at(&mut request, "k", &key, DATE).unwrap(); + assert_eq!( + header(&request, "x-content-sha256"), + B64.encode(Sha256::digest(b"")) + ); + assert_eq!(header(&request, "content-length"), "0"); +} + +#[test] +fn get_request_signs_only_the_basic_headers_and_keeps_ports() { + let key = SigningKey::from_pem(test_pkcs8_pem()).unwrap(); + let mut request = reqwest::Client::new() + .get("https://example.com:8443/a/b?q=1") + .build() + .unwrap(); + sign_request_at(&mut request, "k", &key, DATE).unwrap(); + assert!(request.headers().get("x-content-sha256").is_none()); + let authorization = header(&request, "authorization"); + assert!(authorization.contains("headers=\"(request-target) date host\"")); + let expected = format!("(request-target): get /a/b?q=1\ndate: {DATE}\nhost: example.com:8443"); + assert!(verify(&key, &expected, signature_of(authorization))); +} + +#[test] +fn debug_output_never_contains_key_material() { + let key = SigningKey::from_pem(test_pkcs8_pem()).unwrap(); + assert_eq!(format!("{key:?}"), "SigningKey([redacted])"); +} + +#[test] +fn federation_style_signing_omits_host() { + let key = SigningKey::from_pem(test_pkcs8_pem()).unwrap(); + let mut request = reqwest::Client::new() + .post("https://auth.us-phoenix-1.oraclecloud.com/v1/x509") + .body("{}") + .build() + .unwrap(); + sign_request_with(&mut request, "t/fed-x509-sha256/fp", &key, DATE, false).unwrap(); + let authorization = header(&request, "authorization"); + assert!(authorization.contains( + "headers=\"(request-target) date x-content-sha256 content-type content-length\"" + )); +} diff --git a/crates/tinyinference-llm/src/providers/oci/stream.rs b/crates/tinyinference-llm/src/providers/oci/stream.rs new file mode 100644 index 00000000..cc03b2ed --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/stream.rs @@ -0,0 +1,377 @@ +//! Native chat server-sent-events streaming. +//! +//! Only `data:` lines are read; each payload is one JSON chunk. +//! +//! * GENERIC chunks look like `{"index":0,"message":{"role":"ASSISTANT", +//! "content":[{"type":"TEXT","text":"..."}],"toolCalls":[...], +//! "reasoningContent":"..."}}`, followed by a chunk carrying +//! `finishReason`, and possibly a trailing chunk with `usage`. +//! * COHERE chunks look like `{"apiFormat":"COHERE","text":"..."}`; the final +//! chunk repeats the full `text` alongside `finishReason`, so text on a +//! chunk with a finish reason is not re-emitted. +//! +//! The stream ends at `data: [DONE]` (some models send it) or at EOF; either +//! counts as success only once a `finishReason` was seen. + +use std::collections::VecDeque; +use std::pin::Pin; + +use bytes::Bytes; +use futures::{Stream, StreamExt}; +use serde_json::Value; + +use super::response::{parse_usage, tool_call}; +use crate::message::{AssistantMessage, ContentBlock, MessageDelta, MessageOrigin}; +use crate::model::{ModelResponse, ModelStream, ModelStreamItem, ProviderError}; +use crate::tool::{ToolCall, ToolDelta}; +use crate::usage::Usage; +use crate::{Error, Result}; + +use super::{API_NATIVE, PROVIDER}; + +#[derive(Debug, Default)] +struct CallAcc { + id: String, + name: String, + arguments: String, +} + +#[derive(Debug, Default)] +struct Acc { + text: String, + reasoning: String, + calls: Vec, + usage: Option, + finish: Option, +} + +impl Acc { + fn delta_tool(&mut self, id: &str, name: &str, args: &str, slot_hint: usize) -> usize { + let existing = if id.is_empty() { + // Continuation fragment: attach to the hinted slot, else the last. + // A fragment that arrives before any id opens a placeholder slot. + if self.calls.is_empty() { + self.calls.push(CallAcc::default()); + } + slot_hint.min(self.calls.len() - 1) + } else if let Some(position) = self.calls.iter().position(|call| call.id == id) { + position + } else if let Some(position) = self.calls.iter().position(|call| call.id.is_empty()) { + // The id arrived after id-less fragments: adopt their slot. + self.calls[position].id = id.to_string(); + position + } else { + self.calls.push(CallAcc { + id: id.to_string(), + name: name.to_string(), + arguments: String::new(), + }); + self.calls.len() - 1 + }; + let call = &mut self.calls[existing]; + if call.name.is_empty() { + call.name = name.to_string(); + } + // A repeated id carrying the cumulative arguments replaces rather + // than duplicates what was already streamed. + if !args.is_empty() + && args.len() > call.arguments.len() + && args.starts_with(&call.arguments) + { + call.arguments = args.to_string(); + } else { + call.arguments.push_str(args); + } + existing + } + + fn ingest(&mut self, event: &Value, pending: &mut VecDeque) -> Result<()> { + if event["message"].is_string() && event.get("code").is_some() { + return Err(Error::Provider(Box::new(ProviderError { + provider: PROVIDER.to_string(), + code: event["code"].as_str().map(str::to_string), + message: event["message"].as_str().unwrap_or_default().to_string(), + raw: Some(event.clone()), + ..ProviderError::default() + }))); + } + let cohere = event["apiFormat"].as_str() == Some("COHERE") + || (event.get("text").is_some() && event.get("message").is_none()); + let finishing = event["finishReason"].as_str().is_some(); + if let Some(reason) = event["finishReason"].as_str() { + self.finish = Some(reason.to_string()); + } + if cohere { + if !finishing && let Some(text) = event["text"].as_str().filter(|t| !t.is_empty()) { + self.text.push_str(text); + pending.push_back(ModelStreamItem::MessageDelta(MessageDelta::text( + text.to_string(), + ))); + } + if finishing { + for (index, call) in event["toolCalls"] + .as_array() + .into_iter() + .flatten() + .enumerate() + { + let arguments = call + .get("parameters") + .map(Value::to_string) + .unwrap_or_default(); + let id = format!("call_{index}"); + let name = call["name"].as_str().unwrap_or_default(); + let slot = self.delta_tool(&id, name, &arguments, index); + pending.push_back(ModelStreamItem::ToolCallDelta(ToolDelta { + call_id: id, + content: arguments, + tool_name: Some(name.to_string()), + content_index: Some(slot), + })); + } + } + } else { + let message = &event["message"]; + for part in message["content"].as_array().into_iter().flatten() { + if let Some(text) = part["text"].as_str().filter(|t| !t.is_empty()) { + self.text.push_str(text); + pending.push_back(ModelStreamItem::MessageDelta(MessageDelta::text( + text.to_string(), + ))); + } + } + if let Some(reasoning) = message["reasoningContent"] + .as_str() + .filter(|t| !t.is_empty()) + { + self.reasoning.push_str(reasoning); + pending.push_back(ModelStreamItem::MessageDelta(MessageDelta { + text: String::new(), + reasoning: reasoning.to_string(), + tool_call: None, + })); + } + for (hint, call) in message["toolCalls"] + .as_array() + .into_iter() + .flatten() + .enumerate() + { + let id = call["id"].as_str().unwrap_or_default(); + let name = call["name"].as_str().unwrap_or_default(); + let args = call["arguments"].as_str().unwrap_or_default(); + let slot = self.delta_tool(id, name, args, hint); + let resolved = &self.calls[slot]; + pending.push_back(ModelStreamItem::ToolCallDelta(ToolDelta { + call_id: resolved.id.clone(), + content: args.to_string(), + tool_name: Some(resolved.name.clone()), + content_index: Some(slot), + })); + } + } + if let Some(usage) = event.get("usage").filter(|value| value.is_object()) { + let usage = parse_usage(usage); + self.usage = Some(usage); + pending.push_back(ModelStreamItem::UsageDelta(usage)); + } + Ok(()) + } + + fn into_response(self) -> ModelResponse { + let mut content = Vec::new(); + if !self.reasoning.is_empty() { + content.push(ContentBlock::Thinking { + text: self.reasoning, + signature: None, + }); + } + if !self.text.is_empty() { + content.push(ContentBlock::Text(self.text)); + } + let tool_calls: Vec = self + .calls + .iter() + .map(|call| tool_call(&call.id, &call.name, &call.arguments)) + .collect(); + ModelResponse { + message: AssistantMessage { + id: None, + content, + tool_calls, + usage: self.usage, + origin: None, + }, + usage: self.usage, + finish_reason: self.finish, + raw: None, + resolved_model: None, + continue_turn: None, + served_from_cache: false, + correlation: None, + resolved_route: None, + } + } +} + +struct SseState { + bytes: Pin> + Send>>, + buf: Vec, + pending: VecDeque, + acc: Acc, + model: String, + started: bool, + finished: bool, + terminal_emitted: bool, +} + +impl SseState { + fn failure(&mut self, error: Error) -> ModelStreamItem { + self.pending.clear(); + self.finished = true; + self.terminal_emitted = true; + let mut provider_error = match error { + Error::Provider(error) => *error, + error => { + let message = error.to_string(); + let retryable = + crate::failure::classify_provider_failure(None, None, &message).is_retryable(); + ProviderError { + provider: PROVIDER.to_string(), + message, + retryable, + ..ProviderError::default() + } + } + }; + provider_error.provider = PROVIDER.to_string(); + provider_error.model = Some(self.model.clone()); + provider_error.stop_reason = self.acc.finish.clone(); + let partial = std::mem::take(&mut self.acc).into_response().message; + if !partial.content.is_empty() || !partial.tool_calls.is_empty() { + provider_error.partial_message = Some(partial); + } + ModelStreamItem::ProviderFailed(provider_error) + } + + fn fold_line(&mut self, line: &[u8]) -> Result<()> { + let line = String::from_utf8_lossy(line); + let Some(rest) = line.trim_end_matches('\r').strip_prefix("data:") else { + return Ok(()); + }; + let payload = rest.trim(); + if payload.is_empty() { + return Ok(()); + } + if payload == "[DONE]" { + self.finished = true; + return Ok(()); + } + let event = serde_json::from_str::(payload) + .map_err(|error| Error::Model(format!("invalid OCI SSE data payload: {error}")))?; + self.acc.ingest(&event, &mut self.pending) + } + + fn drain_lines(&mut self) -> Result<()> { + let mut start = 0; + while let Some(offset) = self.buf[start..].iter().position(|byte| *byte == b'\n') { + let end = start + offset; + let line = self.buf[start..end].to_vec(); + start = end + 1; + self.fold_line(&line)?; + if self.finished { + break; + } + } + self.buf.drain(..start); + Ok(()) + } +} + +async fn sse_next(mut state: SseState) -> Option<(ModelStreamItem, SseState)> { + loop { + if let Some(item) = state.pending.pop_front() { + return Some((item, state)); + } + if !state.started { + state.started = true; + return Some((ModelStreamItem::Started, state)); + } + if state.finished { + if state.terminal_emitted { + return None; + } + state.terminal_emitted = true; + if state.acc.finish.is_none() { + let item = state.failure(Error::Model( + "provider stream ended before a completion signal".to_string(), + )); + return Some((item, state)); + } + let mut response = std::mem::take(&mut state.acc).into_response(); + response.message.origin = Some(MessageOrigin { + provider: PROVIDER.to_string(), + api: API_NATIVE.to_string(), + model: state.model.clone(), + }); + return Some((ModelStreamItem::Completed(response), state)); + } + match state.bytes.next().await { + Some(Ok(chunk)) => { + state.buf.extend_from_slice(&chunk); + if let Err(error) = state.drain_lines() { + let item = state.failure(error); + return Some((item, state)); + } + } + Some(Err(error)) => { + let item = state.failure(error); + return Some((item, state)); + } + None => { + if !state.buf.is_empty() { + let line = std::mem::take(&mut state.buf); + if let Err(error) = state.fold_line(&line) { + let item = state.failure(error); + return Some((item, state)); + } + } + // EOF without `[DONE]` is a normal end for most OCI models. + state.finished = true; + } + } + } +} + +fn build(bytes: Pin> + Send>>, model: String) -> ModelStream { + let state = SseState { + bytes, + buf: Vec::new(), + pending: VecDeque::new(), + acc: Acc::default(), + model, + started: false, + finished: false, + terminal_emitted: false, + }; + ModelStream::new(Box::pin(futures::stream::unfold(state, sse_next))) +} + +/// Wraps a successful streaming HTTP response as a [`ModelStream`]. +pub(super) fn into_model_stream(response: reqwest::Response, model: String) -> ModelStream { + let bytes = response + .bytes_stream() + .map(|chunk| chunk.map_err(|error| Error::Model(format!("stream chunk failed: {error}")))); + build(Box::pin(bytes), model) +} + +/// Test seam: feeds raw SSE bytes through the production state machine. +#[cfg(test)] +pub(super) fn stream_from_bytes(chunks: Vec>, model: &str) -> ModelStream { + let bytes = futures::stream::iter(chunks.into_iter().map(|chunk| Ok(Bytes::from(chunk)))); + build(Box::pin(bytes), model.to_string()) +} + +#[cfg(test)] +#[path = "stream_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/oci/stream_tests.rs b/crates/tinyinference-llm/src/providers/oci/stream_tests.rs new file mode 100644 index 00000000..d9b50c30 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/oci/stream_tests.rs @@ -0,0 +1,161 @@ +use super::*; +use futures::StreamExt; +use serde_json::json; + +async fn collect(chunks: Vec<&str>) -> Vec { + stream_from_bytes( + chunks.into_iter().map(|c| c.as_bytes().to_vec()).collect(), + "m", + ) + .collect() + .await +} + +fn completed(items: &[ModelStreamItem]) -> &ModelResponse { + match items.last() { + Some(ModelStreamItem::Completed(response)) => response, + other => panic!("expected Completed, got {other:?}"), + } +} + +#[tokio::test] +async fn generic_text_stream_without_done_sentinel() { + let items = collect(vec![ + "data: {\"index\":0,\"message\":{\"role\":\"ASSISTANT\",\"content\":[{\"type\":\"TEXT\",\"text\":\"Hel\"}]}}\n\n", + "data: {\"index\":0,\"message\":{\"role\":\"ASSISTANT\",\"content\":[{\"type\":\"TEXT\",\"text\":\"lo\"}]}}\n\n", + "data: {\"finishReason\":\"stop\"}\n\n", + "data: {\"usage\":{\"promptTokens\":5,\"completionTokens\":2,\"totalTokens\":7}}\n", + ]) + .await; + assert!(matches!(items[0], ModelStreamItem::Started)); + let response = completed(&items); + assert_eq!( + response.message.content, + vec![ContentBlock::Text("Hello".into())] + ); + assert_eq!(response.finish_reason.as_deref(), Some("stop")); + assert_eq!(response.usage.unwrap().total_tokens, 7); + assert_eq!(response.message.origin.as_ref().unwrap().provider, "oci"); + let deltas = items + .iter() + .filter(|item| matches!(item, ModelStreamItem::MessageDelta(_))) + .count(); + assert_eq!(deltas, 2); +} + +#[tokio::test] +async fn done_sentinel_and_split_chunks_are_handled() { + let line = + "data: {\"index\":0,\"message\":{\"content\":[{\"type\":\"TEXT\",\"text\":\"é\"}]}}\n"; + let bytes = line.as_bytes(); + let (a, b) = bytes.split_at(bytes.len() / 2 + 3); + let items = stream_from_bytes( + vec![ + a.to_vec(), + b.to_vec(), + b"data: {\"finishReason\":\"stop\"}\r\ndata: [DONE]\n".to_vec(), + ], + "m", + ) + .collect::>() + .await; + assert_eq!( + completed(&items).message.content, + vec![ContentBlock::Text("é".into())] + ); +} + +#[tokio::test] +async fn generic_tool_call_fragments_accumulate() { + let items = collect(vec![ + "data: {\"message\":{\"toolCalls\":[{\"id\":\"c1\",\"name\":\"get_weather\",\"arguments\":\"{\\\"ci\"}]}}\n", + "data: {\"message\":{\"toolCalls\":[{\"arguments\":\"ty\\\":\\\"Rome\\\"}\"}]}}\n", + "data: {\"finishReason\":\"tool_calls\"}\n", + ]) + .await; + let response = completed(&items); + assert_eq!(response.message.tool_calls.len(), 1); + assert_eq!(response.message.tool_calls[0].id, "c1"); + assert_eq!( + response.message.tool_calls[0].arguments, + json!({"city": "Rome"}) + ); + let deltas: Vec<_> = items + .iter() + .filter_map(|item| match item { + ModelStreamItem::ToolCallDelta(delta) => Some(delta.call_id.clone()), + _ => None, + }) + .collect(); + assert_eq!(deltas, vec!["c1", "c1"]); +} + +#[tokio::test] +async fn tool_call_fragment_before_any_id_does_not_panic() { + let items = collect(vec![ + "data: {\"message\":{\"toolCalls\":[{\"name\":\"get_weather\",\"arguments\":\"{\\\"ci\"}]}}\n", + "data: {\"message\":{\"toolCalls\":[{\"id\":\"c1\",\"arguments\":\"ty\\\":\\\"Rome\\\"}\"}]}}\n", + "data: {\"finishReason\":\"tool_calls\"}\n", + ]) + .await; + let response = completed(&items); + assert_eq!(response.message.tool_calls.len(), 1); + assert_eq!(response.message.tool_calls[0].id, "c1"); + assert_eq!(response.message.tool_calls[0].name, "get_weather"); + assert_eq!( + response.message.tool_calls[0].arguments, + json!({"city": "Rome"}) + ); +} + +#[tokio::test] +async fn cohere_final_chunk_does_not_duplicate_text() { + let items = collect(vec![ + "data: {\"apiFormat\":\"COHERE\",\"text\":\"Hi \"}\n", + "data: {\"apiFormat\":\"COHERE\",\"text\":\"there\"}\n", + "data: {\"apiFormat\":\"COHERE\",\"text\":\"Hi there\",\"finishReason\":\"COMPLETE\"}\n", + ]) + .await; + let response = completed(&items); + assert_eq!( + response.message.content, + vec![ContentBlock::Text("Hi there".into())] + ); + assert_eq!(response.finish_reason.as_deref(), Some("COMPLETE")); +} + +#[tokio::test] +async fn eof_without_finish_reason_is_a_provider_failure_with_partial() { + let items = collect(vec![ + "data: {\"message\":{\"content\":[{\"type\":\"TEXT\",\"text\":\"partial\"}]}}\n", + ]) + .await; + match items.last() { + Some(ModelStreamItem::ProviderFailed(error)) => { + assert_eq!(error.provider, "oci"); + let partial = error.partial_message.as_ref().unwrap(); + assert_eq!(partial.content, vec![ContentBlock::Text("partial".into())]); + } + other => panic!("expected failure, got {other:?}"), + } +} + +#[tokio::test] +async fn error_event_and_bad_json_fail_the_stream() { + let items = collect(vec![ + "data: {\"code\":\"TooManyRequests\",\"message\":\"slow down\"}\n", + ]) + .await; + match items.last() { + Some(ModelStreamItem::ProviderFailed(error)) => { + assert_eq!(error.code.as_deref(), Some("TooManyRequests")); + assert_eq!(error.message, "slow down"); + } + other => panic!("expected failure, got {other:?}"), + } + let items = collect(vec!["data: {not json}\n"]).await; + assert!(matches!( + items.last(), + Some(ModelStreamItem::ProviderFailed(_)) + )); +} diff --git a/crates/tinyinference-llm/src/providers/openai/mod.rs b/crates/tinyinference-llm/src/providers/openai/mod.rs index 740b0abf..0cc46c4a 100644 --- a/crates/tinyinference-llm/src/providers/openai/mod.rs +++ b/crates/tinyinference-llm/src/providers/openai/mod.rs @@ -93,6 +93,7 @@ pub mod server; pub mod server_http; mod sse; mod transport; +pub(crate) use transport::is_reasoning_model; pub use config::{ OpenAiConfig, build_local_runtime_chat_model, build_openai_chat_model, build_openai_model, diff --git a/crates/tinyinference-llm/src/providers/openai/mod_tests.rs b/crates/tinyinference-llm/src/providers/openai/mod_tests.rs index 8f90e7a1..cb1a0808 100644 --- a/crates/tinyinference-llm/src/providers/openai/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/mod_tests.rs @@ -1631,6 +1631,20 @@ fn routes_max_tokens_to_max_completion_tokens_for_o_series() { assert_eq!(value["max_completion_tokens"], json!(128)); } +#[test] +fn oci_vendor_prefixed_reasoning_models_use_max_completion_tokens() { + // OCI Generative AI names OpenAI models `openai.`. + assert!(is_reasoning_model("openai.gpt-5.5")); + assert!(is_reasoning_model("OpenAI.o4-mini")); + assert!(!is_reasoning_model("openai.gpt-4o")); + assert!(!is_reasoning_model("openai.gpt-oss-120b")); + let request = ModelRequest::new(vec![Message::user("hi")]).with_max_tokens(128); + let model = OpenAiModel::new("k").with_model("openai.gpt-5.5"); + let value = serde_json::to_value(model.translate_request(&request).unwrap()).unwrap(); + assert!(value.get("max_tokens").is_none()); + assert_eq!(value["max_completion_tokens"], json!(128)); +} + #[test] fn keeps_max_tokens_for_classic_models() { let request = ModelRequest::new(vec![Message::user("hi")]).with_max_tokens(128); diff --git a/crates/tinyinference-llm/src/providers/openai/transport.rs b/crates/tinyinference-llm/src/providers/openai/transport.rs index b17b4ebf..3487c9f3 100644 --- a/crates/tinyinference-llm/src/providers/openai/transport.rs +++ b/crates/tinyinference-llm/src/providers/openai/transport.rs @@ -146,6 +146,10 @@ pub struct OpenAiModel { /// [`crate::providers::ProviderRequestOptions`]. Currently applied to the /// Chat Completions transport path only ([`Self::post_json`]). request_options: crate::providers::ProviderRequestOptions, + /// Signs each outbound request just before it is sent, for providers that + /// authenticate with a request signature rather than a header credential. + /// See [`Self::with_request_signer`]. + request_signer: Option>, } impl std::fmt::Debug for OpenAiModel { @@ -316,16 +320,21 @@ pub(super) fn is_anthropic_model(model: &str) -> bool { lower.starts_with("claude") || lower.split('/').any(|seg| seg == "anthropic") } -/// Returns `true` for OpenAI o-series reasoning models (`o1`/`o3`/`o4`), which -/// reject `max_tokens` and require `max_completion_tokens` instead. -pub(super) fn is_reasoning_model(model: &str) -> bool { +/// Returns `true` for OpenAI reasoning models (the `o1`/`o3`/`o4` o-series and +/// the gpt-5 and gpt-6 families), which reject `max_tokens` and require +/// `max_completion_tokens` instead. OCI Generative AI names them with an +/// `openai.` vendor prefix (`openai.gpt-5.5`), which is stripped first. +pub(crate) fn is_reasoning_model(model: &str) -> bool { let lower = model.to_ascii_lowercase(); + let lower = lower.strip_prefix("openai.").unwrap_or(&lower); lower.starts_with("o1") || lower.starts_with("o3") || lower.starts_with("o4") || lower.starts_with("gpt-5") || lower.starts_with("gpt5") || lower.contains("/gpt-5") + || lower.starts_with("gpt-6") + || lower.contains("/gpt-6") } /// Whether `message` is a system message that renders no text on the Chat @@ -456,9 +465,40 @@ impl OpenAiModel { native_tools_on_wire: AtomicBool::new(true), responses_requires_stream: AtomicBool::new(false), request_options: crate::providers::ProviderRequestOptions::default(), + request_signer: None, } } + /// Static headers attached to every request (crate-internal, for tests). + #[cfg(test)] + pub(crate) fn extra_headers(&self) -> &[(String, String)] { + &self.extra_headers + } + + /// Whether a request signer is installed (crate-internal, for tests). + #[cfg(test)] + pub(crate) fn has_request_signer(&self) -> bool { + self.request_signer.is_some() + } + + /// The configured [`AuthStyle`] (crate-internal, for tests). + #[cfg(test)] + pub(crate) fn auth_style(&self) -> &AuthStyle { + &self.auth + } + + /// Installs a [`crate::providers::RequestSigner`] applied to every request + /// after it is fully built (body included) and before it is sent. Pair with + /// [`AuthStyle::None`] when the signature replaces the credential header. + #[must_use] + pub fn with_request_signer( + mut self, + signer: std::sync::Arc, + ) -> Self { + self.request_signer = Some(signer); + self + } + /// Sets host-supplied request hooks and an optional HTTP client override /// applied around Chat Completions calls this adapter makes. See /// [`crate::providers::ProviderRequestOptions`]. @@ -1308,7 +1348,7 @@ impl OpenAiModel { /// Test-only: production paths go through [`Self::build_chat_body`] / /// [`Self::post_chat_with_degrade`], which thread an explicit [`Degrade`]. #[cfg(test)] - pub(super) fn translate_request( + pub(crate) fn translate_request( &self, request: &ModelRequest, ) -> Result { @@ -1757,7 +1797,19 @@ impl OpenAiModel { url: &str, ) -> Result { crate::model::budget::before_physical_attempt()?; - let response = builder.send().await.map_err(|e| { + let sent = match &self.request_signer { + Some(signer) => { + let (client, request) = builder.build_split(); + let signed = request.map_err(|e| { + Error::Validation(format!("{what} to {url} could not be built: {e}")) + }); + let mut request = signed?; + signer.sign(&mut request)?; + client.execute(request).await + } + None => builder.send().await, + }; + let response = sent.map_err(|e| { let error = self.provider_error( format!("{what} to {url} failed: {e}"), None, diff --git a/crates/tinyinference-llm/src/providers/types.rs b/crates/tinyinference-llm/src/providers/types.rs index cbb8d6a2..9ccb1f2d 100644 --- a/crates/tinyinference-llm/src/providers/types.rs +++ b/crates/tinyinference-llm/src/providers/types.rs @@ -75,6 +75,24 @@ impl std::fmt::Debug for ProviderRequestOptions { } } +/// Signs a fully built outbound HTTP request in place, immediately before it +/// is sent. +/// +/// Some hosted providers authenticate with a signature over the method, URL, +/// headers, and body (for example OCI's HTTP Signature) instead of a static +/// credential header, so the signer needs to see the finished +/// [`reqwest::Request`]. Implementations add or replace headers on `request`; +/// they must never log credentials. +pub trait RequestSigner: Send + Sync { + /// Adds authentication headers to `request`. + /// + /// # Errors + /// + /// Returns an error when the credential cannot be loaded or the signature + /// cannot be produced; the request is then not sent. + fn sign(&self, request: &mut reqwest::Request) -> crate::Result<()>; +} + // --------------------------------------------------------------------------- // Per-request credentials // --------------------------------------------------------------------------- diff --git a/crates/tinyinference-providers/src/builtin.rs b/crates/tinyinference-providers/src/builtin.rs index cb96fcef..0a480bc2 100644 --- a/crates/tinyinference-providers/src/builtin.rs +++ b/crates/tinyinference-providers/src/builtin.rs @@ -10,6 +10,10 @@ pub enum BuiltinAuthStyle { Bearer, /// Anthropic x-api-key and version headers. Anthropic, + /// OCI Generative AI: a GenAI API key, OCI request signing, or a + /// principal, served by the `tinyinference-llm` OCI provider rather than + /// a plain OpenAI-compatible client. + Oci, } /// A known cloud-provider preset, independent of any saved host configuration. @@ -185,6 +189,14 @@ pub const BUILTIN_CLOUD_PROVIDERS: &[BuiltinCloudProvider] = &[ endpoint: "https://ai.sumopod.com/v1", auth_style: BuiltinAuthStyle::Bearer, }, + BuiltinCloudProvider { + slug: "oci", + label: "OCI Generative AI", + // Display default only: hosts route OCI entries by region or an + // explicit endpoint through the OCI provider, not by this URL. + endpoint: "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com", + auth_style: BuiltinAuthStyle::Oci, + }, BuiltinCloudProvider { slug: "modelscope", label: "ModelScope", diff --git a/crates/tinyinference-providers/src/builtin_tests.rs b/crates/tinyinference-providers/src/builtin_tests.rs index fe8cd1fa..104a71dd 100644 --- a/crates/tinyinference-providers/src/builtin_tests.rs +++ b/crates/tinyinference-providers/src/builtin_tests.rs @@ -27,6 +27,12 @@ fn builtin_cloud_provider_defaults_cover_phase_one_presets() { "https://ai.sumopod.com/v1", BuiltinAuthStyle::Bearer, ), + ( + "oci", + "OCI Generative AI", + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com", + BuiltinAuthStyle::Oci, + ), ( "modelscope", "ModelScope",