From ca67d6a275bf39d3225c6c196b37e3006cb0e68e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:23:36 +0000 Subject: [PATCH 01/25] test(tinyhumans): cover managed auth failure credential fingerprint Add a test asserting that the fingerprint reported on a managed auth failure matches the bearer the failed request actually carried, not the credential the resolver would return later, and that its debug output does not leak the token. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/tinyhumans/mod_tests.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index 8e3f9190..b1c2568d 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -775,6 +775,44 @@ async fn managed_auth_failure_reports_the_credential_its_request_carried() { ); } +#[tokio::test] +async fn managed_auth_failure_carries_a_non_secret_request_credential_fingerprint() { + let fixture = Fixture::default(); + let base = serve(fixture.clone()).await; + *fixture.response.lock().unwrap() = FixtureResponse { + status: StatusCode::UNAUTHORIZED, + content_type: "application/json", + body: json!({ "error": { "message": "Unauthorized" } }).to_string(), + }; + let token = Arc::new(std::sync::Mutex::new("old-request-token".to_owned())); + let token_for_resolver = token.clone(); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let seen_by_handler = seen.clone(); + let model = TinyHumansModel::new_with_credential(base, "", "default-model", move || { + Ok(TinyHumansCredential { + bearer: token_for_resolver.lock().unwrap().clone(), + kind: TinyHumansCredentialKind::Session, + }) + }) + .unwrap() + .with_managed_auth_failure_handler(move |_, failure| { + seen_by_handler + .lock() + .unwrap() + .push(failure.credential_fingerprint()); + }); + + let request = ModelRequest::new(vec![crate::message::Message::user("hi")]); + assert!(model.invoke(&(), request).await.is_err()); + *token.lock().unwrap() = "new-session-token".to_owned(); + + let fingerprints = seen.lock().unwrap(); + let fingerprint = fingerprints[0].expect("managed credential fingerprint"); + assert!(fingerprint.matches_bearer("old-request-token")); + assert!(!fingerprint.matches_bearer("new-session-token")); + assert!(!format!("{fingerprint:?}").contains("old-request-token")); +} + #[tokio::test] async fn a_plain_bearer_resolver_reports_no_credential_kind() { let fixture = Fixture::default(); From f5e0026747bceed34d5d11c3f6baec4af251a921 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:24:01 +0000 Subject: [PATCH 02/25] feat(tinyinference-llm): fingerprint the credential sent with a request Auth failures now carry an opaque SHA-256 fingerprint of the bearer that the failed request actually used, so a caller can tell whether the credential it sent still matches the host's current one without retaining the secret. The task-local sent-credential slot stores the fingerprint alongside the kind, and the fingerprint compares in constant time and redacts itself in Debug output. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- Cargo.lock | 1 + crates/tinyinference-llm/Cargo.toml | 1 + .../src/providers/tinyhumans/mod.rs | 73 ++++++++++++++++--- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f2ff98b..b31214db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1763,6 +1763,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "tempfile", "thiserror", "tinyinference-core", diff --git a/crates/tinyinference-llm/Cargo.toml b/crates/tinyinference-llm/Cargo.toml index 1663d490..f1e2acb7 100644 --- a/crates/tinyinference-llm/Cargo.toml +++ b/crates/tinyinference-llm/Cargo.toml @@ -28,6 +28,7 @@ futures = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2.workspace = true thiserror = { workspace = true } tinyinference-providers = { version = "0.5.0", path = "../tinyinference-providers" } tinyinference-core = { version = "0.5.0", path = "../tinyinference-core" } diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs index cf1f068b..570a5187 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs @@ -13,6 +13,7 @@ use std::sync::{Arc, OnceLock}; use std::time::Duration; use async_trait::async_trait; +use sha2::{Digest, Sha256}; use tinyinference_core::sanitize; use crate::model::{ @@ -51,11 +52,51 @@ pub struct TinyHumansCredential { type KindedResolver = Arc Result<(String, Option)> + Send + Sync>; +/// Opaque, non-secret identity for a bearer credential used by one request. +/// +/// The fingerprint is a domain-separated SHA-256 digest. It supports comparing +/// the credential used by a failed request with a host's current bearer without +/// retaining or exposing the bearer itself. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct TinyHumansCredentialFingerprint([u8; 32]); + +impl TinyHumansCredentialFingerprint { + fn from_bearer(bearer: &str) -> Self { + let mut digest = Sha256::new(); + digest.update(b"tinyinference/tinyhumans-credential-fingerprint/v1\\0"); + digest.update(bearer.as_bytes()); + Self(digest.finalize().into()) + } + + /// Returns whether this fingerprint belongs to `bearer`. + #[must_use] + pub fn matches_bearer(self, bearer: &str) -> bool { + let candidate = Self::from_bearer(bearer); + // Compare the fixed-width digest without early exit. + self.0 + .iter() + .zip(candidate.0.iter()) + .fold(0_u8, |different, (left, right)| different | (left ^ right)) + == 0 + } +} + +impl std::fmt::Debug for TinyHumansCredentialFingerprint { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("TinyHumansCredentialFingerprint()") + } +} + +#[derive(Clone, Copy)] +struct SentCredential { + kind: Option, + fingerprint: TinyHumansCredentialFingerprint, +} + tokio::task_local! { - /// The credential kind the bearer source handed to the request being - /// dispatched in this task, so a failure is attributed to the credential - /// that request carried rather than to whatever was resolved last. - static SENT_CREDENTIAL: std::cell::Cell>; + /// Request-time credential identity so a failure is attributed to the + /// credential actually sent, rather than whichever one resolves later. + static SENT_CREDENTIAL: std::cell::Cell>; } /// Operation that received a provider authentication failure. @@ -102,6 +143,7 @@ pub struct TinyHumansAuthFailure { operation: TinyHumansOperation, kind: TinyHumansAuthFailureKind, credential: Option, + credential_fingerprint: Option, } impl TinyHumansAuthFailure { @@ -124,6 +166,13 @@ impl TinyHumansAuthFailure { pub fn credential(self) -> Option { self.credential } + + /// The opaque identity of the bearer sent with the failed request, when + /// the request used a managed credential resolver. + #[must_use] + pub fn credential_fingerprint(self) -> Option { + self.credential_fingerprint + } } /// Handles authentication failures from a managed endpoint. @@ -409,7 +458,7 @@ impl TinyHumansModel { &self, error: &Error, operation: TinyHumansOperation, - credential: Option, + credential: Option, ) { if let Error::Provider(provider_error) = error && is_auth_failure(provider_error) @@ -428,7 +477,7 @@ impl TinyHumansModel { &self, error: &ProviderError, operation: TinyHumansOperation, - credential: Option, + credential: Option, ) { if let Some(handler) = &self.auth_failure_handler { handler(error, operation); @@ -439,7 +488,8 @@ impl TinyHumansModel { TinyHumansAuthFailure { operation, kind: auth_failure_kind(error), - credential, + credential: credential.and_then(|sent| sent.kind), + credential_fingerprint: credential.map(|sent| sent.fingerprint), }, ); } @@ -515,7 +565,8 @@ impl ChatModel<()> for TinyHumansModel { TinyHumansAuthFailure { operation: TinyHumansOperation::Stream, kind: auth_failure_kind(&error), - credential, + credential: credential.and_then(|sent| sent.kind), + credential_fingerprint: credential.map(|sent| sent.fingerprint), }, ); } @@ -582,7 +633,11 @@ impl BearerSource for ResolvedBearer { let (bearer, kind) = (self.0)()?; // Outside a dispatch scope (e.g. the readiness probe) there is nothing // to attribute, so a missing task-local is fine. - let _ = SENT_CREDENTIAL.try_with(|sent| sent.set(kind)); + let sent = SentCredential { + kind, + fingerprint: TinyHumansCredentialFingerprint::from_bearer(&bearer), + }; + let _ = SENT_CREDENTIAL.try_with(|credential| credential.set(Some(sent))); Ok(Some(bearer)) } } From 99aa3164cfca2c0480d806a645bca83a7d1869ce Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:24:27 +0000 Subject: [PATCH 03/25] test(tinyhumans): rotate token during request in auth failure test The fixture now rotates the session token while the request is in flight instead of after it returns, so the test exercises the race where the credential changes before the failure is fingerprinted. The assertion confirms the token was rotated by the handler. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/tinyhumans/mod_tests.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index b1c2568d..72d7d75e 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -14,6 +14,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; struct Fixture { requests: Arc>>, response: Arc>, + rotate_token_before_response: Arc>>>>, } #[derive(Clone)] @@ -49,6 +50,9 @@ fn completion_body() -> String { async fn chat(State(fixture): State, headers: HeaderMap, body: String) -> Response { let body: Value = serde_json::from_str(&body).expect("request JSON"); fixture.requests.lock().unwrap().push((headers, body)); + if let Some(token) = fixture.rotate_token_before_response.lock().unwrap().take() { + *token.lock().unwrap() = "new-session-token".to_owned(); + } let response = fixture.response.lock().unwrap().clone(); Response::builder() .status(response.status) @@ -785,6 +789,7 @@ async fn managed_auth_failure_carries_a_non_secret_request_credential_fingerprin body: json!({ "error": { "message": "Unauthorized" } }).to_string(), }; let token = Arc::new(std::sync::Mutex::new("old-request-token".to_owned())); + *fixture.rotate_token_before_response.lock().unwrap() = Some(token.clone()); let token_for_resolver = token.clone(); let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let seen_by_handler = seen.clone(); @@ -804,7 +809,7 @@ async fn managed_auth_failure_carries_a_non_secret_request_credential_fingerprin let request = ModelRequest::new(vec![crate::message::Message::user("hi")]); assert!(model.invoke(&(), request).await.is_err()); - *token.lock().unwrap() = "new-session-token".to_owned(); + assert_eq!(*token.lock().unwrap(), "new-session-token"); let fingerprints = seen.lock().unwrap(); let fingerprint = fingerprints[0].expect("managed credential fingerprint"); From 37e8fd395b6619ac226b7c4f30ab45a51b085949 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:24:53 +0000 Subject: [PATCH 04/25] fix(tinyinference-llm): use a real nul byte in the credential fingerprint domain separator The domain separator was written with an escaped backslash-zero, so the literal two-character sequence was hashed instead of a terminating nul byte. This restores the intended separator, which changes the fingerprint value for every credential. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/tinyhumans/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs index 570a5187..900a0f92 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs @@ -63,7 +63,7 @@ pub struct TinyHumansCredentialFingerprint([u8; 32]); impl TinyHumansCredentialFingerprint { fn from_bearer(bearer: &str) -> Self { let mut digest = Sha256::new(); - digest.update(b"tinyinference/tinyhumans-credential-fingerprint/v1\\0"); + digest.update(b"tinyinference/tinyhumans-credential-fingerprint/v1\0"); digest.update(bearer.as_bytes()); Self(digest.finalize().into()) } From 80cdc36667592811ceb03bf5bdd3e2f155ba094a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:44:47 +0000 Subject: [PATCH 05/25] test(tinyhumans): cover credential fingerprint hex round trip Add a test asserting that a credential fingerprint serializes to canonical lower-case 64-character hex and parses back to the same value, while rejecting upper-case and malformed input. This pins the wire format so future changes cannot silently break stored fingerprints. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/tinyhumans/mod_tests.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index 72d7d75e..0d4145ab 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -818,6 +818,22 @@ async fn managed_auth_failure_carries_a_non_secret_request_credential_fingerprin assert!(!format!("{fingerprint:?}").contains("old-request-token")); } +#[test] +fn credential_fingerprint_has_a_stable_opaque_hex_round_trip() { + let fingerprint = TinyHumansCredentialFingerprint::from_bearer("session-token"); + let encoded = fingerprint.to_hex(); + + assert_eq!(encoded.len(), 64); + assert!(encoded.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))); + assert_eq!(TinyHumansCredentialFingerprint::from_hex(&encoded), Some(fingerprint)); + assert_eq!( + TinyHumansCredentialFingerprint::from_hex(&encoded.to_ascii_uppercase()), + None, + "the serialized form is canonical lower-case hex" + ); + assert_eq!(TinyHumansCredentialFingerprint::from_hex("xyz"), None); +} + #[tokio::test] async fn a_plain_bearer_resolver_reports_no_credential_kind() { let fixture = Fixture::default(); From c9a868752af2404fcbe50a669da9b48156f67fad Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:45:13 +0000 Subject: [PATCH 06/25] feat(tinyhumans): add hex encoding for credential fingerprints Add to_hex and from_hex to TinyHumansCredentialFingerprint so the opaque fingerprint can be serialized to and parsed from a canonical 64-character lower-case hex string without exposing the bearer. Parsing rejects any input that is not exactly 64 lower-case hex digits. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/tinyhumans/mod.rs | 38 +++++++++++++++++++ .../src/providers/tinyhumans/mod_tests.rs | 11 +++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs index 900a0f92..7b224408 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs @@ -68,6 +68,36 @@ impl TinyHumansCredentialFingerprint { Self(digest.finalize().into()) } + /// Returns the canonical lower-case hexadecimal representation of this + /// fingerprint. The bearer is not recoverable from this representation. + #[must_use] + pub fn to_hex(self) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(self.0.len() * 2); + for byte in self.0 { + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + encoded + } + + /// Parses a canonical lower-case hexadecimal fingerprint representation. + /// + /// Returns `None` when `encoded` is not exactly 64 lower-case hex digits. + #[must_use] + pub fn from_hex(encoded: &str) -> Option { + if encoded.len() != 64 { + return None; + } + let mut bytes = [0_u8; 32]; + for (index, pair) in encoded.as_bytes().chunks_exact(2).enumerate() { + let high = decode_lower_hex(pair[0])?; + let low = decode_lower_hex(pair[1])?; + bytes[index] = (high << 4) | low; + } + Some(Self(bytes)) + } + /// Returns whether this fingerprint belongs to `bearer`. #[must_use] pub fn matches_bearer(self, bearer: &str) -> bool { @@ -81,6 +111,14 @@ impl TinyHumansCredentialFingerprint { } } +fn decode_lower_hex(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + _ => None, + } +} + impl std::fmt::Debug for TinyHumansCredentialFingerprint { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str("TinyHumansCredentialFingerprint()") diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index 0d4145ab..009a50d7 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -824,8 +824,15 @@ fn credential_fingerprint_has_a_stable_opaque_hex_round_trip() { let encoded = fingerprint.to_hex(); assert_eq!(encoded.len(), 64); - assert!(encoded.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))); - assert_eq!(TinyHumansCredentialFingerprint::from_hex(&encoded), Some(fingerprint)); + assert!( + encoded + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + ); + assert_eq!( + TinyHumansCredentialFingerprint::from_hex(&encoded), + Some(fingerprint) + ); assert_eq!( TinyHumansCredentialFingerprint::from_hex(&encoded.to_ascii_uppercase()), None, From 5d85552d1077d01999717c92b00bf5f160194150 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 10:49:20 +0000 Subject: [PATCH 07/25] refactor(tinyinference-llm): use as_chunks for hex pair iteration Replaced chunks_exact(2) with as_chunks::<2>() when decoding the credential fingerprint, which lets the compiler verify the pair length at compile time instead of relying on runtime indexing. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/tinyhumans/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs index 7b224408..ae1c3e06 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs @@ -90,7 +90,7 @@ impl TinyHumansCredentialFingerprint { return None; } let mut bytes = [0_u8; 32]; - for (index, pair) in encoded.as_bytes().chunks_exact(2).enumerate() { + for (index, pair) in encoded.as_bytes().as_chunks::<2>().0.iter().enumerate() { let high = decode_lower_hex(pair[0])?; let low = decode_lower_hex(pair[1])?; bytes[index] = (high << 4) | low; From bdd4e594cfcf23c5e4e42bc963eec48ec9d5176f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:21:30 +0000 Subject: [PATCH 08/25] fix(image): resolve bearer credentials asynchronously Co-authored-by: Medulla --- .../src/openrouter_tests.rs | 21 +++++++++++++----- crates/tinyinference-image/src/transport.rs | 22 ++++++++++--------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/crates/tinyinference-image/src/openrouter_tests.rs b/crates/tinyinference-image/src/openrouter_tests.rs index c2d4a13d..d4f547ba 100644 --- a/crates/tinyinference-image/src/openrouter_tests.rs +++ b/crates/tinyinference-image/src/openrouter_tests.rs @@ -257,15 +257,26 @@ fn enveloped_png_reply(_call: usize) -> Response { #[tokio::test] async fn proxied_backend_base_url_and_bearer_resolver() { let fixture = fixture("/agent-integrations/openrouter", enveloped_png_reply, None).await; - let resolver: crate::BearerResolver = Arc::new(|| Ok("session-jwt-abcdefgh".to_owned())); + let resolver_calls = Arc::new(AtomicUsize::new(0)); + let resolver: crate::BearerResolver = { + let resolver_calls = Arc::clone(&resolver_calls); + Arc::new(move || { + let call = resolver_calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { Ok(format!("session-jwt-{call}")) }) + }) + }; let generator = OpenRouterImageGenerator::with_transport( MediaTransport::new(MediaAuth::Bearer(resolver)).with_base_url(&fixture.base_url), - ); + ) + .with_capability_check(false); let response = generator.generate(ImageRequest::new("x")).await.unwrap(); assert_eq!(response.images.len(), 1); assert_eq!(response.cost_usd, Some(0.035)); - let headers = fixture.captured.headers.lock().unwrap()[0].clone(); - assert_eq!(headers["authorization"], "Bearer session-jwt-abcdefgh"); + generator.generate(ImageRequest::new("y")).await.unwrap(); + assert_eq!(resolver_calls.load(Ordering::SeqCst), 2); + let headers = fixture.captured.headers.lock().unwrap(); + assert_eq!(headers[0]["authorization"], "Bearer session-jwt-0"); + assert_eq!(headers[1]["authorization"], "Bearer session-jwt-1"); } #[tokio::test] @@ -288,7 +299,7 @@ async fn failed_envelope_is_an_error_even_with_a_2xx_status() { #[tokio::test] async fn blank_bearer_fails_without_a_request() { let fixture = fixture("/api/v1", png_reply, None).await; - let resolver: crate::BearerResolver = Arc::new(|| Ok(" ".to_owned())); + let resolver: crate::BearerResolver = Arc::new(|| Box::pin(async { Ok(" ".to_owned()) })); let generator = OpenRouterImageGenerator::with_transport( MediaTransport::new(MediaAuth::Bearer(resolver)).with_base_url(&fixture.base_url), ) diff --git a/crates/tinyinference-image/src/transport.rs b/crates/tinyinference-image/src/transport.rs index d86ef6e2..d572e434 100644 --- a/crates/tinyinference-image/src/transport.rs +++ b/crates/tinyinference-image/src/transport.rs @@ -18,6 +18,8 @@ //! connection after a submit is surfaced as-is, because resubmitting could //! bill a second generation. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -40,7 +42,8 @@ pub const DEFAULT_MAX_MEDIA_BYTES: usize = 512 * 1024 * 1024; const MAX_ERROR_BODY_BYTES: usize = 16 * 1024; /// Resolves the current bearer token for each request. -pub type BearerResolver = Arc Result + Send + Sync>; +pub type BearerResolver = + Arc Pin> + Send>> + Send + Sync>; /// How requests authenticate. #[derive(Clone)] @@ -64,10 +67,10 @@ impl MediaAuth { } } - fn token(&self) -> Result { + async fn token(&self) -> Result { let token = match self { Self::ApiKey(key) => key.clone(), - Self::Bearer(resolve) => resolve()?, + Self::Bearer(resolve) => resolve().await?, }; let token = token.trim().to_owned(); if token.is_empty() { @@ -197,7 +200,7 @@ impl MediaTransport { T: DeserializeOwned, { let body = serde_json::to_vec(body)?; - let response = self + let (response, _) = self .send(reqwest::Method::POST, path, Some(body), Billing::Billable) .await?; decode_json_with_limit(response, self.json_limit()).await @@ -210,7 +213,7 @@ impl MediaTransport { /// [`Error::Auth`], [`Error::Http`], [`Error::Transport`] or /// [`Error::Decode`] after retries are exhausted. pub async fn get_json(&self, path: &str) -> Result { - let response = self + let (response, _) = self .send(reqwest::Method::GET, path, None, Billing::Idempotent) .await?; decode_json_with_limit(response, self.json_limit()).await @@ -264,8 +267,7 @@ impl MediaTransport { } async fn get_bytes_once(&self, path: &str) -> Result<(Bytes, Option)> { - let token = self.auth.token()?; - let mut response = self + let (mut response, token) = self .send(reqwest::Method::GET, path, None, Billing::Idempotent) .await?; let content_type = response @@ -317,11 +319,11 @@ impl MediaTransport { path: &str, body: Option>, billing: Billing, - ) -> Result { + ) -> Result<(reqwest::Response, String)> { let url = self.url(path); let mut attempt = 0u32; loop { - let token = self.auth.token()?; + let token = self.auth.token().await?; let mut request = self .client .request(method.clone(), &url) @@ -340,7 +342,7 @@ impl MediaTransport { ); let outcome = request.send().await; let (retry, retry_after, error) = match outcome { - Ok(response) if response.status().is_success() => return Ok(response), + Ok(response) if response.status().is_success() => return Ok((response, token)), Ok(response) => { let status = response.status().as_u16(); let retry_after = response From 4d15d85e21fde8ffa282337d1d57589433f39803 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:22:32 +0000 Subject: [PATCH 09/25] refactor(embeddings): make bearer resolver async The bearer resolver now returns a pinned future so hosts can fetch tokens asynchronously, and the delegate helper awaits it before building the OpenAI-compatible client. Tests were updated to wrap their resolvers in async blocks. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-embeddings/src/cloud.rs | 13 ++++++++----- crates/tinyinference-embeddings/src/cloud_tests.rs | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/tinyinference-embeddings/src/cloud.rs b/crates/tinyinference-embeddings/src/cloud.rs index 87f75aef..5d6c1a63 100644 --- a/crates/tinyinference-embeddings/src/cloud.rs +++ b/crates/tinyinference-embeddings/src/cloud.rs @@ -1,5 +1,7 @@ //! Bearer-authenticated OpenAI-compatible cloud embedding model. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use async_trait::async_trait; @@ -13,7 +15,8 @@ pub const DEFAULT_CLOUD_MODEL: &str = "embedding-v1"; pub const DEFAULT_CLOUD_DIMENSIONS: usize = 1024; /// Resolves the current bearer token for each request. -pub type BearerResolver = Arc Result + Send + Sync>; +pub type BearerResolver = + Arc Pin> + Send>> + Send + Sync>; /// Host policy hook invoked immediately before an embedding batch may leave /// the process. This keeps privacy and audit policy in the host without @@ -81,7 +84,7 @@ impl CloudEmbeddingModel { /// /// [`Error::Validation`] when an input is empty or whitespace, when the /// bearer resolver fails, or when it resolves to a blank token. - fn delegate(&self, texts: &[String]) -> Result> { + async fn delegate(&self, texts: &[String]) -> Result> { if texts.is_empty() { return Ok(None); } @@ -95,7 +98,7 @@ impl CloudEmbeddingModel { if let Some(guard) = &self.egress_guard { guard(&self.model, texts.len())?; } - let bearer = (self.bearer)()?; + let bearer = (self.bearer)().await?; if bearer.trim().is_empty() { return Err(Error::Validation( "No backend session for cloud embeddings".into(), @@ -128,7 +131,7 @@ impl EmbeddingModel for CloudEmbeddingModel { } async fn embed(&self, texts: &[String]) -> Result>> { - let Some(delegate) = self.delegate(texts)? else { + let Some(delegate) = self.delegate(texts).await? else { return Ok(Vec::new()); }; delegate.embed(texts).await @@ -140,7 +143,7 @@ impl EmbeddingModel for CloudEmbeddingModel { &self, texts: &[String], ) -> Result<(Vec>, Option)> { - let Some(delegate) = self.delegate(texts)? else { + let Some(delegate) = self.delegate(texts).await? else { return Ok((Vec::new(), None)); }; delegate.embed_with_usage(texts).await diff --git a/crates/tinyinference-embeddings/src/cloud_tests.rs b/crates/tinyinference-embeddings/src/cloud_tests.rs index 3ff9dab2..3af159c6 100644 --- a/crates/tinyinference-embeddings/src/cloud_tests.rs +++ b/crates/tinyinference-embeddings/src/cloud_tests.rs @@ -1,11 +1,11 @@ use super::*; fn missing_bearer() -> BearerResolver { - Arc::new(|| { + Arc::new(|| Box::pin(async { Err(Error::Validation( "No backend session for cloud embeddings".into(), )) - }) + })) } #[test] @@ -49,7 +49,7 @@ async fn empty_and_blank_batches_skip_egress_guard_and_bearer_for_both_entrypoin 8, Arc::new(move || { bearer_count.fetch_add(1, Ordering::SeqCst); - Ok("token".into()) + Box::pin(async { Ok("token".into()) }) }), ) .with_egress_guard(Arc::new(move |_, _| { From 2f1d2d672c5242e0546df4d5122964e3347d9fe8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:23:11 +0000 Subject: [PATCH 10/25] refactor(tinyhumans): make bearer resolvers async Bearer resolvers now return a boxed future instead of a plain result, so credentials can be fetched asynchronously during a request. The model constructors and the bearer source await the resolver, and the tests were updated to supply async closures. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tinyhumans/cache_identity_tests.rs | 12 +++-- .../src/providers/tinyhumans/mod.rs | 38 +++++++++++--- .../src/providers/tinyhumans/mod_tests.rs | 52 +++++++++++-------- 3 files changed, 68 insertions(+), 34 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs index d859da2b..002e8e0a 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs @@ -35,17 +35,17 @@ fn cache_namespace_is_trimmed_and_blank_input_keeps_the_current_value() { #[test] fn cache_identity_keeps_endpoint_and_default_model_dimensions() { let first = TinyHumansModel::new("https://one.example.test", "", "model-a", || { - Ok("credential-a".into()) + Box::pin(async { Ok("credential-a".into()) }) }) .unwrap() .with_cache_namespace("openhuman"); let other_endpoint = TinyHumansModel::new("https://two.example.test", "", "model-a", || { - Ok("credential-b".into()) + Box::pin(async { Ok("credential-b".into()) }) }) .unwrap() .with_cache_namespace("openhuman"); let other_model = TinyHumansModel::new("https://one.example.test", "", "model-b", || { - Ok("credential-c".into()) + Box::pin(async { Ok("credential-c".into()) }) }) .unwrap() .with_cache_namespace("openhuman"); @@ -63,8 +63,10 @@ fn reading_cache_identity_never_resolves_or_includes_credentials() { let reads = Arc::new(AtomicUsize::new(0)); let resolver_reads = Arc::clone(&reads); let model = TinyHumansModel::new("https://api.example.test", "", "default-model", move || { - resolver_reads.fetch_add(1, Ordering::SeqCst); - Ok("must-not-appear-in-cache-identity".into()) + Box::pin(async move { + resolver_reads.fetch_add(1, Ordering::SeqCst); + Ok("must-not-appear-in-cache-identity".into()) + }) }) .unwrap() .with_cache_namespace("openhuman"); diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs index ae1c3e06..43e52eed 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs @@ -9,6 +9,8 @@ mod catalog; mod reasoning; mod usage; +use std::future::Future; +use std::pin::Pin; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -29,7 +31,9 @@ pub use reasoning::{REASONING_OFF_METADATA_KEY, apply_reasoning, with_thread_id} pub use usage::project_tinyhumans_envelope; /// Resolves the current bearer credential for a managed inference call. -pub type BearerResolver = Arc Result + Send + Sync>; +pub type BearerResolver = Arc< + dyn Fn() -> Pin> + Send>> + Send + Sync, +>; /// Which kind of credential a managed request authenticated with. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -49,8 +53,12 @@ pub struct TinyHumansCredential { pub kind: TinyHumansCredentialKind, } -type KindedResolver = - Arc Result<(String, Option)> + Send + Sync>; +type KindedResolver = Arc< + dyn Fn() -> Pin< + Box)>> + Send>, + > + Send + + Sync, +>; /// Opaque, non-secret identity for a bearer credential used by one request. /// @@ -274,13 +282,19 @@ impl TinyHumansModel { base_url: impl AsRef, model: impl AsRef, default_model: impl AsRef, - resolver: impl Fn() -> Result + Send + Sync + 'static, + resolver: impl Fn() -> Pin> + Send>> + + Send + + Sync + + 'static, ) -> Result { Self::build( base_url, model, default_model, - Arc::new(move || resolver().map(|bearer| (bearer, None))), + Arc::new(move || { + let future = resolver(); + Box::pin(async move { future.await.map(|bearer| (bearer, None)) }) + }), ) } @@ -294,14 +308,22 @@ impl TinyHumansModel { base_url: impl AsRef, model: impl AsRef, default_model: impl AsRef, - resolver: impl Fn() -> Result + Send + Sync + 'static, + resolver: impl Fn() -> Pin> + Send>> + + Send + + Sync + + 'static, ) -> Result { Self::build( base_url, model, default_model, Arc::new(move || { - resolver().map(|credential| (credential.bearer, Some(credential.kind))) + let future = resolver(); + Box::pin(async move { + future + .await + .map(|credential| (credential.bearer, Some(credential.kind))) + }) }), ) } @@ -668,7 +690,7 @@ struct ResolvedBearer(KindedResolver); #[async_trait] impl BearerSource for ResolvedBearer { async fn current(&self) -> Result> { - let (bearer, kind) = (self.0)()?; + let (bearer, kind) = (self.0)().await?; // Outside a dispatch scope (e.g. the readiness probe) there is nothing // to attribute, so a missing task-local is fine. let sent = SentCredential { diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index 009a50d7..1a72f099 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -93,7 +93,9 @@ async fn serve(fixture: Fixture) -> String { } fn fixture_model(base: &str) -> TinyHumansModel { - TinyHumansModel::new(base, "", "default-model", || Ok("test-token".into())) + TinyHumansModel::new(base, "", "default-model", || { + Box::pin(async { Ok("test-token".into()) }) + }) .expect("model construction") } @@ -104,10 +106,12 @@ async fn dynamic_bearer_is_resolved_again_for_invoke_and_stream() { let reads = Arc::new(AtomicUsize::new(0)); let source = reads.clone(); let model = TinyHumansModel::new(&base, "", "default-model", move || { - Ok(format!( - "token-{}", - source.fetch_add(1, Ordering::SeqCst) + 1 - )) + Box::pin(async move { + Ok(format!( + "token-{}", + source.fetch_add(1, Ordering::SeqCst) + 1 + )) + }) }) .unwrap(); model @@ -137,7 +141,7 @@ async fn constructor_model_is_default_and_nonblank_request_model_overrides_it() let fixture = Fixture::default(); let base = serve(fixture.clone()).await; let model = TinyHumansModel::new(&base, " constructor-model ", "fallback-model", || { - Ok("token".into()) + Box::pin(async { Ok("token".into()) }) }) .unwrap(); for requested_model in [None, Some("request-model"), Some(" ")] { @@ -152,8 +156,10 @@ async fn constructor_model_is_default_and_nonblank_request_model_overrides_it() assert_eq!(requests[2].1["model"], "constructor-model"); } - let fallback_model = - TinyHumansModel::new(&base, " ", "fallback-model", || Ok("token".into())).unwrap(); + let fallback_model = TinyHumansModel::new(&base, " ", "fallback-model", || { + Box::pin(async { Ok("token".into()) }) + }) + .unwrap(); fallback_model .invoke( &(), @@ -172,7 +178,7 @@ async fn resolver_failure_stops_before_transport() { let fixture = Fixture::default(); let base = serve(fixture.clone()).await; let model = TinyHumansModel::new(&base, "m", "default-model", || { - Err(Error::Model("credential source failed".into())) + Box::pin(async { Err(Error::Model("credential source failed".into())) }) }) .unwrap(); let error = model @@ -477,7 +483,7 @@ async fn readiness_blocks_only_confirmed_missing_provider_configuration() { #[tokio::test] async fn readiness_fails_open_when_bearer_resolution_fails() { let model = TinyHumansModel::new("http://127.0.0.1:1", "m", "default-model", || { - Err(Error::Model("credential lookup unavailable".into())) + Box::pin(async { Err(Error::Model("credential lookup unavailable".into())) }) }) .unwrap(); model @@ -736,14 +742,16 @@ async fn managed_auth_failure_reports_the_credential_its_request_carried() { let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let seen_by_handler = seen.clone(); let model = TinyHumansModel::new_with_credential(base, "", "default-model", move || { - let api_key = resolver_use_api_key.load(Ordering::SeqCst); - Ok(TinyHumansCredential { - bearer: if api_key { "key".into() } else { "jwt".into() }, - kind: if api_key { - TinyHumansCredentialKind::ApiKey - } else { - TinyHumansCredentialKind::Session - }, + Box::pin(async move { + let api_key = resolver_use_api_key.load(Ordering::SeqCst); + Ok(TinyHumansCredential { + bearer: if api_key { "key".into() } else { "jwt".into() }, + kind: if api_key { + TinyHumansCredentialKind::ApiKey + } else { + TinyHumansCredentialKind::Session + }, + }) }) }) .expect("model construction") @@ -794,9 +802,11 @@ async fn managed_auth_failure_carries_a_non_secret_request_credential_fingerprin let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let seen_by_handler = seen.clone(); let model = TinyHumansModel::new_with_credential(base, "", "default-model", move || { - Ok(TinyHumansCredential { - bearer: token_for_resolver.lock().unwrap().clone(), - kind: TinyHumansCredentialKind::Session, + Box::pin(async move { + Ok(TinyHumansCredential { + bearer: token_for_resolver.lock().unwrap().clone(), + kind: TinyHumansCredentialKind::Session, + }) }) }) .unwrap() From 233c20cc828620b93f38d891fe7e2a8f370c606b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:23:48 +0000 Subject: [PATCH 11/25] refactor(embeddings): make managed model factory async The managed model factory callback now returns a pinned future and the provider construction helpers are async, so hosts can perform asynchronous work such as token acquisition while building a model. Callers in the connection test were updated to await the new signatures. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tinyinference-embeddings/src/factory.rs | 23 +++++++++++-------- crates/tinyinference-embeddings/src/verify.rs | 1 + 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/tinyinference-embeddings/src/factory.rs b/crates/tinyinference-embeddings/src/factory.rs index 8c63793a..0bcf47c8 100644 --- a/crates/tinyinference-embeddings/src/factory.rs +++ b/crates/tinyinference-embeddings/src/factory.rs @@ -1,5 +1,7 @@ //! Provider construction and custom-endpoint validation. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use crate::{ @@ -12,8 +14,11 @@ use crate::{ pub type CredentialResolver = Arc String + Send + Sync>; /// Constructs the host-authenticated managed embedding model. -pub type ManagedModelFactory = - Arc Result> + Send + Sync>; +pub type ManagedModelFactory = Arc< + dyn Fn(&str, usize) -> Pin>> + Send>> + + Send + + Sync, +>; /// Host-independent inputs for selecting an embedding provider. #[derive(Clone, Debug)] @@ -30,7 +35,7 @@ pub struct EmbeddingFactorySettings { /// Builds a provider from persisted settings while keeping credential storage /// and managed authentication in host-supplied callbacks. -pub fn create_configured_embedding_model( +pub async fn create_configured_embedding_model( settings: &EmbeddingFactorySettings, credentials: &CredentialResolver, managed: &ManagedModelFactory, @@ -42,7 +47,7 @@ pub fn create_configured_embedding_model( None => (provider, None), }; if matches!(slug, "cloud" | "managed") { - return managed(&settings.model, settings.dimensions); + return managed(&settings.model, settings.dimensions).await; } let api_key = credentials(slug); create_embedding_model( @@ -57,14 +62,14 @@ pub fn create_configured_embedding_model( /// Builds the configured provider, falling back to the managed model when the /// selection is blank, invalid, or missing a required credential. -pub fn create_default_embedding_model( +pub async fn create_default_embedding_model( settings: &EmbeddingFactorySettings, credentials: &CredentialResolver, managed: &ManagedModelFactory, ) -> Result> { let provider = settings.provider.trim(); if provider.is_empty() || matches!(provider, "cloud" | "managed") { - return managed(DEFAULT_CLOUD_MODEL, DEFAULT_CLOUD_DIMENSIONS); + return managed(DEFAULT_CLOUD_MODEL, DEFAULT_CLOUD_DIMENSIONS).await; } let slug = if provider.starts_with("custom:") { @@ -79,14 +84,14 @@ pub fn create_default_embedding_model( provider = slug, "configured embedding provider has no credential; falling back to managed" ); - return managed(DEFAULT_CLOUD_MODEL, DEFAULT_CLOUD_DIMENSIONS); + return managed(DEFAULT_CLOUD_MODEL, DEFAULT_CLOUD_DIMENSIONS).await; } - match create_configured_embedding_model(settings, credentials, managed) { + match create_configured_embedding_model(settings, credentials, managed).await { Ok(model) => Ok(model), Err(error) => { tracing::warn!(provider = slug, %error, "configured embedding provider failed to build; falling back to managed"); - managed(DEFAULT_CLOUD_MODEL, DEFAULT_CLOUD_DIMENSIONS) + managed(DEFAULT_CLOUD_MODEL, DEFAULT_CLOUD_DIMENSIONS).await } } } diff --git a/crates/tinyinference-embeddings/src/verify.rs b/crates/tinyinference-embeddings/src/verify.rs index 5ac19a20..565df87a 100644 --- a/crates/tinyinference-embeddings/src/verify.rs +++ b/crates/tinyinference-embeddings/src/verify.rs @@ -200,6 +200,7 @@ pub async fn test_embedding_connection_detailed( }); } let model = create_configured_embedding_model(settings, credentials, managed) + .await .map_err(ConnectionTestError::Build)?; let vectors = model .embed(&["connection test".to_owned()]) From 98426c36a38ca6310d6143b3cf044666e2d17d72 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:24:10 +0000 Subject: [PATCH 12/25] test: adapt factory closures to async signatures Update the embedding and tinyhumans test doubles so their factory closures return boxed futures, matching the now-async managed model factory and resolver signatures. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/verify_tests.rs | 17 +++++++++++------ .../src/providers/tinyhumans/observer_tests.rs | 6 ++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/tinyinference-embeddings/src/verify_tests.rs b/crates/tinyinference-embeddings/src/verify_tests.rs index 07633312..5f158a2e 100644 --- a/crates/tinyinference-embeddings/src/verify_tests.rs +++ b/crates/tinyinference-embeddings/src/verify_tests.rs @@ -122,8 +122,9 @@ async fn connection_test_uses_host_managed_factory_and_returns_actual_width() { ollama_base_url: "http://localhost:11434".into(), }; let credentials: CredentialResolver = std::sync::Arc::new(|_| String::new()); - let managed: ManagedModelFactory = - std::sync::Arc::new(|_, dims| Ok(Box::new(crate::MockEmbeddingModel::new(dims)))); + let managed: ManagedModelFactory = std::sync::Arc::new(|_, dims| { + Box::pin(async move { Ok(Box::new(crate::MockEmbeddingModel::new(dims))) }) + }); let result = test_embedding_connection(&settings, &credentials, &managed, None) .await .unwrap(); @@ -139,7 +140,8 @@ async fn detailed_connection_test_identifies_model_build_failures() { ollama_base_url: "http://localhost:11434".into(), }; let credentials: CredentialResolver = std::sync::Arc::new(|_| String::new()); - let managed: ManagedModelFactory = std::sync::Arc::new(|_, _| unreachable!()); + let managed: ManagedModelFactory = + std::sync::Arc::new(|_, _| Box::pin(async { unreachable!() })); assert!(matches!( test_embedding_connection_detailed(&settings, &credentials, &managed, None).await, @@ -177,7 +179,9 @@ async fn detailed_connection_test_identifies_embedding_failures() { ollama_base_url: "http://localhost:11434".into(), }; let credentials: CredentialResolver = std::sync::Arc::new(|_| String::new()); - let managed: ManagedModelFactory = std::sync::Arc::new(|_, _| Ok(Box::new(FailingModel))); + let managed: ManagedModelFactory = std::sync::Arc::new(|_, _| { + Box::pin(async { Ok(Box::new(FailingModel) as Box) }) + }); assert!(matches!( test_embedding_connection_detailed(&settings, &credentials, &managed, None).await, @@ -194,7 +198,8 @@ async fn connection_test_preserves_noop_provider_success() { ollama_base_url: String::new(), }; let credentials: CredentialResolver = std::sync::Arc::new(|_| String::new()); - let managed: ManagedModelFactory = std::sync::Arc::new(|_, _| unreachable!()); + let managed: ManagedModelFactory = + std::sync::Arc::new(|_, _| Box::pin(async { unreachable!() })); let result = test_embedding_connection(&settings, &credentials, &managed, None) .await .unwrap(); @@ -331,7 +336,7 @@ async fn custom_connection_infers_stored_endpoint_and_resolves_custom_credential capture.lock().unwrap().push(slug.to_owned()); "key".into() }); - let managed: ManagedModelFactory = Arc::new(|_, _| unreachable!()); + let managed: ManagedModelFactory = Arc::new(|_, _| Box::pin(async { unreachable!() })); let result = test_embedding_connection(&settings, &credentials, &managed, None) .await .unwrap(); diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs index 2a2cf3d6..2d7f9532 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs @@ -215,8 +215,10 @@ async fn observer_classifies_readiness_and_non_provider_failures() { let errors = Arc::new(std::sync::Mutex::new(Vec::new())); let errors_seen = Arc::clone(&errors); let model = TinyHumansModel::new("http://127.0.0.1:9", "model", "default-model", move || { - calls.fetch_add(1, Ordering::SeqCst); - Err(Error::Model("resolver marker".into())) + Box::pin(async move { + calls.fetch_add(1, Ordering::SeqCst); + Err(Error::Model("resolver marker".into())) + }) }) .unwrap() .with_error_observer(move |error, point| { From 956fea19c3f8fadf3b78426adc1ba207eed8a870 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:24:26 +0000 Subject: [PATCH 13/25] fix(tinyinference-llm): correct malformed bearer resolver type alias The `BearerResolver` type alias in the tinyhumans provider had a broken definition that would not compile, so it has been rewritten to a valid `Arc Pin> + Send>> + Send + Sync>` form. The cloud embeddings test helper and a tinyhumans test were also reformatted to match the corrected closure and indentation. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-embeddings/src/cloud_tests.rs | 12 +++++++----- .../src/providers/tinyhumans/mod.rs | 5 ++--- .../src/providers/tinyhumans/mod_tests.rs | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/tinyinference-embeddings/src/cloud_tests.rs b/crates/tinyinference-embeddings/src/cloud_tests.rs index 3af159c6..e935e500 100644 --- a/crates/tinyinference-embeddings/src/cloud_tests.rs +++ b/crates/tinyinference-embeddings/src/cloud_tests.rs @@ -1,11 +1,13 @@ use super::*; fn missing_bearer() -> BearerResolver { - Arc::new(|| Box::pin(async { - Err(Error::Validation( - "No backend session for cloud embeddings".into(), - )) - })) + Arc::new(|| { + Box::pin(async { + Err(Error::Validation( + "No backend session for cloud embeddings".into(), + )) + }) + }) } #[test] diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs index 43e52eed..f051d117 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod.rs @@ -31,9 +31,8 @@ pub use reasoning::{REASONING_OFF_METADATA_KEY, apply_reasoning, with_thread_id} pub use usage::project_tinyhumans_envelope; /// Resolves the current bearer credential for a managed inference call. -pub type BearerResolver = Arc< - dyn Fn() -> Pin> + Send>> + Send + Sync, ->; +pub type BearerResolver = + Arc Pin> + Send>> + Send + Sync>; /// Which kind of credential a managed request authenticated with. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index 1a72f099..0e7761d4 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -96,7 +96,7 @@ fn fixture_model(base: &str) -> TinyHumansModel { TinyHumansModel::new(base, "", "default-model", || { Box::pin(async { Ok("test-token".into()) }) }) - .expect("model construction") + .expect("model construction") } #[tokio::test] From f590fb4882284a9e73992b0aa3dc4191a3ca9f91 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:24:39 +0000 Subject: [PATCH 14/25] test(embeddings): annotate mock model as trait object in verify test The managed model factory closure now casts the mock embedding model to a boxed EmbeddingModel trait object, matching the factory's expected return type and keeping the connection verification test compiling. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-embeddings/src/verify_tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-embeddings/src/verify_tests.rs b/crates/tinyinference-embeddings/src/verify_tests.rs index 5f158a2e..414c3641 100644 --- a/crates/tinyinference-embeddings/src/verify_tests.rs +++ b/crates/tinyinference-embeddings/src/verify_tests.rs @@ -123,7 +123,9 @@ async fn connection_test_uses_host_managed_factory_and_returns_actual_width() { }; let credentials: CredentialResolver = std::sync::Arc::new(|_| String::new()); let managed: ManagedModelFactory = std::sync::Arc::new(|_, dims| { - Box::pin(async move { Ok(Box::new(crate::MockEmbeddingModel::new(dims))) }) + Box::pin(async move { + Ok(Box::new(crate::MockEmbeddingModel::new(dims)) as Box) + }) }); let result = test_embedding_connection(&settings, &credentials, &managed, None) .await From 4cdf479191a9cc35e14a7f7ea226bcf6b5d3049e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:25:07 +0000 Subject: [PATCH 15/25] test(tinyinference-llm): clone captured state into async resolver blocks The credential resolver closures in the tinyhumans tests moved their captured handles into the returned future, so the surrounding test could no longer use them. Cloning the handles inside the closure before the async block keeps the counters and buffers accessible to the assertions that follow. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/tinyhumans/cache_identity_tests.rs | 1 + crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs | 3 +++ .../src/providers/tinyhumans/observer_tests.rs | 1 + 3 files changed, 5 insertions(+) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs index 002e8e0a..6aa44cab 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/cache_identity_tests.rs @@ -63,6 +63,7 @@ fn reading_cache_identity_never_resolves_or_includes_credentials() { let reads = Arc::new(AtomicUsize::new(0)); let resolver_reads = Arc::clone(&reads); let model = TinyHumansModel::new("https://api.example.test", "", "default-model", move || { + let resolver_reads = Arc::clone(&resolver_reads); Box::pin(async move { resolver_reads.fetch_add(1, Ordering::SeqCst); Ok("must-not-appear-in-cache-identity".into()) diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs index 0e7761d4..c3599440 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/mod_tests.rs @@ -106,6 +106,7 @@ async fn dynamic_bearer_is_resolved_again_for_invoke_and_stream() { let reads = Arc::new(AtomicUsize::new(0)); let source = reads.clone(); let model = TinyHumansModel::new(&base, "", "default-model", move || { + let source = Arc::clone(&source); Box::pin(async move { Ok(format!( "token-{}", @@ -742,6 +743,7 @@ async fn managed_auth_failure_reports_the_credential_its_request_carried() { let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let seen_by_handler = seen.clone(); let model = TinyHumansModel::new_with_credential(base, "", "default-model", move || { + let resolver_use_api_key = Arc::clone(&resolver_use_api_key); Box::pin(async move { let api_key = resolver_use_api_key.load(Ordering::SeqCst); Ok(TinyHumansCredential { @@ -802,6 +804,7 @@ async fn managed_auth_failure_carries_a_non_secret_request_credential_fingerprin let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let seen_by_handler = seen.clone(); let model = TinyHumansModel::new_with_credential(base, "", "default-model", move || { + let token_for_resolver = Arc::clone(&token_for_resolver); Box::pin(async move { Ok(TinyHumansCredential { bearer: token_for_resolver.lock().unwrap().clone(), diff --git a/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs b/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs index 2d7f9532..abd7076f 100644 --- a/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs +++ b/crates/tinyinference-llm/src/providers/tinyhumans/observer_tests.rs @@ -215,6 +215,7 @@ async fn observer_classifies_readiness_and_non_provider_failures() { let errors = Arc::new(std::sync::Mutex::new(Vec::new())); let errors_seen = Arc::clone(&errors); let model = TinyHumansModel::new("http://127.0.0.1:9", "model", "default-model", move || { + let calls = Arc::clone(&calls); Box::pin(async move { calls.fetch_add(1, Ordering::SeqCst); Err(Error::Model("resolver marker".into())) From 3db292be7b13cd430f35b6c8fe146d93a3111ff1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:29:57 +0000 Subject: [PATCH 16/25] refactor(providers): make embedding and catalog limit factories async Embedding factories and catalog limit resolvers now return boxed futures instead of resolving synchronously, and build_embedding is an async method that awaits the selected factory. This lets provider overrides perform asynchronous work while still accepting borrowed, non-clone host contexts. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/overrides.rs | 32 ++++-- .../tests/provider_overrides.rs | 103 ++++++++++-------- 2 files changed, 82 insertions(+), 53 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/overrides.rs b/crates/tinyinference-llm/src/providers/overrides.rs index da169135..764f0a86 100644 --- a/crates/tinyinference-llm/src/providers/overrides.rs +++ b/crates/tinyinference-llm/src/providers/overrides.rs @@ -199,8 +199,11 @@ impl std::fmt::Debug for EmbeddingRequest<'_, Ctx> { /// Builds a provider embedding model. pub type EmbeddingFactory = Arc< - dyn for<'a> Fn(&EmbeddingRequest<'a, Ctx>) -> EmbeddingResult> - + Send + dyn for<'a> Fn( + &'a EmbeddingRequest<'a, Ctx>, + ) -> Pin< + Box>> + Send + 'a>, + > + Send + Sync, >; @@ -224,7 +227,14 @@ pub type CatalogListing = Arc) -> BoxFuture> + Send + Sync>; /// Builds the provider limit discovery request for a model, if supported. -pub type CatalogLimits = Arc Option + Send + Sync>; +pub type CatalogLimits = Arc< + dyn for<'a> Fn( + &'a str, + &'a Ctx, + ) -> Pin> + Send + 'a>> + + Send + + Sync, +>; /// A provider catalog listing and optional model-limit resolver. pub struct CatalogEntry { @@ -338,18 +348,22 @@ impl ProviderOverrides { /// /// # Errors /// Returns the selected factory's construction error. - pub fn build_embedding( - &self, + pub async fn build_embedding<'a, B>( + &'a self, provider_id: &str, - request: &EmbeddingRequest<'_, Ctx>, + request: &'a EmbeddingRequest<'a, Ctx>, builtin: B, ) -> EmbeddingResult> where - B: FnOnce(&EmbeddingRequest<'_, Ctx>) -> EmbeddingResult>, + B: FnOnce( + &'a EmbeddingRequest<'a, Ctx>, + ) -> Pin< + Box>> + Send + 'a>, + >, { match self.embedding_factory(provider_id) { - Some(factory) => factory(request), - None => builtin(request), + Some(factory) => factory(request).await, + None => builtin(request).await, } } diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index 9767a467..ef997c5e 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -17,8 +17,8 @@ struct HostContext<'a> { _ownership_marker: std::marker::PhantomData<&'a mut ()>, } -#[test] -fn synchronous_factories_accept_borrowed_non_clone_host_contexts() { +#[tokio::test] +async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { let config = String::from("borrowed-host-config"); let context = HostContext { borrowed_config: &config, @@ -41,9 +41,11 @@ fn synchronous_factories_accept_borrowed_non_clone_host_contexts() { |id| { (id == "hosted").then(|| { Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| { - assert_eq!(request.context.borrowed_config, "borrowed-host-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "borrowed-host-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) }) as _ }) }, @@ -64,15 +66,17 @@ fn synchronous_factories_accept_borrowed_non_clone_host_contexts() { ); assert_eq!(chat.unwrap().1, "borrowed-model"); - let embedding = overrides.build_embedding( - "hosted", - &EmbeddingRequest { - model: "embed", - dimensions: 3, - context: &context, - }, - |_| panic!("registered embedding factory should be used"), - ); + let embedding = overrides + .build_embedding( + "hosted", + &EmbeddingRequest { + model: "embed", + dimensions: 3, + context: &context, + }, + |_| panic!("registered embedding factory should be used"), + ) + .await; assert_eq!(embedding.unwrap().dimensions(), 3); } @@ -119,9 +123,11 @@ async fn independent_live_family_lookups_override_only_their_builtin() { |id| { (id == "shared").then(|| { Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| { - assert_eq!(request.context.borrowed_config, "scoped-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "scoped-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) }) as _ }) }, @@ -134,14 +140,15 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }) }), limits: Some(Arc::new(|model, context: &HostContext<'static>| { - (model == "catalog-model" && context.borrowed_config == "catalog-config").then( - || { - tinyinference_llm::model::discover::DiscoveryRequest::new( - "https://provider.invalid/v1", - model, - ) - }, - ) + Box::pin(async move { + (model == "catalog-model" && context.borrowed_config == "catalog-config") + .then(|| { + tinyinference_llm::model::discover::DiscoveryRequest::new( + "https://provider.invalid/v1", + model, + ) + }) + }) })), }) }, @@ -176,30 +183,38 @@ async fn independent_live_family_lookups_override_only_their_builtin() { ); assert!(override_error.is_err()); - let embedding = overrides.build_embedding( - "shared", - &EmbeddingRequest { - model: "embed-model", - dimensions: 8, - context: &scoped_context, - }, - |_| panic!("embedding override must run before builtin"), - ); + let embedding = overrides + .build_embedding( + "shared", + &EmbeddingRequest { + model: "embed-model", + dimensions: 8, + context: &scoped_context, + }, + |_| panic!("embedding override must run before builtin"), + ) + .await; assert_eq!(embedding.unwrap().dimensions(), 8); let fallback_context = HostContext { borrowed_config: "fallback-config", _ownership_marker: std::marker::PhantomData, }; - let fallback = overrides.build_embedding( - "unregistered", - &EmbeddingRequest { - model: "embed-model", - dimensions: 4, - context: &fallback_context, - }, - |_| Ok(Box::new(MockEmbeddingModel::new(4)) as Box), - ); + let fallback = overrides + .build_embedding( + "unregistered", + &EmbeddingRequest { + model: "embed-model", + dimensions: 4, + context: &fallback_context, + }, + |_| { + Box::pin(async { + Ok(Box::new(MockEmbeddingModel::new(4)) as Box) + }) + }, + ) + .await; assert_eq!(fallback.unwrap().dimensions(), 4); chat_enabled.store(false, Ordering::SeqCst); @@ -241,7 +256,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { _ownership_marker: std::marker::PhantomData, }; let limits = overrides.catalog_entry("shared").unwrap().limits.unwrap(); - assert!(limits("catalog-model", &catalog_context).is_some()); + assert!(limits("catalog-model", &catalog_context).await.is_some()); let listing = overrides .list_catalog( "shared", From a8469e2a87337e983f670ed3a8f1d6e16005ee3c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:31:49 +0000 Subject: [PATCH 17/25] refactor(providers): extract named future aliases for override factories The inline `Pin>` return types on the embedding and catalog-limits factory aliases are replaced with named `EmbeddingFactoryFuture` and `CatalogLimitsFuture` aliases, and the tests annotate their closures with them. This keeps the signatures readable and gives callers a stable name for the borrowed async result. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/overrides.rs | 34 +++++++------------ .../tests/provider_overrides.rs | 34 +++++++++++-------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/overrides.rs b/crates/tinyinference-llm/src/providers/overrides.rs index 764f0a86..f183112f 100644 --- a/crates/tinyinference-llm/src/providers/overrides.rs +++ b/crates/tinyinference-llm/src/providers/overrides.rs @@ -197,15 +197,13 @@ impl std::fmt::Debug for EmbeddingRequest<'_, Ctx> { } } +/// A provider embedding factory's async result, borrowing its request context. +pub type EmbeddingFactoryFuture<'a> = + Pin>> + Send + 'a>>; + /// Builds a provider embedding model. -pub type EmbeddingFactory = Arc< - dyn for<'a> Fn( - &'a EmbeddingRequest<'a, Ctx>, - ) -> Pin< - Box>> + Send + 'a>, - > + Send - + Sync, ->; +pub type EmbeddingFactory = + Arc Fn(&'a EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a> + Send + Sync>; /// Owned host context passed to a catalog listing implementation. pub struct CatalogRequest { @@ -226,15 +224,13 @@ impl std::fmt::Debug for CatalogRequest { pub type CatalogListing = Arc) -> BoxFuture> + Send + Sync>; +/// Async result of a provider limit-discovery request. +pub type CatalogLimitsFuture<'a> = + Pin> + Send + 'a>>; + /// Builds the provider limit discovery request for a model, if supported. -pub type CatalogLimits = Arc< - dyn for<'a> Fn( - &'a str, - &'a Ctx, - ) -> Pin> + Send + 'a>> - + Send - + Sync, ->; +pub type CatalogLimits = + Arc Fn(&'a str, &'a Ctx) -> CatalogLimitsFuture<'a> + Send + Sync>; /// A provider catalog listing and optional model-limit resolver. pub struct CatalogEntry { @@ -355,11 +351,7 @@ impl ProviderOverrides { builtin: B, ) -> EmbeddingResult> where - B: FnOnce( - &'a EmbeddingRequest<'a, Ctx>, - ) -> Pin< - Box>> + Send + 'a>, - >, + B: FnOnce(&'a EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a>, { match self.embedding_factory(provider_id) { Some(factory) => factory(request).await, diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index ef997c5e..da005695 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -7,8 +7,9 @@ use tinyinference_embeddings::{EmbeddingModel, MockEmbeddingModel}; use tinyinference_llm::model::ChatModel; use tinyinference_llm::providers::MockModel; use tinyinference_llm::providers::overrides::{ - CatalogEntry, CatalogRequest, ChatModelEntry, ChatModelRequest, EmbeddingRequest, - ProviderModelMetadata, ProviderOverrides, ReadinessRequest, + CatalogEntry, CatalogLimitsFuture, CatalogRequest, ChatModelEntry, ChatModelRequest, + EmbeddingFactoryFuture, EmbeddingRequest, ProviderModelMetadata, ProviderOverrides, + ReadinessRequest, }; struct HostContext<'a> { @@ -40,7 +41,7 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { }, |id| { (id == "hosted").then(|| { - Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| { + Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { Box::pin(async move { assert_eq!(request.context.borrowed_config, "borrowed-host-config"); Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) @@ -122,7 +123,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }, |id| { (id == "shared").then(|| { - Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| { + Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { Box::pin(async move { assert_eq!(request.context.borrowed_config, "scoped-config"); Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) @@ -139,17 +140,20 @@ async fn independent_live_family_lookups_override_only_their_builtin() { Ok(serde_json::json!({"models": []})) }) }), - limits: Some(Arc::new(|model, context: &HostContext<'static>| { - Box::pin(async move { - (model == "catalog-model" && context.borrowed_config == "catalog-config") - .then(|| { - tinyinference_llm::model::discover::DiscoveryRequest::new( - "https://provider.invalid/v1", - model, - ) - }) - }) - })), + limits: Some(Arc::new( + |model, context: &HostContext<'static>| -> CatalogLimitsFuture<'_> { + Box::pin(async move { + (model == "catalog-model" + && context.borrowed_config == "catalog-config") + .then(|| { + tinyinference_llm::model::discover::DiscoveryRequest::new( + "https://provider.invalid/v1", + model, + ) + }) + }) + }, + )), }) }, ); From 32ca8b56d9c9fcbcd2fa46771bb67214e7c91372 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:33:17 +0000 Subject: [PATCH 18/25] refactor(providers): pass request structs by value to factories Embedding and catalog limits factories now take owned request structs instead of separate borrowed arguments, and a new CatalogLimitsRequest groups the model and context. This makes the callback signatures uniform and easier to extend without breaking callers. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/overrides.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/overrides.rs b/crates/tinyinference-llm/src/providers/overrides.rs index f183112f..211fed42 100644 --- a/crates/tinyinference-llm/src/providers/overrides.rs +++ b/crates/tinyinference-llm/src/providers/overrides.rs @@ -203,7 +203,7 @@ pub type EmbeddingFactoryFuture<'a> = /// Builds a provider embedding model. pub type EmbeddingFactory = - Arc Fn(&'a EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a> + Send + Sync>; + Arc Fn(EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a> + Send + Sync>; /// Owned host context passed to a catalog listing implementation. pub struct CatalogRequest { @@ -228,9 +228,17 @@ pub type CatalogListing = pub type CatalogLimitsFuture<'a> = Pin> + Send + 'a>>; +/// Input to an async provider catalog limits resolver. +pub struct CatalogLimitsRequest<'a, Ctx> { + /// Model whose supported limits are being discovered. + pub model: &'a str, + /// Borrowed host context such as configuration and credential sources. + pub context: &'a Ctx, +} + /// Builds the provider limit discovery request for a model, if supported. pub type CatalogLimits = - Arc Fn(&'a str, &'a Ctx) -> CatalogLimitsFuture<'a> + Send + Sync>; + Arc Fn(CatalogLimitsRequest<'a, Ctx>) -> CatalogLimitsFuture<'a> + Send + Sync>; /// A provider catalog listing and optional model-limit resolver. pub struct CatalogEntry { @@ -345,13 +353,13 @@ impl ProviderOverrides { /// # Errors /// Returns the selected factory's construction error. pub async fn build_embedding<'a, B>( - &'a self, + &self, provider_id: &str, - request: &'a EmbeddingRequest<'a, Ctx>, + request: EmbeddingRequest<'a, Ctx>, builtin: B, ) -> EmbeddingResult> where - B: FnOnce(&'a EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a>, + B: FnOnce(EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a>, { match self.embedding_factory(provider_id) { Some(factory) => factory(request).await, From ce35612bbbe61be16308cee733cff132d99a81bb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:33:40 +0000 Subject: [PATCH 19/25] test(provider-overrides): update tests for request struct overrides Adapt the provider override tests to the new request-struct signatures for embedding factories and catalog limits, replacing the previous borrowed argument style. This keeps the tests aligned with the updated override API. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tests/provider_overrides.rs | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index da005695..7c8e578d 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -7,9 +7,9 @@ use tinyinference_embeddings::{EmbeddingModel, MockEmbeddingModel}; use tinyinference_llm::model::ChatModel; use tinyinference_llm::providers::MockModel; use tinyinference_llm::providers::overrides::{ - CatalogEntry, CatalogLimitsFuture, CatalogRequest, ChatModelEntry, ChatModelRequest, - EmbeddingFactoryFuture, EmbeddingRequest, ProviderModelMetadata, ProviderOverrides, - ReadinessRequest, + CatalogEntry, CatalogLimitsFuture, CatalogLimitsRequest, CatalogRequest, ChatModelEntry, + ChatModelRequest, EmbeddingFactoryFuture, EmbeddingRequest, ProviderModelMetadata, + ProviderOverrides, ReadinessRequest, }; struct HostContext<'a> { @@ -41,7 +41,7 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { }, |id| { (id == "hosted").then(|| { - Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { + Arc::new(|request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { Box::pin(async move { assert_eq!(request.context.borrowed_config, "borrowed-host-config"); Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) @@ -70,7 +70,7 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { let embedding = overrides .build_embedding( "hosted", - &EmbeddingRequest { + EmbeddingRequest { model: "embed", dimensions: 3, context: &context, @@ -123,7 +123,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }, |id| { (id == "shared").then(|| { - Arc::new(|request: &EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { + Arc::new(|request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { Box::pin(async move { assert_eq!(request.context.borrowed_config, "scoped-config"); Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) @@ -141,14 +141,14 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }) }), limits: Some(Arc::new( - |model, context: &HostContext<'static>| -> CatalogLimitsFuture<'_> { + |request: CatalogLimitsRequest<'_, HostContext<'static>>| -> CatalogLimitsFuture<'_> { Box::pin(async move { - (model == "catalog-model" - && context.borrowed_config == "catalog-config") + (request.model == "catalog-model" + && request.context.borrowed_config == "catalog-config") .then(|| { tinyinference_llm::model::discover::DiscoveryRequest::new( "https://provider.invalid/v1", - model, + request.model, ) }) }) @@ -190,7 +190,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { let embedding = overrides .build_embedding( "shared", - &EmbeddingRequest { + EmbeddingRequest { model: "embed-model", dimensions: 8, context: &scoped_context, @@ -207,7 +207,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { let fallback = overrides .build_embedding( "unregistered", - &EmbeddingRequest { + EmbeddingRequest { model: "embed-model", dimensions: 4, context: &fallback_context, @@ -260,7 +260,14 @@ async fn independent_live_family_lookups_override_only_their_builtin() { _ownership_marker: std::marker::PhantomData, }; let limits = overrides.catalog_entry("shared").unwrap().limits.unwrap(); - assert!(limits("catalog-model", &catalog_context).await.is_some()); + assert!( + limits(CatalogLimitsRequest { + model: "catalog-model", + context: &catalog_context, + }) + .await + .is_some() + ); let listing = overrides .list_catalog( "shared", From 3a106e5b2485b925319abb07d1aae1f3332cedc5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:33:55 +0000 Subject: [PATCH 20/25] test(provider_overrides): reformat test file with rustfmt Reformatted the provider overrides tests to satisfy rustfmt, wrapping closures and fixing indentation of request literals. No behaviour changes. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tests/provider_overrides.rs | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index 7c8e578d..45f45572 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -41,13 +41,15 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { }, |id| { (id == "hosted").then(|| { - Arc::new(|request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { - Box::pin(async move { - assert_eq!(request.context.borrowed_config, "borrowed-host-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) - }) - }) as _ + Arc::new( + |request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "borrowed-host-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) + }, + ) as _ }) }, |_| None, @@ -123,13 +125,15 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }, |id| { (id == "shared").then(|| { - Arc::new(|request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { - Box::pin(async move { - assert_eq!(request.context.borrowed_config, "scoped-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) - }) - }) as _ + Arc::new( + |request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "scoped-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) + }, + ) as _ }) }, |id| { @@ -190,7 +194,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { let embedding = overrides .build_embedding( "shared", - EmbeddingRequest { + EmbeddingRequest { model: "embed-model", dimensions: 8, context: &scoped_context, @@ -207,7 +211,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { let fallback = overrides .build_embedding( "unregistered", - EmbeddingRequest { + EmbeddingRequest { model: "embed-model", dimensions: 4, context: &fallback_context, From e73e79c080394129ae06018ef35f5bd965bcc14e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:34:32 +0000 Subject: [PATCH 21/25] test(provider_overrides): simplify embedding factory closures Replace the explicit closure type annotations and `as _` casts with inferred closures cast to the `EmbeddingFactory` alias, and drop the now unused future type imports. This keeps the tests compiling against the current overrides API while making the factory setup less verbose. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tests/provider_overrides.rs | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index 45f45572..aa58444a 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -7,9 +7,8 @@ use tinyinference_embeddings::{EmbeddingModel, MockEmbeddingModel}; use tinyinference_llm::model::ChatModel; use tinyinference_llm::providers::MockModel; use tinyinference_llm::providers::overrides::{ - CatalogEntry, CatalogLimitsFuture, CatalogLimitsRequest, CatalogRequest, ChatModelEntry, - ChatModelRequest, EmbeddingFactoryFuture, EmbeddingRequest, ProviderModelMetadata, - ProviderOverrides, ReadinessRequest, + CatalogEntry, CatalogLimitsRequest, CatalogRequest, ChatModelEntry, ChatModelRequest, + EmbeddingRequest, ProviderModelMetadata, ProviderOverrides, ReadinessRequest, }; struct HostContext<'a> { @@ -41,15 +40,14 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { }, |id| { (id == "hosted").then(|| { - Arc::new( - |request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { - Box::pin(async move { - assert_eq!(request.context.borrowed_config, "borrowed-host-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) - }) - }, - ) as _ + Arc::new(|request| { + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "borrowed-host-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) + }) + as tinyinference_llm::providers::overrides::EmbeddingFactory> }) }, |_| None, @@ -125,15 +123,14 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }, |id| { (id == "shared").then(|| { - Arc::new( - |request: EmbeddingRequest<'_, HostContext<'_>>| -> EmbeddingFactoryFuture<'_> { - Box::pin(async move { - assert_eq!(request.context.borrowed_config, "scoped-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) - }) - }, - ) as _ + Arc::new(|request| { + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "scoped-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) + }) + as tinyinference_llm::providers::overrides::EmbeddingFactory> }) }, |id| { @@ -145,7 +142,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }) }), limits: Some(Arc::new( - |request: CatalogLimitsRequest<'_, HostContext<'static>>| -> CatalogLimitsFuture<'_> { + |request: CatalogLimitsRequest<'_, HostContext<'static>>| { Box::pin(async move { (request.model == "catalog-model" && request.context.borrowed_config == "catalog-config") From 825aa53efb661e776933940c9b7a8d55f53964b7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:35:40 +0000 Subject: [PATCH 22/25] refactor(providers): own embedding and catalog request data EmbeddingRequest and CatalogLimitsRequest now hold owned model strings and context instead of borrows, and their factory futures use BoxFuture rather than lifetime-parameterised boxed futures. This removes the higher-ranked lifetime bounds from the factory types so callers can build requests without tying them to a borrowed scope. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/overrides.rs | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/overrides.rs b/crates/tinyinference-llm/src/providers/overrides.rs index 211fed42..4a732739 100644 --- a/crates/tinyinference-llm/src/providers/overrides.rs +++ b/crates/tinyinference-llm/src/providers/overrides.rs @@ -177,16 +177,16 @@ impl Clone for ProviderModelMetadata { } /// Inputs a host embedding factory uses to construct an embedding model. -pub struct EmbeddingRequest<'a, Ctx> { +pub struct EmbeddingRequest { /// Embedding model id selected by the host. - pub model: &'a str, + pub model: String, /// Vector dimensionality selected by the host. pub dimensions: usize, - /// Borrowed host context such as configuration and credential scope. - pub context: &'a Ctx, + /// Owned host context such as configuration and credential scope. + pub context: Ctx, } -impl std::fmt::Debug for EmbeddingRequest<'_, Ctx> { +impl std::fmt::Debug for EmbeddingRequest { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("EmbeddingRequest") @@ -197,13 +197,12 @@ impl std::fmt::Debug for EmbeddingRequest<'_, Ctx> { } } -/// A provider embedding factory's async result, borrowing its request context. -pub type EmbeddingFactoryFuture<'a> = - Pin>> + Send + 'a>>; +/// A provider embedding factory's async result. +pub type EmbeddingFactoryFuture = BoxFuture>>; /// Builds a provider embedding model. pub type EmbeddingFactory = - Arc Fn(EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a> + Send + Sync>; + Arc) -> EmbeddingFactoryFuture + Send + Sync>; /// Owned host context passed to a catalog listing implementation. pub struct CatalogRequest { @@ -225,20 +224,19 @@ pub type CatalogListing = Arc) -> BoxFuture> + Send + Sync>; /// Async result of a provider limit-discovery request. -pub type CatalogLimitsFuture<'a> = - Pin> + Send + 'a>>; +pub type CatalogLimitsFuture = BoxFuture>; /// Input to an async provider catalog limits resolver. -pub struct CatalogLimitsRequest<'a, Ctx> { +pub struct CatalogLimitsRequest { /// Model whose supported limits are being discovered. - pub model: &'a str, - /// Borrowed host context such as configuration and credential sources. - pub context: &'a Ctx, + pub model: String, + /// Owned host context such as configuration and credential sources. + pub context: Ctx, } /// Builds the provider limit discovery request for a model, if supported. pub type CatalogLimits = - Arc Fn(CatalogLimitsRequest<'a, Ctx>) -> CatalogLimitsFuture<'a> + Send + Sync>; + Arc) -> CatalogLimitsFuture + Send + Sync>; /// A provider catalog listing and optional model-limit resolver. pub struct CatalogEntry { @@ -352,14 +350,14 @@ impl ProviderOverrides { /// /// # Errors /// Returns the selected factory's construction error. - pub async fn build_embedding<'a, B>( + pub async fn build_embedding( &self, provider_id: &str, - request: EmbeddingRequest<'a, Ctx>, + request: EmbeddingRequest, builtin: B, ) -> EmbeddingResult> where - B: FnOnce(EmbeddingRequest<'a, Ctx>) -> EmbeddingFactoryFuture<'a>, + B: FnOnce(EmbeddingRequest) -> EmbeddingFactoryFuture, { match self.embedding_factory(provider_id) { Some(factory) => factory(request).await, From 8b70332513b56c0a0c9b6a87514ecf79aa5149b8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:36:08 +0000 Subject: [PATCH 23/25] test(providers): cover owned non-clone host contexts in overrides Rework the provider override tests to pass owned, non-clone host contexts through the embedding and catalog limits factories, and derive Debug on CatalogLimitsRequest so the request type can be used in these paths. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/providers/overrides.rs | 1 + .../tests/provider_overrides.rs | 35 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/overrides.rs b/crates/tinyinference-llm/src/providers/overrides.rs index 4a732739..59f4e9a2 100644 --- a/crates/tinyinference-llm/src/providers/overrides.rs +++ b/crates/tinyinference-llm/src/providers/overrides.rs @@ -227,6 +227,7 @@ pub type CatalogListing = pub type CatalogLimitsFuture = BoxFuture>; /// Input to an async provider catalog limits resolver. +#[derive(Debug)] pub struct CatalogLimitsRequest { /// Model whose supported limits are being discovered. pub model: String, diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index aa58444a..e80a0c5f 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -18,10 +18,9 @@ struct HostContext<'a> { } #[tokio::test] -async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { - let config = String::from("borrowed-host-config"); +async fn asynchronous_factories_accept_owned_non_clone_host_contexts() { let context = HostContext { - borrowed_config: &config, + borrowed_config: "borrowed-host-config", _ownership_marker: std::marker::PhantomData, }; let overrides = ProviderOverrides::new( @@ -40,14 +39,16 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { }, |id| { (id == "hosted").then(|| { - Arc::new(|request| { + Arc::new(|request: EmbeddingRequest>| { Box::pin(async move { assert_eq!(request.context.borrowed_config, "borrowed-host-config"); Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) as Box) }) }) - as tinyinference_llm::providers::overrides::EmbeddingFactory> + as tinyinference_llm::providers::overrides::EmbeddingFactory< + HostContext<'static>, + > }) }, |_| None, @@ -71,9 +72,9 @@ async fn asynchronous_factories_accept_borrowed_non_clone_host_contexts() { .build_embedding( "hosted", EmbeddingRequest { - model: "embed", + model: "embed".into(), dimensions: 3, - context: &context, + context, }, |_| panic!("registered embedding factory should be used"), ) @@ -123,14 +124,16 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }, |id| { (id == "shared").then(|| { - Arc::new(|request| { + Arc::new(|request: EmbeddingRequest>| { Box::pin(async move { assert_eq!(request.context.borrowed_config, "scoped-config"); Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) as Box) }) }) - as tinyinference_llm::providers::overrides::EmbeddingFactory> + as tinyinference_llm::providers::overrides::EmbeddingFactory< + HostContext<'static>, + > }) }, |id| { @@ -142,7 +145,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }) }), limits: Some(Arc::new( - |request: CatalogLimitsRequest<'_, HostContext<'static>>| { + |request: CatalogLimitsRequest>| { Box::pin(async move { (request.model == "catalog-model" && request.context.borrowed_config == "catalog-config") @@ -192,9 +195,9 @@ async fn independent_live_family_lookups_override_only_their_builtin() { .build_embedding( "shared", EmbeddingRequest { - model: "embed-model", + model: "embed-model".into(), dimensions: 8, - context: &scoped_context, + context: scoped_context, }, |_| panic!("embedding override must run before builtin"), ) @@ -209,9 +212,9 @@ async fn independent_live_family_lookups_override_only_their_builtin() { .build_embedding( "unregistered", EmbeddingRequest { - model: "embed-model", + model: "embed-model".into(), dimensions: 4, - context: &fallback_context, + context: fallback_context, }, |_| { Box::pin(async { @@ -263,8 +266,8 @@ async fn independent_live_family_lookups_override_only_their_builtin() { let limits = overrides.catalog_entry("shared").unwrap().limits.unwrap(); assert!( limits(CatalogLimitsRequest { - model: "catalog-model", - context: &catalog_context, + model: "catalog-model".into(), + context: catalog_context, }) .await .is_some() From 3bb0f2dec84ecf61e7911e64e6deba78ad1160ca Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:36:20 +0000 Subject: [PATCH 24/25] test(provider_overrides): annotate async factory closures with future types The embedding and catalog limits closures in the provider override tests now declare their return types explicitly using the EmbeddingFactoryFuture and CatalogLimitsFuture aliases, which keeps the boxed futures type-checking against the expected factory signatures. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tests/provider_overrides.rs | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index e80a0c5f..7417586b 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -7,8 +7,9 @@ use tinyinference_embeddings::{EmbeddingModel, MockEmbeddingModel}; use tinyinference_llm::model::ChatModel; use tinyinference_llm::providers::MockModel; use tinyinference_llm::providers::overrides::{ - CatalogEntry, CatalogLimitsRequest, CatalogRequest, ChatModelEntry, ChatModelRequest, - EmbeddingRequest, ProviderModelMetadata, ProviderOverrides, ReadinessRequest, + CatalogEntry, CatalogLimitsFuture, CatalogLimitsRequest, CatalogRequest, ChatModelEntry, + ChatModelRequest, EmbeddingFactoryFuture, EmbeddingRequest, ProviderModelMetadata, + ProviderOverrides, ReadinessRequest, }; struct HostContext<'a> { @@ -39,13 +40,15 @@ async fn asynchronous_factories_accept_owned_non_clone_host_contexts() { }, |id| { (id == "hosted").then(|| { - Arc::new(|request: EmbeddingRequest>| { - Box::pin(async move { - assert_eq!(request.context.borrowed_config, "borrowed-host-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) - }) - }) + Arc::new( + |request: EmbeddingRequest>| -> EmbeddingFactoryFuture { + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "borrowed-host-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) + }, + ) as tinyinference_llm::providers::overrides::EmbeddingFactory< HostContext<'static>, > @@ -124,13 +127,15 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }, |id| { (id == "shared").then(|| { - Arc::new(|request: EmbeddingRequest>| { - Box::pin(async move { - assert_eq!(request.context.borrowed_config, "scoped-config"); - Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) - as Box) - }) - }) + Arc::new( + |request: EmbeddingRequest>| -> EmbeddingFactoryFuture { + Box::pin(async move { + assert_eq!(request.context.borrowed_config, "scoped-config"); + Ok(Box::new(MockEmbeddingModel::new(request.dimensions)) + as Box) + }) + }, + ) as tinyinference_llm::providers::overrides::EmbeddingFactory< HostContext<'static>, > @@ -145,7 +150,7 @@ async fn independent_live_family_lookups_override_only_their_builtin() { }) }), limits: Some(Arc::new( - |request: CatalogLimitsRequest>| { + |request: CatalogLimitsRequest>| -> CatalogLimitsFuture { Box::pin(async move { (request.model == "catalog-model" && request.context.borrowed_config == "catalog-config") From 25f23ec54d46b06d8ccdd21aff4e0914b3e139e7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 07:36:36 +0000 Subject: [PATCH 25/25] test(provider-overrides): use distinct host contexts in override tests The override tests now build separate HostContext values for the fallback and catalog lookups instead of reusing a shared reference, so each call carries its own borrowed config and the assertions can tell the two paths apart. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-llm/tests/provider_overrides.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tinyinference-llm/tests/provider_overrides.rs b/crates/tinyinference-llm/tests/provider_overrides.rs index 7417586b..ac160a9c 100644 --- a/crates/tinyinference-llm/tests/provider_overrides.rs +++ b/crates/tinyinference-llm/tests/provider_overrides.rs @@ -239,7 +239,10 @@ async fn independent_live_family_lookups_override_only_their_builtin() { default_model: Some("builtin-model"), native_tool_calling: false, thread_id: None, - context: &fallback_context, + context: &HostContext { + borrowed_config: "fallback-config", + _ownership_marker: std::marker::PhantomData, + }, }, |_| { Ok(( @@ -281,7 +284,10 @@ async fn independent_live_family_lookups_override_only_their_builtin() { .list_catalog( "shared", CatalogRequest { - context: catalog_context, + context: HostContext { + borrowed_config: "catalog-config", + _ownership_marker: std::marker::PhantomData, + }, }, |_| panic!("catalog override must run before builtin"), )