From 51cae32f360fddbf5d88c19e4f0dd7cf834b6912 Mon Sep 17 00:00:00 2001 From: colafornia Date: Fri, 13 Nov 2026 17:30:44 +0800 Subject: [PATCH 1/4] feat(eval): add experiment contracts and usage pricing semantics - contract: add strict maka.eval.v1 spec/attempt validation, closed maka.eval.v2 enums (AttemptMetering, AttemptCancellation, EgressAuditState), normalized RunFingerprint, DifferenceRegistry, and v2->v1 projections - measurement: add exact 6-counter NormalizedUsage without summing totalTokens, MeteringState/TokenBasis, and standardized DEEPSEEK_V4_FLASH_COST pricing Generated-by: Codex Generated-by: Google Antigravity --- Cargo.lock | 11 + Cargo.toml | 2 + products/eval/Cargo.toml | 35 + products/eval/README.md | 31 + products/eval/README.zh-CN.md | 31 + products/eval/src/contract.rs | 1403 +++++++++++++++++++ products/eval/src/lib.rs | 23 + products/eval/src/measurement.rs | 189 +++ products/eval/tests/integration/contract.rs | 420 ++++++ products/eval/tests/integration/main.rs | 21 + products/eval/tests/integration/support.rs | 101 ++ 11 files changed, 2267 insertions(+) create mode 100644 products/eval/Cargo.toml create mode 100644 products/eval/README.md create mode 100644 products/eval/README.zh-CN.md create mode 100644 products/eval/src/contract.rs create mode 100644 products/eval/src/lib.rs create mode 100644 products/eval/src/measurement.rs create mode 100644 products/eval/tests/integration/contract.rs create mode 100644 products/eval/tests/integration/main.rs create mode 100644 products/eval/tests/integration/support.rs diff --git a/Cargo.lock b/Cargo.lock index 49169f0789..ceb42dc0ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6873,6 +6873,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "maka-eval" +version = "0.2.0" +dependencies = [ + "hex", + "serde", + "serde_json", + "sha2 0.11.1", + "thiserror 2.0.21", +] + [[package]] name = "maka-event-log" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 42d150ce5a..8023006b8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ members = [ "plugins/workspace/workspace-files/native", "plugins/workspace/support", "products/maka/acp", + "products/eval", "plugins/agent/engine", "plugins/agent/native", "plugins/workspace/workspace-files/patch", @@ -149,6 +150,7 @@ imagesize = { version = "=0.15.0", default-features = false, features = ["png", jsonschema = { version = "0.60.1", default-features = false } libc = "0.2.191" maka-acp = { path = "products/maka/acp" } +maka-eval = { path = "products/eval" } maka-agent-engine = { path = "plugins/agent/engine" } maka-agent-native = { path = "plugins/agent/native" } maka-assistant = { path = "plugins/agent/assistant" } diff --git a/products/eval/Cargo.toml b/products/eval/Cargo.toml new file mode 100644 index 0000000000..724bed5572 --- /dev/null +++ b/products/eval/Cargo.toml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "maka-eval" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +autotests = false + +[[test]] +name = "integration" +path = "tests/integration/main.rs" + +[dependencies] +hex.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true diff --git a/products/eval/README.md b/products/eval/README.md new file mode 100644 index 0000000000..b3a1fb91f7 --- /dev/null +++ b/products/eval/README.md @@ -0,0 +1,31 @@ + + +# maka-eval + +[简体中文](README.zh-CN.md) + +Measurement and control plane contracts for multi-arm benchmark experiments (`maka.eval.v1` and `maka.eval.v2`). + +## Architecture + +- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions, condition-only `RunFingerprint`, and the semantic difference registry. +- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), `MeteringState` / `TokenBasis` semantics, and standardized `DEEPSEEK_V4_FLASH_COST` pricing. + +[Source](src/lib.rs) · [Cargo manifest](Cargo.toml) diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md new file mode 100644 index 0000000000..3f348dcd2d --- /dev/null +++ b/products/eval/README.zh-CN.md @@ -0,0 +1,31 @@ + + +# maka-eval + +[English](README.md) + +多臂基准评测实验的测量与控制平面契约底座（支持 `maka.eval.v1` 与 `maka.eval.v2`）。 + +## 模块结构 + +- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度、仅记录运行条件的 `RunFingerprint` 及语义差异注册表。 +- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）、`MeteringState` / `TokenBasis` 语义及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 + +[源码](src/lib.rs) · [Cargo 配置](Cargo.toml) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs new file mode 100644 index 0000000000..7493b785e1 --- /dev/null +++ b/products/eval/src/contract.rs @@ -0,0 +1,1403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Experiment specifications, orthogonal `v2` attempt records, strict `v1` +//! projections, run fingerprinting, and the semantic difference registry. + +use crate::measurement::{MeteringState, NormalizedUsage}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use thiserror::Error; + +pub type JsonObject = Map; + +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ContractError { + #[error("{0}")] + Invalid(String), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum SchemaVersion { + #[serde(rename = "maka.eval.v1")] + V1, + #[serde(rename = "maka.eval.v2")] + V2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SubjectKind { + Maka, + External, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BenchmarkSpec { + pub id: String, + pub version: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutorSpec { + pub kind: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutionSpec { + pub max_concurrent_task_groups: usize, +} + +impl Default for ExecutionSpec { + fn default() -> Self { + Self { + max_concurrent_task_groups: 1, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectSpec { + pub id: String, + pub kind: SubjectKind, + pub credentials: Vec, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TaskSpec { + pub id: String, + pub input: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExperimentSpec { + pub schema_version: SchemaVersion, + pub id: String, + pub benchmark: BenchmarkSpec, + pub executor: ExecutorSpec, + #[serde(default)] + pub execution: ExecutionSpec, + pub subjects: Vec, + pub tasks: Vec, + pub repetitions: u32, + pub budget: JsonObject, + pub verifier: JsonObject, +} + +impl<'de> Deserialize<'de> for ExperimentSpec { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + parse_experiment_spec(&value).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExperimentCell { + pub id: String, + pub experiment_id: String, + pub benchmark: BenchmarkSpec, + pub executor: ExecutorSpec, + pub subject: SubjectSpec, + pub task: TaskSpec, + pub repetition: u32, + pub budget: JsonObject, + pub verifier: JsonObject, +} + +pub fn format_cell_id(task_id: &str, repetition: u32, subject_id: &str) -> String { + format!("{task_id}::{repetition}::{subject_id}") +} + +pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { + let mut cells = + Vec::with_capacity(spec.tasks.len() * (spec.repetitions as usize) * spec.subjects.len()); + for task in &spec.tasks { + for repetition in 1..=spec.repetitions { + for subject in &spec.subjects { + cells.push(ExperimentCell { + id: format_cell_id(&task.id, repetition, &subject.id), + experiment_id: spec.id.clone(), + benchmark: spec.benchmark.clone(), + executor: spec.executor.clone(), + subject: subject.clone(), + task: task.clone(), + repetition, + budget: spec.budget.clone(), + verifier: spec.verifier.clone(), + }); + } + } + } + cells +} + +pub fn parse_experiment_spec(value: &Value) -> Result { + let root = exact_object( + value, + "experiment", + &[ + "schemaVersion", + "id", + "benchmark", + "executor", + "subjects", + "tasks", + "repetitions", + "budget", + "verifier", + ], + &["execution"], + )?; + + let schema_version = match root.get("schemaVersion").and_then(Value::as_str) { + Some("maka.eval.v1") => SchemaVersion::V1, + Some("maka.eval.v2") => SchemaVersion::V2, + _ => { + return Err(ContractError::Invalid( + "unsupported experiment schema".into(), + )); + } + }; + + let benchmark_obj = exact_object( + root.get("benchmark").unwrap(), + "benchmark", + &["id", "version", "config"], + &[], + )?; + let executor_obj = exact_object( + root.get("executor").unwrap(), + "executor", + &["kind", "config"], + &[], + )?; + + let max_concurrent_task_groups = match root.get("execution") { + None => 1, + Some(exec_val) => { + let exec_obj = exact_object(exec_val, "execution", &["maxConcurrentTaskGroups"], &[])?; + positive_usize( + exec_obj.get("maxConcurrentTaskGroups").unwrap(), + "execution.maxConcurrentTaskGroups", + )? + } + }; + + let subjects_arr = root + .get("subjects") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("subjects must be an array".into()))?; + let mut subjects = Vec::with_capacity(subjects_arr.len()); + for (idx, item) in subjects_arr.iter().enumerate() { + let where_label = format!("subjects[{idx}]"); + let sub_obj = exact_object( + item, + &where_label, + &["id", "kind", "credentials", "config"], + &[], + )?; + let kind = match sub_obj.get("kind").and_then(Value::as_str) { + Some("maka") => SubjectKind::Maka, + Some("external") => SubjectKind::External, + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.kind is invalid" + ))); + } + }; + subjects.push(SubjectSpec { + id: identifier(sub_obj.get("id").unwrap(), &format!("{where_label}.id"))?, + kind, + credentials: unique_strings( + sub_obj.get("credentials").unwrap(), + &format!("{where_label}.credentials"), + )?, + config: decode_json_object( + sub_obj.get("config").unwrap(), + &format!("{where_label}.config"), + )?, + }); + } + + let tasks_arr = root + .get("tasks") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("tasks must be an array".into()))?; + let mut tasks = Vec::with_capacity(tasks_arr.len()); + for (idx, item) in tasks_arr.iter().enumerate() { + let where_label = format!("tasks[{idx}]"); + let task_obj = exact_object(item, &where_label, &["id", "input", "config"], &[])?; + tasks.push(TaskSpec { + id: identifier(task_obj.get("id").unwrap(), &format!("{where_label}.id"))?, + input: nonempty_string( + task_obj.get("input").unwrap(), + &format!("{where_label}.input"), + )?, + config: decode_json_object( + task_obj.get("config").unwrap(), + &format!("{where_label}.config"), + )?, + }); + } + + if subjects.is_empty() || tasks.is_empty() { + return Err(ContractError::Invalid("experiment arms are empty".into())); + } + ensure_unique_ids(subjects.iter().map(|s| s.id.as_str()), "subject")?; + ensure_unique_ids(tasks.iter().map(|t| t.id.as_str()), "task")?; + + let repetitions = positive_u32(root.get("repetitions").unwrap(), "repetitions")?; + + Ok(ExperimentSpec { + schema_version, + id: identifier(root.get("id").unwrap(), "experiment.id")?, + benchmark: BenchmarkSpec { + id: identifier(benchmark_obj.get("id").unwrap(), "benchmark.id")?, + version: nonempty_string(benchmark_obj.get("version").unwrap(), "benchmark.version")?, + config: decode_json_object(benchmark_obj.get("config").unwrap(), "benchmark.config")?, + }, + executor: ExecutorSpec { + kind: identifier(executor_obj.get("kind").unwrap(), "executor.kind")?, + config: decode_json_object(executor_obj.get("config").unwrap(), "executor.config")?, + }, + execution: ExecutionSpec { + max_concurrent_task_groups, + }, + subjects, + tasks, + repetitions, + budget: decode_json_object(root.get("budget").unwrap(), "budget")?, + verifier: decode_json_object(root.get("verifier").unwrap(), "verifier")?, + }) +} + +/// Canonical JSON serialization with lexicographically sorted object keys. +pub fn canonical_json_string(value: &Value) -> String { + let normalized = canonicalize_value(value); + serde_json::to_string(&normalized).expect("canonical Value serialization cannot fail") +} + +fn canonicalize_value(value: &Value) -> Value { + match value { + Value::Object(map) => { + let mut sorted: Vec<(&String, &Value)> = map.iter().collect(); + sorted.sort_by_key(|(a, _)| *a); + let mut out = Map::with_capacity(sorted.len()); + for (k, v) in sorted { + out.insert(k.clone(), canonicalize_value(v)); + } + Value::Object(out) + } + Value::Array(items) => Value::Array(items.iter().map(canonicalize_value).collect()), + Value::Number(n) => { + if n.is_u64() || n.is_i64() { + Value::Number(n.clone()) + } else if let Some(f) = n.as_f64() { + if f == 0.0 { + Value::Number(0.into()) + } else if f.is_finite() + && f.fract() == 0.0 + && (-9_007_199_254_740_991.0..=9_007_199_254_740_991.0).contains(&f) + { + if f >= 0.0 { + Value::Number((f as u64).into()) + } else { + Value::Number((f as i64).into()) + } + } else { + Value::Number(n.clone()) + } + } else { + Value::Number(n.clone()) + } + } + other => other.clone(), + } +} + +/// Documents experimental and execution conditions for comparability checks. +/// +/// Fingerprints record the declared configuration and parameters of a run; freezing an +/// eval-only fingerprint does not establish product parity. M3 acceptance and M5 production +/// delivery require a verified `ParityVerificationReport` alongside the run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RunFingerprint { + /// Digest over benchmark, executor kind, subjects, tasks, repetitions, budget, and verifier. + pub semantic_sha256: String, + /// Digest over execution-only parameters (e.g. `maxConcurrentTaskGroups`). + pub execution_sha256: String, +} + +impl RunFingerprint { + pub fn compute(spec: &ExperimentSpec) -> Self { + let semantic_payload = serde_json::json!({ + "benchmark": spec.benchmark, + "executorKind": spec.executor.kind, + "subjects": spec.subjects, + "tasks": spec.tasks, + "repetitions": spec.repetitions, + "budget": spec.budget, + "verifier": spec.verifier, + }); + let execution_payload = serde_json::json!({ + "maxConcurrentTaskGroups": spec.execution.max_concurrent_task_groups, + "executorConfig": spec.executor.config, + }); + Self { + semantic_sha256: sha256_hex(canonical_json_string(&semantic_payload).as_bytes()), + execution_sha256: sha256_hex(canonical_json_string(&execution_payload).as_bytes()), + } + } + + pub fn is_semantically_comparable(&self, other: &Self) -> bool { + self.semantic_sha256 == other.semantic_sha256 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvalResultStatus { + Completed, + SubjectFailed, + InfraFailed, + Indeterminate, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EvalResult { + pub score: Option, + pub usage: Option, + pub cost_usd: Option, + pub duration_ms: u64, + pub status: EvalResultStatus, + pub failure_reason: Option, + pub artifacts: Vec, +} + +impl<'de> Deserialize<'de> for EvalResult { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + decode_eval_result(&value, "result").map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct V1CellAttempt { + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub completed_at: u64, + pub result: EvalResult, +} + +impl<'de> Deserialize<'de> for V1CellAttempt { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + decode_v1_cell_attempt(&value).map_err(serde::de::Error::custom) + } +} + +pub fn is_replaceable_attempt(attempt: &V1CellAttempt) -> bool { + matches!( + attempt.result.status, + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate + ) +} + +/// Canonical cell result selector: selects the earliest valid attempt by sequence (`sequence >= 1`) +/// whose status is neither `infra_failed` nor `indeterminate`. A `completed` attempt with score `0.0` +/// or a `subject_failed` attempt is terminal and never replaced. +pub fn select_cell_result(attempts: &[V1CellAttempt]) -> Option<&V1CellAttempt> { + let mut sorted: Vec<&V1CellAttempt> = attempts.iter().filter(|a| a.sequence >= 1).collect(); + sorted.sort_by_key(|a| a.sequence); + sorted + .into_iter() + .find(|attempt| !is_replaceable_attempt(attempt)) +} + +pub fn decode_eval_result(value: &Value, where_label: &str) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "score", + "usage", + "costUsd", + "durationMs", + "status", + "failureReason", + "artifacts", + ], + &[], + )?; + let status = match obj.get("status").and_then(Value::as_str) { + Some("completed") => EvalResultStatus::Completed, + Some("subject_failed") => EvalResultStatus::SubjectFailed, + Some("infra_failed") => EvalResultStatus::InfraFailed, + Some("indeterminate") => EvalResultStatus::Indeterminate, + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.status is invalid" + ))); + } + }; + let failure_reason = match obj.get("failureReason").unwrap() { + Value::Null => None, + Value::String(s) => Some(s.clone()), + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.failureReason is invalid" + ))); + } + }; + if status == EvalResultStatus::Completed && failure_reason.is_some() { + return Err(ContractError::Invalid(format!( + "{where_label}.failureReason must be null when completed" + ))); + } + let score = nullable_f64(obj.get("score").unwrap(), &format!("{where_label}.score"))?; + let usage = match obj.get("usage").unwrap() { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, &format!("{where_label}.usage"))?), + }; + let cost_usd = nullable_nonnegative_f64( + obj.get("costUsd").unwrap(), + &format!("{where_label}.costUsd"), + )?; + if cost_usd.is_some() && usage.is_none() { + return Err(ContractError::Invalid(format!( + "{where_label}.costUsd requires usage" + ))); + } + let duration_ms = nonnegative_u64( + obj.get("durationMs").unwrap(), + &format!("{where_label}.durationMs"), + )?; + let artifacts_arr = obj + .get("artifacts") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid(format!("{where_label}.artifacts is invalid")))?; + let mut artifacts = Vec::with_capacity(artifacts_arr.len()); + for (idx, item) in artifacts_arr.iter().enumerate() { + artifacts.push(decode_json_object( + item, + &format!("{where_label}.artifacts[{idx}]"), + )?); + } + Ok(EvalResult { + score, + usage, + cost_usd, + duration_ms, + status, + failure_reason, + artifacts, + }) +} + +pub fn decode_v1_cell_attempt(value: &Value) -> Result { + let obj = exact_object( + value, + "attempt", + &["cellId", "sequence", "startedAt", "completedAt", "result"], + &[], + )?; + Ok(V1CellAttempt { + cell_id: nonempty_string(obj.get("cellId").unwrap(), "attempt.cellId")?, + sequence: positive_u32(obj.get("sequence").unwrap(), "attempt.sequence")?, + started_at: nonnegative_u64(obj.get("startedAt").unwrap(), "attempt.startedAt")?, + completed_at: nonnegative_u64(obj.get("completedAt").unwrap(), "attempt.completedAt")?, + result: decode_eval_result(obj.get("result").unwrap(), "attempt.result")?, + }) +} + +/// Orthogonal dimensions for native `maka.eval.v2` attempt records. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptPhase { + Prepared, + Dispatched, + AgentFinished, + Verified, + Settled, + Recovered, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionOutcome { + Completed, + SubjectFailed, + Timeout, + InfraFailed, + Indeterminate, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptCancellation { + #[default] + None, + BeforeVerification, + AfterVerification, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum VerifierOutcome { + Verified { + score: f64, + }, + SubjectFailed { + score: Option, + reason: Option, + }, + /// Missing reward file or empty verifier payload: never defaults to `0.0`. + MissingReward { + reason: String, + }, + VerifierInfraFailed { + reason: String, + }, + Skipped, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CleanupState { + Confirmed, + Unconfirmed { reason: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "snake_case", + rename_all_fields = "camelCase" +)] +pub enum EgressAuditState { + NotRequired, + Recorded { + #[serde(default)] + truncated: bool, + #[serde(default)] + policy_error_count: u64, + }, + Missing, + Unreadable { + reason: String, + }, +} + +impl EgressAuditState { + pub fn clean() -> Self { + Self::Recorded { + truncated: false, + policy_error_count: 0, + } + } + + pub fn truncated() -> Self { + Self::Recorded { + truncated: true, + policy_error_count: 0, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CriticalArtifactsState { + Present, + Missing { paths: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "snake_case", + rename_all_fields = "camelCase" +)] +pub enum PricingEvidence { + Model { + model: String, + cost_usd: Option, + }, + SubjectReported { + cost_usd: f64, + }, +} + +impl PricingEvidence { + pub fn cost_usd(&self) -> Option { + match self { + Self::Model { cost_usd, .. } => *cost_usd, + Self::SubjectReported { cost_usd } => Some(*cost_usd), + } + } + + pub fn validate(&self) -> Result<(), ContractError> { + match self { + Self::Model { model, cost_usd } => { + if model.is_empty() { + return Err(ContractError::Invalid( + "pricing.model must not be empty".into(), + )); + } + if let Some(c) = cost_usd + && (!c.is_finite() || *c < 0.0) + { + return Err(ContractError::Invalid( + "pricing.costUsd must be finite and non-negative".into(), + )); + } + } + Self::SubjectReported { cost_usd } => { + if !cost_usd.is_finite() || *cost_usd < 0.0 { + return Err(ContractError::Invalid( + "pricing.costUsd must be finite and non-negative".into(), + )); + } + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "state", + rename_all = "snake_case", + rename_all_fields = "camelCase" +)] +pub enum AttemptMetering { + #[default] + Unknown, + LowerBound { + usage: NormalizedUsage, + }, + Complete { + usage: NormalizedUsage, + #[serde(default)] + pricing: Option, + }, +} + +impl AttemptMetering { + pub fn state(&self) -> MeteringState { + match self { + Self::Unknown => MeteringState::Unknown, + Self::LowerBound { .. } => MeteringState::LowerBound, + Self::Complete { .. } => MeteringState::Complete, + } + } + + pub fn usage(&self) -> Option<&NormalizedUsage> { + match self { + Self::Unknown => None, + Self::LowerBound { usage } | Self::Complete { usage, .. } => Some(usage), + } + } + + pub fn pricing(&self) -> Option<&PricingEvidence> { + match self { + Self::Complete { pricing, .. } => pricing.as_ref(), + Self::Unknown | Self::LowerBound { .. } => None, + } + } + + pub fn cost_usd(&self) -> Option { + self.pricing().and_then(PricingEvidence::cost_usd) + } + + pub fn validate(&self) -> Result<(), ContractError> { + if let Self::Complete { + pricing: Some(p), .. + } = self + { + p.validate()?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct V2AttemptRecord { + pub schema_version: SchemaVersion, + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub completed_at: u64, + pub duration_ms: u64, + pub phase: AttemptPhase, + pub execution_outcome: ExecutionOutcome, + pub verifier_outcome: VerifierOutcome, + pub metering: AttemptMetering, + pub cleanup_state: CleanupState, + pub audit_state: EgressAuditState, + pub critical_artifacts_state: CriticalArtifactsState, + #[serde(default)] + pub cancellation: AttemptCancellation, + pub recovered_post_hoc: bool, + pub failure_reason: Option, + pub artifacts: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct RawV2AttemptRecord { + schema_version: SchemaVersion, + cell_id: String, + sequence: u32, + started_at: u64, + completed_at: u64, + duration_ms: u64, + phase: AttemptPhase, + execution_outcome: ExecutionOutcome, + verifier_outcome: VerifierOutcome, + metering: AttemptMetering, + cleanup_state: CleanupState, + audit_state: EgressAuditState, + critical_artifacts_state: CriticalArtifactsState, + #[serde(default)] + cancellation: AttemptCancellation, + recovered_post_hoc: bool, + failure_reason: Option, + artifacts: Vec, +} + +impl<'de> Deserialize<'de> for V2AttemptRecord { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let raw = RawV2AttemptRecord::deserialize(deserializer)?; + let record = Self { + schema_version: raw.schema_version, + cell_id: raw.cell_id, + sequence: raw.sequence, + started_at: raw.started_at, + completed_at: raw.completed_at, + duration_ms: raw.duration_ms, + phase: raw.phase, + execution_outcome: raw.execution_outcome, + verifier_outcome: raw.verifier_outcome, + metering: raw.metering, + cleanup_state: raw.cleanup_state, + audit_state: raw.audit_state, + critical_artifacts_state: raw.critical_artifacts_state, + cancellation: raw.cancellation, + recovered_post_hoc: raw.recovered_post_hoc, + failure_reason: raw.failure_reason, + artifacts: raw.artifacts, + }; + record.validate().map_err(serde::de::Error::custom)?; + Ok(record) + } +} + +impl V2AttemptRecord { + pub fn validate(&self) -> Result<(), ContractError> { + if self.schema_version != SchemaVersion::V2 { + return Err(ContractError::Invalid( + "v2 attempt record requires schemaVersion maka.eval.v2".into(), + )); + } + if self.cell_id.is_empty() { + return Err(ContractError::Invalid( + "v2 attempt record cellId is required".into(), + )); + } + if self.sequence < 1 { + return Err(ContractError::Invalid( + "v2 attempt record sequence must be positive".into(), + )); + } + match &self.verifier_outcome { + VerifierOutcome::Verified { score } if !score.is_finite() => { + return Err(ContractError::Invalid( + "verifier score must be finite".into(), + )); + } + VerifierOutcome::SubjectFailed { + score: Some(score), .. + } if !score.is_finite() => { + return Err(ContractError::Invalid( + "verifier score must be finite".into(), + )); + } + _ => {} + } + self.metering.validate()?; + Ok(()) + } + + /// Projects a native `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt` + /// while enforcing all 5 Semantic Difference Registry adjudications. + pub fn project_v1(&self) -> V1CellAttempt { + let mut artifacts = self.artifacts.clone(); + + let usage = self.metering.usage().cloned(); + let cost_usd = self.metering.cost_usd(); + + if self.cancellation == AttemptCancellation::AfterVerification + && !artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification") + }) + { + artifacts.push( + serde_json::json!({"kind": "cancelled-after-verification"}) + .as_object() + .unwrap() + .clone(), + ); + } + + if self.recovered_post_hoc + && !artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("post-hoc-evidence-recovery") + }) + { + let mut marker = Map::new(); + marker.insert( + "kind".into(), + Value::String("post-hoc-evidence-recovery".into()), + ); + artifacts.push(marker); + } + + // Unconfirmed cleanup always resolves to `indeterminate` with `score: null`. + if let CleanupState::Unconfirmed { reason } = &self.cleanup_state { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage, + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::Indeterminate, + failure_reason: Some(reason.clone()), + artifacts, + }, + }; + } + + // Adjudication #4: Late cancellation BEFORE verification completes resolves to + // `indeterminate`, whereas late cancellation AFTER verification and settlement + // preserves the verified result and records the late cancellation signal. + if self.cancellation == AttemptCancellation::BeforeVerification { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage, + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::Indeterminate, + failure_reason: Some(self.failure_reason.clone().unwrap_or_else(|| { + "executor cancelled before verification completed".into() + })), + artifacts, + }, + }; + } + + // Adjudication #3: Missing or unreadable critical artifacts downgrade to `infra_failed`. + if let CriticalArtifactsState::Missing { paths } = &self.critical_artifacts_state { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage, + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some(format!( + "critical artifacts missing: {}", + paths.join(", ") + )), + artifacts, + }, + }; + } + + // Adjudication #2: Egress audit missing or unreadable fails closed (`infra_failed`), + // whereas `Recorded { truncated: true, .. }` (`ruleId == "audit_truncated"`) is preserved + // as a structured artifact flag without discarding a settled verification result. + match &self.audit_state { + EgressAuditState::Missing => { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage, + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some("egress audit log missing".into()), + artifacts, + }, + }; + } + EgressAuditState::Unreadable { reason } => { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage, + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some(reason.clone()), + artifacts, + }, + }; + } + EgressAuditState::Recorded { + truncated, + policy_error_count, + } => { + if *truncated || *policy_error_count > 0 { + if let Some(existing) = artifacts + .iter_mut() + .find(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) + { + if *truncated { + existing.insert("truncated".into(), Value::Bool(true)); + } + if *policy_error_count > 0 { + existing.insert( + "policyErrorCount".into(), + Value::from(*policy_error_count), + ); + } + } else { + let mut audit_obj = Map::new(); + audit_obj.insert("kind".into(), Value::String("egress-audit".into())); + audit_obj.insert("truncated".into(), Value::Bool(*truncated)); + if *policy_error_count > 0 { + audit_obj.insert( + "policyErrorCount".into(), + Value::from(*policy_error_count), + ); + } + artifacts.push(audit_obj); + } + } + } + EgressAuditState::NotRequired => {} + } + + let (status, score, failure_reason) = + match (&self.execution_outcome, &self.verifier_outcome) { + (ExecutionOutcome::InfraFailed, _) => ( + EvalResultStatus::InfraFailed, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject infrastructure failed".into()), + ), + ), + (ExecutionOutcome::Indeterminate, _) => ( + EvalResultStatus::Indeterminate, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject execution indeterminate".into()), + ), + ), + (_, VerifierOutcome::VerifierInfraFailed { reason }) + | (_, VerifierOutcome::MissingReward { reason }) => { + // Missing verifier reward never defaults to 0.0! + (EvalResultStatus::InfraFailed, None, Some(reason.clone())) + } + (ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }) => ( + // Adjudication #1: Agent timed out, verifier still ran and produced a valid + // reward; v1 status is `subject_failed` with the verified `score` preserved. + EvalResultStatus::SubjectFailed, + Some(*score), + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), + ), + ), + (ExecutionOutcome::SubjectFailed, VerifierOutcome::Verified { score }) => ( + EvalResultStatus::SubjectFailed, + Some(*score), + self.failure_reason.clone(), + ), + ( + ExecutionOutcome::Completed + | ExecutionOutcome::SubjectFailed + | ExecutionOutcome::Timeout, + VerifierOutcome::SubjectFailed { score, reason }, + ) => ( + EvalResultStatus::SubjectFailed, + *score, + self.failure_reason.clone().or_else(|| reason.clone()), + ), + (ExecutionOutcome::Completed, VerifierOutcome::Verified { score }) => { + (EvalResultStatus::Completed, Some(*score), None) + } + (ExecutionOutcome::Timeout, VerifierOutcome::Skipped) => ( + EvalResultStatus::SubjectFailed, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), + ), + ), + (ExecutionOutcome::SubjectFailed, VerifierOutcome::Skipped) => ( + EvalResultStatus::SubjectFailed, + None, + self.failure_reason.clone(), + ), + (ExecutionOutcome::Completed, VerifierOutcome::Skipped) => { + (EvalResultStatus::Completed, None, None) + } + }; + + V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score, + usage, + cost_usd, + duration_ms: self.duration_ms, + status, + failure_reason, + artifacts, + }, + } + } +} + +/// The 5 explicit semantic difference scenarios between Legacy TS `v1` and Native Rust `v2`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DifferenceScenarioId { + TimeoutWithRewardIncompleteUsage, + AuditTruncation, + MissingCriticalArtifacts, + LateCancellation, + EvidenceRecoveredLater, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DifferenceAdjudication { + pub id: DifferenceScenarioId, + pub title: String, + pub legacy_v1_behavior: String, + pub rust_v2_behavior: String, + pub v1_projection_rule: String, + pub conditions_comparable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DifferenceRegistry { + entries: Vec, +} + +impl Default for DifferenceRegistry { + fn default() -> Self { + Self::new() + } +} + +impl DifferenceRegistry { + pub fn new() -> Self { + Self { + entries: vec![ + DifferenceAdjudication { + id: DifferenceScenarioId::TimeoutWithRewardIncompleteUsage, + title: "Timeout with verifier reward but incomplete usage".into(), + legacy_v1_behavior: "Projects status='subject_failed', preserves verifier score, records recovered lower-bound usage, and sets costUsd=null because metering did not settle.".into(), + rust_v2_behavior: "Records ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }, and MeteringState::LowerBound orthogonally without conflating execution timeout and metering completeness.".into(), + v1_projection_rule: "Projects status='subject_failed', score=Some(score), usage=Some(lower_bound_usage), costUsd=None.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::AuditTruncation, + title: "Egress audit log truncation at byte cap".into(), + legacy_v1_behavior: "Appends ruleId='audit_truncated' marker to hits.jsonl and sets truncated=true in the egress-audit artifact while keeping the attempt status unchanged.".into(), + rust_v2_behavior: "Captures EgressAuditState::Recorded { truncated, policy_error_count } as an explicit dimension while distinguishing it from Missing or Unreadable audit logs (which fail closed as infra_failed).".into(), + v1_projection_rule: "Preserves completed/subject_failed status and emits the egress-audit artifact with truncated=true and policyErrorCount when nonzero; Missing or Unreadable audit logs project to status='infra_failed'.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::MissingCriticalArtifacts, + title: "0-byte empty file vs. missing or unreadable critical artifact".into(), + legacy_v1_behavior: "Treats a present 0-byte regular file (e.g. empty stderr) as valid with bytes=0, while missing/unreadable required evidence fails the attempt.".into(), + rust_v2_behavior: "Tracks CriticalArtifactsState::Present (including 0-byte regular files verified by inode/stat) separately from CriticalArtifactsState::Missing.".into(), + v1_projection_rule: "0-byte regular files remain valid artifacts with bytes=0; Missing critical artifacts project to status='infra_failed' with score=None.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::LateCancellation, + title: "Cancellation signal before vs. after verification and settlement".into(), + legacy_v1_behavior: "An abort signal observed before attempt append could overwrite a verified result with indeterminate.".into(), + rust_v2_behavior: "Separates AttemptCancellation::BeforeVerification from AttemptCancellation::AfterVerification orthogonally from ExecutionOutcome.".into(), + v1_projection_rule: "Pre-verification cancellation projects to status='indeterminate' with score=None; post-verification cancellation preserves the verified terminal result and emits cancelled-after-verification.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::EvidenceRecoveredLater, + title: "Post-hoc evidence recovered from checkpoint or host journal after crash".into(), + legacy_v1_behavior: "Mutated or re-ran the attempt when the process died before writing the final cell JSON.".into(), + rust_v2_behavior: "Appends a PostHocRecovery journal entry referencing the original attempt sequence and reconstructs V2AttemptRecord with AttemptMetering and recovered_post_hoc=true without mutating prior entries.".into(), + v1_projection_rule: "Projects recovered usage/cost and appends a post-hoc-evidence-recovery artifact while preserving immutable sequence numbering.".into(), + conditions_comparable: true, + }, + ], + } + } + + pub fn entries(&self) -> &[DifferenceAdjudication] { + &self.entries + } + + pub fn get(&self, id: DifferenceScenarioId) -> Option<&DifferenceAdjudication> { + self.entries.iter().find(|entry| entry.id == id) + } +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hex::encode(hasher.finalize()) +} + +pub(crate) fn exact_object<'a>( + value: &'a Value, + where_label: &str, + required: &[&str], + optional: &[&str], +) -> Result<&'a JsonObject, ContractError> { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + for key in obj.keys() { + if !required.contains(&key.as_str()) && !optional.contains(&key.as_str()) { + return Err(ContractError::Invalid(format!( + "{where_label} has unsupported fields" + ))); + } + } + for req in required { + if !obj.contains_key(*req) { + return Err(ContractError::Invalid(format!( + "{where_label}.{req} is required" + ))); + } + } + Ok(obj) +} + +pub(crate) fn decode_json_object( + value: &Value, + where_label: &str, +) -> Result { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + for (k, v) in obj { + validate_json_value(v, &format!("{where_label}.{k}"))?; + } + Ok(obj.clone()) +} + +fn validate_json_value(value: &Value, where_label: &str) -> Result<(), ContractError> { + match value { + Value::Null | Value::Bool(_) | Value::String(_) => Ok(()), + Value::Number(n) => { + if n.as_f64().is_some_and(f64::is_finite) { + Ok(()) + } else { + Err(ContractError::Invalid(format!( + "{where_label} must be JSON" + ))) + } + } + Value::Array(arr) => { + for (idx, item) in arr.iter().enumerate() { + validate_json_value(item, &format!("{where_label}[{idx}]"))?; + } + Ok(()) + } + Value::Object(map) => { + for (k, v) in map { + validate_json_value(v, &format!("{where_label}.{k}"))?; + } + Ok(()) + } + } +} + +pub(crate) fn identifier(value: &Value, where_label: &str) -> Result { + let text = nonempty_string(value, where_label)?; + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return Err(ContractError::Invalid(format!("{where_label} is required"))); + }; + if !first.is_ascii_alphanumeric() + || !chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) + { + return Err(ContractError::Invalid(format!("{where_label} is invalid"))); + } + Ok(text) +} + +pub(crate) fn nonempty_string(value: &Value, where_label: &str) -> Result { + match value.as_str() { + Some(s) if !s.is_empty() => Ok(s.to_owned()), + _ => Err(ContractError::Invalid(format!("{where_label} is required"))), + } +} + +fn unique_strings(value: &Value, where_label: &str) -> Result, ContractError> { + let arr = value + .as_array() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an array")))?; + let mut seen = BTreeSet::new(); + let mut out = Vec::with_capacity(arr.len()); + for item in arr { + let s = nonempty_string(item, where_label)?; + if !seen.insert(s.clone()) { + return Err(ContractError::Invalid(format!( + "{where_label} contains duplicates" + ))); + } + out.push(s); + } + Ok(out) +} + +fn ensure_unique_ids<'a>( + ids: impl Iterator, + label: &str, +) -> Result<(), ContractError> { + let mut seen = BTreeSet::new(); + for id in ids { + if !seen.insert(id) { + return Err(ContractError::Invalid(format!("duplicate {label} id"))); + } + } + Ok(()) +} + +pub(crate) fn positive_u32(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) if n >= 1 && n <= u64::from(u32::MAX) => Ok(n as u32), + _ => Err(ContractError::Invalid(format!( + "{where_label} must be positive" + ))), + } +} + +pub(crate) fn positive_usize(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) if n >= 1 => usize::try_from(n) + .map_err(|_| ContractError::Invalid(format!("{where_label} must be positive"))), + _ => Err(ContractError::Invalid(format!( + "{where_label} must be positive" + ))), + } +} + +pub(crate) fn nonnegative_u64(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) => Ok(n), + None => match value.as_f64() { + Some(f) if f.is_finite() && f >= 0.0 && f.fract() == 0.0 => Ok(f as u64), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + }, + } +} + +pub(crate) fn nullable_f64(value: &Value, where_label: &str) -> Result, ContractError> { + if value.is_null() { + return Ok(None); + } + match value.as_f64() { + Some(f) if f.is_finite() => Ok(Some(f)), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + } +} + +pub(crate) fn nullable_nonnegative_f64( + value: &Value, + where_label: &str, +) -> Result, ContractError> { + if value.is_null() { + return Ok(None); + } + match value.as_f64() { + Some(f) if f.is_finite() && f >= 0.0 => Ok(Some(f)), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + } +} diff --git a/products/eval/src/lib.rs b/products/eval/src/lib.rs new file mode 100644 index 0000000000..237bb1fa2d --- /dev/null +++ b/products/eval/src/lib.rs @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Measurement and control plane for multi-arm Maka evaluation experiments. + +pub mod contract; +pub mod measurement; diff --git a/products/eval/src/measurement.rs b/products/eval/src/measurement.rs new file mode 100644 index 0000000000..93ef29a746 --- /dev/null +++ b/products/eval/src/measurement.rs @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Exact 6-counter token usage preservation, metering state semantics, +//! and standardized model pricing. + +use crate::contract::{ContractError, exact_object, nonnegative_u64}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const COST_USD_TOLERANCE: f64 = 1e-9; + +/// Exact 6-counter normalized token usage. +/// +/// Each counter is preserved as reported by the provider or receipt; `total_tokens` +/// is never synthesized or overwritten by summing the other five counters. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct NormalizedUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub reasoning_tokens: u64, + pub total_tokens: u64, +} + +impl NormalizedUsage { + pub fn decode(value: &Value, where_label: &str) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "inputTokens", + "outputTokens", + "cacheReadTokens", + "cacheWriteTokens", + "reasoningTokens", + "totalTokens", + ], + &[], + )?; + Ok(Self { + input_tokens: nonnegative_u64( + obj.get("inputTokens").unwrap(), + &format!("{where_label}.inputTokens"), + )?, + output_tokens: nonnegative_u64( + obj.get("outputTokens").unwrap(), + &format!("{where_label}.outputTokens"), + )?, + cache_read_tokens: nonnegative_u64( + obj.get("cacheReadTokens").unwrap(), + &format!("{where_label}.cacheReadTokens"), + )?, + cache_write_tokens: nonnegative_u64( + obj.get("cacheWriteTokens").unwrap(), + &format!("{where_label}.cacheWriteTokens"), + )?, + reasoning_tokens: nonnegative_u64( + obj.get("reasoningTokens").unwrap(), + &format!("{where_label}.reasoningTokens"), + )?, + total_tokens: nonnegative_u64( + obj.get("totalTokens").unwrap(), + &format!("{where_label}.totalTokens"), + )?, + }) + } + + /// Accumulates another `NormalizedUsage` record counter-by-counter without + /// synthesizing `total_tokens` from the component fields. + pub fn accumulate_exact(&mut self, other: &Self) { + self.input_tokens = self.input_tokens.saturating_add(other.input_tokens); + self.output_tokens = self.output_tokens.saturating_add(other.output_tokens); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(other.cache_read_tokens); + self.cache_write_tokens = self + .cache_write_tokens + .saturating_add(other.cache_write_tokens); + self.reasoning_tokens = self.reasoning_tokens.saturating_add(other.reasoning_tokens); + self.total_tokens = self.total_tokens.saturating_add(other.total_tokens); + } + + pub fn max_by_counter(&self, other: &Self) -> Self { + Self { + input_tokens: self.input_tokens.max(other.input_tokens), + output_tokens: self.output_tokens.max(other.output_tokens), + cache_read_tokens: self.cache_read_tokens.max(other.cache_read_tokens), + cache_write_tokens: self.cache_write_tokens.max(other.cache_write_tokens), + reasoning_tokens: self.reasoning_tokens.max(other.reasoning_tokens), + total_tokens: self.total_tokens.max(other.total_tokens), + } + } + + pub fn dominates(&self, other: &Self) -> bool { + self.input_tokens >= other.input_tokens + && self.output_tokens >= other.output_tokens + && self.cache_read_tokens >= other.cache_read_tokens + && self.cache_write_tokens >= other.cache_write_tokens + && self.reasoning_tokens >= other.reasoning_tokens + && self.total_tokens >= other.total_tokens + } + + pub fn billing_counters_eq(&self, other: &Self) -> bool { + self.input_tokens == other.input_tokens + && self.output_tokens == other.output_tokens + && self.cache_read_tokens == other.cache_read_tokens + && self.cache_write_tokens == other.cache_write_tokens + && self.reasoning_tokens == other.reasoning_tokens + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MeteringState { + Complete, + LowerBound, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum TokenBasis { + #[serde(rename = "complete")] + Complete, + #[serde(rename = "lower-bound")] + LowerBound, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PricingRates { + pub input_per_million: f64, + pub output_per_million: f64, + pub cache_read_per_million: f64, + pub cache_write_per_million: f64, +} + +/// Published DeepSeek V4 Flash prices in USD per million tokens (off-peak band). +pub const DEEPSEEK_V4_FLASH_COST: PricingRates = PricingRates { + input_per_million: 0.22, + output_per_million: 0.66, + cache_read_per_million: 0.007, + cache_write_per_million: 0.22, +}; + +pub fn deepseek_cost_usd(usage: &NormalizedUsage) -> f64 { + price_usage_usd(usage, &DEEPSEEK_V4_FLASH_COST) +} + +pub fn price_usage_usd(usage: &NormalizedUsage, rates: &PricingRates) -> f64 { + let cached = usage + .cache_read_tokens + .saturating_add(usage.cache_write_tokens); + let uncached = usage.input_tokens.saturating_sub(cached); + ((uncached as f64) * rates.input_per_million + + (usage.cache_read_tokens as f64) * rates.cache_read_per_million + + (usage.cache_write_tokens as f64) * rates.cache_write_per_million + + (usage.output_tokens as f64) * rates.output_per_million) + / 1_000_000.0 +} + +pub fn costs_match_within_tolerance(left: Option, right: Option) -> bool { + match (left, right) { + (None, None) => true, + (Some(a), Some(b)) => (a - b).abs() <= COST_USD_TOLERANCE, + _ => false, + } +} + +pub fn default_model_pricing(model: &str) -> Option { + (model == "deepseek-v4-flash").then_some(DEEPSEEK_V4_FLASH_COST) +} diff --git a/products/eval/tests/integration/contract.rs b/products/eval/tests/integration/contract.rs new file mode 100644 index 0000000000..46b9e58e7c --- /dev/null +++ b/products/eval/tests/integration/contract.rs @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::support::sample_two_arm_spec_json; +use maka_eval::contract::{ + AttemptCancellation, AttemptMetering, AttemptPhase, CleanupState, CriticalArtifactsState, + DifferenceRegistry, EgressAuditState, EvalResult, EvalResultStatus, ExecutionOutcome, + ExperimentSpec, PricingEvidence, RunFingerprint, SchemaVersion, V1CellAttempt, V2AttemptRecord, + VerifierOutcome, decode_v1_cell_attempt, expand_experiment, parse_experiment_spec, + select_cell_result, +}; +use maka_eval::measurement::{NormalizedUsage, costs_match_within_tolerance, deepseek_cost_usd}; +use serde_json::{Value, json}; + +#[test] +fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { + let spec_json = sample_two_arm_spec_json(); + let spec = parse_experiment_spec(&spec_json).unwrap(); + let cells = expand_experiment(&spec); + + assert_eq!(cells.len(), 4); + assert_eq!(cells[0].id, "task-alpha::1::maka"); + assert_eq!(cells[1].id, "task-alpha::1::external-cli"); + assert_eq!(cells[2].id, "task-beta::1::maka"); + assert_eq!(cells[3].id, "task-beta::1::external-cli"); + + // Unknown top-level field must be rejected both by parse_experiment_spec and serde Deserialize. + let mut invalid = spec_json.clone(); + invalid + .as_object_mut() + .unwrap() + .insert("unknownField".into(), json!(123)); + assert!(parse_experiment_spec(&invalid).is_err()); + assert!(serde_json::from_value::(invalid).is_err()); + + // Concurrency changes execution fingerprint but preserves semantic fingerprint. + let mut higher_concurrency_json = spec_json.clone(); + higher_concurrency_json["execution"]["maxConcurrentTaskGroups"] = json!(16); + let higher_spec = parse_experiment_spec(&higher_concurrency_json).unwrap(); + + let fp1 = RunFingerprint::compute(&spec); + let fp2 = RunFingerprint::compute(&higher_spec); + assert!(fp1.is_semantically_comparable(&fp2)); + assert_ne!(fp1.execution_sha256, fp2.execution_sha256); + + // Changing experiment.id or float vs integer representation (1.0 vs 1) preserves semantic fingerprint. + let mut renamed_json = spec_json; + renamed_json["id"] = json!("tb21-deepseek-v4-flash-rerun"); + renamed_json["budget"]["timeoutMultiplier"] = json!(1.0); + let renamed_spec = serde_json::from_value::(renamed_json).unwrap(); + let fp3 = RunFingerprint::compute(&renamed_spec); + assert!(fp1.is_semantically_comparable(&fp3)); +} + +#[test] +fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within_1e9() { + // Provider reports a totalTokens value distinct from the arithmetic sum of the 5 fields. + let u1 = NormalizedUsage { + input_tokens: 1000, + output_tokens: 250, + cache_read_tokens: 400, + cache_write_tokens: 100, + reasoning_tokens: 80, + total_tokens: 9999, + }; + let u2 = NormalizedUsage { + input_tokens: 500, + output_tokens: 150, + cache_read_tokens: 200, + cache_write_tokens: 0, + reasoning_tokens: 20, + total_tokens: 7777, + }; + let mut combined = u1.clone(); + combined.accumulate_exact(&u2); + assert_eq!(combined.input_tokens, 1500); + assert_eq!(combined.output_tokens, 400); + assert_eq!(combined.cache_read_tokens, 600); + assert_eq!(combined.cache_write_tokens, 100); + assert_eq!(combined.reasoning_tokens, 100); + // Must equal 9999 + 7777 = 17776, never the sum of the other 5 counters! + assert_eq!(combined.total_tokens, 17776); + + // Verify DEEPSEEK_V4_FLASH_COST formula: + // uncached = 1000 - 400 - 100 = 500 + // (500 * 0.22 + 400 * 0.007 + 100 * 0.22 + 250 * 0.66) / 1_000_000 + // = (110 + 2.8 + 22 + 165) / 1_000_000 = 299.8 / 1_000_000 = 0.0002998 + let cost = deepseek_cost_usd(&u1); + assert!(costs_match_within_tolerance(Some(cost), Some(0.0002998))); + assert!(!costs_match_within_tolerance( + Some(cost), + Some(0.0002998 + 1e-8) + )); +} + +#[test] +fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_terminal() { + let attempts = vec![ + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 1, + started_at: 10, + completed_at: 20, + result: EvalResult { + score: None, + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::InfraFailed, + failure_reason: Some("docker pull failed".into()), + artifacts: Vec::new(), + }, + }, + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 2, + started_at: 30, + completed_at: 40, + result: EvalResult { + score: Some(0.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 3, + started_at: 50, + completed_at: 60, + result: EvalResult { + score: Some(1.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, + ]; + + let selected = select_cell_result(&attempts).unwrap(); + // Sequence 2 (Completed with score 0.0) is valid and earlier than sequence 3! + assert_eq!(selected.sequence, 2); + assert_eq!(selected.result.score, Some(0.0)); +} + +#[test] +fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly() { + // V1CellAttempt rejects sequence == 0, Completed with failureReason, and costUsd without usage. + let valid_v1 = json!({ + "cellId": "t1::1::maka", + "sequence": 1, + "startedAt": 10, + "completedAt": 20, + "result": { + "score": null, + "usage": null, + "costUsd": null, + "durationMs": 10, + "status": "subject_failed", + "failureReason": null, + "artifacts": [] + } + }); + assert!(serde_json::from_value::(valid_v1.clone()).is_ok()); + + let mut zero_seq = valid_v1.clone(); + zero_seq["sequence"] = json!(0); + assert!(serde_json::from_value::(zero_seq).is_err()); + + let mut cost_without_usage = valid_v1.clone(); + cost_without_usage["result"]["costUsd"] = json!(0.01); + assert!(serde_json::from_value::(cost_without_usage).is_err()); + + let mut completed_with_reason = valid_v1.clone(); + completed_with_reason["result"]["status"] = json!("completed"); + completed_with_reason["result"]["failureReason"] = json!("unexpected"); + assert!(serde_json::from_value::(completed_with_reason).is_err()); + + // V2AttemptRecord rejects schemaVersion != maka.eval.v2 and invalid metering combinations. + let v2_wrong_schema = json!({ + "schemaVersion": "maka.eval.v1", + "cellId": "t1::1::maka", + "sequence": 1, + "startedAt": 10, + "completedAt": 20, + "durationMs": 10, + "phase": "settled", + "executionOutcome": "completed", + "verifierOutcome": { "kind": "skipped" }, + "metering": { "state": "unknown" }, + "cleanupState": { "kind": "confirmed" }, + "auditState": { "kind": "not_required" }, + "criticalArtifactsState": { "kind": "present" }, + "cancellation": "none", + "recoveredPostHoc": false, + "failureReason": null, + "artifacts": [] + }); + assert!(serde_json::from_value::(v2_wrong_schema.clone()).is_err()); + + let mut complete_without_usage = v2_wrong_schema.clone(); + complete_without_usage["schemaVersion"] = json!("maka.eval.v2"); + complete_without_usage["metering"] = json!({ "state": "complete", "usage": null }); + assert!(serde_json::from_value::(complete_without_usage).is_err()); + + let mut negative_cost = v2_wrong_schema; + negative_cost["schemaVersion"] = json!("maka.eval.v2"); + negative_cost["metering"] = json!({ + "state": "complete", + "usage": { + "inputTokens": 10, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 15 + }, + "pricing": { "kind": "subject_reported", "costUsd": -0.5 } + }); + assert!(serde_json::from_value::(negative_cost).is_err()); + + // Lossless V1 serde -> decode_v1_cell_attempt round-trip when SubjectFailed has failureReason: null. + let parsed_v1: V1CellAttempt = serde_json::from_value(valid_v1).unwrap(); + let encoded = serde_json::to_value(&parsed_v1).unwrap(); + assert_eq!(decode_v1_cell_attempt(&encoded).unwrap(), parsed_v1); +} + +#[test] +fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { + let registry = DifferenceRegistry::new(); + assert_eq!(registry.entries().len(), 5); + + let usage = NormalizedUsage { + input_tokens: 100, + output_tokens: 20, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 120, + }; + + // 1. Timeout + Verified(0.75) + LowerBound retains score and usage with costUsd == None; + // with Complete + AfterVerification + Recorded audit, preserves markers as SubjectFailed. + let base_v2 = V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: "t1::1::maka".into(), + sequence: 1, + started_at: 10, + completed_at: 20, + duration_ms: 10, + phase: AttemptPhase::Settled, + execution_outcome: ExecutionOutcome::Timeout, + verifier_outcome: VerifierOutcome::Verified { score: 0.75 }, + metering: AttemptMetering::LowerBound { + usage: usage.clone(), + }, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancellation: AttemptCancellation::None, + recovered_post_hoc: false, + failure_reason: Some("timed out".into()), + artifacts: Vec::new(), + }; + let v1_timeout_lb = base_v2.project_v1(); + assert_eq!(v1_timeout_lb.result.status, EvalResultStatus::SubjectFailed); + assert_eq!(v1_timeout_lb.result.score, Some(0.75)); + assert_eq!(v1_timeout_lb.result.usage, Some(usage.clone())); + assert_eq!(v1_timeout_lb.result.cost_usd, None); + + let mut v2_timeout_complete = base_v2.clone(); + v2_timeout_complete.metering = AttemptMetering::Complete { + usage: usage.clone(), + pricing: Some(PricingEvidence::Model { + model: "deepseek-v4-flash".into(), + cost_usd: Some(0.0000352), + }), + }; + v2_timeout_complete.audit_state = EgressAuditState::Recorded { + truncated: true, + policy_error_count: 2, + }; + v2_timeout_complete.cancellation = AttemptCancellation::AfterVerification; + let v1_timeout_complete = v2_timeout_complete.project_v1(); + assert_eq!( + v1_timeout_complete.result.status, + EvalResultStatus::SubjectFailed + ); + assert_eq!(v1_timeout_complete.result.score, Some(0.75)); + assert_eq!(v1_timeout_complete.result.cost_usd, Some(0.0000352)); + assert!( + v1_timeout_complete + .result + .artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification")) + ); + let audit_art = v1_timeout_complete + .result + .artifacts + .iter() + .find(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) + .unwrap(); + assert_eq!( + audit_art.get("truncated").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + audit_art.get("policyErrorCount").and_then(Value::as_u64), + Some(2) + ); + + // 2. Egress audit truncation preserves Completed + reward, whereas Missing fails closed. + let completed_base = V2AttemptRecord { + execution_outcome: ExecutionOutcome::Completed, + verifier_outcome: VerifierOutcome::Verified { score: 1.0 }, + metering: AttemptMetering::Complete { + usage, + pricing: Some(PricingEvidence::Model { + model: "deepseek-v4-flash".into(), + cost_usd: Some(0.0000352), + }), + }, + audit_state: EgressAuditState::truncated(), + failure_reason: None, + ..base_v2 + }; + let v1_audit_trunc = completed_base.project_v1(); + assert_eq!(v1_audit_trunc.result.status, EvalResultStatus::Completed); + assert_eq!(v1_audit_trunc.result.score, Some(1.0)); + assert!(v1_audit_trunc.result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("egress-audit") + && a.get("truncated").and_then(Value::as_bool) == Some(true) + })); + + let mut v2_audit_missing = completed_base.clone(); + v2_audit_missing.audit_state = EgressAuditState::Missing; + let v1_audit_missing = v2_audit_missing.project_v1(); + assert_eq!( + v1_audit_missing.result.status, + EvalResultStatus::InfraFailed + ); + assert_eq!(v1_audit_missing.result.score, None); + + // 3. CriticalArtifactsState::Present keeps Completed, whereas Missing fails closed. + assert_eq!( + completed_base.project_v1().result.status, + EvalResultStatus::Completed + ); + let mut v2_missing_art = completed_base.clone(); + v2_missing_art.critical_artifacts_state = CriticalArtifactsState::Missing { + paths: vec!["transcript.json".into()], + }; + let v1_missing_art = v2_missing_art.project_v1(); + assert_eq!(v1_missing_art.result.status, EvalResultStatus::InfraFailed); + assert_eq!(v1_missing_art.result.score, None); + + // 4. Late cancellation: BeforeVerification -> Indeterminate(None), AfterVerification -> Completed(score). + let mut v2_cancel_before = completed_base.clone(); + v2_cancel_before.cancellation = AttemptCancellation::BeforeVerification; + let v1_cancel_before = v2_cancel_before.project_v1(); + assert_eq!( + v1_cancel_before.result.status, + EvalResultStatus::Indeterminate + ); + assert_eq!(v1_cancel_before.result.score, None); + + let mut v2_cancel_after = completed_base.clone(); + v2_cancel_after.cancellation = AttemptCancellation::AfterVerification; + let v1_cancel_after = v2_cancel_after.project_v1(); + assert_eq!(v1_cancel_after.result.status, EvalResultStatus::Completed); + assert_eq!(v1_cancel_after.result.score, Some(1.0)); + assert!( + v1_cancel_after + .result + .artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification")) + ); + + // 5. Missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0). + let mut v2_missing_reward = completed_base.clone(); + v2_missing_reward.verifier_outcome = VerifierOutcome::MissingReward { + reason: "reward.json missing".into(), + }; + let v1_missing_reward = v2_missing_reward.project_v1(); + assert_eq!( + v1_missing_reward.result.status, + EvalResultStatus::InfraFailed + ); + assert_eq!(v1_missing_reward.result.score, None); + + let mut v2_zero_score = completed_base; + v2_zero_score.verifier_outcome = VerifierOutcome::Verified { score: 0.0 }; + let v1_zero_score = v2_zero_score.project_v1(); + assert_eq!(v1_zero_score.result.status, EvalResultStatus::Completed); + assert_eq!(v1_zero_score.result.score, Some(0.0)); +} diff --git a/products/eval/tests/integration/main.rs b/products/eval/tests/integration/main.rs new file mode 100644 index 0000000000..ef12ed4834 --- /dev/null +++ b/products/eval/tests/integration/main.rs @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +mod contract; +mod support; diff --git a/products/eval/tests/integration/support.rs b/products/eval/tests/integration/support.rs new file mode 100644 index 0000000000..403516d73f --- /dev/null +++ b/products/eval/tests/integration/support.rs @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use serde_json::{Value, json}; + +pub(crate) fn sample_two_arm_spec_json() -> Value { + json!({ + "schemaVersion": "maka.eval.v1", + "id": "tb21-deepseek-v4-flash-two-arm", + "benchmark": { + "id": "terminal-bench-2.1", + "version": "d49e28f1e4ddd13d289e85a5f312a66750951932", + "config": { + "repository": "https://github.com/harbor-framework/terminal-bench-2-1" + } + }, + "executor": { + "kind": "harbor", + "config": { + "frameworkVersion": "0.20.0", + "pythonPathEnv": "MAKA_EVAL_HARBOR_PYTHON", + "trialsRootEnv": "MAKA_EVAL_TRIALS_DIR", + "environment": { "type": "docker", "delete": true } + } + }, + "execution": { + "maxConcurrentTaskGroups": 2 + }, + "subjects": [ + { + "id": "maka", + "kind": "maka", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "makaExecutable": "/opt/maka-agent/maka", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 120000, + "connectionSlug": "eval-gateway", + "baseUrl": "http://127.0.0.1:43120/v1", + "model": "deepseek-v4-flash", + "sandboxMode": "workspace-write" + } + }, + { + "id": "external-cli", + "kind": "external", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "command": "/opt/maka-node-toolchain/bin/node", + "credentialEnvironment": { + "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY" + }, + "args": [ + "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js", + "deepseek-harness", + "http://127.0.0.1:43120/v1", + "/", + "dsh", + "{{task.input}}" + ] + } + } + ], + "tasks": [ + { + "id": "task-alpha", + "input": "Fix alpha", + "config": { "harbor": { "path": "tasks/task-alpha" } } + }, + { + "id": "task-beta", + "input": "Fix beta", + "config": { "harbor": { "path": "tasks/task-beta" } } + } + ], + "repetitions": 1, + "budget": { + "timeoutMultiplier": 1, + "maxSteps": 100 + }, + "verifier": { + "reward": "reward" + } + }) +} From 805c1404196889d30de7bb3fd0b3568d210545d3 Mon Sep 17 00:00:00 2001 From: colafornia Date: Fri, 13 Nov 2026 18:14:11 +0800 Subject: [PATCH 2/4] fix(eval): preserve frozen v1 result decoding and enforce stricter checks in v2 Generated-by: Google Antigravity --- products/eval/src/contract.rs | 94 ++++++++++++++---- products/eval/tests/integration/contract.rs | 101 ++++++++++++++++---- 2 files changed, 158 insertions(+), 37 deletions(-) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs index 7493b785e1..eb8b71230a 100644 --- a/products/eval/src/contract.rs +++ b/products/eval/src/contract.rs @@ -491,11 +491,6 @@ pub fn decode_eval_result(value: &Value, where_label: &str) -> Result None, @@ -505,11 +500,6 @@ pub fn decode_eval_result(value: &Value, where_label: &str) -> Result }, @@ -657,7 +648,8 @@ pub enum CriticalArtifactsState { #[serde( tag = "kind", rename_all = "snake_case", - rename_all_fields = "camelCase" + rename_all_fields = "camelCase", + deny_unknown_fields )] pub enum PricingEvidence { Model { @@ -705,7 +697,7 @@ impl PricingEvidence { } } -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize)] #[serde( tag = "state", rename_all = "snake_case", @@ -724,6 +716,53 @@ pub enum AttemptMetering { }, } +impl<'de> Deserialize<'de> for AttemptMetering { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + let state = value + .as_object() + .and_then(|obj| obj.get("state")) + .and_then(Value::as_str) + .ok_or_else(|| serde::de::Error::custom("metering.state is required"))?; + let metering = match state { + "unknown" => { + exact_object(&value, "metering", &["state"], &[]) + .map_err(serde::de::Error::custom)?; + Self::Unknown + } + "lower_bound" => { + let obj = exact_object(&value, "metering", &["state", "usage"], &[]) + .map_err(serde::de::Error::custom)?; + Self::LowerBound { + usage: serde_json::from_value(obj.get("usage").unwrap().clone()) + .map_err(serde::de::Error::custom)?, + } + } + "complete" => { + let obj = exact_object(&value, "metering", &["state", "usage"], &["pricing"]) + .map_err(serde::de::Error::custom)?; + let pricing = match obj.get("pricing") { + None | Some(Value::Null) => None, + Some(p) => { + Some(serde_json::from_value(p.clone()).map_err(serde::de::Error::custom)?) + } + }; + Self::Complete { + usage: serde_json::from_value(obj.get("usage").unwrap().clone()) + .map_err(serde::de::Error::custom)?, + pricing, + } + } + _ => return Err(serde::de::Error::custom("metering.state is invalid")), + }; + metering.validate().map_err(serde::de::Error::custom)?; + Ok(metering) + } +} + impl AttemptMetering { pub fn state(&self) -> MeteringState { match self { @@ -839,6 +878,24 @@ impl<'de> Deserialize<'de> for V2AttemptRecord { } impl V2AttemptRecord { + fn is_completed(&self) -> bool { + matches!(self.cleanup_state, CleanupState::Confirmed) + && self.cancellation != AttemptCancellation::BeforeVerification + && matches!( + self.critical_artifacts_state, + CriticalArtifactsState::Present + ) + && matches!( + self.audit_state, + EgressAuditState::NotRequired | EgressAuditState::Recorded { .. } + ) + && self.execution_outcome == ExecutionOutcome::Completed + && matches!( + self.verifier_outcome, + VerifierOutcome::Verified { .. } | VerifierOutcome::Skipped + ) + } + pub fn validate(&self) -> Result<(), ContractError> { if self.schema_version != SchemaVersion::V2 { return Err(ContractError::Invalid( @@ -870,6 +927,11 @@ impl V2AttemptRecord { } _ => {} } + if self.is_completed() && self.failure_reason.is_some() { + return Err(ContractError::Invalid( + "v2 completed attempt record must not set failureReason".into(), + )); + } self.metering.validate()?; Ok(()) } diff --git a/products/eval/tests/integration/contract.rs b/products/eval/tests/integration/contract.rs index 46b9e58e7c..db036b22eb 100644 --- a/products/eval/tests/integration/contract.rs +++ b/products/eval/tests/integration/contract.rs @@ -167,7 +167,8 @@ fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_term #[test] fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly() { - // V1CellAttempt rejects sequence == 0, Completed with failureReason, and costUsd without usage. + // Frozen v1 accepts subject-reported costUsd with usage=null and completed with failureReason, + // while still rejecting sequence == 0 and negative costUsd. Stricter checks belong in v2. let valid_v1 = json!({ "cellId": "t1::1::maka", "sequence": 1, @@ -189,18 +190,53 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly zero_seq["sequence"] = json!(0); assert!(serde_json::from_value::(zero_seq).is_err()); + let mut negative_v1_cost = valid_v1.clone(); + negative_v1_cost["result"]["costUsd"] = json!(-0.01); + assert!(serde_json::from_value::(negative_v1_cost).is_err()); + let mut cost_without_usage = valid_v1.clone(); + cost_without_usage["result"]["score"] = json!(0.0); cost_without_usage["result"]["costUsd"] = json!(0.01); - assert!(serde_json::from_value::(cost_without_usage).is_err()); + let parsed_cost_without_usage: V1CellAttempt = + serde_json::from_value(cost_without_usage).unwrap(); + assert_eq!(parsed_cost_without_usage.result.cost_usd, Some(0.01)); + assert_eq!(parsed_cost_without_usage.result.usage, None); + let encoded_cost_without_usage = serde_json::to_value(&parsed_cost_without_usage).unwrap(); + assert_eq!( + decode_v1_cell_attempt(&encoded_cost_without_usage).unwrap(), + parsed_cost_without_usage + ); + let completed_with_reason_result = json!({ + "score": 0.0, + "usage": null, + "costUsd": null, + "durationMs": 10, + "status": "completed", + "failureReason": "external subject detail", + "artifacts": [] + }); + let parsed_eval_result: EvalResult = + serde_json::from_value(completed_with_reason_result.clone()).unwrap(); + assert_eq!( + parsed_eval_result.failure_reason.as_deref(), + Some("external subject detail") + ); let mut completed_with_reason = valid_v1.clone(); - completed_with_reason["result"]["status"] = json!("completed"); - completed_with_reason["result"]["failureReason"] = json!("unexpected"); - assert!(serde_json::from_value::(completed_with_reason).is_err()); + completed_with_reason["result"] = completed_with_reason_result; + let parsed_completed_with_reason: V1CellAttempt = + serde_json::from_value(completed_with_reason).unwrap(); + let encoded_completed_with_reason = + serde_json::to_value(&parsed_completed_with_reason).unwrap(); + assert_eq!( + decode_v1_cell_attempt(&encoded_completed_with_reason).unwrap(), + parsed_completed_with_reason + ); - // V2AttemptRecord rejects schemaVersion != maka.eval.v2 and invalid metering combinations. - let v2_wrong_schema = json!({ - "schemaVersion": "maka.eval.v1", + // V2AttemptRecord rejects schemaVersion != maka.eval.v2, illegal metering states, + // unknown fields on closed tagged enums, and completed attempts with failureReason. + let v2_base = json!({ + "schemaVersion": "maka.eval.v2", "cellId": "t1::1::maka", "sequence": 1, "startedAt": 10, @@ -218,25 +254,48 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly "failureReason": null, "artifacts": [] }); - assert!(serde_json::from_value::(v2_wrong_schema.clone()).is_err()); + assert!(serde_json::from_value::(v2_base.clone()).is_ok()); - let mut complete_without_usage = v2_wrong_schema.clone(); - complete_without_usage["schemaVersion"] = json!("maka.eval.v2"); + let mut v2_wrong_schema = v2_base.clone(); + v2_wrong_schema["schemaVersion"] = json!("maka.eval.v1"); + assert!(serde_json::from_value::(v2_wrong_schema).is_err()); + + let mut complete_without_usage = v2_base.clone(); complete_without_usage["metering"] = json!({ "state": "complete", "usage": null }); assert!(serde_json::from_value::(complete_without_usage).is_err()); - let mut negative_cost = v2_wrong_schema; - negative_cost["schemaVersion"] = json!("maka.eval.v2"); + let mut unknown_with_pricing = v2_base.clone(); + unknown_with_pricing["metering"] = json!({ + "state": "unknown", + "pricing": { "kind": "subject_reported", "costUsd": 0.01 } + }); + assert!(serde_json::from_value::(unknown_with_pricing).is_err()); + + let valid_usage = json!({ + "inputTokens": 10, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 15 + }); + + let mut lower_bound_with_pricing = v2_base.clone(); + lower_bound_with_pricing["metering"] = json!({ + "state": "lower_bound", + "usage": valid_usage, + "pricing": { "kind": "subject_reported", "costUsd": 0.01 } + }); + assert!(serde_json::from_value::(lower_bound_with_pricing).is_err()); + + let mut completed_v2_with_reason = v2_base.clone(); + completed_v2_with_reason["failureReason"] = json!("unexpected"); + assert!(serde_json::from_value::(completed_v2_with_reason).is_err()); + + let mut negative_cost = v2_base; negative_cost["metering"] = json!({ "state": "complete", - "usage": { - "inputTokens": 10, - "outputTokens": 5, - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "reasoningTokens": 0, - "totalTokens": 15 - }, + "usage": valid_usage, "pricing": { "kind": "subject_reported", "costUsd": -0.5 } }); assert!(serde_json::from_value::(negative_cost).is_err()); From b9469563b3a2267dca3970acbb13e7c960bef441 Mon Sep 17 00:00:00 2001 From: colafornia Date: Fri, 13 Nov 2026 18:30:56 +0800 Subject: [PATCH 3/4] refactor(eval): remove redundant abstractions and dual decoders - contract: drop prose-in-heap DifferenceRegistry, write-only AttemptPhase, wrapper constructors/predicates, and RawV2AttemptRecord duplication; unify v2->v1 resolution and serde Deserialize boundary; drop thiserror dependency - measurement: remove 1c-only MeteringState/TokenBasis and unused usage helpers, unify NormalizedUsage Deserialize, and move pricing onto PricingRates - tests: consolidate multi-file integration test harness into tests/contract.rs with sharper behavioral assertions Generated-by: Google Antigravity --- Cargo.lock | 1 - products/eval/Cargo.toml | 6 - products/eval/README.md | 4 +- products/eval/README.zh-CN.md | 4 +- products/eval/src/contract.rs | 898 +++++++----------- products/eval/src/measurement.rs | 182 ++-- .../eval/tests/{integration => }/contract.rs | 259 +++-- products/eval/tests/integration/main.rs | 21 - products/eval/tests/integration/support.rs | 101 -- 9 files changed, 588 insertions(+), 888 deletions(-) rename products/eval/tests/{integration => }/contract.rs (71%) delete mode 100644 products/eval/tests/integration/main.rs delete mode 100644 products/eval/tests/integration/support.rs diff --git a/Cargo.lock b/Cargo.lock index ceb42dc0ff..694dbb80b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6881,7 +6881,6 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.1", - "thiserror 2.0.21", ] [[package]] diff --git a/products/eval/Cargo.toml b/products/eval/Cargo.toml index 724bed5572..ca6c29a94a 100644 --- a/products/eval/Cargo.toml +++ b/products/eval/Cargo.toml @@ -21,15 +21,9 @@ version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true -autotests = false - -[[test]] -name = "integration" -path = "tests/integration/main.rs" [dependencies] hex.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true -thiserror.workspace = true diff --git a/products/eval/README.md b/products/eval/README.md index b3a1fb91f7..fa11f0eeb1 100644 --- a/products/eval/README.md +++ b/products/eval/README.md @@ -25,7 +25,7 @@ Measurement and control plane contracts for multi-arm benchmark experiments (`ma ## Architecture -- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions, condition-only `RunFingerprint`, and the semantic difference registry. -- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), `MeteringState` / `TokenBasis` semantics, and standardized `DEEPSEEK_V4_FLASH_COST` pricing. +- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions, and condition-only `RunFingerprint`. +- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation) and standardized `DEEPSEEK_V4_FLASH_COST` pricing. [Source](src/lib.rs) · [Cargo manifest](Cargo.toml) diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md index 3f348dcd2d..d56fdde5c7 100644 --- a/products/eval/README.zh-CN.md +++ b/products/eval/README.zh-CN.md @@ -25,7 +25,7 @@ ## 模块结构 -- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度、仅记录运行条件的 `RunFingerprint` 及语义差异注册表。 -- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）、`MeteringState` / `TokenBasis` 语义及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 +- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度，以及仅记录运行条件的 `RunFingerprint`。 +- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 [源码](src/lib.rs) · [Cargo 配置](Cargo.toml) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs index eb8b71230a..14ac06575f 100644 --- a/products/eval/src/contract.rs +++ b/products/eval/src/contract.rs @@ -18,23 +18,38 @@ */ //! Experiment specifications, orthogonal `v2` attempt records, strict `v1` -//! projections, run fingerprinting, and the semantic difference registry. +//! projections, and run fingerprinting. -use crate::measurement::{MeteringState, NormalizedUsage}; -use serde::{Deserialize, Deserializer, Serialize}; +use crate::measurement::{NormalizedUsage, decode_usage}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; -use thiserror::Error; +use std::fmt; pub type JsonObject = Map; -#[derive(Debug, Error, Clone, PartialEq, Eq)] -pub enum ContractError { - #[error("{0}")] - Invalid(String), +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContractError(String); + +impl ContractError { + pub fn new(message: impl Into) -> Self { + Self(message.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } } +impl fmt::Display for ContractError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ContractError {} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum SchemaVersion { #[serde(rename = "maka.eval.v1")] @@ -65,8 +80,8 @@ pub struct ExecutorSpec { pub config: JsonObject, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] pub struct ExecutionSpec { pub max_concurrent_task_groups: usize, } @@ -136,10 +151,6 @@ pub struct ExperimentCell { pub verifier: JsonObject, } -pub fn format_cell_id(task_id: &str, repetition: u32, subject_id: &str) -> String { - format!("{task_id}::{repetition}::{subject_id}") -} - pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { let mut cells = Vec::with_capacity(spec.tasks.len() * (spec.repetitions as usize) * spec.subjects.len()); @@ -147,7 +158,7 @@ pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { for repetition in 1..=spec.repetitions { for subject in &spec.subjects { cells.push(ExperimentCell { - id: format_cell_id(&task.id, repetition, &subject.id), + id: format!("{}::{repetition}::{}", task.id, subject.id), experiment_id: spec.id.clone(), benchmark: spec.benchmark.clone(), executor: spec.executor.clone(), @@ -163,7 +174,7 @@ pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { cells } -pub fn parse_experiment_spec(value: &Value) -> Result { +fn parse_experiment_spec(value: &Value) -> Result { let root = exact_object( value, "experiment", @@ -185,9 +196,7 @@ pub fn parse_experiment_spec(value: &Value) -> Result SchemaVersion::V1, Some("maka.eval.v2") => SchemaVersion::V2, _ => { - return Err(ContractError::Invalid( - "unsupported experiment schema".into(), - )); + return Err(ContractError::new("unsupported experiment schema")); } }; @@ -218,7 +227,7 @@ pub fn parse_experiment_spec(value: &Value) -> Result Result SubjectKind::Maka, Some("external") => SubjectKind::External, _ => { - return Err(ContractError::Invalid(format!( - "{where_label}.kind is invalid" - ))); + return Err(ContractError::new(format!("{where_label}.kind is invalid"))); } }; subjects.push(SubjectSpec { @@ -244,7 +251,7 @@ pub fn parse_experiment_spec(value: &Value) -> Result Result Result Result Result Result String { - let normalized = canonicalize_value(value); - serde_json::to_string(&normalized).expect("canonical Value serialization cannot fail") +fn canonical_sha256_hex(value: &Value) -> String { + let mut buf = Vec::with_capacity(256); + write_canonical_json(value, &mut buf); + let mut hasher = Sha256::new(); + hasher.update(&buf); + hex::encode(hasher.finalize()) } -fn canonicalize_value(value: &Value) -> Value { +fn write_canonical_json(value: &Value, out: &mut Vec) { match value { - Value::Object(map) => { - let mut sorted: Vec<(&String, &Value)> = map.iter().collect(); - sorted.sort_by_key(|(a, _)| *a); - let mut out = Map::with_capacity(sorted.len()); - for (k, v) in sorted { - out.insert(k.clone(), canonicalize_value(v)); - } - Value::Object(out) + Value::Null | Value::Bool(_) | Value::String(_) => { + serde_json::to_writer(&mut *out, value) + .expect("canonical JSON value serialization cannot fail"); } - Value::Array(items) => Value::Array(items.iter().map(canonicalize_value).collect()), Value::Number(n) => { if n.is_u64() || n.is_i64() { - Value::Number(n.clone()) + serde_json::to_writer(&mut *out, n) + .expect("canonical integer serialization cannot fail"); } else if let Some(f) = n.as_f64() { if f == 0.0 { - Value::Number(0.into()) + out.extend_from_slice(b"0"); } else if f.is_finite() && f.fract() == 0.0 && (-9_007_199_254_740_991.0..=9_007_199_254_740_991.0).contains(&f) { if f >= 0.0 { - Value::Number((f as u64).into()) + serde_json::to_writer(&mut *out, &(f as u64)) + .expect("canonical float->u64 serialization cannot fail"); } else { - Value::Number((f as i64).into()) + serde_json::to_writer(&mut *out, &(f as i64)) + .expect("canonical float->i64 serialization cannot fail"); } } else { - Value::Number(n.clone()) + serde_json::to_writer(&mut *out, n) + .expect("canonical float serialization cannot fail"); } } else { - Value::Number(n.clone()) + serde_json::to_writer(&mut *out, n) + .expect("canonical number serialization cannot fail"); + } + } + Value::Array(items) => { + out.push(b'['); + for (idx, item) in items.iter().enumerate() { + if idx > 0 { + out.push(b','); + } + write_canonical_json(item, out); } + out.push(b']'); + } + Value::Object(map) => { + // Explicit key sorting is required because workspace feature unification + // enables serde_json/preserve_order. + let mut sorted: Vec<(&String, &Value)> = map.iter().collect(); + sorted.sort_by_key(|(k, _)| *k); + out.push(b'{'); + for (idx, (k, v)) in sorted.into_iter().enumerate() { + if idx > 0 { + out.push(b','); + } + serde_json::to_writer(&mut *out, k) + .expect("canonical object key serialization cannot fail"); + out.push(b':'); + write_canonical_json(v, out); + } + out.push(b'}'); } - other => other.clone(), } } @@ -377,14 +411,10 @@ impl RunFingerprint { "executorConfig": spec.executor.config, }); Self { - semantic_sha256: sha256_hex(canonical_json_string(&semantic_payload).as_bytes()), - execution_sha256: sha256_hex(canonical_json_string(&execution_payload).as_bytes()), + semantic_sha256: canonical_sha256_hex(&semantic_payload), + execution_sha256: canonical_sha256_hex(&execution_payload), } } - - pub fn is_semantically_comparable(&self, other: &Self) -> bool { - self.semantic_sha256 == other.semantic_sha256 - } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -396,6 +426,12 @@ pub enum EvalResultStatus { Indeterminate, } +impl EvalResultStatus { + pub fn is_selectable(self) -> bool { + matches!(self, Self::Completed | Self::SubjectFailed) + } +} + #[derive(Debug, Clone, PartialEq, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct EvalResult { @@ -438,25 +474,17 @@ impl<'de> Deserialize<'de> for V1CellAttempt { } } -pub fn is_replaceable_attempt(attempt: &V1CellAttempt) -> bool { - matches!( - attempt.result.status, - EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate - ) -} - /// Canonical cell result selector: selects the earliest valid attempt by sequence (`sequence >= 1`) /// whose status is neither `infra_failed` nor `indeterminate`. A `completed` attempt with score `0.0` /// or a `subject_failed` attempt is terminal and never replaced. pub fn select_cell_result(attempts: &[V1CellAttempt]) -> Option<&V1CellAttempt> { - let mut sorted: Vec<&V1CellAttempt> = attempts.iter().filter(|a| a.sequence >= 1).collect(); - sorted.sort_by_key(|a| a.sequence); - sorted - .into_iter() - .find(|attempt| !is_replaceable_attempt(attempt)) + attempts + .iter() + .filter(|a| a.sequence >= 1 && a.result.status.is_selectable()) + .min_by_key(|a| a.sequence) } -pub fn decode_eval_result(value: &Value, where_label: &str) -> Result { +fn decode_eval_result(value: &Value, where_label: &str) -> Result { let obj = exact_object( value, where_label, @@ -477,7 +505,7 @@ pub fn decode_eval_result(value: &Value, where_label: &str) -> Result EvalResultStatus::InfraFailed, Some("indeterminate") => EvalResultStatus::Indeterminate, _ => { - return Err(ContractError::Invalid(format!( + return Err(ContractError::new(format!( "{where_label}.status is invalid" ))); } @@ -486,7 +514,7 @@ pub fn decode_eval_result(value: &Value, where_label: &str) -> Result None, Value::String(s) => Some(s.clone()), _ => { - return Err(ContractError::Invalid(format!( + return Err(ContractError::new(format!( "{where_label}.failureReason is invalid" ))); } @@ -494,7 +522,7 @@ pub fn decode_eval_result(value: &Value, where_label: &str) -> Result None, - u => Some(NormalizedUsage::decode(u, &format!("{where_label}.usage"))?), + u => Some(decode_usage(u, &format!("{where_label}.usage"))?), }; let cost_usd = nullable_nonnegative_f64( obj.get("costUsd").unwrap(), @@ -507,10 +535,10 @@ pub fn decode_eval_result(value: &Value, where_label: &str) -> Result Result Result { +fn decode_v1_cell_attempt(value: &Value) -> Result { let obj = exact_object( value, "attempt", @@ -542,18 +570,6 @@ pub fn decode_v1_cell_attempt(value: &Value) -> Result Self { - Self::Recorded { - truncated: false, - policy_error_count: 0, - } - } - - pub fn truncated() -> Self { - Self::Recorded { - truncated: true, - policy_error_count: 0, - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum CriticalArtifactsState { @@ -646,6 +646,7 @@ pub enum CriticalArtifactsState { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde( + remote = "Self", tag = "kind", rename_all = "snake_case", rename_all_fields = "camelCase", @@ -661,6 +662,20 @@ pub enum PricingEvidence { }, } +impl Serialize for PricingEvidence { + fn serialize(&self, serializer: S) -> Result { + Self::serialize(self, serializer) + } +} + +impl<'de> Deserialize<'de> for PricingEvidence { + fn deserialize>(deserializer: D) -> Result { + let value = Self::deserialize(deserializer)?; + value.validate().map_err(serde::de::Error::custom)?; + Ok(value) + } +} + impl PricingEvidence { pub fn cost_usd(&self) -> Option { match self { @@ -673,22 +688,20 @@ impl PricingEvidence { match self { Self::Model { model, cost_usd } => { if model.is_empty() { - return Err(ContractError::Invalid( - "pricing.model must not be empty".into(), - )); + return Err(ContractError::new("pricing.model must not be empty")); } if let Some(c) = cost_usd && (!c.is_finite() || *c < 0.0) { - return Err(ContractError::Invalid( - "pricing.costUsd must be finite and non-negative".into(), + return Err(ContractError::new( + "pricing.costUsd must be finite and non-negative", )); } } Self::SubjectReported { cost_usd } => { if !cost_usd.is_finite() || *cost_usd < 0.0 { - return Err(ContractError::Invalid( - "pricing.costUsd must be finite and non-negative".into(), + return Err(ContractError::new( + "pricing.costUsd must be finite and non-negative", )); } } @@ -737,7 +750,7 @@ impl<'de> Deserialize<'de> for AttemptMetering { let obj = exact_object(&value, "metering", &["state", "usage"], &[]) .map_err(serde::de::Error::custom)?; Self::LowerBound { - usage: serde_json::from_value(obj.get("usage").unwrap().clone()) + usage: decode_usage(obj.get("usage").unwrap(), "metering.usage") .map_err(serde::de::Error::custom)?, } } @@ -751,7 +764,7 @@ impl<'de> Deserialize<'de> for AttemptMetering { } }; Self::Complete { - usage: serde_json::from_value(obj.get("usage").unwrap().clone()) + usage: decode_usage(obj.get("usage").unwrap(), "metering.usage") .map_err(serde::de::Error::custom)?, pricing, } @@ -764,14 +777,6 @@ impl<'de> Deserialize<'de> for AttemptMetering { } impl AttemptMetering { - pub fn state(&self) -> MeteringState { - match self { - Self::Unknown => MeteringState::Unknown, - Self::LowerBound { .. } => MeteringState::LowerBound, - Self::Complete { .. } => MeteringState::Complete, - } - } - pub fn usage(&self) -> Option<&NormalizedUsage> { match self { Self::Unknown => None, @@ -779,17 +784,16 @@ impl AttemptMetering { } } - pub fn pricing(&self) -> Option<&PricingEvidence> { + pub fn cost_usd(&self) -> Option { match self { - Self::Complete { pricing, .. } => pricing.as_ref(), - Self::Unknown | Self::LowerBound { .. } => None, + Self::Complete { + pricing: Some(pricing), + .. + } => pricing.cost_usd(), + Self::Unknown | Self::LowerBound { .. } | Self::Complete { pricing: None, .. } => None, } } - pub fn cost_usd(&self) -> Option { - self.pricing().and_then(PricingEvidence::cost_usd) - } - pub fn validate(&self) -> Result<(), ContractError> { if let Self::Complete { pricing: Some(p), .. @@ -801,8 +805,8 @@ impl AttemptMetering { } } -#[derive(Debug, Clone, PartialEq, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(remote = "Self", rename_all = "camelCase", deny_unknown_fields)] pub struct V2AttemptRecord { pub schema_version: SchemaVersion, pub cell_id: String, @@ -810,7 +814,6 @@ pub struct V2AttemptRecord { pub started_at: u64, pub completed_at: u64, pub duration_ms: u64, - pub phase: AttemptPhase, pub execution_outcome: ExecutionOutcome, pub verifier_outcome: VerifierOutcome, pub metering: AttemptMetering, @@ -824,362 +827,187 @@ pub struct V2AttemptRecord { pub artifacts: Vec, } -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct RawV2AttemptRecord { - schema_version: SchemaVersion, - cell_id: String, - sequence: u32, - started_at: u64, - completed_at: u64, - duration_ms: u64, - phase: AttemptPhase, - execution_outcome: ExecutionOutcome, - verifier_outcome: VerifierOutcome, - metering: AttemptMetering, - cleanup_state: CleanupState, - audit_state: EgressAuditState, - critical_artifacts_state: CriticalArtifactsState, - #[serde(default)] - cancellation: AttemptCancellation, - recovered_post_hoc: bool, - failure_reason: Option, - artifacts: Vec, +impl Serialize for V2AttemptRecord { + fn serialize(&self, serializer: S) -> Result { + Self::serialize(self, serializer) + } } impl<'de> Deserialize<'de> for V2AttemptRecord { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let raw = RawV2AttemptRecord::deserialize(deserializer)?; - let record = Self { - schema_version: raw.schema_version, - cell_id: raw.cell_id, - sequence: raw.sequence, - started_at: raw.started_at, - completed_at: raw.completed_at, - duration_ms: raw.duration_ms, - phase: raw.phase, - execution_outcome: raw.execution_outcome, - verifier_outcome: raw.verifier_outcome, - metering: raw.metering, - cleanup_state: raw.cleanup_state, - audit_state: raw.audit_state, - critical_artifacts_state: raw.critical_artifacts_state, - cancellation: raw.cancellation, - recovered_post_hoc: raw.recovered_post_hoc, - failure_reason: raw.failure_reason, - artifacts: raw.artifacts, - }; + fn deserialize>(deserializer: D) -> Result { + let record = Self::deserialize(deserializer)?; record.validate().map_err(serde::de::Error::custom)?; Ok(record) } } impl V2AttemptRecord { - fn is_completed(&self) -> bool { - matches!(self.cleanup_state, CleanupState::Confirmed) - && self.cancellation != AttemptCancellation::BeforeVerification - && matches!( - self.critical_artifacts_state, - CriticalArtifactsState::Present - ) - && matches!( - self.audit_state, - EgressAuditState::NotRequired | EgressAuditState::Recorded { .. } - ) - && self.execution_outcome == ExecutionOutcome::Completed - && matches!( - self.verifier_outcome, - VerifierOutcome::Verified { .. } | VerifierOutcome::Skipped - ) - } - pub fn validate(&self) -> Result<(), ContractError> { if self.schema_version != SchemaVersion::V2 { - return Err(ContractError::Invalid( - "v2 attempt record requires schemaVersion maka.eval.v2".into(), + return Err(ContractError::new( + "v2 attempt record requires schemaVersion maka.eval.v2", )); } if self.cell_id.is_empty() { - return Err(ContractError::Invalid( - "v2 attempt record cellId is required".into(), - )); + return Err(ContractError::new("v2 attempt record cellId is required")); } if self.sequence < 1 { - return Err(ContractError::Invalid( - "v2 attempt record sequence must be positive".into(), + return Err(ContractError::new( + "v2 attempt record sequence must be positive", )); } match &self.verifier_outcome { VerifierOutcome::Verified { score } if !score.is_finite() => { - return Err(ContractError::Invalid( - "verifier score must be finite".into(), - )); + return Err(ContractError::new("verifier score must be finite")); } VerifierOutcome::SubjectFailed { score: Some(score), .. } if !score.is_finite() => { - return Err(ContractError::Invalid( - "verifier score must be finite".into(), - )); + return Err(ContractError::new("verifier score must be finite")); } _ => {} } - if self.is_completed() && self.failure_reason.is_some() { - return Err(ContractError::Invalid( - "v2 completed attempt record must not set failureReason".into(), + if self.resolve_outcome().0 == EvalResultStatus::Completed && self.failure_reason.is_some() + { + return Err(ContractError::new( + "v2 completed attempt record must not set failureReason", )); } self.metering.validate()?; Ok(()) } - /// Projects a native `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt` - /// while enforcing all 5 Semantic Difference Registry adjudications. - pub fn project_v1(&self) -> V1CellAttempt { - let mut artifacts = self.artifacts.clone(); - - let usage = self.metering.usage().cloned(); - let cost_usd = self.metering.cost_usd(); - - if self.cancellation == AttemptCancellation::AfterVerification - && !artifacts.iter().any(|a| { - a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification") - }) - { - artifacts.push( - serde_json::json!({"kind": "cancelled-after-verification"}) - .as_object() - .unwrap() - .clone(), - ); - } - - if self.recovered_post_hoc - && !artifacts.iter().any(|a| { - a.get("kind").and_then(Value::as_str) == Some("post-hoc-evidence-recovery") - }) - { - let mut marker = Map::new(); - marker.insert( - "kind".into(), - Value::String("post-hoc-evidence-recovery".into()), - ); - artifacts.push(marker); - } - + fn resolve_outcome(&self) -> (EvalResultStatus, Option, Option) { // Unconfirmed cleanup always resolves to `indeterminate` with `score: null`. if let CleanupState::Unconfirmed { reason } = &self.cleanup_state { - return V1CellAttempt { - cell_id: self.cell_id.clone(), - sequence: self.sequence, - started_at: self.started_at, - completed_at: self.completed_at, - result: EvalResult { - score: None, - usage, - cost_usd, - duration_ms: self.duration_ms, - status: EvalResultStatus::Indeterminate, - failure_reason: Some(reason.clone()), - artifacts, - }, - }; + return (EvalResultStatus::Indeterminate, None, Some(reason.clone())); } - // Adjudication #4: Late cancellation BEFORE verification completes resolves to - // `indeterminate`, whereas late cancellation AFTER verification and settlement - // preserves the verified result and records the late cancellation signal. + // Late cancellation before verification completes resolves to `indeterminate`. if self.cancellation == AttemptCancellation::BeforeVerification { - return V1CellAttempt { - cell_id: self.cell_id.clone(), - sequence: self.sequence, - started_at: self.started_at, - completed_at: self.completed_at, - result: EvalResult { - score: None, - usage, - cost_usd, - duration_ms: self.duration_ms, - status: EvalResultStatus::Indeterminate, - failure_reason: Some(self.failure_reason.clone().unwrap_or_else(|| { + return ( + EvalResultStatus::Indeterminate, + None, + Some( + self.failure_reason.clone().unwrap_or_else(|| { "executor cancelled before verification completed".into() - })), - artifacts, - }, - }; + }), + ), + ); } - // Adjudication #3: Missing or unreadable critical artifacts downgrade to `infra_failed`. if let CriticalArtifactsState::Missing { paths } = &self.critical_artifacts_state { - return V1CellAttempt { - cell_id: self.cell_id.clone(), - sequence: self.sequence, - started_at: self.started_at, - completed_at: self.completed_at, - result: EvalResult { - score: None, - usage, - cost_usd, - duration_ms: self.duration_ms, - status: EvalResultStatus::InfraFailed, - failure_reason: Some(format!( - "critical artifacts missing: {}", - paths.join(", ") - )), - artifacts, - }, - }; + return ( + EvalResultStatus::InfraFailed, + None, + Some(format!("critical artifacts missing: {}", paths.join(", "))), + ); } - // Adjudication #2: Egress audit missing or unreadable fails closed (`infra_failed`), - // whereas `Recorded { truncated: true, .. }` (`ruleId == "audit_truncated"`) is preserved - // as a structured artifact flag without discarding a settled verification result. match &self.audit_state { EgressAuditState::Missing => { - return V1CellAttempt { - cell_id: self.cell_id.clone(), - sequence: self.sequence, - started_at: self.started_at, - completed_at: self.completed_at, - result: EvalResult { - score: None, - usage, - cost_usd, - duration_ms: self.duration_ms, - status: EvalResultStatus::InfraFailed, - failure_reason: Some("egress audit log missing".into()), - artifacts, - }, - }; + return ( + EvalResultStatus::InfraFailed, + None, + Some("egress audit log missing".into()), + ); } EgressAuditState::Unreadable { reason } => { - return V1CellAttempt { - cell_id: self.cell_id.clone(), - sequence: self.sequence, - started_at: self.started_at, - completed_at: self.completed_at, - result: EvalResult { - score: None, - usage, - cost_usd, - duration_ms: self.duration_ms, - status: EvalResultStatus::InfraFailed, - failure_reason: Some(reason.clone()), - artifacts, - }, - }; - } - EgressAuditState::Recorded { - truncated, - policy_error_count, - } => { - if *truncated || *policy_error_count > 0 { - if let Some(existing) = artifacts - .iter_mut() - .find(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) - { - if *truncated { - existing.insert("truncated".into(), Value::Bool(true)); - } - if *policy_error_count > 0 { - existing.insert( - "policyErrorCount".into(), - Value::from(*policy_error_count), - ); - } - } else { - let mut audit_obj = Map::new(); - audit_obj.insert("kind".into(), Value::String("egress-audit".into())); - audit_obj.insert("truncated".into(), Value::Bool(*truncated)); - if *policy_error_count > 0 { - audit_obj.insert( - "policyErrorCount".into(), - Value::from(*policy_error_count), - ); - } - artifacts.push(audit_obj); - } - } + return (EvalResultStatus::InfraFailed, None, Some(reason.clone())); } - EgressAuditState::NotRequired => {} + EgressAuditState::NotRequired | EgressAuditState::Recorded { .. } => {} } - let (status, score, failure_reason) = - match (&self.execution_outcome, &self.verifier_outcome) { - (ExecutionOutcome::InfraFailed, _) => ( - EvalResultStatus::InfraFailed, - None, - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject infrastructure failed".into()), - ), - ), - (ExecutionOutcome::Indeterminate, _) => ( - EvalResultStatus::Indeterminate, - None, - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject execution indeterminate".into()), - ), + match (&self.execution_outcome, &self.verifier_outcome) { + (ExecutionOutcome::InfraFailed, _) => ( + EvalResultStatus::InfraFailed, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject infrastructure failed".into()), ), - (_, VerifierOutcome::VerifierInfraFailed { reason }) - | (_, VerifierOutcome::MissingReward { reason }) => { - // Missing verifier reward never defaults to 0.0! - (EvalResultStatus::InfraFailed, None, Some(reason.clone())) - } - (ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }) => ( - // Adjudication #1: Agent timed out, verifier still ran and produced a valid - // reward; v1 status is `subject_failed` with the verified `score` preserved. - EvalResultStatus::SubjectFailed, - Some(*score), - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject exceeded the framework timeout".into()), - ), - ), - (ExecutionOutcome::SubjectFailed, VerifierOutcome::Verified { score }) => ( - EvalResultStatus::SubjectFailed, - Some(*score), - self.failure_reason.clone(), - ), - ( - ExecutionOutcome::Completed - | ExecutionOutcome::SubjectFailed - | ExecutionOutcome::Timeout, - VerifierOutcome::SubjectFailed { score, reason }, - ) => ( - EvalResultStatus::SubjectFailed, - *score, - self.failure_reason.clone().or_else(|| reason.clone()), + ), + (ExecutionOutcome::Indeterminate, _) => ( + EvalResultStatus::Indeterminate, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject execution indeterminate".into()), ), - (ExecutionOutcome::Completed, VerifierOutcome::Verified { score }) => { - (EvalResultStatus::Completed, Some(*score), None) - } - (ExecutionOutcome::Timeout, VerifierOutcome::Skipped) => ( - EvalResultStatus::SubjectFailed, - None, - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject exceeded the framework timeout".into()), - ), + ), + (_, VerifierOutcome::VerifierInfraFailed { reason }) + | (_, VerifierOutcome::MissingReward { reason }) => { + // Missing verifier reward never defaults to 0.0! + (EvalResultStatus::InfraFailed, None, Some(reason.clone())) + } + (ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }) => ( + EvalResultStatus::SubjectFailed, + Some(*score), + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), ), - (ExecutionOutcome::SubjectFailed, VerifierOutcome::Skipped) => ( - EvalResultStatus::SubjectFailed, - None, - self.failure_reason.clone(), + ), + (ExecutionOutcome::SubjectFailed, VerifierOutcome::Verified { score }) => ( + EvalResultStatus::SubjectFailed, + Some(*score), + self.failure_reason.clone(), + ), + ( + ExecutionOutcome::Completed + | ExecutionOutcome::SubjectFailed + | ExecutionOutcome::Timeout, + VerifierOutcome::SubjectFailed { score, reason }, + ) => ( + EvalResultStatus::SubjectFailed, + *score, + self.failure_reason.clone().or_else(|| reason.clone()), + ), + (ExecutionOutcome::Completed, VerifierOutcome::Verified { score }) => { + (EvalResultStatus::Completed, Some(*score), None) + } + (ExecutionOutcome::Timeout, VerifierOutcome::Skipped) => ( + EvalResultStatus::SubjectFailed, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), ), - (ExecutionOutcome::Completed, VerifierOutcome::Skipped) => { - (EvalResultStatus::Completed, None, None) - } - }; + ), + (ExecutionOutcome::SubjectFailed, VerifierOutcome::Skipped) => ( + EvalResultStatus::SubjectFailed, + None, + self.failure_reason.clone(), + ), + (ExecutionOutcome::Completed, VerifierOutcome::Skipped) => { + (EvalResultStatus::Completed, None, None) + } + } + } + + /// Projects a native `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt`. + pub fn project_v1(&self) -> V1CellAttempt { + let mut artifacts = self.artifacts.clone(); + if self.cancellation == AttemptCancellation::AfterVerification { + push_artifact_marker(&mut artifacts, "cancelled-after-verification"); + } + if self.recovered_post_hoc { + push_artifact_marker(&mut artifacts, "post-hoc-evidence-recovery"); + } + if let EgressAuditState::Recorded { + truncated, + policy_error_count, + } = &self.audit_state + && (*truncated || *policy_error_count > 0) + { + record_audit_artifact(&mut artifacts, *truncated, *policy_error_count); + } + let (status, score, failure_reason) = self.resolve_outcome(); V1CellAttempt { cell_id: self.cell_id.clone(), sequence: self.sequence, @@ -1187,8 +1015,8 @@ impl V2AttemptRecord { completed_at: self.completed_at, result: EvalResult { score, - usage, - cost_usd, + usage: self.metering.usage().cloned(), + cost_usd: self.metering.cost_usd(), duration_ms: self.duration_ms, status, failure_reason, @@ -1198,100 +1026,41 @@ impl V2AttemptRecord { } } -/// The 5 explicit semantic difference scenarios between Legacy TS `v1` and Native Rust `v2`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum DifferenceScenarioId { - TimeoutWithRewardIncompleteUsage, - AuditTruncation, - MissingCriticalArtifacts, - LateCancellation, - EvidenceRecoveredLater, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DifferenceAdjudication { - pub id: DifferenceScenarioId, - pub title: String, - pub legacy_v1_behavior: String, - pub rust_v2_behavior: String, - pub v1_projection_rule: String, - pub conditions_comparable: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DifferenceRegistry { - entries: Vec, -} - -impl Default for DifferenceRegistry { - fn default() -> Self { - Self::new() +fn push_artifact_marker(artifacts: &mut Vec, kind: &str) { + if !artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some(kind)) + { + let mut marker = Map::new(); + marker.insert("kind".into(), Value::String(kind.into())); + artifacts.push(marker); } } -impl DifferenceRegistry { - pub fn new() -> Self { - Self { - entries: vec![ - DifferenceAdjudication { - id: DifferenceScenarioId::TimeoutWithRewardIncompleteUsage, - title: "Timeout with verifier reward but incomplete usage".into(), - legacy_v1_behavior: "Projects status='subject_failed', preserves verifier score, records recovered lower-bound usage, and sets costUsd=null because metering did not settle.".into(), - rust_v2_behavior: "Records ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }, and MeteringState::LowerBound orthogonally without conflating execution timeout and metering completeness.".into(), - v1_projection_rule: "Projects status='subject_failed', score=Some(score), usage=Some(lower_bound_usage), costUsd=None.".into(), - conditions_comparable: true, - }, - DifferenceAdjudication { - id: DifferenceScenarioId::AuditTruncation, - title: "Egress audit log truncation at byte cap".into(), - legacy_v1_behavior: "Appends ruleId='audit_truncated' marker to hits.jsonl and sets truncated=true in the egress-audit artifact while keeping the attempt status unchanged.".into(), - rust_v2_behavior: "Captures EgressAuditState::Recorded { truncated, policy_error_count } as an explicit dimension while distinguishing it from Missing or Unreadable audit logs (which fail closed as infra_failed).".into(), - v1_projection_rule: "Preserves completed/subject_failed status and emits the egress-audit artifact with truncated=true and policyErrorCount when nonzero; Missing or Unreadable audit logs project to status='infra_failed'.".into(), - conditions_comparable: true, - }, - DifferenceAdjudication { - id: DifferenceScenarioId::MissingCriticalArtifacts, - title: "0-byte empty file vs. missing or unreadable critical artifact".into(), - legacy_v1_behavior: "Treats a present 0-byte regular file (e.g. empty stderr) as valid with bytes=0, while missing/unreadable required evidence fails the attempt.".into(), - rust_v2_behavior: "Tracks CriticalArtifactsState::Present (including 0-byte regular files verified by inode/stat) separately from CriticalArtifactsState::Missing.".into(), - v1_projection_rule: "0-byte regular files remain valid artifacts with bytes=0; Missing critical artifacts project to status='infra_failed' with score=None.".into(), - conditions_comparable: true, - }, - DifferenceAdjudication { - id: DifferenceScenarioId::LateCancellation, - title: "Cancellation signal before vs. after verification and settlement".into(), - legacy_v1_behavior: "An abort signal observed before attempt append could overwrite a verified result with indeterminate.".into(), - rust_v2_behavior: "Separates AttemptCancellation::BeforeVerification from AttemptCancellation::AfterVerification orthogonally from ExecutionOutcome.".into(), - v1_projection_rule: "Pre-verification cancellation projects to status='indeterminate' with score=None; post-verification cancellation preserves the verified terminal result and emits cancelled-after-verification.".into(), - conditions_comparable: true, - }, - DifferenceAdjudication { - id: DifferenceScenarioId::EvidenceRecoveredLater, - title: "Post-hoc evidence recovered from checkpoint or host journal after crash".into(), - legacy_v1_behavior: "Mutated or re-ran the attempt when the process died before writing the final cell JSON.".into(), - rust_v2_behavior: "Appends a PostHocRecovery journal entry referencing the original attempt sequence and reconstructs V2AttemptRecord with AttemptMetering and recovered_post_hoc=true without mutating prior entries.".into(), - v1_projection_rule: "Projects recovered usage/cost and appends a post-hoc-evidence-recovery artifact while preserving immutable sequence numbering.".into(), - conditions_comparable: true, - }, - ], +fn record_audit_artifact( + artifacts: &mut Vec, + truncated: bool, + policy_error_count: u64, +) { + if let Some(existing) = artifacts + .iter_mut() + .find(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) + { + if truncated { + existing.insert("truncated".into(), Value::Bool(true)); } + if policy_error_count > 0 { + existing.insert("policyErrorCount".into(), Value::from(policy_error_count)); + } + } else { + let mut audit_obj = Map::new(); + audit_obj.insert("kind".into(), Value::String("egress-audit".into())); + audit_obj.insert("truncated".into(), Value::Bool(truncated)); + if policy_error_count > 0 { + audit_obj.insert("policyErrorCount".into(), Value::from(policy_error_count)); + } + artifacts.push(audit_obj); } - - pub fn entries(&self) -> &[DifferenceAdjudication] { - &self.entries - } - - pub fn get(&self, id: DifferenceScenarioId) -> Option<&DifferenceAdjudication> { - self.entries.iter().find(|entry| entry.id == id) - } -} - -pub fn sha256_hex(bytes: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(bytes); - hex::encode(hasher.finalize()) } pub(crate) fn exact_object<'a>( @@ -1302,17 +1071,17 @@ pub(crate) fn exact_object<'a>( ) -> Result<&'a JsonObject, ContractError> { let obj = value .as_object() - .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + .ok_or_else(|| ContractError::new(format!("{where_label} must be an object")))?; for key in obj.keys() { if !required.contains(&key.as_str()) && !optional.contains(&key.as_str()) { - return Err(ContractError::Invalid(format!( + return Err(ContractError::new(format!( "{where_label} has unsupported fields" ))); } } for req in required { if !obj.contains_key(*req) { - return Err(ContractError::Invalid(format!( + return Err(ContractError::new(format!( "{where_label}.{req} is required" ))); } @@ -1320,77 +1089,44 @@ pub(crate) fn exact_object<'a>( Ok(obj) } -pub(crate) fn decode_json_object( - value: &Value, - where_label: &str, -) -> Result { - let obj = value +fn json_object(value: &Value, where_label: &str) -> Result { + value .as_object() - .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; - for (k, v) in obj { - validate_json_value(v, &format!("{where_label}.{k}"))?; - } - Ok(obj.clone()) -} - -fn validate_json_value(value: &Value, where_label: &str) -> Result<(), ContractError> { - match value { - Value::Null | Value::Bool(_) | Value::String(_) => Ok(()), - Value::Number(n) => { - if n.as_f64().is_some_and(f64::is_finite) { - Ok(()) - } else { - Err(ContractError::Invalid(format!( - "{where_label} must be JSON" - ))) - } - } - Value::Array(arr) => { - for (idx, item) in arr.iter().enumerate() { - validate_json_value(item, &format!("{where_label}[{idx}]"))?; - } - Ok(()) - } - Value::Object(map) => { - for (k, v) in map { - validate_json_value(v, &format!("{where_label}.{k}"))?; - } - Ok(()) - } - } + .cloned() + .ok_or_else(|| ContractError::new(format!("{where_label} must be an object"))) } -pub(crate) fn identifier(value: &Value, where_label: &str) -> Result { +fn identifier(value: &Value, where_label: &str) -> Result { let text = nonempty_string(value, where_label)?; let mut chars = text.chars(); let Some(first) = chars.next() else { - return Err(ContractError::Invalid(format!("{where_label} is required"))); + return Err(ContractError::new(format!("{where_label} is required"))); }; if !first.is_ascii_alphanumeric() || !chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) { - return Err(ContractError::Invalid(format!("{where_label} is invalid"))); + return Err(ContractError::new(format!("{where_label} is invalid"))); } Ok(text) } -pub(crate) fn nonempty_string(value: &Value, where_label: &str) -> Result { +fn nonempty_string(value: &Value, where_label: &str) -> Result { match value.as_str() { Some(s) if !s.is_empty() => Ok(s.to_owned()), - _ => Err(ContractError::Invalid(format!("{where_label} is required"))), + _ => Err(ContractError::new(format!("{where_label} is required"))), } } fn unique_strings(value: &Value, where_label: &str) -> Result, ContractError> { let arr = value .as_array() - .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an array")))?; + .ok_or_else(|| ContractError::new(format!("{where_label} must be an array")))?; let mut seen = BTreeSet::new(); let mut out = Vec::with_capacity(arr.len()); for item in arr { let s = nonempty_string(item, where_label)?; if !seen.insert(s.clone()) { - return Err(ContractError::Invalid(format!( + return Err(ContractError::new(format!( "{where_label} contains duplicates" ))); } @@ -1406,26 +1142,26 @@ fn ensure_unique_ids<'a>( let mut seen = BTreeSet::new(); for id in ids { if !seen.insert(id) { - return Err(ContractError::Invalid(format!("duplicate {label} id"))); + return Err(ContractError::new(format!("duplicate {label} id"))); } } Ok(()) } -pub(crate) fn positive_u32(value: &Value, where_label: &str) -> Result { +fn positive_u32(value: &Value, where_label: &str) -> Result { match value.as_u64() { Some(n) if n >= 1 && n <= u64::from(u32::MAX) => Ok(n as u32), - _ => Err(ContractError::Invalid(format!( + _ => Err(ContractError::new(format!( "{where_label} must be positive" ))), } } -pub(crate) fn positive_usize(value: &Value, where_label: &str) -> Result { +fn positive_usize(value: &Value, where_label: &str) -> Result { match value.as_u64() { Some(n) if n >= 1 => usize::try_from(n) - .map_err(|_| ContractError::Invalid(format!("{where_label} must be positive"))), - _ => Err(ContractError::Invalid(format!( + .map_err(|_| ContractError::new(format!("{where_label} must be positive"))), + _ => Err(ContractError::new(format!( "{where_label} must be positive" ))), } @@ -1436,22 +1172,22 @@ pub(crate) fn nonnegative_u64(value: &Value, where_label: &str) -> Result Ok(n), None => match value.as_f64() { Some(f) if f.is_finite() && f >= 0.0 && f.fract() == 0.0 => Ok(f as u64), - _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + _ => Err(ContractError::new(format!("{where_label} is invalid"))), }, } } -pub(crate) fn nullable_f64(value: &Value, where_label: &str) -> Result, ContractError> { +fn nullable_f64(value: &Value, where_label: &str) -> Result, ContractError> { if value.is_null() { return Ok(None); } match value.as_f64() { Some(f) if f.is_finite() => Ok(Some(f)), - _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + _ => Err(ContractError::new(format!("{where_label} is invalid"))), } } -pub(crate) fn nullable_nonnegative_f64( +fn nullable_nonnegative_f64( value: &Value, where_label: &str, ) -> Result, ContractError> { @@ -1460,6 +1196,6 @@ pub(crate) fn nullable_nonnegative_f64( } match value.as_f64() { Some(f) if f.is_finite() && f >= 0.0 => Ok(Some(f)), - _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + _ => Err(ContractError::new(format!("{where_label} is invalid"))), } } diff --git a/products/eval/src/measurement.rs b/products/eval/src/measurement.rs index 93ef29a746..1a260c3c26 100644 --- a/products/eval/src/measurement.rs +++ b/products/eval/src/measurement.rs @@ -17,11 +17,10 @@ * under the License. */ -//! Exact 6-counter token usage preservation, metering state semantics, -//! and standardized model pricing. +//! Exact 6-counter token usage preservation and standardized model pricing. use crate::contract::{ContractError, exact_object, nonnegative_u64}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; pub const COST_USD_TOLERANCE: f64 = 1e-9; @@ -30,8 +29,8 @@ pub const COST_USD_TOLERANCE: f64 = 1e-9; /// /// Each counter is preserved as reported by the provider or receipt; `total_tokens` /// is never synthesized or overwritten by summing the other five counters. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] pub struct NormalizedUsage { pub input_tokens: u64, pub output_tokens: u64, @@ -41,49 +40,62 @@ pub struct NormalizedUsage { pub total_tokens: u64, } -impl NormalizedUsage { - pub fn decode(value: &Value, where_label: &str) -> Result { - let obj = exact_object( - value, - where_label, - &[ - "inputTokens", - "outputTokens", - "cacheReadTokens", - "cacheWriteTokens", - "reasoningTokens", - "totalTokens", - ], - &[], - )?; - Ok(Self { - input_tokens: nonnegative_u64( - obj.get("inputTokens").unwrap(), - &format!("{where_label}.inputTokens"), - )?, - output_tokens: nonnegative_u64( - obj.get("outputTokens").unwrap(), - &format!("{where_label}.outputTokens"), - )?, - cache_read_tokens: nonnegative_u64( - obj.get("cacheReadTokens").unwrap(), - &format!("{where_label}.cacheReadTokens"), - )?, - cache_write_tokens: nonnegative_u64( - obj.get("cacheWriteTokens").unwrap(), - &format!("{where_label}.cacheWriteTokens"), - )?, - reasoning_tokens: nonnegative_u64( - obj.get("reasoningTokens").unwrap(), - &format!("{where_label}.reasoningTokens"), - )?, - total_tokens: nonnegative_u64( - obj.get("totalTokens").unwrap(), - &format!("{where_label}.totalTokens"), - )?, - }) +impl<'de> Deserialize<'de> for NormalizedUsage { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + decode_usage(&value, "usage").map_err(serde::de::Error::custom) } +} + +pub(crate) fn decode_usage( + value: &Value, + where_label: &str, +) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "inputTokens", + "outputTokens", + "cacheReadTokens", + "cacheWriteTokens", + "reasoningTokens", + "totalTokens", + ], + &[], + )?; + Ok(NormalizedUsage { + input_tokens: nonnegative_u64( + obj.get("inputTokens").unwrap(), + &format!("{where_label}.inputTokens"), + )?, + output_tokens: nonnegative_u64( + obj.get("outputTokens").unwrap(), + &format!("{where_label}.outputTokens"), + )?, + cache_read_tokens: nonnegative_u64( + obj.get("cacheReadTokens").unwrap(), + &format!("{where_label}.cacheReadTokens"), + )?, + cache_write_tokens: nonnegative_u64( + obj.get("cacheWriteTokens").unwrap(), + &format!("{where_label}.cacheWriteTokens"), + )?, + reasoning_tokens: nonnegative_u64( + obj.get("reasoningTokens").unwrap(), + &format!("{where_label}.reasoningTokens"), + )?, + total_tokens: nonnegative_u64( + obj.get("totalTokens").unwrap(), + &format!("{where_label}.totalTokens"), + )?, + }) +} +impl NormalizedUsage { /// Accumulates another `NormalizedUsage` record counter-by-counter without /// synthesizing `total_tokens` from the component fields. pub fn accumulate_exact(&mut self, other: &Self) { @@ -98,50 +110,6 @@ impl NormalizedUsage { self.reasoning_tokens = self.reasoning_tokens.saturating_add(other.reasoning_tokens); self.total_tokens = self.total_tokens.saturating_add(other.total_tokens); } - - pub fn max_by_counter(&self, other: &Self) -> Self { - Self { - input_tokens: self.input_tokens.max(other.input_tokens), - output_tokens: self.output_tokens.max(other.output_tokens), - cache_read_tokens: self.cache_read_tokens.max(other.cache_read_tokens), - cache_write_tokens: self.cache_write_tokens.max(other.cache_write_tokens), - reasoning_tokens: self.reasoning_tokens.max(other.reasoning_tokens), - total_tokens: self.total_tokens.max(other.total_tokens), - } - } - - pub fn dominates(&self, other: &Self) -> bool { - self.input_tokens >= other.input_tokens - && self.output_tokens >= other.output_tokens - && self.cache_read_tokens >= other.cache_read_tokens - && self.cache_write_tokens >= other.cache_write_tokens - && self.reasoning_tokens >= other.reasoning_tokens - && self.total_tokens >= other.total_tokens - } - - pub fn billing_counters_eq(&self, other: &Self) -> bool { - self.input_tokens == other.input_tokens - && self.output_tokens == other.output_tokens - && self.cache_read_tokens == other.cache_read_tokens - && self.cache_write_tokens == other.cache_write_tokens - && self.reasoning_tokens == other.reasoning_tokens - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum MeteringState { - Complete, - LowerBound, - Unknown, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum TokenBasis { - #[serde(rename = "complete")] - Complete, - #[serde(rename = "lower-bound")] - LowerBound, } #[derive(Debug, Clone, Copy, PartialEq)] @@ -152,6 +120,20 @@ pub struct PricingRates { pub cache_write_per_million: f64, } +impl PricingRates { + pub fn cost_usd(&self, usage: &NormalizedUsage) -> f64 { + let cached = usage + .cache_read_tokens + .saturating_add(usage.cache_write_tokens); + let uncached = usage.input_tokens.saturating_sub(cached); + ((uncached as f64) * self.input_per_million + + (usage.cache_read_tokens as f64) * self.cache_read_per_million + + (usage.cache_write_tokens as f64) * self.cache_write_per_million + + (usage.output_tokens as f64) * self.output_per_million) + / 1_000_000.0 + } +} + /// Published DeepSeek V4 Flash prices in USD per million tokens (off-peak band). pub const DEEPSEEK_V4_FLASH_COST: PricingRates = PricingRates { input_per_million: 0.22, @@ -160,22 +142,6 @@ pub const DEEPSEEK_V4_FLASH_COST: PricingRates = PricingRates { cache_write_per_million: 0.22, }; -pub fn deepseek_cost_usd(usage: &NormalizedUsage) -> f64 { - price_usage_usd(usage, &DEEPSEEK_V4_FLASH_COST) -} - -pub fn price_usage_usd(usage: &NormalizedUsage, rates: &PricingRates) -> f64 { - let cached = usage - .cache_read_tokens - .saturating_add(usage.cache_write_tokens); - let uncached = usage.input_tokens.saturating_sub(cached); - ((uncached as f64) * rates.input_per_million - + (usage.cache_read_tokens as f64) * rates.cache_read_per_million - + (usage.cache_write_tokens as f64) * rates.cache_write_per_million - + (usage.output_tokens as f64) * rates.output_per_million) - / 1_000_000.0 -} - pub fn costs_match_within_tolerance(left: Option, right: Option) -> bool { match (left, right) { (None, None) => true, @@ -183,7 +149,3 @@ pub fn costs_match_within_tolerance(left: Option, right: Option) -> bo _ => false, } } - -pub fn default_model_pricing(model: &str) -> Option { - (model == "deepseek-v4-flash").then_some(DEEPSEEK_V4_FLASH_COST) -} diff --git a/products/eval/tests/integration/contract.rs b/products/eval/tests/contract.rs similarity index 71% rename from products/eval/tests/integration/contract.rs rename to products/eval/tests/contract.rs index db036b22eb..ba0c01b973 100644 --- a/products/eval/tests/integration/contract.rs +++ b/products/eval/tests/contract.rs @@ -17,21 +17,87 @@ * under the License. */ -use crate::support::sample_two_arm_spec_json; use maka_eval::contract::{ - AttemptCancellation, AttemptMetering, AttemptPhase, CleanupState, CriticalArtifactsState, - DifferenceRegistry, EgressAuditState, EvalResult, EvalResultStatus, ExecutionOutcome, - ExperimentSpec, PricingEvidence, RunFingerprint, SchemaVersion, V1CellAttempt, V2AttemptRecord, - VerifierOutcome, decode_v1_cell_attempt, expand_experiment, parse_experiment_spec, - select_cell_result, + AttemptCancellation, AttemptMetering, CleanupState, CriticalArtifactsState, EgressAuditState, + EvalResult, EvalResultStatus, ExecutionOutcome, ExperimentSpec, PricingEvidence, + RunFingerprint, SchemaVersion, V1CellAttempt, V2AttemptRecord, VerifierOutcome, + expand_experiment, select_cell_result, }; -use maka_eval::measurement::{NormalizedUsage, costs_match_within_tolerance, deepseek_cost_usd}; -use serde_json::{Value, json}; +use maka_eval::measurement::{ + DEEPSEEK_V4_FLASH_COST, NormalizedUsage, costs_match_within_tolerance, +}; +use serde_json::{Map, Value, json}; + +fn sample_two_arm_spec_json() -> Value { + json!({ + "schemaVersion": "maka.eval.v1", + "id": "tb21-deepseek-v4-flash-two-arm", + "benchmark": { + "id": "terminal-bench-2.1", + "version": "d49e28f1e4ddd13d289e85a5f312a66750951932", + "config": { + "repository": "https://github.com/harbor-framework/terminal-bench-2-1" + } + }, + "executor": { + "kind": "harbor", + "config": { + "frameworkVersion": "0.20.0", + "pythonPathEnv": "MAKA_EVAL_HARBOR_PYTHON", + "trialsRootEnv": "MAKA_EVAL_TRIALS_DIR", + "environment": { "type": "docker", "delete": true } + } + }, + "execution": { + "maxConcurrentTaskGroups": 2 + }, + "subjects": [ + { + "id": "maka", + "kind": "maka", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "model": "deepseek-v4-flash", + "sandboxMode": "workspace-write" + } + }, + { + "id": "external-cli", + "kind": "external", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "command": "/opt/maka-node-toolchain/bin/node", + "args": ["harbor-external-subject.js", "{{task.input}}"] + } + } + ], + "tasks": [ + { + "id": "task-alpha", + "input": "Fix alpha", + "config": { "harbor": { "path": "tasks/task-alpha" } } + }, + { + "id": "task-beta", + "input": "Fix beta", + "config": { "harbor": { "path": "tasks/task-beta" } } + } + ], + "repetitions": 1, + "budget": { + "timeoutMultiplier": 1, + "maxSteps": 100 + }, + "verifier": { + "reward": "reward" + } + }) +} #[test] fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { let spec_json = sample_two_arm_spec_json(); - let spec = parse_experiment_spec(&spec_json).unwrap(); + let spec: ExperimentSpec = serde_json::from_value(spec_json.clone()).unwrap(); let cells = expand_experiment(&spec); assert_eq!(cells.len(), 4); @@ -40,45 +106,47 @@ fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { assert_eq!(cells[2].id, "task-beta::1::maka"); assert_eq!(cells[3].id, "task-beta::1::external-cli"); - // Unknown top-level field must be rejected both by parse_experiment_spec and serde Deserialize. let mut invalid = spec_json.clone(); invalid .as_object_mut() .unwrap() .insert("unknownField".into(), json!(123)); - assert!(parse_experiment_spec(&invalid).is_err()); assert!(serde_json::from_value::(invalid).is_err()); // Concurrency changes execution fingerprint but preserves semantic fingerprint. let mut higher_concurrency_json = spec_json.clone(); higher_concurrency_json["execution"]["maxConcurrentTaskGroups"] = json!(16); - let higher_spec = parse_experiment_spec(&higher_concurrency_json).unwrap(); + let higher_spec: ExperimentSpec = serde_json::from_value(higher_concurrency_json).unwrap(); let fp1 = RunFingerprint::compute(&spec); let fp2 = RunFingerprint::compute(&higher_spec); - assert!(fp1.is_semantically_comparable(&fp2)); + assert_eq!(fp1.semantic_sha256, fp2.semantic_sha256); assert_ne!(fp1.execution_sha256, fp2.execution_sha256); - // Changing experiment.id or float vs integer representation (1.0 vs 1) preserves semantic fingerprint. - let mut renamed_json = spec_json; - renamed_json["id"] = json!("tb21-deepseek-v4-flash-rerun"); - renamed_json["budget"]["timeoutMultiplier"] = json!(1.0); - let renamed_spec = serde_json::from_value::(renamed_json).unwrap(); - let fp3 = RunFingerprint::compute(&renamed_spec); - assert!(fp1.is_semantically_comparable(&fp3)); + // Changing experiment.id, float/integer equivalence (1.0 vs 1), and nested object + // key insertion order all preserve semantic_sha256 even under preserve_order. + let mut reordered_json = spec_json; + reordered_json["id"] = json!("tb21-deepseek-v4-flash-rerun"); + let mut reordered_budget = Map::new(); + reordered_budget.insert("maxSteps".into(), json!(100)); + reordered_budget.insert("timeoutMultiplier".into(), json!(1.0)); + reordered_json["budget"] = Value::Object(reordered_budget); + let reordered_spec: ExperimentSpec = serde_json::from_value(reordered_json).unwrap(); + let fp3 = RunFingerprint::compute(&reordered_spec); + assert_eq!(fp1.semantic_sha256, fp3.semantic_sha256); } #[test] fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within_1e9() { - // Provider reports a totalTokens value distinct from the arithmetic sum of the 5 fields. - let u1 = NormalizedUsage { - input_tokens: 1000, - output_tokens: 250, - cache_read_tokens: 400, - cache_write_tokens: 100, - reasoning_tokens: 80, - total_tokens: 9999, - }; + let u1: NormalizedUsage = serde_json::from_value(json!({ + "inputTokens": 1000.0, + "outputTokens": 250, + "cacheReadTokens": 400, + "cacheWriteTokens": 100, + "reasoningTokens": 80, + "totalTokens": 9999 + })) + .unwrap(); let u2 = NormalizedUsage { input_tokens: 500, output_tokens: 150, @@ -87,6 +155,41 @@ fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within reasoning_tokens: 20, total_tokens: 7777, }; + + assert!( + serde_json::from_value::(json!({ + "inputTokens": 10, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0 + })) + .is_err() + ); + assert!( + serde_json::from_value::(json!({ + "inputTokens": -1, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 4 + })) + .is_err() + ); + assert!( + serde_json::from_value::(json!({ + "inputTokens": 10, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 15, + "extra": 1 + })) + .is_err() + ); + let mut combined = u1.clone(); combined.accumulate_exact(&u2); assert_eq!(combined.input_tokens, 1500); @@ -97,11 +200,9 @@ fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within // Must equal 9999 + 7777 = 17776, never the sum of the other 5 counters! assert_eq!(combined.total_tokens, 17776); - // Verify DEEPSEEK_V4_FLASH_COST formula: // uncached = 1000 - 400 - 100 = 500 - // (500 * 0.22 + 400 * 0.007 + 100 * 0.22 + 250 * 0.66) / 1_000_000 - // = (110 + 2.8 + 22 + 165) / 1_000_000 = 299.8 / 1_000_000 = 0.0002998 - let cost = deepseek_cost_usd(&u1); + // (500 * 0.22 + 400 * 0.007 + 100 * 0.22 + 250 * 0.66) / 1_000_000 = 0.0002998 + let cost = DEEPSEEK_V4_FLASH_COST.cost_usd(&u1); assert!(costs_match_within_tolerance(Some(cost), Some(0.0002998))); assert!(!costs_match_within_tolerance( Some(cost), @@ -112,6 +213,21 @@ fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within #[test] fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_terminal() { let attempts = vec![ + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 3, + started_at: 50, + completed_at: 60, + result: EvalResult { + score: Some(1.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, V1CellAttempt { cell_id: "t1::1::maka".into(), sequence: 1, @@ -142,33 +258,15 @@ fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_term artifacts: Vec::new(), }, }, - V1CellAttempt { - cell_id: "t1::1::maka".into(), - sequence: 3, - started_at: 50, - completed_at: 60, - result: EvalResult { - score: Some(1.0), - usage: None, - cost_usd: None, - duration_ms: 10, - status: EvalResultStatus::Completed, - failure_reason: None, - artifacts: Vec::new(), - }, - }, ]; let selected = select_cell_result(&attempts).unwrap(); - // Sequence 2 (Completed with score 0.0) is valid and earlier than sequence 3! assert_eq!(selected.sequence, 2); assert_eq!(selected.result.score, Some(0.0)); } #[test] fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly() { - // Frozen v1 accepts subject-reported costUsd with usage=null and completed with failureReason, - // while still rejecting sequence == 0 and negative costUsd. Stricter checks belong in v2. let valid_v1 = json!({ "cellId": "t1::1::maka", "sequence": 1, @@ -186,6 +284,13 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly }); assert!(serde_json::from_value::(valid_v1.clone()).is_ok()); + let mut omitted_score = valid_v1.clone(); + omitted_score["result"] + .as_object_mut() + .unwrap() + .remove("score"); + assert!(serde_json::from_value::(omitted_score).is_err()); + let mut zero_seq = valid_v1.clone(); zero_seq["sequence"] = json!(0); assert!(serde_json::from_value::(zero_seq).is_err()); @@ -203,7 +308,7 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly assert_eq!(parsed_cost_without_usage.result.usage, None); let encoded_cost_without_usage = serde_json::to_value(&parsed_cost_without_usage).unwrap(); assert_eq!( - decode_v1_cell_attempt(&encoded_cost_without_usage).unwrap(), + serde_json::from_value::(encoded_cost_without_usage).unwrap(), parsed_cost_without_usage ); @@ -229,12 +334,13 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly let encoded_completed_with_reason = serde_json::to_value(&parsed_completed_with_reason).unwrap(); assert_eq!( - decode_v1_cell_attempt(&encoded_completed_with_reason).unwrap(), + serde_json::from_value::(encoded_completed_with_reason).unwrap(), parsed_completed_with_reason ); // V2AttemptRecord rejects schemaVersion != maka.eval.v2, illegal metering states, - // unknown fields on closed tagged enums, and completed attempts with failureReason. + // unknown fields on metering unit variants, completed attempts with failureReason, + // and standalone invalid PricingEvidence. let v2_base = json!({ "schemaVersion": "maka.eval.v2", "cellId": "t1::1::maka", @@ -242,7 +348,6 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly "startedAt": 10, "completedAt": 20, "durationMs": 10, - "phase": "settled", "executionOutcome": "completed", "verifierOutcome": { "kind": "skipped" }, "metering": { "state": "unknown" }, @@ -300,17 +405,24 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly }); assert!(serde_json::from_value::(negative_cost).is_err()); - // Lossless V1 serde -> decode_v1_cell_attempt round-trip when SubjectFailed has failureReason: null. + assert!( + serde_json::from_value::(json!({ + "kind": "subject_reported", + "costUsd": -0.5 + })) + .is_err() + ); + let parsed_v1: V1CellAttempt = serde_json::from_value(valid_v1).unwrap(); let encoded = serde_json::to_value(&parsed_v1).unwrap(); - assert_eq!(decode_v1_cell_attempt(&encoded).unwrap(), parsed_v1); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + parsed_v1 + ); } #[test] fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { - let registry = DifferenceRegistry::new(); - assert_eq!(registry.entries().len(), 5); - let usage = NormalizedUsage { input_tokens: 100, output_tokens: 20, @@ -329,7 +441,6 @@ fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { started_at: 10, completed_at: 20, duration_ms: 10, - phase: AttemptPhase::Settled, execution_outcome: ExecutionOutcome::Timeout, verifier_outcome: VerifierOutcome::Verified { score: 0.75 }, metering: AttemptMetering::LowerBound { @@ -396,13 +507,16 @@ fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { execution_outcome: ExecutionOutcome::Completed, verifier_outcome: VerifierOutcome::Verified { score: 1.0 }, metering: AttemptMetering::Complete { - usage, + usage: usage.clone(), pricing: Some(PricingEvidence::Model { model: "deepseek-v4-flash".into(), cost_usd: Some(0.0000352), }), }, - audit_state: EgressAuditState::truncated(), + audit_state: EgressAuditState::Recorded { + truncated: true, + policy_error_count: 0, + }, failure_reason: None, ..base_v2 }; @@ -459,7 +573,24 @@ fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { .any(|a| a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification")) ); - // 5. Missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0). + // 5. Post-hoc recovery emits artifact marker while keeping recovered usage/cost and sequence; + // missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0). + let mut v2_recovered = completed_base.clone(); + v2_recovered.sequence = 2; + v2_recovered.recovered_post_hoc = true; + let v1_recovered = v2_recovered.project_v1(); + assert_eq!(v1_recovered.sequence, 2); + assert_eq!(v1_recovered.result.status, EvalResultStatus::Completed); + assert_eq!(v1_recovered.result.usage, Some(usage)); + assert_eq!(v1_recovered.result.cost_usd, Some(0.0000352)); + assert!( + v1_recovered + .result + .artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some("post-hoc-evidence-recovery")) + ); + let mut v2_missing_reward = completed_base.clone(); v2_missing_reward.verifier_outcome = VerifierOutcome::MissingReward { reason: "reward.json missing".into(), diff --git a/products/eval/tests/integration/main.rs b/products/eval/tests/integration/main.rs deleted file mode 100644 index ef12ed4834..0000000000 --- a/products/eval/tests/integration/main.rs +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -mod contract; -mod support; diff --git a/products/eval/tests/integration/support.rs b/products/eval/tests/integration/support.rs deleted file mode 100644 index 403516d73f..0000000000 --- a/products/eval/tests/integration/support.rs +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -use serde_json::{Value, json}; - -pub(crate) fn sample_two_arm_spec_json() -> Value { - json!({ - "schemaVersion": "maka.eval.v1", - "id": "tb21-deepseek-v4-flash-two-arm", - "benchmark": { - "id": "terminal-bench-2.1", - "version": "d49e28f1e4ddd13d289e85a5f312a66750951932", - "config": { - "repository": "https://github.com/harbor-framework/terminal-bench-2-1" - } - }, - "executor": { - "kind": "harbor", - "config": { - "frameworkVersion": "0.20.0", - "pythonPathEnv": "MAKA_EVAL_HARBOR_PYTHON", - "trialsRootEnv": "MAKA_EVAL_TRIALS_DIR", - "environment": { "type": "docker", "delete": true } - } - }, - "execution": { - "maxConcurrentTaskGroups": 2 - }, - "subjects": [ - { - "id": "maka", - "kind": "maka", - "credentials": ["DEEPSEEK_API_KEY"], - "config": { - "makaExecutable": "/opt/maka-agent/maka", - "runtimeHostsPath": "/tmp/maka-runtime-hosts", - "hostSettlementTimeoutMs": 120000, - "connectionSlug": "eval-gateway", - "baseUrl": "http://127.0.0.1:43120/v1", - "model": "deepseek-v4-flash", - "sandboxMode": "workspace-write" - } - }, - { - "id": "external-cli", - "kind": "external", - "credentials": ["DEEPSEEK_API_KEY"], - "config": { - "command": "/opt/maka-node-toolchain/bin/node", - "credentialEnvironment": { - "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY" - }, - "args": [ - "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js", - "deepseek-harness", - "http://127.0.0.1:43120/v1", - "/", - "dsh", - "{{task.input}}" - ] - } - } - ], - "tasks": [ - { - "id": "task-alpha", - "input": "Fix alpha", - "config": { "harbor": { "path": "tasks/task-alpha" } } - }, - { - "id": "task-beta", - "input": "Fix beta", - "config": { "harbor": { "path": "tasks/task-beta" } } - } - ], - "repetitions": 1, - "budget": { - "timeoutMultiplier": 1, - "maxSteps": 100 - }, - "verifier": { - "reward": "reward" - } - }) -} From c5577bc779549845a5468c972817338b3ea402f6 Mon Sep 17 00:00:00 2001 From: colafornia Date: Fri, 13 Nov 2026 21:41:18 +0800 Subject: [PATCH 4/4] refactor(eval): enforce causal v2 funnel and single pricing authority Generated-by: Google Antigravity --- products/eval/README.md | 14 +- products/eval/README.zh-CN.md | 14 +- products/eval/src/contract.rs | 1142 ++++++++++++++++++++---------- products/eval/src/measurement.rs | 26 +- products/eval/tests/contract.rs | 807 ++++++++++++--------- 5 files changed, 1281 insertions(+), 722 deletions(-) diff --git a/products/eval/README.md b/products/eval/README.md index fa11f0eeb1..523965d18a 100644 --- a/products/eval/README.md +++ b/products/eval/README.md @@ -25,7 +25,17 @@ Measurement and control plane contracts for multi-arm benchmark experiments (`ma ## Architecture -- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions, and condition-only `RunFingerprint`. -- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation) and standardized `DEEPSEEK_V4_FLASH_COST` pricing. +- [`contract`](src/contract.rs): Strict `maka.eval.v1` spec validation and cell expansion, immutable `V2AttemptRecord` with 3-gate causal adjudication (`BlockedIndeterminate` -> `RetryableInfra` -> `Selected`), leaf `v2 -> v1` projection, and spec-only `SpecFingerprint`. +- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens` normalized inclusive of cache reads/writes; `totalTokens` preserved without summation) and `PricingRates` (`DEEPSEEK_V4_FLASH_COST`) as the sole USD pricing authority. + +## Adjudicated `v2 -> v1` Rules + +| Scenario | `v2` Disposition | `v1` Projection & Selection | +| --- | --- | --- | +| Timeout + reward + `LowerBound` | `Selected(SubjectFailed)` | `subject_failed`, raw `score`, `costUsd: null`; selected & never resampled | +| Audit truncated / policy error | `Selected(Completed)` | `completed` with `egress-audit` metadata (`bytes`, `sha256`, `malformedLineCount`) | +| Missing audit / critical artifacts | `RetryableInfra` (unless Gate 1 indeterminate) | `infra_failed`, `score: null` (stricter than legacy TS when score existed) | +| Late cancellation | Before verify: `BlockedIndeterminate`; After: `Selected` | `indeterminate` (`score: null`) vs preserved score + marker | +| Post-hoc evidence recovery | `PostHocRecovery` appended monotonically | Derived view updates status/usage/cost without mutating the base attempt | [Source](src/lib.rs) · [Cargo manifest](Cargo.toml) diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md index d56fdde5c7..4d05620c44 100644 --- a/products/eval/README.zh-CN.md +++ b/products/eval/README.zh-CN.md @@ -25,7 +25,17 @@ ## 模块结构 -- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度，以及仅记录运行条件的 `RunFingerprint`。 -- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 +- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验规格校验与单元展开、不可变 `V2AttemptRecord` 及三关因果裁定（`BlockedIndeterminate` -> `RetryableInfra` -> `Selected`）、末端只读 `v2 -> v1` 投影，以及仅覆盖声明规格的 `SpecFingerprint`。 +- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（`inputTokens` 归一化后包含读写缓存；`totalTokens` 原样保留不重算求和），并以 `PricingRates`（`DEEPSEEK_V4_FLASH_COST`）作为唯一 USD 计价权威。 + +## 已裁定 `v2 -> v1` 语义规则 + +| 场景 | `v2` 裁定 | `v1` 投影与选样 | +| --- | --- | --- | +| 超时 + 有 reward + `LowerBound` | `Selected(SubjectFailed)` | `subject_failed`、保留 `score`、`costUsd: null`；直接入选且不重跑 | +| 审计日志截断 / 策略错误 | `Selected(Completed)` | `completed` 并输出完整 `egress-audit` 产物（含 `bytes`、`sha256`、`malformedLineCount`） | +| 缺失审计日志 / 关键产物 | `RetryableInfra`（若已被第一关判未决则保持未决） | `infra_failed`、`score: null`（比旧版 TS 保留分数更严格） | +| 迟到取消 | 验证前 `BlockedIndeterminate`；验证后 `Selected` | `indeterminate`（`score: null`）vs 保留分数并附加标记 | +| 事后证据恢复 | 通过 `PostHocRecovery` 单调叠加 | 派生视图更新状态与计量，不原地改写原始 attempt | [源码](src/lib.rs) · [Cargo 配置](Cargo.toml) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs index 14ac06575f..d6a434e23e 100644 --- a/products/eval/src/contract.rs +++ b/products/eval/src/contract.rs @@ -17,18 +17,23 @@ * under the License. */ -//! Experiment specifications, orthogonal `v2` attempt records, strict `v1` -//! projections, and run fingerprinting. +//! Experiment specifications, immutable `v2` attempt records with 3-gate +//! causal adjudication, strict `v1` leaf projections, and spec fingerprinting. -use crate::measurement::{NormalizedUsage, decode_usage}; +use crate::measurement::{NormalizedUsage, PricingRates, decode_usage}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; use std::fmt; +use std::num::NonZeroU32; pub type JsonObject = Map; +pub const EGRESS_AUDIT_ARTIFACT_PATH: &str = "artifacts/egress-hits.jsonl"; + +const MAX_SAFE_INTEGER_F64: f64 = 9_007_199_254_740_991.0; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ContractError(String); @@ -80,16 +85,16 @@ pub struct ExecutorSpec { pub config: JsonObject, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ExecutionSpec { - pub max_concurrent_task_groups: usize, + pub max_concurrent_task_groups: NonZeroU32, } impl Default for ExecutionSpec { fn default() -> Self { Self { - max_concurrent_task_groups: 1, + max_concurrent_task_groups: NonZeroU32::MIN, } } } @@ -122,7 +127,7 @@ pub struct ExperimentSpec { pub execution: ExecutionSpec, pub subjects: Vec, pub tasks: Vec, - pub repetitions: u32, + pub repetitions: NonZeroU32, pub budget: JsonObject, pub verifier: JsonObject, } @@ -146,19 +151,21 @@ pub struct ExperimentCell { pub executor: ExecutorSpec, pub subject: SubjectSpec, pub task: TaskSpec, - pub repetition: u32, + pub repetition: NonZeroU32, pub budget: JsonObject, pub verifier: JsonObject, } pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { - let mut cells = - Vec::with_capacity(spec.tasks.len() * (spec.repetitions as usize) * spec.subjects.len()); + let mut cells = Vec::with_capacity( + spec.tasks.len() * (spec.repetitions.get() as usize) * spec.subjects.len(), + ); for task in &spec.tasks { - for repetition in 1..=spec.repetitions { + for r in 1..=spec.repetitions.get() { + let repetition = NonZeroU32::new(r).expect("range starts at 1"); for subject in &spec.subjects { cells.push(ExperimentCell { - id: format!("{}::{repetition}::{}", task.id, subject.id), + id: format!("{}::{r}::{}", task.id, subject.id), experiment_id: spec.id.clone(), benchmark: spec.benchmark.clone(), executor: spec.executor.clone(), @@ -194,7 +201,6 @@ fn parse_experiment_spec(value: &Value) -> Result let schema_version = match root.get("schemaVersion").and_then(Value::as_str) { Some("maka.eval.v1") => SchemaVersion::V1, - Some("maka.eval.v2") => SchemaVersion::V2, _ => { return Err(ContractError::new("unsupported experiment schema")); } @@ -214,10 +220,10 @@ fn parse_experiment_spec(value: &Value) -> Result )?; let max_concurrent_task_groups = match root.get("execution") { - None => 1, + None => NonZeroU32::MIN, Some(exec_val) => { let exec_obj = exact_object(exec_val, "execution", &["maxConcurrentTaskGroups"], &[])?; - positive_usize( + positive_nonzero_u32( exec_obj.get("maxConcurrentTaskGroups").unwrap(), "execution.maxConcurrentTaskGroups", )? @@ -285,7 +291,7 @@ fn parse_experiment_spec(value: &Value) -> Result ensure_unique_ids(subjects.iter().map(|s| s.id.as_str()), "subject")?; ensure_unique_ids(tasks.iter().map(|t| t.id.as_str()), "task")?; - let repetitions = positive_u32(root.get("repetitions").unwrap(), "repetitions")?; + let repetitions = positive_nonzero_u32(root.get("repetitions").unwrap(), "repetitions")?; Ok(ExperimentSpec { schema_version, @@ -333,7 +339,7 @@ fn write_canonical_json(value: &Value, out: &mut Vec) { out.extend_from_slice(b"0"); } else if f.is_finite() && f.fract() == 0.0 - && (-9_007_199_254_740_991.0..=9_007_199_254_740_991.0).contains(&f) + && (-MAX_SAFE_INTEGER_F64..=MAX_SAFE_INTEGER_F64).contains(&f) { if f >= 0.0 { serde_json::to_writer(&mut *out, &(f as u64)) @@ -381,23 +387,23 @@ fn write_canonical_json(value: &Value, out: &mut Vec) { } } -/// Documents experimental and execution conditions for comparability checks. +/// Canonical digests of declared `ExperimentSpec` semantic and execution parameters. /// -/// Fingerprints record the declared configuration and parameters of a run; freezing an -/// eval-only fingerprint does not establish product parity. M3 acceptance and M5 production -/// delivery require a verified `ParityVerificationReport` alongside the run. +/// Fingerprints cover only the declared experiment specification; they do not capture +/// host/runtime binaries, prompt or tool schemas, container image digests, or product parity. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct RunFingerprint { - /// Digest over benchmark, executor kind, subjects, tasks, repetitions, budget, and verifier. +pub struct SpecFingerprint { + /// Digest over schema version, benchmark, executor kind, subjects, tasks, repetitions, budget, and verifier. pub semantic_sha256: String, - /// Digest over execution-only parameters (e.g. `maxConcurrentTaskGroups`). + /// Digest over execution-only spec parameters (e.g. `maxConcurrentTaskGroups`, `executor.config`). pub execution_sha256: String, } -impl RunFingerprint { +impl SpecFingerprint { pub fn compute(spec: &ExperimentSpec) -> Self { let semantic_payload = serde_json::json!({ + "schemaVersion": spec.schema_version, "benchmark": spec.benchmark, "executorKind": spec.executor.kind, "subjects": spec.subjects, @@ -458,7 +464,7 @@ impl<'de> Deserialize<'de> for EvalResult { #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct V1CellAttempt { pub cell_id: String, - pub sequence: u32, + pub sequence: NonZeroU32, pub started_at: u64, pub completed_at: u64, pub result: EvalResult, @@ -474,13 +480,12 @@ impl<'de> Deserialize<'de> for V1CellAttempt { } } -/// Canonical cell result selector: selects the earliest valid attempt by sequence (`sequence >= 1`) -/// whose status is neither `infra_failed` nor `indeterminate`. A `completed` attempt with score `0.0` -/// or a `subject_failed` attempt is terminal and never replaced. +/// Legacy `v1` cell result selector for frozen oracle compatibility: picks the earliest +/// attempt by sequence whose status is `completed` or `subject_failed`. pub fn select_cell_result(attempts: &[V1CellAttempt]) -> Option<&V1CellAttempt> { attempts .iter() - .filter(|a| a.sequence >= 1 && a.result.status.is_selectable()) + .filter(|a| a.result.status.is_selectable()) .min_by_key(|a| a.sequence) } @@ -528,7 +533,7 @@ fn decode_eval_result(value: &Value, where_label: &str) -> Result Result )?; Ok(V1CellAttempt { cell_id: nonempty_string(obj.get("cellId").unwrap(), "attempt.cellId")?, - sequence: positive_u32(obj.get("sequence").unwrap(), "attempt.sequence")?, - started_at: nonnegative_u64(obj.get("startedAt").unwrap(), "attempt.startedAt")?, - completed_at: nonnegative_u64(obj.get("completedAt").unwrap(), "attempt.completedAt")?, + sequence: positive_nonzero_u32(obj.get("sequence").unwrap(), "attempt.sequence")?, + started_at: safe_u64(obj.get("startedAt").unwrap(), "attempt.startedAt")?, + completed_at: safe_u64(obj.get("completedAt").unwrap(), "attempt.completedAt")?, result: decode_eval_result(obj.get("result").unwrap(), "attempt.result")?, }) } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ExecutionOutcome { - Completed, - SubjectFailed, - Timeout, - InfraFailed, - Indeterminate, +/// Monotonic phase durations (in milliseconds) recorded on native `v2` attempts. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PhaseDurationsMs { + pub queue_wait_ms: u64, + pub prepare_ms: u64, + pub subject_ms: u64, + pub verify_ms: u64, + pub cleanup_ms: u64, + pub total_ms: u64, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AttemptCancellation { - #[default] - None, - BeforeVerification, - AfterVerification, +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SubjectOutcome { + Completed {}, + Failed { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + Timeout { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -596,121 +608,47 @@ pub enum VerifierOutcome { score: f64, }, SubjectFailed { + #[serde(default, skip_serializing_if = "Option::is_none")] score: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] reason: Option, }, /// Missing reward file or empty verifier payload: never defaults to `0.0`. MissingReward { reason: String, }, - VerifierInfraFailed { + InfraFailed { reason: String, }, - Skipped, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum CleanupState { - Confirmed, - Unconfirmed { reason: String }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde( - tag = "kind", + tag = "stage", rename_all = "snake_case", rename_all_fields = "camelCase", deny_unknown_fields )] -pub enum EgressAuditState { - NotRequired, - Recorded { - #[serde(default)] - truncated: bool, - #[serde(default)] - policy_error_count: u64, +pub enum TrialOutcome { + InfraFailed { + reason: String, }, - Missing, - Unreadable { + Indeterminate { reason: String, }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum CriticalArtifactsState { - Present, - Missing { paths: Vec }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde( - remote = "Self", - tag = "kind", - rename_all = "snake_case", - rename_all_fields = "camelCase", - deny_unknown_fields -)] -pub enum PricingEvidence { - Model { - model: String, - cost_usd: Option, + CancelledBeforeVerification { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, }, - SubjectReported { - cost_usd: f64, + Evaluated { + subject: SubjectOutcome, + verifier: VerifierOutcome, + #[serde(default)] + cancelled_after_verification: bool, }, } -impl Serialize for PricingEvidence { - fn serialize(&self, serializer: S) -> Result { - Self::serialize(self, serializer) - } -} - -impl<'de> Deserialize<'de> for PricingEvidence { - fn deserialize>(deserializer: D) -> Result { - let value = Self::deserialize(deserializer)?; - value.validate().map_err(serde::de::Error::custom)?; - Ok(value) - } -} - -impl PricingEvidence { - pub fn cost_usd(&self) -> Option { - match self { - Self::Model { cost_usd, .. } => *cost_usd, - Self::SubjectReported { cost_usd } => Some(*cost_usd), - } - } - - pub fn validate(&self) -> Result<(), ContractError> { - match self { - Self::Model { model, cost_usd } => { - if model.is_empty() { - return Err(ContractError::new("pricing.model must not be empty")); - } - if let Some(c) = cost_usd - && (!c.is_finite() || *c < 0.0) - { - return Err(ContractError::new( - "pricing.costUsd must be finite and non-negative", - )); - } - } - Self::SubjectReported { cost_usd } => { - if !cost_usd.is_finite() || *cost_usd < 0.0 { - return Err(ContractError::new( - "pricing.costUsd must be finite and non-negative", - )); - } - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, Default, PartialEq, Serialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] #[serde( tag = "state", rename_all = "snake_case", @@ -724,8 +662,8 @@ pub enum AttemptMetering { }, Complete { usage: NormalizedUsage, - #[serde(default)] - pricing: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + model: Option, }, } @@ -755,18 +693,18 @@ impl<'de> Deserialize<'de> for AttemptMetering { } } "complete" => { - let obj = exact_object(&value, "metering", &["state", "usage"], &["pricing"]) + let obj = exact_object(&value, "metering", &["state", "usage"], &["model"]) .map_err(serde::de::Error::custom)?; - let pricing = match obj.get("pricing") { + let model = match obj.get("model") { None | Some(Value::Null) => None, - Some(p) => { - Some(serde_json::from_value(p.clone()).map_err(serde::de::Error::custom)?) - } + Some(m) => Some( + nonempty_string(m, "metering.model").map_err(serde::de::Error::custom)?, + ), }; Self::Complete { usage: decode_usage(obj.get("usage").unwrap(), "metering.usage") .map_err(serde::de::Error::custom)?, - pricing, + model, } } _ => return Err(serde::de::Error::custom("metering.state is invalid")), @@ -784,248 +722,686 @@ impl AttemptMetering { } } + /// Standardized USD cost derived strictly from `PricingRates::for_model` on `Complete` metering. pub fn cost_usd(&self) -> Option { match self { Self::Complete { - pricing: Some(pricing), - .. - } => pricing.cost_usd(), - Self::Unknown | Self::LowerBound { .. } | Self::Complete { pricing: None, .. } => None, + usage, + model: Some(model), + } => PricingRates::for_model(model).map(|rates| rates.cost_usd(usage)), + Self::Unknown | Self::LowerBound { .. } | Self::Complete { model: None, .. } => None, } } pub fn validate(&self) -> Result<(), ContractError> { if let Self::Complete { - pricing: Some(p), .. + model: Some(model), .. } = self + && model.is_empty() { - p.validate()?; + return Err(ContractError::new("metering.model must not be empty")); } Ok(()) } + + fn rank(&self) -> u8 { + match self { + Self::Unknown => 0, + Self::LowerBound { .. } => 1, + Self::Complete { .. } => 2, + } + } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(remote = "Self", rename_all = "camelCase", deny_unknown_fields)] -pub struct V2AttemptRecord { - pub schema_version: SchemaVersion, +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CleanupState { + Confirmed {}, + Unconfirmed { reason: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "snake_case", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum EgressAuditState { + NotRequired {}, + Recorded { + bytes: u64, + sha256: String, + #[serde(default)] + truncated: bool, + #[serde(default)] + policy_error_count: u64, + #[serde(default)] + malformed_line_count: u64, + }, + Missing {}, + Unreadable { + reason: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CriticalArtifactsState { + Present {}, + Missing { paths: Vec }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct V2AttemptParts { pub cell_id: String, - pub sequence: u32, + pub sequence: NonZeroU32, pub started_at: u64, pub completed_at: u64, - pub duration_ms: u64, - pub execution_outcome: ExecutionOutcome, - pub verifier_outcome: VerifierOutcome, + pub durations: PhaseDurationsMs, + pub trial_outcome: TrialOutcome, pub metering: AttemptMetering, pub cleanup_state: CleanupState, pub audit_state: EgressAuditState, pub critical_artifacts_state: CriticalArtifactsState, - #[serde(default)] - pub cancellation: AttemptCancellation, - pub recovered_post_hoc: bool, - pub failure_reason: Option, pub artifacts: Vec, } -impl Serialize for V2AttemptRecord { - fn serialize(&self, serializer: S) -> Result { - Self::serialize(self, serializer) - } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct V2AttemptWire { + schema_version: SchemaVersion, + cell_id: String, + sequence: NonZeroU32, + started_at: u64, + completed_at: u64, + durations: PhaseDurationsMs, + trial_outcome: TrialOutcome, + metering: AttemptMetering, + cleanup_state: CleanupState, + audit_state: EgressAuditState, + critical_artifacts_state: CriticalArtifactsState, + artifacts: Vec, } -impl<'de> Deserialize<'de> for V2AttemptRecord { - fn deserialize>(deserializer: D) -> Result { - let record = Self::deserialize(deserializer)?; - record.validate().map_err(serde::de::Error::custom)?; - Ok(record) - } -} +/// Immutable, validated `maka.eval.v2` attempt record. +#[derive(Debug, Clone, PartialEq)] +pub struct V2AttemptRecord(V2AttemptParts); impl V2AttemptRecord { - pub fn validate(&self) -> Result<(), ContractError> { - if self.schema_version != SchemaVersion::V2 { - return Err(ContractError::new( - "v2 attempt record requires schemaVersion maka.eval.v2", - )); - } - if self.cell_id.is_empty() { + pub fn try_new(parts: V2AttemptParts) -> Result { + if parts.cell_id.is_empty() { return Err(ContractError::new("v2 attempt record cellId is required")); } - if self.sequence < 1 { + if parts.completed_at < parts.started_at { return Err(ContractError::new( - "v2 attempt record sequence must be positive", + "v2 attempt record completedAt must not precede startedAt", )); } - match &self.verifier_outcome { - VerifierOutcome::Verified { score } if !score.is_finite() => { - return Err(ContractError::new("verifier score must be finite")); - } - VerifierOutcome::SubjectFailed { - score: Some(score), .. - } if !score.is_finite() => { - return Err(ContractError::new("verifier score must be finite")); + validate_trial_outcome(&parts.trial_outcome)?; + parts.metering.validate()?; + validate_cleanup_state(&parts.cleanup_state)?; + validate_audit_state(&parts.audit_state)?; + validate_critical_artifacts_state(&parts.critical_artifacts_state)?; + Ok(Self(parts)) + } + + pub fn cell_id(&self) -> &str { + &self.0.cell_id + } + + pub fn sequence(&self) -> NonZeroU32 { + self.0.sequence + } + + pub fn started_at(&self) -> u64 { + self.0.started_at + } + + pub fn completed_at(&self) -> u64 { + self.0.completed_at + } + + pub fn durations(&self) -> &PhaseDurationsMs { + &self.0.durations + } + + pub fn trial_outcome(&self) -> &TrialOutcome { + &self.0.trial_outcome + } + + pub fn metering(&self) -> &AttemptMetering { + &self.0.metering + } + + pub fn cleanup_state(&self) -> &CleanupState { + &self.0.cleanup_state + } + + pub fn audit_state(&self) -> &EgressAuditState { + &self.0.audit_state + } + + pub fn critical_artifacts_state(&self) -> &CriticalArtifactsState { + &self.0.critical_artifacts_state + } + + pub fn artifacts(&self) -> &[JsonObject] { + &self.0.artifacts + } + + pub fn into_parts(self) -> V2AttemptParts { + self.0 + } + + pub fn disposition(&self) -> V2Disposition { + self.disposition_with_recovery(&[]) + } + + pub fn disposition_with_recovery(&self, recoveries: &[PostHocRecovery]) -> V2Disposition { + let (effective_cleanup, effective_metering) = self.effective_evidence(recoveries); + self.adjudicate(effective_cleanup, effective_metering) + } + + /// Projects this `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt`. + pub fn project_v1(&self) -> V1CellAttempt { + self.project_v1_with_recovery(&[]) + } + + /// Projects this `maka.eval.v2` record plus appended post-hoc recovery entries + /// into a strict `maka.eval.v1` `V1CellAttempt` without mutating the record. + pub fn project_v1_with_recovery(&self, recoveries: &[PostHocRecovery]) -> V1CellAttempt { + let (effective_cleanup, effective_metering) = self.effective_evidence(recoveries); + let disposition = self.adjudicate(effective_cleanup, effective_metering); + + let mut artifacts = self.0.artifacts.clone(); + if matches!( + self.0.trial_outcome, + TrialOutcome::Evaluated { + cancelled_after_verification: true, + .. } - _ => {} + ) { + push_artifact_marker(&mut artifacts, "cancelled-after-verification"); } - if self.resolve_outcome().0 == EvalResultStatus::Completed && self.failure_reason.is_some() - { - return Err(ContractError::new( - "v2 completed attempt record must not set failureReason", - )); + if !recoveries.is_empty() { + push_artifact_marker(&mut artifacts, "post-hoc-evidence-recovery"); + } + append_egress_audit_artifact(&mut artifacts, &self.0.audit_state); + + let (status, score, failure_reason) = match disposition { + V2Disposition::Selected { + status: SelectedStatus::Completed, + score, + failure_reason, + } => (EvalResultStatus::Completed, score, failure_reason), + V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score, + failure_reason, + } => (EvalResultStatus::SubjectFailed, score, failure_reason), + V2Disposition::RetryableInfra { failure_reason } => { + (EvalResultStatus::InfraFailed, None, Some(failure_reason)) + } + V2Disposition::BlockedIndeterminate { failure_reason } => { + (EvalResultStatus::Indeterminate, None, Some(failure_reason)) + } + }; + + V1CellAttempt { + cell_id: self.0.cell_id.clone(), + sequence: self.0.sequence, + started_at: self.0.started_at, + completed_at: self.0.completed_at, + result: EvalResult { + score, + usage: effective_metering.usage().cloned(), + cost_usd: effective_metering.cost_usd(), + duration_ms: self.0.durations.subject_ms, + status, + failure_reason, + artifacts, + }, } - self.metering.validate()?; - Ok(()) } - fn resolve_outcome(&self) -> (EvalResultStatus, Option, Option) { - // Unconfirmed cleanup always resolves to `indeterminate` with `score: null`. - if let CleanupState::Unconfirmed { reason } = &self.cleanup_state { - return (EvalResultStatus::Indeterminate, None, Some(reason.clone())); + fn effective_evidence<'a>( + &'a self, + recoveries: &'a [PostHocRecovery], + ) -> (&'a CleanupState, &'a AttemptMetering) { + let mut cleanup = &self.0.cleanup_state; + let mut metering = &self.0.metering; + for recovery in recoveries { + if let Some(next_cleanup) = recovery.cleanup_state() + && matches!(cleanup, CleanupState::Unconfirmed { .. }) + && matches!(next_cleanup, CleanupState::Confirmed {}) + { + cleanup = next_cleanup; + } + if let Some(next_metering) = recovery.metering() + && next_metering.rank() > metering.rank() + { + metering = next_metering; + } } + (cleanup, metering) + } - // Late cancellation before verification completes resolves to `indeterminate`. - if self.cancellation == AttemptCancellation::BeforeVerification { - return ( - EvalResultStatus::Indeterminate, - None, - Some( - self.failure_reason.clone().unwrap_or_else(|| { + fn adjudicate( + &self, + effective_cleanup: &CleanupState, + effective_metering: &AttemptMetering, + ) -> V2Disposition { + // Gate 1: Epistemic certainty — unresolved cleanup, pre-verification cancellation, + // indeterminate execution, or unknown metering on an evaluated trial block retry. + if let CleanupState::Unconfirmed { reason } = effective_cleanup { + return V2Disposition::BlockedIndeterminate { + failure_reason: reason.clone(), + }; + } + match &self.0.trial_outcome { + TrialOutcome::CancelledBeforeVerification { reason } => { + return V2Disposition::BlockedIndeterminate { + failure_reason: reason.clone().unwrap_or_else(|| { "executor cancelled before verification completed".into() }), - ), - ); + }; + } + TrialOutcome::Indeterminate { reason } => { + return V2Disposition::BlockedIndeterminate { + failure_reason: reason.clone(), + }; + } + TrialOutcome::Evaluated { .. } + if matches!(effective_metering, AttemptMetering::Unknown) => + { + return V2Disposition::BlockedIndeterminate { + failure_reason: "metering state unknown".into(), + }; + } + TrialOutcome::InfraFailed { .. } | TrialOutcome::Evaluated { .. } => {} } - if let CriticalArtifactsState::Missing { paths } = &self.critical_artifacts_state { - return ( - EvalResultStatus::InfraFailed, - None, - Some(format!("critical artifacts missing: {}", paths.join(", "))), - ); + // Gate 2: Infrastructure & evidence integrity — preserve root infra failures before + // checking downstream artifacts/audit, and reject LowerBound metering on Completed runs. + match &self.0.trial_outcome { + TrialOutcome::InfraFailed { reason } => { + return V2Disposition::RetryableInfra { + failure_reason: reason.clone(), + }; + } + TrialOutcome::Evaluated { + verifier: + VerifierOutcome::InfraFailed { reason } | VerifierOutcome::MissingReward { reason }, + .. + } => { + return V2Disposition::RetryableInfra { + failure_reason: reason.clone(), + }; + } + TrialOutcome::Evaluated { .. } => {} + TrialOutcome::Indeterminate { .. } + | TrialOutcome::CancelledBeforeVerification { .. } => { + unreachable!("handled in Gate 1"); + } } - - match &self.audit_state { - EgressAuditState::Missing => { - return ( - EvalResultStatus::InfraFailed, - None, - Some("egress audit log missing".into()), - ); + if let CriticalArtifactsState::Missing { paths } = &self.0.critical_artifacts_state { + return V2Disposition::RetryableInfra { + failure_reason: format!("critical artifacts missing: {}", paths.join(", ")), + }; + } + match &self.0.audit_state { + EgressAuditState::Missing {} => { + return V2Disposition::RetryableInfra { + failure_reason: "egress audit log missing".into(), + }; } EgressAuditState::Unreadable { reason } => { - return (EvalResultStatus::InfraFailed, None, Some(reason.clone())); + return V2Disposition::RetryableInfra { + failure_reason: reason.clone(), + }; } - EgressAuditState::NotRequired | EgressAuditState::Recorded { .. } => {} + EgressAuditState::NotRequired {} | EgressAuditState::Recorded { .. } => {} + } + if matches!( + self.0.trial_outcome, + TrialOutcome::Evaluated { + subject: SubjectOutcome::Completed {}, + .. + } + ) && matches!(effective_metering, AttemptMetering::LowerBound { .. }) + { + return V2Disposition::RetryableInfra { + failure_reason: "incomplete lower-bound metering on completed subject".into(), + }; } - match (&self.execution_outcome, &self.verifier_outcome) { - (ExecutionOutcome::InfraFailed, _) => ( - EvalResultStatus::InfraFailed, - None, - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject infrastructure failed".into()), - ), - ), - (ExecutionOutcome::Indeterminate, _) => ( - EvalResultStatus::Indeterminate, - None, - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject execution indeterminate".into()), - ), - ), - (_, VerifierOutcome::VerifierInfraFailed { reason }) - | (_, VerifierOutcome::MissingReward { reason }) => { - // Missing verifier reward never defaults to 0.0! - (EvalResultStatus::InfraFailed, None, Some(reason.clone())) + // Gate 3: Capability settlement — permanent selection, never resampled. + let TrialOutcome::Evaluated { + subject, verifier, .. + } = &self.0.trial_outcome + else { + unreachable!("non-evaluated outcomes handled in Gates 1 and 2"); + }; + match (subject, verifier) { + (SubjectOutcome::Completed {}, VerifierOutcome::Verified { score }) => { + V2Disposition::Selected { + status: SelectedStatus::Completed, + score: Some(*score), + failure_reason: None, + } } - (ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }) => ( - EvalResultStatus::SubjectFailed, - Some(*score), - Some( - self.failure_reason - .clone() - .unwrap_or_else(|| "subject exceeded the framework timeout".into()), - ), - ), - (ExecutionOutcome::SubjectFailed, VerifierOutcome::Verified { score }) => ( - EvalResultStatus::SubjectFailed, - Some(*score), - self.failure_reason.clone(), - ), - ( - ExecutionOutcome::Completed - | ExecutionOutcome::SubjectFailed - | ExecutionOutcome::Timeout, - VerifierOutcome::SubjectFailed { score, reason }, - ) => ( - EvalResultStatus::SubjectFailed, - *score, - self.failure_reason.clone().or_else(|| reason.clone()), - ), - (ExecutionOutcome::Completed, VerifierOutcome::Verified { score }) => { - (EvalResultStatus::Completed, Some(*score), None) + (SubjectOutcome::Completed {}, VerifierOutcome::SubjectFailed { score, reason }) => { + V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score: *score, + failure_reason: reason.clone(), + } } - (ExecutionOutcome::Timeout, VerifierOutcome::Skipped) => ( - EvalResultStatus::SubjectFailed, - None, - Some( - self.failure_reason + (SubjectOutcome::Timeout { reason }, VerifierOutcome::Verified { score }) => { + V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score: Some(*score), + failure_reason: Some( + reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), + ), + } + } + ( + SubjectOutcome::Timeout { reason }, + VerifierOutcome::SubjectFailed { + score, + reason: v_reason, + }, + ) => V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score: *score, + failure_reason: Some( + reason .clone() + .or_else(|| v_reason.clone()) .unwrap_or_else(|| "subject exceeded the framework timeout".into()), ), - ), - (ExecutionOutcome::SubjectFailed, VerifierOutcome::Skipped) => ( - EvalResultStatus::SubjectFailed, - None, - self.failure_reason.clone(), - ), - (ExecutionOutcome::Completed, VerifierOutcome::Skipped) => { - (EvalResultStatus::Completed, None, None) + }, + (SubjectOutcome::Failed { reason }, VerifierOutcome::Verified { score }) => { + V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score: Some(*score), + failure_reason: reason.clone(), + } + } + ( + SubjectOutcome::Failed { reason }, + VerifierOutcome::SubjectFailed { + score, + reason: v_reason, + }, + ) => V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score: *score, + failure_reason: reason.clone().or_else(|| v_reason.clone()), + }, + (_, VerifierOutcome::MissingReward { .. } | VerifierOutcome::InfraFailed { .. }) => { + unreachable!("handled in Gate 2") } } } +} - /// Projects a native `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt`. - pub fn project_v1(&self) -> V1CellAttempt { - let mut artifacts = self.artifacts.clone(); - if self.cancellation == AttemptCancellation::AfterVerification { - push_artifact_marker(&mut artifacts, "cancelled-after-verification"); +impl Serialize for V2AttemptRecord { + fn serialize(&self, serializer: S) -> Result { + let wire = V2AttemptWire { + schema_version: SchemaVersion::V2, + cell_id: self.0.cell_id.clone(), + sequence: self.0.sequence, + started_at: self.0.started_at, + completed_at: self.0.completed_at, + durations: self.0.durations, + trial_outcome: self.0.trial_outcome.clone(), + metering: self.0.metering.clone(), + cleanup_state: self.0.cleanup_state.clone(), + audit_state: self.0.audit_state.clone(), + critical_artifacts_state: self.0.critical_artifacts_state.clone(), + artifacts: self.0.artifacts.clone(), + }; + wire.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for V2AttemptRecord { + fn deserialize>(deserializer: D) -> Result { + let wire = V2AttemptWire::deserialize(deserializer)?; + if wire.schema_version != SchemaVersion::V2 { + return Err(serde::de::Error::custom( + "v2 attempt record requires schemaVersion maka.eval.v2", + )); } - if self.recovered_post_hoc { - push_artifact_marker(&mut artifacts, "post-hoc-evidence-recovery"); + Self::try_new(V2AttemptParts { + cell_id: wire.cell_id, + sequence: wire.sequence, + started_at: wire.started_at, + completed_at: wire.completed_at, + durations: wire.durations, + trial_outcome: wire.trial_outcome, + metering: wire.metering, + cleanup_state: wire.cleanup_state, + audit_state: wire.audit_state, + critical_artifacts_state: wire.critical_artifacts_state, + artifacts: wire.artifacts, + }) + .map_err(serde::de::Error::custom) + } +} + +/// Append-only post-hoc evidence recovery event applied on top of an immutable `V2AttemptRecord`. +#[derive(Debug, Clone, PartialEq)] +pub struct PostHocRecovery { + recovered_at: u64, + metering: Option, + cleanup_state: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct PostHocRecoveryWire { + recovered_at: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + metering: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cleanup_state: Option, +} + +impl PostHocRecovery { + pub fn try_new( + recovered_at: u64, + metering: Option, + cleanup_state: Option, + ) -> Result { + if metering.is_none() && cleanup_state.is_none() { + return Err(ContractError::new( + "post-hoc recovery must provide metering or cleanupState", + )); } - if let EgressAuditState::Recorded { - truncated, - policy_error_count, - } = &self.audit_state - && (*truncated || *policy_error_count > 0) - { - record_audit_artifact(&mut artifacts, *truncated, *policy_error_count); + if let Some(m) = &metering { + m.validate()?; + } + if let Some(c) = &cleanup_state { + validate_cleanup_state(c)?; } + Ok(Self { + recovered_at, + metering, + cleanup_state, + }) + } - let (status, score, failure_reason) = self.resolve_outcome(); - V1CellAttempt { - cell_id: self.cell_id.clone(), - sequence: self.sequence, - started_at: self.started_at, - completed_at: self.completed_at, - result: EvalResult { - score, - usage: self.metering.usage().cloned(), - cost_usd: self.metering.cost_usd(), - duration_ms: self.duration_ms, - status, - failure_reason, - artifacts, - }, + pub fn recovered_at(&self) -> u64 { + self.recovered_at + } + + pub fn metering(&self) -> Option<&AttemptMetering> { + self.metering.as_ref() + } + + pub fn cleanup_state(&self) -> Option<&CleanupState> { + self.cleanup_state.as_ref() + } +} + +impl Serialize for PostHocRecovery { + fn serialize(&self, serializer: S) -> Result { + PostHocRecoveryWire { + recovered_at: self.recovered_at, + metering: self.metering.clone(), + cleanup_state: self.cleanup_state.clone(), } + .serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for PostHocRecovery { + fn deserialize>(deserializer: D) -> Result { + let wire = PostHocRecoveryWire::deserialize(deserializer)?; + Self::try_new(wire.recovered_at, wire.metering, wire.cleanup_state) + .map_err(serde::de::Error::custom) } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SelectedStatus { + Completed, + SubjectFailed, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum V2Disposition { + Selected { + status: SelectedStatus, + score: Option, + failure_reason: Option, + }, + RetryableInfra { + failure_reason: String, + }, + BlockedIndeterminate { + failure_reason: String, + }, +} + +/// Authoritative `v2` cell result selector: picks the earliest attempt by sequence +/// whose `V2Disposition` is `Selected`. +pub fn select_v2_cell_result(attempts: &[V2AttemptRecord]) -> Option<&V2AttemptRecord> { + attempts + .iter() + .filter(|a| matches!(a.disposition(), V2Disposition::Selected { .. })) + .min_by_key(|a| a.sequence()) +} + +/// Authoritative `v2` retry admission check: returns `false` if any attempt is already +/// `Selected` or `BlockedIndeterminate`, or if `attempts.len() > max_infra_retries as usize`. +pub fn can_admit_v2_attempt(attempts: &[V2AttemptRecord], max_infra_retries: u32) -> bool { + !attempts.iter().any(|a| { + matches!( + a.disposition(), + V2Disposition::Selected { .. } | V2Disposition::BlockedIndeterminate { .. } + ) + }) && attempts.len() <= max_infra_retries as usize +} + +fn validate_trial_outcome(outcome: &TrialOutcome) -> Result<(), ContractError> { + match outcome { + TrialOutcome::InfraFailed { reason } | TrialOutcome::Indeterminate { reason } => { + if reason.is_empty() { + return Err(ContractError::new("trialOutcome.reason must not be empty")); + } + } + TrialOutcome::CancelledBeforeVerification { reason: Some(r) } => { + if r.is_empty() { + return Err(ContractError::new("trialOutcome.reason must not be empty")); + } + } + TrialOutcome::CancelledBeforeVerification { reason: None } => {} + TrialOutcome::Evaluated { + subject, verifier, .. + } => { + match subject { + SubjectOutcome::Completed {} => {} + SubjectOutcome::Failed { reason: Some(r) } + | SubjectOutcome::Timeout { reason: Some(r) } => { + if r.is_empty() { + return Err(ContractError::new("subject.reason must not be empty")); + } + } + SubjectOutcome::Failed { reason: None } + | SubjectOutcome::Timeout { reason: None } => {} + } + match verifier { + VerifierOutcome::Verified { score } => { + if !score.is_finite() { + return Err(ContractError::new("verifier score must be finite")); + } + } + VerifierOutcome::SubjectFailed { score, reason } => { + if score.is_some_and(|s| !s.is_finite()) { + return Err(ContractError::new("verifier score must be finite")); + } + if reason.as_deref().is_some_and(str::is_empty) { + return Err(ContractError::new("verifier.reason must not be empty")); + } + } + VerifierOutcome::MissingReward { reason } + | VerifierOutcome::InfraFailed { reason } => { + if reason.is_empty() { + return Err(ContractError::new("verifier.reason must not be empty")); + } + } + } + } + } + Ok(()) +} + +fn validate_cleanup_state(state: &CleanupState) -> Result<(), ContractError> { + if let CleanupState::Unconfirmed { reason } = state + && reason.is_empty() + { + return Err(ContractError::new("cleanupState.reason must not be empty")); + } + Ok(()) +} + +fn validate_audit_state(state: &EgressAuditState) -> Result<(), ContractError> { + match state { + EgressAuditState::NotRequired {} | EgressAuditState::Missing {} => {} + EgressAuditState::Recorded { sha256, .. } => { + if !is_lower_hex_sha256(sha256) { + return Err(ContractError::new( + "auditState.sha256 must be 64 lowercase hex characters", + )); + } + } + EgressAuditState::Unreadable { reason } => { + if reason.is_empty() { + return Err(ContractError::new("auditState.reason must not be empty")); + } + } + } + Ok(()) +} + +fn validate_critical_artifacts_state(state: &CriticalArtifactsState) -> Result<(), ContractError> { + if let CriticalArtifactsState::Missing { paths } = state + && (paths.is_empty() || paths.iter().any(String::is_empty)) + { + return Err(ContractError::new( + "criticalArtifactsState.paths must be non-empty strings", + )); + } + Ok(()) +} + fn push_artifact_marker(artifacts: &mut Vec, kind: &str) { if !artifacts .iter() @@ -1037,29 +1413,63 @@ fn push_artifact_marker(artifacts: &mut Vec, kind: &str) { } } -fn record_audit_artifact( - artifacts: &mut Vec, - truncated: bool, - policy_error_count: u64, -) { - if let Some(existing) = artifacts - .iter_mut() - .find(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) - { - if truncated { - existing.insert("truncated".into(), Value::Bool(true)); +fn append_egress_audit_artifact(artifacts: &mut Vec, audit_state: &EgressAuditState) { + match audit_state { + EgressAuditState::NotRequired {} => {} + EgressAuditState::Recorded { + bytes, + sha256, + truncated, + policy_error_count, + malformed_line_count, + } => { + if !artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) + { + let mut audit_obj = Map::new(); + audit_obj.insert("kind".into(), Value::String("egress-audit".into())); + audit_obj.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + audit_obj.insert("bytes".into(), Value::from(*bytes)); + audit_obj.insert("sha256".into(), Value::String(format!("sha256:{sha256}"))); + audit_obj.insert("truncated".into(), Value::Bool(*truncated)); + audit_obj.insert("policyErrorCount".into(), Value::from(*policy_error_count)); + audit_obj.insert( + "malformedLineCount".into(), + Value::from(*malformed_line_count), + ); + artifacts.push(audit_obj); + } } - if policy_error_count > 0 { - existing.insert("policyErrorCount".into(), Value::from(policy_error_count)); + EgressAuditState::Missing {} => { + push_path_artifact( + artifacts, + "egress-audit-missing", + EGRESS_AUDIT_ARTIFACT_PATH, + ); } - } else { - let mut audit_obj = Map::new(); - audit_obj.insert("kind".into(), Value::String("egress-audit".into())); - audit_obj.insert("truncated".into(), Value::Bool(truncated)); - if policy_error_count > 0 { - audit_obj.insert("policyErrorCount".into(), Value::from(policy_error_count)); + EgressAuditState::Unreadable { .. } => { + push_path_artifact( + artifacts, + "egress-audit-unreadable", + EGRESS_AUDIT_ARTIFACT_PATH, + ); } - artifacts.push(audit_obj); + } +} + +fn push_path_artifact(artifacts: &mut Vec, kind: &str, path: &str) { + if !artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some(kind)) + { + let mut obj = Map::new(); + obj.insert("kind".into(), Value::String(kind.into())); + obj.insert("path".into(), Value::String(path.into())); + artifacts.push(obj); } } @@ -1148,33 +1558,33 @@ fn ensure_unique_ids<'a>( Ok(()) } -fn positive_u32(value: &Value, where_label: &str) -> Result { - match value.as_u64() { - Some(n) if n >= 1 && n <= u64::from(u32::MAX) => Ok(n as u32), - _ => Err(ContractError::new(format!( - "{where_label} must be positive" - ))), - } +fn is_lower_hex_sha256(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } -fn positive_usize(value: &Value, where_label: &str) -> Result { - match value.as_u64() { - Some(n) if n >= 1 => usize::try_from(n) - .map_err(|_| ContractError::new(format!("{where_label} must be positive"))), - _ => Err(ContractError::new(format!( - "{where_label} must be positive" - ))), +pub(crate) fn safe_u64(value: &Value, where_label: &str) -> Result { + if let Some(n) = value.as_u64() { + return Ok(n); + } + match value.as_f64() { + Some(f) + if f.is_finite() && (0.0..=MAX_SAFE_INTEGER_F64).contains(&f) && f.fract() == 0.0 => + { + Ok(f as u64) + } + _ => Err(ContractError::new(format!("{where_label} is invalid"))), } } -pub(crate) fn nonnegative_u64(value: &Value, where_label: &str) -> Result { - match value.as_u64() { - Some(n) => Ok(n), - None => match value.as_f64() { - Some(f) if f.is_finite() && f >= 0.0 && f.fract() == 0.0 => Ok(f as u64), - _ => Err(ContractError::new(format!("{where_label} is invalid"))), - }, - } +fn positive_nonzero_u32(value: &Value, where_label: &str) -> Result { + let n = safe_u64(value, where_label)?; + let raw = u32::try_from(n) + .ok() + .and_then(NonZeroU32::new) + .ok_or_else(|| ContractError::new(format!("{where_label} must be positive")))?; + Ok(raw) } fn nullable_f64(value: &Value, where_label: &str) -> Result, ContractError> { diff --git a/products/eval/src/measurement.rs b/products/eval/src/measurement.rs index 1a260c3c26..7790dea1bd 100644 --- a/products/eval/src/measurement.rs +++ b/products/eval/src/measurement.rs @@ -19,7 +19,7 @@ //! Exact 6-counter token usage preservation and standardized model pricing. -use crate::contract::{ContractError, exact_object, nonnegative_u64}; +use crate::contract::{ContractError, exact_object, safe_u64}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; @@ -27,8 +27,9 @@ pub const COST_USD_TOLERANCE: f64 = 1e-9; /// Exact 6-counter normalized token usage. /// -/// Each counter is preserved as reported by the provider or receipt; `total_tokens` -/// is never synthesized or overwritten by summing the other five counters. +/// `input_tokens` is normalized to total input tokens **inclusive** of `cache_read_tokens` +/// and `cache_write_tokens`; `total_tokens` is preserved as reported and is never +/// synthesized or overwritten by summing the other five counters. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct NormalizedUsage { @@ -68,27 +69,27 @@ pub(crate) fn decode_usage( &[], )?; Ok(NormalizedUsage { - input_tokens: nonnegative_u64( + input_tokens: safe_u64( obj.get("inputTokens").unwrap(), &format!("{where_label}.inputTokens"), )?, - output_tokens: nonnegative_u64( + output_tokens: safe_u64( obj.get("outputTokens").unwrap(), &format!("{where_label}.outputTokens"), )?, - cache_read_tokens: nonnegative_u64( + cache_read_tokens: safe_u64( obj.get("cacheReadTokens").unwrap(), &format!("{where_label}.cacheReadTokens"), )?, - cache_write_tokens: nonnegative_u64( + cache_write_tokens: safe_u64( obj.get("cacheWriteTokens").unwrap(), &format!("{where_label}.cacheWriteTokens"), )?, - reasoning_tokens: nonnegative_u64( + reasoning_tokens: safe_u64( obj.get("reasoningTokens").unwrap(), &format!("{where_label}.reasoningTokens"), )?, - total_tokens: nonnegative_u64( + total_tokens: safe_u64( obj.get("totalTokens").unwrap(), &format!("{where_label}.totalTokens"), )?, @@ -121,6 +122,13 @@ pub struct PricingRates { } impl PricingRates { + pub fn for_model(model: &str) -> Option { + match model { + "deepseek-v4-flash" => Some(DEEPSEEK_V4_FLASH_COST), + _ => None, + } + } + pub fn cost_usd(&self, usage: &NormalizedUsage) -> f64 { let cached = usage .cache_read_tokens diff --git a/products/eval/tests/contract.rs b/products/eval/tests/contract.rs index ba0c01b973..6206c2bd72 100644 --- a/products/eval/tests/contract.rs +++ b/products/eval/tests/contract.rs @@ -18,15 +18,32 @@ */ use maka_eval::contract::{ - AttemptCancellation, AttemptMetering, CleanupState, CriticalArtifactsState, EgressAuditState, - EvalResult, EvalResultStatus, ExecutionOutcome, ExperimentSpec, PricingEvidence, - RunFingerprint, SchemaVersion, V1CellAttempt, V2AttemptRecord, VerifierOutcome, - expand_experiment, select_cell_result, -}; -use maka_eval::measurement::{ - DEEPSEEK_V4_FLASH_COST, NormalizedUsage, costs_match_within_tolerance, + AttemptMetering, CleanupState, CriticalArtifactsState, EGRESS_AUDIT_ARTIFACT_PATH, + EgressAuditState, EvalResultStatus, ExperimentSpec, PhaseDurationsMs, PostHocRecovery, + SelectedStatus, SpecFingerprint, SubjectOutcome, TrialOutcome, V1CellAttempt, V2AttemptParts, + V2AttemptRecord, V2Disposition, VerifierOutcome, can_admit_v2_attempt, expand_experiment, + select_cell_result, select_v2_cell_result, }; +use maka_eval::measurement::{NormalizedUsage, PricingRates, costs_match_within_tolerance}; use serde_json::{Map, Value, json}; +use std::num::NonZeroU32; + +const SAMPLE_SHA256: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +fn nz(v: u32) -> NonZeroU32 { + NonZeroU32::new(v).unwrap() +} + +fn sample_usage() -> NormalizedUsage { + NormalizedUsage { + input_tokens: 100, + output_tokens: 20, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 120, + } +} fn sample_two_arm_spec_json() -> Value { json!({ @@ -94,6 +111,36 @@ fn sample_two_arm_spec_json() -> Value { }) } +fn base_v2_parts() -> V2AttemptParts { + V2AttemptParts { + cell_id: "t1::1::maka".into(), + sequence: nz(1), + started_at: 10, + completed_at: 20, + durations: PhaseDurationsMs { + queue_wait_ms: 1, + prepare_ms: 2, + subject_ms: 5, + verify_ms: 1, + cleanup_ms: 1, + total_ms: 10, + }, + trial_outcome: TrialOutcome::Evaluated { + subject: SubjectOutcome::Completed {}, + verifier: VerifierOutcome::Verified { score: 1.0 }, + cancelled_after_verification: false, + }, + metering: AttemptMetering::Complete { + usage: sample_usage(), + model: Some("deepseek-v4-flash".into()), + }, + cleanup_state: CleanupState::Confirmed {}, + audit_state: EgressAuditState::NotRequired {}, + critical_artifacts_state: CriticalArtifactsState::Present {}, + artifacts: Vec::new(), + } +} + #[test] fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { let spec_json = sample_two_arm_spec_json(); @@ -106,25 +153,48 @@ fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { assert_eq!(cells[2].id, "task-beta::1::maka"); assert_eq!(cells[3].id, "task-beta::1::external-cli"); - let mut invalid = spec_json.clone(); - invalid + // Frozen TS safe-integer parity: 1.0 is accepted for integer counts. + let mut float_int_spec = spec_json.clone(); + float_int_spec["repetitions"] = json!(1.0); + float_int_spec["execution"]["maxConcurrentTaskGroups"] = json!(1.0); + let parsed_float_int: ExperimentSpec = serde_json::from_value(float_int_spec).unwrap(); + assert_eq!(parsed_float_int.repetitions, nz(1)); + assert_eq!(parsed_float_int.execution.max_concurrent_task_groups, nz(1)); + + // Reject v2 schemaVersion on ExperimentSpec, unknown fields, 0, fractional, and oversized floats. + let mut v2_spec = spec_json.clone(); + v2_spec["schemaVersion"] = json!("maka.eval.v2"); + assert!(serde_json::from_value::(v2_spec).is_err()); + + let mut unknown = spec_json.clone(); + unknown .as_object_mut() .unwrap() .insert("unknownField".into(), json!(123)); - assert!(serde_json::from_value::(invalid).is_err()); + assert!(serde_json::from_value::(unknown).is_err()); + + for bad_num in [json!(0), json!(1.5), json!(1e30)] { + let mut bad_rep = spec_json.clone(); + bad_rep["repetitions"] = bad_num.clone(); + assert!(serde_json::from_value::(bad_rep).is_err()); + + let mut bad_conc = spec_json.clone(); + bad_conc["execution"]["maxConcurrentTaskGroups"] = bad_num; + assert!(serde_json::from_value::(bad_conc).is_err()); + } // Concurrency changes execution fingerprint but preserves semantic fingerprint. let mut higher_concurrency_json = spec_json.clone(); higher_concurrency_json["execution"]["maxConcurrentTaskGroups"] = json!(16); let higher_spec: ExperimentSpec = serde_json::from_value(higher_concurrency_json).unwrap(); - let fp1 = RunFingerprint::compute(&spec); - let fp2 = RunFingerprint::compute(&higher_spec); + let fp1 = SpecFingerprint::compute(&spec); + let fp2 = SpecFingerprint::compute(&higher_spec); assert_eq!(fp1.semantic_sha256, fp2.semantic_sha256); assert_ne!(fp1.execution_sha256, fp2.execution_sha256); // Changing experiment.id, float/integer equivalence (1.0 vs 1), and nested object - // key insertion order all preserve semantic_sha256 even under preserve_order. + // key insertion order preserve semantic_sha256 even under preserve_order. let mut reordered_json = spec_json; reordered_json["id"] = json!("tb21-deepseek-v4-flash-rerun"); let mut reordered_budget = Map::new(); @@ -132,12 +202,12 @@ fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { reordered_budget.insert("timeoutMultiplier".into(), json!(1.0)); reordered_json["budget"] = Value::Object(reordered_budget); let reordered_spec: ExperimentSpec = serde_json::from_value(reordered_json).unwrap(); - let fp3 = RunFingerprint::compute(&reordered_spec); + let fp3 = SpecFingerprint::compute(&reordered_spec); assert_eq!(fp1.semantic_sha256, fp3.semantic_sha256); } #[test] -fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within_1e9() { +fn six_counter_usage_and_authoritative_model_pricing() { let u1: NormalizedUsage = serde_json::from_value(json!({ "inputTokens": 1000.0, "outputTokens": 250, @@ -156,29 +226,31 @@ fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within total_tokens: 7777, }; - assert!( - serde_json::from_value::(json!({ + for invalid in [ + json!({ "inputTokens": 10, "outputTokens": 5, "cacheReadTokens": 0, "cacheWriteTokens": 0, "reasoningTokens": 0 - })) - .is_err() - ); - assert!( - serde_json::from_value::(json!({ + }), + json!({ "inputTokens": -1, "outputTokens": 5, "cacheReadTokens": 0, "cacheWriteTokens": 0, "reasoningTokens": 0, "totalTokens": 4 - })) - .is_err() - ); - assert!( - serde_json::from_value::(json!({ + }), + json!({ + "inputTokens": 1e30, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 15 + }), + json!({ "inputTokens": 10, "outputTokens": 5, "cacheReadTokens": 0, @@ -186,9 +258,10 @@ fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within "reasoningTokens": 0, "totalTokens": 15, "extra": 1 - })) - .is_err() - ); + }), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } let mut combined = u1.clone(); combined.accumulate_exact(&u2); @@ -197,79 +270,124 @@ fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within assert_eq!(combined.cache_read_tokens, 600); assert_eq!(combined.cache_write_tokens, 100); assert_eq!(combined.reasoning_tokens, 100); - // Must equal 9999 + 7777 = 17776, never the sum of the other 5 counters! assert_eq!(combined.total_tokens, 17776); // uncached = 1000 - 400 - 100 = 500 // (500 * 0.22 + 400 * 0.007 + 100 * 0.22 + 250 * 0.66) / 1_000_000 = 0.0002998 - let cost = DEEPSEEK_V4_FLASH_COST.cost_usd(&u1); + let rates = PricingRates::for_model("deepseek-v4-flash").unwrap(); + let cost = rates.cost_usd(&u1); assert!(costs_match_within_tolerance(Some(cost), Some(0.0002998))); assert!(!costs_match_within_tolerance( Some(cost), Some(0.0002998 + 1e-8) )); + assert!(PricingRates::for_model("custom-local").is_none()); } #[test] -fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_terminal() { - let attempts = vec![ - V1CellAttempt { - cell_id: "t1::1::maka".into(), - sequence: 3, - started_at: 50, - completed_at: 60, - result: EvalResult { - score: Some(1.0), - usage: None, - cost_usd: None, - duration_ms: 10, - status: EvalResultStatus::Completed, - failure_reason: None, - artifacts: Vec::new(), - }, +fn strict_boundary_decoding_rejects_unknown_fields_and_illegal_states() { + let v2_base = json!({ + "schemaVersion": "maka.eval.v2", + "cellId": "t1::1::maka", + "sequence": 1, + "startedAt": 10, + "completedAt": 20, + "durations": { + "queueWaitMs": 1, + "prepareMs": 2, + "subjectMs": 5, + "verifyMs": 1, + "cleanupMs": 1, + "totalMs": 10 }, - V1CellAttempt { - cell_id: "t1::1::maka".into(), - sequence: 1, - started_at: 10, - completed_at: 20, - result: EvalResult { - score: None, - usage: None, - cost_usd: None, - duration_ms: 10, - status: EvalResultStatus::InfraFailed, - failure_reason: Some("docker pull failed".into()), - artifacts: Vec::new(), - }, + "trialOutcome": { + "stage": "evaluated", + "subject": { "kind": "completed" }, + "verifier": { "kind": "verified", "score": 1.0 }, + "cancelledAfterVerification": false }, - V1CellAttempt { - cell_id: "t1::1::maka".into(), - sequence: 2, - started_at: 30, - completed_at: 40, - result: EvalResult { - score: Some(0.0), - usage: None, - cost_usd: None, - duration_ms: 10, - status: EvalResultStatus::Completed, - failure_reason: None, - artifacts: Vec::new(), + "metering": { + "state": "complete", + "usage": { + "inputTokens": 10, + "outputTokens": 5, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 15 }, + "model": "deepseek-v4-flash" }, - ]; + "cleanupState": { "kind": "confirmed" }, + "auditState": { "kind": "not_required" }, + "criticalArtifactsState": { "kind": "present" }, + "artifacts": [] + }); + let parsed_v2: V2AttemptRecord = serde_json::from_value(v2_base.clone()).unwrap(); + let encoded_v2 = serde_json::to_value(&parsed_v2).unwrap(); + assert_eq!( + serde_json::from_value::(encoded_v2).unwrap(), + parsed_v2 + ); - let selected = select_cell_result(&attempts).unwrap(); - assert_eq!(selected.sequence, 2); - assert_eq!(selected.result.score, Some(0.0)); -} + // Every empty-struct tagged variant and metering.unknown must reject extra fields. + let mut bad_subject = v2_base.clone(); + bad_subject["trialOutcome"]["subject"] = json!({ "kind": "completed", "extra": 1 }); + assert!(serde_json::from_value::(bad_subject).is_err()); + + let mut bad_cleanup = v2_base.clone(); + bad_cleanup["cleanupState"] = json!({ "kind": "confirmed", "extra": true }); + assert!(serde_json::from_value::(bad_cleanup).is_err()); + + let mut bad_audit_nr = v2_base.clone(); + bad_audit_nr["auditState"] = json!({ "kind": "not_required", "extra": "x" }); + assert!(serde_json::from_value::(bad_audit_nr).is_err()); + + let mut bad_audit_missing = v2_base.clone(); + bad_audit_missing["auditState"] = json!({ "kind": "missing", "extra": "x" }); + assert!(serde_json::from_value::(bad_audit_missing).is_err()); + + let mut bad_artifacts = v2_base.clone(); + bad_artifacts["criticalArtifactsState"] = json!({ "kind": "present", "extra": 1 }); + assert!(serde_json::from_value::(bad_artifacts).is_err()); + + let mut bad_metering_unknown = v2_base.clone(); + bad_metering_unknown["metering"] = json!({ "state": "unknown", "model": "deepseek-v4-flash" }); + assert!(serde_json::from_value::(bad_metering_unknown).is_err()); + + // Illegal v2 invariants on wire & Rust boundary. + let mut wrong_schema = v2_base.clone(); + wrong_schema["schemaVersion"] = json!("maka.eval.v1"); + assert!(serde_json::from_value::(wrong_schema).is_err()); + + let mut reversed_times = v2_base.clone(); + reversed_times["completedAt"] = json!(5); + assert!(serde_json::from_value::(reversed_times).is_err()); + + let mut empty_missing_paths = v2_base.clone(); + empty_missing_paths["criticalArtifactsState"] = json!({ "kind": "missing", "paths": [] }); + assert!(serde_json::from_value::(empty_missing_paths).is_err()); + + let mut empty_reason = v2_base.clone(); + empty_reason["trialOutcome"] = json!({ "stage": "infra_failed", "reason": "" }); + assert!(serde_json::from_value::(empty_reason).is_err()); + + let mut bad_sha = v2_base; + bad_sha["auditState"] = json!({ "kind": "recorded", "bytes": 12, "sha256": "NOT-HEX" }); + assert!(serde_json::from_value::(bad_sha).is_err()); + + let mut non_finite = base_v2_parts(); + non_finite.trial_outcome = TrialOutcome::Evaluated { + subject: SubjectOutcome::Completed {}, + verifier: VerifierOutcome::Verified { score: f64::NAN }, + cancelled_after_verification: false, + }; + assert!(V2AttemptRecord::try_new(non_finite).is_err()); -#[test] -fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly() { + // V1CellAttempt: safe integer parity (1.0 ok, 1e30 & 0 rejected), lossless round-trip. let valid_v1 = json!({ "cellId": "t1::1::maka", - "sequence": 1, + "sequence": 1.0, "startedAt": 10, "completedAt": 20, "result": { @@ -282,217 +400,161 @@ fn contract_deserialization_rejects_illegal_states_and_v1_round_trips_losslessly "artifacts": [] } }); - assert!(serde_json::from_value::(valid_v1.clone()).is_ok()); - - let mut omitted_score = valid_v1.clone(); - omitted_score["result"] - .as_object_mut() - .unwrap() - .remove("score"); - assert!(serde_json::from_value::(omitted_score).is_err()); + let parsed_v1: V1CellAttempt = serde_json::from_value(valid_v1.clone()).unwrap(); + assert_eq!(parsed_v1.sequence, nz(1)); + let encoded_v1 = serde_json::to_value(&parsed_v1).unwrap(); + assert_eq!( + serde_json::from_value::(encoded_v1).unwrap(), + parsed_v1 + ); let mut zero_seq = valid_v1.clone(); zero_seq["sequence"] = json!(0); assert!(serde_json::from_value::(zero_seq).is_err()); - let mut negative_v1_cost = valid_v1.clone(); - negative_v1_cost["result"]["costUsd"] = json!(-0.01); - assert!(serde_json::from_value::(negative_v1_cost).is_err()); - - let mut cost_without_usage = valid_v1.clone(); - cost_without_usage["result"]["score"] = json!(0.0); - cost_without_usage["result"]["costUsd"] = json!(0.01); - let parsed_cost_without_usage: V1CellAttempt = - serde_json::from_value(cost_without_usage).unwrap(); - assert_eq!(parsed_cost_without_usage.result.cost_usd, Some(0.01)); - assert_eq!(parsed_cost_without_usage.result.usage, None); - let encoded_cost_without_usage = serde_json::to_value(&parsed_cost_without_usage).unwrap(); - assert_eq!( - serde_json::from_value::(encoded_cost_without_usage).unwrap(), - parsed_cost_without_usage - ); + let mut saturated_ts = valid_v1; + saturated_ts["startedAt"] = json!(1e30); + assert!(serde_json::from_value::(saturated_ts).is_err()); +} - let completed_with_reason_result = json!({ - "score": 0.0, - "usage": null, - "costUsd": null, - "durationMs": 10, - "status": "completed", - "failureReason": "external subject detail", - "artifacts": [] - }); - let parsed_eval_result: EvalResult = - serde_json::from_value(completed_with_reason_result.clone()).unwrap(); +#[test] +fn three_gate_v2_funnel_selection_and_v1_projections() { + // Gate 1 dominance: Indeterminate overrides missing critical artifacts and missing audit. + let mut indet_parts = base_v2_parts(); + indet_parts.trial_outcome = TrialOutcome::Indeterminate { + reason: "relay disconnected".into(), + }; + indet_parts.metering = AttemptMetering::Unknown; + indet_parts.critical_artifacts_state = CriticalArtifactsState::Missing { + paths: vec!["transcript.json".into()], + }; + indet_parts.audit_state = EgressAuditState::Missing {}; + let indet_rec = V2AttemptRecord::try_new(indet_parts).unwrap(); assert_eq!( - parsed_eval_result.failure_reason.as_deref(), - Some("external subject detail") + indet_rec.disposition(), + V2Disposition::BlockedIndeterminate { + failure_reason: "relay disconnected".into(), + } ); - let mut completed_with_reason = valid_v1.clone(); - completed_with_reason["result"] = completed_with_reason_result; - let parsed_completed_with_reason: V1CellAttempt = - serde_json::from_value(completed_with_reason).unwrap(); - let encoded_completed_with_reason = - serde_json::to_value(&parsed_completed_with_reason).unwrap(); assert_eq!( - serde_json::from_value::(encoded_completed_with_reason).unwrap(), - parsed_completed_with_reason - ); - - // V2AttemptRecord rejects schemaVersion != maka.eval.v2, illegal metering states, - // unknown fields on metering unit variants, completed attempts with failureReason, - // and standalone invalid PricingEvidence. - let v2_base = json!({ - "schemaVersion": "maka.eval.v2", - "cellId": "t1::1::maka", - "sequence": 1, - "startedAt": 10, - "completedAt": 20, - "durationMs": 10, - "executionOutcome": "completed", - "verifierOutcome": { "kind": "skipped" }, - "metering": { "state": "unknown" }, - "cleanupState": { "kind": "confirmed" }, - "auditState": { "kind": "not_required" }, - "criticalArtifactsState": { "kind": "present" }, - "cancellation": "none", - "recoveredPostHoc": false, - "failureReason": null, - "artifacts": [] - }); - assert!(serde_json::from_value::(v2_base.clone()).is_ok()); - - let mut v2_wrong_schema = v2_base.clone(); - v2_wrong_schema["schemaVersion"] = json!("maka.eval.v1"); - assert!(serde_json::from_value::(v2_wrong_schema).is_err()); - - let mut complete_without_usage = v2_base.clone(); - complete_without_usage["metering"] = json!({ "state": "complete", "usage": null }); - assert!(serde_json::from_value::(complete_without_usage).is_err()); - - let mut unknown_with_pricing = v2_base.clone(); - unknown_with_pricing["metering"] = json!({ - "state": "unknown", - "pricing": { "kind": "subject_reported", "costUsd": 0.01 } - }); - assert!(serde_json::from_value::(unknown_with_pricing).is_err()); - - let valid_usage = json!({ - "inputTokens": 10, - "outputTokens": 5, - "cacheReadTokens": 0, - "cacheWriteTokens": 0, - "reasoningTokens": 0, - "totalTokens": 15 - }); - - let mut lower_bound_with_pricing = v2_base.clone(); - lower_bound_with_pricing["metering"] = json!({ - "state": "lower_bound", - "usage": valid_usage, - "pricing": { "kind": "subject_reported", "costUsd": 0.01 } - }); - assert!(serde_json::from_value::(lower_bound_with_pricing).is_err()); - - let mut completed_v2_with_reason = v2_base.clone(); - completed_v2_with_reason["failureReason"] = json!("unexpected"); - assert!(serde_json::from_value::(completed_v2_with_reason).is_err()); - - let mut negative_cost = v2_base; - negative_cost["metering"] = json!({ - "state": "complete", - "usage": valid_usage, - "pricing": { "kind": "subject_reported", "costUsd": -0.5 } - }); - assert!(serde_json::from_value::(negative_cost).is_err()); - - assert!( - serde_json::from_value::(json!({ - "kind": "subject_reported", - "costUsd": -0.5 - })) - .is_err() + indet_rec.project_v1().result.status, + EvalResultStatus::Indeterminate ); + assert!(!can_admit_v2_attempt(&[indet_rec], 3)); - let parsed_v1: V1CellAttempt = serde_json::from_value(valid_v1).unwrap(); - let encoded = serde_json::to_value(&parsed_v1).unwrap(); + // Gate 2 root-cause preservation: pre-verification InfraFailed keeps its own reason + // even with Unknown metering and missing artifacts, and admits retries within budget. + let mut infra_parts = base_v2_parts(); + infra_parts.trial_outcome = TrialOutcome::InfraFailed { + reason: "docker pull failed".into(), + }; + infra_parts.metering = AttemptMetering::Unknown; + infra_parts.critical_artifacts_state = CriticalArtifactsState::Missing { + paths: vec!["transcript.json".into()], + }; + let infra_rec = V2AttemptRecord::try_new(infra_parts).unwrap(); assert_eq!( - serde_json::from_value::(encoded).unwrap(), - parsed_v1 + infra_rec.disposition(), + V2Disposition::RetryableInfra { + failure_reason: "docker pull failed".into(), + } ); -} + assert!(!can_admit_v2_attempt(std::slice::from_ref(&infra_rec), 0)); + assert!(can_admit_v2_attempt(std::slice::from_ref(&infra_rec), 1)); + + // Metering gates on Completed: Unknown -> BlockedIndeterminate; LowerBound -> RetryableInfra. + let mut completed_unknown = base_v2_parts(); + completed_unknown.metering = AttemptMetering::Unknown; + let completed_unknown_rec = V2AttemptRecord::try_new(completed_unknown).unwrap(); + assert!(matches!( + completed_unknown_rec.disposition(), + V2Disposition::BlockedIndeterminate { .. } + )); -#[test] -fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { - let usage = NormalizedUsage { - input_tokens: 100, - output_tokens: 20, - cache_read_tokens: 0, - cache_write_tokens: 0, - reasoning_tokens: 0, - total_tokens: 120, + let mut completed_lb = base_v2_parts(); + completed_lb.metering = AttemptMetering::LowerBound { + usage: sample_usage(), }; + let completed_lb_rec = V2AttemptRecord::try_new(completed_lb).unwrap(); + assert!(matches!( + completed_lb_rec.disposition(), + V2Disposition::RetryableInfra { .. } + )); - // 1. Timeout + Verified(0.75) + LowerBound retains score and usage with costUsd == None; - // with Complete + AfterVerification + Recorded audit, preserves markers as SubjectFailed. - let base_v2 = V2AttemptRecord { - schema_version: SchemaVersion::V2, - cell_id: "t1::1::maka".into(), - sequence: 1, - started_at: 10, - completed_at: 20, - duration_ms: 10, - execution_outcome: ExecutionOutcome::Timeout, - verifier_outcome: VerifierOutcome::Verified { score: 0.75 }, - metering: AttemptMetering::LowerBound { - usage: usage.clone(), + // Scenario 1: Timeout + Verified(0.75) + LowerBound -> Selected(SubjectFailed, Some(0.75)), + // costUsd == None, blocks resampling; Complete derives standardized cost via PricingRates. + let mut timeout_lb_parts = base_v2_parts(); + timeout_lb_parts.trial_outcome = TrialOutcome::Evaluated { + subject: SubjectOutcome::Timeout { + reason: Some("timed out".into()), }, - cleanup_state: CleanupState::Confirmed, - audit_state: EgressAuditState::NotRequired, - critical_artifacts_state: CriticalArtifactsState::Present, - cancellation: AttemptCancellation::None, - recovered_post_hoc: false, - failure_reason: Some("timed out".into()), - artifacts: Vec::new(), + verifier: VerifierOutcome::Verified { score: 0.75 }, + cancelled_after_verification: false, }; - let v1_timeout_lb = base_v2.project_v1(); - assert_eq!(v1_timeout_lb.result.status, EvalResultStatus::SubjectFailed); - assert_eq!(v1_timeout_lb.result.score, Some(0.75)); - assert_eq!(v1_timeout_lb.result.usage, Some(usage.clone())); - assert_eq!(v1_timeout_lb.result.cost_usd, None); - - let mut v2_timeout_complete = base_v2.clone(); - v2_timeout_complete.metering = AttemptMetering::Complete { - usage: usage.clone(), - pricing: Some(PricingEvidence::Model { - model: "deepseek-v4-flash".into(), - cost_usd: Some(0.0000352), - }), + timeout_lb_parts.metering = AttemptMetering::LowerBound { + usage: sample_usage(), + }; + let timeout_lb_rec = V2AttemptRecord::try_new(timeout_lb_parts.clone()).unwrap(); + assert_eq!( + timeout_lb_rec.disposition(), + V2Disposition::Selected { + status: SelectedStatus::SubjectFailed, + score: Some(0.75), + failure_reason: Some("timed out".into()), + } + ); + assert!(!can_admit_v2_attempt( + std::slice::from_ref(&timeout_lb_rec), + 3 + )); + let timeout_lb_v1 = timeout_lb_rec.project_v1(); + assert_eq!(timeout_lb_v1.result.status, EvalResultStatus::SubjectFailed); + assert_eq!(timeout_lb_v1.result.score, Some(0.75)); + assert_eq!(timeout_lb_v1.result.usage, Some(sample_usage())); + assert_eq!(timeout_lb_v1.result.cost_usd, None); + + timeout_lb_parts.metering = AttemptMetering::Complete { + usage: sample_usage(), + model: Some("deepseek-v4-flash".into()), }; - v2_timeout_complete.audit_state = EgressAuditState::Recorded { + let timeout_complete_v1 = V2AttemptRecord::try_new(timeout_lb_parts) + .unwrap() + .project_v1(); + assert!(costs_match_within_tolerance( + timeout_complete_v1.result.cost_usd, + Some(0.0000352) + )); + + // Scenario 2: EgressAuditState::Recorded keeps Completed(1.0) and projects full audit fields; + // Missing -> RetryableInfra + egress-audit-missing artifact. + let mut audit_parts = base_v2_parts(); + audit_parts.audit_state = EgressAuditState::Recorded { + bytes: 512, + sha256: SAMPLE_SHA256.into(), truncated: true, policy_error_count: 2, + malformed_line_count: 1, }; - v2_timeout_complete.cancellation = AttemptCancellation::AfterVerification; - let v1_timeout_complete = v2_timeout_complete.project_v1(); - assert_eq!( - v1_timeout_complete.result.status, - EvalResultStatus::SubjectFailed - ); - assert_eq!(v1_timeout_complete.result.score, Some(0.75)); - assert_eq!(v1_timeout_complete.result.cost_usd, Some(0.0000352)); - assert!( - v1_timeout_complete - .result - .artifacts - .iter() - .any(|a| a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification")) - ); - let audit_art = v1_timeout_complete + let audit_v1 = V2AttemptRecord::try_new(audit_parts.clone()) + .unwrap() + .project_v1(); + assert_eq!(audit_v1.result.status, EvalResultStatus::Completed); + assert_eq!(audit_v1.result.score, Some(1.0)); + let audit_art = audit_v1 .result .artifacts .iter() .find(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) .unwrap(); + assert_eq!( + audit_art.get("path").and_then(Value::as_str), + Some(EGRESS_AUDIT_ARTIFACT_PATH) + ); + assert_eq!(audit_art.get("bytes").and_then(Value::as_u64), Some(512)); + assert_eq!( + audit_art.get("sha256").and_then(Value::as_str), + Some(format!("sha256:{SAMPLE_SHA256}").as_str()) + ); assert_eq!( audit_art.get("truncated").and_then(Value::as_bool), Some(true) @@ -501,110 +563,169 @@ fn v2_to_v1_projections_cover_all_five_semantic_difference_scenarios() { audit_art.get("policyErrorCount").and_then(Value::as_u64), Some(2) ); - - // 2. Egress audit truncation preserves Completed + reward, whereas Missing fails closed. - let completed_base = V2AttemptRecord { - execution_outcome: ExecutionOutcome::Completed, - verifier_outcome: VerifierOutcome::Verified { score: 1.0 }, - metering: AttemptMetering::Complete { - usage: usage.clone(), - pricing: Some(PricingEvidence::Model { - model: "deepseek-v4-flash".into(), - cost_usd: Some(0.0000352), - }), - }, - audit_state: EgressAuditState::Recorded { - truncated: true, - policy_error_count: 0, - }, - failure_reason: None, - ..base_v2 - }; - let v1_audit_trunc = completed_base.project_v1(); - assert_eq!(v1_audit_trunc.result.status, EvalResultStatus::Completed); - assert_eq!(v1_audit_trunc.result.score, Some(1.0)); - assert!(v1_audit_trunc.result.artifacts.iter().any(|a| { - a.get("kind").and_then(Value::as_str) == Some("egress-audit") - && a.get("truncated").and_then(Value::as_bool) == Some(true) - })); - - let mut v2_audit_missing = completed_base.clone(); - v2_audit_missing.audit_state = EgressAuditState::Missing; - let v1_audit_missing = v2_audit_missing.project_v1(); assert_eq!( - v1_audit_missing.result.status, - EvalResultStatus::InfraFailed + audit_art.get("malformedLineCount").and_then(Value::as_u64), + Some(1) ); - assert_eq!(v1_audit_missing.result.score, None); - // 3. CriticalArtifactsState::Present keeps Completed, whereas Missing fails closed. + audit_parts.audit_state = EgressAuditState::Missing {}; + let missing_audit_v1 = V2AttemptRecord::try_new(audit_parts).unwrap().project_v1(); assert_eq!( - completed_base.project_v1().result.status, - EvalResultStatus::Completed + missing_audit_v1.result.status, + EvalResultStatus::InfraFailed ); - let mut v2_missing_art = completed_base.clone(); - v2_missing_art.critical_artifacts_state = CriticalArtifactsState::Missing { + assert_eq!(missing_audit_v1.result.score, None); + assert!(missing_audit_v1.result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("egress-audit-missing") + && a.get("path").and_then(Value::as_str) == Some(EGRESS_AUDIT_ARTIFACT_PATH) + })); + + // Scenario 3: CriticalArtifactsState::Missing -> RetryableInfra (score: None). + let mut missing_art_parts = base_v2_parts(); + missing_art_parts.critical_artifacts_state = CriticalArtifactsState::Missing { paths: vec!["transcript.json".into()], }; - let v1_missing_art = v2_missing_art.project_v1(); - assert_eq!(v1_missing_art.result.status, EvalResultStatus::InfraFailed); - assert_eq!(v1_missing_art.result.score, None); - - // 4. Late cancellation: BeforeVerification -> Indeterminate(None), AfterVerification -> Completed(score). - let mut v2_cancel_before = completed_base.clone(); - v2_cancel_before.cancellation = AttemptCancellation::BeforeVerification; - let v1_cancel_before = v2_cancel_before.project_v1(); + let missing_art_v1 = V2AttemptRecord::try_new(missing_art_parts) + .unwrap() + .project_v1(); + assert_eq!(missing_art_v1.result.status, EvalResultStatus::InfraFailed); + assert_eq!(missing_art_v1.result.score, None); + + // Scenario 4: CancelledBeforeVerification -> BlockedIndeterminate; + // Evaluated { cancelled_after_verification: true } -> Selected(Completed) + marker. + let mut cancel_before_parts = base_v2_parts(); + cancel_before_parts.trial_outcome = TrialOutcome::CancelledBeforeVerification { reason: None }; + let cancel_before_v1 = V2AttemptRecord::try_new(cancel_before_parts) + .unwrap() + .project_v1(); assert_eq!( - v1_cancel_before.result.status, + cancel_before_v1.result.status, EvalResultStatus::Indeterminate ); - assert_eq!(v1_cancel_before.result.score, None); + assert_eq!(cancel_before_v1.result.score, None); - let mut v2_cancel_after = completed_base.clone(); - v2_cancel_after.cancellation = AttemptCancellation::AfterVerification; - let v1_cancel_after = v2_cancel_after.project_v1(); - assert_eq!(v1_cancel_after.result.status, EvalResultStatus::Completed); - assert_eq!(v1_cancel_after.result.score, Some(1.0)); + let mut cancel_after_parts = base_v2_parts(); + cancel_after_parts.trial_outcome = TrialOutcome::Evaluated { + subject: SubjectOutcome::Completed {}, + verifier: VerifierOutcome::Verified { score: 1.0 }, + cancelled_after_verification: true, + }; + let cancel_after_v1 = V2AttemptRecord::try_new(cancel_after_parts) + .unwrap() + .project_v1(); + assert_eq!(cancel_after_v1.result.status, EvalResultStatus::Completed); + assert_eq!(cancel_after_v1.result.score, Some(1.0)); assert!( - v1_cancel_after + cancel_after_v1 .result .artifacts .iter() .any(|a| a.get("kind").and_then(Value::as_str) == Some("cancelled-after-verification")) ); - // 5. Post-hoc recovery emits artifact marker while keeping recovered usage/cost and sequence; - // missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0). - let mut v2_recovered = completed_base.clone(); - v2_recovered.sequence = 2; - v2_recovered.recovered_post_hoc = true; - let v1_recovered = v2_recovered.project_v1(); - assert_eq!(v1_recovered.sequence, 2); - assert_eq!(v1_recovered.result.status, EvalResultStatus::Completed); - assert_eq!(v1_recovered.result.usage, Some(usage)); - assert_eq!(v1_recovered.result.cost_usd, Some(0.0000352)); + // Scenario 5: PostHocRecovery upgrades immutable record monotonically via projection; + // MissingReward -> RetryableInfra (score: None); genuine 0.0 is terminal and earliest-selected. + let mut unrecovered_parts = base_v2_parts(); + unrecovered_parts.sequence = nz(2); + unrecovered_parts.cleanup_state = CleanupState::Unconfirmed { + reason: "cleanup pending".into(), + }; + unrecovered_parts.metering = AttemptMetering::Unknown; + let unrecovered_rec = V2AttemptRecord::try_new(unrecovered_parts).unwrap(); + assert!(matches!( + unrecovered_rec.disposition(), + V2Disposition::BlockedIndeterminate { .. } + )); + + let recovery = PostHocRecovery::try_new( + 30, + Some(AttemptMetering::Complete { + usage: sample_usage(), + model: Some("deepseek-v4-flash".into()), + }), + Some(CleanupState::Confirmed {}), + ) + .unwrap(); + let recovered_disp = unrecovered_rec.disposition_with_recovery(std::slice::from_ref(&recovery)); + assert_eq!( + recovered_disp, + V2Disposition::Selected { + status: SelectedStatus::Completed, + score: Some(1.0), + failure_reason: None, + } + ); + let recovered_v1 = unrecovered_rec.project_v1_with_recovery(&[recovery]); + assert_eq!(recovered_v1.sequence, nz(2)); + assert_eq!(recovered_v1.result.status, EvalResultStatus::Completed); + assert_eq!(recovered_v1.result.usage, Some(sample_usage())); + assert!(costs_match_within_tolerance( + recovered_v1.result.cost_usd, + Some(0.0000352) + )); assert!( - v1_recovered + recovered_v1 .result .artifacts .iter() .any(|a| a.get("kind").and_then(Value::as_str) == Some("post-hoc-evidence-recovery")) ); + // Original record stays immutable. + assert!(matches!( + unrecovered_rec.cleanup_state(), + CleanupState::Unconfirmed { .. } + )); - let mut v2_missing_reward = completed_base.clone(); - v2_missing_reward.verifier_outcome = VerifierOutcome::MissingReward { - reason: "reward.json missing".into(), + let mut missing_reward_parts = base_v2_parts(); + missing_reward_parts.trial_outcome = TrialOutcome::Evaluated { + subject: SubjectOutcome::Completed {}, + verifier: VerifierOutcome::MissingReward { + reason: "reward.json missing".into(), + }, + cancelled_after_verification: false, }; - let v1_missing_reward = v2_missing_reward.project_v1(); + let missing_reward_v1 = V2AttemptRecord::try_new(missing_reward_parts) + .unwrap() + .project_v1(); assert_eq!( - v1_missing_reward.result.status, + missing_reward_v1.result.status, EvalResultStatus::InfraFailed ); - assert_eq!(v1_missing_reward.result.score, None); + assert_eq!(missing_reward_v1.result.score, None); + + // Selection picks earliest valid attempt (sequence 2, score 0.0) over sequence 1 (InfraFailed) + // and sequence 3 (Completed, score 1.0) in both v2 and v1. + let mut zero_seq2_parts = base_v2_parts(); + zero_seq2_parts.sequence = nz(2); + zero_seq2_parts.trial_outcome = TrialOutcome::Evaluated { + subject: SubjectOutcome::Completed {}, + verifier: VerifierOutcome::Verified { score: 0.0 }, + cancelled_after_verification: false, + }; + let zero_seq2 = V2AttemptRecord::try_new(zero_seq2_parts).unwrap(); + + let mut win_seq3_parts = base_v2_parts(); + win_seq3_parts.sequence = nz(3); + let win_seq3 = V2AttemptRecord::try_new(win_seq3_parts).unwrap(); + + let v2_attempts = [win_seq3.clone(), infra_rec.clone(), zero_seq2.clone()]; + let selected_v2 = select_v2_cell_result(&v2_attempts).unwrap(); + assert_eq!(selected_v2.sequence(), nz(2)); + assert_eq!( + selected_v2.disposition(), + V2Disposition::Selected { + status: SelectedStatus::Completed, + score: Some(0.0), + failure_reason: None, + } + ); - let mut v2_zero_score = completed_base; - v2_zero_score.verifier_outcome = VerifierOutcome::Verified { score: 0.0 }; - let v1_zero_score = v2_zero_score.project_v1(); - assert_eq!(v1_zero_score.result.status, EvalResultStatus::Completed); - assert_eq!(v1_zero_score.result.score, Some(0.0)); + let v1_attempts = [ + win_seq3.project_v1(), + infra_rec.project_v1(), + zero_seq2.project_v1(), + ]; + let selected_v1 = select_cell_result(&v1_attempts).unwrap(); + assert_eq!(selected_v1.sequence, nz(2)); + assert_eq!(selected_v1.result.score, Some(0.0)); }