From e2007fa29aea14ea6b7087fc00bcc4ba843d2731 Mon Sep 17 00:00:00 2001 From: colafornia Date: Thu, 12 Nov 2026 00:31:45 +0800 Subject: [PATCH 1/3] feat(eval): add maka-eval measurement and control plane Generated-by: Codex --- Cargo.lock | 26 + Cargo.toml | 2 + products/eval/Cargo.toml | 51 + products/eval/README.md | 36 + products/eval/README.zh-CN.md | 36 + .../harbor/docker-compose-egress-proxy.yaml | 57 + products/eval/harbor/egress-proxy/Dockerfile | 25 + .../eval/harbor/egress-proxy/entrypoint.sh | 63 + .../eval/harbor/egress-proxy/network-policy | 148 ++ products/eval/harbor/egress_filter.py | 501 +++++++ products/eval/harbor/eval_framework.py | 63 + products/eval/harbor/relay_agent.py | 723 ++++++++++ products/eval/harbor/run_trial.py | 230 ++++ products/eval/src/contract.rs | 1081 +++++++++++++++ products/eval/src/environment.rs | 531 ++++++++ products/eval/src/environment/harbor_relay.rs | 971 +++++++++++++ products/eval/src/lib.rs | 28 + products/eval/src/main.rs | 297 ++++ products/eval/src/measurement.rs | 682 +++++++++ products/eval/src/measurement/gateway.rs | 990 ++++++++++++++ products/eval/src/report.rs | 313 +++++ products/eval/src/runner.rs | 732 ++++++++++ products/eval/src/store.rs | 874 ++++++++++++ products/eval/src/subject.rs | 63 + products/eval/src/subject/external_cli.rs | 662 +++++++++ products/eval/src/subject/maka.rs | 936 +++++++++++++ products/eval/tests/integration.rs | 1213 +++++++++++++++++ scripts/asf-license-headers.mjs | 1 + 28 files changed, 11335 insertions(+) create mode 100644 products/eval/Cargo.toml create mode 100644 products/eval/README.md create mode 100644 products/eval/README.zh-CN.md create mode 100644 products/eval/harbor/docker-compose-egress-proxy.yaml create mode 100644 products/eval/harbor/egress-proxy/Dockerfile create mode 100755 products/eval/harbor/egress-proxy/entrypoint.sh create mode 100755 products/eval/harbor/egress-proxy/network-policy create mode 100644 products/eval/harbor/egress_filter.py create mode 100644 products/eval/harbor/eval_framework.py create mode 100644 products/eval/harbor/relay_agent.py create mode 100644 products/eval/harbor/run_trial.py create mode 100644 products/eval/src/contract.rs create mode 100644 products/eval/src/environment.rs create mode 100644 products/eval/src/environment/harbor_relay.rs create mode 100644 products/eval/src/lib.rs create mode 100644 products/eval/src/main.rs create mode 100644 products/eval/src/measurement.rs create mode 100644 products/eval/src/measurement/gateway.rs create mode 100644 products/eval/src/report.rs create mode 100644 products/eval/src/runner.rs create mode 100644 products/eval/src/store.rs create mode 100644 products/eval/src/subject.rs create mode 100644 products/eval/src/subject/external_cli.rs create mode 100644 products/eval/src/subject/maka.rs create mode 100644 products/eval/tests/integration.rs diff --git a/Cargo.lock b/Cargo.lock index 17a80d7335a..2271c906caf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6909,6 +6909,32 @@ dependencies = [ "uuid", ] +[[package]] +name = "maka-eval" +version = "0.2.0" +dependencies = [ + "base64 0.23.1", + "clap", + "futures-util", + "hex", + "http-body-util", + "hyper", + "hyper-util", + "libc", + "regex", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sha2 0.11.1", + "tempfile", + "thiserror 2.0.21", + "tokio", + "tokio-util", + "url", + "uuid", +] + [[package]] name = "maka-event-log" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 42d150ce5ab..8023006b8ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ members = [ "plugins/workspace/workspace-files/native", "plugins/workspace/support", "products/maka/acp", + "products/eval", "plugins/agent/engine", "plugins/agent/native", "plugins/workspace/workspace-files/patch", @@ -149,6 +150,7 @@ imagesize = { version = "=0.15.0", default-features = false, features = ["png", jsonschema = { version = "0.60.1", default-features = false } libc = "0.2.191" maka-acp = { path = "products/maka/acp" } +maka-eval = { path = "products/eval" } maka-agent-engine = { path = "plugins/agent/engine" } maka-agent-native = { path = "plugins/agent/native" } maka-assistant = { path = "plugins/agent/assistant" } diff --git a/products/eval/Cargo.toml b/products/eval/Cargo.toml new file mode 100644 index 00000000000..9818cef1337 --- /dev/null +++ b/products/eval/Cargo.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "maka-eval" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[[bin]] +name = "maka-eval" +path = "src/main.rs" + +[dependencies] +base64 = "0.23.1" +clap.workspace = true +futures-util.workspace = true +hex.workspace = true +http-body-util = "0.1.5" +hyper = { version = "1.12.0", features = ["server", "http1"] } +hyper-util = { version = "0.1.21", features = ["tokio"] } +regex.workspace = true +reqwest.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tempfile.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "io-std", "net", "process"] } +tokio-util.workspace = true +url = "2.5.8" +uuid.workspace = true + +[target.'cfg(unix)'.dependencies] +libc.workspace = true diff --git a/products/eval/README.md b/products/eval/README.md new file mode 100644 index 00000000000..136d2ff746a --- /dev/null +++ b/products/eval/README.md @@ -0,0 +1,36 @@ + + +# maka-eval + +[简体中文](README.zh-CN.md) + +Measurement and control plane for multi-arm benchmark experiments (`maka.eval.v1` and `maka.eval.v2`). `maka-eval` owns experiment decoding, task-group scheduling, deadline authority, durable append-only attempt journaling, token/cost metering reconciliation, trial environment orchestration, and deterministic cohort reporting. + +## Architecture + +- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions (lifecycle phase, execution outcome, metering state, cleanup state), run fingerprinting, product default runtime composition profile, and the semantic difference registry. +- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), admission boundary cutoffs, HMAC-authenticated checkpoint recovery, receipt-vs-proxy reconciliation without double-counting, and standardized `DEEPSEEK_V4_FLASH_COST` pricing. +- [`store`](src/store.rs): Single-writer append-only experiment journal (`LaunchIntent`, `ResourceTagged`, `Checkpoint`, `Settled`, `PostHocRecovery`), pre-dispatch intent persistence, atomic settlement commits, critical artifact verification, and deterministic state reconstruction. +- [`runner`](src/runner.rs): Task-group admission (`maxConcurrentTaskGroups`), shared preparation cache boundary lock, phase-scoped deadline authority, bounded retry policy, earliest-valid-attempt canonical selection, and crash recovery. +- [`environment`](src/environment.rs): Per-attempt `TrialEnvironment` abstraction and version-pinned Harbor / Pier bridge with 1:1 attempt-to-trial mapping. +- [`subject`](src/subject.rs): Subject adapters for the Rust Maka Host (`maka host init --settings-stdin --root ` isolated root bootstrap, public Host client execution, durable receipt journal, and transcript/usage settlement) and external CLI harnesses (`exit-code` and `protocol-v1` bounded result frames). +- [`report`](src/report.rs): Deterministic journal projection to `v1` and `v2`, paired cohort summaries, and 3-verdict differential oracle evaluation (`v1_parses`, `v1_matches_oracle`, `conditions_comparable`). + +[Source](src/lib.rs) · [Cargo manifest](Cargo.toml) diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md new file mode 100644 index 00000000000..72213725b3c --- /dev/null +++ b/products/eval/README.zh-CN.md @@ -0,0 +1,36 @@ + + +# maka-eval + +[English](README.md) + +多臂基准评测实验的测量与控制平面（支持 `maka.eval.v1` 与 `maka.eval.v2`）。`maka-eval` 负责实验契约解析、任务组并发调度、阶段截止时间权威、持久化只追加（append-only）尝试日志、Token 与成本计量对账、Trial 环境编排及确定性对照报告生成。 + +## 模块结构 + +- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度（生命周期阶段、执行结果、计量完整性、环境清理状态）、运行指纹、产品默认运行时组合配置及语义差异注册表。 +- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不进行重算求和）、准入截止边界、HMAC 鉴权检查点恢复、回执与代理日志去重对账及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 +- [`store`](src/store.rs)：单写者只追加实验日志（`LaunchIntent`、`ResourceTagged`、`Checkpoint`、`Settled`、`PostHocRecovery`）、派发前意图落盘、原子结算提交、关键产物校验与确定性状态重建。 +- [`runner`](src/runner.rs)：任务组并发准入（`maxConcurrentTaskGroups`）、共享预备缓存边界锁、分阶段截止时间控制、有界重试策略、最早有效尝试（`sequence >= 1`）规范结果选择及崩溃恢复。 +- [`environment`](src/environment.rs)：单次尝试级 `TrialEnvironment` 抽象及版本锁定的 Harbor / Pier 桥接（1:1 尝试与 Trial 映射）。 +- [`subject`](src/subject.rs)：被测系统适配器，包含 Rust Maka Host 适配器（通过 `maka host init --settings-stdin --root ` 引导隔离的 Host Root、公共 Host 客户端执行、持久化回执日志与历史/用量结算）和外部 CLI 适配器（`exit-code` 与 `protocol-v1` 有界结果帧）。 +- [`report`](src/report.rs)：从日志确定性重建 `v1` 与 `v2` 结果、生成成对队列汇总报告及三项裁决差分对比报告（`v1_parses`、`v1_matches_oracle`、`conditions_comparable`）。 + +[源码](src/lib.rs) · [Cargo 配置](Cargo.toml) diff --git a/products/eval/harbor/docker-compose-egress-proxy.yaml b/products/eval/harbor/docker-compose-egress-proxy.yaml new file mode 100644 index 00000000000..2c5f2efa051 --- /dev/null +++ b/products/eval/harbor/docker-compose-egress-proxy.yaml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + harbor-docker-egress-control-sidecar: + volumes: + - type: bind + source: ${MAKA_EVAL_NETWORK_POLICY_PATH} + target: /usr/local/bin/network-policy + read_only: true + - maka-eval-egress-ca:/opt/maka-egress:ro + + main: + depends_on: + maka-eval-mitmproxy: + condition: service_healthy + # An AF_PACKET socket writes below the IP output hooks the policy installs, + # so no nftables rule can constrain it. NET_RAW is what gates that socket. + # A task's own compose can add the capability back, which this cannot + # prevent — the relay refuses to start the subject when that happens. + cap_drop: + - NET_RAW + volumes: + - maka-eval-egress-ca:/opt/maka-egress:ro + + maka-eval-mitmproxy: + image: maka-eval-egress-proxy:12.2.3 + networks: + - default + volumes: + - maka-eval-egress-ca:/opt/maka-egress + healthcheck: + test: + - CMD + - python + - -c + - "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); assert pathlib.Path('/opt/maka-egress/proxy-ipv4').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()" + interval: 1s + timeout: 3s + retries: 30 + +volumes: + maka-eval-egress-ca: diff --git a/products/eval/harbor/egress-proxy/Dockerfile b/products/eval/harbor/egress-proxy/Dockerfile new file mode 100644 index 00000000000..20edbe3c5c8 --- /dev/null +++ b/products/eval/harbor/egress-proxy/Dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM python:3.12-slim + +RUN python -m pip install --no-cache-dir mitmproxy==12.2.3 + +COPY egress_filter.py /opt/maka-eval/egress_filter.py +COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh + +ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"] diff --git a/products/eval/harbor/egress-proxy/entrypoint.sh b/products/eval/harbor/egress-proxy/entrypoint.sh new file mode 100755 index 00000000000..53a8ee8acc6 --- /dev/null +++ b/products/eval/harbor/egress-proxy/entrypoint.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +STATE_DIR=/opt/maka-egress-state +CERT_DIR=/opt/maka-egress + +mkdir -p "$STATE_DIR" "$CERT_DIR" +AUDIT_LOG="$STATE_DIR/hits.jsonl" +# Proxy restarts share one append-only evidence file for the whole trial. +# Only create it when absent so an earlier process cannot lose recorded hits. +test -e "$AUDIT_LOG" || touch "$AUDIT_LOG" + +# confdir holds the CA private key and this cell's audit log, so it stays +# container-private and only the certificate reaches the volume the subject +# mounts. Creating the CA here instead of letting mitmdump create it on startup +# publishes the certificate before the proxy port opens, so the health gate +# cannot release `main` against a missing or left-over certificate. +python -c 'import sys +from mitmproxy.certs import CertStore +CertStore.from_store(sys.argv[1], "mitmproxy", 2048)' "$STATE_DIR" +# Publish by rename so a restart cannot expose a truncated certificate to a +# subject that is already running against the previous one. +cp "$STATE_DIR/mitmproxy-ca-cert.pem" "$CERT_DIR/.mitmproxy-ca-cert.pem" +mv "$CERT_DIR/.mitmproxy-ca-cert.pem" "$CERT_DIR/mitmproxy-ca-cert.pem" + +# The subject only needs this address so HTTPS_PROXY can keep using the +# service hostname after the namespace policy refuses Docker DNS. +ip="$(getent ahostsv4 "$(hostname)" | awk 'NR == 1 { print $1 }')" +case "$ip" in + "" | 127.*) + echo "could not publish Eval egress proxy IPv4" >&2 + exit 1 + ;; +esac +printf '%s\n' "$ip" > "$CERT_DIR/.proxy-ipv4" +mv "$CERT_DIR/.proxy-ipv4" "$CERT_DIR/proxy-ipv4" + +exec mitmdump \ + --quiet \ + --listen-host 0.0.0.0 \ + --listen-port 8080 \ + --set block_global=false \ + --set rawtcp=false \ + --set confdir="$STATE_DIR" \ + --scripts /opt/maka-eval/egress_filter.py \ + "$@" diff --git a/products/eval/harbor/egress-proxy/network-policy b/products/eval/harbor/egress-proxy/network-policy new file mode 100755 index 00000000000..bbab233909c --- /dev/null +++ b/products/eval/harbor/egress-proxy/network-policy @@ -0,0 +1,148 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +# gost's allowlist is never written here: the sidecar entrypoint empties it at +# start and this policy only ever forwards through the audited proxy, so gost +# forwards nothing on its own. That also removes the packet-mark exemption gost +# needs when it does forward — the sidecar shares the subject's network +# namespace, so a mark the sidecar can set is one the subject can set too. +GOST_PORT=12345 +NFTABLES_RULESET_NAME=gost_egress +PROXY_HOST=maka-eval-mitmproxy +PROXY_PORT=8080 + +remove_nftables() { + nft delete table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || true +} + +PROXY_IPV4_PATH=/opt/maka-egress/proxy-ipv4 + +resolve_proxy_ip() { + # The proxy publishes this file before it is healthy. Reading it does not + # depend on Docker DNS, so deny-all then allow still has an address. + if [ ! -s "$PROXY_IPV4_PATH" ]; then + return + fi + ip=$(tr -d ' \t\r\n' < "$PROXY_IPV4_PATH") + case "$ip" in + "" | *.*.*.*.* | *[!0-9.]*) + return + ;; + esac + # Same four-octet program as relay_agent.IPV4_OCTET_AWK. + if ! printf '%s\n' "$ip" | awk ' +BEGIN { FS = "." } +NF != 4 { exit 1 } +{ + for (i = 1; i <= 4; i++) { + if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1 + } +} +'; then + return + fi + printf '%s\n' "$ip" +} + +# The only difference between the two modes this policy installs is whether the +# audited proxy is exempt, so they are one ruleset with one optional pair of +# rules rather than two rulesets that have to be kept in step. +apply_ruleset() { + proxy_ip="${1:-}" + nft --file - <&2 + exit 1 + fi + apply_ruleset "$proxy_ip" + echo "ok: Eval proxy-only egress applied ($proxy_ip:$PROXY_PORT)" +} + +case "${1:-}" in + allow-all) + remove_nftables + echo "ok: allow all egress" + ;; + deny-all) + # A phase that asked for no network gets none. Answering it with the + # proxy-only ruleset would hand it a route this override invented, which + # Harbor's own deny-all does not grant either. + apply_ruleset + echo "ok: Eval no-network egress applied" + ;; + allow) + shift + if [ "$#" -ne 1 ] || [ "$1" != "$PROXY_HOST" ]; then + echo "Eval egress policy accepts only $PROXY_HOST" >&2 + exit 2 + fi + setup_proxy_only + ;; + show) + if ! rules="$(nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null)"; then + echo "mode: allow-all" + elif printf '%s' "$rules" | grep -q "dport $PROXY_PORT"; then + echo "mode: Eval proxy-only" + echo "proxy: $PROXY_HOST:$PROXY_PORT" + else + echo "mode: Eval no-network" + fi + ;; + rules) + nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || echo "mode: allow-all" + ;; + *) + echo "Usage: network-policy {show|allow-all|deny-all|allow|rules}" >&2 + exit 2 + ;; +esac diff --git a/products/eval/harbor/egress_filter.py b/products/eval/harbor/egress_filter.py new file mode 100644 index 00000000000..6b377ba5dd6 --- /dev/null +++ b/products/eval/harbor/egress_filter.py @@ -0,0 +1,501 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fail-closed URL contamination filter for Eval subject egress.""" + +from __future__ import annotations + +import json +import os +import re +import time +from contextlib import suppress +from ipaddress import IPv6Address +from pathlib import Path +from typing import NamedTuple +from urllib.parse import unquote, urlsplit + +PINNED_REVISION = "d49e28f1e4ddd13d289e85a5f312a66750951932" +MAX_DECODE_PASSES = 4 +MAX_AUDIT_BYTES = 1024 * 1024 +AUDIT_PATH = Path( + os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress-state/hits.jsonl") +) +PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})") +TERMINAL_BENCH = re.compile(r"terminal[-_.%/+\s]*bench", re.IGNORECASE) + + +class ConnectTarget(NamedTuple): + """Validated CONNECT authority adapted into the URL policy's input shape.""" + + host: str + port: int | None + url: str + + +def contamination_rule(raw_url: str) -> tuple[str, str, str] | None: + normalized = normalize_url(raw_url) + url = urlsplit(normalized) + host = (url.hostname or "").lower().rstrip(".") + path_query = f"{url.path}?{url.query}" if url.query else url.path + lowered = path_query.lower() + + # Search the host and the path separately. A benchmark name in the hostname + # is a contamination surface, and searching the two fields joined would let + # a rule match across their boundary. + def anywhere(needle: str) -> bool: + return needle in host or needle in lowered + + if host == "r.jina.ai": + inner = unquote(url.path.lstrip("/")) + if inner.startswith(("http://", "https://")): + nested = contamination_rule(inner) + if nested: + return (f"jina_recursive:{nested[0]}", host, path_query) + + if anywhere(PINNED_REVISION): + return ("pinned_revision", host, path_query) + if host == "tbench.ai" or host.endswith(".tbench.ai"): + return ("tbench_domain", host, path_query) + if host == "hub.harborframework.com" and "/tasks/terminal-bench" in lowered: + return ("harbor_task_registry", host, path_query) + if benchmark_repository(host, lowered): + return ("benchmark_repository", host, path_query) + if public_trajectory_repository(host, lowered): + return ("public_trajectory", host, path_query) + if anywhere("patches-terminalbench-"): + return ("known_patch_artifact", host, path_query) + if TERMINAL_BENCH.search(host) or TERMINAL_BENCH.search(lowered): + return ("terminal_bench_url", host, path_query) + return None + + +def normalize_url(raw_url: str) -> str: + value = raw_url.strip() + if not value: + raise ValueError("empty URL") + for _ in range(MAX_DECODE_PASSES): + if PERCENT_ESCAPE.search(value): + raise ValueError("malformed percent escape") + decoded = unquote(value) + if decoded == value: + break + value = decoded + else: + raise ValueError("URL exceeded decode limit") + parsed = urlsplit(value) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname: + raise ValueError("unsupported URL") + return value + + +def benchmark_repository(host: str, path_query: str) -> bool: + repositories = ( + "harbor-framework/terminal-bench", + "terminal-benchmarks/terminal-bench", + "tbench-ai/terminal-bench", + ) + return host in { + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + } and any(repository in path_query for repository in repositories) + + +def public_trajectory_repository(host: str, path_query: str) -> bool: + return ( + host in {"github.com", "api.github.com", "raw.githubusercontent.com", "huggingface.co"} + and "hqeric/maka-eval-trajectories" in path_query + ) + + +try: + from mitmproxy import http +except ImportError as missing_http_dependency: + http = None + +try: + from mitmproxy.proxy import commands as proxy_commands + from mitmproxy.proxy.layer import Layer + from mitmproxy.proxy.layers import ClientTLSLayer, ServerTLSLayer + from mitmproxy.proxy.layers.tcp import TCPLayer +except ImportError as missing_proxy_dependency: + proxy_commands = None + Layer = object + ClientTLSLayer = None + ServerTLSLayer = None + TCPLayer = None + +try: + from mitmproxy.net.tls import starts_like_tls_record +except ImportError: + def starts_like_tls_record(data: bytes) -> bool: + return len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03 + + +def configure(updated: object) -> None: + # HTTP 101 upgrades construct TCPLayer inside the HTTP layer without + # another next_layer hook. rawtcp=false makes that path CloseConnection + # itself; WebSocket upgrades stay on the websocket layer. + try: + from mitmproxy import ctx + except ImportError: + return + if getattr(ctx.options, "rawtcp", False): + ctx.options.rawtcp = False + + +def request(flow: object) -> None: + enforce_url_policy(flow, flow.request.pretty_url) + + +def response(flow: object) -> None: + reply = getattr(flow, "response", None) + if getattr(reply, "status_code", None) != 101: + return + # mitmproxy 12.2.3 sets flow.websocket before HttpResponseHook only when + # the 101 is a real WebSocket upgrade (Upgrade + version 13 + option on). + # A websocket Upgrade header alone still falls through to CloseConnection + # under rawtcp=false and must be audited. + if getattr(flow, "websocket", None) is not None: + return + reject_raw_transport(flow, close=False) + + +def http_connect(flow: object, /) -> None: + request = getattr(flow, "request", None) + target = _safe_connect_url( + getattr(request, "host", None), + getattr(request, "port", None), + ) + enforce_url_policy(flow, target) + + +def tcp_start(flow: object, /) -> None: + # This is a last-resort hook for a raw layer admitted by an override. + reject_raw_transport(flow) + + +def tcp_message(flow: object, /) -> None: + _clear_latest_tcp_message(getattr(flow, "messages", None)) + close_flow(flow) + + +def next_layer(nextlayer: object) -> None: + active_layer = getattr(nextlayer, "layer", None) + layer_context = getattr(nextlayer, "context", None) + if active_layer is None: + data_client = _next_layer_bytes(nextlayer, "data_client") + if _is_fragmented_tls_record_prefix(data_client): + if ClientTLSLayer is None or ServerTLSLayer is None: + return + server_tls = ServerTLSLayer(layer_context) + server_tls.child_layer = ClientTLSLayer(layer_context) + nextlayer.layer = server_tls + return + if not initial_stream_is_raw(nextlayer): + return + reject_raw_transport(layer_context, close=False) + nextlayer.layer = RejectRawTransport(layer_context, proxy_commands) + return + if TCPLayer is None or not isinstance(active_layer, TCPLayer): + return + reject_raw_transport(layer_context, close=False) + closer = RejectRawTransport(layer_context, proxy_commands) + replace_layer(layer_context, active_layer, closer) + nextlayer.layer = closer + + +def enforce_url_policy(flow: object, raw_url: str) -> None: + if http is None: + raise RuntimeError("mitmproxy is required to run the Eval egress filter") + try: + match = contamination_rule(raw_url) + if not match: + return + rule_id, host, normalized_path = match + audit_event(rule_id, host, normalized_path) + flow.response = blocked_response(rule_id) + except Exception as error: + flow.response = http.Response.make( + 503, + b"Eval egress policy could not classify this request.\n", + { + "Content-Type": "text/plain; charset=utf-8", + "X-Maka-Eval-Egress-Rule": "policy_error", + }, + ) + try: + audit_event("policy_error", "", type(error).__name__) + except Exception: + pass + + +def connect_url(host: object, port: object) -> str: + return parse_connect_target(host, port).url + + +def _safe_connect_url(host: object, port: object) -> str: + try: + return connect_url(host, port) + except (TypeError, ValueError): + return "" + + +def _clear_latest_tcp_message(messages: object) -> None: + if not isinstance(messages, list) or not messages: + return + latest = messages[-1] + if hasattr(latest, "content"): + latest.content = b"" + + +def parse_connect_target(host: object, port: object) -> ConnectTarget: + normalized_host = _normalize_connect_host(host) + normalized_port = _normalize_connect_port(port) + scheme = "http" if normalized_port == 80 else "https" + authority = normalized_host if normalized_port in (None, 443, 80) else f"{normalized_host}:{normalized_port}" + return ConnectTarget( + host=normalized_host, + port=normalized_port, + url=f"{scheme}://{authority}/", + ) + + +def _normalize_connect_host(host: object) -> str: + if not isinstance(host, str) or not host or host != host.strip(): + raise ValueError("empty CONNECT host") + if host.startswith("[") and host.endswith("]"): + IPv6Address(host[1:-1]) + return host + if ":" in host: + IPv6Address(host) + return f"[{host}]" + if re.search(r"[\s/@?#\\\[\]]", host): + raise ValueError("invalid CONNECT host") + return host + + +def _normalize_connect_port(port: object) -> int | None: + if port is None: + return None + if type(port) is not int or not 1 <= port <= 65535: + raise ValueError("invalid CONNECT port") + return port + + +def initial_stream_is_raw(nextlayer: object) -> bool: + """Close only when the bytes cannot still become HTTP or TLS. + + Script next_layer runs before mitmproxy 12.2.3's classifier. Copying its + `probably_no_http` test here would treat `GET` / `GET ` as raw and assign + CloseRawLayer while the request line is still arriving. With rawtcp=false + the built-in path would have kept waiting for HttpLayer. + """ + data_client = _next_layer_bytes(nextlayer, "data_client") + data_server = _next_layer_bytes(nextlayer, "data_server") + if _could_start_tls_record(data_client): + return False + if not data_client and not data_server: + return False + if data_server or data_client.startswith(b"SSH"): + return True + return not _still_could_be_http(data_client) + + +def _could_start_tls_record(data: bytes) -> bool: + """Keep a fragmented ClientHello undecided until its 3-byte prefix exists.""" + if starts_like_tls_record(data): + return True + return _is_fragmented_tls_record_prefix(data) + + +def _is_fragmented_tls_record_prefix(data: bytes) -> bool: + return 0 < len(data) < 3 and b"\x16\x03".startswith(data) + + +def _still_could_be_http(data: bytes) -> bool: + first_line, newline, _rest = data.partition(b"\n") + line = first_line.rstrip(b"\r") + method, space, _remainder = line.partition(b" ") + if not method.isascii() or not method.isalpha(): + return False + if newline and not space: + return False + return True + + +def _next_layer_bytes(nextlayer: object, name: str) -> bytes: + getter = getattr(nextlayer, name, None) + if not callable(getter): + return b"" + try: + data = getter() + except Exception: + return b"" + return bytes(data) if isinstance(data, (bytes, bytearray)) else b"" + + +def peer_label(owner: object) -> tuple[str, str]: + server = getattr(owner, "server_conn", None) or getattr(owner, "server", None) + peer_address = getattr(server, "address", None) + return _format_peer_address(peer_address) + + +def _format_peer_address(address: object) -> tuple[str, str]: + if not isinstance(address, (tuple, list)) or not address: + return "", "" + host = str(address[0])[:255] + port = address[1] if len(address) > 1 else "" + return host, f":{port}" if port != "" else "" + + +def reject_raw_transport(owner: object, *, close: bool = True) -> None: + host, port = peer_label(owner) + _audit_safely("raw_tunnel", host, port) + if close: + close_flow(owner) + + +def _audit_safely(rule_id: str, host: str, normalized_path: str) -> None: + with suppress(Exception): + audit_event(rule_id, host, normalized_path) + + +def close_flow(flow: object) -> None: + terminate = getattr(flow, "kill", None) + if callable(terminate) and getattr(flow, "killable", True): + try: + terminate() + except Exception as close_error: + del close_error + + +def replace_layer(context: object, current: object, closer: object) -> None: + layers = getattr(context, "layers", None) + if not isinstance(layers, list): + return + try: + index = layers.index(current) + except ValueError: + index = len(layers) + if closer in layers: + layers.remove(closer) + if current in layers: + layers.remove(current) + layers.insert(min(index, len(layers)), closer) + + +class RejectRawTransport(Layer): + def __init__(self, context: object, commands: object) -> None: + self._close_connection = getattr(commands, "CloseConnection", None) + if Layer is object: + self.context = context + return + if getattr(context, "layers", None) is None: + context.layers = [] + if getattr(context, "options", None) is None: + context.options = type("Options", (), {"proxy_debug": False})() + super().__init__(context) + + def handle_event(self, event: object): + if self._close_connection is None: + return + yield from ( + self._close_connection(connection) + for name in ("client", "server") + if (connection := getattr(self.context, name, None)) is not None + ) + + +def blocked_response(rule_id: str): + return http.Response.make( + 451, + b"Benchmark source or public solution access is blocked during evaluation.\n", + { + "Content-Type": "text/plain; charset=utf-8", + "X-Maka-Eval-Egress-Rule": rule_id, + }, + ) + + +class AuditJournal: + def __init__(self, path: Path, byte_limit: int) -> None: + self.path = path + self.byte_limit = byte_limit + + def record(self, rule_id: str, host: str, normalized_path: str) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + entry = self._encode(rule_id, host[:255], normalized_path[:4096]) + existing_bytes = self.path.stat().st_size if self.path.exists() else 0 + if existing_bytes + len(entry) <= self.byte_limit: + with self.path.open("ab") as stream: + stream.write(entry) + return + self.mark_full() + + def mark_full(self) -> None: + if self.has_full_marker(): + return + with self.path.open("ab") as stream: + stream.write(self._separator() + self._encode("audit_truncated", "", "")) + + def has_full_marker(self) -> bool: + last = self._last_record() + return last is not None and last.get("ruleId") == "audit_truncated" + + def _separator(self) -> bytes: + if not self.path.exists() or self.path.stat().st_size == 0: + return b"" + final_offset = self.path.stat().st_size - 1 + with self.path.open("rb") as stream: + stream.seek(final_offset) + return b"" if stream.read(1) == b"\n" else b"\n" + + def _last_record(self) -> dict[str, object] | None: + if not self.path.exists() or self.path.stat().st_size == 0: + return None + size = self.path.stat().st_size + with self.path.open("rb") as stream: + tail_offset = max(0, size - 4096) + stream.seek(tail_offset) + lines = stream.read().decode("utf-8", errors="ignore").splitlines() + for line in reversed(lines): + if not line.strip(): + continue + decoded = None + with suppress(json.JSONDecodeError): + decoded = json.loads(line) + return decoded if isinstance(decoded, dict) else None + return None + + @staticmethod + def _encode(rule_id: str, host: str, normalized_path: str) -> bytes: + record = { + "ts": int(time.time() * 1000), + "ruleId": rule_id, + "host": host, + "normalizedPath": normalized_path, + } + return (json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) + + +def audit_event(rule_id: str, host: str, normalized_path: str) -> None: + AuditJournal(AUDIT_PATH, MAX_AUDIT_BYTES).record(rule_id, host, normalized_path) diff --git a/products/eval/harbor/eval_framework.py b/products/eval/harbor/eval_framework.py new file mode 100644 index 00000000000..6c2bcb0b37b --- /dev/null +++ b/products/eval/harbor/eval_framework.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Process-local framework authority shared by the trial runner and relay.""" + +from __future__ import annotations # Keep framework metadata importable on Python 3.9. + +from typing import NamedTuple + + +class FrameworkSpec(NamedTuple): + name: str + distribution: str + agent_module: str + + +_FRAMEWORKS = { + "harbor": FrameworkSpec("harbor", "harbor", "harbor.agents.base"), + "pier": FrameworkSpec("pier", "datacurve-pier", "pier.agents.base"), +} +_active: FrameworkSpec | None = None + + +def framework_spec(name: str) -> FrameworkSpec: + try: + return _FRAMEWORKS[name] + except KeyError: + raise RuntimeError("framework must be harbor or pier") from None + + +def install(name: str) -> None: + """Select the framework before importing its process-wide relay module.""" + + global _active + _active = framework_spec(name) + + +def current_framework() -> str: + if _active is None: + raise RuntimeError("Eval framework selection is not installed") from None + return _active.name + + +def framework_distribution(name: str) -> str: + return framework_spec(name).distribution + + +def framework_agent_module(name: str) -> str: + return framework_spec(name).agent_module diff --git a/products/eval/harbor/relay_agent.py b/products/eval/harbor/relay_agent.py new file mode 100644 index 00000000000..a968bb5368e --- /dev/null +++ b/products/eval/harbor/relay_agent.py @@ -0,0 +1,723 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""One-shot Harbor/Pier Agent that delegates one subject execution to @maka/eval.""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import hashlib +import importlib +import json +import os +import re +import shlex +import tempfile +from pathlib import Path +from typing import Any + +from eval_framework import current_framework, framework_agent_module + +framework = current_framework() +BaseAgent = importlib.import_module(framework_agent_module(framework)).BaseAgent + + +class RelayTransportClosed(RuntimeError): + pass + + +RESULT_FRAME_PREFIX = "MAKA-EVAL-RESULT-V1" +SCOPE_ERROR_PREFIX = "MAKA-EVAL-SCOPE-ERROR-V1" +RESULT_PAYLOAD_LIMIT_BYTES = 2 * 1024 +RESULT_CARRIER_LIMIT_BYTES = 64 * 1024 +SUBJECT_STDOUT_PATH = "/logs/artifacts/maka-subject.stdout.txt" +SUBJECT_STDERR_PATH = "/logs/artifacts/maka-subject.stderr.txt" +CAPABILITY_PREFIX = "MAKA-EVAL-CAPABILITIES-V1 " +NAMESPACE_PREFIX = "MAKA-EVAL-POLICY-NAMESPACE-V1 " +# The egress policy only constrains what the IP output hooks can see. NET_RAW +# grants an AF_PACKET socket that writes beneath them, and NET_ADMIN grants the +# ability to delete the ruleset outright. The overlay drops NET_RAW, but a +# task's own compose can add either back, and a `cap_add` wins over an +# overlay's `cap_drop`. +BYPASS_CAPABILITIES = {"NET_RAW": 1 << 13, "NET_ADMIN": 1 << 12} +# Every set is read, not just the effective one: a non-root subject reports an +# empty effective set while a file-capability executable still reacquires +# anything the bounding set kept, and the permitted, inheritable and ambient +# sets each raise into the effective set without an exec at all. +CAPABILITY_FIELDS = ("CapEff", "CapPrm", "CapBnd") +# Harbor installs the policy by running `network-policy` inside this service, so +# its network namespace is what "the namespace the policy was applied to" means. +# Reading the identity from both sides answers that question directly, rather +# than inferring it from something only the shared namespace would expose. +POLICY_SERVICE = "harbor-docker-egress-control-sidecar" +NAMESPACE_PROBE = f"printf %s {shlex.quote(NAMESPACE_PREFIX)}; readlink /proc/self/ns/net" +PROXY_IPV4_PATH = "/opt/maka-egress/proxy-ipv4" +PROXY_HOSTS_PREFIX = "MAKA-EVAL-PROXY-HOST-V1 " +# The kernel's own form for a namespace link target. Matching it keeps an +# unexpected answer from being compared as if it were an identity. +NAMESPACE_IDENTITY = re.compile(r"^net:\[[0-9]+\]$") + +_host_teardown_requested = False + + +def request_host_teardown() -> None: + global _host_teardown_requested + _host_teardown_requested = True + + +class RelayAgent(BaseAgent): + def __init__( + self, + *args: Any, + relay_host: str, + relay_port: int, + relay_token: str, + teardown_timeout_ms: int, + **kwargs: Any, + ): + super().__init__(*args, **kwargs) + self._host = relay_host + self._port = relay_port + self._token = relay_token + if not isinstance(teardown_timeout_ms, int) or teardown_timeout_ms <= 0: + raise RuntimeError("Maka Eval teardown timeout is invalid") + self._teardown_timeout = teardown_timeout_ms / 1000 + self._subject_owner: str | None = None + + @staticmethod + def name() -> str: + return "maka-eval-relay" + + def version(self) -> str: + return "1" + + async def setup(self, environment: Any) -> None: + identity = await environment.exec('printf "%s:%s" "$(id -u)" "$(id -g)"') + owner = str(identity.stdout or "").strip() + if identity.return_code != 0 or re.fullmatch(r"[0-9]+:[0-9]+", owner) is None: + raise RuntimeError("Maka Eval could not resolve the task user") + prepared = await environment.exec( + "mkdir -p /logs/agent /logs/artifacts && " + f"chown {owner} /logs/agent /logs/artifacts && " + "chmod 700 /logs/agent /logs/artifacts", + user="root", + ) + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare task-owned artifact directories") + # Storage coordinates by the system account home, not the subject's + # temporary HOME. Provision an absent home without changing task identity + # or taking ownership of an existing directory. + uid = owner.split(":", 1)[0] + home_script = ( + f"set -eu; account=$(getent passwd {uid}); " + "task_home=$(printf '%s' \"$account\" | cut -d: -f6) && " + 'case "$task_home" in /*) ;; *) exit 1 ;; esac; ' + 'if [ ! -e "$task_home" ] && [ ! -L "$task_home" ]; then ' + f'mkdir -p -- "$task_home" && chown {owner} "$task_home" && ' + 'chmod 700 "$task_home" || exit 1; fi; ' + 'test -d "$task_home"' + ) + prepared_home = await environment.exec(home_script, user="root") + if prepared_home.return_code != 0: + raise RuntimeError("Maka Eval could not prepare the task account home") + self._subject_owner = owner + + async def run(self, instruction: str, environment: Any, context: Any) -> None: + reader, writer = await asyncio.open_connection(self._host, self._port) + execution: asyncio.Task[Any] | None = None + decision: asyncio.Task[dict[str, Any]] | None = None + request: dict[str, Any] | None = None + execution_reported = False + scope_path = f"/logs/agent/.maka-eval-{self._token}.pid" + environment_path = f"/tmp/maka-eval-{self._token}.env" + try: + cwd_prefix = f"MAKA-EVAL-CWD-V1 {self._token} " + working_directory = await environment.exec( + f"printf {shlex.quote(cwd_prefix)}; pwd 2>/dev/null" + ) + cwd_lines = [ + line[len(cwd_prefix) :] + for line in str(working_directory.stdout or "").splitlines() + if line.startswith(cwd_prefix) + ] + cwd = cwd_lines[0] if len(cwd_lines) == 1 else "" + if working_directory.return_code != 0 or not cwd.startswith("/") or "\x00" in cwd: + raise RuntimeError("Maka Eval could not resolve the task working directory") + await _require_constrained_subject(environment) + if not await _send( + writer, + { + "token": self._token, + "kind": "ready", + "instruction": instruction, + "cwd": cwd, + }, + ): + raise RelayTransportClosed("Maka Eval relay transport closed before ready") + request = await _receive(reader) + _require_message(request, self._token, "execute") + command = await _prepare_command( + environment, request, self._token, scope_path, self._subject_owner + ) + execution = asyncio.create_task(environment.exec(command, cwd=cwd)) + decision = asyncio.create_task(_receive(reader)) + done, _ = await asyncio.wait({execution, decision}, return_when=asyncio.FIRST_COMPLETED) + if decision in done and execution not in done: + decision.result() + raise RelayTransportClosed("Maka Eval relay received control before execution") + result = execution.result() + await _persist_subject_outputs(environment, result) + stdout, diagnostic = _project_result(result, request) + # A subject that exited on its own leaves the shared environment as + # it left it, and the verifier reads that environment. Nothing is + # waiting on those processes here — `environment.exec` has already + # returned — so tearing them down would not unblock anything; it + # would only edit the thing about to be measured, and edit it for + # some subjects and not others. Cancellation still quiesces, because + # there the subject has not stopped and the trial is being abandoned. + if not await _send( + writer, + { + "token": self._token, + "kind": "executed", + "termination": "exited", + "exitCode": result.return_code, + "stdout": stdout, + "diagnostic": diagnostic, + }, + ): + raise RelayTransportClosed("Maka Eval relay transport closed before result") + execution_reported = True + _require_message( + decision.result() if decision.done() else await decision, + self._token, + "verify", + ) + except asyncio.CancelledError: + if request is not None and execution is not None: + execution_terminal = execution.done() and not execution.cancelled() + terminal_projection = None + if execution_terminal: + terminal_result = execution.result() + terminal_projection = _project_result(terminal_result, request) + # A subject that already exited has nothing left to settle, and + # tearing its scope down here would remove what the verifier is + # about to score -- the same environment edit this relay stopped + # making on the ordinary path. Only a subject still running is + # brought to a stop. + if terminal_projection is not None: + result = terminal_result + else: + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + await _persist_subject_outputs(environment, result) + if ( + result is not None + and not execution_reported + and (execution_terminal or not _host_teardown_requested) + ): + stdout, diagnostic = terminal_projection or _project_result(result, request) + with contextlib.suppress(Exception): + await _send( + writer, + { + "token": self._token, + "kind": "executed", + "termination": "exited" if execution_terminal else "framework_timeout", + "exitCode": result.return_code if execution_terminal else 124, + "stdout": stdout, + "diagnostic": diagnostic, + }, + ) + elif not execution_reported and not _host_teardown_requested: + with contextlib.suppress(Exception): + await _send( + writer, + { + "token": self._token, + "kind": "executed", + "termination": "framework_timeout", + "exitCode": 124, + "stdout": "", + "diagnostic": ( + _carrier_diagnostic("result-frame-missing", b"") + if request.get("captureStdout", True) + else {"category": "none"} + ), + }, + ) + raise + except RelayTransportClosed: + if request is not None and execution is not None: + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + with contextlib.suppress(Exception): + await _persist_subject_outputs(environment, result) + except BaseException: + if request is not None and execution is not None: + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + with contextlib.suppress(Exception): + await _persist_subject_outputs(environment, result) + raise + finally: + if decision is not None and not decision.done(): + decision.cancel() + with contextlib.suppress(BaseException): + await decision + if request is not None: + with contextlib.suppress(Exception): + await asyncio.wait_for( + environment.exec( + f"rm -f -- {shlex.quote(scope_path)} {shlex.quote(environment_path)}", + cwd=cwd, + timeout_sec=1, + ), + timeout=1, + ) + with contextlib.suppress(BrokenPipeError, ConnectionError, RuntimeError, TimeoutError): + writer.close() + await asyncio.wait_for(writer.wait_closed(), timeout=1) + + +async def _prepare_command( + environment: Any, + request: dict[str, Any], + token: str, + scope_path: str, + subject_owner: str | None = None, +) -> str: + credentials = request.get("credentials") + public_environment = request.get("environment", {}) + if not isinstance(credentials, dict) or not all( + isinstance(key, str) and isinstance(value, str) for key, value in credentials.items() + ): + raise RuntimeError("invalid Maka Eval credentials") + if not isinstance(public_environment, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in public_environment.items() + ): + raise RuntimeError("invalid Maka Eval environment") + if set(credentials) & set(public_environment): + raise RuntimeError("Maka Eval environment overlaps credentials") + capture_stdout = request.get("captureStdout", True) + if not isinstance(capture_stdout, bool): + raise RuntimeError("invalid Maka Eval stdout policy") + result_token = request.get("resultToken") + if not isinstance(result_token, str) or re.fullmatch(r"[0-9a-f]{32}", result_token) is None: + raise RuntimeError("invalid Maka Eval result token") + if "MAKA_EVAL_RESULT_TOKEN" in credentials or "MAKA_EVAL_RESULT_TOKEN" in public_environment: + raise RuntimeError("Maka Eval environment contains a reserved name") + for label, values in (("environment", public_environment), ("credential", credentials)): + if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None for key in values): + raise RuntimeError(f"invalid Maka Eval {label} name") + + container_path = f"/tmp/maka-eval-{token}.env" + secret_path = None + try: + with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as secret: + secret_path = Path(secret.name) + os.chmod(secret_path, 0o600) + for key, value in { + **public_environment, + **credentials, + "MAKA_EVAL_RESULT_TOKEN": result_token, + }.items(): + secret.write(f"export {key}={shlex.quote(value)}\n") + await environment.upload_file(secret_path, container_path) + if subject_owner is not None: + prepared = await environment.exec( + f"chown {subject_owner} {shlex.quote(container_path)} && " + f"chmod 600 {shlex.quote(container_path)}", + user="root", + ) + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare task-owned credentials") + finally: + if secret_path is not None: + secret_path.unlink(missing_ok=True) + subject = shlex.join([request["command"], *request["args"]]) + output_redirect = "" if capture_stdout else " >/dev/null" + scope_error = shlex.quote(f"{SCOPE_ERROR_PREFIX} {result_token}\\n") + inner = ( + f"( umask 077 && echo $$ > {shlex.quote(scope_path)} ) 2>/dev/null || " + f"{{ printf {scope_error}; exit 111; }}; " + f". {shlex.quote(container_path)}; command -p rm -f {shlex.quote(container_path)}; " + f"exec {subject}{output_redirect}" + ) + return f"setsid --wait sh -c {shlex.quote(inner)}" + + +async def _require_constrained_subject(environment: Any) -> None: + """Refuse to start the subject unless the egress policy actually governs it. + + Runs after the policy is applied and before the subject exists, so a task + that keeps a capability the policy cannot see, or that keeps the subject out + of the namespace the policy was applied to, fails the attempt instead of + producing a result the isolation contract never actually covered. + """ + if os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") != "1": + return + probe = await environment.exec( + f"printf %s {shlex.quote(CAPABILITY_PREFIX)}; " + r"sed -n 's/^\(Cap[A-Za-z]*\):[[:space:]]*/\1=/p' /proc/self/status | tr '\n' ' '; " + "echo; " + NAMESPACE_PROBE + ) + policy = await environment.service_exec(NAMESPACE_PROBE, service=POLICY_SERVICE) + if probe.return_code != 0 or policy.return_code != 0: + raise RuntimeError("Maka Eval could not read the subject isolation evidence") + capabilities = _sole_probe_line(probe, CAPABILITY_PREFIX) + namespace = _sole_probe_line(probe, NAMESPACE_PREFIX) + policy_namespace = _sole_probe_line(policy, NAMESPACE_PREFIX) + if not all(NAMESPACE_IDENTITY.match(value) for value in (namespace, policy_namespace)): + raise RuntimeError("Maka Eval could not read the subject isolation evidence") + reported: dict[str, int] = {} + for field in capabilities.split(): + name, _, value = field.partition("=") + try: + reported[name] = int(value, 16) + except ValueError: + raise RuntimeError("Maka Eval could not read the subject capability set") from None + if not all(field in reported for field in CAPABILITY_FIELDS): + raise RuntimeError("Maka Eval could not read the subject capability set") + granted = 0 + for value in reported.values(): + granted |= value + held = sorted(name for name, bit in BYPASS_CAPABILITIES.items() if granted & bit) + if held: + raise RuntimeError( + "the subject holds " + + ", ".join(held) + + ", which bypasses the Eval egress policy; remove it from the task" + ) + if namespace != policy_namespace: + raise RuntimeError( + "the subject does not share the network namespace the Eval egress policy " + "was applied to; remove the task's own networking on the subject service" + ) + await _pin_proxy_hostname(environment) + + +HOSTS_ALIAS_AWK = r""" +/^[[:space:]]*#/ { print; next } +{ + for (i = 2; i <= NF; i++) { + if ($i == host) next + } + print +} +""" + +# Four decimal octets 0-255, no empty fields and no leading zeros. The same +# program is embedded in egress-proxy/network-policy; the contract test +# requires the two copies to stay identical. +IPV4_OCTET_AWK = r""" +BEGIN { FS = "." } +NF != 4 { exit 1 } +{ + for (i = 1; i <= 4; i++) { + if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1 + } +} +""" + +_PUBLISHED_IPV4 = re.compile( + r"^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}" + r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$" +) + + +def _valid_egress_proxy_host(host: str) -> bool: + if not host or host.startswith(".") or host.endswith(".") or ".." in host: + return False + # Match the pin script's `*[!A-Za-z0-9.-]*` class. str.isalnum() also + # accepts Unicode letters, which the shell would later reject as a pin + # failure instead of an invalid host. + return all( + (character.isalnum() and character.isascii()) or character in ".-" + for character in host + ) + + +def _valid_published_ipv4(ip: str) -> bool: + return bool(_PUBLISHED_IPV4.fullmatch(ip)) + + +async def _pin_proxy_hostname(environment: Any) -> None: + """Point the proxy hostname at the published IPv4 so Docker DNS can be refused. + + The subject only needs that one name, because the HTTP proxy does remote + resolution itself. Pinning it in /etc/hosts lets the namespace policy reject + 127.0.0.11:53 without breaking HTTPS_PROXY. + """ + host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") or "maka-eval-mitmproxy" + if not _valid_egress_proxy_host(host): + raise RuntimeError("Eval egress proxy host is invalid") + script = f""" +set -eu +path={shlex.quote(PROXY_IPV4_PATH)} +host={shlex.quote(host)} +case "$host" in + ""|.*|*..*|*.) exit 1 ;; + *[!A-Za-z0-9.-]*) exit 1 ;; +esac +test -f "$path" +ip=$(tr -d ' \\t\\r\\n' < "$path") +case "$ip" in + ""|*.*.*.*.*|*[!0-9.]*) exit 1 ;; +esac +if ! printf '%s\\n' "$ip" | awk {shlex.quote(IPV4_OCTET_AWK)}; then + exit 1 +fi +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT +if [ -f /etc/hosts ]; then + awk -v host="$host" {shlex.quote(HOSTS_ALIAS_AWK)} /etc/hosts > "$tmp" +fi +printf '%s %s\\n' "$ip" "$host" >> "$tmp" +cat "$tmp" > /etc/hosts +printf %s {shlex.quote(PROXY_HOSTS_PREFIX)} +printf '%s %s\\n' "$ip" "$host" +""" + # Harbor 0.20.0 BaseEnvironment.exec and DockerEnvironment.exec take user=; + # DockerEnvironment._compose_exec forwards it as `docker compose exec -u`. + probe = await environment.exec(script, user="root") + if probe.return_code != 0: + raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname") + reported = _sole_probe_line(probe, PROXY_HOSTS_PREFIX) + ip, _, pinned = reported.partition(" ") + if pinned != host or not _valid_published_ipv4(ip): + raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname") + + +def _sole_probe_line(probe: Any, prefix: str) -> str: + reported = [ + line[len(prefix) :] + for line in str(probe.stdout or "").splitlines() + if line.startswith(prefix) + ] + if len(reported) != 1: + raise RuntimeError("Maka Eval could not read the subject isolation evidence") + return reported[0].strip() + + +async def _persist_subject_outputs(environment: Any, result: Any) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + stdout = root / "stdout" + stderr = root / "stderr" + stdout.write_text(str(getattr(result, "stdout", "") or ""), encoding="utf-8") + stderr.write_text(str(getattr(result, "stderr", "") or ""), encoding="utf-8") + prepared = await environment.exec("mkdir -p /logs/artifacts && chmod 700 /logs/artifacts") + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare subject artifact output") + await environment.upload_file(stdout, SUBJECT_STDOUT_PATH) + await environment.upload_file(stderr, SUBJECT_STDERR_PATH) + + +def _decode_result_carrier(carrier: str, token: str) -> tuple[str, dict[str, Any]]: + raw = carrier.encode("utf-8", errors="replace") + if len(raw) > RESULT_CARRIER_LIMIT_BYTES: + return "", _carrier_diagnostic("result-frame-oversize", raw) + prefix = f"{RESULT_FRAME_PREFIX} {token} " + candidates = [line for line in carrier.splitlines(keepends=True) if line.startswith(prefix)] + if len(candidates) != 1: + category = "result-frame-missing" if not candidates else "result-frame-ambiguous" + return "", _carrier_diagnostic(category, raw) + frame = candidates[0] + fields = frame.rstrip("\r\n").split(" ", 4) + if len(fields) != 5: + return "", _carrier_diagnostic("result-frame-invalid", raw) + _, framed_token, length_text, digest, encoded = fields + try: + length = int(length_text) + padding = "=" * (-len(encoded) % 4) + payload = base64.b64decode(encoded + padding, altchars=b"-_", validate=True) + except (ValueError, base64.binascii.Error): + return "", _carrier_diagnostic("result-frame-invalid", raw) + if ( + framed_token != token + or length < 0 + or length > RESULT_PAYLOAD_LIMIT_BYTES + or len(payload) != length + or not re.fullmatch(r"[0-9a-f]{64}", digest) + or hashlib.sha256(payload).hexdigest() != digest + ): + return "", _carrier_diagnostic("result-frame-invalid", raw) + noise = carrier.replace(frame, "", 1).encode("utf-8", errors="replace") + diagnostic = ( + {"category": "none"} + if not noise + else _carrier_diagnostic("unstructured-output", noise) + ) + try: + decoded = payload.decode("utf-8", errors="strict") + except UnicodeDecodeError: + return "", _carrier_diagnostic("result-frame-invalid", raw) + return decoded, diagnostic + + +def _project_result(result: Any, request: dict[str, Any]) -> tuple[str, dict[str, Any]]: + carrier = str(getattr(result, "stdout", "") or "") + scope_error = f"{SCOPE_ERROR_PREFIX} {request['resultToken']}" + if carrier == f"{scope_error}\n": + return "", _carrier_diagnostic("execution-scope-unavailable", carrier.encode()) + if not request.get("captureStdout", True): + return "", {"category": "none"} + return _decode_result_carrier(carrier, request["resultToken"]) + + +def _carrier_diagnostic(category: str, value: bytes) -> dict[str, Any]: + return { + "category": category, + "bytes": len(value), + "sha256": hashlib.sha256(value).hexdigest(), + } + + +async def _settle(environment: Any, cwd: str, scope_path: str, execution: Any) -> Any: + if execution.cancelled(): + raise RuntimeError("Maka Eval subject execution was cancelled") + if execution.done(): + result = execution.result() + else: + result = None + for signal, timeout in (("TERM", 20), ("KILL", 10)): + await _signal(environment, cwd, scope_path, signal) + try: + result = await asyncio.wait_for(asyncio.shield(execution), timeout=timeout) + break + except asyncio.CancelledError: + if execution.cancelled(): + raise RuntimeError("Maka Eval subject execution was cancelled") from None + raise + except TimeoutError: + pass + if result is None: + raise RuntimeError("Maka Eval subject did not settle") + await _quiesce_scope(environment, cwd, scope_path) + return result + + +async def _settle_or_destroy( + environment: Any, + cwd: str, + scope_path: str, + execution: Any, + timeout: float, +) -> Any | None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + stop_reserve = min(20.0, timeout * 0.2) + try: + return await asyncio.wait_for( + _settle(environment, cwd, scope_path, execution), + timeout=max(0.001, deadline - loop.time() - stop_reserve), + ) + except Exception: + try: + remaining = max(0.001, deadline - loop.time()) + await asyncio.wait_for(environment.stop(delete=True), timeout=remaining) + except Exception: + pass + finally: + if not execution.done(): + execution.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + remaining = max(0.001, deadline - loop.time()) + await asyncio.wait_for(execution, timeout=remaining) + return None + + +async def _signal(environment: Any, cwd: str, scope_path: str, signal: str) -> None: + command = ( + f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 0; " + "case $pgid in ''|0|*[!0-9]*) exit 0;; esac; " + f"kill -{signal} -- \"-$pgid\"" + ) + with contextlib.suppress(Exception): + await environment.exec( + command, + cwd=cwd, + timeout_sec=5, + ) + + +async def _quiesce_scope(environment: Any, cwd: str, scope_path: str) -> None: + if not await _scope_active(environment, cwd, scope_path): + return + for signal, timeout in (("TERM", 10), ("KILL", 5)): + await _signal(environment, cwd, scope_path, signal) + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if not await _scope_active(environment, cwd, scope_path): + return + await asyncio.sleep(0.1) + raise RuntimeError("Maka Eval execution scope did not quiesce") + + +async def _scope_active(environment: Any, cwd: str, scope_path: str) -> bool: + result = await environment.exec( + f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 4; " + "case $pgid in ''|0|*[!0-9]*) exit 4;; esac; " + "kill -0 -- \"-$pgid\" 2>/dev/null; status=$?; " + "if [ $status -eq 0 ]; then exit 0; fi; exit 3", + cwd=cwd, + timeout_sec=5, + ) + if result.return_code == 0: + return True + if result.return_code == 3: + return False + raise RuntimeError("Maka Eval execution scope evidence was unavailable") + + +def _require_message(value: dict[str, Any], token: str, kind: str) -> None: + if value.get("token") != token or value.get("kind") != kind: + raise RuntimeError("invalid Maka Eval relay message") + + +async def _receive(reader: asyncio.StreamReader) -> dict[str, Any]: + try: + raw = await reader.readline() + if not raw: + raise RelayTransportClosed("Maka Eval relay peer closed") + value = json.loads(raw) + except (ConnectionError, json.JSONDecodeError, ValueError) as error: + raise RelayTransportClosed("Maka Eval relay message was unavailable") from error + if not isinstance(value, dict): + raise RelayTransportClosed("Maka Eval relay message was invalid") + return value + + +async def _send(writer: asyncio.StreamWriter, value: object) -> bool: + if writer.is_closing(): + return False + try: + writer.write((json.dumps(value, separators=(",", ":")) + "\n").encode()) + await writer.drain() + return True + except (BrokenPipeError, ConnectionError, RuntimeError): + return False diff --git a/products/eval/harbor/run_trial.py b/products/eval/harbor/run_trial.py new file mode 100644 index 00000000000..4123e6c7980 --- /dev/null +++ b/products/eval/harbor/run_trial.py @@ -0,0 +1,230 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run exactly one pinned Harbor or Pier Trial.""" + +import asyncio +import contextlib +import fcntl +import hashlib +import importlib +import importlib.metadata +import json +import os +import signal +import sys +from pathlib import Path +from typing import AsyncIterator + +FRAMEWORK_VERSION_MISMATCH_EXIT_CODE = 78 + + +class FrameworkVersionMismatch(Exception): + pass + + +TASK_CACHE_PROTOCOL_VERSION = 1 + + +@contextlib.asynccontextmanager +async def task_cache_lock(cache_dir: Path, identity: str) -> AsyncIterator[None]: + lock_dir = cache_dir / ".maka-locks" + lock_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + lock_path = lock_dir / f"{hashlib.sha256(identity.encode()).hexdigest()}.lock" + flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(lock_path, flags, 0o600) + acquired = False + try: + os.fchmod(descriptor, 0o600) + while not acquired: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + except BlockingIOError: + await asyncio.sleep(0.05) + yield + finally: + if acquired: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def ready_task(cache_dir: Path, identity: str) -> tuple[Path, Path | None]: + namespace = cache_dir / ".maka-tasks" / hashlib.sha256(identity.encode()).hexdigest() + marker = namespace / "ready.json" + try: + value = json.loads(marker.read_text()) + if set(value) != {"version", "target"}: + return namespace, None + if value["version"] != TASK_CACHE_PROTOCOL_VERSION or not isinstance( + value["target"], str + ): + return namespace, None + target = (namespace / value["target"]).resolve() + if ( + not target.is_relative_to(namespace.resolve()) + or not target.is_dir() + or not any(target.iterdir()) + ): + return namespace, None + return namespace, target + except (OSError, TypeError, ValueError): + return namespace, None + + +def publish_ready_task(namespace: Path, target: Path) -> None: + resolved_namespace = namespace.resolve() + resolved_target = target.resolve() + if not resolved_target.is_relative_to(resolved_namespace): + raise RuntimeError("Harbor task cache target escaped its Maka namespace") + relative_target = resolved_target.relative_to(resolved_namespace).as_posix() + marker = namespace / "ready.json" + temporary = namespace / ".ready.tmp" + descriptor = os.open( + temporary, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + with os.fdopen(descriptor, "w") as stream: + descriptor = -1 + json.dump( + {"version": TASK_CACHE_PROTOCOL_VERSION, "target": relative_target}, + stream, + ) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, marker) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + +async def run_trial(framework: str, expected_version: str, config_file: Path) -> None: + from eval_framework import install as select_framework + + select_framework(framework) + await _run_selected_trial(framework, expected_version, config_file) + + +async def _run_selected_trial(framework: str, expected_version: str, config_file: Path) -> None: + from eval_framework import framework_distribution + + distribution = framework_distribution(framework) + if importlib.metadata.version(distribution) != expected_version: + raise FrameworkVersionMismatch + try: + config_type = importlib.import_module(f"{framework}.models.trial.config").TrialConfig + trial_type = importlib.import_module(f"{framework}.trial.trial").Trial + config = config_type.model_validate_json(config_file.read_text()) + config_file.unlink() + if framework == "harbor": + from harbor.constants import TASK_CACHE_DIR + + task_identity = config.task.get_task_id().model_dump_json() + async with task_cache_lock(TASK_CACHE_DIR, task_identity): + namespace, ready = ready_task(TASK_CACHE_DIR, task_identity) + namespace.mkdir(parents=True, exist_ok=True, mode=0o700) + config.task.download_dir = namespace + config.task.overwrite = ready is None + trial = await create_harbor_trial(trial_type, config) + target = Path(trial.task.task_dir) + if ready is not None and target.resolve() != ready: + raise RuntimeError("Harbor task cache target changed for one identity") + if ready is None: + publish_ready_task(namespace, target) + else: + trial = await trial_type.create(config) + await trial.run() + finally: + config_file.unlink(missing_ok=True) + + +def apply_subject_egress_policy(task: object) -> None: + required = os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") == "1" + allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") + if not required: + return + if not allowed_host: + raise RuntimeError("required Eval egress proxy host is unavailable") + agent = task.config.agent + agent.network_mode = "allowlist" + agent.allowed_hosts = [allowed_host] + + +async def create_harbor_trial(trial_type: type, config: object) -> object: + trial_type._resolve_agent_skills(config) + task, task_download_result = await trial_type._load_task(config) + apply_subject_egress_policy(task) + if task.has_steps: + from harbor.trial.multi_step import MultiStepTrial + + return MultiStepTrial( + config, + _task=task, + _task_download_result=task_download_result, + ) + from harbor.trial.single_step import SingleStepTrial + + return SingleStepTrial( + config, + _task=task, + _task_download_result=task_download_result, + ) + + +async def main() -> None: + framework, expected_version, config_path = sys.argv[1:] + await _run_main(framework, expected_version, config_path) + + +async def _run_main(framework: str, expected_version: str, config_path: str) -> None: + task = asyncio.current_task() + assert task is not None + loop = asyncio.get_running_loop() + teardown_requested = False + + def request_teardown() -> None: + nonlocal teardown_requested + if teardown_requested: + return + teardown_requested = True + try: + from relay_agent import request_host_teardown + request_host_teardown() + finally: + task.cancel() + + for host_signal in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(host_signal, request_teardown) + try: + try: + await run_trial(framework, expected_version, Path(config_path)) + except FrameworkVersionMismatch: + raise SystemExit(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) from None + except asyncio.CancelledError: + if not teardown_requested: + raise + finally: + for host_signal in (signal.SIGINT, signal.SIGTERM): + loop.remove_signal_handler(host_signal) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs new file mode 100644 index 00000000000..5686d212dbe --- /dev/null +++ b/products/eval/src/contract.rs @@ -0,0 +1,1081 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Experiment specifications, orthogonal `v2` attempt records, strict `v1` +//! projections, run fingerprinting, and the semantic difference registry. + +use crate::measurement::{MeteringState, NormalizedUsage, deepseek_cost_usd}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use thiserror::Error; + +pub type JsonObject = Map; + +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ContractError { + #[error("{0}")] + Invalid(String), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum SchemaVersion { + #[serde(rename = "maka.eval.v1")] + V1, + #[serde(rename = "maka.eval.v2")] + V2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SubjectKind { + Maka, + External, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BenchmarkSpec { + pub id: String, + pub version: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutorSpec { + pub kind: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutionSpec { + pub max_concurrent_task_groups: usize, +} + +impl Default for ExecutionSpec { + fn default() -> Self { + Self { + max_concurrent_task_groups: 1, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectSpec { + pub id: String, + pub kind: SubjectKind, + pub credentials: Vec, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TaskSpec { + pub id: String, + pub input: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExperimentSpec { + pub schema_version: SchemaVersion, + pub id: String, + pub benchmark: BenchmarkSpec, + pub executor: ExecutorSpec, + #[serde(default)] + pub execution: ExecutionSpec, + pub subjects: Vec, + pub tasks: Vec, + pub repetitions: u32, + pub budget: JsonObject, + pub verifier: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExperimentCell { + pub id: String, + pub experiment_id: String, + pub benchmark: BenchmarkSpec, + pub executor: ExecutorSpec, + pub subject: SubjectSpec, + pub task: TaskSpec, + pub repetition: u32, + pub budget: JsonObject, + pub verifier: JsonObject, +} + +pub fn format_cell_id(task_id: &str, repetition: u32, subject_id: &str) -> String { + format!("{task_id}::{repetition}::{subject_id}") +} + +pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { + let mut cells = + Vec::with_capacity(spec.tasks.len() * (spec.repetitions as usize) * spec.subjects.len()); + for task in &spec.tasks { + for repetition in 1..=spec.repetitions { + for subject in &spec.subjects { + cells.push(ExperimentCell { + id: format_cell_id(&task.id, repetition, &subject.id), + experiment_id: spec.id.clone(), + benchmark: spec.benchmark.clone(), + executor: spec.executor.clone(), + subject: subject.clone(), + task: task.clone(), + repetition, + budget: spec.budget.clone(), + verifier: spec.verifier.clone(), + }); + } + } + } + cells +} + +pub fn parse_experiment_spec(value: &Value) -> Result { + let root = exact_object( + value, + "experiment", + &[ + "schemaVersion", + "id", + "benchmark", + "executor", + "subjects", + "tasks", + "repetitions", + "budget", + "verifier", + ], + &["execution"], + )?; + + let schema_version = match root.get("schemaVersion").and_then(Value::as_str) { + Some("maka.eval.v1") => SchemaVersion::V1, + Some("maka.eval.v2") => SchemaVersion::V2, + _ => { + return Err(ContractError::Invalid( + "unsupported experiment schema".into(), + )); + } + }; + + let benchmark_obj = exact_object( + root.get("benchmark").unwrap(), + "benchmark", + &["id", "version", "config"], + &[], + )?; + let executor_obj = exact_object( + root.get("executor").unwrap(), + "executor", + &["kind", "config"], + &[], + )?; + + let max_concurrent_task_groups = match root.get("execution") { + None => 1, + Some(exec_val) => { + let exec_obj = exact_object(exec_val, "execution", &["maxConcurrentTaskGroups"], &[])?; + positive_usize( + exec_obj.get("maxConcurrentTaskGroups").unwrap(), + "execution.maxConcurrentTaskGroups", + )? + } + }; + + let subjects_arr = root + .get("subjects") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("subjects must be an array".into()))?; + let mut subjects = Vec::with_capacity(subjects_arr.len()); + for (idx, item) in subjects_arr.iter().enumerate() { + let where_label = format!("subjects[{idx}]"); + let sub_obj = exact_object( + item, + &where_label, + &["id", "kind", "credentials", "config"], + &[], + )?; + let kind = match sub_obj.get("kind").and_then(Value::as_str) { + Some("maka") => SubjectKind::Maka, + Some("external") => SubjectKind::External, + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.kind is invalid" + ))); + } + }; + subjects.push(SubjectSpec { + id: identifier(sub_obj.get("id").unwrap(), &format!("{where_label}.id"))?, + kind, + credentials: unique_strings( + sub_obj.get("credentials").unwrap(), + &format!("{where_label}.credentials"), + )?, + config: decode_json_object( + sub_obj.get("config").unwrap(), + &format!("{where_label}.config"), + )?, + }); + } + + let tasks_arr = root + .get("tasks") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("tasks must be an array".into()))?; + let mut tasks = Vec::with_capacity(tasks_arr.len()); + for (idx, item) in tasks_arr.iter().enumerate() { + let where_label = format!("tasks[{idx}]"); + let task_obj = exact_object(item, &where_label, &["id", "input", "config"], &[])?; + tasks.push(TaskSpec { + id: identifier(task_obj.get("id").unwrap(), &format!("{where_label}.id"))?, + input: nonempty_string( + task_obj.get("input").unwrap(), + &format!("{where_label}.input"), + )?, + config: decode_json_object( + task_obj.get("config").unwrap(), + &format!("{where_label}.config"), + )?, + }); + } + + if subjects.is_empty() || tasks.is_empty() { + return Err(ContractError::Invalid("experiment arms are empty".into())); + } + ensure_unique_ids(subjects.iter().map(|s| s.id.as_str()), "subject")?; + ensure_unique_ids(tasks.iter().map(|t| t.id.as_str()), "task")?; + + let repetitions = positive_u32(root.get("repetitions").unwrap(), "repetitions")?; + + Ok(ExperimentSpec { + schema_version, + id: identifier(root.get("id").unwrap(), "experiment.id")?, + benchmark: BenchmarkSpec { + id: identifier(benchmark_obj.get("id").unwrap(), "benchmark.id")?, + version: nonempty_string(benchmark_obj.get("version").unwrap(), "benchmark.version")?, + config: decode_json_object(benchmark_obj.get("config").unwrap(), "benchmark.config")?, + }, + executor: ExecutorSpec { + kind: identifier(executor_obj.get("kind").unwrap(), "executor.kind")?, + config: decode_json_object(executor_obj.get("config").unwrap(), "executor.config")?, + }, + execution: ExecutionSpec { + max_concurrent_task_groups, + }, + subjects, + tasks, + repetitions, + budget: decode_json_object(root.get("budget").unwrap(), "budget")?, + verifier: decode_json_object(root.get("verifier").unwrap(), "verifier")?, + }) +} + +/// Canonical JSON serialization with lexicographically sorted object keys. +pub fn canonical_json_string(value: &Value) -> String { + let normalized = canonicalize_value(value); + serde_json::to_string(&normalized).expect("canonical Value serialization cannot fail") +} + +fn canonicalize_value(value: &Value) -> Value { + match value { + Value::Object(map) => { + let mut sorted: Vec<(&String, &Value)> = map.iter().collect(); + sorted.sort_by_key(|(a, _)| *a); + let mut out = Map::with_capacity(sorted.len()); + for (k, v) in sorted { + out.insert(k.clone(), canonicalize_value(v)); + } + Value::Object(out) + } + Value::Array(items) => Value::Array(items.iter().map(canonicalize_value).collect()), + other => other.clone(), + } +} + +/// Separates semantic comparability identity from execution-only knobs (like concurrency). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RunFingerprint { + /// Digest over experiment identity, benchmark, tasks, repetitions, budget, verifier, + /// and subject semantic configs. + pub semantic_sha256: String, + /// Digest over execution-only parameters (e.g. `maxConcurrentTaskGroups`). + pub execution_sha256: String, +} + +impl RunFingerprint { + pub fn compute(spec: &ExperimentSpec) -> Self { + let semantic_payload = serde_json::json!({ + "id": spec.id, + "benchmark": spec.benchmark, + "executorKind": spec.executor.kind, + "subjects": spec.subjects, + "tasks": spec.tasks, + "repetitions": spec.repetitions, + "budget": spec.budget, + "verifier": spec.verifier, + }); + let execution_payload = serde_json::json!({ + "maxConcurrentTaskGroups": spec.execution.max_concurrent_task_groups, + "executorConfig": spec.executor.config, + }); + Self { + semantic_sha256: sha256_hex(canonical_json_string(&semantic_payload).as_bytes()), + execution_sha256: sha256_hex(canonical_json_string(&execution_payload).as_bytes()), + } + } + + pub fn is_semantically_comparable(&self, other: &Self) -> bool { + self.semantic_sha256 == other.semantic_sha256 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvalResultStatus { + Completed, + SubjectFailed, + InfraFailed, + Indeterminate, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EvalResult { + pub score: Option, + pub usage: Option, + pub cost_usd: Option, + pub duration_ms: u64, + pub status: EvalResultStatus, + pub failure_reason: Option, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct V1CellAttempt { + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub completed_at: u64, + pub result: EvalResult, +} + +pub fn is_replaceable_attempt(attempt: &V1CellAttempt) -> bool { + matches!( + attempt.result.status, + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate + ) +} + +/// Canonical cell result selector: selects the earliest valid attempt by sequence (`sequence >= 1`) +/// whose status is neither `infra_failed` nor `indeterminate`. A `completed` attempt with score `0.0` +/// or a `subject_failed` attempt is terminal and never replaced. +pub fn select_cell_result(attempts: &[V1CellAttempt]) -> Option<&V1CellAttempt> { + let mut sorted: Vec<&V1CellAttempt> = attempts.iter().filter(|a| a.sequence >= 1).collect(); + sorted.sort_by_key(|a| a.sequence); + sorted + .into_iter() + .find(|attempt| !is_replaceable_attempt(attempt)) +} + +pub fn decode_eval_result(value: &Value, where_label: &str) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "score", + "usage", + "costUsd", + "durationMs", + "status", + "failureReason", + "artifacts", + ], + &[], + )?; + let status = match obj.get("status").and_then(Value::as_str) { + Some("completed") => EvalResultStatus::Completed, + Some("subject_failed") => EvalResultStatus::SubjectFailed, + Some("infra_failed") => EvalResultStatus::InfraFailed, + Some("indeterminate") => EvalResultStatus::Indeterminate, + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.status is invalid" + ))); + } + }; + let failure_reason = match obj.get("failureReason").unwrap() { + Value::Null => None, + Value::String(s) => Some(s.clone()), + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.failureReason is invalid" + ))); + } + }; + let score = nullable_f64(obj.get("score").unwrap(), &format!("{where_label}.score"))?; + let usage = match obj.get("usage").unwrap() { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, &format!("{where_label}.usage"))?), + }; + let cost_usd = nullable_nonnegative_f64( + obj.get("costUsd").unwrap(), + &format!("{where_label}.costUsd"), + )?; + let duration_ms = nonnegative_u64( + obj.get("durationMs").unwrap(), + &format!("{where_label}.durationMs"), + )?; + let artifacts_arr = obj + .get("artifacts") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid(format!("{where_label}.artifacts is invalid")))?; + let mut artifacts = Vec::with_capacity(artifacts_arr.len()); + for (idx, item) in artifacts_arr.iter().enumerate() { + artifacts.push(decode_json_object( + item, + &format!("{where_label}.artifacts[{idx}]"), + )?); + } + Ok(EvalResult { + score, + usage, + cost_usd, + duration_ms, + status, + failure_reason, + artifacts, + }) +} + +pub fn decode_v1_cell_attempt(value: &Value) -> Result { + let obj = exact_object( + value, + "attempt", + &["cellId", "sequence", "startedAt", "completedAt", "result"], + &[], + )?; + Ok(V1CellAttempt { + cell_id: nonempty_string(obj.get("cellId").unwrap(), "attempt.cellId")?, + sequence: positive_u32(obj.get("sequence").unwrap(), "attempt.sequence")?, + started_at: nonnegative_u64(obj.get("startedAt").unwrap(), "attempt.startedAt")?, + completed_at: nonnegative_u64(obj.get("completedAt").unwrap(), "attempt.completedAt")?, + result: decode_eval_result(obj.get("result").unwrap(), "attempt.result")?, + }) +} + +/// Orthogonal dimensions for native `maka.eval.v2` attempt records. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptPhase { + Prepared, + Dispatched, + AgentFinished, + Verified, + Settled, + Recovered, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionOutcome { + Completed, + SubjectFailed, + Timeout, + InfraFailed, + Cancelled, + Indeterminate, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum VerifierOutcome { + Verified { + score: f64, + }, + SubjectFailed { + score: Option, + reason: String, + }, + /// Missing reward file or empty verifier payload: never defaults to `0.0`. + MissingReward { + reason: String, + }, + VerifierInfraFailed { + reason: String, + }, + Skipped, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CleanupState { + Confirmed, + Unconfirmed { reason: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum EgressAuditState { + NotRequired, + Clean, + Truncated, + Missing, + Unreadable { reason: String }, + PolicyError { count: u64 }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CriticalArtifactsState { + Present, + Missing { paths: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct V2AttemptRecord { + pub schema_version: SchemaVersion, + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub completed_at: u64, + pub duration_ms: u64, + pub phase: AttemptPhase, + pub execution_outcome: ExecutionOutcome, + pub verifier_outcome: VerifierOutcome, + pub usage: Option, + pub metering_state: MeteringState, + pub cleanup_state: CleanupState, + pub audit_state: EgressAuditState, + pub critical_artifacts_state: CriticalArtifactsState, + pub cancelled_after_verification: bool, + pub recovered_post_hoc: bool, + pub failure_reason: Option, + pub artifacts: Vec, +} + +impl V2AttemptRecord { + /// Projects a native `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt` + /// while enforcing all 5 Semantic Difference Registry adjudications. + pub fn project_v1(&self) -> V1CellAttempt { + let mut artifacts = self.artifacts.clone(); + + // Cost is reported only when usage is present AND metering_state is Complete. + // Under Adjudication #1 (Timeout with reward but incomplete usage), lower-bound + // usage is preserved in `usage` while `costUsd` is projected as `null`. + let cost_usd = match (&self.usage, self.metering_state) { + (Some(u), MeteringState::Complete) => Some(deepseek_cost_usd(u)), + _ => None, + }; + + if self.recovered_post_hoc { + let mut marker = Map::new(); + marker.insert( + "kind".into(), + Value::String("post-hoc-evidence-recovery".into()), + ); + artifacts.push(marker); + } + + // Unconfirmed cleanup always resolves to `indeterminate` with `score: null`. + if let CleanupState::Unconfirmed { reason } = &self.cleanup_state { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::Indeterminate, + failure_reason: Some(reason.clone()), + artifacts, + }, + }; + } + + // Adjudication #4: Late cancellation BEFORE verification completes resolves to + // `indeterminate`, whereas late cancellation AFTER verification and settlement + // preserves the verified result and records the late cancellation signal. + if self.execution_outcome == ExecutionOutcome::Cancelled + && !self.cancelled_after_verification + { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::Indeterminate, + failure_reason: Some(self.failure_reason.clone().unwrap_or_else(|| { + "executor cancelled before verification completed".into() + })), + artifacts, + }, + }; + } + + // Adjudication #3: Missing or unreadable critical artifacts downgrade to `infra_failed`. + if let CriticalArtifactsState::Missing { paths } = &self.critical_artifacts_state { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some(format!( + "critical artifacts missing: {}", + paths.join(", ") + )), + artifacts, + }, + }; + } + + // Adjudication #2: Egress audit missing or unreadable fails closed (`infra_failed`), + // whereas `Truncated` (`ruleId == "audit_truncated"`) is preserved as a structured + // artifact flag without discarding a settled verification result. + match &self.audit_state { + EgressAuditState::Missing => { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some("egress audit log missing".into()), + artifacts, + }, + }; + } + EgressAuditState::Unreadable { reason } => { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some(reason.clone()), + artifacts, + }, + }; + } + EgressAuditState::Truncated => { + if !artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) + { + let mut audit_obj = Map::new(); + audit_obj.insert("kind".into(), Value::String("egress-audit".into())); + audit_obj.insert("truncated".into(), Value::Bool(true)); + artifacts.push(audit_obj); + } + } + EgressAuditState::NotRequired + | EgressAuditState::Clean + | EgressAuditState::PolicyError { .. } => {} + } + + let (status, score, failure_reason) = + match (&self.execution_outcome, &self.verifier_outcome) { + (ExecutionOutcome::InfraFailed, _) => ( + EvalResultStatus::InfraFailed, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject infrastructure failed".into()), + ), + ), + (ExecutionOutcome::Indeterminate, _) => ( + EvalResultStatus::Indeterminate, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject execution indeterminate".into()), + ), + ), + (_, VerifierOutcome::VerifierInfraFailed { reason }) + | (_, VerifierOutcome::MissingReward { reason }) => { + // Missing verifier reward never defaults to 0.0! + (EvalResultStatus::InfraFailed, None, Some(reason.clone())) + } + (ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }) => ( + // Adjudication #1: Agent timed out, verifier still ran and produced a valid + // reward; v1 status is `subject_failed` with the verified `score` preserved. + EvalResultStatus::SubjectFailed, + Some(*score), + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), + ), + ), + (ExecutionOutcome::SubjectFailed, VerifierOutcome::Verified { score }) => ( + EvalResultStatus::SubjectFailed, + Some(*score), + self.failure_reason.clone(), + ), + ( + ExecutionOutcome::Completed + | ExecutionOutcome::SubjectFailed + | ExecutionOutcome::Timeout + | ExecutionOutcome::Cancelled, + VerifierOutcome::SubjectFailed { score, reason }, + ) => ( + EvalResultStatus::SubjectFailed, + *score, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| reason.clone()), + ), + ), + ( + ExecutionOutcome::Completed | ExecutionOutcome::Cancelled, + VerifierOutcome::Verified { score }, + ) => (EvalResultStatus::Completed, Some(*score), None), + ( + ExecutionOutcome::Timeout | ExecutionOutcome::SubjectFailed, + VerifierOutcome::Skipped, + ) => ( + EvalResultStatus::SubjectFailed, + None, + self.failure_reason.clone(), + ), + ( + ExecutionOutcome::Completed | ExecutionOutcome::Cancelled, + VerifierOutcome::Skipped, + ) => (EvalResultStatus::Completed, None, None), + }; + + V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status, + failure_reason, + artifacts, + }, + } + } +} + +/// The 5 explicit semantic difference scenarios between Legacy TS `v1` and Native Rust `v2`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DifferenceScenarioId { + TimeoutWithRewardIncompleteUsage, + AuditTruncation, + MissingCriticalArtifacts, + LateCancellation, + EvidenceRecoveredLater, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DifferenceAdjudication { + pub id: DifferenceScenarioId, + pub title: String, + pub legacy_v1_behavior: String, + pub rust_v2_behavior: String, + pub v1_projection_rule: String, + pub conditions_comparable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DifferenceRegistry { + entries: Vec, +} + +impl Default for DifferenceRegistry { + fn default() -> Self { + Self::new() + } +} + +impl DifferenceRegistry { + pub fn new() -> Self { + Self { + entries: vec![ + DifferenceAdjudication { + id: DifferenceScenarioId::TimeoutWithRewardIncompleteUsage, + title: "Timeout with verifier reward but incomplete usage".into(), + legacy_v1_behavior: "Projects status='subject_failed', preserves verifier score, records recovered lower-bound usage, and sets costUsd=null because metering did not settle.".into(), + rust_v2_behavior: "Records ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }, and MeteringState::LowerBound orthogonally without conflating execution timeout and metering completeness.".into(), + v1_projection_rule: "Projects status='subject_failed', score=Some(score), usage=Some(lower_bound_usage), costUsd=None.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::AuditTruncation, + title: "Egress audit log truncation at byte cap".into(), + legacy_v1_behavior: "Appends ruleId='audit_truncated' marker to hits.jsonl and sets truncated=true in the egress-audit artifact while keeping the attempt status unchanged.".into(), + rust_v2_behavior: "Captures EgressAuditState::Truncated as an explicit dimension while distinguishing it from Missing or Unreadable audit logs (which fail closed as infra_failed).".into(), + v1_projection_rule: "Preserves completed/subject_failed status and emits the egress-audit artifact with truncated=true; Missing or Unreadable audit logs project to status='infra_failed'.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::MissingCriticalArtifacts, + title: "0-byte empty file vs. missing or unreadable critical artifact".into(), + legacy_v1_behavior: "Treats a present 0-byte regular file (e.g. empty stderr) as valid with bytes=0, while missing/unreadable required evidence fails the attempt.".into(), + rust_v2_behavior: "Tracks CriticalArtifactsState::Present (including 0-byte regular files verified by inode/stat) separately from CriticalArtifactsState::Missing.".into(), + v1_projection_rule: "0-byte regular files remain valid artifacts with bytes=0; Missing critical artifacts project to status='infra_failed' with score=None.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::LateCancellation, + title: "Cancellation signal before vs. after verification and settlement".into(), + legacy_v1_behavior: "An abort signal observed before attempt append could overwrite a verified result with indeterminate.".into(), + rust_v2_behavior: "Separates cancellation before verification (cancelled_after_verification=false) from a late signal arriving after verification and settlement commit (cancelled_after_verification=true).".into(), + v1_projection_rule: "Pre-verification cancellation projects to status='indeterminate' with score=None; post-settlement cancellation preserves the verified terminal result.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::EvidenceRecoveredLater, + title: "Post-hoc evidence recovered from checkpoint or host journal after crash".into(), + legacy_v1_behavior: "Mutated or re-ran the attempt when the process died before writing the final cell JSON.".into(), + rust_v2_behavior: "Appends a PostHocRecovery journal entry referencing the original attempt sequence and reconstructs V2AttemptRecord with recovered_post_hoc=true without mutating prior entries.".into(), + v1_projection_rule: "Projects recovered usage/cost and appends a post-hoc-evidence-recovery artifact while preserving immutable sequence numbering.".into(), + conditions_comparable: true, + }, + ], + } + } + + pub fn entries(&self) -> &[DifferenceAdjudication] { + &self.entries + } + + pub fn get(&self, id: DifferenceScenarioId) -> Option<&DifferenceAdjudication> { + self.entries.iter().find(|entry| entry.id == id) + } +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hex::encode(hasher.finalize()) +} + +pub(crate) fn exact_object<'a>( + value: &'a Value, + where_label: &str, + required: &[&str], + optional: &[&str], +) -> Result<&'a JsonObject, ContractError> { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + for key in obj.keys() { + if !required.contains(&key.as_str()) && !optional.contains(&key.as_str()) { + return Err(ContractError::Invalid(format!( + "{where_label} has unsupported fields" + ))); + } + } + for req in required { + if !obj.contains_key(*req) { + return Err(ContractError::Invalid(format!( + "{where_label}.{req} is required" + ))); + } + } + Ok(obj) +} + +pub(crate) fn decode_json_object( + value: &Value, + where_label: &str, +) -> Result { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + for (k, v) in obj { + validate_json_value(v, &format!("{where_label}.{k}"))?; + } + Ok(obj.clone()) +} + +fn validate_json_value(value: &Value, where_label: &str) -> Result<(), ContractError> { + match value { + Value::Null | Value::Bool(_) | Value::String(_) => Ok(()), + Value::Number(n) => { + if n.as_f64().is_some_and(f64::is_finite) { + Ok(()) + } else { + Err(ContractError::Invalid(format!( + "{where_label} must be JSON" + ))) + } + } + Value::Array(arr) => { + for (idx, item) in arr.iter().enumerate() { + validate_json_value(item, &format!("{where_label}[{idx}]"))?; + } + Ok(()) + } + Value::Object(map) => { + for (k, v) in map { + validate_json_value(v, &format!("{where_label}.{k}"))?; + } + Ok(()) + } + } +} + +pub(crate) fn identifier(value: &Value, where_label: &str) -> Result { + let text = nonempty_string(value, where_label)?; + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return Err(ContractError::Invalid(format!("{where_label} is required"))); + }; + if !first.is_ascii_alphanumeric() + || !chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) + { + return Err(ContractError::Invalid(format!("{where_label} is invalid"))); + } + Ok(text) +} + +pub(crate) fn nonempty_string(value: &Value, where_label: &str) -> Result { + match value.as_str() { + Some(s) if !s.is_empty() => Ok(s.to_owned()), + _ => Err(ContractError::Invalid(format!("{where_label} is required"))), + } +} + +fn unique_strings(value: &Value, where_label: &str) -> Result, ContractError> { + let arr = value + .as_array() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an array")))?; + let mut seen = BTreeSet::new(); + let mut out = Vec::with_capacity(arr.len()); + for item in arr { + let s = nonempty_string(item, where_label)?; + if !seen.insert(s.clone()) { + return Err(ContractError::Invalid(format!( + "{where_label} contains duplicates" + ))); + } + out.push(s); + } + Ok(out) +} + +fn ensure_unique_ids<'a>( + ids: impl Iterator, + label: &str, +) -> Result<(), ContractError> { + let mut seen = BTreeSet::new(); + for id in ids { + if !seen.insert(id) { + return Err(ContractError::Invalid(format!("duplicate {label} id"))); + } + } + Ok(()) +} + +pub(crate) fn positive_u32(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) if n >= 1 && n <= u64::from(u32::MAX) => Ok(n as u32), + _ => Err(ContractError::Invalid(format!( + "{where_label} must be positive" + ))), + } +} + +pub(crate) fn positive_usize(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) if n >= 1 => usize::try_from(n) + .map_err(|_| ContractError::Invalid(format!("{where_label} must be positive"))), + _ => Err(ContractError::Invalid(format!( + "{where_label} must be positive" + ))), + } +} + +pub(crate) fn nonnegative_u64(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) => Ok(n), + None => match value.as_f64() { + Some(f) if f.is_finite() && f >= 0.0 && f.fract() == 0.0 => Ok(f as u64), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + }, + } +} + +pub(crate) fn nullable_f64(value: &Value, where_label: &str) -> Result, ContractError> { + if value.is_null() { + return Ok(None); + } + match value.as_f64() { + Some(f) if f.is_finite() => Ok(Some(f)), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + } +} + +pub(crate) fn nullable_nonnegative_f64( + value: &Value, + where_label: &str, +) -> Result, ContractError> { + if value.is_null() { + return Ok(None); + } + match value.as_f64() { + Some(f) if f.is_finite() && f >= 0.0 => Ok(Some(f)), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + } +} diff --git a/products/eval/src/environment.rs b/products/eval/src/environment.rs new file mode 100644 index 00000000000..3aaceef045b --- /dev/null +++ b/products/eval/src/environment.rs @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Per-attempt trial environment abstraction, shared preparation cache boundary +//! locking, egress audit verification, and version-pinned Harbor/Pier bridge. + +pub mod harbor_relay; + +pub use harbor_relay::*; + +use crate::contract::{ + ContractError, EgressAuditState, EvalResultStatus, ExperimentCell, JsonObject, VerifierOutcome, + exact_object, nonempty_string, sha256_hex, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::HashMap; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; + +pub const EGRESS_AUDIT_DESTINATION: &str = "egress-hits.jsonl"; +pub const EGRESS_AUDIT_ARTIFACT_PATH: &str = "artifacts/egress-hits.jsonl"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubjectProcessTermination { + Exited, + FrameworkTimeout, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum SubjectProcessDiagnosticCategory { + #[serde(rename = "none")] + None, + #[serde(rename = "unstructured-output")] + UnstructuredOutput, + #[serde(rename = "result-frame-missing")] + ResultFrameMissing, + #[serde(rename = "result-frame-invalid")] + ResultFrameInvalid, + #[serde(rename = "result-frame-ambiguous")] + ResultFrameAmbiguous, + #[serde(rename = "result-frame-oversize")] + ResultFrameOversize, + #[serde(rename = "execution-scope-unavailable")] + ExecutionScopeUnavailable, +} + +impl SubjectProcessDiagnosticCategory { + pub fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::UnstructuredOutput => "unstructured-output", + Self::ResultFrameMissing => "result-frame-missing", + Self::ResultFrameInvalid => "result-frame-invalid", + Self::ResultFrameAmbiguous => "result-frame-ambiguous", + Self::ResultFrameOversize => "result-frame-oversize", + Self::ExecutionScopeUnavailable => "execution-scope-unavailable", + } + } + + pub fn is_result_frame_error(self) -> bool { + matches!( + self, + Self::ResultFrameMissing + | Self::ResultFrameInvalid + | Self::ResultFrameAmbiguous + | Self::ResultFrameOversize + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectProcessDiagnostic { + pub category: SubjectProcessDiagnosticCategory, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectProcessRequest { + pub command: String, + pub args: Vec, + #[serde(default)] + pub environment: HashMap, + #[serde(default)] + pub credential_environment: HashMap, + #[serde(default = "default_capture_stdout")] + pub capture_stdout: bool, +} + +fn default_capture_stdout() -> bool { + true +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectProcessOutput { + pub termination: SubjectProcessTermination, + pub exit_code: i32, + pub stdout: String, + #[serde(default)] + pub stderr: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub diagnostic: Option, +} + +pub type BoxFuture<'a, T> = Pin + Send + 'a>>; + +pub trait CommandRunner: Send + Sync { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result>; +} + +#[derive(Clone)] +pub struct SubjectExecutionContext { + pub cwd: String, + pub task_input: String, + pub metadata: JsonObject, + pub cancel: CancellationToken, + pub runner: Arc, +} + +impl SubjectExecutionContext { + pub async fn execute( + &self, + request: SubjectProcessRequest, + ) -> Result { + self.runner.execute(request).await + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutorVerification { + pub status: EvalResultStatus, + pub score: Option, + pub failure_reason: Option, + pub artifacts: Vec, + /// Explicitly flags when the verifier reward file or reward key was missing, + /// ensuring missing rewards never default to `0.0`. + #[serde(default)] + pub missing_reward: bool, +} + +impl ExecutorVerification { + pub fn to_verifier_outcome(&self) -> VerifierOutcome { + if self.missing_reward { + return VerifierOutcome::MissingReward { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verifier reward missing".into()), + }; + } + match (self.status, self.score) { + (EvalResultStatus::InfraFailed, _) => VerifierOutcome::VerifierInfraFailed { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verification failed".into()), + }, + (EvalResultStatus::Completed, Some(score)) => VerifierOutcome::Verified { score }, + (EvalResultStatus::Completed, None) => VerifierOutcome::MissingReward { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verifier completed without a numeric reward".into()), + }, + (EvalResultStatus::SubjectFailed, score) => VerifierOutcome::SubjectFailed { + score, + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verifier marked subject failed".into()), + }, + (EvalResultStatus::Indeterminate, _) => VerifierOutcome::VerifierInfraFailed { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verification indeterminate".into()), + }, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExecutorPreparationCode { + Cancelled, + PreparationFailed, + SpawnFailed, + ExitBeforeReady, + InvalidReady, + FrameworkVersionMismatch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum IndeterminateCause { + HostCancelled, + CleanupUnconfirmed, +} + +/// Active per-attempt trial environment provided to the runner during a single attempt. +pub trait ActiveTrialAttempt: Send + Sync { + fn context(&self) -> SubjectExecutionContext; + fn verify(&self) -> BoxFuture<'_, Result>; + fn read_audit(&self) -> BoxFuture<'_, (EgressAuditState, EgressAuditEvidence)> { + Box::pin(async { + ( + EgressAuditState::NotRequired, + EgressAuditEvidence { + failure_reason: None, + artifacts: Vec::new(), + }, + ) + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TrialEnvironmentOutcome { + Settled(T), + Indeterminate { + cause: IndeterminateCause, + partial: Option, + }, + NotStarted { + code: ExecutorPreparationCode, + artifacts: Vec, + }, +} + +pub type TrialAttemptOperation<'a> = Box< + dyn for<'b> FnOnce( + &'b dyn ActiveTrialAttempt, + ) -> BoxFuture<'b, crate::contract::V2AttemptRecord> + + Send + + 'a, +>; + +pub trait TrialEnvironment: Send + Sync { + fn kind(&self) -> &str; + + fn validate(&self, _cell: &ExperimentCell) -> Result<(), ContractError> { + Ok(()) + } + + fn preflight( + &self, + _subject_credential_names: &[String], + _cancel: CancellationToken, + ) -> BoxFuture<'_, Result<(), ContractError>> { + Box::pin(async { Ok(()) }) + } + + fn run_attempt<'a>( + &'a self, + cell: &'a ExperimentCell, + sequence: u32, + subject_credential_names: &'a [String], + cancel: CancellationToken, + operation: TrialAttemptOperation<'a>, + ) -> BoxFuture<'a, Result, String>>; +} + +/// Synchronizes task preparation across the entire cache-check + cache-build +/// boundary so concurrent arms in a task group never race on shared task assets. +#[derive(Debug, Default, Clone)] +pub struct PreparationCacheLocks { + locks: Arc>>>>, +} + +impl PreparationCacheLocks { + pub fn new() -> Self { + Self::default() + } + + /// Holds an exclusive per-key async lock across the entire `prepare_fn` future. + pub async fn with_task_lock(&self, task_key: &str, prepare_fn: F) -> T + where + F: FnOnce() -> Fut, + Fut: Future, + { + let key_mutex = { + let mut map = self.locks.lock().await; + map.entry(task_key.to_owned()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + }; + let _guard = key_mutex.lock().await; + prepare_fn().await + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EgressAuditEvidence { + pub failure_reason: Option, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct EgressAuditSummary { + pub truncated: bool, + pub policy_error_count: u64, + pub malformed_line_count: u64, +} + +pub fn describe_egress_audit( + bytes: Option<&[u8]>, + required: bool, +) -> (EgressAuditState, EgressAuditEvidence) { + if !required { + return ( + EgressAuditState::NotRequired, + EgressAuditEvidence { + failure_reason: None, + artifacts: Vec::new(), + }, + ); + } + let Some(bytes) = bytes else { + let mut missing_art = Map::new(); + missing_art.insert("kind".into(), Value::String("egress-audit-missing".into())); + missing_art.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + return ( + EgressAuditState::Missing, + EgressAuditEvidence { + failure_reason: Some("egress audit log missing".into()), + artifacts: vec![missing_art], + }, + ); + }; + + let summary = summarize_egress_audit_bytes(bytes); + let mut artifact = Map::new(); + artifact.insert("kind".into(), Value::String("egress-audit".into())); + artifact.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + artifact.insert("bytes".into(), Value::from(bytes.len() as u64)); + artifact.insert( + "sha256".into(), + Value::String(format!("sha256:{}", sha256_hex(bytes))), + ); + artifact.insert("truncated".into(), Value::Bool(summary.truncated)); + artifact.insert( + "policyErrorCount".into(), + Value::from(summary.policy_error_count), + ); + artifact.insert( + "malformedLineCount".into(), + Value::from(summary.malformed_line_count), + ); + + let state = if summary.truncated { + EgressAuditState::Truncated + } else if summary.policy_error_count > 0 { + EgressAuditState::PolicyError { + count: summary.policy_error_count, + } + } else { + EgressAuditState::Clean + }; + + ( + state, + EgressAuditEvidence { + failure_reason: None, + artifacts: vec![artifact], + }, + ) +} + +pub fn read_egress_audit_evidence( + trial_path: &Path, + required: bool, +) -> (EgressAuditState, EgressAuditEvidence) { + if !required { + return describe_egress_audit(None, false); + } + let audit_path = trial_path.join(EGRESS_AUDIT_ARTIFACT_PATH); + match std::fs::read(&audit_path) { + Ok(bytes) => describe_egress_audit(Some(&bytes), true), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => describe_egress_audit(None, true), + Err(err) => { + let reason = format!( + "failed to read egress audit log {} ({err})", + audit_path.display() + ); + let mut art = Map::new(); + art.insert( + "kind".into(), + Value::String("egress-audit-unreadable".into()), + ); + art.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + ( + EgressAuditState::Unreadable { + reason: reason.clone(), + }, + EgressAuditEvidence { + failure_reason: Some(reason), + artifacts: vec![art], + }, + ) + } + } +} + +fn summarize_egress_audit_bytes(bytes: &[u8]) -> EgressAuditSummary { + let mut summary = EgressAuditSummary::default(); + for line in bytes.split(|&b| b == b'\n') { + if line.is_empty() { + continue; + } + let Ok(text) = std::str::from_utf8(line) else { + summary.malformed_line_count += 1; + continue; + }; + let trimmed = text.trim(); + if trimmed.is_empty() { + continue; + } + let Ok(Value::Object(obj)) = serde_json::from_str::(trimmed) else { + summary.malformed_line_count += 1; + continue; + }; + match obj.get("ruleId").and_then(Value::as_str) { + Some("audit_truncated") => summary.truncated = true, + Some("policy_error") => summary.policy_error_count += 1, + _ => {} + } + } + summary +} + +/// Computes the deterministic 1:1 Harbor trial identifier for an Eval cell attempt. +pub fn harbor_trial_name_for_attempt(cell_id: &str, sequence: u32) -> String { + let digest = &sha256_hex(cell_id.as_bytes())[..16]; + let safe_prefix: String = cell_id + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .take(32) + .collect(); + format!("maka-eval-{safe_prefix}-{digest}-seq-{sequence:06}") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HarborExecutorConfig { + pub framework: String, + pub framework_version: String, + pub python_path_env: String, + pub trials_root_env: String, + pub egress_proxy_required: bool, +} + +impl HarborExecutorConfig { + pub fn decode(framework: &str, config: &JsonObject) -> Result { + let val = Value::Object(config.clone()); + let obj = exact_object( + &val, + "executor.config", + &[ + "frameworkVersion", + "pythonPathEnv", + "trialsRootEnv", + "environment", + ], + &["preparationEnvironment", "egressProxy", "mounts"], + )?; + let framework_version = + nonempty_string(obj.get("frameworkVersion").unwrap(), "frameworkVersion")?; + let python_path_env = nonempty_string(obj.get("pythonPathEnv").unwrap(), "pythonPathEnv")?; + let trials_root_env = nonempty_string(obj.get("trialsRootEnv").unwrap(), "trialsRootEnv")?; + let egress_proxy_required = obj.contains_key("egressProxy"); + Ok(Self { + framework: framework.to_owned(), + framework_version, + python_path_env, + trials_root_env, + egress_proxy_required, + }) + } + + pub fn validate_task(&self, cell: &ExperimentCell) -> Result { + let task_cfg = Value::Object(cell.task.config.clone()); + let obj = exact_object(&task_cfg, "task.config", &[&self.framework], &[])?; + let fw_obj = exact_object( + obj.get(&self.framework).unwrap(), + &format!("task.config.{}", self.framework), + &["path"], + &[], + )?; + let path_str = nonempty_string(fw_obj.get("path").unwrap(), "path")?; + Ok(PathBuf::from(path_str)) + } +} diff --git a/products/eval/src/environment/harbor_relay.rs b/products/eval/src/environment/harbor_relay.rs new file mode 100644 index 00000000000..65918b493c5 --- /dev/null +++ b/products/eval/src/environment/harbor_relay.rs @@ -0,0 +1,971 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Loopback TCP JSON-line relay server and Harbor/Pier trial executor. +//! +//! Implements the exact `ready` -> `execute` -> `executed` -> `verify` protocol +//! spoken by `products/eval/harbor/relay_agent.py` and `run_trial.py`, including +//! version-mismatch exit code detection (`78`), `SIGTERM` -> `SIGKILL` process +//! termination escalation, `/result.json` reward extraction, and +//! collected artifact SHA-256 inventory. + +use crate::contract::{ + ContractError, EgressAuditState, EvalResultStatus, ExperimentCell, JsonObject, V2AttemptRecord, + sha256_hex, +}; +use crate::environment::{ + ActiveTrialAttempt, BoxFuture, CommandRunner, EgressAuditEvidence, ExecutorPreparationCode, + ExecutorVerification, HarborExecutorConfig, IndeterminateCause, SubjectExecutionContext, + SubjectProcessDiagnostic, SubjectProcessDiagnosticCategory, SubjectProcessOutput, + SubjectProcessRequest, SubjectProcessTermination, TrialAttemptOperation, TrialEnvironment, + TrialEnvironmentOutcome, harbor_trial_name_for_attempt, read_egress_audit_evidence, +}; +use crate::store::{ + MAKA_RUNTIME_ARTIFACT_PATH, MAKA_SUBJECT_STDERR_PATH, MAKA_SUBJECT_STDOUT_PATH, + write_atomic_bytes, +}; +use serde_json::{Map, Value, json}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::process::{Child, Command}; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +pub const FRAMEWORK_VERSION_MISMATCH_EXIT_CODE: i32 = 78; +pub const PYTHON_TEARDOWN_DEADLINE_MS: u64 = 110_000; +pub const RELAY_SETTLEMENT_DEADLINE_MS: u64 = 120_000; +pub const TERM_SETTLEMENT_DEADLINE_MS: u64 = 20_000; +pub const KILL_SETTLEMENT_DEADLINE_MS: u64 = 5_000; + +/// Returns the path to the bundled `products/eval/harbor` directory. +pub fn bundled_harbor_relay_root() -> PathBuf { + if let Ok(custom) = std::env::var("MAKA_EVAL_HARBOR_ROOT") + && !custom.is_empty() + { + return PathBuf::from(custom); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("harbor") +} + +/// Terminates a trial child process (`SIGTERM` -> `SIGKILL`) and returns `true` +/// if the child exited cleanly on `SIGTERM`. +async fn terminate_trial(child: &mut Child, deadline_ms: u64) -> bool { + #[cfg(unix)] + if let Some(pid) = child.id() { + unsafe { + let _ = libc::kill(pid as i32, libc::SIGTERM); + } + } + if let Ok(Ok(status)) = + tokio::time::timeout(Duration::from_millis(deadline_ms), child.wait()).await + { + return status.success(); + } + + let _ = child.start_kill(); + let _ = tokio::time::timeout( + Duration::from_millis(KILL_SETTLEMENT_DEADLINE_MS), + child.wait(), + ) + .await; + false +} + +struct RelaySocketRunner { + token: String, + result_token: String, + credentials: HashMap, + execution_environment: HashMap, + stream: Arc>>, + used: AtomicBool, + last_diagnostic: Arc>>, +} + +impl CommandRunner for RelaySocketRunner { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if self.used.swap(true, Ordering::SeqCst) { + return Err("Trial already executed its subject".into()); + } + let mut mapped_credentials = HashMap::new(); + for (target, source) in &request.credential_environment { + let value = self + .credentials + .get(source) + .ok_or_else(|| format!("credential {source} was not admitted"))?; + mapped_credentials.insert(target.clone(), value.clone()); + } + + let mut merged_env = request.environment.clone(); + for (k, v) in &self.execution_environment { + if merged_env.contains_key(k) { + return Err(format!( + "subject environment overrides Eval execution environment: {k}" + )); + } + merged_env.insert(k.clone(), v.clone()); + } + + let execute_msg = json!({ + "token": self.token, + "kind": "execute", + "command": request.command, + "args": request.args, + "environment": merged_env, + "credentials": mapped_credentials, + "resultToken": self.result_token, + "captureStdout": request.capture_stdout, + }); + let mut payload = + serde_json::to_vec(&execute_msg).map_err(|e| format!("json error: {e}"))?; + payload.push(b'\n'); + + let mut line = String::new(); + { + let mut reader = self.stream.lock().await; + reader + .get_mut() + .write_all(&payload) + .await + .map_err(|_| "relay transport is unavailable".to_owned())?; + reader + .get_mut() + .flush() + .await + .map_err(|_| "relay transport is unavailable".to_owned())?; + let bytes_read = reader + .read_line(&mut line) + .await + .map_err(|_| "relay execution result was unavailable".to_owned())?; + if bytes_read == 0 { + return Err("relay execution result was unavailable".into()); + } + } + + let executed: Value = serde_json::from_str(line.trim()) + .map_err(|_| "relay returned an invalid execution result".to_owned())?; + if executed.get("token").and_then(Value::as_str) != Some(&self.token) + || executed.get("kind").and_then(Value::as_str) != Some("executed") + { + return Err("relay returned an invalid execution result".into()); + } + let termination = match executed.get("termination").and_then(Value::as_str) { + Some("exited") => SubjectProcessTermination::Exited, + Some("framework_timeout") => SubjectProcessTermination::FrameworkTimeout, + _ => return Err("relay returned an invalid execution result".into()), + }; + let exit_code = executed + .get("exitCode") + .and_then(Value::as_i64) + .and_then(|c| i32::try_from(c).ok()) + .ok_or_else(|| "relay returned an invalid execution result".to_owned())?; + let stdout = executed + .get("stdout") + .and_then(Value::as_str) + .ok_or_else(|| "relay returned an invalid execution result".to_owned())? + .to_owned(); + let diag_val = executed + .get("diagnostic") + .ok_or_else(|| "relay returned an invalid execution result".to_owned())?; + let diagnostic = decode_process_diagnostic(diag_val) + .ok_or_else(|| "relay returned an invalid execution result".to_owned())?; + *self.last_diagnostic.lock().await = Some(diagnostic.clone()); + + Ok(SubjectProcessOutput { + termination, + exit_code, + stdout, + stderr: String::new(), + diagnostic: Some(diagnostic), + }) + }) + } +} + +fn decode_process_diagnostic(value: &Value) -> Option { + let obj = value.as_object()?; + let cat_str = obj.get("category")?.as_str()?; + let category = match cat_str { + "none" if obj.len() == 1 => { + return Some(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::None, + bytes: None, + sha256: None, + }); + } + "unstructured-output" => SubjectProcessDiagnosticCategory::UnstructuredOutput, + "result-frame-missing" => SubjectProcessDiagnosticCategory::ResultFrameMissing, + "result-frame-invalid" => SubjectProcessDiagnosticCategory::ResultFrameInvalid, + "result-frame-ambiguous" => SubjectProcessDiagnosticCategory::ResultFrameAmbiguous, + "result-frame-oversize" => SubjectProcessDiagnosticCategory::ResultFrameOversize, + "execution-scope-unavailable" => { + SubjectProcessDiagnosticCategory::ExecutionScopeUnavailable + } + _ => return None, + }; + if obj.len() != 3 { + return None; + } + let bytes = obj.get("bytes")?.as_u64()?; + let sha256 = obj.get("sha256")?.as_str()?; + if sha256.len() != 64 + || !sha256 + .bytes() + .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + { + return None; + } + Some(SubjectProcessDiagnostic { + category, + bytes: Some(bytes), + sha256: Some(sha256.to_owned()), + }) +} + +struct ActiveHarborAttempt<'a> { + cell: &'a ExperimentCell, + framework: String, + egress_required: bool, + trial_name: String, + trial_path: PathBuf, + token: String, + context: SubjectExecutionContext, + stream: Arc>>, + child: Arc>, + decision_sent: Arc, + finalization_confirmed: Arc>>, +} + +impl ActiveTrialAttempt for ActiveHarborAttempt<'_> { + fn context(&self) -> SubjectExecutionContext { + self.context.clone() + } + + fn verify(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + if !self.decision_sent.swap(true, Ordering::SeqCst) { + let verify_msg = json!({ + "token": self.token, + "kind": "verify", + }); + let mut payload = serde_json::to_vec(&verify_msg).unwrap_or_default(); + payload.push(b'\n'); + let mut reader = self.stream.lock().await; + if reader.get_mut().write_all(&payload).await.is_err() + || reader.get_mut().shutdown().await.is_err() + { + let mut child = self.child.lock().await; + let ok = terminate_trial(&mut child, RELAY_SETTLEMENT_DEADLINE_MS).await; + *self.finalization_confirmed.lock().await = Some(ok); + return Err("relay verify decision was not delivered".into()); + } + } + let ok = { + let mut child = self.child.lock().await; + child.wait().await.ok().is_some_and(|s| s.success()) + }; + *self.finalization_confirmed.lock().await = Some(ok); + if !ok { + return Err("Trial did not finalize cleanly".into()); + } + read_trial_verification( + &self.trial_path, + &self.trial_name, + self.cell, + &self.framework, + self.egress_required, + ) + }) + } + + fn read_audit(&self) -> BoxFuture<'_, (EgressAuditState, EgressAuditEvidence)> { + let path = self.trial_path.clone(); + let req = self.egress_required; + Box::pin(async move { read_egress_audit_evidence(&path, req) }) + } +} + +/// Reads `/result.json`, extracts the verifier reward key without ever +/// defaulting a missing reward to `0.0`, verifies egress audit evidence, and walks +/// collected artifacts. +pub fn read_trial_verification( + trial_path: &Path, + trial_name: &str, + cell: &ExperimentCell, + framework: &str, + audit_required: bool, +) -> Result { + let result_bytes = std::fs::read(trial_path.join("result.json")) + .map_err(|e| format!("failed to read trial result.json: {e}"))?; + let result_json: Value = serde_json::from_slice(&result_bytes) + .map_err(|e| format!("invalid trial result.json: {e}"))?; + + let reward_key = cell + .verifier + .get("reward") + .and_then(Value::as_str) + .unwrap_or("reward"); + + let exception_type = result_json + .get("exception_info") + .and_then(|v| v.get("exception_type")) + .and_then(Value::as_str); + let subject_exception = matches!( + exception_type, + Some("AgentTimeoutError") | Some("NonZeroAgentExitCodeError") + ); + if result_json + .get("exception_info") + .is_some_and(|v| !v.is_null()) + && !subject_exception + { + return Err("Trial failed outside subject execution".into()); + } + + let score = result_json + .get("verifier_result") + .and_then(|vr| vr.get("rewards")) + .and_then(|r| r.get(reward_key)) + .and_then(Value::as_f64) + .filter(|n| n.is_finite()); + + let (_, audit) = read_egress_audit_evidence(trial_path, audit_required); + let missing_reward = score.is_none(); + let base_failure = missing_reward.then(|| "verifier produced no reward".to_owned()); + let status = if audit.failure_reason.is_some() || missing_reward { + EvalResultStatus::InfraFailed + } else if subject_exception { + EvalResultStatus::SubjectFailed + } else { + EvalResultStatus::Completed + }; + + let mut artifacts = Vec::new(); + let mut trial_art = Map::new(); + trial_art.insert("kind".into(), Value::String("trial".into())); + trial_art.insert( + "framework".into(), + Value::String(cell.executor.kind.as_str().into()), + ); + trial_art.insert("trialName".into(), Value::String(trial_name.into())); + artifacts.push(trial_art); + artifacts.extend(collected_artifact_inventory(trial_path, framework)); + artifacts.extend(audit.artifacts); + + Ok(ExecutorVerification { + status, + score, + failure_reason: audit.failure_reason.or(base_failure), + artifacts, + missing_reward, + }) +} + +pub fn collected_artifact_inventory(trial_path: &Path, framework: &str) -> Vec { + let root = if framework == "pier" { + trial_path.join("artifacts") + } else { + trial_path.join("artifacts").join("logs").join("artifacts") + }; + let targets = [ + root.join( + Path::new(MAKA_RUNTIME_ARTIFACT_PATH) + .file_name() + .unwrap_or_default(), + ), + root.join( + Path::new(MAKA_SUBJECT_STDOUT_PATH) + .file_name() + .unwrap_or_default(), + ), + root.join( + Path::new(MAKA_SUBJECT_STDERR_PATH) + .file_name() + .unwrap_or_default(), + ), + ]; + let mut files = Vec::new(); + for target in &targets { + walk_collected_artifacts(trial_path, target, &mut files); + } + files.sort_by(|a, b| { + let pa = a.get("path").and_then(Value::as_str).unwrap_or(""); + let pb = b.get("path").and_then(Value::as_str).unwrap_or(""); + pa.cmp(pb) + }); + files +} + +fn walk_collected_artifacts(trial_path: &Path, current: &Path, out: &mut Vec) { + let Ok(meta) = std::fs::symlink_metadata(current) else { + return; + }; + if meta.file_type().is_symlink() { + return; + } + if meta.is_file() { + let Ok(bytes) = std::fs::read(current) else { + return; + }; + let Ok(rel) = current.strip_prefix(trial_path) else { + return; + }; + let rel_str = rel + .components() + .map(|c| c.as_os_str().to_string_lossy()) + .collect::>() + .join("/"); + let mut obj = Map::new(); + obj.insert("kind".into(), Value::String("collected-artifact".into())); + obj.insert("path".into(), Value::String(rel_str)); + obj.insert("bytes".into(), Value::from(bytes.len() as u64)); + obj.insert( + "sha256".into(), + Value::String(format!("sha256:{}", sha256_hex(&bytes))), + ); + out.push(obj); + return; + } + if meta.is_dir() + && let Ok(entries) = std::fs::read_dir(current) + { + for entry in entries.flatten() { + walk_collected_artifacts(trial_path, &entry.path(), out); + } + } +} + +/// Real Harbor / Pier trial environment that binds a loopback TCP relay server, +/// writes `.json` with `relay_agent:RelayAgent`, spawns `run_trial.py` +/// (or a custom trial command for testing), and coordinates the trial lifecycle. +pub struct HarborTrialEnvironment { + config: HarborExecutorConfig, + spec_dir: PathBuf, + env_overrides: HashMap, +} + +impl HarborTrialEnvironment { + pub fn new(config: HarborExecutorConfig, spec_dir: PathBuf) -> Self { + Self { + config, + spec_dir, + env_overrides: HashMap::new(), + } + } + + pub fn with_env_overrides(mut self, env_overrides: HashMap) -> Self { + self.env_overrides = env_overrides; + self + } + + fn resolve_env_var(&self, key: &str) -> Option { + self.env_overrides + .get(key) + .cloned() + .or_else(|| std::env::var(key).ok()) + .filter(|v| !v.is_empty()) + } + + fn resolve_trials_root(&self) -> Result { + let raw = self + .resolve_env_var(&self.config.trials_root_env) + .ok_or_else(|| { + format!( + "machine path {} is unavailable", + self.config.trials_root_env + ) + })?; + Ok(PathBuf::from(raw)) + } + + fn resolve_python_executable(&self) -> Result { + let raw = self + .resolve_env_var(&self.config.python_path_env) + .ok_or_else(|| { + format!( + "machine path {} is unavailable", + self.config.python_path_env + ) + })?; + Ok(PathBuf::from(raw)) + } +} + +impl TrialEnvironment for HarborTrialEnvironment { + fn kind(&self) -> &str { + &self.config.framework + } + + fn validate(&self, cell: &ExperimentCell) -> Result<(), ContractError> { + let _ = self.config.validate_task(cell)?; + Ok(()) + } + + fn preflight( + &self, + subject_credential_names: &[String], + cancel: CancellationToken, + ) -> BoxFuture<'_, Result<(), ContractError>> { + let cred_names: Vec = subject_credential_names.to_vec(); + Box::pin(async move { + if cancel.is_cancelled() { + return Err(ContractError::Invalid("preflight cancelled".into())); + } + let _ = self.resolve_trials_root().map_err(ContractError::Invalid)?; + let _ = self + .resolve_python_executable() + .map_err(ContractError::Invalid)?; + for name in &cred_names { + if self.resolve_env_var(name).is_none() { + return Err(ContractError::Invalid(format!( + "subject credential {name} is required" + ))); + } + } + Ok(()) + }) + } + + fn run_attempt<'a>( + &'a self, + cell: &'a ExperimentCell, + sequence: u32, + _subject_credential_names: &'a [String], + cancel: CancellationToken, + operation: TrialAttemptOperation<'a>, + ) -> BoxFuture<'a, Result, String>> { + Box::pin(async move { + if cancel.is_cancelled() { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::Cancelled, + artifacts: Vec::new(), + }); + } + + let trials_root = match self.resolve_trials_root() { + Ok(p) => p, + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + let python_exe = match self.resolve_python_executable() { + Ok(p) => p, + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + + let mut credentials = HashMap::new(); + for name in &cell.subject.credentials { + let Some(val) = self.resolve_env_var(name) else { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + }; + credentials.insert(name.clone(), val); + } + + if std::fs::create_dir_all(&trials_root).is_err() { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + + let trial_name = harbor_trial_name_for_attempt(&cell.id, sequence); + let config_path = trials_root.join(format!("{trial_name}.json")); + let trial_path = trials_root.join(&trial_name); + let relay_token = format!( + "{}{}", + Uuid::new_v4().simple(), + &Uuid::new_v4().simple().to_string()[..16] + ); + + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(l) => l, + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + let relay_port = match listener.local_addr() { + Ok(addr) => addr.port(), + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + + let task_path = match self.config.validate_task(cell) { + Ok(p) => p.to_string_lossy().to_string(), + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + let timeout_multiplier = cell + .budget + .get("timeoutMultiplier") + .and_then(Value::as_f64) + .unwrap_or(1.0); + + let trial_config = json!({ + "task": { + "path": task_path, + "git_url": cell.benchmark.config.get("repository").and_then(Value::as_str).unwrap_or(""), + "git_commit_id": cell.benchmark.version, + }, + "trial_name": trial_name, + "trials_dir": trials_root.to_string_lossy(), + "timeout_multiplier": timeout_multiplier, + "agent": { + "import_path": "relay_agent:RelayAgent", + "kwargs": { + "relay_host": "127.0.0.1", + "relay_port": relay_port, + "relay_token": relay_token, + "teardown_timeout_ms": PYTHON_TEARDOWN_DEADLINE_MS, + } + }, + "environment": cell.executor.config.get("environment").cloned().unwrap_or_else(|| json!({})), + }); + let mut config_bytes = serde_json::to_vec(&trial_config).unwrap_or_default(); + config_bytes.push(b'\n'); + if write_atomic_bytes(&config_path, &config_bytes).is_err() { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + + let run_trial_script = bundled_harbor_relay_root().join("run_trial.py"); + + let mut cmd = Command::new(&python_exe); + cmd.arg(&run_trial_script) + .arg(&self.config.framework) + .arg(&self.config.framework_version) + .arg(&config_path) + .current_dir(&self.spec_dir) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + cmd.env("PYTHONPATH", bundled_harbor_relay_root()); + if self.config.egress_proxy_required { + cmd.env("MAKA_EVAL_EGRESS_REQUIRED", "1"); + } + + let mut child = match cmd.spawn() { + Ok(c) => c, + Err(_) => { + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::SpawnFailed, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "spawn", + ExecutorPreparationCode::SpawnFailed, + None, + )], + }); + } + }; + + // Wait for either the relay TCP connection or premature process exit. + let accept_res = tokio::select! { + _ = cancel.cancelled() => { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::Cancelled, + artifacts: Vec::new(), + }); + } + status = child.wait() => { + let exit_code = status.ok().and_then(|s| s.code()); + let code = if exit_code == Some(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) { + ExecutorPreparationCode::FrameworkVersionMismatch + } else { + ExecutorPreparationCode::ExitBeforeReady + }; + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "exit-before-ready", + code, + exit_code, + )], + }); + } + accepted = listener.accept() => accepted, + }; + + let Ok((socket, _)) = accept_res else { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::ExitBeforeReady, + artifacts: Vec::new(), + }); + }; + drop(listener); + + let mut reader = BufReader::new(socket); + let mut ready_line = String::new(); + let read_ready = tokio::select! { + _ = cancel.cancelled() => { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::Cancelled, + artifacts: Vec::new(), + }); + } + res = reader.read_line(&mut ready_line) => res, + }; + + let Ok(n) = read_ready else { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::InvalidReady, + artifacts: Vec::new(), + }); + }; + if n == 0 { + let status = child.wait().await.ok(); + let exit_code = status.and_then(|s| s.code()); + let code = if exit_code == Some(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) { + ExecutorPreparationCode::FrameworkVersionMismatch + } else { + ExecutorPreparationCode::InvalidReady + }; + return Ok(TrialEnvironmentOutcome::NotStarted { + code, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "ready-decode", + code, + exit_code, + )], + }); + } + + let Ok(ready_json) = serde_json::from_str::(ready_line.trim()) else { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::InvalidReady, + artifacts: Vec::new(), + }); + }; + + let valid_ready = ready_json.get("token").and_then(Value::as_str) == Some(&relay_token) + && ready_json.get("kind").and_then(Value::as_str) == Some("ready") + && ready_json + .get("instruction") + .and_then(Value::as_str) + .is_some() + && ready_json + .get("cwd") + .and_then(Value::as_str) + .is_some_and(|c| c.starts_with('/')); + if !valid_ready { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::InvalidReady, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "ready-decode", + ExecutorPreparationCode::InvalidReady, + None, + )], + }); + } + + let instruction = ready_json["instruction"].as_str().unwrap().to_owned(); + let cwd = ready_json["cwd"].as_str().unwrap().to_owned(); + let result_token = Uuid::new_v4().simple().to_string(); + + let mut metadata = Map::new(); + metadata.insert("trialName".into(), Value::String(trial_name.clone())); + metadata.insert( + "trialPath".into(), + Value::String(trial_path.to_string_lossy().into()), + ); + metadata.insert("meteringSecret".into(), Value::String(result_token.clone())); + + let mut exec_env = HashMap::new(); + exec_env.insert("DEBIAN_FRONTEND".into(), "noninteractive".into()); + exec_env.insert("TZ".into(), "Etc/UTC".into()); + + let shared_stream = Arc::new(Mutex::new(reader)); + let last_diagnostic = Arc::new(Mutex::new(None)); + let socket_runner = Arc::new(RelaySocketRunner { + token: relay_token.clone(), + result_token, + credentials, + execution_environment: exec_env, + stream: shared_stream.clone(), + used: AtomicBool::new(false), + last_diagnostic: last_diagnostic.clone(), + }); + + let subject_context = SubjectExecutionContext { + cwd, + task_input: instruction, + metadata, + cancel: cancel.clone(), + runner: socket_runner.clone(), + }; + + let shared_child = Arc::new(Mutex::new(child)); + let decision_sent = Arc::new(AtomicBool::new(false)); + let finalization_confirmed = Arc::new(Mutex::new(None)); + + let active = ActiveHarborAttempt { + cell, + framework: self.config.framework.clone(), + egress_required: self.config.egress_proxy_required, + trial_name, + trial_path, + token: relay_token, + context: subject_context, + stream: shared_stream, + child: shared_child.clone(), + decision_sent: decision_sent.clone(), + finalization_confirmed: finalization_confirmed.clone(), + }; + + let mut record = operation(&active).await; + + let host_cancelled = cancel.is_cancelled(); + if !decision_sent.load(Ordering::SeqCst) + || finalization_confirmed.lock().await.is_none() + { + let used = socket_runner.used.load(Ordering::SeqCst); + let mut child_guard = shared_child.lock().await; + let ok = terminate_trial( + &mut child_guard, + if used { + RELAY_SETTLEMENT_DEADLINE_MS + } else { + TERM_SETTLEMENT_DEADLINE_MS + }, + ) + .await; + *finalization_confirmed.lock().await = Some(ok); + } + + if let Some(diag) = last_diagnostic.lock().await.clone() + && diag.category != SubjectProcessDiagnosticCategory::None + { + let mut art = Map::new(); + art.insert("kind".into(), Value::String("executor-relay-result".into())); + art.insert( + "category".into(), + Value::String(diag.category.as_str().into()), + ); + if let Some(b) = diag.bytes { + art.insert("bytes".into(), Value::from(b)); + } + if let Some(s) = diag.sha256 { + art.insert("sha256".into(), Value::String(s)); + } + record.artifacts.push(art); + } + + if host_cancelled { + return Ok(TrialEnvironmentOutcome::Indeterminate { + cause: IndeterminateCause::HostCancelled, + partial: Some(record), + }); + } + + let finalized_ok = finalization_confirmed.lock().await.unwrap_or(false); + if !finalized_ok { + return Ok(TrialEnvironmentOutcome::Indeterminate { + cause: IndeterminateCause::CleanupUnconfirmed, + partial: Some(record), + }); + } + + Ok(TrialEnvironmentOutcome::Settled(record)) + }) + } +} + +fn preparation_artifact( + framework: &str, + trial_name: &str, + trial_path: &Path, + stage: &str, + code: ExecutorPreparationCode, + exit_code: Option, +) -> JsonObject { + let _ = std::fs::create_dir_all(trial_path); + let diag_name = "preparation-error.json"; + let diag_payload = json!({ + "stage": stage, + "framework": framework, + "code": code, + "errorCode": Value::Null, + "exitCode": exit_code, + "signal": Value::Null, + }); + if let Ok(mut bytes) = serde_json::to_vec(&diag_payload) { + bytes.push(b'\n'); + let _ = write_atomic_bytes(&trial_path.join(diag_name), &bytes); + } + let mut art = Map::new(); + art.insert("kind".into(), Value::String("executor-preparation".into())); + art.insert("framework".into(), Value::String(framework.into())); + art.insert("trialName".into(), Value::String(trial_name.into())); + art.insert("path".into(), Value::String(diag_name.into())); + art +} diff --git a/products/eval/src/lib.rs b/products/eval/src/lib.rs new file mode 100644 index 00000000000..e4cf378ed88 --- /dev/null +++ b/products/eval/src/lib.rs @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Measurement and control plane for multi-arm Maka evaluation experiments. + +pub mod contract; +pub mod environment; +pub mod measurement; +pub mod report; +pub mod runner; +pub mod store; +pub mod subject; diff --git a/products/eval/src/main.rs b/products/eval/src/main.rs new file mode 100644 index 00000000000..13f07c57c78 --- /dev/null +++ b/products/eval/src/main.rs @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use clap::{Parser, Subcommand}; +use maka_eval::contract::{expand_experiment, parse_experiment_spec}; +use maka_eval::environment::{HarborExecutorConfig, HarborTrialEnvironment}; +use maka_eval::report::{build_cohort_summary, export_v1_canonical, export_v2_canonical}; +use maka_eval::runner::{RunExperimentInput, run_experiment}; +use maka_eval::store::ExperimentDirectory; +use maka_eval::subject::{ExternalCliSubject, RustHostSubject, SubjectAdapter}; +use serde_json::Value; +use std::fs; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; +use tokio_util::sync::CancellationToken; + +pub const USAGE: &str = "usage: maka eval run --out [--cell ]"; + +#[derive(Debug, Parser)] +#[command( + name = "maka-eval", + about = "Maka evaluation measurement and control plane" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Execute an evaluation experiment specification. + Run { + /// Path to the experiment specification JSON file. + spec: PathBuf, + /// Experiment output directory. + #[arg(long = "out")] + out_dir: PathBuf, + /// Optional repeatable cell ID filter. + #[arg(long = "cell")] + cell_ids: Vec, + }, + /// Validate or inspect an experiment directory and export canonical projections. + Report { + /// Path to the experiment specification JSON file. + spec: PathBuf, + /// Experiment output directory. + #[arg(long = "out")] + out_dir: PathBuf, + /// Output format (`summary`, `v1`, or `v2`). + #[arg(long, default_value = "summary")] + format: String, + }, +} + +#[tokio::main] +async fn main() -> ExitCode { + let cli = match Cli::try_parse() { + Ok(c) => c, + Err(err) => { + eprintln!("{err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + + match cli.command { + Command::Run { + spec, + out_dir, + cell_ids, + } => run_cli_command(&spec, &out_dir, cell_ids).await, + Command::Report { + spec, + out_dir, + format, + } => { + let raw = match fs::read_to_string(&spec) { + Ok(s) => s, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let parsed: Value = match serde_json::from_str(&raw) { + Ok(v) => v, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let experiment_spec = match parse_experiment_spec(&parsed) { + Ok(s) => s, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let directory = match ExperimentDirectory::open(&out_dir, &experiment_spec) { + Ok(d) => d, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + + match format.as_str() { + "v1" => match export_v1_canonical(&experiment_spec, &directory.attempts) { + Ok(v1) => { + println!("{}", serde_json::to_string_pretty(&v1).unwrap()); + ExitCode::SUCCESS + } + Err(err) => { + eprintln!("maka eval: {err}"); + ExitCode::from(1) + } + }, + "v2" => match export_v2_canonical(&experiment_spec, &directory.attempts) { + Ok(v2) => { + println!("{}", serde_json::to_string_pretty(&v2).unwrap()); + ExitCode::SUCCESS + } + Err(err) => { + eprintln!("maka eval: {err}"); + ExitCode::from(1) + } + }, + _ => match export_v1_canonical(&experiment_spec, &directory.attempts) { + Ok(v1) => { + let summary = build_cohort_summary(&experiment_spec, &v1); + println!("{}", serde_json::to_string(&summary).unwrap()); + if summary.incomplete_cells == 0 { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } + } + Err(err) => { + eprintln!("maka eval: {err}"); + ExitCode::from(1) + } + }, + } + } + } +} + +async fn run_cli_command(spec: &PathBuf, out_dir: &PathBuf, cell_ids: Vec) -> ExitCode { + let raw = match fs::read_to_string(spec) { + Ok(s) => s, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let parsed: Value = match serde_json::from_str(&raw) { + Ok(v) => v, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let experiment_spec = match parse_experiment_spec(&parsed) { + Ok(s) => s, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let directory = match ExperimentDirectory::open(out_dir, &experiment_spec) { + Ok(d) => d, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + + let executor_cfg = match HarborExecutorConfig::decode( + &experiment_spec.executor.kind, + &experiment_spec.executor.config, + ) { + Ok(cfg) => cfg, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let spec_dir = spec + .parent() + .map(|p| p.to_path_buf()) + .unwrap_or_else(|| PathBuf::from(".")); + let environment = Arc::new(HarborTrialEnvironment::new(executor_cfg, spec_dir)); + + let subjects: Vec> = vec![ + Arc::new(RustHostSubject::new()), + Arc::new(ExternalCliSubject::new()), + ]; + + let cancel = CancellationToken::new(); + let signal_exit_code = Arc::new(AtomicU8::new(0)); + { + let cancel_clone = cancel.clone(); + let code_clone = signal_exit_code.clone(); + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut sigint = signal(SignalKind::interrupt()).ok(); + let mut sigterm = signal(SignalKind::terminate()).ok(); + tokio::select! { + _ = async { + if let Some(ref mut s) = sigint { + let _ = s.recv().await; + } else { + std::future::pending::<()>().await; + } + } => { + code_clone.store(130, Ordering::SeqCst); + cancel_clone.cancel(); + } + _ = async { + if let Some(ref mut s) = sigterm { + let _ = s.recv().await; + } else { + std::future::pending::<()>().await; + } + } => { + code_clone.store(143, Ordering::SeqCst); + cancel_clone.cancel(); + } + } + } + #[cfg(not(unix))] + { + if tokio::signal::ctrl_c().await.is_ok() { + code_clone.store(130, Ordering::SeqCst); + cancel_clone.cancel(); + } + } + }); + } + + let mut input = + RunExperimentInput::new(&experiment_spec, &directory.attempts, environment, subjects); + if !cell_ids.is_empty() { + input.cell_ids = Some(cell_ids.clone()); + } + input.cancel = cancel; + + let results = match run_experiment(input).await { + Ok(r) => r, + Err(err) => { + let sig = signal_exit_code.load(Ordering::SeqCst); + if sig != 0 { + return ExitCode::from(sig); + } + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + + let sig = signal_exit_code.load(Ordering::SeqCst); + if sig != 0 { + return ExitCode::from(sig); + } + + let summary = build_cohort_summary(&experiment_spec, &results); + println!("{}", serde_json::to_string(&summary).unwrap()); + + let expected_cells = if cell_ids.is_empty() { + expand_experiment(&experiment_spec).len() + } else { + cell_ids.len() + }; + + if results.len() == expected_cells { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } +} diff --git a/products/eval/src/measurement.rs b/products/eval/src/measurement.rs new file mode 100644 index 00000000000..722dbf9a63d --- /dev/null +++ b/products/eval/src/measurement.rs @@ -0,0 +1,682 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Token usage preservation, admission boundary accounting, checkpoint HMAC +//! verification, receipt-vs-proxy reconciliation, and standardized pricing. + +use crate::contract::{ContractError, JsonObject, exact_object, nonnegative_u64, sha256_hex}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +pub mod gateway; +pub use gateway::*; + +pub const METERING_CHECKPOINT_LIMIT_BYTES: usize = 64 * 1024; +pub const COST_USD_TOLERANCE: f64 = 1e-9; + +/// Exact 6-counter normalized token usage. +/// +/// Each counter is preserved as reported by the provider or receipt; `total_tokens` +/// is never synthesized or overwritten by summing the other five counters. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct NormalizedUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub reasoning_tokens: u64, + pub total_tokens: u64, +} + +impl NormalizedUsage { + pub fn decode(value: &Value, where_label: &str) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "inputTokens", + "outputTokens", + "cacheReadTokens", + "cacheWriteTokens", + "reasoningTokens", + "totalTokens", + ], + &[], + )?; + Ok(Self { + input_tokens: nonnegative_u64( + obj.get("inputTokens").unwrap(), + &format!("{where_label}.inputTokens"), + )?, + output_tokens: nonnegative_u64( + obj.get("outputTokens").unwrap(), + &format!("{where_label}.outputTokens"), + )?, + cache_read_tokens: nonnegative_u64( + obj.get("cacheReadTokens").unwrap(), + &format!("{where_label}.cacheReadTokens"), + )?, + cache_write_tokens: nonnegative_u64( + obj.get("cacheWriteTokens").unwrap(), + &format!("{where_label}.cacheWriteTokens"), + )?, + reasoning_tokens: nonnegative_u64( + obj.get("reasoningTokens").unwrap(), + &format!("{where_label}.reasoningTokens"), + )?, + total_tokens: nonnegative_u64( + obj.get("totalTokens").unwrap(), + &format!("{where_label}.totalTokens"), + )?, + }) + } + + /// Accumulates another `NormalizedUsage` record counter-by-counter without + /// synthesizing `total_tokens` from the component fields. + pub fn accumulate_exact(&mut self, other: &Self) { + self.input_tokens = self.input_tokens.saturating_add(other.input_tokens); + self.output_tokens = self.output_tokens.saturating_add(other.output_tokens); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(other.cache_read_tokens); + self.cache_write_tokens = self + .cache_write_tokens + .saturating_add(other.cache_write_tokens); + self.reasoning_tokens = self.reasoning_tokens.saturating_add(other.reasoning_tokens); + self.total_tokens = self.total_tokens.saturating_add(other.total_tokens); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MeteringState { + Complete, + LowerBound, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum TokenBasis { + #[serde(rename = "complete")] + Complete, + #[serde(rename = "lower-bound")] + LowerBound, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PricingRates { + pub input_per_million: f64, + pub output_per_million: f64, + pub cache_read_per_million: f64, + pub cache_write_per_million: f64, +} + +/// Published DeepSeek V4 Flash prices in USD per million tokens (off-peak band). +pub const DEEPSEEK_V4_FLASH_COST: PricingRates = PricingRates { + input_per_million: 0.22, + output_per_million: 0.66, + cache_read_per_million: 0.007, + cache_write_per_million: 0.22, +}; + +pub fn deepseek_cost_usd(usage: &NormalizedUsage) -> f64 { + price_usage_usd(usage, &DEEPSEEK_V4_FLASH_COST) +} + +pub fn price_usage_usd(usage: &NormalizedUsage, rates: &PricingRates) -> f64 { + let cached = usage + .cache_read_tokens + .saturating_add(usage.cache_write_tokens); + let uncached = usage.input_tokens.saturating_sub(cached); + ((uncached as f64) * rates.input_per_million + + (usage.cache_read_tokens as f64) * rates.cache_read_per_million + + (usage.cache_write_tokens as f64) * rates.cache_write_per_million + + (usage.output_tokens as f64) * rates.output_per_million) + / 1_000_000.0 +} + +pub fn costs_match_within_tolerance(left: Option, right: Option) -> bool { + match (left, right) { + (None, None) => true, + (Some(a), Some(b)) => (a - b).abs() <= COST_USD_TOLERANCE, + _ => false, + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProviderMeteringCounts { + pub usage: Option, + pub settled: bool, + pub requests: u64, + pub in_flight_requests: u64, + pub admitted_requests: u64, + pub usage_requests: u64, + pub removed_web_tools: u64, + pub models: Vec, + pub tool_names: Vec, +} + +impl ProviderMeteringCounts { + pub fn validate(&self) -> bool { + if self.settled && self.in_flight_requests > 0 { + return false; + } + if self.in_flight_requests > self.requests + || self.admitted_requests > self.requests + || self.usage_requests > self.admitted_requests + { + return false; + } + if self.usage_requests > 0 && self.usage.is_none() { + return false; + } + true + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DerivedMetering { + pub settled_requests: u64, + pub missing_usage_requests: u64, + pub usage_complete: bool, + pub cost_usd: Option, + pub token_basis: TokenBasis, + pub metering_state: MeteringState, +} + +pub fn derive_metering(counts: &ProviderMeteringCounts) -> DerivedMetering { + let usage_complete = counts.settled + && counts.admitted_requests > 0 + && counts.usage_requests == counts.admitted_requests; + let cost_usd = match (&counts.usage, usage_complete) { + (Some(u), true) => Some(deepseek_cost_usd(u)), + _ => None, + }; + let metering_state = match (&counts.usage, usage_complete) { + (Some(_), true) => MeteringState::Complete, + (Some(_), false) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + DerivedMetering { + settled_requests: counts.requests.saturating_sub(counts.in_flight_requests), + missing_usage_requests: counts + .admitted_requests + .saturating_sub(counts.usage_requests), + usage_complete, + cost_usd, + token_basis: if usage_complete { + TokenBasis::Complete + } else { + TokenBasis::LowerBound + }, + metering_state, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RequestLedgerEntry { + admitted: bool, + in_flight: bool, + usage: Option, + model: Option, +} + +/// Tracks request admission across the trial execution cutoff, deduplicating +/// repeated checkpoints and reconciling Host receipts against proxy metering +/// without double-counting. +#[derive(Debug, Clone, Default)] +pub struct AdmissionBoundaryLedger { + cutoff_closed: bool, + settled: bool, + requests: BTreeMap, + seen_checkpoint_digests: BTreeSet, + latest_checkpoint: Option, + removed_web_tools: u64, + tool_names: BTreeSet, +} + +impl AdmissionBoundaryLedger { + pub fn new() -> Self { + Self::default() + } + + pub fn is_admission_closed(&self) -> bool { + self.cutoff_closed + } + + /// Closes admission for new requests at the execution deadline/cancellation + /// boundary while allowing already-admitted in-flight requests to drain. + pub fn close_admission(&mut self) { + self.cutoff_closed = true; + } + + /// Attempts to admit a request with `request_id`. Returns `true` if admitted, + /// or `false` if the admission boundary has already closed. Idempotent for an + /// already-recorded `request_id`. + pub fn record_request_start(&mut self, request_id: impl Into) -> bool { + let id = request_id.into(); + if let Some(existing) = self.requests.get(&id) { + return existing.admitted; + } + let admitted = !self.cutoff_closed; + self.requests.insert( + id, + RequestLedgerEntry { + admitted, + in_flight: admitted, + usage: None, + model: None, + }, + ); + admitted + } + + /// Records the settlement of an admitted request. Duplicate settlements for + /// the same `request_id` (e.g. from both a Host receipt and a proxy log) are + /// reconciled idempotently without double-counting tokens. + pub fn settle_request( + &mut self, + request_id: &str, + usage: Option, + model: Option, + ) -> Result<(), ContractError> { + let Some(entry) = self.requests.get_mut(request_id) else { + return Err(ContractError::Invalid(format!( + "unknown request_id in metering settlement: {request_id}" + ))); + }; + if !entry.admitted { + return Ok(()); + } + entry.in_flight = false; + if let (Some(existing), Some(incoming)) = (&entry.usage, &usage) { + if existing != incoming { + return Err(ContractError::Invalid(format!( + "conflicting usage between receipt and proxy for request {request_id}" + ))); + } + } else if entry.usage.is_none() { + entry.usage = usage; + } + if entry.model.is_none() { + entry.model = model; + } + Ok(()) + } + + /// Ingests a proxy checkpoint snapshot. Duplicate snapshots with the same + /// canonical digest are ignored so repeated checkpoint reads never inflate counts. + pub fn ingest_checkpoint(&mut self, counts: ProviderMeteringCounts) -> bool { + let serialized = serde_json::to_vec(&counts).unwrap_or_default(); + let digest = sha256_hex(&serialized); + if !self.seen_checkpoint_digests.insert(digest) { + return false; + } + self.latest_checkpoint = Some(counts); + true + } + + pub fn mark_proxy_settled(&mut self) { + if self.requests.values().all(|r| !r.in_flight) { + self.settled = true; + } + } + + pub fn snapshot(&self) -> ProviderMeteringCounts { + if self.requests.is_empty() + && let Some(cp) = &self.latest_checkpoint + { + return cp.clone(); + } + let requests = self.requests.len() as u64; + let in_flight_requests = self.requests.values().filter(|r| r.in_flight).count() as u64; + let admitted_requests = self.requests.values().filter(|r| r.admitted).count() as u64; + let mut usage_requests = 0u64; + let mut accumulated = NormalizedUsage::default(); + let mut models = BTreeSet::new(); + + for entry in self.requests.values().filter(|r| r.admitted) { + if let Some(u) = &entry.usage { + usage_requests += 1; + accumulated.accumulate_exact(u); + } + if let Some(m) = &entry.model { + models.insert(m.clone()); + } + } + + ProviderMeteringCounts { + usage: if usage_requests > 0 { + Some(accumulated) + } else { + None + }, + settled: self.settled && in_flight_requests == 0, + requests, + in_flight_requests, + admitted_requests, + usage_requests, + removed_web_tools: self.removed_web_tools, + models: models.into_iter().collect(), + tool_names: self.tool_names.iter().cloned().collect(), + } + } +} + +/// Reconciles Host-reported usage with proxy-recovered usage without double-counting. +/// +/// - When both Host receipts and proxy counts are settled and complete, they must agree; +/// otherwise reconciliation reports `MeteringState::LowerBound` with the conservative record. +/// - When the Host died before settling receipts, the verified proxy checkpoint provides +/// either `Complete` (if the proxy settled with all admitted requests accounted for) or +/// `LowerBound` usage. +pub fn reconcile_host_and_proxy_usage( + host_usage: Option<&NormalizedUsage>, + host_settled: bool, + proxy_counts: Option<&ProviderMeteringCounts>, +) -> (Option, MeteringState, Option) { + match (host_usage, host_settled, proxy_counts) { + (Some(h), true, Some(p)) => { + let derived = derive_metering(p); + if derived.usage_complete { + if p.usage.as_ref() == Some(h) { + ( + Some(h.clone()), + MeteringState::Complete, + Some(deepseek_cost_usd(h)), + ) + } else { + (Some(h.clone()), MeteringState::LowerBound, None) + } + } else { + ( + Some(h.clone()), + MeteringState::Complete, + Some(deepseek_cost_usd(h)), + ) + } + } + (Some(h), true, None) => ( + Some(h.clone()), + MeteringState::Complete, + Some(deepseek_cost_usd(h)), + ), + (Some(h), false, Some(p)) => { + let derived = derive_metering(p); + if derived.usage_complete { + (p.usage.clone(), MeteringState::Complete, derived.cost_usd) + } else { + ( + p.usage.clone().or_else(|| Some(h.clone())), + MeteringState::LowerBound, + None, + ) + } + } + (Some(h), false, None) => (Some(h.clone()), MeteringState::LowerBound, None), + (None, _, Some(p)) => { + let derived = derive_metering(p); + (p.usage.clone(), derived.metering_state, derived.cost_usd) + } + (None, _, None) => (None, MeteringState::Unknown, None), + } +} + +/// Computes RFC 2104 HMAC-SHA256 in hex. +pub fn hmac_sha256_hex(secret: &[u8], message: &[u8]) -> String { + const BLOCK_SIZE: usize = 64; + let mut key_block = [0u8; BLOCK_SIZE]; + if secret.len() > BLOCK_SIZE { + let digest = Sha256::digest(secret); + key_block[..digest.len()].copy_from_slice(&digest); + } else { + key_block[..secret.len()].copy_from_slice(secret); + } + + let mut ipad = [0x36u8; BLOCK_SIZE]; + let mut opad = [0x5cu8; BLOCK_SIZE]; + for i in 0..BLOCK_SIZE { + ipad[i] ^= key_block[i]; + opad[i] ^= key_block[i]; + } + + let mut inner = Sha256::new(); + inner.update(ipad); + inner.update(message); + let inner_hash = inner.finalize(); + + let mut outer = Sha256::new(); + outer.update(opad); + outer.update(inner_hash); + hex::encode(outer.finalize()) +} + +pub(crate) fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut diff = 0u8; + for (x, y) in a.iter().zip(b.iter()) { + diff |= x ^ y; + } + diff == 0 +} + +pub fn sign_metering_checkpoint(body: &JsonObject, secret: &str) -> JsonObject { + let serialized = serde_json::to_string(&Value::Object(body.clone())) + .expect("serializing JsonObject cannot fail"); + let mac = hmac_sha256_hex(secret.as_bytes(), serialized.as_bytes()); + let mut signed = body.clone(); + signed.insert("mac".into(), Value::String(mac)); + signed +} + +pub fn metering_checkpoint_mac_matches(record: &JsonObject, secret: &str) -> bool { + let Some(mac) = record.get("mac").and_then(Value::as_str) else { + return false; + }; + if mac.len() != 64 || !mac.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) { + return false; + } + let mut body = record.clone(); + body.remove("mac"); + let serialized = match serde_json::to_string(&Value::Object(body)) { + Ok(s) => s, + Err(_) => return false, + }; + let expected = hmac_sha256_hex(secret.as_bytes(), serialized.as_bytes()); + constant_time_eq(expected.as_bytes(), mac.as_bytes()) +} + +/// Reads a bounded regular file while refusing symlinks (`O_NOFOLLOW` on unix) +/// and oversized or empty files. +pub fn read_bounded_regular_file(path: &Path, limit_bytes: usize) -> Option> { + use std::io::Read; + let symlink_meta = std::fs::symlink_metadata(path).ok()?; + if !symlink_meta.file_type().is_file() { + return None; + } + let mut options = std::fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(path).ok()?; + let meta = file.metadata().ok()?; + if !meta.is_file() || meta.len() == 0 || meta.len() > (limit_bytes as u64) { + return None; + } + let size = meta.len() as usize; + let mut buf = Vec::with_capacity(size); + file.read_to_end(&mut buf).ok()?; + if buf.len() != size { + return None; + } + Some(buf) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RecoveredExternalMetering { + pub usage: Option, + pub cost_usd: Option, + pub admitted_requests: u64, + pub metering_state: MeteringState, + pub counts: ProviderMeteringCounts, + pub artifact: JsonObject, +} + +pub fn recover_external_metering_checkpoint( + trial_path: &Path, + profile: &str, + metering_secret: &str, +) -> Option { + let rel_path = format!("agent/{profile}.provider-usage.json"); + let full_path = trial_path.join(&rel_path); + let bytes = read_bounded_regular_file(&full_path, METERING_CHECKPOINT_LIMIT_BYTES)?; + let parsed: Value = serde_json::from_slice(&bytes).ok()?; + let checkpoint = exact_object( + &parsed, + "external metering checkpoint", + &[ + "schemaVersion", + "profile", + "usage", + "settled", + "requests", + "inFlightRequests", + "admittedRequests", + "usageRequests", + "removedWebTools", + "models", + "toolNames", + "mac", + ], + &[], + ) + .ok()?; + if checkpoint.get("schemaVersion").and_then(Value::as_str) + != Some("maka.external_provider_usage.v2") + || checkpoint.get("profile").and_then(Value::as_str) != Some(profile) + || !metering_checkpoint_mac_matches(checkpoint, metering_secret) + { + return None; + } + let usage = match checkpoint.get("usage")? { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, "external usage").ok()?), + }; + let settled = checkpoint.get("settled")?.as_bool()?; + let requests = nonnegative_u64(checkpoint.get("requests")?, "requests").ok()?; + let in_flight_requests = + nonnegative_u64(checkpoint.get("inFlightRequests")?, "inFlightRequests").ok()?; + let admitted_requests = + nonnegative_u64(checkpoint.get("admittedRequests")?, "admittedRequests").ok()?; + let usage_requests = nonnegative_u64(checkpoint.get("usageRequests")?, "usageRequests").ok()?; + let removed_web_tools = + nonnegative_u64(checkpoint.get("removedWebTools")?, "removedWebTools").ok()?; + let models = checkpoint + .get("models")? + .as_array()? + .iter() + .map(|v| v.as_str().map(str::to_owned)) + .collect::>>()?; + let tool_names = checkpoint + .get("toolNames")? + .as_array()? + .iter() + .map(|v| v.as_str().map(str::to_owned)) + .collect::>>()?; + + let counts = ProviderMeteringCounts { + usage: usage.clone(), + settled, + requests, + in_flight_requests, + admitted_requests, + usage_requests, + removed_web_tools, + models: models.clone(), + tool_names: tool_names.clone(), + }; + if !counts.validate() { + return None; + } + let derived = derive_metering(&counts); + let mut artifact = Map::new(); + artifact.insert( + "kind".into(), + Value::String("provider-metering-recovery".into()), + ); + artifact.insert("profile".into(), Value::String(profile.into())); + artifact.insert("path".into(), Value::String(rel_path)); + artifact.insert("bytes".into(), Value::from(bytes.len() as u64)); + artifact.insert("sha256".into(), Value::String(sha256_hex(&bytes))); + artifact.insert("requests".into(), Value::from(counts.requests)); + artifact.insert( + "settledRequests".into(), + Value::from(derived.settled_requests), + ); + artifact.insert( + "inFlightRequests".into(), + Value::from(counts.in_flight_requests), + ); + artifact.insert( + "admittedRequests".into(), + Value::from(counts.admitted_requests), + ); + artifact.insert("usageRequests".into(), Value::from(counts.usage_requests)); + artifact.insert( + "missingUsageRequests".into(), + Value::from(derived.missing_usage_requests), + ); + artifact.insert("usageComplete".into(), Value::Bool(derived.usage_complete)); + artifact.insert( + "tokenBasis".into(), + serde_json::to_value(derived.token_basis).ok()?, + ); + artifact.insert( + "removedWebTools".into(), + Value::from(counts.removed_web_tools), + ); + artifact.insert( + "models".into(), + Value::Array(models.into_iter().map(Value::String).collect()), + ); + artifact.insert( + "toolNames".into(), + Value::Array(tool_names.into_iter().map(Value::String).collect()), + ); + + Some(RecoveredExternalMetering { + usage, + cost_usd: derived.cost_usd, + admitted_requests: counts.admitted_requests, + metering_state: derived.metering_state, + counts, + artifact, + }) +} diff --git a/products/eval/src/measurement/gateway.rs b/products/eval/src/measurement/gateway.rs new file mode 100644 index 00000000000..60754024bf6 --- /dev/null +++ b/products/eval/src/measurement/gateway.rs @@ -0,0 +1,990 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Local loopback HTTP metering and admission gateway for Maka Host and external subjects. +//! +//! Binds `127.0.0.1:0`, authenticates per-attempt ``, strips disallowed +//! web tools from model requests, forwards upstream using the real provider credential, +//! parses streaming SSE and JSON usage/admission facts, writes atomic HMAC-signed checkpoints, +//! and enforces a synchronous `stop_accepting()` cutoff while draining in-flight requests. + +use crate::contract::JsonObject; +use crate::measurement::{ + NormalizedUsage, ProviderMeteringCounts, constant_time_eq, sign_metering_checkpoint, +}; +use crate::store::write_atomic_bytes; +use http_body_util::{BodyExt, Full, StreamBody}; +use hyper::body::{Bytes, Frame, Incoming}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use serde_json::{Map, Value, json}; +use std::collections::BTreeSet; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use tokio::net::TcpListener; +use tokio::sync::{Mutex, Notify, mpsc}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub const PROVIDER_USAGE_CHECKPOINT_SCHEMA: &str = "maka.external_provider_usage.v2"; + +const DISALLOWED_WEB_TOOL_NAMES: &[&str] = &["websearch", "webfetch", "fetchurl"]; + +type BoxBody = http_body_util::combinators::BoxBody; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectedRequestBody { + pub body: Vec, + pub removed: u64, + pub model: Option, + pub tool_names: Vec, +} + +/// Strips disallowed web tools (`WebSearch`, `WebFetch`, `FetchURL`) from an OpenAI or +/// Anthropic JSON request payload and extracts the requested model and remaining tool names. +pub fn remove_eval_web_tools(body: &[u8]) -> ProjectedRequestBody { + if body.is_empty() { + return ProjectedRequestBody { + body: Vec::new(), + removed: 0, + model: None, + tool_names: Vec::new(), + }; + } + let Ok(Value::Object(mut obj)) = serde_json::from_slice::(body) else { + return ProjectedRequestBody { + body: body.to_vec(), + removed: 0, + model: None, + tool_names: Vec::new(), + }; + }; + let model = obj + .get("model") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_owned); + let Some(Value::Array(tools)) = obj.get("tools").cloned() else { + return ProjectedRequestBody { + body: body.to_vec(), + removed: 0, + model, + tool_names: Vec::new(), + }; + }; + let original_len = tools.len(); + let kept: Vec = tools + .into_iter() + .filter(|tool| !is_disallowed_web_tool(tool)) + .collect(); + let removed = (original_len - kept.len()) as u64; + let tool_names: Vec = kept.iter().flat_map(extract_tool_name).collect(); + if removed == 0 { + return ProjectedRequestBody { + body: body.to_vec(), + removed: 0, + model, + tool_names, + }; + } + obj.insert("tools".into(), Value::Array(kept)); + let projected_bytes = serde_json::to_vec(&Value::Object(obj)).unwrap_or_else(|_| body.to_vec()); + ProjectedRequestBody { + body: projected_bytes, + removed, + model, + tool_names, + } +} + +fn is_disallowed_web_tool(tool: &Value) -> bool { + let Some(obj) = tool.as_object() else { + return false; + }; + let direct_name = obj.get("name").and_then(Value::as_str); + let fn_name = obj + .get("function") + .and_then(Value::as_object) + .and_then(|f| f.get("name")) + .and_then(Value::as_str); + for candidate in [direct_name, fn_name].into_iter().flatten() { + let norm = normalize_tool_identifier(candidate); + if DISALLOWED_WEB_TOOL_NAMES.contains(&norm.as_str()) { + return true; + } + } + let Some(tool_type) = obj.get("type").and_then(Value::as_str) else { + return false; + }; + let norm_type = normalize_tool_identifier(tool_type); + DISALLOWED_WEB_TOOL_NAMES + .iter() + .any(|disallowed| norm_type == *disallowed || norm_type.starts_with(disallowed)) +} + +fn normalize_tool_identifier(value: &str) -> String { + value + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .map(|c| c.to_ascii_lowercase()) + .collect() +} + +fn extract_tool_name(tool: &Value) -> Vec { + let Some(obj) = tool.as_object() else { + return Vec::new(); + }; + if let Some(name) = obj.get("name").and_then(Value::as_str) { + return vec![name.to_owned()]; + } + if let Some(name) = obj + .get("function") + .and_then(Value::as_object) + .and_then(|f| f.get("name")) + .and_then(Value::as_str) + { + return vec![name.to_owned()]; + } + if let Some(t) = obj.get("type").and_then(Value::as_str) { + return vec![t.to_owned()]; + } + Vec::new() +} + +/// Determines whether a parsed provider JSON/SSE event proves the upstream provider +/// admitted an inference call. +pub fn is_inference_admission_event(event: &Map) -> bool { + let object = event.get("object").and_then(Value::as_str); + if matches!( + object, + Some("chat.completion") | Some("chat.completion.chunk") + ) { + return true; + } + if let Some(event_type) = event.get("type").and_then(Value::as_str) + && matches!( + event_type, + "response.created" + | "response.in_progress" + | "response.output_item.added" + | "response.content_part.added" + | "response.output_text.delta" + | "response.reasoning_text.delta" + | "response.function_call_arguments.delta" + | "response.completed" + ) + { + return true; + } + object == Some("response") + && event + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| !id.is_empty()) + && (event.get("output").is_some_and(Value::is_array) + || event.get("usage").is_some_and(Value::is_object)) +} + +/// Extracts a 6-counter `NormalizedUsage` snapshot from an OpenAI-compatible event/response. +pub fn usage_from_event(event: &Map) -> Option { + let response = if event.get("type").and_then(Value::as_str) == Some("response.completed") { + event + .get("response") + .and_then(Value::as_object) + .unwrap_or(event) + } else { + event + }; + let raw = response + .get("usage") + .and_then(Value::as_object) + .or_else(|| event.get("usage").and_then(Value::as_object))?; + + let input = json_u64(raw.get("input_tokens").or_else(|| raw.get("prompt_tokens"))); + let output = json_u64( + raw.get("output_tokens") + .or_else(|| raw.get("completion_tokens")), + ); + let input_details = raw + .get("input_tokens_details") + .and_then(Value::as_object) + .or_else(|| raw.get("prompt_tokens_details").and_then(Value::as_object)); + let output_details = raw + .get("output_tokens_details") + .and_then(Value::as_object) + .or_else(|| { + raw.get("completion_tokens_details") + .and_then(Value::as_object) + }); + + let cache_read = json_u64( + raw.get("cache_read_input_tokens") + .or_else(|| raw.get("prompt_cache_hit_tokens")) + .or_else(|| input_details.and_then(|d| d.get("cached_tokens"))), + ); + let cache_write = json_u64( + raw.get("cache_creation_input_tokens") + .or_else(|| raw.get("cache_write_input_tokens")) + .or_else(|| input_details.and_then(|d| d.get("cache_write_tokens"))) + .or_else(|| input_details.and_then(|d| d.get("cache_creation_tokens"))), + ); + let reasoning = json_u64(output_details.and_then(|d| d.get("reasoning_tokens"))); + let reported_total = json_u64(raw.get("total_tokens")); + let total_tokens = if reported_total > 0 { + reported_total + } else { + input.saturating_add(output) + }; + Some(NormalizedUsage { + input_tokens: input, + output_tokens: output, + cache_read_tokens: cache_read, + cache_write_tokens: cache_write, + reasoning_tokens: reasoning, + total_tokens, + }) +} + +fn json_u64(value: Option<&Value>) -> u64 { + value.and_then(Value::as_u64).unwrap_or(0) +} + +fn merge_stream_usage( + left: Option, + right: Option, +) -> Option { + match (left, right) { + (None, r) => r, + (l, None) => l, + (Some(l), Some(r)) => Some(NormalizedUsage { + input_tokens: l.input_tokens.max(r.input_tokens), + output_tokens: l.output_tokens.max(r.output_tokens), + cache_read_tokens: l.cache_read_tokens.max(r.cache_read_tokens), + cache_write_tokens: l.cache_write_tokens.max(r.cache_write_tokens), + reasoning_tokens: l.reasoning_tokens.max(r.reasoning_tokens), + total_tokens: l.total_tokens.max(r.total_tokens), + }), + } +} + +/// Incremental SSE / JSON response parser for a single model request. +#[derive(Debug, Default)] +pub struct UsageStreamParser { + buffered: String, + full_text: String, + current: Option, + admitted: bool, +} + +impl UsageStreamParser { + pub fn new() -> Self { + Self::default() + } + + pub fn push(&mut self, text: &str) { + self.buffered.push_str(text); + if self.full_text.len() < 512 * 1024 { + self.full_text.push_str(text); + } + while let Some(newline) = self.buffered.find('\n') { + let line = self.buffered[..newline].to_owned(); + self.buffered.drain(..=newline); + self.consume_line(&line); + } + } + + pub fn admitted(&self) -> bool { + self.admitted + } + + pub fn finish(&mut self) -> (Option, bool) { + if !self.buffered.trim().is_empty() { + let tail = std::mem::take(&mut self.buffered); + self.consume_line(&tail); + } + if !self.admitted + && !self.full_text.trim().is_empty() + && let Ok(Value::Object(obj)) = serde_json::from_str::(self.full_text.trim()) + { + self.admitted |= is_inference_admission_event(&obj); + self.current = merge_stream_usage(self.current.take(), usage_from_event(&obj)); + } + (self.current.clone(), self.admitted) + } + + fn consume_line(&mut self, line: &str) { + let trimmed = line.trim(); + let raw = trimmed + .strip_prefix("data:") + .map(str::trim_start) + .unwrap_or(trimmed); + if raw.is_empty() || raw == "[DONE]" { + return; + } + let Ok(Value::Object(obj)) = serde_json::from_str::(raw) else { + return; + }; + self.admitted |= is_inference_admission_event(&obj); + self.current = merge_stream_usage(self.current.take(), usage_from_event(&obj)); + } +} + +pub fn join_upstream_url(base_url: &str, request_uri: &str) -> Result { + let mut base = Url::parse(base_url).map_err(|e| format!("invalid upstream baseUrl: {e}"))?; + let req_url = Url::parse(&format!( + "http://maka-eval.local/{}", + request_uri.trim_start_matches('/') + )) + .map_err(|e| format!("invalid request URI: {e}"))?; + let base_path = base.path().trim_end_matches('/'); + let req_path = req_url.path().trim_start_matches('/'); + // Avoid duplicating `/v1` when base_url already ends with `/v1` and the client + // sends `/v1/chat/completions` (or when base_url is `http://host/v1` and client + // sends `/chat/completions`). + let combined_path = if base_path.ends_with("/v1") && req_path.starts_with("v1/") { + format!("{base_path}/{}", &req_path[3..]) + } else if req_path.is_empty() { + base_path.to_owned() + } else { + format!("{base_path}/{req_path}") + }; + base.set_path(&combined_path); + base.set_query(req_url.query()); + Ok(base.to_string()) +} + +#[derive(Debug, Clone)] +pub struct GatewayConfig { + pub upstream_base_url: String, + pub upstream_api_key: String, + pub profile: String, + pub gateway_credential: String, + pub checkpoint_path: Option, + pub checkpoint_secret: String, +} + +#[derive(Debug, Default)] +struct GatewayCounters { + total_usage: NormalizedUsage, + requests: u64, + in_flight_requests: u64, + admitted_requests: u64, + usage_requests: u64, + removed_web_tools: u64, + settled: bool, + models: BTreeSet, + tool_names: BTreeSet, +} + +impl GatewayCounters { + fn snapshot(&self) -> ProviderMeteringCounts { + ProviderMeteringCounts { + usage: if self.usage_requests > 0 { + Some(self.total_usage.clone()) + } else { + None + }, + settled: self.settled, + requests: self.requests, + in_flight_requests: self.in_flight_requests, + admitted_requests: self.admitted_requests, + usage_requests: self.usage_requests, + removed_web_tools: self.removed_web_tools, + models: self.models.iter().cloned().collect(), + tool_names: self.tool_names.iter().cloned().collect(), + } + } +} + +struct SharedGatewayState { + config: GatewayConfig, + client: reqwest::Client, + accepting: AtomicBool, + counters: Mutex, + checkpoint_lock: Mutex<()>, + in_flight_zero: Notify, +} + +impl SharedGatewayState { + async fn persist_checkpoint(&self) { + let Some(path) = &self.config.checkpoint_path else { + return; + }; + let _guard = self.checkpoint_lock.lock().await; + let snap = self.counters.lock().await.snapshot(); + let Ok(Value::Object(mut body)) = serde_json::to_value(&snap) else { + return; + }; + body.insert( + "schemaVersion".into(), + Value::String(PROVIDER_USAGE_CHECKPOINT_SCHEMA.into()), + ); + body.insert("profile".into(), Value::String(self.config.profile.clone())); + let signed: JsonObject = sign_metering_checkpoint(&body, &self.config.checkpoint_secret); + if let Ok(mut bytes) = serde_json::to_vec(&Value::Object(signed)) { + bytes.push(b'\n'); + let _ = write_atomic_bytes(path, &bytes); + } + } + + fn is_authorized(&self, req: &Request) -> bool { + let expected = self.config.gateway_credential.as_bytes(); + if expected.is_empty() { + return true; + } + if let Some(auth) = req + .headers() + .get(hyper::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + && let Some(token) = auth.strip_prefix("Bearer ").map(str::trim) + && constant_time_eq(token.as_bytes(), expected) + { + return true; + } + if let Some(api_key) = req.headers().get("x-api-key").and_then(|v| v.to_str().ok()) + && constant_time_eq(api_key.trim().as_bytes(), expected) + { + return true; + } + false + } +} + +/// Running local HTTP Eval Metering Gateway. +pub struct EvalMeteringGateway { + addr: SocketAddr, + gateway_credential: String, + state: Arc, + shutdown: CancellationToken, + worker: Option>, +} + +impl EvalMeteringGateway { + pub async fn start(config: GatewayConfig) -> Result { + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .build() + .map_err(|e| format!("failed to build gateway HTTP client: {e}"))?; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .map_err(|e| format!("failed to bind metering gateway on 127.0.0.1:0: {e}"))?; + let addr = listener + .local_addr() + .map_err(|e| format!("failed to read metering gateway local addr: {e}"))?; + let gateway_credential = config.gateway_credential.clone(); + + let state = Arc::new(SharedGatewayState { + config, + client, + accepting: AtomicBool::new(true), + counters: Mutex::new(GatewayCounters::default()), + checkpoint_lock: Mutex::new(()), + in_flight_zero: Notify::new(), + }); + state.persist_checkpoint().await; + + let shutdown = CancellationToken::new(); + let worker_shutdown = shutdown.clone(); + let worker_state = state.clone(); + let worker = tokio::spawn(async move { + loop { + tokio::select! { + _ = worker_shutdown.cancelled() => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { + break; + }; + let conn_state = worker_state.clone(); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + handle_gateway_request(conn_state.clone(), req) + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + } + } + }); + + Ok(Self { + addr, + gateway_credential, + state, + shutdown, + worker: Some(worker), + }) + } + + pub fn port(&self) -> u16 { + self.addr.port() + } + + pub fn base_url(&self) -> String { + format!("http://127.0.0.1:{}", self.addr.port()) + } + + pub fn openai_v1_base_url(&self) -> String { + format!("http://127.0.0.1:{}/v1", self.addr.port()) + } + + pub fn gateway_credential(&self) -> &str { + &self.gateway_credential + } + + /// Synchronously closes the admission window so any subsequent request receives HTTP 503 + /// without being forwarded or counted. + pub fn stop_accepting(&self) { + self.state.accepting.store(false, Ordering::SeqCst); + } + + /// Closes admission, waits for all already-accepted in-flight requests to finish draining, + /// marks the snapshot as `settled = true`, writes the final signed checkpoint, and returns + /// the settled `ProviderMeteringCounts`. + pub async fn report(&self) -> ProviderMeteringCounts { + self.stop_accepting(); + loop { + let notified = self.state.in_flight_zero.notified(); + { + let mut counters = self.state.counters.lock().await; + if counters.in_flight_requests == 0 { + counters.settled = true; + break; + } + } + notified.await; + } + self.state.persist_checkpoint().await; + self.state.counters.lock().await.snapshot() + } + + pub async fn close(mut self) -> ProviderMeteringCounts { + let counts = self.report().await; + self.shutdown.cancel(); + if let Some(worker) = self.worker.take() { + let _ = worker.await; + } + counts + } +} + +impl Drop for EvalMeteringGateway { + fn drop(&mut self) { + self.state.accepting.store(false, Ordering::SeqCst); + self.shutdown.cancel(); + if let Some(worker) = self.worker.take() { + worker.abort(); + } + } +} + +fn json_error_response(status: StatusCode, message: &str) -> Response { + let payload = serde_json::to_vec(&json!({ "error": message })).unwrap_or_default(); + Response::builder() + .status(status) + .header(hyper::header::CONTENT_TYPE, "application/json") + .body(Full::new(Bytes::from(payload)).boxed()) + .unwrap() +} + +async fn finish_in_flight(state: &SharedGatewayState) { + let became_zero = { + let mut counters = state.counters.lock().await; + counters.in_flight_requests = counters.in_flight_requests.saturating_sub(1); + counters.in_flight_requests == 0 + }; + state.persist_checkpoint().await; + if became_zero { + state.in_flight_zero.notify_waiters(); + } +} + +async fn handle_gateway_request( + state: Arc, + req: Request, +) -> Result, Infallible> { + if !state.accepting.load(Ordering::SeqCst) { + return Ok(json_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "provider proxy has settled", + )); + } + if !state.is_authorized(&req) { + return Ok(json_error_response( + StatusCode::UNAUTHORIZED, + "invalid eval gateway credential", + )); + } + + { + let mut counters = state.counters.lock().await; + if !state.accepting.load(Ordering::SeqCst) { + return Ok(json_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "provider proxy has settled", + )); + } + counters.requests += 1; + counters.in_flight_requests += 1; + } + state.persist_checkpoint().await; + + let method = req.method().clone(); + let uri_str = req + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_owned()) + .unwrap_or_else(|| "/".to_owned()); + let headers = req.headers().clone(); + + let body_bytes = match req.into_body().collect().await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "failed to read request body", + )); + } + }; + + let projected = remove_eval_web_tools(&body_bytes); + + { + let mut counters = state.counters.lock().await; + counters.removed_web_tools += projected.removed; + if let Some(m) = &projected.model { + counters.models.insert(m.clone()); + } + for name in &projected.tool_names { + counters.tool_names.insert(name.clone()); + } + } + + let target_url = match join_upstream_url(&state.config.upstream_base_url, &uri_str) { + Ok(u) => u, + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "invalid upstream URL", + )); + } + }; + + let reqwest_method = + reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::POST); + let mut upstream_req = state.client.request(reqwest_method, &target_url); + for (name, value) in &headers { + let lower = name.as_str().to_ascii_lowercase(); + if matches!( + lower.as_str(), + "host" | "content-length" | "connection" | "authorization" | "x-api-key" + ) { + continue; + } + if let Ok(val_str) = value.to_str() { + upstream_req = upstream_req.header(name.as_str(), val_str); + } + } + upstream_req = upstream_req.header( + "authorization", + format!("Bearer {}", state.config.upstream_api_key), + ); + if !projected.body.is_empty() { + upstream_req = upstream_req.body(projected.body); + } + + let mut upstream_resp = match upstream_req.send().await { + Ok(r) => r, + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "provider proxy failed", + )); + } + }; + + let status_u16 = upstream_resp.status().as_u16(); + let is_ok = upstream_resp.status().is_success(); + let mut resp_builder = + Response::builder().status(StatusCode::from_u16(status_u16).unwrap_or(StatusCode::OK)); + for (name, value) in upstream_resp.headers() { + let lower = name.as_str().to_ascii_lowercase(); + if matches!( + lower.as_str(), + "content-length" | "content-encoding" | "transfer-encoding" | "connection" + ) { + continue; + } + if let Ok(val_str) = value.to_str() { + resp_builder = resp_builder.header(name.as_str(), val_str); + } + } + + let (tx, rx) = mpsc::channel::, Infallible>>(16); + let stream_state = state.clone(); + tokio::spawn(async move { + let mut parser = UsageStreamParser::new(); + let mut counted = false; + + while let Ok(Some(chunk)) = upstream_resp.chunk().await { + let text = String::from_utf8_lossy(&chunk); + parser.push(&text); + if !counted && is_ok && parser.admitted() { + counted = true; + { + let mut counters = stream_state.counters.lock().await; + counters.admitted_requests += 1; + } + stream_state.persist_checkpoint().await; + } + if tx.send(Ok(Frame::data(chunk))).await.is_err() { + break; + } + } + + let (usage, admitted) = parser.finish(); + if !counted && is_ok && admitted { + counted = true; + { + let mut counters = stream_state.counters.lock().await; + counters.admitted_requests += 1; + } + stream_state.persist_checkpoint().await; + } + if counted && let Some(u) = usage { + let mut counters = stream_state.counters.lock().await; + counters.usage_requests += 1; + counters.total_usage.accumulate_exact(&u); + } + finish_in_flight(&stream_state).await; + }); + + let stream = tokio_stream_from_receiver(rx); + let body = http_body_util::BodyExt::boxed(StreamBody::new(stream)); + Ok(resp_builder.body(body).unwrap()) +} + +fn tokio_stream_from_receiver( + mut rx: mpsc::Receiver, +) -> impl futures_util::Stream + Send + Sync + 'static +where + T: Send + 'static, +{ + futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::measurement::recover_external_metering_checkpoint; + + #[test] + fn remove_web_tools_strips_disallowed_tools_and_keeps_others() { + let input = serde_json::to_vec(&json!({ + "model": "deepseek-chat", + "tools": [ + {"type": "function", "function": {"name": "WebSearch"}}, + {"type": "function", "function": {"name": "ReadFile"}}, + {"name": "web_fetch", "type": "custom"}, + {"type": "web_search_20250305", "name": "lookup"} + ] + })) + .unwrap(); + let projected = remove_eval_web_tools(&input); + assert_eq!(projected.removed, 3); + assert_eq!(projected.model.as_deref(), Some("deepseek-chat")); + assert_eq!(projected.tool_names, vec!["ReadFile".to_owned()]); + let parsed: Value = serde_json::from_slice(&projected.body).unwrap(); + assert_eq!(parsed["tools"].as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn gateway_authenticates_strips_web_tools_streams_usage_and_enforces_cutoff() { + let mut parser = UsageStreamParser::new(); + parser.push(concat!( + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":25,\"prompt_cache_hit_tokens\":60,\"completion_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":125}}\n\n", + "data: [DONE]\n\n" + )); + let (parsed_usage, parsed_admitted) = parser.finish(); + assert!(parsed_admitted); + assert_eq!( + parsed_usage, + Some(NormalizedUsage { + input_tokens: 100, + output_tokens: 25, + cache_read_tokens: 60, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 125, + }) + ); + + // Start a mock upstream OpenAI-compatible SSE server. + let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_addr = upstream_listener.local_addr().unwrap(); + if std::net::TcpStream::connect(upstream_addr) + .is_err_and(|e| e.kind() == std::io::ErrorKind::PermissionDenied) + { + return; + } + let upstream_task = tokio::spawn(async move { + loop { + let Ok((stream, _)) = upstream_listener.accept().await else { + break; + }; + tokio::spawn(async move { + let service = service_fn(|req: Request| async move { + assert_eq!(req.uri().path(), "/v1/chat/completions"); + assert_eq!( + req.headers() + .get("authorization") + .and_then(|v| v.to_str().ok()), + Some("Bearer real-upstream-secret") + ); + let body = req.into_body().collect().await.unwrap().to_bytes(); + let json_body: Value = serde_json::from_slice(&body).unwrap(); + let tools = json_body["tools"].as_array().unwrap(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0]["function"]["name"], "Bash"); + + let sse = concat!( + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":25,\"prompt_cache_hit_tokens\":60,\"completion_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":125}}\n\n", + "data: [DONE]\n\n" + ); + Ok::<_, Infallible>( + Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/event-stream") + .body(Full::new(Bytes::from(sse))) + .unwrap(), + ) + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + + let tmp = tempfile::tempdir().unwrap(); + let agent_dir = tmp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let checkpoint_path = agent_dir.join("maka.provider-usage.json"); + let checkpoint_secret = "0123456789abcdef0123456789abcdef"; + + let gateway = EvalMeteringGateway::start(GatewayConfig { + upstream_base_url: format!("http://{upstream_addr}/v1"), + upstream_api_key: "real-upstream-secret".into(), + profile: "maka".into(), + gateway_credential: "attempt-gateway-token".into(), + checkpoint_path: Some(checkpoint_path), + checkpoint_secret: checkpoint_secret.into(), + }) + .await + .unwrap(); + + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + + // 1. Unauthorized request is rejected with 401 and not counted. + let unauth = client + .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) + .header("authorization", "Bearer wrong-token") + .header("content-type", "application/json") + .body(serde_json::to_vec(&json!({"model": "deepseek-chat"})).unwrap()) + .send() + .await + .unwrap(); + assert_eq!(unauth.status(), reqwest::StatusCode::UNAUTHORIZED); + + // 2. Authorized request strips web tools, forwards upstream, and records SSE usage. + let resp = client + .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) + .header("authorization", "Bearer attempt-gateway-token") + .header("content-type", "application/json") + .body( + serde_json::to_vec(&json!({ + "model": "deepseek-chat", + "tools": [ + {"type": "function", "function": {"name": "WebSearch"}}, + {"type": "function", "function": {"name": "Bash"}} + ] + })) + .unwrap(), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), reqwest::StatusCode::OK); + let text = resp.text().await.unwrap(); + assert!(text.contains("[DONE]")); + + // 3. Stop accepting and verify subsequent requests get 503 without inflating counts. + gateway.stop_accepting(); + let refused = client + .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) + .header("authorization", "Bearer attempt-gateway-token") + .header("content-type", "application/json") + .body(serde_json::to_vec(&json!({"model": "deepseek-chat"})).unwrap()) + .send() + .await + .unwrap(); + assert_eq!(refused.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); + + let counts = gateway.close().await; + upstream_task.abort(); + + assert!(counts.settled); + assert_eq!(counts.requests, 1); + assert_eq!(counts.admitted_requests, 1); + assert_eq!(counts.usage_requests, 1); + assert_eq!(counts.removed_web_tools, 1); + assert_eq!(counts.models, vec!["deepseek-chat".to_owned()]); + assert_eq!(counts.tool_names, vec!["Bash".to_owned()]); + assert_eq!( + counts.usage, + Some(NormalizedUsage { + input_tokens: 100, + output_tokens: 25, + cache_read_tokens: 60, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 125, + }) + ); + + // Verify the HMAC-signed checkpoint on disk is recoverable. + let recovered = + recover_external_metering_checkpoint(tmp.path(), "maka", checkpoint_secret).unwrap(); + assert_eq!(recovered.counts, counts); + } +} diff --git a/products/eval/src/report.rs b/products/eval/src/report.rs new file mode 100644 index 00000000000..1a8aba41b9f --- /dev/null +++ b/products/eval/src/report.rs @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Deterministic journal projection (`v1` and `v2`), paired cohort comparison +//! summaries, and 3-verdict differential oracle evaluation. + +use crate::contract::{ + DifferenceRegistry, DifferenceScenarioId, EvalResultStatus, ExperimentSpec, RunFingerprint, + V1CellAttempt, V2AttemptRecord, decode_v1_cell_attempt, expand_experiment, format_cell_id, + is_replaceable_attempt, select_cell_result, +}; +use crate::measurement::{NormalizedUsage, costs_match_within_tolerance}; +use crate::store::{FileAttemptStore, ReconstructedStoreState, StoreError}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; + +/// Reconstructs all `v1` and `v2` attempt records deterministically from the append-only journal. +pub fn rebuild_from_journal( + store: &FileAttemptStore, +) -> Result { + store.reconstruct_state() +} + +/// Exports the canonical `maka.eval.v1` attempt for each cell in `spec` using +/// earliest-valid-attempt selection (`sequence >= 1`). +pub fn export_v1_canonical( + spec: &ExperimentSpec, + store: &FileAttemptStore, +) -> Result, StoreError> { + let cells = expand_experiment(spec); + let mut out = BTreeMap::new(); + for cell in cells { + let attempts = store.list_v1(&cell.id)?; + if let Some(selected) = select_cell_result(&attempts) { + out.insert(cell.id, selected.clone()); + } + } + Ok(out) +} + +/// Exports the canonical `maka.eval.v2` record for each cell in `spec` from the journal. +pub fn export_v2_canonical( + spec: &ExperimentSpec, + store: &FileAttemptStore, +) -> Result, StoreError> { + let state = store.reconstruct_state()?; + let cells = expand_experiment(spec); + let mut out = BTreeMap::new(); + for cell in cells { + if let Some(records) = state.settled_v2.get(&cell.id) { + let mut sorted: Vec<&V2AttemptRecord> = + records.iter().filter(|r| r.sequence >= 1).collect(); + sorted.sort_by_key(|r| r.sequence); + for rec in sorted { + let projected = rec.project_v1(); + if !is_replaceable_attempt(&projected) { + out.insert(cell.id, rec.clone()); + break; + } + } + } + } + Ok(out) +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectArmSummary { + pub subject_id: String, + pub total_cells: usize, + pub settled_cells: usize, + pub completed_cells: usize, + pub subject_failed_cells: usize, + pub total_score: f64, + pub mean_score: Option, + pub total_usage: NormalizedUsage, + pub total_cost_usd: f64, + pub mean_duration_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CohortSummary { + pub experiment_id: String, + pub total_cells: usize, + pub settled_cells: usize, + pub incomplete_cells: usize, + pub paired_task_groups: usize, + pub arms: BTreeMap, +} + +pub fn build_cohort_summary( + spec: &ExperimentSpec, + canonical_v1: &BTreeMap, +) -> CohortSummary { + let cells = expand_experiment(spec); + let total_cells = cells.len(); + let total_cells_per_arm = spec.tasks.len() * (spec.repetitions as usize); + + let mut arms = BTreeMap::new(); + for subject in &spec.subjects { + let mut settled_cells = 0usize; + let mut completed_cells = 0usize; + let mut subject_failed_cells = 0usize; + let mut total_score = 0.0f64; + let mut scored_count = 0usize; + let mut total_usage = NormalizedUsage::default(); + let mut total_cost_usd = 0.0f64; + let mut total_duration_ms = 0u64; + + for task in &spec.tasks { + for rep in 1..=spec.repetitions { + let cell_id = format_cell_id(&task.id, rep, &subject.id); + if let Some(attempt) = canonical_v1.get(&cell_id) { + settled_cells += 1; + total_duration_ms = + total_duration_ms.saturating_add(attempt.result.duration_ms); + match attempt.result.status { + EvalResultStatus::Completed => completed_cells += 1, + EvalResultStatus::SubjectFailed => subject_failed_cells += 1, + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate => {} + } + if let Some(score) = attempt.result.score { + total_score += score; + scored_count += 1; + } + if let Some(u) = &attempt.result.usage { + total_usage.accumulate_exact(u); + } + if let Some(cost) = attempt.result.cost_usd { + total_cost_usd += cost; + } + } + } + } + + arms.insert( + subject.id.clone(), + SubjectArmSummary { + subject_id: subject.id.clone(), + total_cells: total_cells_per_arm, + settled_cells, + completed_cells, + subject_failed_cells, + total_score, + mean_score: (scored_count > 0).then(|| total_score / (scored_count as f64)), + total_usage, + total_cost_usd, + mean_duration_ms: (settled_cells > 0) + .then(|| (total_duration_ms as f64) / (settled_cells as f64)), + }, + ); + } + + let mut paired_task_groups = 0usize; + for task in &spec.tasks { + for rep in 1..=spec.repetitions { + let all_arms_settled = spec.subjects.iter().all(|s| { + let id = format_cell_id(&task.id, rep, &s.id); + canonical_v1.contains_key(&id) + }); + if all_arms_settled { + paired_task_groups += 1; + } + } + } + + let settled_cells = canonical_v1.len(); + CohortSummary { + experiment_id: spec.id.clone(), + total_cells, + settled_cells, + incomplete_cells: total_cells.saturating_sub(settled_cells), + paired_task_groups, + arms, + } +} + +/// 3-verdict differential report comparing a candidate `maka.eval.v1` projection +/// against a reference baseline oracle and the Semantic Difference Registry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DifferentialReport { + /// 1. Whether the candidate JSON parses strictly as a valid `maka.eval.v1` attempt. + pub v1_parses: bool, + /// 2. Whether the candidate matches the oracle (either byte/tolerance-exact or via an adjudicated difference scenario). + pub v1_matches_oracle: bool, + /// 3. Whether the experimental conditions (semantic fingerprint and any adjudicated difference) remain comparable. + pub conditions_comparable: bool, + pub adjudicated_scenario: Option, + pub diagnostics: Vec, +} + +pub fn evaluate_differential_oracle( + candidate_v1_json: &Value, + oracle: &V1CellAttempt, + candidate_fingerprint: &RunFingerprint, + oracle_fingerprint: &RunFingerprint, + adjudicated_scenario: Option, + registry: &DifferenceRegistry, +) -> DifferentialReport { + let mut diagnostics = Vec::new(); + let candidate = match decode_v1_cell_attempt(candidate_v1_json) { + Ok(c) => c, + Err(err) => { + diagnostics.push(format!("v1 schema validation failed: {err}")); + return DifferentialReport { + v1_parses: false, + v1_matches_oracle: false, + conditions_comparable: false, + adjudicated_scenario, + diagnostics, + }; + } + }; + + let fingerprint_comparable = + candidate_fingerprint.is_semantically_comparable(oracle_fingerprint); + if !fingerprint_comparable { + diagnostics.push("semantic run fingerprint differs from oracle".into()); + } + + let scenario_entry = adjudicated_scenario.and_then(|id| registry.get(id)); + if adjudicated_scenario.is_some() && scenario_entry.is_none() { + diagnostics.push("unregistered semantic difference scenario".into()); + } + + let direct_match = attempts_equivalent(&candidate, oracle, &mut diagnostics); + let matches_oracle = direct_match || scenario_entry.is_some(); + let conditions_comparable = fingerprint_comparable + && matches_oracle + && scenario_entry.is_none_or(|entry| entry.conditions_comparable); + + DifferentialReport { + v1_parses: true, + v1_matches_oracle: matches_oracle, + conditions_comparable, + adjudicated_scenario, + diagnostics, + } +} + +fn attempts_equivalent( + candidate: &V1CellAttempt, + oracle: &V1CellAttempt, + diagnostics: &mut Vec, +) -> bool { + let mut ok = true; + if candidate.cell_id != oracle.cell_id { + diagnostics.push(format!( + "cellId mismatch: {} != {}", + candidate.cell_id, oracle.cell_id + )); + ok = false; + } + if candidate.sequence != oracle.sequence { + diagnostics.push(format!( + "sequence mismatch: {} != {}", + candidate.sequence, oracle.sequence + )); + ok = false; + } + if candidate.result.status != oracle.result.status { + diagnostics.push(format!( + "status mismatch: {:?} != {:?}", + candidate.result.status, oracle.result.status + )); + ok = false; + } + if candidate.result.score != oracle.result.score { + diagnostics.push(format!( + "score mismatch: {:?} != {:?}", + candidate.result.score, oracle.result.score + )); + ok = false; + } + if candidate.result.usage != oracle.result.usage { + diagnostics.push("usage counters mismatch".into()); + ok = false; + } + if !costs_match_within_tolerance(candidate.result.cost_usd, oracle.result.cost_usd) { + diagnostics.push(format!( + "costUsd mismatch beyond 1e-9 tolerance: {:?} != {:?}", + candidate.result.cost_usd, oracle.result.cost_usd + )); + ok = false; + } + if candidate.result.failure_reason != oracle.result.failure_reason { + diagnostics.push(format!( + "failureReason mismatch: {:?} != {:?}", + candidate.result.failure_reason, oracle.result.failure_reason + )); + ok = false; + } + ok +} diff --git a/products/eval/src/runner.rs b/products/eval/src/runner.rs new file mode 100644 index 00000000000..965a30e371d --- /dev/null +++ b/products/eval/src/runner.rs @@ -0,0 +1,732 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Task-group admission scheduler, phase-scoped deadline authority, bounded +//! retry policy, canonical attempt selection, and crash recovery loop. + +use crate::contract::{ + AttemptPhase, CleanupState, ContractError, CriticalArtifactsState, EgressAuditState, + EvalResultStatus, ExecutionOutcome, ExperimentCell, ExperimentSpec, SchemaVersion, SubjectKind, + V1CellAttempt, V2AttemptRecord, VerifierOutcome, expand_experiment, select_cell_result, +}; +use crate::environment::{ + ExecutorPreparationCode, IndeterminateCause, TrialEnvironment, TrialEnvironmentOutcome, + harbor_trial_name_for_attempt, +}; +use crate::measurement::MeteringState; +use crate::store::{FileAttemptStore, StoreError, UnsettledAttempt}; +use crate::subject::SubjectAdapter; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use thiserror::Error; +use tokio::sync::{Mutex, Semaphore}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +#[derive(Debug, Error)] +pub enum RunnerError { + #[error("Contract error: {0}")] + Contract(#[from] ContractError), + #[error("Store error: {0}")] + Store(#[from] StoreError), + #[error("{0}")] + Execution(String), +} + +/// Retry policy governing automatic retries within a run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryPolicy { + /// Maximum automatic retries for `infra_failed` attempts within a single run (`0` = single try per invocation). + pub max_infra_retries: u32, + /// When `true` (default), an `indeterminate` outcome with unconfirmed resource cleanup + /// blocks automatic resubmission in the same run until cleanup is confirmed. + pub block_automatic_retry_on_indeterminate: bool, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_infra_retries: 0, + block_automatic_retry_on_indeterminate: true, + } + } +} + +/// Probe used during crash recovery to verify whether a tagged resource from an +/// interrupted run has been cleaned up. +pub trait ResourceCleanupProbe: Send + Sync { + fn confirm_cleaned_up(&self, resource_kind: &str, resource_id: &str) -> bool; +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct ConservativeCleanupProbe; + +impl ResourceCleanupProbe for ConservativeCleanupProbe { + fn confirm_cleaned_up(&self, _resource_kind: &str, _resource_id: &str) -> bool { + false + } +} + +/// Recovers unsettled attempts left behind by a previous crashed process before +/// admitting new work. +pub fn recover_unsettled_attempts( + store: &FileAttemptStore, + probe: &dyn ResourceCleanupProbe, + now_ms: u64, +) -> Result, RunnerError> { + let state = store.reconstruct_state()?; + let mut recovered = Vec::new(); + + for ((cell_id, sequence), unsettled) in state.unsettled { + let record = settle_interrupted_attempt(&unsettled, probe, now_ms); + store.append_v2(&record)?; + store.record_post_hoc_recovery( + &cell_id, + sequence, + now_ms, + record.usage.clone(), + record.metering_state, + matches!(record.cleanup_state, CleanupState::Confirmed), + record.failure_reason.clone(), + )?; + recovered.push(record); + } + + Ok(recovered) +} + +fn settle_interrupted_attempt( + unsettled: &UnsettledAttempt, + probe: &dyn ResourceCleanupProbe, + now_ms: u64, +) -> V2AttemptRecord { + let duration_ms = now_ms.saturating_sub(unsettled.started_at); + match (&unsettled.resource_kind, &unsettled.resource_id) { + (None, None) => { + // Crashed after LaunchIntent but before any external resource or Host operation was created. + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: unsettled.cell_id.clone(), + sequence: unsettled.sequence, + started_at: unsettled.started_at, + completed_at: now_ms, + duration_ms, + phase: AttemptPhase::Recovered, + execution_outcome: ExecutionOutcome::InfraFailed, + verifier_outcome: VerifierOutcome::Skipped, + usage: unsettled.latest_usage.clone(), + metering_state: unsettled.latest_metering_state, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some("attempt interrupted before resource creation".into()), + artifacts: Vec::new(), + } + } + (Some(kind), Some(id)) => { + let confirmed = probe.confirm_cleaned_up(kind, id); + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: unsettled.cell_id.clone(), + sequence: unsettled.sequence, + started_at: unsettled.started_at, + completed_at: now_ms, + duration_ms, + phase: AttemptPhase::Recovered, + execution_outcome: if confirmed { + ExecutionOutcome::InfraFailed + } else { + ExecutionOutcome::Indeterminate + }, + verifier_outcome: VerifierOutcome::Skipped, + usage: unsettled.latest_usage.clone(), + metering_state: unsettled.latest_metering_state, + cleanup_state: if confirmed { + CleanupState::Confirmed + } else { + CleanupState::Unconfirmed { + reason: "executor cleanup did not settle".into(), + } + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some(if confirmed { + "attempt interrupted and resource cleanup confirmed".into() + } else { + "executor cleanup did not settle".into() + }), + artifacts: Vec::new(), + } + } + _ => V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: unsettled.cell_id.clone(), + sequence: unsettled.sequence, + started_at: unsettled.started_at, + completed_at: now_ms, + duration_ms, + phase: AttemptPhase::Recovered, + execution_outcome: ExecutionOutcome::Indeterminate, + verifier_outcome: VerifierOutcome::Skipped, + usage: unsettled.latest_usage.clone(), + metering_state: unsettled.latest_metering_state, + cleanup_state: CleanupState::Unconfirmed { + reason: "executor cleanup did not settle".into(), + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some("executor cleanup did not settle".into()), + artifacts: Vec::new(), + }, + } +} + +/// Groups expanded cells by `(task.id, repetition)` in deterministic declaration order. +pub fn group_task_cells(cells: &[ExperimentCell]) -> Vec> { + let mut index_by_key: HashMap<(String, u32), usize> = HashMap::new(); + let mut groups: Vec> = Vec::new(); + for cell in cells { + let key = (cell.task.id.clone(), cell.repetition); + if let Some(&idx) = index_by_key.get(&key) { + groups[idx].push(cell.clone()); + } else { + let idx = groups.len(); + index_by_key.insert(key, idx); + groups.push(vec![cell.clone()]); + } + } + groups +} + +pub struct RunExperimentInput<'a> { + pub spec: &'a ExperimentSpec, + pub store: &'a FileAttemptStore, + pub environment: Arc, + pub subjects: Vec>, + pub cell_ids: Option>, + pub retry_policy: RetryPolicy, + pub cancel: CancellationToken, + pub now_ms: Arc u64 + Send + Sync>, +} + +impl<'a> RunExperimentInput<'a> { + pub fn new( + spec: &'a ExperimentSpec, + store: &'a FileAttemptStore, + environment: Arc, + subjects: Vec>, + ) -> Self { + Self { + spec, + store, + environment, + subjects, + cell_ids: None, + retry_policy: RetryPolicy::default(), + cancel: CancellationToken::new(), + now_ms: Arc::new(default_now_ms), + } + } +} + +pub fn default_now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +pub async fn run_experiment( + input: RunExperimentInput<'_>, +) -> Result, RunnerError> { + let _writer_guard = input.store.acquire_writer()?; + + let all_cells = expand_experiment(input.spec); + let selected_cells = select_cells(&all_cells, input.cell_ids.as_deref())?; + + if input.environment.kind() != input.spec.executor.kind { + return Err(RunnerError::Execution("executor kind mismatch".into())); + } + + let mut subjects_by_kind: HashMap> = HashMap::new(); + for adapter in &input.subjects { + subjects_by_kind.insert(adapter.kind(), adapter.clone()); + } + + let credential_names: Vec = { + let mut set = BTreeSet::new(); + for s in &input.spec.subjects { + for c in &s.credentials { + set.insert(c.clone()); + } + } + set.into_iter().collect() + }; + + for cell in &selected_cells { + input.environment.validate(cell)?; + let Some(subject) = subjects_by_kind.get(&cell.subject.kind) else { + return Err(RunnerError::Execution(format!( + "missing subject adapter: {:?}", + cell.subject.kind + ))); + }; + subject.validate(cell)?; + } + + input + .environment + .preflight(&credential_names, input.cancel.clone()) + .await?; + + let groups = group_task_cells(&selected_cells); + let max_concurrent = input.spec.execution.max_concurrent_task_groups.max(1); + let semaphore = Arc::new(Semaphore::new(max_concurrent)); + let store_lock = Arc::new(Mutex::new(())); + let first_error: Arc>> = Arc::new(Mutex::new(None)); + + let mut group_handles = Vec::with_capacity(groups.len()); + for group in groups { + let sem = semaphore.clone(); + let env = input.environment.clone(); + let subjects_map = subjects_by_kind.clone(); + let creds = credential_names.clone(); + let cancel = input.cancel.clone(); + let now_fn = input.now_ms.clone(); + let store = input.store.clone(); + let store_mutex = store_lock.clone(); + let err_slot = first_error.clone(); + let retry_policy = input.retry_policy; + + group_handles.push(tokio::spawn(async move { + let Ok(_permit) = sem.acquire_owned().await else { + return; + }; + if cancel.is_cancelled() || err_slot.lock().await.is_some() { + return; + } + + let mut arm_handles = Vec::with_capacity(group.len()); + for cell in group { + let env = env.clone(); + let subject = subjects_map.get(&cell.subject.kind).unwrap().clone(); + let creds = creds.clone(); + let cancel = cancel.clone(); + let now_fn = now_fn.clone(); + let store = store.clone(); + let store_mutex = store_mutex.clone(); + let err_slot = err_slot.clone(); + + arm_handles.push(tokio::spawn(async move { + let mut retries_used = 0u32; + loop { + if cancel.is_cancelled() { + return; + } + let (should_run, next_seq, started_at) = { + let _g = store_mutex.lock().await; + let attempts = match store.list_v1(&cell.id) { + Ok(a) => a, + Err(e) => { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + }; + if cancel.is_cancelled() { + return; + } + if select_cell_result(&attempts).is_some() { + (false, 0, 0) + } else { + let seq = attempts.last().map_or(1, |a| a.sequence + 1); + let started = now_fn(); + let intent_id = Uuid::new_v4().to_string(); + if let Err(e) = + store.record_launch_intent(&cell.id, seq, started, &intent_id) + { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + let trial_id = harbor_trial_name_for_attempt(&cell.id, seq); + let _ = store.record_resource_tagged( + &cell.id, + seq, + env.kind(), + &trial_id, + None, + None, + ); + (true, seq, started) + } + }; + + if !should_run { + return; + } + + let mut record = execute_cell_v2( + env.as_ref(), + subject.clone(), + &cell, + next_seq, + started_at, + &creds, + cancel.clone(), + now_fn.as_ref(), + ) + .await; + record.completed_at = now_fn(); + + let append_res = { + let _g = store_mutex.lock().await; + store.append_v2(&record) + }; + let v1 = match append_res { + Ok(v1) => v1, + Err(e) => { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + }; + + match v1.result.status { + EvalResultStatus::Completed | EvalResultStatus::SubjectFailed => { + return; + } + EvalResultStatus::Indeterminate + if retry_policy.block_automatic_retry_on_indeterminate => + { + return; + } + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate => { + if retries_used >= retry_policy.max_infra_retries { + return; + } + retries_used += 1; + } + } + } + })); + } + + for handle in arm_handles { + let _ = handle.await; + } + })); + } + + for handle in group_handles { + let _ = handle.await; + } + + if let Some(err) = first_error.lock().await.take() { + return Err(err); + } + + let mut canonical_results = BTreeMap::new(); + for cell in &all_cells { + let attempts = input.store.list_v1(&cell.id)?; + if let Some(selected) = select_cell_result(&attempts) { + canonical_results.insert(cell.id.clone(), selected.clone()); + } + } + + Ok(canonical_results) +} + +#[allow(clippy::too_many_arguments)] +async fn execute_cell_v2( + environment: &dyn TrialEnvironment, + subject: Arc, + cell: &ExperimentCell, + sequence: u32, + started_at: u64, + subject_credential_names: &[String], + cancel: CancellationToken, + now_ms: &(dyn Fn() -> u64 + Send + Sync), +) -> V2AttemptRecord { + let cell_id = cell.id.clone(); + let cell_for_inner = cell.clone(); + let cancel_for_inner = cancel.clone(); + + let env_res = environment + .run_attempt( + cell, + sequence, + subject_credential_names, + cancel.clone(), + Box::new(move |active_attempt| { + Box::pin(async move { + let mut ctx = active_attempt.context(); + ctx.cancel = cancel_for_inner.clone(); + let subject_exec = subject.execute(&cell_for_inner, ctx).await; + + if cancel_for_inner.is_cancelled() + || matches!( + subject_exec.outcome, + ExecutionOutcome::InfraFailed | ExecutionOutcome::Indeterminate + ) + { + let outcome = if cancel_for_inner.is_cancelled() { + ExecutionOutcome::Cancelled + } else { + subject_exec.outcome + }; + return V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: cell_for_inner.id.clone(), + sequence, + started_at, + completed_at: 0, + duration_ms: subject_exec.duration_ms, + phase: AttemptPhase::AgentFinished, + execution_outcome: outcome, + verifier_outcome: VerifierOutcome::Skipped, + usage: subject_exec.usage, + metering_state: subject_exec.metering_state, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: subject_exec.failure_reason, + artifacts: subject_exec.artifacts, + }; + } + + let verify_res = active_attempt.verify().await; + let (audit_state, audit_evidence) = active_attempt.read_audit().await; + + let mut artifacts = subject_exec.artifacts; + match verify_res { + Ok(verified) => { + let verifier_outcome = verified.to_verifier_outcome(); + artifacts.extend(verified.artifacts); + artifacts.extend(audit_evidence.artifacts); + + let failure_reason = if verified.status == EvalResultStatus::InfraFailed + || verified.missing_reward + { + verified.failure_reason.or(audit_evidence.failure_reason) + } else { + subject_exec + .failure_reason + .or(verified.failure_reason) + .or(audit_evidence.failure_reason) + }; + + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: cell_for_inner.id.clone(), + sequence, + started_at, + completed_at: 0, + duration_ms: subject_exec.duration_ms, + phase: AttemptPhase::Verified, + execution_outcome: subject_exec.outcome, + verifier_outcome, + usage: subject_exec.usage, + metering_state: subject_exec.metering_state, + cleanup_state: CleanupState::Confirmed, + audit_state, + critical_artifacts_state: CriticalArtifactsState::Present, + // Verification already completed; any late cancellation from here + // on is post-verification and must not erase the verified score! + cancelled_after_verification: true, + recovered_post_hoc: false, + failure_reason, + artifacts, + } + } + Err(_) => V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: cell_for_inner.id.clone(), + sequence, + started_at, + completed_at: 0, + duration_ms: subject_exec.duration_ms, + phase: AttemptPhase::AgentFinished, + execution_outcome: subject_exec.outcome, + verifier_outcome: VerifierOutcome::VerifierInfraFailed { + reason: "verification failed".into(), + }, + usage: subject_exec.usage, + metering_state: subject_exec.metering_state, + cleanup_state: CleanupState::Confirmed, + audit_state, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some("verification failed".into()), + artifacts, + }, + } + }) + }), + ) + .await; + + let completed_at = now_ms(); + match env_res { + Ok(TrialEnvironmentOutcome::Settled(mut rec)) => { + rec.phase = AttemptPhase::Settled; + rec.completed_at = completed_at; + rec + } + Ok(TrialEnvironmentOutcome::NotStarted { code, artifacts }) => { + let cancelled = code == ExecutorPreparationCode::Cancelled; + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at, + completed_at, + duration_ms: 0, + phase: AttemptPhase::Prepared, + execution_outcome: if cancelled { + ExecutionOutcome::Cancelled + } else { + ExecutionOutcome::InfraFailed + }, + verifier_outcome: VerifierOutcome::Skipped, + usage: None, + metering_state: MeteringState::Unknown, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some(if cancelled { + "executor preparation cancelled".into() + } else { + "executor preparation failed".into() + }), + artifacts, + } + } + Ok(TrialEnvironmentOutcome::Indeterminate { cause, partial }) => { + let reason = match cause { + IndeterminateCause::HostCancelled => { + "executor cancelled before verification completed" + } + IndeterminateCause::CleanupUnconfirmed => "executor cleanup did not settle", + }; + if let Some(mut rec) = partial { + rec.completed_at = completed_at; + if cause == IndeterminateCause::CleanupUnconfirmed { + rec.cleanup_state = CleanupState::Unconfirmed { + reason: reason.into(), + }; + } else { + rec.execution_outcome = ExecutionOutcome::Cancelled; + rec.cancelled_after_verification = false; + rec.failure_reason = Some(reason.into()); + } + rec + } else { + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at, + completed_at, + duration_ms: 0, + phase: AttemptPhase::Settled, + execution_outcome: ExecutionOutcome::Indeterminate, + verifier_outcome: VerifierOutcome::Skipped, + usage: None, + metering_state: MeteringState::Unknown, + cleanup_state: if cause == IndeterminateCause::CleanupUnconfirmed { + CleanupState::Unconfirmed { + reason: reason.into(), + } + } else { + CleanupState::Confirmed + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some(reason.into()), + artifacts: Vec::new(), + } + } + } + Err(_) => V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at, + completed_at, + duration_ms: 0, + phase: AttemptPhase::Prepared, + execution_outcome: ExecutionOutcome::InfraFailed, + verifier_outcome: VerifierOutcome::Skipped, + usage: None, + metering_state: MeteringState::Unknown, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some("executor preparation failed".into()), + artifacts: Vec::new(), + }, + } +} + +fn select_cells( + cells: &[ExperimentCell], + selected_ids: Option<&[String]>, +) -> Result, ContractError> { + let Some(ids) = selected_ids else { + return Ok(cells.to_vec()); + }; + let known: BTreeSet<&str> = cells.iter().map(|c| c.id.as_str()).collect(); + let mut filter = BTreeSet::new(); + for id in ids { + if !known.contains(id.as_str()) { + return Err(ContractError::Invalid(format!( + "unknown experiment cell: {id}" + ))); + } + filter.insert(id.as_str()); + } + Ok(cells + .iter() + .filter(|c| filter.contains(c.id.as_str())) + .cloned() + .collect()) +} diff --git a/products/eval/src/store.rs b/products/eval/src/store.rs new file mode 100644 index 00000000000..0611106a44a --- /dev/null +++ b/products/eval/src/store.rs @@ -0,0 +1,874 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Single-writer append-only experiment journal, atomic attempt settlement, +//! critical artifact verification, and deterministic state reconstruction. + +use crate::contract::{ + AttemptPhase, CleanupState, ContractError, CriticalArtifactsState, EgressAuditState, + ExecutionOutcome, ExperimentSpec, SchemaVersion, V1CellAttempt, V2AttemptRecord, + VerifierOutcome, canonical_json_string, decode_v1_cell_attempt, parse_experiment_spec, + sha256_hex, +}; +use crate::measurement::{MeteringState, NormalizedUsage}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use thiserror::Error; +use uuid::Uuid; + +pub const MAKA_RUNTIME_ARTIFACT_PATH: &str = "/logs/artifacts/maka-runtime-host"; +pub const MAKA_SUBJECT_STDOUT_PATH: &str = "/logs/artifacts/maka-subject.stdout.txt"; +pub const MAKA_SUBJECT_STDERR_PATH: &str = "/logs/artifacts/maka-subject.stderr.txt"; + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + #[error("Contract error: {0}")] + Contract(#[from] ContractError), + #[error("{0}: experiment already has an active writer")] + ActiveWriter(String), + #[error("{0}")] + Integrity(String), +} + +/// Exclusive single-writer lock guard held for the duration of a store write session. +#[derive(Debug)] +pub struct WriterGuard { + lock_path: PathBuf, + _file: File, +} + +impl WriterGuard { + pub fn acquire(dir: &Path) -> Result { + fs::create_dir_all(dir)?; + let lock_path = dir.join(".writer.lock"); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(&lock_path) { + Ok(mut file) => { + let _ = writeln!(file, "{}", std::process::id()); + let _ = file.sync_all(); + Ok(Self { + lock_path, + _file: file, + }) + } + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + Err(StoreError::ActiveWriter(lock_path.display().to_string())) + } + Err(err) => Err(StoreError::Io(err)), + } + } +} + +impl Drop for WriterGuard { + fn drop(&mut self) { + let _ = fs::remove_file(&self.lock_path); + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum JournalEntry { + LaunchIntent { + cell_id: String, + sequence: u32, + started_at: u64, + intent_id: String, + }, + ResourceTagged { + cell_id: String, + sequence: u32, + resource_kind: String, + resource_id: String, + operation_id: Option, + invocation_id: Option, + }, + Checkpoint { + cell_id: String, + sequence: u32, + recorded_at: u64, + usage: Option, + metering_state: MeteringState, + }, + Settled { + attempt: Box, + }, + PostHocRecovery { + cell_id: String, + sequence: u32, + recovered_at: u64, + usage: Option, + metering_state: MeteringState, + cleanup_confirmed: bool, + reason: Option, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct UnsettledAttempt { + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub intent_id: String, + pub resource_kind: Option, + pub resource_id: Option, + pub operation_id: Option, + pub invocation_id: Option, + pub latest_usage: Option, + pub latest_metering_state: MeteringState, +} + +#[derive(Debug, Clone, Default)] +pub struct ReconstructedStoreState { + pub settled_v2: BTreeMap>, + pub settled_v1: BTreeMap>, + pub unsettled: BTreeMap<(String, u32), UnsettledAttempt>, +} + +#[derive(Debug, Clone)] +pub struct FileAttemptStore { + root: PathBuf, + journal_path: PathBuf, +} + +impl FileAttemptStore { + pub fn new(root: impl Into) -> Self { + let root = root.into(); + let journal_path = root.join("journal.jsonl"); + Self { root, journal_path } + } + + pub fn path(&self) -> &Path { + &self.root + } + + pub fn journal_path(&self) -> &Path { + &self.journal_path + } + + pub fn acquire_writer(&self) -> Result { + WriterGuard::acquire(&self.root) + } + + pub fn run_exclusive(&self, operation: F) -> Result + where + F: FnOnce(&WriterGuard) -> Result, + { + let guard = self.acquire_writer()?; + operation(&guard) + } + + pub async fn run_exclusive_async(&self, operation: F) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + let _guard = self.acquire_writer()?; + operation().await + } + + /// Appends a single journal entry to `journal.jsonl` and fsyncs it before returning. + pub fn append_journal_entry(&self, entry: &JournalEntry) -> Result<(), StoreError> { + fs::create_dir_all(&self.root)?; + let mut options = OpenOptions::new(); + options.create(true).append(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&self.journal_path)?; + let line = serde_json::to_string(entry)?; + file.write_all(line.as_bytes())?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) + } + + /// Records a pre-dispatch `LaunchIntent` before any external resource or Host operation starts. + pub fn record_launch_intent( + &self, + cell_id: &str, + sequence: u32, + started_at: u64, + intent_id: &str, + ) -> Result<(), StoreError> { + let expected = self.next_sequence(cell_id)?; + if sequence != expected { + return Err(StoreError::Integrity(format!( + "attempt sequence must be {expected}" + ))); + } + self.append_journal_entry(&JournalEntry::LaunchIntent { + cell_id: cell_id.to_owned(), + sequence, + started_at, + intent_id: intent_id.to_owned(), + }) + } + + pub fn record_resource_tagged( + &self, + cell_id: &str, + sequence: u32, + resource_kind: &str, + resource_id: &str, + operation_id: Option, + invocation_id: Option, + ) -> Result<(), StoreError> { + self.append_journal_entry(&JournalEntry::ResourceTagged { + cell_id: cell_id.to_owned(), + sequence, + resource_kind: resource_kind.to_owned(), + resource_id: resource_id.to_owned(), + operation_id, + invocation_id, + }) + } + + pub fn record_checkpoint( + &self, + cell_id: &str, + sequence: u32, + recorded_at: u64, + usage: Option, + metering_state: MeteringState, + ) -> Result<(), StoreError> { + self.append_journal_entry(&JournalEntry::Checkpoint { + cell_id: cell_id.to_owned(), + sequence, + recorded_at, + usage, + metering_state, + }) + } + + /// Atomically commits a settled `V2AttemptRecord`, writing its `v1` projection file, + /// `v2` native file, and `Settled` journal entry. + pub fn append_v2(&self, record: &V2AttemptRecord) -> Result { + let expected = self.next_settled_sequence(&record.cell_id)?; + if record.sequence != expected { + return Err(StoreError::Integrity(format!( + "attempt sequence must be {expected}" + ))); + } + let v1 = record.project_v1(); + let _ = decode_v1_cell_attempt(&serde_json::to_value(&v1)?)?; + + let cell_dir = self.cell_directory(&record.cell_id); + fs::create_dir_all(&cell_dir)?; + + let v1_final = cell_dir.join(format!("{:06}.json", record.sequence)); + let v2_final = cell_dir.join(format!("{:06}.v2.json", record.sequence)); + + write_immutable_json(&v1_final, &v1)?; + write_immutable_json(&v2_final, record)?; + + self.append_journal_entry(&JournalEntry::Settled { + attempt: Box::new(record.clone()), + })?; + + Ok(v1) + } + + /// Appends a `maka.eval.v1` attempt directly (for legacy compatibility tests and stores) + /// while synthesizing the corresponding `v2` record in the journal. + pub fn append_v1(&self, attempt: &V1CellAttempt) -> Result<(), StoreError> { + let validated = decode_v1_cell_attempt(&serde_json::to_value(attempt)?)?; + let expected = self.next_settled_sequence(&validated.cell_id)?; + if validated.sequence != expected { + return Err(StoreError::Integrity(format!( + "attempt sequence must be {expected}" + ))); + } + let cell_dir = self.cell_directory(&validated.cell_id); + fs::create_dir_all(&cell_dir)?; + let v1_final = cell_dir.join(format!("{:06}.json", validated.sequence)); + write_immutable_json(&v1_final, &validated)?; + + let v2 = v2_from_v1(&validated); + let v2_final = cell_dir.join(format!("{:06}.v2.json", validated.sequence)); + let _ = write_immutable_json(&v2_final, &v2); + self.append_journal_entry(&JournalEntry::Settled { + attempt: Box::new(v2), + })?; + Ok(()) + } + + /// Appends a post-hoc recovery entry without mutating any existing attempt file or journal entry. + #[allow(clippy::too_many_arguments)] + pub fn record_post_hoc_recovery( + &self, + cell_id: &str, + sequence: u32, + recovered_at: u64, + usage: Option, + metering_state: MeteringState, + cleanup_confirmed: bool, + reason: Option, + ) -> Result<(), StoreError> { + self.append_journal_entry(&JournalEntry::PostHocRecovery { + cell_id: cell_id.to_owned(), + sequence, + recovered_at, + usage, + metering_state, + cleanup_confirmed, + reason, + }) + } + + /// Lists immutable `v1` attempt files for `cell_id`, verifying each file's identity against + /// its 6-digit sequence filename. + pub fn list_v1(&self, cell_id: &str) -> Result, StoreError> { + let dir = self.cell_directory(cell_id); + let entries = match fs::read_dir(&dir) { + Ok(rd) => rd, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(StoreError::Io(err)), + }; + + let mut names = Vec::new(); + for entry in entries { + let entry = entry?; + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + continue; + }; + if is_six_digit_json_filename(name) { + names.push(name.to_owned()); + } + } + names.sort(); + + let mut attempts = Vec::with_capacity(names.len()); + for name in names { + let seq: u32 = name[..6].parse().map_err(|_| { + StoreError::Integrity("attempt file does not match its immutable identity".into()) + })?; + let content = fs::read_to_string(dir.join(&name))?; + let parsed: Value = serde_json::from_str(&content)?; + let attempt = decode_v1_cell_attempt(&parsed)?; + if attempt.cell_id != cell_id || attempt.sequence != seq { + return Err(StoreError::Integrity( + "attempt file does not match its immutable identity".into(), + )); + } + attempts.push(attempt); + } + Ok(attempts) + } + + pub fn read_journal_entries(&self) -> Result, StoreError> { + let file = match File::open(&self.journal_path) { + Ok(f) => f, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(StoreError::Io(err)), + }; + let reader = BufReader::new(file); + let mut entries = Vec::new(); + for line in reader.lines() { + let line = line?; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let entry: JournalEntry = serde_json::from_str(trimmed)?; + entries.push(entry); + } + Ok(entries) + } + + /// Deterministically reconstructs all settled `v2` records, `v1` projections, and + /// unsettled in-flight attempts from the append-only journal. + pub fn reconstruct_state(&self) -> Result { + let entries = self.read_journal_entries()?; + let mut state = ReconstructedStoreState::default(); + let mut settled_map: BTreeMap<(String, u32), V2AttemptRecord> = BTreeMap::new(); + + for entry in entries { + match entry { + JournalEntry::LaunchIntent { + cell_id, + sequence, + started_at, + intent_id, + } => { + state.unsettled.insert( + (cell_id.clone(), sequence), + UnsettledAttempt { + cell_id, + sequence, + started_at, + intent_id, + resource_kind: None, + resource_id: None, + operation_id: None, + invocation_id: None, + latest_usage: None, + latest_metering_state: MeteringState::Unknown, + }, + ); + } + JournalEntry::ResourceTagged { + cell_id, + sequence, + resource_kind, + resource_id, + operation_id, + invocation_id, + } => { + if let Some(unsettled) = state.unsettled.get_mut(&(cell_id, sequence)) { + unsettled.resource_kind = Some(resource_kind); + unsettled.resource_id = Some(resource_id); + unsettled.operation_id = operation_id; + unsettled.invocation_id = invocation_id; + } + } + JournalEntry::Checkpoint { + cell_id, + sequence, + usage, + metering_state, + .. + } => { + if let Some(unsettled) = state.unsettled.get_mut(&(cell_id, sequence)) { + if usage.is_some() { + unsettled.latest_usage = usage; + } + unsettled.latest_metering_state = metering_state; + } + } + JournalEntry::Settled { attempt } => { + let key = (attempt.cell_id.clone(), attempt.sequence); + state.unsettled.remove(&key); + settled_map.insert(key, *attempt); + } + JournalEntry::PostHocRecovery { + cell_id, + sequence, + recovered_at, + usage, + metering_state, + cleanup_confirmed, + reason, + } => { + let key = (cell_id.clone(), sequence); + if let Some(existing) = settled_map.get_mut(&key) { + existing.recovered_post_hoc = true; + existing.phase = AttemptPhase::Recovered; + if usage.is_some() { + existing.usage = usage; + existing.metering_state = metering_state; + } + if cleanup_confirmed { + existing.cleanup_state = CleanupState::Confirmed; + } + if let Some(r) = reason { + existing.failure_reason = Some(r); + } + } else if let Some(unsettled) = state.unsettled.remove(&key) { + let recovered_usage = usage.or(unsettled.latest_usage); + let record = + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at: unsettled.started_at, + completed_at: recovered_at, + duration_ms: recovered_at.saturating_sub(unsettled.started_at), + phase: AttemptPhase::Recovered, + execution_outcome: if cleanup_confirmed { + ExecutionOutcome::InfraFailed + } else { + ExecutionOutcome::Indeterminate + }, + verifier_outcome: VerifierOutcome::Skipped, + usage: recovered_usage, + metering_state, + cleanup_state: if cleanup_confirmed { + CleanupState::Confirmed + } else { + CleanupState::Unconfirmed { + reason: reason.clone().unwrap_or_else(|| { + "executor cleanup did not settle".into() + }), + } + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some(reason.unwrap_or_else(|| { + "recovered after interrupted attempt".into() + })), + artifacts: Vec::new(), + }; + settled_map.insert(key, record); + } + } + } + } + + for ((cell_id, _seq), record) in settled_map { + let v1 = record.project_v1(); + state + .settled_v2 + .entry(cell_id.clone()) + .or_default() + .push(record); + state.settled_v1.entry(cell_id).or_default().push(v1); + } + + Ok(state) + } + + pub fn next_settled_sequence(&self, cell_id: &str) -> Result { + let v1_list = self.list_v1(cell_id)?; + Ok(v1_list.last().map_or(1, |a| a.sequence + 1)) + } + + pub fn next_sequence(&self, cell_id: &str) -> Result { + let settled_next = self.next_settled_sequence(cell_id)?; + let reconstructed = self.reconstruct_state()?; + let max_journal_seq = reconstructed + .settled_v2 + .get(cell_id) + .and_then(|v| v.last().map(|r| r.sequence)) + .into_iter() + .chain( + reconstructed + .unsettled + .keys() + .filter(|(c, _)| c == cell_id) + .map(|(_, s)| *s), + ) + .max() + .map_or(1, |s| s + 1); + Ok(settled_next.max(max_journal_seq)) + } + + fn cell_directory(&self, cell_id: &str) -> PathBuf { + self.root.join(sha256_hex(cell_id.as_bytes())) + } +} + +#[derive(Debug, Clone)] +pub struct ExperimentDirectory { + pub root: PathBuf, + pub spec_path: PathBuf, + pub attempts: FileAttemptStore, +} + +impl ExperimentDirectory { + pub fn open(root: impl Into, spec: &ExperimentSpec) -> Result { + let root = root.into(); + fs::create_dir_all(&root)?; + let spec_path = root.join("experiment.json"); + let spec_val = serde_json::to_value(spec)?; + let canonical = format!("{}\n", canonical_json_string(&spec_val)); + let temp_path = root.join(format!(".experiment-{}.tmp", Uuid::new_v4())); + + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut temp_file = options.open(&temp_path)?; + let link_res = (|| -> Result<(), std::io::Error> { + temp_file.write_all(canonical.as_bytes())?; + temp_file.sync_all()?; + drop(temp_file); + fs::hard_link(&temp_path, &spec_path) + })(); + let _ = fs::remove_file(&temp_path); + + match link_res { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + let existing_raw = fs::read_to_string(&spec_path)?; + let existing_val: Value = serde_json::from_str(&existing_raw)?; + let existing_spec = parse_experiment_spec(&existing_val)?; + let existing_canonical = + canonical_json_string(&serde_json::to_value(&existing_spec)?); + if existing_canonical != canonical_json_string(&spec_val) { + return Err(StoreError::Integrity("experiment spec differs".into())); + } + } + Err(err) => return Err(StoreError::Io(err)), + } + + Ok(Self { + spec_path, + attempts: FileAttemptStore::new(root.join("attempts")), + root, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CapturedArtifactFile { + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaRuntimeArtifactManifest { + pub schema_version: String, + pub reason: String, + pub captured_at: u64, + pub files: Vec, +} + +/// Verifies whether every required critical artifact path exists as a readable regular file. +/// +/// Explicitly distinguishes a `0-byte` regular file (`Present`, valid empty file) from a +/// missing, symlink, directory, or unreadable path (`Missing`). +pub fn verify_critical_artifacts( + base_dir: &Path, + required_paths: &[&str], +) -> (CriticalArtifactsState, Vec) { + let mut missing = Vec::new(); + let mut captured = Vec::new(); + + for rel in required_paths { + let full = base_dir.join(rel); + match describe_regular_file(&full, rel) { + Some(desc) => captured.push(desc), + None => missing.push((*rel).to_owned()), + } + } + + if missing.is_empty() { + (CriticalArtifactsState::Present, captured) + } else { + (CriticalArtifactsState::Missing { paths: missing }, captured) + } +} + +pub fn describe_regular_file(path: &Path, reported_name: &str) -> Option { + let meta = fs::symlink_metadata(path).ok()?; + if !meta.file_type().is_file() { + return None; + } + let bytes = fs::read(path).ok()?; + Some(CapturedArtifactFile { + path: reported_name.to_owned(), + bytes: bytes.len() as u64, + sha256: format!("sha256:{}", sha256_hex(&bytes)), + }) +} + +/// Captures Maka runtime artifacts (`runtime.sqlite`, logs, and `manifest.json`) into +/// `destination_root` using atomic directory staging. +pub fn capture_maka_runtime_artifacts( + state_root: &Path, + destination_root: &Path, + reason: &str, + captured_at: u64, +) -> Result { + let staging_root = + destination_root.with_extension(format!("tmp.{}.{}", std::process::id(), Uuid::new_v4())); + let _ = fs::remove_dir_all(&staging_root); + fs::create_dir_all(&staging_root)?; + + let res = (|| -> Result { + let sqlite_src = state_root.join("runtime.sqlite"); + let sqlite_dst = staging_root.join("runtime.sqlite"); + if sqlite_src.is_file() { + let src_conn = rusqlite::Connection::open_with_flags( + &sqlite_src, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| StoreError::Integrity(format!("sqlite open failed: {e}")))?; + let dst_str = sqlite_dst + .to_str() + .ok_or_else(|| StoreError::Integrity("invalid sqlite destination path".into()))?; + src_conn + .execute("VACUUM INTO ?1", [dst_str]) + .map_err(|e| StoreError::Integrity(format!("sqlite backup failed: {e}")))?; + } + + for candidate in [ + "runtime-host-candidate.log", + "runtime-policy.json", + "collection-error.json", + ] { + let src = state_root.join(candidate); + if src.is_file() { + fs::copy(&src, staging_root.join(candidate))?; + } + } + + let mut files = Vec::new(); + for name in [ + "runtime.sqlite", + "runtime-host-candidate.log", + "runtime-policy.json", + "collection-error.json", + ] { + if let Some(desc) = describe_regular_file(&staging_root.join(name), name) { + files.push(desc); + } + } + + let manifest = MakaRuntimeArtifactManifest { + schema_version: "maka.eval.runtime_artifacts.v1".into(), + reason: reason.to_owned(), + captured_at, + files, + }; + let manifest_bytes = format!("{}\n", serde_json::to_string_pretty(&manifest)?); + fs::write(staging_root.join("manifest.json"), manifest_bytes)?; + + if let Some(parent) = destination_root.parent() { + fs::create_dir_all(parent)?; + } + let _ = fs::remove_dir_all(destination_root); + fs::rename(&staging_root, destination_root)?; + Ok(manifest) + })(); + + if res.is_err() { + let _ = fs::remove_dir_all(&staging_root); + } + res +} + +pub fn write_atomic_bytes(final_path: &Path, bytes: &[u8]) -> Result<(), StoreError> { + if let Some(parent) = final_path.parent() { + fs::create_dir_all(parent)?; + } + let temp_path = + final_path.with_extension(format!("tmp.{}.{}", std::process::id(), Uuid::new_v4())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut temp_file = options.open(&temp_path)?; + let res = (|| -> Result<(), StoreError> { + temp_file.write_all(bytes)?; + temp_file.sync_all()?; + drop(temp_file); + fs::rename(&temp_path, final_path)?; + Ok(()) + })(); + if res.is_err() { + let _ = fs::remove_file(&temp_path); + } + res +} + +fn write_immutable_json(final_path: &Path, value: &T) -> Result<(), StoreError> { + let temp_path = final_path.with_extension(format!("json.{}.tmp", Uuid::new_v4())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut temp_file = options.open(&temp_path)?; + let res = (|| -> Result<(), StoreError> { + let serialized = format!("{}\n", serde_json::to_string(value)?); + temp_file.write_all(serialized.as_bytes())?; + temp_file.sync_all()?; + drop(temp_file); + fs::hard_link(&temp_path, final_path)?; + Ok(()) + })(); + let _ = fs::remove_file(&temp_path); + res +} + +fn is_six_digit_json_filename(name: &str) -> bool { + name.len() == 11 && name.ends_with(".json") && name[..6].bytes().all(|b| b.is_ascii_digit()) +} + +fn v2_from_v1(v1: &V1CellAttempt) -> V2AttemptRecord { + use crate::contract::EvalResultStatus; + let execution_outcome = match v1.result.status { + EvalResultStatus::Completed => ExecutionOutcome::Completed, + EvalResultStatus::SubjectFailed => ExecutionOutcome::SubjectFailed, + EvalResultStatus::InfraFailed => ExecutionOutcome::InfraFailed, + EvalResultStatus::Indeterminate => ExecutionOutcome::Indeterminate, + }; + let verifier_outcome = match (v1.result.status, v1.result.score) { + (EvalResultStatus::Completed, Some(score)) => VerifierOutcome::Verified { score }, + (EvalResultStatus::SubjectFailed, score) => VerifierOutcome::SubjectFailed { + score, + reason: v1 + .result + .failure_reason + .clone() + .unwrap_or_else(|| "subject failed".into()), + }, + _ => VerifierOutcome::Skipped, + }; + let metering_state = match (&v1.result.usage, v1.result.cost_usd) { + (Some(_), Some(_)) => MeteringState::Complete, + (Some(_), None) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + let mut clean_artifacts: Vec> = Vec::new(); + for a in &v1.result.artifacts { + clean_artifacts.push(a.clone()); + } + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: v1.cell_id.clone(), + sequence: v1.sequence, + started_at: v1.started_at, + completed_at: v1.completed_at, + duration_ms: v1.result.duration_ms, + phase: AttemptPhase::Settled, + execution_outcome, + verifier_outcome, + usage: v1.result.usage.clone(), + metering_state, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: v1.result.failure_reason.clone(), + artifacts: clean_artifacts, + } +} diff --git a/products/eval/src/subject.rs b/products/eval/src/subject.rs new file mode 100644 index 00000000000..8918da8b898 --- /dev/null +++ b/products/eval/src/subject.rs @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Subject adapter trait and implementations for the Rust Maka Host and external CLI harnesses. + +pub mod external_cli; +pub mod maka; + +pub use external_cli::ExternalCliSubject; +pub use maka::RustHostSubject; + +use crate::contract::{ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind}; +use crate::environment::{BoxFuture, SubjectExecutionContext}; +use crate::measurement::{MeteringState, NormalizedUsage}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectExecutionResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + pub usage: Option, + pub metering_state: MeteringState, + pub cost_usd: Option, + pub duration_ms: u64, + pub outcome: ExecutionOutcome, + pub failure_reason: Option, + pub artifacts: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub operation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub invocation_id: Option, +} + +pub trait SubjectAdapter: Send + Sync { + fn kind(&self) -> SubjectKind; + + fn validate(&self, _cell: &ExperimentCell) -> Result<(), ContractError> { + Ok(()) + } + + fn execute<'a>( + &'a self, + cell: &'a ExperimentCell, + context: SubjectExecutionContext, + ) -> BoxFuture<'a, SubjectExecutionResult>; +} diff --git a/products/eval/src/subject/external_cli.rs b/products/eval/src/subject/external_cli.rs new file mode 100644 index 00000000000..e89c7a69a37 --- /dev/null +++ b/products/eval/src/subject/external_cli.rs @@ -0,0 +1,662 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! External CLI subject adapter (`exit-code` and `protocol-v1` bounded result frames, +//! placeholder expansion, and checkpoint metering recovery). + +use crate::contract::{ + ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind, exact_object, + nonempty_string, nullable_nonnegative_f64, sha256_hex, +}; +use crate::environment::{ + BoxFuture, SubjectExecutionContext, SubjectProcessDiagnostic, SubjectProcessDiagnosticCategory, + SubjectProcessRequest, SubjectProcessTermination, +}; +use crate::measurement::{ + MeteringState, NormalizedUsage, RecoveredExternalMetering, recover_external_metering_checkpoint, +}; +use crate::subject::{SubjectAdapter, SubjectExecutionResult}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::HashMap; +use std::path::Path; +use std::time::Instant; + +pub const RESULT_FRAME_PREFIX: &str = "MAKA-EVAL-RESULT-V1"; +pub const RESULT_PAYLOAD_LIMIT_BYTES: usize = 2 * 1024; +pub const RESULT_CARRIER_LIMIT_BYTES: usize = 64 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ExternalResultMode { + #[serde(rename = "protocol-v1")] + ProtocolV1, + #[serde(rename = "exit-code")] + ExitCode, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExternalSubjectConfig { + pub command: String, + pub args: Vec, + pub environment: HashMap, + pub credential_environment: HashMap, + pub result_mode: ExternalResultMode, +} + +impl ExternalSubjectConfig { + pub fn decode( + config: &JsonObject, + declared_credentials: &[String], + ) -> Result { + let val = Value::Object(config.clone()); + let obj = exact_object( + &val, + "external subject config", + &["command", "args"], + &["environment", "credentialEnvironment", "result"], + )?; + let command = nonempty_string(obj.get("command").unwrap(), "external subject command")?; + let args_arr = obj + .get("args") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("external subject args are invalid".into()))?; + let mut args = Vec::with_capacity(args_arr.len()); + for item in args_arr { + let Some(s) = item.as_str() else { + return Err(ContractError::Invalid( + "external subject args are invalid".into(), + )); + }; + args.push(s.to_owned()); + } + + let environment = match obj.get("environment") { + None => HashMap::new(), + Some(env_val) => decode_env_map(env_val, "environment")?, + }; + + let credential_environment = match obj.get("credentialEnvironment") { + None => declared_credentials + .iter() + .map(|c| (c.clone(), c.clone())) + .collect(), + Some(cred_val) => decode_env_map(cred_val, "credentialEnvironment")?, + }; + for (target, source) in &credential_environment { + if !declared_credentials.iter().any(|c| c == source) { + return Err(ContractError::Invalid(format!( + "credentialEnvironment.{target} must reference a declared credential" + ))); + } + if environment.contains_key(target) { + return Err(ContractError::Invalid(format!( + "environment and credentialEnvironment overlap at {target}" + ))); + } + } + + let result_mode = match obj.get("result").and_then(Value::as_str) { + None | Some("protocol-v1") => ExternalResultMode::ProtocolV1, + Some("exit-code") => ExternalResultMode::ExitCode, + Some(_) => { + return Err(ContractError::Invalid( + "external subject result is invalid".into(), + )); + } + }; + + Ok(Self { + command, + args, + environment, + credential_environment, + result_mode, + }) + } +} + +/// Encodes a bounded `MAKA-EVAL-RESULT-V1` frame matching `relay-result-frame.ts`. +pub fn encode_relay_result_frame(token: &str, value: &Value) -> Result { + if token.len() != 32 + || !token + .bytes() + .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + { + return Err(ContractError::Invalid( + "relay result token is invalid".into(), + )); + } + let payload = serde_json::to_vec(value).map_err(|e| ContractError::Invalid(e.to_string()))?; + if payload.len() > RESULT_PAYLOAD_LIMIT_BYTES { + return Err(ContractError::Invalid( + "relay result exceeds the payload limit".into(), + )); + } + let digest = sha256_hex(&payload); + let encoded = URL_SAFE_NO_PAD.encode(&payload); + Ok(format!( + "{RESULT_FRAME_PREFIX} {token} {} {digest} {encoded}\n", + payload.len() + )) +} + +/// Parses and validates a `MAKA-EVAL-RESULT-V1` frame from raw process stdout. +pub fn parse_relay_result_frame( + carrier: &str, + expected_token: &str, +) -> Result { + if carrier.len() > RESULT_CARRIER_LIMIT_BYTES { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameOversize, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + let matching_lines: Vec<&str> = carrier + .lines() + .filter(|line| { + line.starts_with(RESULT_FRAME_PREFIX) + && line + .split_whitespace() + .nth(1) + .is_some_and(|tok| tok == expected_token) + }) + .collect(); + + if matching_lines.is_empty() { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameMissing, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + if matching_lines.len() > 1 { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameAmbiguous, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + + let parts: Vec<&str> = matching_lines[0].split_whitespace().collect(); + if parts.len() != 5 { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + let Ok(expected_len) = parts[2].parse::() else { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: None, + sha256: None, + }); + }; + if expected_len > RESULT_PAYLOAD_LIMIT_BYTES { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameOversize, + bytes: Some(expected_len as u64), + sha256: None, + }); + } + let expected_digest = parts[3]; + let Ok(decoded_bytes) = URL_SAFE_NO_PAD.decode(parts[4]) else { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: None, + sha256: None, + }); + }; + if decoded_bytes.len() != expected_len || sha256_hex(&decoded_bytes) != expected_digest { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: Some(decoded_bytes.len() as u64), + sha256: Some(sha256_hex(&decoded_bytes)), + }); + } + String::from_utf8(decoded_bytes).map_err(|_| SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: None, + sha256: None, + }) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct DecodedExternalResult { + pub output: Option, + pub usage: Option, + pub cost_usd: Option, + pub status: Option, + pub failure_reason: Option, + pub artifacts: Vec, +} + +pub fn decode_external_result_json(stdout: &str) -> Result { + let parsed: Value = serde_json::from_str(stdout) + .map_err(|_| ContractError::Invalid("external result must be valid JSON".into()))?; + let obj = exact_object( + &parsed, + "external result", + &["schemaVersion", "usage", "costUsd", "artifacts"], + &["output", "status", "failureReason"], + )?; + let schema_version = obj.get("schemaVersion").and_then(Value::as_str); + if !matches!( + schema_version, + Some("maka.external_subject_result.v1" | "maka.external_subject_result.v2") + ) { + return Err(ContractError::Invalid( + "external result schema is invalid".into(), + )); + } + if schema_version == Some("maka.external_subject_result.v2") + && (!obj.contains_key("status") || !obj.contains_key("failureReason")) + { + return Err(ContractError::Invalid( + "external result v2 fields are invalid".into(), + )); + } + let output = match obj.get("output") { + None => None, + Some(Value::String(s)) => Some(s.clone()), + Some(_) => { + return Err(ContractError::Invalid( + "external result output is invalid".into(), + )); + } + }; + let status = match obj.get("status") { + None => None, + Some(Value::String(s)) + if matches!( + s.as_str(), + "completed" | "failed" | "infra_failed" | "indeterminate" + ) => + { + Some(s.clone()) + } + Some(_) => { + return Err(ContractError::Invalid( + "external result status is invalid".into(), + )); + } + }; + let failure_reason = match obj.get("failureReason") { + None | Some(Value::Null) => None, + Some(Value::String(s)) => Some(s.clone()), + Some(_) => { + return Err(ContractError::Invalid( + "external result failureReason is invalid".into(), + )); + } + }; + let usage = match obj.get("usage").unwrap() { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, "external usage")?), + }; + let cost_usd = nullable_nonnegative_f64(obj.get("costUsd").unwrap(), "external result cost")?; + let artifacts_arr = obj + .get("artifacts") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("external result artifacts are invalid".into()))?; + let mut artifacts = Vec::with_capacity(artifacts_arr.len()); + for item in artifacts_arr { + let Some(map) = item.as_object() else { + return Err(ContractError::Invalid( + "external result artifacts are invalid".into(), + )); + }; + artifacts.push(map.clone()); + } + + Ok(DecodedExternalResult { + output, + usage, + cost_usd, + status, + failure_reason, + artifacts, + }) +} + +#[derive(Debug, Default, Clone)] +pub struct ExternalCliSubject; + +impl ExternalCliSubject { + pub fn new() -> Self { + Self + } +} + +impl SubjectAdapter for ExternalCliSubject { + fn kind(&self) -> SubjectKind { + SubjectKind::External + } + + fn validate(&self, cell: &ExperimentCell) -> Result<(), ContractError> { + let _ = ExternalSubjectConfig::decode(&cell.subject.config, &cell.subject.credentials)?; + Ok(()) + } + + fn execute<'a>( + &'a self, + cell: &'a ExperimentCell, + context: SubjectExecutionContext, + ) -> BoxFuture<'a, SubjectExecutionResult> { + Box::pin(async move { + let started = Instant::now(); + let Ok(config) = + ExternalSubjectConfig::decode(&cell.subject.config, &cell.subject.credentials) + else { + return SubjectExecutionResult { + output: None, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms: 0, + outcome: ExecutionOutcome::InfraFailed, + failure_reason: Some("external subject config invalid".into()), + artifacts: Vec::new(), + operation_id: None, + invocation_id: None, + }; + }; + + let profile = Some(cell.subject.id.as_str()); + let args: Vec = config + .args + .iter() + .map(|arg| { + arg.replace("{{task.input}}", &context.task_input) + .replace("{{task.id}}", &cell.task.id) + .replace("{{task.cwd}}", &context.cwd) + }) + .collect(); + + let execution = match context + .execute(SubjectProcessRequest { + command: config.command, + args, + environment: config.environment, + credential_environment: config.credential_environment, + capture_stdout: config.result_mode != ExternalResultMode::ExitCode, + }) + .await + { + Ok(exec) => exec, + Err(_) => { + let recovered = recover_from_metadata(&context.metadata, profile); + let cancelled = context.cancel.is_cancelled(); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms: started.elapsed().as_millis() as u64, + outcome: if cancelled { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::InfraFailed + }, + failure_reason: Some(if cancelled { + "external subject cancelled".into() + } else { + "external subject execution failed".into() + }), + artifacts: recovered.into_iter().map(|r| r.artifact).collect(), + operation_id: None, + invocation_id: None, + }; + } + }; + + let duration_ms = started.elapsed().as_millis() as u64; + let decoded = if execution.stdout.is_empty() { + None + } else { + decode_external_result_json(&execution.stdout).ok() + }; + let recovered = recover_from_metadata(&context.metadata, profile); + + let mut process_art = Map::new(); + process_art.insert("kind".into(), Value::String("external_process".into())); + process_art.insert("exitCode".into(), Value::from(execution.exit_code)); + + if execution.termination == SubjectProcessTermination::FrameworkTimeout { + let usage = decoded + .as_ref() + .and_then(|d| d.usage.clone()) + .or_else(|| recovered.as_ref().and_then(|r| r.usage.clone())); + let cost_usd = decoded + .as_ref() + .and_then(|d| d.cost_usd) + .or_else(|| recovered.as_ref().and_then(|r| r.cost_usd)); + let metering_state = match (&usage, cost_usd) { + (Some(_), Some(_)) => MeteringState::Complete, + (Some(_), None) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + let mut artifacts = match &decoded { + Some(d) => d.artifacts.clone(), + None => recovered.iter().map(|r| r.artifact.clone()).collect(), + }; + if decoded.as_ref().is_some_and(|d| d.usage.is_none()) + && let Some(rec) = &recovered + { + artifacts.push(rec.artifact.clone()); + } + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage, + metering_state, + cost_usd, + duration_ms, + outcome: ExecutionOutcome::Timeout, + failure_reason: Some("external subject exceeded the framework timeout".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + } + + if execution.diagnostic.as_ref().map(|d| d.category) + == Some(SubjectProcessDiagnosticCategory::ExecutionScopeUnavailable) + { + let mut artifacts: Vec = + recovered.iter().map(|r| r.artifact.clone()).collect(); + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms, + outcome: if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::InfraFailed + }, + failure_reason: Some("external subject execution scope was unavailable".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + } + + if config.result_mode == ExternalResultMode::ExitCode { + let outcome = if execution.exit_code == 0 { + ExecutionOutcome::Completed + } else if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::SubjectFailed + }; + return SubjectExecutionResult { + output: None, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms, + outcome, + failure_reason: (execution.exit_code != 0) + .then(|| format!("external subject exited {}", execution.exit_code)), + artifacts: vec![process_art], + operation_id: None, + invocation_id: None, + }; + } + + if execution + .diagnostic + .as_ref() + .is_some_and(|d| d.category.is_result_frame_error()) + { + let outcome = if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else if recovered.as_ref().is_some_and(|r| r.admitted_requests > 0) { + ExecutionOutcome::SubjectFailed + } else { + ExecutionOutcome::InfraFailed + }; + let mut artifacts: Vec = + recovered.iter().map(|r| r.artifact.clone()).collect(); + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms, + outcome, + failure_reason: Some("external subject result transport failed".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + } + + let Some(decoded) = decoded else { + let outcome = if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else if recovered.as_ref().is_some_and(|r| r.admitted_requests > 0) { + ExecutionOutcome::SubjectFailed + } else { + ExecutionOutcome::InfraFailed + }; + let mut artifacts: Vec = + recovered.iter().map(|r| r.artifact.clone()).collect(); + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms, + outcome, + failure_reason: Some("external subject returned no result".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + }; + + let outcome = match decoded.status.as_deref() { + Some("completed") => ExecutionOutcome::Completed, + Some("failed") => ExecutionOutcome::SubjectFailed, + Some("infra_failed") => ExecutionOutcome::InfraFailed, + Some("indeterminate") => ExecutionOutcome::Indeterminate, + None if execution.exit_code == 0 => ExecutionOutcome::Completed, + None => ExecutionOutcome::SubjectFailed, + Some(_) => ExecutionOutcome::InfraFailed, + }; + let metering_state = match (&decoded.usage, decoded.cost_usd) { + (Some(_), Some(_)) => MeteringState::Complete, + (Some(_), None) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + let failure_reason = decoded.failure_reason.or_else(|| { + (execution.exit_code != 0 && outcome != ExecutionOutcome::Completed) + .then(|| format!("external subject exited {}", execution.exit_code)) + }); + + SubjectExecutionResult { + output: decoded.output, + usage: decoded.usage, + metering_state, + cost_usd: decoded.cost_usd, + duration_ms, + outcome, + failure_reason, + artifacts: decoded.artifacts, + operation_id: None, + invocation_id: None, + } + }) + } +} + +fn recover_from_metadata( + metadata: &JsonObject, + profile: Option<&str>, +) -> Option { + let profile = profile?; + let trial_path = metadata.get("trialPath")?.as_str()?; + let secret = metadata.get("meteringSecret")?.as_str()?; + recover_external_metering_checkpoint(Path::new(trial_path), profile, secret) +} + +fn decode_env_map( + value: &Value, + where_label: &str, +) -> Result, ContractError> { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + let mut map = HashMap::with_capacity(obj.len()); + for (name, item) in obj { + let mut chars = name.chars(); + let valid_name = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + let Some(val_str) = item.as_str() else { + return Err(ContractError::Invalid(format!( + "{where_label}.{name} is invalid" + ))); + }; + if !valid_name { + return Err(ContractError::Invalid(format!( + "{where_label}.{name} is invalid" + ))); + } + map.insert(name.clone(), val_str.to_owned()); + } + Ok(map) +} diff --git a/products/eval/src/subject/maka.rs b/products/eval/src/subject/maka.rs new file mode 100644 index 00000000000..cd1217805a6 --- /dev/null +++ b/products/eval/src/subject/maka.rs @@ -0,0 +1,936 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Rust Maka Host subject adapter (`maka host init --settings-stdin --root `, +//! durable operation/invocation receipt tracking, disconnect re-observation, +//! paginated transcript export, and usage settlement reconciliation). + +use crate::contract::{ + ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind, exact_object, + nonempty_string, positive_u32, +}; +use crate::environment::{ + BoxFuture, SubjectExecutionContext, SubjectProcessOutput, SubjectProcessRequest, + SubjectProcessTermination, +}; +use crate::measurement::{ + MeteringState, NormalizedUsage, ProviderMeteringCounts, deepseek_cost_usd, + reconcile_host_and_proxy_usage, +}; +use crate::store::{ + MAKA_RUNTIME_ARTIFACT_PATH, MAKA_SUBJECT_STDERR_PATH, MAKA_SUBJECT_STDOUT_PATH, +}; +use crate::subject::{SubjectAdapter, SubjectExecutionResult}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; +use std::process::Stdio; +use std::time::Instant; +use tokio::io::AsyncWriteExt; +use tokio::process::Command; +use uuid::Uuid; + +pub const DEFAULT_EVAL_GATEWAY_SLUG: &str = "eval-gateway"; +pub const DEFAULT_EVAL_GATEWAY_NAME: &str = "Eval Gateway"; +pub const DEFAULT_EVAL_GATEWAY_PROVIDER: &str = "openai-compatible"; +pub const DEFAULT_EVAL_CONTEXT_WINDOW: u64 = 200_000; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaGatewayConfiguration { + pub base_url: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaModelOverride { + pub context_window: u64, + pub thinking_levels: Vec, +} + +impl Default for MakaModelOverride { + fn default() -> Self { + Self { + context_window: DEFAULT_EVAL_CONTEXT_WINDOW, + thinking_levels: vec!["off".into()], + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaConnectionInit { + pub slug: String, + pub name: String, + pub provider: String, + pub configuration: MakaGatewayConfiguration, + pub api_key: String, + pub enabled: bool, + pub enabled_model_ids: Vec, + pub model_overrides: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaDefaultTargetInit { + pub slug: String, + pub model_id: String, +} + +/// Exact stdin JSON payload accepted by `maka host init --settings-stdin --root `. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaHostInitSettingsPayload { + pub connections: Vec, + pub default_target: MakaDefaultTargetInit, +} + +impl MakaHostInitSettingsPayload { + /// Constructs the canonical `eval-gateway` settings payload for a local metering gateway + /// port, credential, and model ID. + pub fn for_eval_gateway( + port: u16, + gateway_credential: impl Into, + model_id: impl Into, + ) -> Self { + Self::for_eval_gateway_base_url( + format!("http://127.0.0.1:{port}/v1"), + gateway_credential, + model_id, + ) + } + + pub fn for_eval_gateway_base_url( + base_url: impl Into, + gateway_credential: impl Into, + model_id: impl Into, + ) -> Self { + let model_id = model_id.into(); + let mut model_overrides = BTreeMap::new(); + model_overrides.insert(model_id.clone(), MakaModelOverride::default()); + + Self { + connections: vec![MakaConnectionInit { + slug: DEFAULT_EVAL_GATEWAY_SLUG.into(), + name: DEFAULT_EVAL_GATEWAY_NAME.into(), + provider: DEFAULT_EVAL_GATEWAY_PROVIDER.into(), + configuration: MakaGatewayConfiguration { + base_url: base_url.into(), + }, + api_key: gateway_credential.into(), + enabled: true, + enabled_model_ids: vec![model_id.clone()], + model_overrides, + }], + default_target: MakaDefaultTargetInit { + slug: DEFAULT_EVAL_GATEWAY_SLUG.into(), + model_id, + }, + } + } + + pub fn to_json_value(&self) -> Value { + serde_json::to_value(self).expect("MakaHostInitSettingsPayload serialization cannot fail") + } +} + +/// Bootstraps an isolated Maka Host Root by invoking: +/// `maka host init --settings-stdin --root ` +/// and piping `payload` as JSON over `stdin` so credentials never appear in argv or env. +pub async fn bootstrap_maka_host_root( + maka_executable: &Path, + root_dir: &Path, + payload: &MakaHostInitSettingsPayload, +) -> Result<(), String> { + let mut child = Command::new(maka_executable) + .arg("host") + .arg("init") + .arg("--settings-stdin") + .arg("--root") + .arg(root_dir) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| format!("failed to spawn maka host init: {err}"))?; + + let json_bytes = serde_json::to_vec(payload) + .map_err(|err| format!("failed to serialize init settings: {err}"))?; + + if let Some(mut stdin) = child.stdin.take() { + stdin + .write_all(&json_bytes) + .await + .map_err(|err| format!("failed writing settings to maka host init stdin: {err}"))?; + stdin + .shutdown() + .await + .map_err(|err| format!("failed closing maka host init stdin: {err}"))?; + } + + let output = child + .wait_with_output() + .await + .map_err(|err| format!("failed waiting for maka host init: {err}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + return Err(format!( + "maka host init exited with {}: {}", + output.status, stderr + )); + } + Ok(()) +} + +/// Durable receipt identifying an admitted Host operation and Agent invocation across reconnects. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct HostExecutionReceipt { + pub execution_id: String, + pub operation_id: String, + pub invocation_id: String, +} + +/// Paginated transcript history exporter that drains all pages until `next_cursor` is `None`. +pub async fn export_paginated_transcript( + page_limit: usize, + mut fetch_page: F, +) -> Result, String> +where + F: FnMut(Option, usize) -> Fut, + Fut: std::future::Future, Option), String>>, +{ + let limit = page_limit.max(1); + let mut cursor: Option = None; + let mut items = Vec::new(); + let mut visited_cursors = std::collections::BTreeSet::new(); + + loop { + let (page, next) = fetch_page(cursor.clone(), limit).await?; + items.extend(page); + match next { + None => break, + Some(next_cursor) => { + if !visited_cursors.insert(next_cursor.clone()) { + return Err(format!( + "transcript pagination cycle detected at cursor {next_cursor}" + )); + } + cursor = Some(next_cursor); + } + } + } + Ok(items) +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum HostedExecutionProjection { + Settled { + #[serde(rename = "executionId")] + execution_id: String, + status: String, + #[serde(rename = "failureReason")] + failure_reason: Option, + usage: NormalizedUsage, + #[serde(rename = "costUsd")] + cost_usd: Option, + #[serde(rename = "operationId", default)] + operation_id: Option, + #[serde(rename = "invocationId", default)] + invocation_id: Option, + }, + Indeterminate { + #[serde(rename = "executionId")] + execution_id: String, + #[serde(rename = "failureReason")] + failure_reason: String, + #[serde(rename = "operationId", default)] + operation_id: Option, + #[serde(rename = "invocationId", default)] + invocation_id: Option, + }, +} + +impl HostedExecutionProjection { + pub fn execution_id(&self) -> &str { + match self { + Self::Settled { execution_id, .. } | Self::Indeterminate { execution_id, .. } => { + execution_id + } + } + } + + pub fn operation_id(&self) -> Option<&str> { + match self { + Self::Settled { operation_id, .. } | Self::Indeterminate { operation_id, .. } => { + operation_id.as_deref() + } + } + } + + pub fn invocation_id(&self) -> Option<&str> { + match self { + Self::Settled { invocation_id, .. } | Self::Indeterminate { invocation_id, .. } => { + invocation_id.as_deref() + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MakaSubjectConfig { + pub maka_executable: Option, + pub node_path: String, + pub shim_path: String, + pub runtime_hosts_path: String, + pub host_settlement_timeout_ms: u32, + pub base_url: String, + pub connection_slug: String, + pub model: String, + pub provider_type: Option, + pub default_api_protocol: Option, + pub api_key_environment: Option, + pub thinking_level: Option, + pub permission_mode: String, + pub collaboration_mode: String, + pub orchestration_mode: String, + pub tool_profile: String, +} + +impl MakaSubjectConfig { + pub fn decode( + config: &JsonObject, + declared_credentials: &[String], + ) -> Result { + let val = Value::Object(config.clone()); + let mut required = vec![ + "runtimeHostsPath", + "baseUrl", + "connectionSlug", + "model", + "permissionMode", + "collaborationMode", + "orchestrationMode", + "hostSettlementTimeoutMs", + "toolProfile", + ]; + if config.contains_key("makaExecutable") { + required.push("makaExecutable"); + if config.contains_key("nodePath") { + required.push("nodePath"); + } + if config.contains_key("shimPath") { + required.push("shimPath"); + } + } else { + required.push("nodePath"); + required.push("shimPath"); + } + if config.contains_key("providerType") { + required.push("providerType"); + required.push("apiKeyEnvironment"); + } + if config.get("providerType").and_then(Value::as_str) == Some("custom") { + required.push("defaultApiProtocol"); + } + if config.contains_key("thinkingLevel") { + required.push("thinkingLevel"); + } + + let obj = exact_object(&val, "Maka config", &required, &[])?; + for field in &required { + if *field == "hostSettlementTimeoutMs" { + continue; + } + let _ = nonempty_string(obj.get(*field).unwrap(), &format!("Maka config.{field}"))?; + } + + let base_url = nonempty_string(obj.get("baseUrl").unwrap(), "Maka config.baseUrl")?; + if url::Url::parse(&base_url).is_err() { + return Err(ContractError::Invalid("Maka baseUrl is invalid".into())); + } + + let host_settlement_timeout_ms = positive_u32( + obj.get("hostSettlementTimeoutMs").unwrap(), + "Maka config.hostSettlementTimeoutMs", + )?; + + let api_key_environment = obj + .get("apiKeyEnvironment") + .and_then(Value::as_str) + .map(str::to_owned); + if let Some(env_name) = &api_key_environment + && !declared_credentials.iter().any(|c| c == env_name) + { + return Err(ContractError::Invalid( + "Maka apiKeyEnvironment must name a declared subject credential".into(), + )); + } + + let thinking_level = obj + .get("thinkingLevel") + .and_then(Value::as_str) + .map(str::to_owned); + if let Some(level) = &thinking_level + && !matches!( + level.as_str(), + "off" | "minimal" | "low" | "medium" | "high" | "max" + ) + { + return Err(ContractError::Invalid( + "Maka config.thinkingLevel is invalid".into(), + )); + } + + let tool_profile = + nonempty_string(obj.get("toolProfile").unwrap(), "Maka config.toolProfile")?; + if !matches!( + tool_profile.as_str(), + "default" | "headless-coding-v1" | "minimal" + ) { + return Err(ContractError::Invalid( + "Maka config.toolProfile is invalid".into(), + )); + } + + let maka_executable = obj + .get("makaExecutable") + .and_then(Value::as_str) + .map(str::to_owned); + let node_path = obj + .get("nodePath") + .and_then(Value::as_str) + .unwrap_or("node") + .to_owned(); + let shim_path = obj + .get("shimPath") + .and_then(Value::as_str) + .unwrap_or("") + .to_owned(); + + Ok(Self { + maka_executable, + node_path, + shim_path, + runtime_hosts_path: nonempty_string( + obj.get("runtimeHostsPath").unwrap(), + "Maka config.runtimeHostsPath", + )?, + host_settlement_timeout_ms, + base_url, + connection_slug: nonempty_string( + obj.get("connectionSlug").unwrap(), + "Maka config.connectionSlug", + )?, + model: nonempty_string(obj.get("model").unwrap(), "Maka config.model")?, + provider_type: obj + .get("providerType") + .and_then(Value::as_str) + .map(str::to_owned), + default_api_protocol: obj + .get("defaultApiProtocol") + .and_then(Value::as_str) + .map(str::to_owned), + api_key_environment, + thinking_level, + permission_mode: nonempty_string( + obj.get("permissionMode").unwrap(), + "Maka config.permissionMode", + )?, + collaboration_mode: nonempty_string( + obj.get("collaborationMode").unwrap(), + "Maka config.collaborationMode", + )?, + orchestration_mode: nonempty_string( + obj.get("orchestrationMode").unwrap(), + "Maka config.orchestrationMode", + )?, + tool_profile, + }) + } +} + +#[derive(Debug, Default, Clone)] +pub struct RustHostSubject; + +impl RustHostSubject { + pub fn new() -> Self { + Self + } +} + +impl SubjectAdapter for RustHostSubject { + fn kind(&self) -> SubjectKind { + SubjectKind::Maka + } + + fn validate(&self, cell: &ExperimentCell) -> Result<(), ContractError> { + let _ = MakaSubjectConfig::decode(&cell.subject.config, &cell.subject.credentials)?; + let max_steps = cell + .budget + .get("maxSteps") + .ok_or_else(|| ContractError::Invalid("budget.maxSteps is invalid".into()))?; + let _ = positive_u32(max_steps, "budget.maxSteps")?; + Ok(()) + } + + fn execute<'a>( + &'a self, + cell: &'a ExperimentCell, + context: SubjectExecutionContext, + ) -> BoxFuture<'a, SubjectExecutionResult> { + Box::pin(async move { + let started = Instant::now(); + let Ok(config) = + MakaSubjectConfig::decode(&cell.subject.config, &cell.subject.credentials) + else { + return subject_failure("config-decode", 0, context.cancel.is_cancelled(), None); + }; + let Ok(max_steps) = cell + .budget + .get("maxSteps") + .ok_or_else(|| ContractError::Invalid("budget.maxSteps is invalid".into())) + .and_then(|v| positive_u32(v, "budget.maxSteps")) + else { + return subject_failure("budget-decode", 0, context.cancel.is_cancelled(), None); + }; + + let execution_id = Uuid::new_v4().to_string(); + let root_path = format!("{}/{execution_id}", config.runtime_hosts_path); + + let gateway_settings = MakaHostInitSettingsPayload::for_eval_gateway_base_url( + &config.base_url, + config + .api_key_environment + .as_deref() + .unwrap_or("EVAL_GATEWAY_API_KEY"), + &config.model, + ); + + let mut session_obj = serde_json::json!({ + "workspace": { "kind": "host_path", "path": context.cwd }, + "modelTarget": { + "kind": "explicit", + "connectionSlug": config.connection_slug, + "model": config.model, + }, + "permissionMode": config.permission_mode, + "collaborationMode": config.collaboration_mode, + "orchestrationMode": config.orchestration_mode, + "toolProfile": config.tool_profile, + }); + if let Some(level) = &config.thinking_level { + session_obj + .as_object_mut() + .unwrap() + .insert("thinkingLevel".into(), Value::String(level.clone())); + } + + let mut envelope = serde_json::json!({ + "rootPath": root_path, + "artifactRoot": MAKA_RUNTIME_ARTIFACT_PATH, + "baseUrl": config.base_url, + "hostSettlementTimeoutMs": config.host_settlement_timeout_ms, + "hostInitSettings": gateway_settings, + "execution": { + "executionId": execution_id, + "session": session_obj, + "content": { "text": context.task_input }, + "maxSteps": max_steps, + } + }); + if let Some(provider_type) = &config.provider_type { + let mut conn = serde_json::json!({ + "providerType": provider_type, + "apiKeyEnvironment": config.api_key_environment, + }); + if let Some(proto) = &config.default_api_protocol { + conn.as_object_mut() + .unwrap() + .insert("defaultApiProtocol".into(), Value::String(proto.clone())); + } + envelope + .as_object_mut() + .unwrap() + .insert("connection".into(), conn); + } + + let encoded_payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&envelope).unwrap()); + let credential_environment: HashMap = cell + .subject + .credentials + .iter() + .map(|c| (c.clone(), c.clone())) + .collect(); + + let process = match context + .execute(SubjectProcessRequest { + command: config.node_path.clone(), + args: vec![config.shim_path.clone(), encoded_payload], + environment: HashMap::new(), + credential_environment, + capture_stdout: true, + }) + .await + { + Ok(p) => p, + Err(_) => { + return subject_failure( + "relay-execute", + started.elapsed().as_millis() as u64, + context.cancel.is_cancelled(), + None, + ); + } + }; + + let duration_ms = started.elapsed().as_millis() as u64; + + if process.termination != SubjectProcessTermination::Exited { + let projection = try_decode_matching_projection(&process.stdout, &execution_id); + let (usage, metering_state, cost_usd, subject_reported_cost, op_id, inv_id) = + match projection { + Some(HostedExecutionProjection::Settled { + usage, + cost_usd, + operation_id, + invocation_id, + .. + }) => ( + Some(usage.clone()), + MeteringState::Complete, + estimate_deepseek_cost(&usage, &config.model), + cost_usd, + operation_id, + invocation_id, + ), + _ => (None, MeteringState::Unknown, None, None, None, None), + }; + return SubjectExecutionResult { + output: None, + usage, + metering_state, + cost_usd, + duration_ms, + outcome: ExecutionOutcome::Timeout, + failure_reason: Some("Maka subject exceeded the framework timeout".into()), + artifacts: maka_artifacts( + Some(&execution_id), + Some(&process), + subject_reported_cost, + ), + operation_id: op_id, + invocation_id: inv_id, + }; + } + + if process.stdout.is_empty() { + return subject_failure( + "empty-output", + duration_ms, + context.cancel.is_cancelled(), + Some(&process), + ); + } + + let Ok(parsed) = serde_json::from_str::(&process.stdout) else { + return subject_failure( + "json-parse", + duration_ms, + context.cancel.is_cancelled(), + Some(&process), + ); + }; + + let Ok(projection) = serde_json::from_value::(parsed) else { + return subject_failure( + "result-decode", + duration_ms, + context.cancel.is_cancelled(), + Some(&process), + ); + }; + + if projection.execution_id() != execution_id { + return subject_failure( + "identity-check", + duration_ms, + context.cancel.is_cancelled(), + Some(&process), + ); + } + + let proxy_counts = context + .metadata + .get("proxyMeteringCounts") + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + project_into_execution_result( + projection, + None, + proxy_counts.as_ref(), + &config, + &execution_id, + duration_ms, + Some(&process), + ) + }) + } +} + +fn project_into_execution_result( + projection: HostedExecutionProjection, + output: Option, + proxy_counts: Option<&ProviderMeteringCounts>, + config: &MakaSubjectConfig, + execution_id: &str, + duration_ms: u64, + process: Option<&SubjectProcessOutput>, +) -> SubjectExecutionResult { + match projection { + HostedExecutionProjection::Indeterminate { + failure_reason, + operation_id, + invocation_id, + .. + } => SubjectExecutionResult { + output, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms, + outcome: ExecutionOutcome::Indeterminate, + failure_reason: Some(safe_maka_failure_reason( + &failure_reason, + "Maka execution did not settle", + )), + artifacts: maka_artifacts(Some(execution_id), process, None), + operation_id, + invocation_id, + }, + HostedExecutionProjection::Settled { + status, + failure_reason, + usage, + cost_usd: subject_cost, + operation_id, + invocation_id, + .. + } => { + let (reconciled_usage, metering_state, _) = + reconcile_host_and_proxy_usage(Some(&usage), true, proxy_counts); + let final_usage = reconciled_usage.unwrap_or(usage); + let cost_usd = if metering_state == MeteringState::Complete { + estimate_deepseek_cost(&final_usage, &config.model) + } else { + None + }; + + if status == "cancelled" { + return SubjectExecutionResult { + output, + usage: Some(final_usage), + metering_state, + cost_usd, + duration_ms, + outcome: ExecutionOutcome::Indeterminate, + failure_reason: Some("Maka subject cancelled".into()), + artifacts: maka_artifacts(Some(execution_id), process, subject_cost), + operation_id, + invocation_id, + }; + } + + let outcome = if status == "completed" { + ExecutionOutcome::Completed + } else { + ExecutionOutcome::SubjectFailed + }; + let reason = if status == "completed" { + None + } else { + failure_reason + .as_deref() + .map(|r| safe_maka_failure_reason(r, "Maka execution failed")) + }; + + SubjectExecutionResult { + output, + usage: Some(final_usage), + metering_state, + cost_usd, + duration_ms, + outcome, + failure_reason: reason, + artifacts: maka_artifacts(Some(execution_id), process, subject_cost), + operation_id, + invocation_id, + } + } + } +} + +fn try_decode_matching_projection( + stdout: &str, + expected_execution_id: &str, +) -> Option { + if stdout.is_empty() { + return None; + } + let proj: HostedExecutionProjection = serde_json::from_str(stdout).ok()?; + (proj.execution_id() == expected_execution_id).then_some(proj) +} + +fn estimate_deepseek_cost(usage: &NormalizedUsage, model: &str) -> Option { + (model == "deepseek-v4-flash").then(|| deepseek_cost_usd(usage)) +} + +pub fn safe_maka_failure_reason(value: &str, fallback: &str) -> String { + const SAFE_REASONS: &[&str] = &[ + "Hosted execution is not active", + "Hosted execution was cancelled", + "Hosted execution was cancelled before admission", + "Runtime Host connection failed before execution settlement", + "Runtime Host could not settle execution", + "Runtime Host did not exit cleanly", + ]; + if SAFE_REASONS.contains(&value) { + return value.to_owned(); + } + if let Some(rest) = value.strip_prefix("Runtime Host did not start: ") + && matches!( + rest, + "composition_mismatch" + | "existing_host" + | "host_unresponsive" + | "incompatible" + | "startup_timeout" + | "upgrade_required" + ) + { + return value.to_owned(); + } + if let Some(rest) = value.strip_prefix("Runtime Host usage did not settle: ") + && matches!( + rest, + "missing_attempt_usage" + | "partial_attempt_usage" + | "pending_usage_repair" + | "unreadable_usage_record" + ) + { + return value.to_owned(); + } + fallback.to_owned() +} + +pub fn maka_artifacts( + execution_id: Option<&str>, + process: Option<&SubjectProcessOutput>, + subject_reported_cost_usd: Option, +) -> Vec { + let mut out = Vec::new(); + if let Some(cost) = subject_reported_cost_usd { + let mut obj = Map::new(); + obj.insert("kind".into(), Value::String("subject-reported-cost".into())); + obj.insert("costUsd".into(), Value::from(cost)); + out.push(obj); + } + let mut runtime_state = Map::new(); + runtime_state.insert("kind".into(), Value::String("maka-runtime-state".into())); + runtime_state.insert( + "path".into(), + Value::String(MAKA_RUNTIME_ARTIFACT_PATH.into()), + ); + if let Some(id) = execution_id { + runtime_state.insert("executionId".into(), Value::String(id.into())); + } + out.push(runtime_state); + + let mut stdout_art = Map::new(); + stdout_art.insert("kind".into(), Value::String("subject-stdout".into())); + stdout_art.insert( + "path".into(), + Value::String(MAKA_SUBJECT_STDOUT_PATH.into()), + ); + if let Some(proc) = process { + stdout_art.insert("bytes".into(), Value::from(proc.stdout.len() as u64)); + } + out.push(stdout_art); + + let mut stderr_art = Map::new(); + stderr_art.insert("kind".into(), Value::String("subject-stderr".into())); + stderr_art.insert( + "path".into(), + Value::String(MAKA_SUBJECT_STDERR_PATH.into()), + ); + out.push(stderr_art); + out +} + +fn subject_failure( + stage: &str, + duration_ms: u64, + cancelled: bool, + process: Option<&SubjectProcessOutput>, +) -> SubjectExecutionResult { + let mut artifacts = maka_artifacts(None, process, None); + let mut failure_art = Map::new(); + failure_art.insert("kind".into(), Value::String("subject-failure".into())); + failure_art.insert("stage".into(), Value::String(stage.into())); + if let Some(proc) = process { + failure_art.insert( + "termination".into(), + serde_json::to_value(proc.termination).unwrap_or(Value::Null), + ); + failure_art.insert("exitCode".into(), Value::from(proc.exit_code)); + failure_art.insert("stdoutBytes".into(), Value::from(proc.stdout.len() as u64)); + if let Some(diag) = &proc.diagnostic { + failure_art.insert( + "diagnostic".into(), + serde_json::to_value(diag).unwrap_or(Value::Null), + ); + } + } + artifacts.push(failure_art); + + SubjectExecutionResult { + output: None, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms, + outcome: if cancelled { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::InfraFailed + }, + failure_reason: Some(if cancelled { + "Maka subject cancelled".into() + } else { + format!("Maka subject failed during {stage}") + }), + artifacts, + operation_id: None, + invocation_id: None, + } +} diff --git a/products/eval/tests/integration.rs b/products/eval/tests/integration.rs new file mode 100644 index 00000000000..d4b486268e6 --- /dev/null +++ b/products/eval/tests/integration.rs @@ -0,0 +1,1213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use maka_eval::contract::{ + AttemptPhase, CleanupState, CriticalArtifactsState, DifferenceRegistry, DifferenceScenarioId, + EgressAuditState, EvalResult, EvalResultStatus, ExecutionOutcome, ExperimentCell, + RunFingerprint, SchemaVersion, V1CellAttempt, V2AttemptRecord, VerifierOutcome, + decode_v1_cell_attempt, expand_experiment, parse_experiment_spec, select_cell_result, +}; +use maka_eval::environment::{ + ActiveTrialAttempt, BoxFuture, CommandRunner, EgressAuditEvidence, ExecutorPreparationCode, + ExecutorVerification, HarborExecutorConfig, HarborTrialEnvironment, PreparationCacheLocks, + SubjectExecutionContext, SubjectProcessDiagnostic, SubjectProcessDiagnosticCategory, + SubjectProcessOutput, SubjectProcessRequest, SubjectProcessTermination, TrialAttemptOperation, + TrialEnvironment, TrialEnvironmentOutcome, describe_egress_audit, read_trial_verification, +}; +use maka_eval::measurement::{ + AdmissionBoundaryLedger, DEEPSEEK_V4_FLASH_COST, MeteringState, NormalizedUsage, + costs_match_within_tolerance, deepseek_cost_usd, derive_metering, + reconcile_host_and_proxy_usage, recover_external_metering_checkpoint, sign_metering_checkpoint, +}; +use maka_eval::report::{ + build_cohort_summary, evaluate_differential_oracle, export_v1_canonical, export_v2_canonical, + rebuild_from_journal, +}; +use maka_eval::runner::{ + ConservativeCleanupProbe, RetryPolicy, RunExperimentInput, recover_unsettled_attempts, + run_experiment, +}; +use maka_eval::store::{ + ExperimentDirectory, capture_maka_runtime_artifacts, verify_critical_artifacts, +}; +use maka_eval::subject::SubjectAdapter; +use maka_eval::subject::external_cli::{ + ExternalCliSubject, encode_relay_result_frame, parse_relay_result_frame, +}; +use maka_eval::subject::maka::{ + HostedExecutionProjection, MakaHostInitSettingsPayload, RustHostSubject, + bootstrap_maka_host_root, export_paginated_transcript, +}; +use serde_json::{Map, Value, json}; +use std::fs; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tempfile::TempDir; +use tokio::sync::Barrier; +use tokio_util::sync::CancellationToken; + +#[test] +fn maka_host_init_settings_stdin_payload_matches_exact_session_contract() { + let payload = MakaHostInitSettingsPayload::for_eval_gateway( + 43120, + "gw-secret-credential-99", + "deepseek-v4-flash", + ); + let actual = payload.to_json_value(); + let expected = json!({ + "connections": [ + { + "slug": "eval-gateway", + "name": "Eval Gateway", + "provider": "openai-compatible", + "configuration": { + "baseUrl": "http://127.0.0.1:43120/v1" + }, + "apiKey": "gw-secret-credential-99", + "enabled": true, + "enabledModelIds": ["deepseek-v4-flash"], + "modelOverrides": { + "deepseek-v4-flash": { + "contextWindow": 200000, + "thinkingLevels": ["off"] + } + } + } + ], + "defaultTarget": { + "slug": "eval-gateway", + "modelId": "deepseek-v4-flash" + } + }); + assert_eq!(actual, expected); +} + +#[cfg(unix)] +#[tokio::test] +async fn bootstrap_maka_host_root_pipes_settings_stdin_without_leaking_secret_in_argv() { + use std::os::unix::fs::PermissionsExt; + + let temp = TempDir::new().unwrap(); + let fake_maka = temp.path().join("fake-maka.sh"); + let captured_args = temp.path().join("captured-args.txt"); + let captured_stdin = temp.path().join("captured-stdin.json"); + let host_root = temp.path().join("isolated-host-root"); + + let script = format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"{}\"\ncat > \"{}\"\n", + captured_args.display(), + captured_stdin.display() + ); + fs::write(&fake_maka, script).unwrap(); + let mut perms = fs::metadata(&fake_maka).unwrap().permissions(); + perms.set_mode(0o755); + fs::set_permissions(&fake_maka, perms).unwrap(); + + let payload = MakaHostInitSettingsPayload::for_eval_gateway( + 18080, + "super-secret-gateway-token", + "deepseek-v4-flash", + ); + bootstrap_maka_host_root(&fake_maka, &host_root, &payload) + .await + .unwrap(); + + let args_text = fs::read_to_string(&captured_args).unwrap(); + let args_lines: Vec<&str> = args_text.lines().collect(); + assert_eq!( + args_lines, + vec![ + "host", + "init", + "--settings-stdin", + "--root", + host_root.to_str().unwrap() + ] + ); + assert!(!args_text.contains("super-secret-gateway-token")); + + let stdin_json: Value = + serde_json::from_str(&fs::read_to_string(&captured_stdin).unwrap()).unwrap(); + assert_eq!(stdin_json, payload.to_json_value()); +} + +#[test] +fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { + let spec_json = sample_two_arm_spec_json(); + let spec = parse_experiment_spec(&spec_json).unwrap(); + let cells = expand_experiment(&spec); + + assert_eq!(cells.len(), 4); + assert_eq!(cells[0].id, "task-alpha::1::maka"); + assert_eq!(cells[1].id, "task-alpha::1::external-cli"); + assert_eq!(cells[2].id, "task-beta::1::maka"); + assert_eq!(cells[3].id, "task-beta::1::external-cli"); + + // Unknown top-level field must be rejected. + let mut invalid = spec_json.clone(); + invalid + .as_object_mut() + .unwrap() + .insert("unknownField".into(), json!(123)); + assert!(parse_experiment_spec(&invalid).is_err()); + + // Concurrency changes execution fingerprint but preserves semantic fingerprint. + let mut higher_concurrency_json = spec_json; + higher_concurrency_json["execution"]["maxConcurrentTaskGroups"] = json!(16); + let higher_spec = parse_experiment_spec(&higher_concurrency_json).unwrap(); + + let fp1 = RunFingerprint::compute(&spec); + let fp2 = RunFingerprint::compute(&higher_spec); + assert!(fp1.is_semantically_comparable(&fp2)); + assert_ne!(fp1.execution_sha256, fp2.execution_sha256); +} + +#[test] +fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within_1e9() { + // Provider reports a totalTokens value distinct from the arithmetic sum of the 5 fields. + let u1 = NormalizedUsage { + input_tokens: 1000, + output_tokens: 250, + cache_read_tokens: 400, + cache_write_tokens: 100, + reasoning_tokens: 80, + total_tokens: 9999, + }; + let u2 = NormalizedUsage { + input_tokens: 500, + output_tokens: 150, + cache_read_tokens: 200, + cache_write_tokens: 0, + reasoning_tokens: 20, + total_tokens: 7777, + }; + let mut combined = u1.clone(); + combined.accumulate_exact(&u2); + assert_eq!(combined.input_tokens, 1500); + assert_eq!(combined.output_tokens, 400); + assert_eq!(combined.cache_read_tokens, 600); + assert_eq!(combined.cache_write_tokens, 100); + assert_eq!(combined.reasoning_tokens, 100); + // Must equal 9999 + 7777 = 17776, never the sum of the other 5 counters! + assert_eq!(combined.total_tokens, 17776); + + // Verify DEEPSEEK_V4_FLASH_COST formula: + // uncached = 1000 - 400 - 100 = 500 + // (500 * 0.22 + 400 * 0.007 + 100 * 0.22 + 250 * 0.66) / 1_000_000 + // = (110 + 2.8 + 22 + 165) / 1_000_000 = 299.8 / 1_000_000 = 0.0002998 + let cost = deepseek_cost_usd(&u1); + assert_eq!(DEEPSEEK_V4_FLASH_COST.input_per_million, 0.22); + assert_eq!(DEEPSEEK_V4_FLASH_COST.output_per_million, 0.66); + assert_eq!(DEEPSEEK_V4_FLASH_COST.cache_read_per_million, 0.007); + assert_eq!(DEEPSEEK_V4_FLASH_COST.cache_write_per_million, 0.22); + assert!(costs_match_within_tolerance(Some(cost), Some(0.0002998))); + assert!(!costs_match_within_tolerance( + Some(cost), + Some(0.0002998 + 1e-8) + )); +} + +#[test] +fn admission_cutoff_checkpoint_deduplication_and_receipt_proxy_reconciliation() { + let mut ledger = AdmissionBoundaryLedger::new(); + assert!(ledger.record_request_start("req-1")); + assert!(ledger.record_request_start("req-2")); + + ledger.close_admission(); + assert!(ledger.is_admission_closed()); + // Request arriving after cutoff is rejected. + assert!(!ledger.record_request_start("req-after-cutoff")); + + let u1 = NormalizedUsage { + input_tokens: 100, + output_tokens: 50, + cache_read_tokens: 20, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 150, + }; + // Both Host receipt and proxy log settle req-1; idempotent settlement must not double-count. + ledger + .settle_request("req-1", Some(u1.clone()), Some("deepseek-v4-flash".into())) + .unwrap(); + ledger + .settle_request("req-1", Some(u1.clone()), Some("deepseek-v4-flash".into())) + .unwrap(); + + // While req-2 is still in flight, snapshot is lower-bound (`settled == false`). + let mid_snapshot = ledger.snapshot(); + let mid_derived = derive_metering(&mid_snapshot); + assert!(!mid_derived.usage_complete); + assert_eq!(mid_derived.metering_state, MeteringState::LowerBound); + assert_eq!(mid_derived.cost_usd, None); + + // Duplicate checkpoint ingestion is deduplicated. + assert!(ledger.ingest_checkpoint(mid_snapshot.clone())); + assert!(!ledger.ingest_checkpoint(mid_snapshot)); + + // Once req-2 drains and proxy settles, metering becomes Complete with costUsd populated. + ledger + .settle_request("req-2", Some(u1.clone()), Some("deepseek-v4-flash".into())) + .unwrap(); + ledger.mark_proxy_settled(); + let final_snapshot = ledger.snapshot(); + let final_derived = derive_metering(&final_snapshot); + assert!(final_derived.usage_complete); + assert_eq!(final_derived.metering_state, MeteringState::Complete); + assert_eq!(final_snapshot.usage.as_ref().unwrap().total_tokens, 300); + assert!(final_derived.cost_usd.is_some()); + + // Reconciling matching Host and proxy usage yields Complete without double-counting. + let (reconciled, state, cost) = + reconcile_host_and_proxy_usage(final_snapshot.usage.as_ref(), true, Some(&final_snapshot)); + assert_eq!(state, MeteringState::Complete); + assert_eq!(reconciled.unwrap().total_tokens, 300); + assert!(cost.is_some()); +} + +#[test] +fn all_five_semantic_difference_scenarios_and_three_verdict_differential_oracle() { + let registry = DifferenceRegistry::new(); + assert_eq!(registry.entries().len(), 5); + + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let fp = RunFingerprint::compute(&spec); + + let sample_usage = NormalizedUsage { + input_tokens: 200, + output_tokens: 100, + cache_read_tokens: 50, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 300, + }; + + // 1. Timeout with verifier reward (1.0) but incomplete (lower-bound) usage: + // Status is `subject_failed`, score is preserved (`Some(1.0)`), usage is preserved, + // and `costUsd` is `None` because usage is lower-bound. + let v2_timeout_reward = V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: "task-alpha::1::maka".into(), + sequence: 1, + started_at: 100, + completed_at: 200, + duration_ms: 100, + phase: AttemptPhase::Settled, + execution_outcome: ExecutionOutcome::Timeout, + verifier_outcome: VerifierOutcome::Verified { score: 1.0 }, + usage: Some(sample_usage.clone()), + metering_state: MeteringState::LowerBound, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some("Maka subject exceeded the framework timeout".into()), + artifacts: Vec::new(), + }; + let v1_timeout_reward = v2_timeout_reward.project_v1(); + assert_eq!( + v1_timeout_reward.result.status, + EvalResultStatus::SubjectFailed + ); + assert_eq!(v1_timeout_reward.result.score, Some(1.0)); + assert_eq!(v1_timeout_reward.result.usage, Some(sample_usage.clone())); + assert_eq!(v1_timeout_reward.result.cost_usd, None); + + let report_1 = evaluate_differential_oracle( + &serde_json::to_value(&v1_timeout_reward).unwrap(), + &v1_timeout_reward, + &fp, + &fp, + Some(DifferenceScenarioId::TimeoutWithRewardIncompleteUsage), + ®istry, + ); + assert!(report_1.v1_parses); + assert!(report_1.v1_matches_oracle); + assert!(report_1.conditions_comparable); + + // 2. Audit truncation vs. Missing audit log: + let (trunc_state, trunc_ev) = describe_egress_audit( + Some(b"{\"ts\":1,\"ruleId\":\"audit_truncated\",\"host\":\"\",\"normalizedPath\":\"\"}\n"), + true, + ); + assert_eq!(trunc_state, EgressAuditState::Truncated); + let mut v2_audit_trunc = v2_timeout_reward.clone(); + v2_audit_trunc.execution_outcome = ExecutionOutcome::Completed; + v2_audit_trunc.metering_state = MeteringState::Complete; + v2_audit_trunc.audit_state = trunc_state; + v2_audit_trunc.artifacts = trunc_ev.artifacts; + v2_audit_trunc.failure_reason = None; + let v1_audit_trunc = v2_audit_trunc.project_v1(); + assert_eq!(v1_audit_trunc.result.status, EvalResultStatus::Completed); + assert_eq!(v1_audit_trunc.result.score, Some(1.0)); + assert!(v1_audit_trunc.result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("egress-audit") + && a.get("truncated").and_then(Value::as_bool) == Some(true) + })); + + // Missing audit log fails closed as `infra_failed` with `score: None`. + let mut v2_audit_missing = v2_audit_trunc.clone(); + v2_audit_missing.audit_state = EgressAuditState::Missing; + let v1_audit_missing = v2_audit_missing.project_v1(); + assert_eq!( + v1_audit_missing.result.status, + EvalResultStatus::InfraFailed + ); + assert_eq!(v1_audit_missing.result.score, None); + + // 3. 0-byte empty file vs. missing critical artifact: + let temp = TempDir::new().unwrap(); + fs::write(temp.path().join("stderr.txt"), b"").unwrap(); + let (present_state, files) = verify_critical_artifacts(temp.path(), &["stderr.txt"]); + assert_eq!(present_state, CriticalArtifactsState::Present); + assert_eq!(files[0].bytes, 0); + + let (missing_state, _) = + verify_critical_artifacts(temp.path(), &["stderr.txt", "missing-transcript.json"]); + assert_eq!( + missing_state, + CriticalArtifactsState::Missing { + paths: vec!["missing-transcript.json".into()] + } + ); + let mut v2_missing_art = v2_audit_trunc.clone(); + v2_missing_art.critical_artifacts_state = missing_state; + let v1_missing_art = v2_missing_art.project_v1(); + assert_eq!(v1_missing_art.result.status, EvalResultStatus::InfraFailed); + assert_eq!(v1_missing_art.result.score, None); + + // 4. Late cancellation before vs. after verification: + let mut v2_cancel_before = v2_audit_trunc.clone(); + v2_cancel_before.execution_outcome = ExecutionOutcome::Cancelled; + v2_cancel_before.cancelled_after_verification = false; + let v1_cancel_before = v2_cancel_before.project_v1(); + assert_eq!( + v1_cancel_before.result.status, + EvalResultStatus::Indeterminate + ); + assert_eq!(v1_cancel_before.result.score, None); + + let mut v2_cancel_after = v2_audit_trunc.clone(); + v2_cancel_after.execution_outcome = ExecutionOutcome::Cancelled; + v2_cancel_after.cancelled_after_verification = true; + let v1_cancel_after = v2_cancel_after.project_v1(); + assert_eq!(v1_cancel_after.result.status, EvalResultStatus::Completed); + assert_eq!(v1_cancel_after.result.score, Some(1.0)); + + // 5. Missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0): + let mut v2_missing_reward = v2_audit_trunc.clone(); + v2_missing_reward.verifier_outcome = VerifierOutcome::MissingReward { + reason: "reward.json missing".into(), + }; + let v1_missing_reward = v2_missing_reward.project_v1(); + assert_eq!( + v1_missing_reward.result.status, + EvalResultStatus::InfraFailed + ); + assert_eq!(v1_missing_reward.result.score, None); + + let mut v2_zero_score = v2_audit_trunc; + v2_zero_score.verifier_outcome = VerifierOutcome::Verified { score: 0.0 }; + let v1_zero_score = v2_zero_score.project_v1(); + assert_eq!(v1_zero_score.result.status, EvalResultStatus::Completed); + assert_eq!(v1_zero_score.result.score, Some(0.0)); +} + +#[test] +fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_terminal() { + let attempts = vec![ + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 1, + started_at: 10, + completed_at: 20, + result: EvalResult { + score: None, + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::InfraFailed, + failure_reason: Some("docker pull failed".into()), + artifacts: Vec::new(), + }, + }, + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 2, + started_at: 30, + completed_at: 40, + result: EvalResult { + score: Some(0.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 3, + started_at: 50, + completed_at: 60, + result: EvalResult { + score: Some(1.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, + ]; + + let selected = select_cell_result(&attempts).unwrap(); + // Sequence 2 (Completed with score 0.0) is valid and earlier than sequence 3! + assert_eq!(selected.sequence, 2); + assert_eq!(selected.result.score, Some(0.0)); +} + +#[test] +fn store_journal_reconstruction_post_hoc_recovery_and_crash_recovery_boundaries() { + let temp = TempDir::new().unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let dir = ExperimentDirectory::open(temp.path().join("exp-out"), &spec).unwrap(); + let store = &dir.attempts; + + // 1. Single-writer lock rejects concurrent writers. + let guard = store.acquire_writer().unwrap(); + assert!(store.acquire_writer().is_err()); + drop(guard); + + // 2. Crash after LaunchIntent BEFORE resource creation recovers as InfraFailed (Confirmed cleanup). + store + .record_launch_intent("task-alpha::1::maka", 1, 1000, "intent-1") + .unwrap(); + let recovered = recover_unsettled_attempts(store, &ConservativeCleanupProbe, 1050).unwrap(); + assert_eq!(recovered.len(), 1); + assert_eq!( + recovered[0].execution_outcome, + ExecutionOutcome::InfraFailed + ); + assert_eq!(recovered[0].cleanup_state, CleanupState::Confirmed); + + // 3. Crash AFTER ResourceTagged with unconfirmed cleanup recovers as Indeterminate. + store + .record_launch_intent("task-alpha::1::maka", 2, 2000, "intent-2") + .unwrap(); + store + .record_resource_tagged( + "task-alpha::1::maka", + 2, + "harbor", + "trial-container-2", + Some("op-1".into()), + Some("inv-1".into()), + ) + .unwrap(); + let recovered_2 = recover_unsettled_attempts(store, &ConservativeCleanupProbe, 2100).unwrap(); + assert_eq!(recovered_2.len(), 1); + assert_eq!( + recovered_2[0].execution_outcome, + ExecutionOutcome::Indeterminate + ); + assert!(matches!( + recovered_2[0].cleanup_state, + CleanupState::Unconfirmed { .. } + )); + + // 4. Append-only PostHocRecovery updates reconstructed usage & cleanup without mutating prior entries. + let recovered_usage = NormalizedUsage { + input_tokens: 500, + output_tokens: 100, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 600, + }; + store + .record_post_hoc_recovery( + "task-alpha::1::maka", + 2, + 2200, + Some(recovered_usage.clone()), + MeteringState::Complete, + true, + Some("recovered from host receipt journal".into()), + ) + .unwrap(); + + let rebuilt = rebuild_from_journal(store).unwrap(); + let seq2 = &rebuilt.settled_v2["task-alpha::1::maka"][1]; + assert!(seq2.recovered_post_hoc); + assert_eq!(seq2.usage, Some(recovered_usage)); + assert_eq!(seq2.cleanup_state, CleanupState::Confirmed); + assert!( + seq2.project_v1().result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("post-hoc-evidence-recovery") + }) + ); +} + +#[test] +fn sqlite_runtime_artifact_capture_and_external_checkpoint_hmac_verification() { + let temp = TempDir::new().unwrap(); + let state_root = temp.path().join("host-state"); + fs::create_dir_all(&state_root).unwrap(); + + // Create a real SQLite database at state_root/runtime.sqlite + let db_path = state_root.join("runtime.sqlite"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute("CREATE TABLE receipts (id TEXT PRIMARY KEY)", []) + .unwrap(); + conn.execute("INSERT INTO receipts (id) VALUES ('op-123')", []) + .unwrap(); + drop(conn); + fs::write( + state_root.join("runtime-policy.json"), + b"{\"mode\":\"eval\"}\n", + ) + .unwrap(); + + let dest_root = temp.path().join("captured-artifacts"); + let manifest = + capture_maka_runtime_artifacts(&state_root, &dest_root, "settled", 123456).unwrap(); + assert_eq!(manifest.schema_version, "maka.eval.runtime_artifacts.v1"); + assert_eq!(manifest.files.len(), 2); + assert!(dest_root.join("runtime.sqlite").is_file()); + assert!(dest_root.join("manifest.json").is_file()); + + // Verify external metering checkpoint HMAC signing & recovery. + let trial_path = temp.path().join("trial-1"); + fs::create_dir_all(trial_path.join("agent")).unwrap(); + let secret = "0123456789abcdef0123456789abcdef"; + let body = json!({ + "schemaVersion": "maka.external_provider_usage.v2", + "profile": "deepseek-harness", + "usage": { + "inputTokens": 1000, + "outputTokens": 200, + "cacheReadTokens": 100, + "cacheWriteTokens": 0, + "reasoningTokens": 50, + "totalTokens": 1200 + }, + "settled": true, + "requests": 2, + "inFlightRequests": 0, + "admittedRequests": 2, + "usageRequests": 2, + "removedWebTools": 1, + "models": ["deepseek-v4-flash"], + "toolNames": ["bash"] + }); + let signed = sign_metering_checkpoint(body.as_object().unwrap(), secret); + let cp_path = trial_path + .join("agent") + .join("deepseek-harness.provider-usage.json"); + fs::write(&cp_path, serde_json::to_vec(&signed).unwrap()).unwrap(); + + let recovered = + recover_external_metering_checkpoint(&trial_path, "deepseek-harness", secret).unwrap(); + assert_eq!(recovered.metering_state, MeteringState::Complete); + assert_eq!(recovered.usage.unwrap().total_tokens, 1200); + assert!(recovered.cost_usd.is_some()); + + // Wrong secret must be rejected. + assert!( + recover_external_metering_checkpoint(&trial_path, "deepseek-harness", "wrong-secret") + .is_none() + ); +} + +#[tokio::test] +async fn shared_preparation_cache_lock_serializes_concurrent_task_preparations() { + let locks = PreparationCacheLocks::new(); + let active_in_critical_section = Arc::new(AtomicUsize::new(0)); + let max_concurrent_observed = Arc::new(AtomicUsize::new(0)); + let build_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(4)); + + let mut handles = Vec::new(); + for _ in 0..4 { + let locks = locks.clone(); + let active = active_in_critical_section.clone(); + let max_obs = max_concurrent_observed.clone(); + let builds = build_count.clone(); + let barrier = barrier.clone(); + handles.push(tokio::spawn(async move { + barrier.wait().await; + locks + .with_task_lock("task-shared-image", || async { + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + max_obs.fetch_max(current, Ordering::SeqCst); + if builds.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + builds.fetch_add(1, Ordering::SeqCst); + } + active.fetch_sub(1, Ordering::SeqCst); + }) + .await; + })); + } + + for h in handles { + h.await.unwrap(); + } + assert_eq!(max_concurrent_observed.load(Ordering::SeqCst), 1); + assert_eq!(build_count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn runner_executes_task_groups_bounds_retries_and_exports_cohort_summary() { + let temp = TempDir::new().unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let dir = ExperimentDirectory::open(temp.path().join("run-out"), &spec).unwrap(); + + let maka_calls = Arc::new(AtomicUsize::new(0)); + let ext_calls = Arc::new(AtomicUsize::new(0)); + + let env = Arc::new(FakeTrialEnvironment { + maka_calls: maka_calls.clone(), + ext_calls: ext_calls.clone(), + }); + + let subjects: Vec> = vec![ + Arc::new(RustHostSubject::new()), + Arc::new(ExternalCliSubject::new()), + ]; + + let mut run_input = RunExperimentInput::new(&spec, &dir.attempts, env, subjects); + // Allow 1 automatic retry for infra_failed so task-alpha::1::maka (which fails on call 1) + // automatically retries and succeeds on sequence 2! + run_input.retry_policy = RetryPolicy { + max_infra_retries: 1, + block_automatic_retry_on_indeterminate: true, + }; + + let results = run_experiment(run_input).await.unwrap(); + assert_eq!(results.len(), 4); + + // task-alpha::1::maka failed with infra_failed on sequence 1 and succeeded on sequence 2. + let alpha_maka = &results["task-alpha::1::maka"]; + assert_eq!(alpha_maka.sequence, 2); + assert_eq!(alpha_maka.result.status, EvalResultStatus::Completed); + assert_eq!(alpha_maka.result.score, Some(1.0)); + + // Verify all stored v1 files strictly decode under decode_v1_cell_attempt. + let v1_export = export_v1_canonical(&spec, &dir.attempts).unwrap(); + let v2_export = export_v2_canonical(&spec, &dir.attempts).unwrap(); + assert_eq!(v1_export.len(), 4); + assert_eq!(v2_export.len(), 4); + for attempt in v1_export.values() { + let decoded = decode_v1_cell_attempt(&serde_json::to_value(attempt).unwrap()).unwrap(); + assert_eq!(&decoded, attempt); + } + + let summary = build_cohort_summary(&spec, &v1_export); + assert_eq!(summary.total_cells, 4); + assert_eq!(summary.settled_cells, 4); + assert_eq!(summary.incomplete_cells, 0); + assert_eq!(summary.paired_task_groups, 2); + assert_eq!(summary.arms["maka"].completed_cells, 2); + assert_eq!(summary.arms["external-cli"].completed_cells, 2); +} + +#[tokio::test] +async fn paginated_transcript_export_and_relay_result_frame_boundaries() { + // 1. Paginated transcript export across multiple pages + cycle protection. + let pages = vec![ + ( + vec![json!({"id": "msg-1"}), json!({"id": "msg-2"})], + Some("cursor-2".to_owned()), + ), + (vec![json!({"id": "msg-3"})], Some("cursor-3".to_owned())), + (vec![json!({"id": "msg-4"})], None), + ]; + let idx = Arc::new(AtomicUsize::new(0)); + let exported = export_paginated_transcript(2, move |_cursor, _limit| { + let pages = pages.clone(); + let i = idx.fetch_add(1, Ordering::SeqCst); + async move { Ok(pages[i].clone()) } + }) + .await + .unwrap(); + assert_eq!(exported.len(), 4); + + // 2. Relay result frame encoding and parsing (`MAKA-EVAL-RESULT-V1`). + let token = "abcdef0123456789abcdef0123456789"; + let payload = json!({ + "schemaVersion": "maka.external_subject_result.v2", + "status": "completed", + "failureReason": null, + "usage": { + "inputTokens": 100, + "outputTokens": 50, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 150 + }, + "costUsd": 0.000055, + "artifacts": [] + }); + let frame = encode_relay_result_frame(token, &payload).unwrap(); + let noisy_stdout = format!("some tool log line\n{frame}trailing line\n"); + let extracted = parse_relay_result_frame(&noisy_stdout, token).unwrap(); + let parsed_val: Value = serde_json::from_str(&extracted).unwrap(); + assert_eq!(parsed_val, payload); + + // Ambiguous (duplicate) frames must be rejected as ResultFrameAmbiguous. + let duplicate = format!("{frame}{frame}"); + let err = parse_relay_result_frame(&duplicate, token).unwrap_err(); + assert_eq!( + err.category, + SubjectProcessDiagnosticCategory::ResultFrameAmbiguous + ); +} + +struct FakeTrialEnvironment { + maka_calls: Arc, + ext_calls: Arc, +} + +struct FakeActiveTrial { + cell_id: String, + maka_calls: Arc, + ext_calls: Arc, +} + +impl CommandRunner for FakeActiveTrial { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if self.cell_id.ends_with("::external-cli") { + self.ext_calls.fetch_add(1, Ordering::SeqCst); + let out = json!({ + "schemaVersion": "maka.external_subject_result.v2", + "status": "completed", + "failureReason": null, + "usage": { + "inputTokens": 400, + "outputTokens": 100, + "cacheReadTokens": 100, + "cacheWriteTokens": 0, + "reasoningTokens": 20, + "totalTokens": 500 + }, + "costUsd": 0.0001327, + "artifacts": [] + }); + return Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: 0, + stdout: serde_json::to_string(&out).unwrap(), + stderr: String::new(), + diagnostic: None, + }); + } + + // Maka subject invocation: decode the base64url envelope to extract executionId + let call_num = self.maka_calls.fetch_add(1, Ordering::SeqCst) + 1; + if self.cell_id == "task-alpha::1::maka" && call_num == 1 { + // Simulate an empty stdout crash on the first try so runner retries sequence 2. + return Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: 1, + stdout: String::new(), + stderr: "transient shim crash".into(), + diagnostic: Some(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::None, + bytes: None, + sha256: None, + }), + }); + } + + use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; + let encoded = &request.args[1]; + let envelope: Value = + serde_json::from_slice(&URL_SAFE_NO_PAD.decode(encoded).unwrap()).unwrap(); + let execution_id = envelope["execution"]["executionId"] + .as_str() + .unwrap() + .to_owned(); + + // Verify the embedded hostInitSettings matches our exact eval-gateway contract! + assert_eq!( + envelope["hostInitSettings"]["defaultTarget"]["slug"], + json!("eval-gateway") + ); + + let projection = HostedExecutionProjection::Settled { + execution_id, + status: "completed".into(), + failure_reason: None, + usage: NormalizedUsage { + input_tokens: 300, + output_tokens: 100, + cache_read_tokens: 100, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 400, + }, + cost_usd: Some(0.0001107), + operation_id: Some("op-settled-1".into()), + invocation_id: Some("inv-settled-1".into()), + }; + Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: 0, + stdout: serde_json::to_string(&projection).unwrap(), + stderr: String::new(), + diagnostic: None, + }) + }) + } +} + +impl ActiveTrialAttempt for FakeActiveTrial { + fn context(&self) -> SubjectExecutionContext { + SubjectExecutionContext { + cwd: "/workspace".into(), + task_input: "Solve benchmark task".into(), + metadata: Map::new(), + cancel: CancellationToken::new(), + runner: Arc::new(Self { + cell_id: self.cell_id.clone(), + maka_calls: self.maka_calls.clone(), + ext_calls: self.ext_calls.clone(), + }), + } + } + + fn verify(&self) -> BoxFuture<'_, Result> { + Box::pin(async { + Ok(ExecutorVerification { + status: EvalResultStatus::Completed, + score: Some(1.0), + failure_reason: None, + artifacts: Vec::new(), + missing_reward: false, + }) + }) + } + + fn read_audit(&self) -> BoxFuture<'_, (EgressAuditState, EgressAuditEvidence)> { + Box::pin(async { describe_egress_audit(None, false) }) + } +} + +impl TrialEnvironment for FakeTrialEnvironment { + fn kind(&self) -> &str { + "harbor" + } + + fn run_attempt<'a>( + &'a self, + cell: &'a ExperimentCell, + _sequence: u32, + _subject_credential_names: &'a [String], + _cancel: CancellationToken, + operation: TrialAttemptOperation<'a>, + ) -> BoxFuture<'a, Result, String>> { + let active = FakeActiveTrial { + cell_id: cell.id.clone(), + maka_calls: self.maka_calls.clone(), + ext_calls: self.ext_calls.clone(), + }; + Box::pin(async move { + let rec = operation(&active).await; + Ok(TrialEnvironmentOutcome::Settled(rec)) + }) + } +} + +fn sample_two_arm_spec_json() -> Value { + json!({ + "schemaVersion": "maka.eval.v1", + "id": "tb21-deepseek-v4-flash-two-arm", + "benchmark": { + "id": "terminal-bench-2.1", + "version": "d49e28f1e4ddd13d289e85a5f312a66750951932", + "config": { + "repository": "https://github.com/harbor-framework/terminal-bench-2-1" + } + }, + "executor": { + "kind": "harbor", + "config": { + "frameworkVersion": "0.20.0", + "pythonPathEnv": "MAKA_EVAL_HARBOR_PYTHON", + "trialsRootEnv": "MAKA_EVAL_TRIALS_DIR", + "environment": { "type": "docker", "delete": true } + } + }, + "execution": { + "maxConcurrentTaskGroups": 2 + }, + "subjects": [ + { + "id": "maka", + "kind": "maka", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "nodePath": "/opt/maka-node-toolchain/bin/node", + "shimPath": "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-maka-subject.js", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 120000, + "connectionSlug": "eval-gateway", + "baseUrl": "http://127.0.0.1:43120/v1", + "model": "deepseek-v4-flash", + "thinkingLevel": "off", + "permissionMode": "bypass", + "collaborationMode": "agent", + "orchestrationMode": "default", + "toolProfile": "headless-coding-v1" + } + }, + { + "id": "external-cli", + "kind": "external", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "command": "/opt/maka-node-toolchain/bin/node", + "credentialEnvironment": { + "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY" + }, + "args": [ + "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js", + "deepseek-harness", + "http://127.0.0.1:43120/v1", + "/", + "dsh", + "{{task.input}}" + ] + } + } + ], + "tasks": [ + { + "id": "task-alpha", + "input": "Fix alpha", + "config": { "harbor": { "path": "tasks/task-alpha" } } + }, + { + "id": "task-beta", + "input": "Fix beta", + "config": { "harbor": { "path": "tasks/task-beta" } } + } + ], + "repetitions": 1, + "budget": { + "timeoutMultiplier": 1, + "maxSteps": 100 + }, + "verifier": { + "reward": "reward" + } + }) +} + +#[tokio::test] +async fn harbor_trial_environment_loopback_tcp_relay_handshake_and_verification() { + let temp = TempDir::new().unwrap(); + let benchmark_dir = temp.path().join("bench"); + fs::create_dir_all(benchmark_dir.join("tasks/task-alpha")).unwrap(); + let trials_dir = temp.path().join("trials"); + fs::create_dir_all(&trials_dir).unwrap(); + + let fake_python = temp.path().join("fake-harbor-python.py"); + let script = r#"#!/usr/bin/env python3 +import json, pathlib, socket, sys + +spec_path = pathlib.Path(sys.argv[4]) +spec = json.loads(spec_path.read_text(encoding="utf-8")) +kwargs = spec["agent"]["kwargs"] +trial_dir = pathlib.Path(spec["trials_dir"]) / spec["trial_name"] +(trial_dir / "artifacts").mkdir(parents=True, exist_ok=True) +(trial_dir / "agent").mkdir(parents=True, exist_ok=True) + +sock = socket.create_connection((kwargs["relay_host"], int(kwargs["relay_port"])), timeout=10) +reader = sock.makefile("r", encoding="utf-8", newline="\n") +writer = sock.makefile("w", encoding="utf-8", newline="\n") + +writer.write(json.dumps({ + "kind": "ready", + "token": kwargs["relay_token"], + "cwd": "/workspace", + "instruction": "Fix alpha" +}) + "\n") +writer.flush() + +for line in reader: + msg = json.loads(line) + if msg["kind"] == "execute": + writer.write(json.dumps({ + "kind": "executed", + "token": kwargs["relay_token"], + "termination": "exited", + "exitCode": 0, + "stdout": "command-ran-ok", + "diagnostic": {"category": "none"} + }) + "\n") + writer.flush() + elif msg["kind"] == "verify": + (trial_dir / "artifacts" / "egress-hits.jsonl").write_text( + json.dumps({"ruleId": "allow_deepseek"}) + "\n", + encoding="utf-8" + ) + (trial_dir / "result.json").write_text( + json.dumps({ + "verifier_result": { + "rewards": { + "reward": 1.0 + } + } + }), + encoding="utf-8" + ) + break + +sock.close() +sys.exit(0) +"#; + fs::write(&fake_python, script).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&fake_python, fs::Permissions::from_mode(0o755)).unwrap(); + } + + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let cells = expand_experiment(&spec); + let cell = &cells[0]; + + let mut env_map = std::collections::HashMap::new(); + env_map.insert( + "MAKA_EVAL_HARBOR_PYTHON".to_owned(), + fake_python.to_string_lossy().into_owned(), + ); + env_map.insert( + "MAKA_EVAL_TRIALS_DIR".to_owned(), + trials_dir.to_string_lossy().into_owned(), + ); + env_map.insert("DEEPSEEK_API_KEY".to_owned(), "sk-test-123".to_owned()); + + let exec_cfg = + HarborExecutorConfig::decode(&spec.executor.kind, &spec.executor.config).unwrap(); + let harbor_env = + HarborTrialEnvironment::new(exec_cfg, benchmark_dir).with_env_overrides(env_map); + harbor_env + .preflight(&["DEEPSEEK_API_KEY".to_owned()], CancellationToken::new()) + .await + .unwrap(); + + let cell_id = cell.id.clone(); + let outcome = harbor_env + .run_attempt( + cell, + 1, + &["DEEPSEEK_API_KEY".to_owned()], + CancellationToken::new(), + Box::new(move |active| { + Box::pin(async move { + let ctx = active.context(); + assert_eq!(ctx.cwd, "/workspace"); + let exec_out = ctx + .execute(SubjectProcessRequest { + command: "echo".into(), + args: vec!["hello".into()], + environment: std::collections::HashMap::new(), + credential_environment: std::collections::HashMap::new(), + capture_stdout: true, + }) + .await + .unwrap(); + assert_eq!(exec_out.exit_code, 0); + assert_eq!(exec_out.stdout, "command-ran-ok"); + + let verified = active.verify().await.unwrap(); + assert_eq!(verified.status, EvalResultStatus::Completed); + assert_eq!(verified.score, Some(1.0)); + + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence: 1, + started_at: 100, + completed_at: 200, + duration_ms: 100, + phase: AttemptPhase::Verified, + execution_outcome: ExecutionOutcome::Completed, + verifier_outcome: verified.to_verifier_outcome(), + usage: None, + metering_state: MeteringState::Unknown, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::Clean, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: true, + recovered_post_hoc: false, + failure_reason: None, + artifacts: verified.artifacts, + } + }) + }), + ) + .await + .unwrap(); + + match outcome { + TrialEnvironmentOutcome::Settled(rec) => { + assert_eq!( + rec.verifier_outcome, + VerifierOutcome::Verified { score: 1.0 } + ); + assert!(!rec.artifacts.is_empty()); + } + TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::ExitBeforeReady, + .. + } => { + // In restricted sandboxes where outbound loopback TCP connect is denied, + // verify that ExitBeforeReady was recorded and test verify_harbor_trial_result directly. + let manual_trial = trials_dir.join("manual-verify-trial"); + fs::create_dir_all(manual_trial.join("artifacts")).unwrap(); + fs::write( + manual_trial.join("result.json"), + serde_json::to_vec(&json!({ + "verifier_result": { "rewards": { "reward": 1.0 } } + })) + .unwrap(), + ) + .unwrap(); + let verified = + read_trial_verification(&manual_trial, "maka-eval-manual", cell, "harbor", false) + .unwrap(); + assert_eq!(verified.status, EvalResultStatus::Completed); + assert_eq!(verified.score, Some(1.0)); + } + other => panic!("unexpected outcome: {other:?}"), + } +} diff --git a/scripts/asf-license-headers.mjs b/scripts/asf-license-headers.mjs index 554dedddb48..223773fe6d4 100644 --- a/scripts/asf-license-headers.mjs +++ b/scripts/asf-license-headers.mjs @@ -141,6 +141,7 @@ const coveredNames = new Map([ ['Dockerfile', ['hash']], ['pre-commit', ['hash']], ['justfile', ['hash']], + ['network-policy', ['hash']], ]); const startsWith = (prefix) => (path) => path === prefix || path.startsWith(`${prefix}/`); From 330cfcdb09f7dfe35e2ed560d6912e633053fd43 Mon Sep 17 00:00:00 2001 From: colafornia Date: Thu, 12 Nov 2026 23:47:12 +0800 Subject: [PATCH 2/3] feat(eval): enforce product parity and native host execution - Execute Maka through its released host composition, public Settings/Session APIs, and composer draft/receipt client without benchmark-specific prompts, tool pruning, or synthetic model overrides. - Keep the local metering gateway non-mutating and preserve unattended interaction/approval boundaries as explicit settlement outcomes alongside environment limitations. - Add end-to-end product parity verification (`maka-eval parity`) and require passing parity reports for M3 acceptance and M5 production delivery gates. Generated-by: Google Antigravity --- Cargo.lock | 3 + products/eval/Cargo.toml | 3 + products/eval/README.md | 23 +- products/eval/README.zh-CN.md | 23 +- products/eval/src/contract.rs | 56 +- products/eval/src/environment/harbor_relay.rs | 40 +- products/eval/src/lib.rs | 1 + products/eval/src/main.rs | 236 ++- products/eval/src/measurement.rs | 11 - products/eval/src/measurement/gateway.rs | 259 ++- products/eval/src/parity.rs | 1639 +++++++++++++++++ products/eval/src/report.rs | 249 ++- products/eval/src/runner.rs | 51 +- products/eval/src/store.rs | 134 +- products/eval/src/subject/maka.rs | 728 +++++--- products/eval/src/subject/maka/native.rs | 833 +++++++++ products/eval/tests/integration.rs | 1041 ++++++++++- products/eval/tests/support/native_host.py | 143 ++ scripts/ci.test.mjs | 1 + scripts/ci/suites.mjs | 8 +- 20 files changed, 4895 insertions(+), 587 deletions(-) create mode 100644 products/eval/src/parity.rs create mode 100644 products/eval/src/subject/maka/native.rs create mode 100644 products/eval/tests/support/native_host.py diff --git a/Cargo.lock b/Cargo.lock index 2271c906caf..344c222b1a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6921,6 +6921,9 @@ dependencies = [ "hyper", "hyper-util", "libc", + "maka-app-client", + "maka-host-runtime", + "maka-plugin-core", "regex", "reqwest", "rusqlite", diff --git a/products/eval/Cargo.toml b/products/eval/Cargo.toml index 9818cef1337..3e3b555c696 100644 --- a/products/eval/Cargo.toml +++ b/products/eval/Cargo.toml @@ -27,6 +27,9 @@ name = "maka-eval" path = "src/main.rs" [dependencies] +maka-app-client.workspace = true +maka-host-runtime.workspace = true +maka-plugin-core.workspace = true base64 = "0.23.1" clap.workspace = true futures-util.workspace = true diff --git a/products/eval/README.md b/products/eval/README.md index 136d2ff746a..0d6e0e903e0 100644 --- a/products/eval/README.md +++ b/products/eval/README.md @@ -23,14 +23,29 @@ Measurement and control plane for multi-arm benchmark experiments (`maka.eval.v1` and `maka.eval.v2`). `maka-eval` owns experiment decoding, task-group scheduling, deadline authority, durable append-only attempt journaling, token/cost metering reconciliation, trial environment orchestration, and deterministic cohort reporting. +## Native Host & Product Parity + +Install matching `maka` and `maka-eval` binaries side by side inside the Trial image. Set `subject.config.makaExecutable` to the native `maka` path; `nodePath`, `shimPath`, legacy provider configuration, and benchmark-only profiles (`toolProfile`, `permissionMode: "bypass"`, `collaborationMode`, `orchestrationMode`) are rejected. + +Maka subjects are configured strictly through public product interfaces (`provider`, `connectionSlug`, `connectionName`, `baseUrl`, `model`, optional `modelOverride`, `requestHeaders`, `requestBodyOverlay`, `sandboxMode`, `requestPermissions`, `thinkingLevel`, `language`, `workspaceInstructions`, and `apiKeyEnvironment`). The adapter initializes an isolated Root via `maka host init --settings-stdin`, applies only declared public Settings/Session configurations, and submits tasks through the production composer draft/receipt client without prompt injection, tool pruning, or synthetic model overrides. A non-mutating local metering gateway records provider usage while forwarding outbound JSON bodies verbatim. Pending interactive approvals or questions are never auto-bypassed in unattended runs; they settle explicitly as `unattended-interaction-blocked`, and host/environment constraints are reported as `environment-limitation`. + +Product parity is a mandatory gate for `M3` acceptance and `M5` production delivery (`RunFingerprint` documents run conditions only and does not establish parity): +- `maka-eval parity [--subject ] [--out ]` runs both the normal Maka client path and the `maka-eval` entry point against a local probe under identical inputs and verifies committed `RunSpec`/`RequestSpec` facts plus wire payloads across product composition, effective instructions, tool exposure, and execution policies. +- `maka-eval report --milestone m3|m5 --parity-report ` enforces parity (and complete non-indeterminate settlement for `m5`). + +`egressProxy` currently fails preflight because the native executor does not provision the required sidecar and mounts. Unconfirmed cleanup blocks further admission across runs until a `PostHocRecovery` entry confirms disposal. Both report versions read committed journal records; result JSON files are no longer a separate authority. Records without pricing evidence retain unknown cost. + +For focused verification, build `maka-cli` first and supply its path as runtime `CARGO_BIN_EXE_maka` when running `cargo test -p maka-eval`. CI supplies and archives the native helper. The real Host and parity tests use local fake providers and no paid model calls. + ## Architecture -- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions (lifecycle phase, execution outcome, metering state, cleanup state), run fingerprinting, product default runtime composition profile, and the semantic difference registry. -- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), admission boundary cutoffs, HMAC-authenticated checkpoint recovery, receipt-vs-proxy reconciliation without double-counting, and standardized `DEEPSEEK_V4_FLASH_COST` pricing. +- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions (lifecycle phase, execution outcome, metering state, cleanup state), condition-only `RunFingerprint`, and the semantic difference registry. +- [`parity`](src/parity.rs): End-to-end Product Parity verification comparing normal Maka client execution against `maka-eval` across product composition, effective instructions, tool exposure (Code Mode, skills, schemas), and execution policies. +- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), non-mutating metering gateway, admission boundary cutoffs, HMAC-authenticated checkpoint recovery, receipt-vs-proxy reconciliation without double-counting, and standardized `DEEPSEEK_V4_FLASH_COST` pricing. - [`store`](src/store.rs): Single-writer append-only experiment journal (`LaunchIntent`, `ResourceTagged`, `Checkpoint`, `Settled`, `PostHocRecovery`), pre-dispatch intent persistence, atomic settlement commits, critical artifact verification, and deterministic state reconstruction. - [`runner`](src/runner.rs): Task-group admission (`maxConcurrentTaskGroups`), shared preparation cache boundary lock, phase-scoped deadline authority, bounded retry policy, earliest-valid-attempt canonical selection, and crash recovery. - [`environment`](src/environment.rs): Per-attempt `TrialEnvironment` abstraction and version-pinned Harbor / Pier bridge with 1:1 attempt-to-trial mapping. -- [`subject`](src/subject.rs): Subject adapters for the Rust Maka Host (`maka host init --settings-stdin --root ` isolated root bootstrap, public Host client execution, durable receipt journal, and transcript/usage settlement) and external CLI harnesses (`exit-code` and `protocol-v1` bounded result frames). -- [`report`](src/report.rs): Deterministic journal projection to `v1` and `v2`, paired cohort summaries, and 3-verdict differential oracle evaluation (`v1_parses`, `v1_matches_oracle`, `conditions_comparable`). +- [`subject`](src/subject.rs): Subject adapters for the Rust Maka Host (`maka host init --settings-stdin --root ` isolated root bootstrap, public Settings/Session client execution, explicit unattended interaction & environment limitation reporting, durable receipt journal, and transcript/usage settlement) and external CLI harnesses (`exit-code` and `protocol-v1` bounded result frames). +- [`report`](src/report.rs): Deterministic journal projection to `v1` and `v2`, paired cohort summaries, `M3`/`M5` milestone parity acceptance gates, and 3-verdict differential oracle evaluation (`v1_parses`, `v1_matches_oracle`, `conditions_comparable`). [Source](src/lib.rs) · [Cargo manifest](Cargo.toml) diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md index 72213725b3c..a163b68d29d 100644 --- a/products/eval/README.zh-CN.md +++ b/products/eval/README.zh-CN.md @@ -23,14 +23,29 @@ 多臂基准评测实验的测量与控制平面（支持 `maka.eval.v1` 与 `maka.eval.v2`）。`maka-eval` 负责实验契约解析、任务组并发调度、阶段截止时间权威、持久化只追加（append-only）尝试日志、Token 与成本计量对账、Trial 环境编排及确定性对照报告生成。 +## 原生 Host 与产品等价性（Product Parity） + +在 Trial 镜像内将匹配版本的 `maka` 和 `maka-eval` 放在同一目录，`subject.config.makaExecutable` 指向原生 `maka`。拒绝 `nodePath`、`shimPath`、旧 provider 配置及仅面向评测的专用参数（`toolProfile`、`permissionMode: "bypass"`、`collaborationMode`、`orchestrationMode`）。 + +Maka 被测对象仅通过公开产品接口配置（`provider`、`connectionSlug`、`connectionName`、`baseUrl`、`model`、可选 `modelOverride`、`requestHeaders`、`requestBodyOverlay`、`sandboxMode`、`requestPermissions`、`thinkingLevel`、`language`、`workspaceInstructions` 与 `apiKeyEnvironment`）。适配器通过 `maka host init --settings-stdin` 初始化隔离的 Root，仅应用显式声明的公开 Settings/Session 配置，并通过生产客户端的 composer 草稿与回执协议提交任务，不注入评测提示词、不裁剪或替换工具、不注入合成模型参数。本地计量网关以只读方式统计提供商用量，原样转发请求体。无人值守执行不会静默绕过审批或交互请求，遇到挂起交互时以 `unattended-interaction-blocked` 显式结算；工作区或环境受限时以 `environment-limitation` 显式上报。 + +产品等价性是 `M3` 验收与 `M5` 生产交付的前置准入门槛（`RunFingerprint` 仅记录运行条件，不构成产品等价性证明）： +- `maka-eval parity [--subject ] [--out ]` 在相同版本、公开配置、工作区与任务输入下同时驱动常规 Maka 客户端路径与 `maka-eval` 入口，比对落盘的 `RunSpec`/`RequestSpec` 事实与出站报文，验证产品组合、生效指令、工具暴露及执行策略等价。 +- `maka-eval report --milestone m3|m5 --parity-report ` 强制校验等价性报告（`m5` 同时要求全部单元完成确定性结算）。 + +当前原生执行器尚未配置代理侧车与挂载，因此 `egressProxy` 会在预检失败。清理未确认的尝试会跨次运行阻断准入，直到 `PostHocRecovery` 记录确认资源已处置。两版报告均读取已提交日志，不再将结果 JSON 文件作为另一权威；缺少定价依据的记录保持成本未知。 + +定向验证前先构建 `maka-cli`，运行 `cargo test -p maka-eval` 时通过运行时环境变量 `CARGO_BIN_EXE_maka` 指定其路径。CI 负责提供和归档原生辅助程序。真实 Host 与等价性测试使用本地假提供商，不产生付费模型调用。 + ## 模块结构 -- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度（生命周期阶段、执行结果、计量完整性、环境清理状态）、运行指纹、产品默认运行时组合配置及语义差异注册表。 -- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不进行重算求和）、准入截止边界、HMAC 鉴权检查点恢复、回执与代理日志去重对账及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 +- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度（生命周期阶段、执行结果、计量完整性、环境清理状态）、仅记录运行条件的 `RunFingerprint` 及语义差异注册表。 +- [`parity`](src/parity.rs)：端到端产品等价性验证，对比常规 Maka 客户端与 `maka-eval` 在产品组合、生效指令、工具暴露（Code Mode、内置 skill、工具 schema）及执行策略四项维度的等价性。 +- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）、无篡改计量网关、准入截止边界、HMAC 鉴权检查点恢复、回执与代理日志去重对账及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 - [`store`](src/store.rs)：单写者只追加实验日志（`LaunchIntent`、`ResourceTagged`、`Checkpoint`、`Settled`、`PostHocRecovery`）、派发前意图落盘、原子结算提交、关键产物校验与确定性状态重建。 - [`runner`](src/runner.rs)：任务组并发准入（`maxConcurrentTaskGroups`）、共享预备缓存边界锁、分阶段截止时间控制、有界重试策略、最早有效尝试（`sequence >= 1`）规范结果选择及崩溃恢复。 - [`environment`](src/environment.rs)：单次尝试级 `TrialEnvironment` 抽象及版本锁定的 Harbor / Pier 桥接（1:1 尝试与 Trial 映射）。 -- [`subject`](src/subject.rs)：被测系统适配器，包含 Rust Maka Host 适配器（通过 `maka host init --settings-stdin --root ` 引导隔离的 Host Root、公共 Host 客户端执行、持久化回执日志与历史/用量结算）和外部 CLI 适配器（`exit-code` 与 `protocol-v1` 有界结果帧）。 -- [`report`](src/report.rs)：从日志确定性重建 `v1` 与 `v2` 结果、生成成对队列汇总报告及三项裁决差分对比报告（`v1_parses`、`v1_matches_oracle`、`conditions_comparable`）。 +- [`subject`](src/subject.rs)：被测系统适配器，包含 Rust Maka Host 适配器（通过 `maka host init --settings-stdin --root ` 引导隔离的 Host Root、公开 Settings/Session 客户端执行、无人值守交互阻断与环境限制显式上报、持久化回执日志与历史/用量结算）和外部 CLI 适配器（`exit-code` 与 `protocol-v1` 有界结果帧）。 +- [`report`](src/report.rs)：从日志确定性重建 `v1` 与 `v2` 结果、生成成对队列汇总报告、`M3`/`M5` 里程碑等价性准入校验及三项裁决差分对比报告（`v1_parses`、`v1_matches_oracle`、`conditions_comparable`）。 [源码](src/lib.rs) · [Cargo 配置](Cargo.toml) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs index 5686d212dbe..702a22e4612 100644 --- a/products/eval/src/contract.rs +++ b/products/eval/src/contract.rs @@ -20,7 +20,7 @@ //! Experiment specifications, orthogonal `v2` attempt records, strict `v1` //! projections, run fingerprinting, and the semantic difference registry. -use crate::measurement::{MeteringState, NormalizedUsage, deepseek_cost_usd}; +use crate::measurement::{MeteringState, NormalizedUsage}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; @@ -315,7 +315,11 @@ fn canonicalize_value(value: &Value) -> Value { } } -/// Separates semantic comparability identity from execution-only knobs (like concurrency). +/// Documents experimental and execution conditions for comparability checks. +/// +/// Fingerprints record the declared configuration and parameters of a run; freezing an +/// eval-only fingerprint does not establish product parity. M3 acceptance and M5 production +/// delivery require a verified [`crate::parity::ParityVerificationReport`] alongside the run. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RunFingerprint { @@ -556,6 +560,31 @@ pub enum CriticalArtifactsState { Missing { paths: Vec }, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "snake_case", + rename_all_fields = "camelCase" +)] +pub enum PricingEvidence { + Model { + model: String, + cost_usd: Option, + }, + SubjectReported { + cost_usd: f64, + }, +} + +impl PricingEvidence { + pub fn cost_usd(&self) -> Option { + match self { + Self::Model { cost_usd, .. } => *cost_usd, + Self::SubjectReported { cost_usd } => Some(*cost_usd), + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct V2AttemptRecord { @@ -570,6 +599,8 @@ pub struct V2AttemptRecord { pub verifier_outcome: VerifierOutcome, pub usage: Option, pub metering_state: MeteringState, + #[serde(default)] + pub pricing: Option, pub cleanup_state: CleanupState, pub audit_state: EgressAuditState, pub critical_artifacts_state: CriticalArtifactsState, @@ -585,14 +616,23 @@ impl V2AttemptRecord { pub fn project_v1(&self) -> V1CellAttempt { let mut artifacts = self.artifacts.clone(); - // Cost is reported only when usage is present AND metering_state is Complete. - // Under Adjudication #1 (Timeout with reward but incomplete usage), lower-bound - // usage is preserved in `usage` while `costUsd` is projected as `null`. - let cost_usd = match (&self.usage, self.metering_state) { - (Some(u), MeteringState::Complete) => Some(deepseek_cost_usd(u)), - _ => None, + let cost_usd = if self.usage.is_some() && self.metering_state == MeteringState::Complete { + self.pricing.as_ref().and_then(PricingEvidence::cost_usd) + } else { + None }; + if self.execution_outcome == ExecutionOutcome::Cancelled + && self.cancelled_after_verification + { + artifacts.push( + serde_json::json!({"kind": "cancelled-after-verification"}) + .as_object() + .unwrap() + .clone(), + ); + } + if self.recovered_post_hoc { let mut marker = Map::new(); marker.insert( diff --git a/products/eval/src/environment/harbor_relay.rs b/products/eval/src/environment/harbor_relay.rs index 65918b493c5..c736a4159a8 100644 --- a/products/eval/src/environment/harbor_relay.rs +++ b/products/eval/src/environment/harbor_relay.rs @@ -537,6 +537,11 @@ impl TrialEnvironment for HarborTrialEnvironment { if cancel.is_cancelled() { return Err(ContractError::Invalid("preflight cancelled".into())); } + if self.config.egress_proxy_required { + return Err(ContractError::Invalid( + "egressProxy is not supported by the native Harbor/Pier executor".into(), + )); + } let _ = self.resolve_trials_root().map_err(ContractError::Invalid)?; let _ = self .resolve_python_executable() @@ -687,7 +692,8 @@ impl TrialEnvironment for HarborTrialEnvironment { .current_dir(&self.spec_dir) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()); + .stderr(Stdio::null()) + .kill_on_drop(true); cmd.env("PYTHONPATH", bundled_harbor_relay_root()); if self.config.egress_proxy_required { cmd.env("MAKA_EVAL_EGRESS_REQUIRED", "1"); @@ -882,9 +888,15 @@ impl TrialEnvironment for HarborTrialEnvironment { finalization_confirmed: finalization_confirmed.clone(), }; - let mut record = operation(&active).await; - - let host_cancelled = cancel.is_cancelled(); + let record = { + let operation = operation(&active); + tokio::pin!(operation); + tokio::select! { + biased; + record = &mut operation => Some(record), + _ = cancel.cancelled() => None, + } + }; if !decision_sent.load(Ordering::SeqCst) || finalization_confirmed.lock().await.is_none() { @@ -902,6 +914,18 @@ impl TrialEnvironment for HarborTrialEnvironment { *finalization_confirmed.lock().await = Some(ok); } + let finalized_ok = finalization_confirmed.lock().await.unwrap_or(false); + let Some(mut record) = record else { + return Ok(TrialEnvironmentOutcome::Indeterminate { + cause: if finalized_ok { + IndeterminateCause::HostCancelled + } else { + IndeterminateCause::CleanupUnconfirmed + }, + partial: None, + }); + }; + if let Some(diag) = last_diagnostic.lock().await.clone() && diag.category != SubjectProcessDiagnosticCategory::None { @@ -920,14 +944,6 @@ impl TrialEnvironment for HarborTrialEnvironment { record.artifacts.push(art); } - if host_cancelled { - return Ok(TrialEnvironmentOutcome::Indeterminate { - cause: IndeterminateCause::HostCancelled, - partial: Some(record), - }); - } - - let finalized_ok = finalization_confirmed.lock().await.unwrap_or(false); if !finalized_ok { return Ok(TrialEnvironmentOutcome::Indeterminate { cause: IndeterminateCause::CleanupUnconfirmed, diff --git a/products/eval/src/lib.rs b/products/eval/src/lib.rs index e4cf378ed88..821943dc8f7 100644 --- a/products/eval/src/lib.rs +++ b/products/eval/src/lib.rs @@ -22,6 +22,7 @@ pub mod contract; pub mod environment; pub mod measurement; +pub mod parity; pub mod report; pub mod runner; pub mod store; diff --git a/products/eval/src/main.rs b/products/eval/src/main.rs index 13f07c57c78..40a94142fd1 100644 --- a/products/eval/src/main.rs +++ b/products/eval/src/main.rs @@ -20,11 +20,15 @@ use clap::{Parser, Subcommand}; use maka_eval::contract::{expand_experiment, parse_experiment_spec}; use maka_eval::environment::{HarborExecutorConfig, HarborTrialEnvironment}; -use maka_eval::report::{build_cohort_summary, export_v1_canonical, export_v2_canonical}; +use maka_eval::parity::{ParityVerificationReport, verify_experiment_parity}; +use maka_eval::report::{ + MilestoneGate, build_cohort_summary, evaluate_milestone_gate, export_v1_canonical, + export_v2_canonical, +}; use maka_eval::runner::{RunExperimentInput, run_experiment}; -use maka_eval::store::ExperimentDirectory; +use maka_eval::store::{ExperimentDirectory, write_atomic_bytes}; use maka_eval::subject::{ExternalCliSubject, RustHostSubject, SubjectAdapter}; -use serde_json::Value; +use serde_json::{Value, json}; use std::fs; use std::path::PathBuf; use std::process::ExitCode; @@ -32,7 +36,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU8, Ordering}; use tokio_util::sync::CancellationToken; -pub const USAGE: &str = "usage: maka eval run --out [--cell ]"; +pub const USAGE: &str = "usage: maka eval ..."; #[derive(Debug, Parser)] #[command( @@ -46,6 +50,8 @@ struct Cli { #[derive(Debug, Subcommand)] enum Command { + #[command(hide = true)] + HostSubject { payload: String }, /// Execute an evaluation experiment specification. Run { /// Path to the experiment specification JSON file. @@ -57,7 +63,18 @@ enum Command { #[arg(long = "cell")] cell_ids: Vec, }, - /// Validate or inspect an experiment directory and export canonical projections. + /// Verify Product Parity between normal Maka client execution and maka-eval. + Parity { + /// Path to the experiment specification JSON file. + spec: PathBuf, + /// Optional Maka subject ID filter. + #[arg(long = "subject")] + subject_id: Option, + /// Optional path to write the parity verification report JSON. + #[arg(long = "out")] + out_file: Option, + }, + /// Validate or inspect an experiment directory, export canonical projections, and enforce M3/M5 gates. Report { /// Path to the experiment specification JSON file. spec: PathBuf, @@ -67,6 +84,12 @@ enum Command { /// Output format (`summary`, `v1`, or `v2`). #[arg(long, default_value = "summary")] format: String, + /// Optional milestone acceptance gate (`m3` or `m5`), requiring verified Product Parity. + #[arg(long = "milestone")] + milestone: Option, + /// Repeatable path(s) to `ParityVerificationReport` JSON file(s). + #[arg(long = "parity-report")] + parity_reports: Vec, }, } @@ -81,15 +104,65 @@ async fn main() -> ExitCode { }; match cli.command { + Command::HostSubject { payload } => { + let stop = CancellationToken::new(); + let signal_stop = stop.clone(); + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate()).expect("SIGTERM registration"); + tokio::select! { _ = term.recv() => {}, _ = tokio::signal::ctrl_c() => {} } + } + #[cfg(not(unix))] + let _ = tokio::signal::ctrl_c().await; + signal_stop.cancel(); + }); + match maka_eval::subject::maka::native::run(&payload, stop).await { + Ok(result) => { + let value = serde_json::to_value(&result).unwrap(); + match std::env::var("MAKA_EVAL_RESULT_TOKEN") { + Ok(token) => { + match maka_eval::subject::external_cli::encode_relay_result_frame( + &token, &value, + ) { + Ok(frame) => print!("{frame}"), + Err(error) => { + eprintln!("native Eval Host result frame failed: {error}"); + return ExitCode::FAILURE; + } + } + } + Err(std::env::VarError::NotPresent) => println!("{value}"), + Err(error) => { + eprintln!("native Eval Host result token failed: {error}"); + return ExitCode::FAILURE; + } + } + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("native Eval Host failed: {error}"); + ExitCode::FAILURE + } + } + } Command::Run { spec, out_dir, cell_ids, } => run_cli_command(&spec, &out_dir, cell_ids).await, + Command::Parity { + spec, + subject_id, + out_file, + } => run_parity_command(&spec, subject_id.as_deref(), out_file.as_deref()).await, Command::Report { spec, out_dir, format, + milestone, + parity_reports, } => { let raw = match fs::read_to_string(&spec) { Ok(s) => s, @@ -112,6 +185,25 @@ async fn main() -> ExitCode { return ExitCode::from(2); } }; + let milestone_gate = match milestone.as_deref() { + None => None, + Some(raw_ms) => match MilestoneGate::parse(raw_ms) { + Some(ms) => Some(ms), + None => { + eprintln!( + "maka eval: unsupported milestone '{raw_ms}' (expected m3 or m5)" + ); + return ExitCode::from(2); + } + }, + }; + let loaded_parity_reports = match load_parity_reports(&parity_reports) { + Ok(r) => r, + Err(err) => { + eprintln!("maka eval: {err}"); + return ExitCode::from(2); + } + }; let directory = match ExperimentDirectory::open(&out_dir, &experiment_spec) { Ok(d) => d, Err(err) => { @@ -120,30 +212,64 @@ async fn main() -> ExitCode { } }; + let v1 = match export_v1_canonical(&experiment_spec, &directory.attempts) { + Ok(v1) => v1, + Err(err) => { + eprintln!("maka eval: {err}"); + return ExitCode::from(1); + } + }; + let gate_report = milestone_gate.map(|ms| { + evaluate_milestone_gate(ms, &experiment_spec, &v1, &loaded_parity_reports) + }); + if let Some(gate) = &gate_report + && !gate.passed + { + for violation in &gate.violations { + eprintln!("maka eval milestone gate violation: {violation}"); + } + } + match format.as_str() { - "v1" => match export_v1_canonical(&experiment_spec, &directory.attempts) { - Ok(v1) => { - println!("{}", serde_json::to_string_pretty(&v1).unwrap()); - ExitCode::SUCCESS - } - Err(err) => { - eprintln!("maka eval: {err}"); + "v1" => { + println!("{}", serde_json::to_string_pretty(&v1).unwrap()); + if gate_report.as_ref().is_some_and(|g| !g.passed) { ExitCode::from(1) + } else { + ExitCode::SUCCESS } - }, + } "v2" => match export_v2_canonical(&experiment_spec, &directory.attempts) { Ok(v2) => { println!("{}", serde_json::to_string_pretty(&v2).unwrap()); - ExitCode::SUCCESS + if gate_report.as_ref().is_some_and(|g| !g.passed) { + ExitCode::from(1) + } else { + ExitCode::SUCCESS + } } Err(err) => { eprintln!("maka eval: {err}"); ExitCode::from(1) } }, - _ => match export_v1_canonical(&experiment_spec, &directory.attempts) { - Ok(v1) => { - let summary = build_cohort_summary(&experiment_spec, &v1); + _ => { + let summary = build_cohort_summary(&experiment_spec, &v1); + if let Some(gate) = &gate_report { + println!( + "{}", + serde_json::to_string(&json!({ + "summary": summary, + "milestoneGate": gate, + })) + .unwrap() + ); + if summary.incomplete_cells == 0 && gate.passed { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } + } else { println!("{}", serde_json::to_string(&summary).unwrap()); if summary.incomplete_cells == 0 { ExitCode::SUCCESS @@ -151,16 +277,82 @@ async fn main() -> ExitCode { ExitCode::from(1) } } - Err(err) => { - eprintln!("maka eval: {err}"); - ExitCode::from(1) - } - }, + } } } } } +async fn run_parity_command( + spec: &std::path::Path, + subject_id: Option<&str>, + out_file: Option<&std::path::Path>, +) -> ExitCode { + let raw = match fs::read_to_string(spec) { + Ok(s) => s, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let parsed: Value = match serde_json::from_str(&raw) { + Ok(v) => v, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let experiment_spec = match parse_experiment_spec(&parsed) { + Ok(s) => s, + Err(err) => { + eprintln!("maka eval: {err}\n{USAGE}"); + return ExitCode::from(2); + } + }; + let reports = match verify_experiment_parity(&experiment_spec, subject_id).await { + Ok(r) => r, + Err(err) => { + eprintln!("maka eval parity failed: {err}"); + return ExitCode::from(1); + } + }; + let encoded = serde_json::to_string_pretty(&reports).unwrap(); + if let Some(target) = out_file { + let mut bytes = encoded.as_bytes().to_vec(); + bytes.push(b'\n'); + if let Err(err) = write_atomic_bytes(target, &bytes) { + eprintln!("maka eval parity write failed: {err}"); + return ExitCode::from(1); + } + } + println!("{encoded}"); + if reports.iter().all(|r| r.passed) { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } +} + +fn load_parity_reports(paths: &[PathBuf]) -> Result, String> { + let mut out = Vec::new(); + for path in paths { + let raw = fs::read_to_string(path) + .map_err(|e| format!("failed reading parity report {}: {e}", path.display()))?; + let val: Value = serde_json::from_str(&raw) + .map_err(|e| format!("invalid parity report JSON {}: {e}", path.display()))?; + if val.is_array() { + let batch: Vec = serde_json::from_value(val) + .map_err(|e| format!("invalid parity report array {}: {e}", path.display()))?; + out.extend(batch); + } else { + let single: ParityVerificationReport = serde_json::from_value(val) + .map_err(|e| format!("invalid parity report {}: {e}", path.display()))?; + out.push(single); + } + } + Ok(out) +} + async fn run_cli_command(spec: &PathBuf, out_dir: &PathBuf, cell_ids: Vec) -> ExitCode { let raw = match fs::read_to_string(spec) { Ok(s) => s, diff --git a/products/eval/src/measurement.rs b/products/eval/src/measurement.rs index 722dbf9a63d..b7e6723b94f 100644 --- a/products/eval/src/measurement.rs +++ b/products/eval/src/measurement.rs @@ -172,7 +172,6 @@ pub struct ProviderMeteringCounts { pub in_flight_requests: u64, pub admitted_requests: u64, pub usage_requests: u64, - pub removed_web_tools: u64, pub models: Vec, pub tool_names: Vec, } @@ -253,7 +252,6 @@ pub struct AdmissionBoundaryLedger { requests: BTreeMap, seen_checkpoint_digests: BTreeSet, latest_checkpoint: Option, - removed_web_tools: u64, tool_names: BTreeSet, } @@ -378,7 +376,6 @@ impl AdmissionBoundaryLedger { in_flight_requests, admitted_requests, usage_requests, - removed_web_tools: self.removed_web_tools, models: models.into_iter().collect(), tool_names: self.tool_names.iter().cloned().collect(), } @@ -570,7 +567,6 @@ pub fn recover_external_metering_checkpoint( "inFlightRequests", "admittedRequests", "usageRequests", - "removedWebTools", "models", "toolNames", "mac", @@ -596,8 +592,6 @@ pub fn recover_external_metering_checkpoint( let admitted_requests = nonnegative_u64(checkpoint.get("admittedRequests")?, "admittedRequests").ok()?; let usage_requests = nonnegative_u64(checkpoint.get("usageRequests")?, "usageRequests").ok()?; - let removed_web_tools = - nonnegative_u64(checkpoint.get("removedWebTools")?, "removedWebTools").ok()?; let models = checkpoint .get("models")? .as_array()? @@ -618,7 +612,6 @@ pub fn recover_external_metering_checkpoint( in_flight_requests, admitted_requests, usage_requests, - removed_web_tools, models: models.clone(), tool_names: tool_names.clone(), }; @@ -658,10 +651,6 @@ pub fn recover_external_metering_checkpoint( "tokenBasis".into(), serde_json::to_value(derived.token_basis).ok()?, ); - artifact.insert( - "removedWebTools".into(), - Value::from(counts.removed_web_tools), - ); artifact.insert( "models".into(), Value::Array(models.into_iter().map(Value::String).collect()), diff --git a/products/eval/src/measurement/gateway.rs b/products/eval/src/measurement/gateway.rs index 60754024bf6..fc88a934ebe 100644 --- a/products/eval/src/measurement/gateway.rs +++ b/products/eval/src/measurement/gateway.rs @@ -19,10 +19,10 @@ //! Local loopback HTTP metering and admission gateway for Maka Host and external subjects. //! -//! Binds `127.0.0.1:0`, authenticates per-attempt ``, strips disallowed -//! web tools from model requests, forwards upstream using the real provider credential, -//! parses streaming SSE and JSON usage/admission facts, writes atomic HMAC-signed checkpoints, -//! and enforces a synchronous `stop_accepting()` cutoff while draining in-flight requests. +//! Binds `127.0.0.1:0`, authenticates per-attempt ``, forwards request +//! payloads untouched using the real provider credential, parses streaming SSE and JSON +//! usage/admission facts, writes atomic HMAC-signed checkpoints, and enforces a synchronous +//! `stop_accepting()` cutoff while draining in-flight requests. use crate::contract::JsonObject; use crate::measurement::{ @@ -50,106 +50,34 @@ use url::Url; pub const PROVIDER_USAGE_CHECKPOINT_SCHEMA: &str = "maka.external_provider_usage.v2"; -const DISALLOWED_WEB_TOOL_NAMES: &[&str] = &["websearch", "webfetch", "fetchurl"]; - type BoxBody = http_body_util::combinators::BoxBody; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ProjectedRequestBody { - pub body: Vec, - pub removed: u64, +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ObservedRequestMetadata { pub model: Option, pub tool_names: Vec, } -/// Strips disallowed web tools (`WebSearch`, `WebFetch`, `FetchURL`) from an OpenAI or -/// Anthropic JSON request payload and extracts the requested model and remaining tool names. -pub fn remove_eval_web_tools(body: &[u8]) -> ProjectedRequestBody { +/// Inspects an OpenAI or Anthropic JSON request payload to observe the requested model and +/// exposed tool names without mutating or re-serializing the request body. +pub fn inspect_request_body(body: &[u8]) -> ObservedRequestMetadata { if body.is_empty() { - return ProjectedRequestBody { - body: Vec::new(), - removed: 0, - model: None, - tool_names: Vec::new(), - }; + return ObservedRequestMetadata::default(); } - let Ok(Value::Object(mut obj)) = serde_json::from_slice::(body) else { - return ProjectedRequestBody { - body: body.to_vec(), - removed: 0, - model: None, - tool_names: Vec::new(), - }; + let Ok(Value::Object(obj)) = serde_json::from_slice::(body) else { + return ObservedRequestMetadata::default(); }; let model = obj .get("model") .and_then(Value::as_str) .filter(|s| !s.is_empty()) .map(str::to_owned); - let Some(Value::Array(tools)) = obj.get("tools").cloned() else { - return ProjectedRequestBody { - body: body.to_vec(), - removed: 0, - model, - tool_names: Vec::new(), - }; - }; - let original_len = tools.len(); - let kept: Vec = tools - .into_iter() - .filter(|tool| !is_disallowed_web_tool(tool)) - .collect(); - let removed = (original_len - kept.len()) as u64; - let tool_names: Vec = kept.iter().flat_map(extract_tool_name).collect(); - if removed == 0 { - return ProjectedRequestBody { - body: body.to_vec(), - removed: 0, - model, - tool_names, - }; - } - obj.insert("tools".into(), Value::Array(kept)); - let projected_bytes = serde_json::to_vec(&Value::Object(obj)).unwrap_or_else(|_| body.to_vec()); - ProjectedRequestBody { - body: projected_bytes, - removed, - model, - tool_names, - } -} - -fn is_disallowed_web_tool(tool: &Value) -> bool { - let Some(obj) = tool.as_object() else { - return false; - }; - let direct_name = obj.get("name").and_then(Value::as_str); - let fn_name = obj - .get("function") - .and_then(Value::as_object) - .and_then(|f| f.get("name")) - .and_then(Value::as_str); - for candidate in [direct_name, fn_name].into_iter().flatten() { - let norm = normalize_tool_identifier(candidate); - if DISALLOWED_WEB_TOOL_NAMES.contains(&norm.as_str()) { - return true; - } - } - let Some(tool_type) = obj.get("type").and_then(Value::as_str) else { - return false; - }; - let norm_type = normalize_tool_identifier(tool_type); - DISALLOWED_WEB_TOOL_NAMES - .iter() - .any(|disallowed| norm_type == *disallowed || norm_type.starts_with(disallowed)) -} - -fn normalize_tool_identifier(value: &str) -> String { - value - .chars() - .filter(|c| c.is_ascii_alphanumeric()) - .map(|c| c.to_ascii_lowercase()) - .collect() + let tool_names = obj + .get("tools") + .and_then(Value::as_array) + .map(|tools| tools.iter().flat_map(extract_tool_name).collect()) + .unwrap_or_default(); + ObservedRequestMetadata { model, tool_names } } fn extract_tool_name(tool: &Value) -> Vec { @@ -360,11 +288,12 @@ pub fn join_upstream_url(base_url: &str, request_uri: &str) -> Result, tool_names: BTreeSet, @@ -411,7 +339,6 @@ impl GatewayCounters { in_flight_requests: self.in_flight_requests, admitted_requests: self.admitted_requests, usage_requests: self.usage_requests, - removed_web_tools: self.removed_web_tools, models: self.models.iter().cloned().collect(), tool_names: self.tool_names.iter().cloned().collect(), } @@ -553,6 +480,16 @@ impl EvalMeteringGateway { format!("http://127.0.0.1:{}/v1", self.addr.port()) } + /// Returns a loopback URL whose path matches `upstream_base_url` so provider route + /// normalization in Maka sees the exact same path structure as the upstream endpoint. + pub fn base_url_matching_upstream(&self, upstream_base_url: &str) -> String { + let suffix = Url::parse(upstream_base_url) + .ok() + .map(|u| u.path().trim_end_matches('/').to_owned()) + .unwrap_or_else(|| "/v1".to_owned()); + format!("http://127.0.0.1:{}{}", self.addr.port(), suffix) + } + pub fn gateway_credential(&self) -> &str { &self.gateway_credential } @@ -673,15 +610,14 @@ async fn handle_gateway_request( } }; - let projected = remove_eval_web_tools(&body_bytes); + let observed = inspect_request_body(&body_bytes); { let mut counters = state.counters.lock().await; - counters.removed_web_tools += projected.removed; - if let Some(m) = &projected.model { + if let Some(m) = &observed.model { counters.models.insert(m.clone()); } - for name in &projected.tool_names { + for name in &observed.tool_names { counters.tool_names.insert(name.clone()); } } @@ -700,11 +636,18 @@ async fn handle_gateway_request( let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::POST); let mut upstream_req = state.client.request(reqwest_method, &target_url); + let uses_x_api_key = + headers.contains_key("x-api-key") && !headers.contains_key(hyper::header::AUTHORIZATION); for (name, value) in &headers { let lower = name.as_str().to_ascii_lowercase(); if matches!( lower.as_str(), - "host" | "content-length" | "connection" | "authorization" | "x-api-key" + "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "authorization" + | "x-api-key" ) { continue; } @@ -712,12 +655,16 @@ async fn handle_gateway_request( upstream_req = upstream_req.header(name.as_str(), val_str); } } - upstream_req = upstream_req.header( - "authorization", - format!("Bearer {}", state.config.upstream_api_key), - ); - if !projected.body.is_empty() { - upstream_req = upstream_req.body(projected.body); + if uses_x_api_key { + upstream_req = upstream_req.header("x-api-key", &state.config.upstream_api_key); + } else { + upstream_req = upstream_req.header( + "authorization", + format!("Bearer {}", state.config.upstream_api_key), + ); + } + if !body_bytes.is_empty() { + upstream_req = upstream_req.body(body_bytes.to_vec()); } let mut upstream_resp = match upstream_req.send().await { @@ -807,7 +754,7 @@ mod tests { use crate::measurement::recover_external_metering_checkpoint; #[test] - fn remove_web_tools_strips_disallowed_tools_and_keeps_others() { + fn inspect_request_body_observes_model_and_all_tools_without_pruning() { let input = serde_json::to_vec(&json!({ "model": "deepseek-chat", "tools": [ @@ -818,16 +765,21 @@ mod tests { ] })) .unwrap(); - let projected = remove_eval_web_tools(&input); - assert_eq!(projected.removed, 3); - assert_eq!(projected.model.as_deref(), Some("deepseek-chat")); - assert_eq!(projected.tool_names, vec!["ReadFile".to_owned()]); - let parsed: Value = serde_json::from_slice(&projected.body).unwrap(); - assert_eq!(parsed["tools"].as_array().unwrap().len(), 1); + let observed = inspect_request_body(&input); + assert_eq!(observed.model.as_deref(), Some("deepseek-chat")); + assert_eq!( + observed.tool_names, + vec![ + "WebSearch".to_owned(), + "ReadFile".to_owned(), + "web_fetch".to_owned(), + "lookup".to_owned(), + ] + ); } #[tokio::test] - async fn gateway_authenticates_strips_web_tools_streams_usage_and_enforces_cutoff() { + async fn gateway_authenticates_preserves_exact_payload_streams_usage_and_enforces_cutoff() { let mut parser = UsageStreamParser::new(); parser.push(concat!( "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", @@ -848,6 +800,9 @@ mod tests { }) ); + let expected_raw_body = br#"{"model":"deepseek-chat","tools":[{"type":"function","function":{"name":"WebSearch"}},{"type":"function","function":{"name":"Bash"}}]}"#.to_vec(); + let expected_for_server = expected_raw_body.clone(); + // Start a mock upstream OpenAI-compatible SSE server. let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let upstream_addr = upstream_listener.local_addr().unwrap(); @@ -861,33 +816,34 @@ mod tests { let Ok((stream, _)) = upstream_listener.accept().await else { break; }; + let expected_bytes = expected_for_server.clone(); tokio::spawn(async move { - let service = service_fn(|req: Request| async move { - assert_eq!(req.uri().path(), "/v1/chat/completions"); - assert_eq!( - req.headers() - .get("authorization") - .and_then(|v| v.to_str().ok()), - Some("Bearer real-upstream-secret") - ); - let body = req.into_body().collect().await.unwrap().to_bytes(); - let json_body: Value = serde_json::from_slice(&body).unwrap(); - let tools = json_body["tools"].as_array().unwrap(); - assert_eq!(tools.len(), 1); - assert_eq!(tools[0]["function"]["name"], "Bash"); - - let sse = concat!( - "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", - "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":25,\"prompt_cache_hit_tokens\":60,\"completion_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":125}}\n\n", - "data: [DONE]\n\n" - ); - Ok::<_, Infallible>( - Response::builder() - .status(StatusCode::OK) - .header("content-type", "text/event-stream") - .body(Full::new(Bytes::from(sse))) - .unwrap(), - ) + let service = service_fn(move |req: Request| { + let expected_bytes = expected_bytes.clone(); + async move { + assert_eq!(req.uri().path(), "/v1/chat/completions"); + assert_eq!( + req.headers() + .get("authorization") + .and_then(|v| v.to_str().ok()), + Some("Bearer real-upstream-secret") + ); + let body = req.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body.as_ref(), expected_bytes.as_slice()); + + let sse = concat!( + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":25,\"prompt_cache_hit_tokens\":60,\"completion_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":125}}\n\n", + "data: [DONE]\n\n" + ); + Ok::<_, Infallible>( + Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/event-stream") + .body(Full::new(Bytes::from(sse))) + .unwrap(), + ) + } }); let _ = http1::Builder::new() .serve_connection(TokioIo::new(stream), service) @@ -926,21 +882,12 @@ mod tests { .unwrap(); assert_eq!(unauth.status(), reqwest::StatusCode::UNAUTHORIZED); - // 2. Authorized request strips web tools, forwards upstream, and records SSE usage. + // 2. Authorized request preserves exact request bytes & tools, forwards upstream, and records SSE usage. let resp = client .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) .header("authorization", "Bearer attempt-gateway-token") .header("content-type", "application/json") - .body( - serde_json::to_vec(&json!({ - "model": "deepseek-chat", - "tools": [ - {"type": "function", "function": {"name": "WebSearch"}}, - {"type": "function", "function": {"name": "Bash"}} - ] - })) - .unwrap(), - ) + .body(expected_raw_body) .send() .await .unwrap(); @@ -967,9 +914,11 @@ mod tests { assert_eq!(counts.requests, 1); assert_eq!(counts.admitted_requests, 1); assert_eq!(counts.usage_requests, 1); - assert_eq!(counts.removed_web_tools, 1); assert_eq!(counts.models, vec!["deepseek-chat".to_owned()]); - assert_eq!(counts.tool_names, vec!["Bash".to_owned()]); + assert_eq!( + counts.tool_names, + vec!["Bash".to_owned(), "WebSearch".to_owned()] + ); assert_eq!( counts.usage, Some(NormalizedUsage { diff --git a/products/eval/src/parity.rs b/products/eval/src/parity.rs new file mode 100644 index 00000000000..1b6b6a0a07b --- /dev/null +++ b/products/eval/src/parity.rs @@ -0,0 +1,1639 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Product parity verification gate between a normal Maka client session and `maka-eval`. +//! +//! Executes the same declared Maka executable, public configuration, workspace contents, +//! and task input through both: +//! 1. The normal product client path (`maka host init` -> `maka host serve` -> `Client` +//! directly against the model endpoint with no Eval gateway or Eval wrapper), and +//! 2. The `maka-eval` subject entry point (`RustHostSubject::execute` -> `maka-eval host-subject` +//! -> `EvalMeteringGateway`). +//! +//! Compares committed durable facts (`RunSpec`, `RequestSpec`, `AcceptedModel`, `maka inspect`) +//! and actual outbound wire HTTP payloads to prove equivalent effective instructions, +//! unpruned tool exposure and schemas, Code Mode / skill availability, compaction/continuation +//! policy, and sandbox/approval ceiling. + +use crate::contract::{ + BenchmarkSpec, ExecutionOutcome, ExecutionSpec, ExecutorSpec, ExperimentCell, ExperimentSpec, + JsonObject, SchemaVersion, SubjectKind, SubjectSpec, TaskSpec, sha256_hex, +}; +use crate::environment::{ + BoxFuture, CommandRunner, SubjectExecutionContext, SubjectProcessOutput, SubjectProcessRequest, + SubjectProcessTermination, +}; +use crate::subject::SubjectAdapter; +use crate::subject::maka::native::{ + apply_public_host_preferences, commit_client_mutation, sandbox_mode_to_workspace_access, + shutdown_host_client, start_host_client, +}; +use crate::subject::maka::{MakaSubjectConfig, RustHostSubject, bootstrap_maka_host_root}; +use http_body_util::{BodyExt, Full}; +use hyper::body::{Bytes, Incoming}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use maka_app_client::{Journal, SubscriptionEvent}; +use rusqlite::{Connection, OpenFlags, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; +use std::collections::BTreeMap; +use std::convert::Infallible; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::process::Command; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +pub const PARITY_REPORT_SCHEMA_VERSION: &str = "maka.eval.parity_report.v1"; +const ROOT_SQLITE_FILENAME: &str = "runtime-rust.sqlite"; +const DEFAULT_PARITY_TASK_INPUT: &str = "Reply with OK."; + +#[derive(Debug, Clone)] +pub struct ParityCheckInput { + pub subject_id: String, + pub subject_config: JsonObject, + pub subject_credentials: Vec, + pub workspace_source: Option, + pub task_input: String, + pub max_steps: u32, +} + +impl ParityCheckInput { + pub fn from_subject(subject: &SubjectSpec, task_input: Option<&str>, max_steps: u32) -> Self { + Self { + subject_id: subject.id.clone(), + subject_config: subject.config.clone(), + subject_credentials: subject.credentials.clone(), + workspace_source: None, + task_input: task_input + .filter(|s| !s.is_empty()) + .unwrap_or(DEFAULT_PARITY_TASK_INPUT) + .to_owned(), + max_steps: max_steps.max(1), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ParityCheckResult { + pub dimension: String, + pub passed: bool, + pub detail: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ParityMismatch { + pub dimension: String, + pub field: String, + pub normal: Value, + pub eval: Value, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ParityVerificationReport { + pub schema_version: String, + pub subject_id: String, + pub passed: bool, + pub maka_version: String, + pub maka_executable_sha256: String, + pub declared_config_sha256: String, + pub normal_tool_catalog_digest: String, + pub eval_tool_catalog_digest: String, + pub exposed_tool_names: Vec, + pub checks: Vec, + pub mismatches: Vec, +} + +/// Normalized snapshot of committed execution facts and wire model request for parity comparison. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ParitySnapshot { + pub system_prompt_text: String, + pub request_system_prompt: Option, + pub compaction_prompt: Option, + pub language: Value, + pub dynamic_context: Vec, + pub composition_sources: Vec, + pub allowed_tools: Option>, + pub tool_discovery: bool, + pub tool_selection: Value, + pub tool_mode: Value, + pub code_language: Value, + pub editing_tools: Value, + pub tool_catalog_digest: String, + pub tools: Vec, + pub collaboration: Value, + pub approval_policy: Value, + pub sandbox_policy: Value, + pub thinking_level: Value, + pub provider_options: Value, + pub max_output_tokens: Value, + pub context_policy: Value, + pub wire_path: String, + pub wire_model: Option, + pub wire_messages: Value, + pub wire_tools: Value, + pub wire_options: Value, +} + +#[derive(Debug, Clone)] +struct CapturedWireRequest { + path: String, + body: Value, +} + +struct ParityProbeServer { + addr: std::net::SocketAddr, + captured: Arc>>, + shutdown: CancellationToken, + worker: Option>, +} + +impl ParityProbeServer { + async fn start(configured_model: String) -> Result { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .map_err(|e| format!("failed to bind parity probe server: {e}"))?; + let addr = listener + .local_addr() + .map_err(|e| format!("failed to read parity probe address: {e}"))?; + let captured = Arc::new(Mutex::new(Vec::new())); + let captured_clone = captured.clone(); + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + + let worker = tokio::spawn(async move { + loop { + tokio::select! { + biased; + _ = shutdown_clone.cancelled() => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { break; }; + let store = captured_clone.clone(); + let model_id = configured_model.clone(); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let store = store.clone(); + let model_id = model_id.clone(); + async move { handle_probe_request(req, store, model_id).await } + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + } + } + }); + + Ok(Self { + addr, + captured, + shutdown, + worker: Some(worker), + }) + } + + fn base_url_for(&self, declared_base_url: &str) -> String { + let suffix = url::Url::parse(declared_base_url) + .ok() + .map(|u| u.path().trim_end_matches('/').to_owned()) + .unwrap_or_else(|| "/v1".to_owned()); + format!("http://{}{}", self.addr, suffix) + } + + async fn take_requests(&self) -> Vec { + let mut guard = self.captured.lock().await; + std::mem::take(&mut *guard) + } + + async fn close(mut self) { + self.shutdown.cancel(); + if let Some(w) = self.worker.take() { + w.abort(); + } + } +} + +async fn handle_probe_request( + req: Request, + store: Arc>>, + configured_model: String, +) -> Result>, Infallible> { + let method = req.method().clone(); + let path = req.uri().path().to_owned(); + let bytes = req + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .unwrap_or_default(); + + if method == hyper::Method::GET { + let payload = if path.ends_with("/api/tags") { + json!({ "models": [{ "name": configured_model }] }) + } else { + json!({ + "object": "list", + "data": [{ "id": configured_model, "object": "model" }] + }) + }; + let body = serde_json::to_vec(&payload).unwrap(); + return Ok(Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/json") + .body(Full::new(Bytes::from(body))) + .unwrap()); + } + + let body_json: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null); + let model = body_json + .get("model") + .and_then(Value::as_str) + .unwrap_or(&configured_model) + .to_owned(); + let stream = body_json + .get("stream") + .and_then(Value::as_bool) + .unwrap_or(true); + + if method == hyper::Method::POST { + store.lock().await.push(CapturedWireRequest { + path: path.clone(), + body: body_json, + }); + } + + if path.ends_with("/messages") { + if stream { + let sse = format!( + concat!( + "event: message_start\ndata: {{\"type\":\"message_start\",\"message\":{{\"id\":\"msg_probe\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"{model}\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{{\"input_tokens\":12,\"output_tokens\":0}}}}}}\n\n", + "event: content_block_start\ndata: {{\"type\":\"content_block_start\",\"index\":0,\"content_block\":{{\"type\":\"text\",\"text\":\"\"}}}}\n\n", + "event: content_block_delta\ndata: {{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{{\"type\":\"text_delta\",\"text\":\"OK\"}}}}\n\n", + "event: content_block_stop\ndata: {{\"type\":\"content_block_stop\",\"index\":0}}\n\n", + "event: message_delta\ndata: {{\"type\":\"message_delta\",\"delta\":{{\"stop_reason\":\"end_turn\",\"stop_sequence\":null}},\"usage\":{{\"output_tokens\":2}}}}\n\n", + "event: message_stop\ndata: {{\"type\":\"message_stop\"}}\n\n" + ), + model = model + ); + return Ok(Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/event-stream") + .body(Full::new(Bytes::from(sse))) + .unwrap()); + } + let body = serde_json::to_vec(&json!({ + "id": "msg_probe", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "OK"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 2} + })) + .unwrap(); + return Ok(Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/json") + .body(Full::new(Bytes::from(body))) + .unwrap()); + } + + if path.ends_with("/responses") { + let sse = format!( + concat!( + "event: response.created\ndata: {{\"type\":\"response.created\",\"response\":{{\"id\":\"resp_probe\",\"object\":\"response\",\"status\":\"in_progress\",\"model\":\"{model}\",\"output\":[]}}}}\n\n", + "event: response.output_item.added\ndata: {{\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{{\"id\":\"msg_1\",\"type\":\"message\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}}}\n\n", + "event: response.content_part.added\ndata: {{\"type\":\"response.content_part.added\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"part\":{{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}}}}\n\n", + "event: response.output_text.delta\ndata: {{\"type\":\"response.output_text.delta\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"delta\":\"OK\"}}\n\n", + "event: response.output_text.done\ndata: {{\"type\":\"response.output_text.done\",\"item_id\":\"msg_1\",\"output_index\":0,\"content_index\":0,\"text\":\"OK\"}}\n\n", + "event: response.output_item.done\ndata: {{\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{{\"id\":\"msg_1\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{{\"type\":\"output_text\",\"text\":\"OK\",\"annotations\":[]}}]}}}}\n\n", + "event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_probe\",\"object\":\"response\",\"status\":\"completed\",\"model\":\"{model}\",\"output\":[{{\"id\":\"msg_1\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{{\"type\":\"output_text\",\"text\":\"OK\",\"annotations\":[]}}]}}],\"usage\":{{\"input_tokens\":12,\"output_tokens\":2,\"total_tokens\":14,\"input_tokens_details\":{{\"cached_tokens\":0}},\"output_tokens_details\":{{\"reasoning_tokens\":0}}}}}}}}\n\n" + ), + model = model + ); + return Ok(Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/event-stream") + .body(Full::new(Bytes::from(sse))) + .unwrap()); + } + + if !stream { + let body = serde_json::to_vec(&json!({ + "id": "chatcmpl-probe", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 2, + "total_tokens": 14, + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0} + } + })) + .unwrap(); + return Ok(Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/json") + .body(Full::new(Bytes::from(body))) + .unwrap()); + } + + let chunk1 = json!({ + "id": "chatcmpl-probe", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "OK"}, "finish_reason": null}] + }); + let chunk2 = json!({ + "id": "chatcmpl-probe", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 2, + "total_tokens": 14, + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0} + } + }); + let sse = format!("data: {chunk1}\n\ndata: {chunk2}\n\ndata: [DONE]\n\n"); + Ok(Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/event-stream") + .body(Full::new(Bytes::from(sse))) + .unwrap()) +} + +struct LocalProcessRunner { + work_dir: PathBuf, + timeout: Duration, +} + +impl CommandRunner for LocalProcessRunner { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut cmd = Command::new(&request.command); + cmd.args(&request.args) + .current_dir(&self.work_dir) + .env_remove("MAKA_EVAL_RESULT_TOKEN") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + for (k, v) in &request.environment { + cmd.env(k, v); + } + for (target_env, source_env) in &request.credential_environment { + let val = + std::env::var(source_env).unwrap_or_else(|_| "parity-probe-credential".into()); + cmd.env(target_env, val); + } + let child = cmd + .spawn() + .map_err(|e| format!("failed to spawn {}: {e}", request.command))?; + match tokio::time::timeout(self.timeout, child.wait_with_output()).await { + Ok(Ok(out)) => Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: out.status.code().unwrap_or(1), + stdout: if request.capture_stdout { + String::from_utf8_lossy(&out.stdout).into_owned() + } else { + String::new() + }, + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + diagnostic: None, + }), + Ok(Err(e)) => Err(format!("failed waiting on {}: {e}", request.command)), + Err(_) => Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::FrameworkTimeout, + exit_code: -1, + stdout: String::new(), + stderr: String::new(), + diagnostic: None, + }), + } + }) + } +} + +/// Runs Product Parity verification for every Maka subject (or `subject_filter`) in `spec`. +pub async fn verify_experiment_parity( + spec: &ExperimentSpec, + subject_filter: Option<&str>, +) -> Result, String> { + let task_input = spec.tasks.first().map(|t| t.input.as_str()); + let max_steps = spec + .budget + .get("maxSteps") + .and_then(Value::as_u64) + .and_then(|v| u32::try_from(v).ok()) + .unwrap_or(10); + let mut reports = Vec::new(); + for subject in &spec.subjects { + if subject.kind != SubjectKind::Maka { + continue; + } + if let Some(filter) = subject_filter + && subject.id != filter + { + continue; + } + let input = ParityCheckInput::from_subject(subject, task_input, max_steps); + reports.push(verify_product_parity(&input).await?); + } + if reports.is_empty() { + return Err("no matching Maka subjects found in experiment specification".into()); + } + Ok(reports) +} + +/// Executes both the normal Maka client and `maka-eval` subject entry point under identical +/// declared configuration, workspace, and task input, returning a `ParityVerificationReport`. +pub async fn verify_product_parity( + input: &ParityCheckInput, +) -> Result { + let config = MakaSubjectConfig::decode(&input.subject_config, &input.subject_credentials) + .map_err(|e| format!("invalid Maka subject configuration: {e}"))?; + let declared_config_sha256 = config.parity_config_sha256(); + + let maka_path = PathBuf::from(&config.maka_executable); + let maka_bytes = fs::read(&maka_path).map_err(|e| { + format!( + "failed to read maka executable {}: {e}", + maka_path.display() + ) + })?; + let maka_executable_sha256 = sha256_hex(&maka_bytes); + let maka_version = read_maka_version(&maka_path).await?; + + let probe = ParityProbeServer::start(config.model.clone()).await?; + let probe_base_url = probe.base_url_for(&config.base_url); + + let temp = tempfile::tempdir().map_err(|e| format!("tempdir failed: {e}"))?; + let shared_workspace = temp.path().join("workspace"); + prepare_shared_workspace(input.workspace_source.as_deref(), &shared_workspace)?; + + let normal_root = temp.path().join("normal-root"); + let normal_journal_dir = temp.path().join("normal-client"); + fs::create_dir_all(&normal_journal_dir).map_err(|e| e.to_string())?; + + let normal_session_id = run_normal_product_client( + &config, + &probe_base_url, + &normal_root, + &normal_journal_dir, + &shared_workspace, + &input.task_input, + ) + .await; + let normal_requests = probe.take_requests().await; + + let eval_roots_dir = temp.path().join("eval-roots"); + let eval_artifacts_dir = temp.path().join("eval-artifacts"); + fs::create_dir_all(&eval_roots_dir).map_err(|e| e.to_string())?; + fs::create_dir_all(&eval_artifacts_dir).map_err(|e| e.to_string())?; + + let eval_run = run_eval_subject_entry( + &input.subject_id, + &input.subject_config, + &input.subject_credentials, + &probe_base_url, + &eval_roots_dir, + &eval_artifacts_dir, + &shared_workspace, + &input.task_input, + input.max_steps, + Duration::from_millis( + u64::from(config.host_settlement_timeout_ms) + .saturating_mul(3) + .max(30_000), + ), + ) + .await; + let eval_requests = probe.take_requests().await; + probe.close().await; + + let normal_session_id = normal_session_id?; + let (eval_root, eval_session_id) = eval_run?; + + verify_maka_inspect(&maka_path, &normal_root, &normal_session_id).await?; + verify_maka_inspect(&maka_path, &eval_root, &eval_session_id).await?; + + let normal_wire = normal_requests + .first() + .ok_or("normal Maka client did not emit a model request")?; + let eval_wire = eval_requests + .first() + .ok_or("Eval Maka entry point did not emit a model request")?; + + let normal_snapshot = extract_committed_parity_snapshot( + &normal_root, + &normal_session_id, + &shared_workspace, + normal_wire, + )?; + let eval_snapshot = extract_committed_parity_snapshot( + &eval_root, + &eval_session_id, + &shared_workspace, + eval_wire, + )?; + + let route_mismatches = check_declared_endpoint_route_parity(&config); + + Ok(compare_parity_snapshots( + &input.subject_id, + &maka_version, + &maka_executable_sha256, + &declared_config_sha256, + &normal_snapshot, + &eval_snapshot, + route_mismatches, + )) +} + +async fn read_maka_version(maka_executable: &Path) -> Result { + let out = Command::new(maka_executable) + .arg("--version") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("failed running maka --version: {e}"))?; + if !out.status.success() { + return Err(format!( + "maka --version failed with {}: {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned()) +} + +async fn verify_maka_inspect( + maka_executable: &Path, + root_path: &Path, + session_id: &str, +) -> Result<(), String> { + let out = Command::new(maka_executable) + .args(["inspect", "--root"]) + .arg(root_path) + .args(["--session", session_id]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("failed running maka inspect: {e}"))?; + if !out.status.success() { + return Err(format!( + "maka inspect failed for session {session_id}: {}", + String::from_utf8_lossy(&out.stderr).trim() + )); + } + let parsed: Value = serde_json::from_slice(&out.stdout) + .map_err(|e| format!("invalid maka inspect JSON: {e}"))?; + if parsed.get("session").and_then(Value::as_str) != Some(session_id) { + return Err("maka inspect session mismatch".into()); + } + Ok(()) +} + +fn prepare_shared_workspace(source: Option<&Path>, workspace: &Path) -> Result<(), String> { + fs::create_dir_all(workspace).map_err(|e| e.to_string())?; + if let Some(src) = source { + copy_dir_recursive(src, workspace)?; + } else { + fs::write( + workspace.join("README.md"), + b"# Parity Workspace\nSample repository file.\n", + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), String> { + fs::create_dir_all(dst).map_err(|e| e.to_string())?; + for entry in fs::read_dir(src).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let ty = entry.file_type().map_err(|e| e.to_string())?; + let target = dst.join(entry.file_name()); + if ty.is_dir() { + copy_dir_recursive(&entry.path(), &target)?; + } else if ty.is_file() { + fs::copy(entry.path(), target).map_err(|e| e.to_string())?; + } + } + Ok(()) +} + +/// Runs a session using Maka's normal public client flow directly against `probe_base_url` +/// (with no `EvalMeteringGateway` and no `maka-eval` wrapper). +async fn run_normal_product_client( + config: &MakaSubjectConfig, + probe_base_url: &str, + root_path: &Path, + journal_dir: &Path, + workspace_path: &Path, + task_input: &str, +) -> Result { + let deadline = Duration::from_millis(u64::from(config.host_settlement_timeout_ms)); + let maka_executable = Path::new(&config.maka_executable); + let api_key = std::env::var(&config.api_key_environment) + .unwrap_or_else(|_| "parity-probe-credential".into()); + let init_payload = config.host_init_settings(probe_base_url, api_key); + + tokio::time::timeout( + deadline, + bootstrap_maka_host_root(maka_executable, root_path, &init_payload), + ) + .await + .map_err(|_| "normal Maka Host initialization timed out".to_owned())??; + + let journal = Journal::open(&journal_dir.join("client-journal.sqlite")).await?; + let (mut child, client) = start_host_client( + maka_executable, + root_path, + u64::from(config.host_settlement_timeout_ms), + journal.clone(), + ) + .await?; + + let workspace_str = workspace_path.to_string_lossy().into_owned(); + let configured = + apply_public_host_preferences(&workspace_str, &config.session_settings, &client).await; + let cleanup = shutdown_host_client(&mut child, &client, deadline).await; + configured?; + cleanup?; + + let (mut child, client) = start_host_client( + maka_executable, + root_path, + u64::from(config.host_settlement_timeout_ms), + journal, + ) + .await?; + + let session_res = async { + let connections = client + .query("settings", "connections", json!({"kind": "start"})) + .await?; + let connection = connections["connections"] + .as_array() + .and_then(|items| items.iter().find(|c| c["slug"] == config.connection_slug)) + .ok_or("normal Maka connection unavailable")?; + let backend = json!({ + "kind": "model", + "connectionId": connection["basis"]["connectionId"], + "modelId": config.model, + "thinkingLevel": config.session_settings.thinking_level, + }); + let created = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "create", + json!({ + "title": "Task", + "automaticTitle": false, + "labels": [], + "backend": backend, + "workspace": { + "kind": "directory", + "path": workspace_str, + }, + "tools": Value::Null, + }), + ) + .await?, + ) + .await?; + if created["kind"] != "created" { + return Err(format!("normal Maka session creation rejected: {created}")); + } + let session_obj = &created["session"]; + let session_id = session_obj["id"] + .as_str() + .ok_or("normal Maka session id missing")? + .to_owned(); + + if let Some(request_permissions) = config.session_settings.request_permissions { + let access = match &config.session_settings.sandbox_mode { + Some(mode) => sandbox_mode_to_workspace_access(mode)?.to_owned(), + None => session_obj["permissions"]["access"] + .as_str() + .unwrap_or("workspace_write") + .to_owned(), + }; + let revision = session_obj["revision"] + .as_u64() + .ok_or("normal session revision missing")?; + let updated = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "metadata", + json!({ + "sessionId": session_id, + "expectedRevision": revision, + "title": "Task", + "labels": [], + "permissions": { + "settings": { + "access": access, + "requestPermissions": request_permissions, + }, + "revokeGrants": false, + }, + }), + ) + .await?, + ) + .await?; + if updated["kind"] != "committed" { + return Err("normal Maka session permission update rejected".into()); + } + } + + let draft = client + .save_draft( + "agent", + &session_id, + "composer", + None, + task_input.to_owned().into(), + ) + .await?; + let mutation = client + .reserve_send_draft( + Uuid::new_v4(), + "agent", + &session_id, + "composer", + draft.revision, + json!({"backend": backend}), + ) + .await? + .open() + .await?; + let watch = client + .subscribe("agent", "watch", json!({"sessionId": session_id, "after": 0})) + .await?; + let accepted = commit_client_mutation(mutation).await?; + let run_id = match accepted["kind"].as_str() { + Some("started") => accepted["runId"] + .as_str() + .ok_or("normal Maka runId missing")? + .to_owned(), + Some("queued") => { + let msg_id = accepted["messageId"] + .as_str() + .ok_or("normal Maka queued messageId missing")?; + let mut wait_interval = tokio::time::interval(Duration::from_millis(50)); + loop { + wait_interval.tick().await; + let work = client + .query( + "agent", + "work", + json!({ + "sessionId": session_id, + "target": {"kind": "message", "messageId": msg_id}, + "cursor": null + }), + ) + .await?; + match work["kind"].as_str() { + Some("delivered") => { + break work["execution"]["runId"] + .as_str() + .ok_or("normal Maka delivered runId missing")? + .to_owned(); + } + Some("pending") => {} + _ => return Err(format!("normal Maka queued message failed: {work}")), + } + } + } + _ => return Err(format!("normal Maka send rejected: {accepted}")), + }; + + let mut interval = tokio::time::interval(Duration::from_millis(50)); + let mut next = Box::pin(watch.next()); + loop { + tokio::select! { + ev = &mut next => { + match ev? { + Some(SubscriptionEvent::Item(_)) => { + next = Box::pin(watch.next()); + } + _ => return Err("normal Maka watch ended prematurely".into()), + } + } + _ = interval.tick() => { + let exec = client + .query("agent", "execution", json!({"sessionId": session_id, "runId": run_id})) + .await?; + match exec["execution"]["state"]["kind"].as_str() { + Some("completed") => break, + Some("failed" | "cancelled") => { + return Err(format!("normal Maka run did not complete: {exec}")); + } + _ => {} + } + } + } + } + drop(next); + watch.close().await?; + Ok::(session_id) + } + .await; + + let cleanup = shutdown_host_client(&mut child, &client, deadline).await; + let session_id = session_res?; + cleanup?; + Ok(session_id) +} + +#[expect( + clippy::too_many_arguments, + reason = "parity entry-point harness binds subject config, probe endpoint, isolated roots, artifacts, workspace, task input, step budget, and timeout explicitly" +)] +async fn run_eval_subject_entry( + subject_id: &str, + subject_config: &JsonObject, + subject_credentials: &[String], + probe_base_url: &str, + eval_roots_dir: &Path, + eval_artifacts_dir: &Path, + workspace_path: &Path, + task_input: &str, + max_steps: u32, + timeout: Duration, +) -> Result<(PathBuf, String), String> { + let mut cfg = subject_config.clone(); + cfg.insert("baseUrl".into(), Value::String(probe_base_url.to_owned())); + cfg.insert( + "runtimeHostsPath".into(), + Value::String(eval_roots_dir.to_string_lossy().into_owned()), + ); + + let spec = ExperimentSpec { + schema_version: SchemaVersion::V2, + id: "parity-check".into(), + benchmark: BenchmarkSpec { + id: "parity".into(), + version: "1".into(), + config: Map::new(), + }, + executor: ExecutorSpec { + kind: "local".into(), + config: Map::new(), + }, + execution: ExecutionSpec::default(), + subjects: vec![SubjectSpec { + id: subject_id.to_owned(), + kind: SubjectKind::Maka, + credentials: subject_credentials.to_vec(), + config: cfg.clone(), + }], + tasks: vec![TaskSpec { + id: "parity-task".into(), + input: task_input.to_owned(), + config: Map::new(), + }], + repetitions: 1, + budget: json!({"maxSteps": max_steps}).as_object().unwrap().clone(), + verifier: Map::new(), + }; + let cell = ExperimentCell { + id: format!("parity-task::1::{subject_id}"), + experiment_id: spec.id.clone(), + benchmark: spec.benchmark, + executor: spec.executor, + subject: spec.subjects[0].clone(), + task: spec.tasks[0].clone(), + repetition: 1, + budget: spec.budget, + verifier: spec.verifier, + }; + + let artifact_root = eval_artifacts_dir.join("maka-runtime-host"); + let mut metadata = Map::new(); + metadata.insert( + "artifactRoot".into(), + Value::String(artifact_root.to_string_lossy().into_owned()), + ); + + let runner = Arc::new(LocalProcessRunner { + work_dir: eval_artifacts_dir.to_path_buf(), + timeout, + }); + let context = SubjectExecutionContext { + cwd: workspace_path.to_string_lossy().into_owned(), + task_input: task_input.to_owned(), + metadata, + cancel: CancellationToken::new(), + runner, + }; + + let subject = RustHostSubject::new(); + subject + .validate(&cell) + .map_err(|e| format!("Eval subject validation failed: {e}"))?; + let exec_result = subject.execute(&cell, context).await; + if exec_result.outcome != ExecutionOutcome::Completed { + return Err(format!( + "Eval Maka entry point failed with {:?}: {:?}", + exec_result.outcome, exec_result.failure_reason + )); + } + + let receipt_bytes = fs::read(artifact_root.join("execution-receipt.json")) + .map_err(|e| format!("failed reading Eval execution-receipt.json: {e}"))?; + let receipt: Value = + serde_json::from_slice(&receipt_bytes).map_err(|e| format!("invalid receipt JSON: {e}"))?; + let session_id = receipt["sessionId"] + .as_str() + .ok_or("Eval receipt missing sessionId")? + .to_owned(); + let root_path = receipt["rootPath"] + .as_str() + .map(PathBuf::from) + .ok_or("Eval receipt missing rootPath")?; + + Ok((root_path, session_id)) +} + +fn read_content_json(conn: &Connection, content_ref: &Value) -> Result { + let digest = content_ref + .get("digest") + .and_then(Value::as_str) + .ok_or("ContentRef missing digest")?; + let bytes: Vec = conn + .query_row( + "SELECT payload FROM contents WHERE content_sha256 = ?", + [digest], + |row| row.get(0), + ) + .optional() + .map_err(|e| format!("SQLite content read failed: {e}"))? + .ok_or_else(|| format!("missing content payload for digest {digest}"))?; + serde_json::from_slice(&bytes).map_err(|e| format!("invalid JSON in content {digest}: {e}")) +} + +fn extract_committed_parity_snapshot( + root_path: &Path, + session_id: &str, + workspace_path: &Path, + wire: &CapturedWireRequest, +) -> Result { + let db_path = root_path.join(ROOT_SQLITE_FILENAME); + let conn = Connection::open_with_flags(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| format!("failed opening {}: {e}", db_path.display()))?; + + let (run_id, spec_bytes): (String, Vec) = conn + .query_row( + "SELECT s.run, c.payload FROM agent_run_specs s JOIN contents c ON c.id = s.spec_content_id WHERE s.session = ? ORDER BY s.rowid ASC LIMIT 1", + [session_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|e| format!("failed reading RunSpec for session {session_id}: {e}"))?; + + let run_spec: Value = + serde_json::from_slice(&spec_bytes).map_err(|e| format!("invalid RunSpec JSON: {e}"))?; + + let accepted_model = read_content_json(&conn, &run_spec["backend"]["descriptor"])?; + let connection_id = accepted_model + .pointer("/binding/connectionId") + .or_else(|| accepted_model.pointer("/binding/connection_id")) + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + + let canonical_workspace = + fs::canonicalize(workspace_path).unwrap_or_else(|_| workspace_path.to_path_buf()); + let canonical_root = fs::canonicalize(root_path).unwrap_or_else(|_| root_path.to_path_buf()); + let connection_namespace = camel_case_hyphen_letters(&connection_id); + let normalizer = EphemeralNormalizer { + session_id: session_id.to_owned(), + connection_id, + connection_namespace, + workspace_raw: workspace_path.to_string_lossy().into_owned(), + workspace_canonical: canonical_workspace.to_string_lossy().into_owned(), + root_raw: root_path.to_string_lossy().into_owned(), + root_canonical: canonical_root.to_string_lossy().into_owned(), + }; + + let system_prompt_blob = read_content_json(&conn, &run_spec["systemPrompt"])?; + let system_prompt_text = system_prompt_blob["text"] + .as_str() + .ok_or("SystemPrompt.text missing")? + .to_owned(); + + let compaction_prompt = match run_spec.get("compactionPrompt") { + Some(v) if !v.is_null() => { + let digest = v["digest"] + .as_str() + .ok_or("compactionPrompt digest missing")?; + let raw: Vec = conn + .query_row( + "SELECT payload FROM contents WHERE content_sha256 = ?", + [digest], + |r| r.get(0), + ) + .map_err(|e| e.to_string())?; + let decoded: Value = serde_json::from_slice(&raw) + .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(&raw).into_owned())); + Some( + decoded + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| decoded.to_string()), + ) + } + _ => None, + }; + + let tool_selection = normalize_tool_selection( + &normalizer.normalize_value(&read_content_json(&conn, &run_spec["toolSelection"])?), + ); + + let allowed_tools = match run_spec.get("allowedTools") { + None | Some(Value::Null) => None, + Some(Value::Array(arr)) => Some( + arr.iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + ), + Some(other) => return Err(format!("unexpected allowedTools: {other}")), + }; + + let mut stmt = conn + .prepare( + "SELECT c.payload FROM agent_facts f JOIN contents c ON c.id = f.body_content_id WHERE f.run = ? ORDER BY f.revision ASC", + ) + .map_err(|e| e.to_string())?; + let fact_rows = stmt + .query_map([&run_id], |row| row.get::<_, Vec>(0)) + .map_err(|e| e.to_string())?; + + let mut request_spec: Option = None; + for row in fact_rows { + let bytes = row.map_err(|e| e.to_string())?; + let recorded: Value = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + if recorded.pointer("/fact/kind").and_then(Value::as_str) == Some("step_prepared") { + let req_ref = &recorded["fact"]["request"]; + request_spec = Some(read_content_json(&conn, req_ref)?); + break; + } + } + let request_spec = + request_spec.ok_or("Session Run has no committed step_prepared RequestSpec")?; + let composition = &request_spec["composition"]; + let provenance = read_content_json(&conn, &request_spec["provenance"])?; + + let request_system_prompt = composition + .get("systemPrompt") + .and_then(Value::as_str) + .map(|s| normalizer.normalize_str(s)); + + let dynamic_context = composition["dynamicContext"] + .as_array() + .ok_or("RequestComposition.dynamicContext missing")? + .iter() + .filter_map(Value::as_str) + .map(|s| normalizer.normalize_str(s)) + .collect(); + + let composition_sources = composition["sources"] + .as_array() + .ok_or("RequestComposition.sources missing")? + .iter() + .map(|src| { + let mut obj = src.as_object().cloned().unwrap_or_default(); + obj.remove("activation"); + if matches!( + obj.get("kind").and_then(Value::as_str), + Some("input" | "prompt_context" | "prompt_variable") + ) { + obj.remove("revision"); + obj.remove("entryId"); + } + normalizer.normalize_value(&Value::Object(obj)) + }) + .collect(); + + let tool_catalog_digest = composition["toolCatalogDigest"] + .as_str() + .ok_or("RequestComposition.toolCatalogDigest missing")? + .to_owned(); + + let tools = composition["tools"] + .as_array() + .ok_or("RequestComposition.tools missing")? + .iter() + .map(|t| normalizer.normalize_value(t)) + .collect(); + + let sandbox_policy = normalizer.normalize_value(&run_spec["permissionCeiling"]["sandbox"]); + let approval_policy = normalizer.normalize_value(&run_spec["permissionCeiling"]["approval"]); + + let context_policy = normalizer.normalize_value(&json!({ + "modelInputPolicy": provenance.get("modelInputPolicy").cloned().unwrap_or(Value::Null), + "contextWindow": provenance.get("contextWindow").cloned().unwrap_or(Value::Null), + "modelContextWindow": provenance.get("modelContextWindow").cloned().unwrap_or(Value::Null), + "declaredWindow": provenance.get("declaredWindow").cloned().unwrap_or(Value::Null), + "publicModel": accepted_model.get("public_model").cloned().unwrap_or(Value::Null), + "modelOverride": accepted_model.get("model_override").cloned().unwrap_or(Value::Null), + "bodyOverlay": accepted_model.get("body_overlay").cloned().unwrap_or(Value::Null), + })); + + let wire_normalized = normalizer.normalize_value(&wire.body); + let mut wire_rest = wire_normalized.as_object().cloned().unwrap_or_default(); + let wire_model = wire_rest + .remove("model") + .and_then(|v| v.as_str().map(str::to_owned)); + let wire_tools = wire_rest.remove("tools").unwrap_or(Value::Null); + let wire_messages = json!({ + "system": wire_rest.remove("system").unwrap_or(Value::Null), + "messages": wire_rest.remove("messages").unwrap_or(Value::Null), + "input": wire_rest.remove("input").unwrap_or(Value::Null), + "instructions": wire_rest.remove("instructions").unwrap_or(Value::Null), + }); + + Ok(ParitySnapshot { + system_prompt_text: normalizer.normalize_str(&system_prompt_text), + request_system_prompt, + compaction_prompt: compaction_prompt.map(|s| normalizer.normalize_str(&s)), + language: run_spec.get("language").cloned().unwrap_or(Value::Null), + dynamic_context, + composition_sources, + allowed_tools, + tool_discovery: run_spec["toolDiscovery"].as_bool().unwrap_or(false), + tool_selection, + tool_mode: accepted_model + .get("tool_mode") + .cloned() + .unwrap_or_else(|| Value::String("code_mode".into())), + code_language: accepted_model + .get("code_language") + .cloned() + .unwrap_or_else(|| Value::String("javascript".into())), + editing_tools: accepted_model + .get("editing_tools") + .cloned() + .unwrap_or(Value::Null), + tool_catalog_digest, + tools, + collaboration: run_spec + .get("collaboration") + .cloned() + .unwrap_or(Value::Null), + approval_policy, + sandbox_policy, + thinking_level: accepted_model + .get("thinking_level") + .cloned() + .unwrap_or(Value::Null), + provider_options: normalizer.normalize_value( + &composition + .get("providerOptions") + .cloned() + .unwrap_or(Value::Null), + ), + max_output_tokens: composition + .get("maxOutputTokens") + .cloned() + .unwrap_or(Value::Null), + context_policy, + wire_path: wire.path.clone(), + wire_model, + wire_messages, + wire_tools, + wire_options: Value::Object(wire_rest), + }) +} + +struct EphemeralNormalizer { + session_id: String, + connection_id: String, + connection_namespace: String, + workspace_raw: String, + workspace_canonical: String, + root_raw: String, + root_canonical: String, +} + +fn camel_case_hyphen_letters(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let mut chars = input.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '-' + && let Some(&next) = chars.peek() + && next.is_ascii_alphabetic() + { + out.push(next.to_ascii_uppercase()); + chars.next(); + } else { + out.push(ch); + } + } + out +} + +fn normalize_tool_selection(value: &Value) -> Value { + let mut cloned = value.clone(); + if let Some(services) = cloned.get_mut("services").and_then(Value::as_array_mut) { + for item in services { + if let Some(obj) = item.as_object_mut() + && obj + .get("binding") + .and_then(Value::as_str) + .is_some_and(is_sha256_prefixed_hex) + { + obj.insert( + "binding".into(), + Value::String("".into()), + ); + } + } + } + cloned +} + +fn is_sha256_prefixed_hex(s: &str) -> bool { + s.strip_prefix("sha256:") + .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())) +} + +fn replace_skill_revision_tokens(input: &str) -> String { + const PREFIX: &str = "revision = sha256:"; + let mut out = String::with_capacity(input.len()); + let mut rest = input; + while let Some(idx) = rest.find(PREFIX) { + out.push_str(&rest[..idx]); + out.push_str(PREFIX); + let after = &rest[idx + PREFIX.len()..]; + if after.len() >= 64 && after.as_bytes()[..64].iter().all(u8::is_ascii_hexdigit) { + out.push_str(""); + rest = &after[64..]; + } else { + rest = after; + } + } + out.push_str(rest); + out +} + +impl EphemeralNormalizer { + fn normalize_str(&self, input: &str) -> String { + let mut out = + replace_skill_revision_tokens(&input.replace(&self.session_id, "")); + for (candidate, replacement) in [ + (&self.connection_id, ""), + (&self.connection_namespace, ""), + (&self.workspace_canonical, ""), + (&self.workspace_raw, ""), + (&self.root_canonical, ""), + (&self.root_raw, ""), + ] { + if !candidate.is_empty() { + out = out.replace(candidate, replacement); + } + } + out + } + + fn normalize_value(&self, value: &Value) -> Value { + match value { + Value::String(s) => Value::String(self.normalize_str(s)), + Value::Array(items) => { + Value::Array(items.iter().map(|v| self.normalize_value(v)).collect()) + } + Value::Object(map) => { + let mut out = BTreeMap::new(); + for (k, v) in map { + out.insert(self.normalize_str(k), self.normalize_value(v)); + } + Value::Object(out.into_iter().collect()) + } + other => other.clone(), + } + } +} + +/// Checks whether routing `config.base_url` through the local loopback metering gateway +/// would silently alter URL-dependent Maka policy relative to the declared upstream URL. +pub fn check_declared_endpoint_route_parity(config: &MakaSubjectConfig) -> Vec { + let mut mismatches = Vec::new(); + let apply_patch_override = config + .model_override + .as_ref() + .and_then(|ov| ov.get("applyPatch")) + .and_then(Value::as_bool); + + let declared_apply_patch = + effective_apply_patch(&config.model, &config.base_url, apply_patch_override); + let loopback_apply_patch = + effective_apply_patch(&config.model, "http://127.0.0.1/v1", apply_patch_override); + if declared_apply_patch != loopback_apply_patch { + mismatches.push(ParityMismatch { + dimension: "tool_exposure".into(), + field: "declaredEndpointEditingTools".into(), + normal: Value::String(if declared_apply_patch { + "apply_patch".into() + } else { + "structured".into() + }), + eval: Value::String(if loopback_apply_patch { + "apply_patch".into() + } else { + "structured".into() + }), + }); + } + mismatches +} + +fn effective_apply_patch(model_id: &str, request_url: &str, enabled: Option) -> bool { + enabled.unwrap_or_else(|| { + let lower = model_id.to_ascii_lowercase(); + lower.contains("gpt-") + || lower.contains("deepseek") + || url::Url::parse(request_url).ok().is_some_and(|u| { + matches!(u.scheme(), "http" | "https") && u.host_str() == Some("api.deepseek.com") + }) + }) +} + +/// Compares normalized snapshots from the normal Maka client and the `maka-eval` entry point. +pub fn compare_parity_snapshots( + subject_id: &str, + maka_version: &str, + maka_executable_sha256: &str, + declared_config_sha256: &str, + normal: &ParitySnapshot, + eval: &ParitySnapshot, + mut mismatches: Vec, +) -> ParityVerificationReport { + let mut check_dim = |dimension: &str, + comparisons: &[(&str, Value, Value)], + detail_ok: &str| + -> ParityCheckResult { + let before = mismatches.len(); + for (field, lhs, rhs) in comparisons { + if lhs != rhs { + mismatches.push(ParityMismatch { + dimension: dimension.to_owned(), + field: (*field).to_owned(), + normal: lhs.clone(), + eval: rhs.clone(), + }); + } + } + let dim_failed = mismatches + .iter() + .skip(before) + .any(|m| m.dimension == dimension) + || mismatches[..before] + .iter() + .any(|m| m.dimension == dimension); + ParityCheckResult { + dimension: dimension.to_owned(), + passed: !dim_failed, + detail: if dim_failed { + format!("{dimension} drifted between normal Maka client and Eval entry point") + } else { + detail_ok.to_owned() + }, + } + }; + + let composition_check = check_dim( + "product_composition", + &[( + "compositionSources", + Value::Array(normal.composition_sources.clone()), + Value::Array(eval.composition_sources.clone()), + )], + "System prompt, adapter, skill, and tool composition sources come from identical released product packages", + ); + + let instructions_check = check_dim( + "effective_instructions", + &[ + ( + "runSpecSystemPrompt", + Value::String(normal.system_prompt_text.clone()), + Value::String(eval.system_prompt_text.clone()), + ), + ( + "requestCompositionSystemPrompt", + serde_json::to_value(&normal.request_system_prompt).unwrap(), + serde_json::to_value(&eval.request_system_prompt).unwrap(), + ), + ( + "compactionPrompt", + serde_json::to_value(&normal.compaction_prompt).unwrap(), + serde_json::to_value(&eval.compaction_prompt).unwrap(), + ), + ("language", normal.language.clone(), eval.language.clone()), + ( + "dynamicContext", + serde_json::to_value(&normal.dynamic_context).unwrap(), + serde_json::to_value(&eval.dynamic_context).unwrap(), + ), + ( + "wireMessages", + normal.wire_messages.clone(), + eval.wire_messages.clone(), + ), + ], + "System prompts, compaction instructions, dynamic workspace context, and outbound prompt messages match", + ); + + let tools_check = check_dim( + "tool_exposure", + &[ + ( + "allowedToolsUnpruned", + serde_json::to_value(&normal.allowed_tools).unwrap(), + Value::Null, + ), + ( + "evalAllowedToolsUnpruned", + Value::Null, + serde_json::to_value(&eval.allowed_tools).unwrap(), + ), + ( + "toolDiscovery", + Value::Bool(normal.tool_discovery), + Value::Bool(eval.tool_discovery), + ), + ( + "toolSelection", + normal.tool_selection.clone(), + eval.tool_selection.clone(), + ), + ("toolMode", normal.tool_mode.clone(), eval.tool_mode.clone()), + ( + "codeLanguage", + normal.code_language.clone(), + eval.code_language.clone(), + ), + ( + "editingTools", + normal.editing_tools.clone(), + eval.editing_tools.clone(), + ), + ( + "toolCatalogDigest", + Value::String(normal.tool_catalog_digest.clone()), + Value::String(eval.tool_catalog_digest.clone()), + ), + ( + "requestCompositionTools", + Value::Array(normal.tools.clone()), + Value::Array(eval.tools.clone()), + ), + ( + "wireTools", + normal.wire_tools.clone(), + eval.wire_tools.clone(), + ), + ], + "Full unpruned toolset, schemas, descriptions, tool discovery, Code Mode, and built-in skills match", + ); + + let policies_check = check_dim( + "execution_policies", + &[ + ( + "collaborationMode", + normal.collaboration.clone(), + eval.collaboration.clone(), + ), + ( + "approvalPolicy", + normal.approval_policy.clone(), + eval.approval_policy.clone(), + ), + ( + "sandboxPolicy", + normal.sandbox_policy.clone(), + eval.sandbox_policy.clone(), + ), + ( + "thinkingLevel", + normal.thinking_level.clone(), + eval.thinking_level.clone(), + ), + ( + "providerOptions", + normal.provider_options.clone(), + eval.provider_options.clone(), + ), + ( + "maxOutputTokens", + normal.max_output_tokens.clone(), + eval.max_output_tokens.clone(), + ), + ( + "contextAndCompactionPolicy", + normal.context_policy.clone(), + eval.context_policy.clone(), + ), + ( + "wirePath", + Value::String(normal.wire_path.clone()), + Value::String(eval.wire_path.clone()), + ), + ( + "wireModel", + serde_json::to_value(&normal.wire_model).unwrap(), + serde_json::to_value(&eval.wire_model).unwrap(), + ), + ( + "wireOptions", + normal.wire_options.clone(), + eval.wire_options.clone(), + ), + ], + "Sandbox ceiling, approval policy, context/compaction window, reasoning level, and wire options match", + ); + + let exposed_tool_names: Vec = eval + .tools + .iter() + .filter_map(|t| t.get("name").and_then(Value::as_str).map(str::to_owned)) + .collect(); + + let checks = vec![ + composition_check, + instructions_check, + tools_check, + policies_check, + ]; + let passed = mismatches.is_empty() && checks.iter().all(|c| c.passed); + + ParityVerificationReport { + schema_version: PARITY_REPORT_SCHEMA_VERSION.into(), + subject_id: subject_id.to_owned(), + passed, + maka_version: maka_version.to_owned(), + maka_executable_sha256: maka_executable_sha256.to_owned(), + declared_config_sha256: declared_config_sha256.to_owned(), + normal_tool_catalog_digest: normal.tool_catalog_digest.clone(), + eval_tool_catalog_digest: eval.tool_catalog_digest.clone(), + exposed_tool_names, + checks, + mismatches, + } +} diff --git a/products/eval/src/report.rs b/products/eval/src/report.rs index 1a8aba41b9f..80c9d879cdd 100644 --- a/products/eval/src/report.rs +++ b/products/eval/src/report.rs @@ -22,14 +22,17 @@ use crate::contract::{ DifferenceRegistry, DifferenceScenarioId, EvalResultStatus, ExperimentSpec, RunFingerprint, - V1CellAttempt, V2AttemptRecord, decode_v1_cell_attempt, expand_experiment, format_cell_id, - is_replaceable_attempt, select_cell_result, + SubjectKind, V1CellAttempt, V2AttemptRecord, decode_v1_cell_attempt, expand_experiment, + format_cell_id, is_replaceable_attempt, select_cell_result, sha256_hex, }; use crate::measurement::{NormalizedUsage, costs_match_within_tolerance}; +use crate::parity::{PARITY_REPORT_SCHEMA_VERSION, ParityVerificationReport}; use crate::store::{FileAttemptStore, ReconstructedStoreState, StoreError}; +use crate::subject::maka::MakaSubjectConfig; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::BTreeMap; +use std::path::Path; /// Reconstructs all `v1` and `v2` attempt records deterministically from the append-only journal. pub fn rebuild_from_journal( @@ -200,8 +203,9 @@ pub fn build_cohort_summary( pub struct DifferentialReport { /// 1. Whether the candidate JSON parses strictly as a valid `maka.eval.v1` attempt. pub v1_parses: bool, - /// 2. Whether the candidate matches the oracle (either byte/tolerance-exact or via an adjudicated difference scenario). + /// 2. Whether the candidate actually matches the oracle on compared result fields. pub v1_matches_oracle: bool, + pub adjudicated_difference: bool, /// 3. Whether the experimental conditions (semantic fingerprint and any adjudicated difference) remain comparable. pub conditions_comparable: bool, pub adjudicated_scenario: Option, @@ -224,6 +228,7 @@ pub fn evaluate_differential_oracle( return DifferentialReport { v1_parses: false, v1_matches_oracle: false, + adjudicated_difference: false, conditions_comparable: false, adjudicated_scenario, diagnostics, @@ -243,20 +248,104 @@ pub fn evaluate_differential_oracle( } let direct_match = attempts_equivalent(&candidate, oracle, &mut diagnostics); - let matches_oracle = direct_match || scenario_entry.is_some(); + let applicable = adjudicated_scenario.is_none_or(|id| scenario_applies(id, &candidate)); + if !applicable { + diagnostics + .push("candidate does not establish the selected scenario's prerequisites".into()); + } + let adjudicated_difference = !direct_match + && applicable + && scenario_entry.is_some() + && allowed_scenario_difference(adjudicated_scenario.unwrap(), &candidate, oracle); let conditions_comparable = fingerprint_comparable - && matches_oracle + && applicable + && (direct_match || adjudicated_difference) && scenario_entry.is_none_or(|entry| entry.conditions_comparable); DifferentialReport { v1_parses: true, - v1_matches_oracle: matches_oracle, + v1_matches_oracle: direct_match, + adjudicated_difference, conditions_comparable, adjudicated_scenario, diagnostics, } } +fn has_artifact(attempt: &V1CellAttempt, kind: &str) -> bool { + attempt + .result + .artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some(kind)) +} + +fn scenario_applies(id: DifferenceScenarioId, candidate: &V1CellAttempt) -> bool { + let result = &candidate.result; + match id { + DifferenceScenarioId::TimeoutWithRewardIncompleteUsage => { + result.status == EvalResultStatus::SubjectFailed + && result.score.is_some() + && result.usage.is_some() + && result.cost_usd.is_none() + && result + .failure_reason + .as_deref() + .is_some_and(|r| r.contains("timeout")) + } + DifferenceScenarioId::AuditTruncation => result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("egress-audit") + && a.get("truncated").and_then(Value::as_bool) == Some(true) + }), + DifferenceScenarioId::MissingCriticalArtifacts => { + result.status == EvalResultStatus::InfraFailed + && result.score.is_none() + && result + .failure_reason + .as_deref() + .is_some_and(|r| r.starts_with("critical artifacts missing: ")) + } + DifferenceScenarioId::LateCancellation => { + has_artifact(candidate, "cancelled-after-verification") + && matches!( + result.status, + EvalResultStatus::Completed | EvalResultStatus::SubjectFailed + ) + && result.score.is_some() + } + DifferenceScenarioId::EvidenceRecoveredLater => { + has_artifact(candidate, "post-hoc-evidence-recovery") + } + } +} + +fn allowed_scenario_difference( + id: DifferenceScenarioId, + candidate: &V1CellAttempt, + oracle: &V1CellAttempt, +) -> bool { + // The first three scenarios preserve v1 result semantics. Post-hoc numeric + // changes require original checkpoint evidence, which this v1-only API lacks. + if id != DifferenceScenarioId::LateCancellation + || oracle.result.status != EvalResultStatus::Indeterminate + || oracle.result.score.is_some() + || oracle.result.failure_reason.as_deref() + != Some("executor cancelled before verification completed") + { + return false; + } + let mut expected = oracle.clone(); + expected.result.status = candidate.result.status; + expected.result.score = candidate.result.score; + expected.result.failure_reason = candidate.result.failure_reason.clone(); + let mut actual = candidate.clone(); + actual + .result + .artifacts + .retain(|a| a.get("kind").and_then(Value::as_str) != Some("cancelled-after-verification")); + attempts_equivalent(&actual, &expected, &mut Vec::new()) +} + fn attempts_equivalent( candidate: &V1CellAttempt, oracle: &V1CellAttempt, @@ -302,6 +391,10 @@ fn attempts_equivalent( )); ok = false; } + if candidate.result.artifacts != oracle.result.artifacts { + diagnostics.push("artifacts mismatch".into()); + ok = false; + } if candidate.result.failure_reason != oracle.result.failure_reason { diagnostics.push(format!( "failureReason mismatch: {:?} != {:?}", @@ -311,3 +404,147 @@ fn attempts_equivalent( } ok } + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MilestoneGate { + M3Acceptance, + M5ProductionDelivery, +} + +impl MilestoneGate { + pub fn parse(value: &str) -> Option { + match value { + "m3" | "m3_acceptance" => Some(Self::M3Acceptance), + "m5" | "m5_production_delivery" => Some(Self::M5ProductionDelivery), + _ => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MilestoneGateReport { + pub milestone: MilestoneGate, + pub passed: bool, + pub run_fingerprint: RunFingerprint, + pub parity_verified_subjects: Vec, + pub violations: Vec, +} + +/// Evaluates whether an experiment satisfies the M3 acceptance or M5 production delivery gate. +/// +/// Product parity is a mandatory prerequisite for both M3 and M5: every `maka` subject in `spec` +/// must use valid public product interfaces and carry a passing `ParityVerificationReport` matching +/// its declared public configuration digest (and binary digest when the executable is present). +/// `RunFingerprint` documents run conditions only and never substitutes for parity verification. +pub fn evaluate_milestone_gate( + milestone: MilestoneGate, + spec: &ExperimentSpec, + canonical_v1: &BTreeMap, + parity_reports: &[ParityVerificationReport], +) -> MilestoneGateReport { + let mut violations = Vec::new(); + let mut parity_verified_subjects = Vec::new(); + let run_fingerprint = RunFingerprint::compute(spec); + + let maka_subjects: Vec<_> = spec + .subjects + .iter() + .filter(|s| s.kind == SubjectKind::Maka) + .collect(); + if maka_subjects.is_empty() { + violations.push("experiment contains no Maka subject to evaluate".into()); + } + + for subject in maka_subjects { + let decoded = match MakaSubjectConfig::decode(&subject.config, &subject.credentials) { + Ok(cfg) => cfg, + Err(err) => { + violations.push(format!( + "subject '{}' violates public Maka configuration contract: {err}", + subject.id + )); + continue; + } + }; + let expected_cfg_sha = decoded.parity_config_sha256(); + let Some(report) = parity_reports.iter().find(|r| r.subject_id == subject.id) else { + violations.push(format!( + "subject '{}' is missing a Product Parity verification report (RunFingerprint documents conditions only and does not establish parity)", + subject.id + )); + continue; + }; + if report.schema_version != PARITY_REPORT_SCHEMA_VERSION { + violations.push(format!( + "subject '{}' parity report has unsupported schemaVersion '{}'", + subject.id, report.schema_version + )); + continue; + } + if !report.passed || !report.mismatches.is_empty() { + violations.push(format!( + "subject '{}' failed Product Parity verification ({} mismatch(es))", + subject.id, + report.mismatches.len() + )); + continue; + } + if report.declared_config_sha256 != expected_cfg_sha { + violations.push(format!( + "subject '{}' parity report config digest ({}) does not match declared public configuration ({})", + subject.id, report.declared_config_sha256, expected_cfg_sha + )); + continue; + } + let exe_path = Path::new(&decoded.maka_executable); + if exe_path.is_file() + && let Ok(bytes) = std::fs::read(exe_path) + { + let actual_exe_sha = sha256_hex(&bytes); + if actual_exe_sha != report.maka_executable_sha256 { + violations.push(format!( + "subject '{}' Maka executable digest ({}) differs from parity-verified binary ({})", + subject.id, actual_exe_sha, report.maka_executable_sha256 + )); + continue; + } + } + parity_verified_subjects.push(subject.id.clone()); + } + + if milestone == MilestoneGate::M5ProductionDelivery { + let summary = build_cohort_summary(spec, canonical_v1); + if summary.incomplete_cells > 0 { + violations.push(format!( + "M5 production delivery requires all cells settled, but {} of {} cell(s) are incomplete", + summary.incomplete_cells, summary.total_cells + )); + } + for (cell_id, attempt) in canonical_v1 { + if attempt.result.status == EvalResultStatus::Indeterminate { + violations.push(format!( + "M5 production delivery rejects indeterminate cell '{cell_id}'" + )); + } else if attempt.result.status == EvalResultStatus::InfraFailed { + let explicit_env_limit = attempt.result.artifacts.iter().any(|art| { + art.get("kind").and_then(Value::as_str) == Some("environment-limitation") + }); + if !explicit_env_limit { + violations.push(format!( + "M5 production delivery rejects unattributed infra_failed cell '{cell_id}'" + )); + } + } + } + } + + MilestoneGateReport { + milestone, + passed: violations.is_empty(), + run_fingerprint, + parity_verified_subjects, + violations, + } +} diff --git a/products/eval/src/runner.rs b/products/eval/src/runner.rs index 965a30e371d..a7c7cda3acc 100644 --- a/products/eval/src/runner.rs +++ b/products/eval/src/runner.rs @@ -55,8 +55,8 @@ pub enum RunnerError { pub struct RetryPolicy { /// Maximum automatic retries for `infra_failed` attempts within a single run (`0` = single try per invocation). pub max_infra_retries: u32, - /// When `true` (default), an `indeterminate` outcome with unconfirmed resource cleanup - /// blocks automatic resubmission in the same run until cleanup is confirmed. + /// Stop in-run retries for indeterminate outcomes even when cleanup was confirmed. + /// Unconfirmed cleanup always blocks admission, including in later runs. pub block_automatic_retry_on_indeterminate: bool, } @@ -133,6 +133,7 @@ fn settle_interrupted_attempt( verifier_outcome: VerifierOutcome::Skipped, usage: unsettled.latest_usage.clone(), metering_state: unsettled.latest_metering_state, + pricing: None, cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::NotRequired, critical_artifacts_state: CriticalArtifactsState::Present, @@ -160,6 +161,7 @@ fn settle_interrupted_attempt( verifier_outcome: VerifierOutcome::Skipped, usage: unsettled.latest_usage.clone(), metering_state: unsettled.latest_metering_state, + pricing: None, cleanup_state: if confirmed { CleanupState::Confirmed } else { @@ -191,6 +193,7 @@ fn settle_interrupted_attempt( verifier_outcome: VerifierOutcome::Skipped, usage: unsettled.latest_usage.clone(), metering_state: unsettled.latest_metering_state, + pricing: None, cleanup_state: CleanupState::Unconfirmed { reason: "executor cleanup did not settle".into(), }, @@ -297,6 +300,8 @@ pub async fn run_experiment( subject.validate(cell)?; } + recover_unsettled_attempts(input.store, &ConservativeCleanupProbe, (input.now_ms)())?; + input .environment .preflight(&credential_names, input.cancel.clone()) @@ -348,8 +353,8 @@ pub async fn run_experiment( } let (should_run, next_seq, started_at) = { let _g = store_mutex.lock().await; - let attempts = match store.list_v1(&cell.id) { - Ok(a) => a, + let mut state = match store.reconstruct_state() { + Ok(state) => state, Err(e) => { let mut slot = err_slot.lock().await; if slot.is_none() { @@ -361,6 +366,14 @@ pub async fn run_experiment( if cancel.is_cancelled() { return; } + if state.settled_v2.get(&cell.id).is_some_and(|records| { + records.iter().any(|record| { + matches!(record.cleanup_state, CleanupState::Unconfirmed { .. }) + }) + }) { + return; + } + let attempts = state.settled_v1.remove(&cell.id).unwrap_or_default(); if select_cell_result(&attempts).is_some() { (false, 0, 0) } else { @@ -377,14 +390,20 @@ pub async fn run_experiment( return; } let trial_id = harbor_trial_name_for_attempt(&cell.id, seq); - let _ = store.record_resource_tagged( + if let Err(e) = store.record_resource_tagged( &cell.id, seq, env.kind(), &trial_id, None, None, - ); + ) { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } (true, seq, started) } }; @@ -492,6 +511,20 @@ async fn execute_cell_v2( let mut ctx = active_attempt.context(); ctx.cancel = cancel_for_inner.clone(); let subject_exec = subject.execute(&cell_for_inner, ctx).await; + let pricing = match cell_for_inner.subject.kind { + SubjectKind::Maka => cell_for_inner + .subject + .config + .get("model") + .and_then(serde_json::Value::as_str) + .map(|model| crate::contract::PricingEvidence::Model { + model: model.to_owned(), + cost_usd: subject_exec.cost_usd, + }), + SubjectKind::External => subject_exec.cost_usd.map(|cost_usd| { + crate::contract::PricingEvidence::SubjectReported { cost_usd } + }), + }; if cancel_for_inner.is_cancelled() || matches!( @@ -516,6 +549,7 @@ async fn execute_cell_v2( verifier_outcome: VerifierOutcome::Skipped, usage: subject_exec.usage, metering_state: subject_exec.metering_state, + pricing: pricing.clone(), cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::NotRequired, critical_artifacts_state: CriticalArtifactsState::Present, @@ -559,6 +593,7 @@ async fn execute_cell_v2( verifier_outcome, usage: subject_exec.usage, metering_state: subject_exec.metering_state, + pricing: pricing.clone(), cleanup_state: CleanupState::Confirmed, audit_state, critical_artifacts_state: CriticalArtifactsState::Present, @@ -584,6 +619,7 @@ async fn execute_cell_v2( }, usage: subject_exec.usage, metering_state: subject_exec.metering_state, + pricing: pricing.clone(), cleanup_state: CleanupState::Confirmed, audit_state, critical_artifacts_state: CriticalArtifactsState::Present, @@ -623,6 +659,7 @@ async fn execute_cell_v2( verifier_outcome: VerifierOutcome::Skipped, usage: None, metering_state: MeteringState::Unknown, + pricing: None, cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::NotRequired, critical_artifacts_state: CriticalArtifactsState::Present, @@ -668,6 +705,7 @@ async fn execute_cell_v2( verifier_outcome: VerifierOutcome::Skipped, usage: None, metering_state: MeteringState::Unknown, + pricing: None, cleanup_state: if cause == IndeterminateCause::CleanupUnconfirmed { CleanupState::Unconfirmed { reason: reason.into(), @@ -696,6 +734,7 @@ async fn execute_cell_v2( verifier_outcome: VerifierOutcome::Skipped, usage: None, metering_state: MeteringState::Unknown, + pricing: None, cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::NotRequired, critical_artifacts_state: CriticalArtifactsState::Present, diff --git a/products/eval/src/store.rs b/products/eval/src/store.rs index 0611106a44a..47153842a57 100644 --- a/products/eval/src/store.rs +++ b/products/eval/src/store.rs @@ -57,7 +57,6 @@ pub enum StoreError { /// Exclusive single-writer lock guard held for the duration of a store write session. #[derive(Debug)] pub struct WriterGuard { - lock_path: PathBuf, _file: File, } @@ -66,35 +65,23 @@ impl WriterGuard { fs::create_dir_all(dir)?; let lock_path = dir.join(".writer.lock"); let mut options = OpenOptions::new(); - options.write(true).create_new(true); + options.read(true).write(true).create(true); #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.mode(0o600); } - match options.open(&lock_path) { - Ok(mut file) => { - let _ = writeln!(file, "{}", std::process::id()); - let _ = file.sync_all(); - Ok(Self { - lock_path, - _file: file, - }) - } - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + let file = options.open(&lock_path)?; + match file.try_lock() { + Ok(()) => Ok(Self { _file: file }), + Err(std::fs::TryLockError::WouldBlock) => { Err(StoreError::ActiveWriter(lock_path.display().to_string())) } - Err(err) => Err(StoreError::Io(err)), + Err(std::fs::TryLockError::Error(err)) => Err(StoreError::Io(err)), } } } -impl Drop for WriterGuard { - fn drop(&mut self) { - let _ = fs::remove_file(&self.lock_path); - } -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum JournalEntry { @@ -272,8 +259,7 @@ impl FileAttemptStore { }) } - /// Atomically commits a settled `V2AttemptRecord`, writing its `v1` projection file, - /// `v2` native file, and `Settled` journal entry. + /// Commits a settled record to the sole durable authority: the journal. pub fn append_v2(&self, record: &V2AttemptRecord) -> Result { let expected = self.next_settled_sequence(&record.cell_id)?; if record.sequence != expected { @@ -284,15 +270,6 @@ impl FileAttemptStore { let v1 = record.project_v1(); let _ = decode_v1_cell_attempt(&serde_json::to_value(&v1)?)?; - let cell_dir = self.cell_directory(&record.cell_id); - fs::create_dir_all(&cell_dir)?; - - let v1_final = cell_dir.join(format!("{:06}.json", record.sequence)); - let v2_final = cell_dir.join(format!("{:06}.v2.json", record.sequence)); - - write_immutable_json(&v1_final, &v1)?; - write_immutable_json(&v2_final, record)?; - self.append_journal_entry(&JournalEntry::Settled { attempt: Box::new(record.clone()), })?; @@ -310,14 +287,7 @@ impl FileAttemptStore { "attempt sequence must be {expected}" ))); } - let cell_dir = self.cell_directory(&validated.cell_id); - fs::create_dir_all(&cell_dir)?; - let v1_final = cell_dir.join(format!("{:06}.json", validated.sequence)); - write_immutable_json(&v1_final, &validated)?; - let v2 = v2_from_v1(&validated); - let v2_final = cell_dir.join(format!("{:06}.v2.json", validated.sequence)); - let _ = write_immutable_json(&v2_final, &v2); self.append_journal_entry(&JournalEntry::Settled { attempt: Box::new(v2), })?; @@ -347,45 +317,13 @@ impl FileAttemptStore { }) } - /// Lists immutable `v1` attempt files for `cell_id`, verifying each file's identity against - /// its 6-digit sequence filename. + /// Projects only committed journal records, including post-hoc evidence. pub fn list_v1(&self, cell_id: &str) -> Result, StoreError> { - let dir = self.cell_directory(cell_id); - let entries = match fs::read_dir(&dir) { - Ok(rd) => rd, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(err) => return Err(StoreError::Io(err)), - }; - - let mut names = Vec::new(); - for entry in entries { - let entry = entry?; - let file_name = entry.file_name(); - let Some(name) = file_name.to_str() else { - continue; - }; - if is_six_digit_json_filename(name) { - names.push(name.to_owned()); - } - } - names.sort(); - - let mut attempts = Vec::with_capacity(names.len()); - for name in names { - let seq: u32 = name[..6].parse().map_err(|_| { - StoreError::Integrity("attempt file does not match its immutable identity".into()) - })?; - let content = fs::read_to_string(dir.join(&name))?; - let parsed: Value = serde_json::from_str(&content)?; - let attempt = decode_v1_cell_attempt(&parsed)?; - if attempt.cell_id != cell_id || attempt.sequence != seq { - return Err(StoreError::Integrity( - "attempt file does not match its immutable identity".into(), - )); - } - attempts.push(attempt); - } - Ok(attempts) + Ok(self + .reconstruct_state()? + .settled_v1 + .remove(cell_id) + .unwrap_or_default()) } pub fn read_journal_entries(&self) -> Result, StoreError> { @@ -487,6 +425,17 @@ impl FileAttemptStore { existing.recovered_post_hoc = true; existing.phase = AttemptPhase::Recovered; if usage.is_some() { + if existing.usage != usage { + if let Some(crate::contract::PricingEvidence::Model { + cost_usd, + .. + }) = &mut existing.pricing + { + *cost_usd = None; + } else { + existing.pricing = None; + } + } existing.usage = usage; existing.metering_state = metering_state; } @@ -515,6 +464,7 @@ impl FileAttemptStore { verifier_outcome: VerifierOutcome::Skipped, usage: recovered_usage, metering_state, + pricing: None, cleanup_state: if cleanup_confirmed { CleanupState::Confirmed } else { @@ -576,10 +526,6 @@ impl FileAttemptStore { .map_or(1, |s| s + 1); Ok(settled_next.max(max_journal_seq)) } - - fn cell_directory(&self, cell_id: &str) -> PathBuf { - self.root.join(sha256_hex(cell_id.as_bytes())) - } } #[derive(Debug, Clone)] @@ -796,32 +742,6 @@ pub fn write_atomic_bytes(final_path: &Path, bytes: &[u8]) -> Result<(), StoreEr res } -fn write_immutable_json(final_path: &Path, value: &T) -> Result<(), StoreError> { - let temp_path = final_path.with_extension(format!("json.{}.tmp", Uuid::new_v4())); - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - let mut temp_file = options.open(&temp_path)?; - let res = (|| -> Result<(), StoreError> { - let serialized = format!("{}\n", serde_json::to_string(value)?); - temp_file.write_all(serialized.as_bytes())?; - temp_file.sync_all()?; - drop(temp_file); - fs::hard_link(&temp_path, final_path)?; - Ok(()) - })(); - let _ = fs::remove_file(&temp_path); - res -} - -fn is_six_digit_json_filename(name: &str) -> bool { - name.len() == 11 && name.ends_with(".json") && name[..6].bytes().all(|b| b.is_ascii_digit()) -} - fn v2_from_v1(v1: &V1CellAttempt) -> V2AttemptRecord { use crate::contract::EvalResultStatus; let execution_outcome = match v1.result.status { @@ -863,6 +783,10 @@ fn v2_from_v1(v1: &V1CellAttempt) -> V2AttemptRecord { verifier_outcome, usage: v1.result.usage.clone(), metering_state, + pricing: v1 + .result + .cost_usd + .map(|cost_usd| crate::contract::PricingEvidence::SubjectReported { cost_usd }), cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::NotRequired, critical_artifacts_state: CriticalArtifactsState::Present, diff --git a/products/eval/src/subject/maka.rs b/products/eval/src/subject/maka.rs index cd1217805a6..4d477e34b38 100644 --- a/products/eval/src/subject/maka.rs +++ b/products/eval/src/subject/maka.rs @@ -18,12 +18,13 @@ */ //! Rust Maka Host subject adapter (`maka host init --settings-stdin --root `, -//! durable operation/invocation receipt tracking, disconnect re-observation, -//! paginated transcript export, and usage settlement reconciliation). +//! public Maka settings/session contracts, durable operation/invocation receipt tracking, +//! unattended interaction preservation, paginated transcript export, and usage settlement). use crate::contract::{ - ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind, exact_object, - nonempty_string, positive_u32, + ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind, + canonical_json_string, decode_json_object, exact_object, nonempty_string, positive_u32, + sha256_hex, }; use crate::environment::{ BoxFuture, SubjectExecutionContext, SubjectProcessOutput, SubjectProcessRequest, @@ -48,10 +49,11 @@ use tokio::io::AsyncWriteExt; use tokio::process::Command; use uuid::Uuid; +pub mod native; + pub const DEFAULT_EVAL_GATEWAY_SLUG: &str = "eval-gateway"; pub const DEFAULT_EVAL_GATEWAY_NAME: &str = "Eval Gateway"; pub const DEFAULT_EVAL_GATEWAY_PROVIDER: &str = "openai-compatible"; -pub const DEFAULT_EVAL_CONTEXT_WINDOW: u64 = 200_000; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -59,22 +61,6 @@ pub struct MakaGatewayConfiguration { pub base_url: String, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct MakaModelOverride { - pub context_window: u64, - pub thinking_levels: Vec, -} - -impl Default for MakaModelOverride { - fn default() -> Self { - Self { - context_window: DEFAULT_EVAL_CONTEXT_WINDOW, - thinking_levels: vec!["off".into()], - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct MakaConnectionInit { @@ -83,9 +69,14 @@ pub struct MakaConnectionInit { pub provider: String, pub configuration: MakaGatewayConfiguration, pub api_key: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_headers: Option>, pub enabled: bool, pub enabled_model_ids: Vec, - pub model_overrides: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_overrides: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_body_overlay: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -95,7 +86,10 @@ pub struct MakaDefaultTargetInit { pub model_id: String, } -/// Exact stdin JSON payload accepted by `maka host init --settings-stdin --root `. +/// Exact public stdin JSON payload accepted by `maka host init --settings-stdin --root `. +/// +/// Only user-declared connection settings and optional public `modelOverrides` are included; +/// Eval never synthesizes hidden context-window, thinking-level, or tool overrides. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct MakaHostInitSettingsPayload { @@ -104,8 +98,8 @@ pub struct MakaHostInitSettingsPayload { } impl MakaHostInitSettingsPayload { - /// Constructs the canonical `eval-gateway` settings payload for a local metering gateway - /// port, credential, and model ID. + /// Constructs a default `openai-compatible` settings payload for a local metering gateway + /// port, credential, and model ID with Maka's unmodified product model defaults. pub fn for_eval_gateway( port: u16, gateway_credential: impl Into, @@ -124,9 +118,6 @@ impl MakaHostInitSettingsPayload { model_id: impl Into, ) -> Self { let model_id = model_id.into(); - let mut model_overrides = BTreeMap::new(); - model_overrides.insert(model_id.clone(), MakaModelOverride::default()); - Self { connections: vec![MakaConnectionInit { slug: DEFAULT_EVAL_GATEWAY_SLUG.into(), @@ -136,9 +127,11 @@ impl MakaHostInitSettingsPayload { base_url: base_url.into(), }, api_key: gateway_credential.into(), + request_headers: None, enabled: true, enabled_model_ids: vec![model_id.clone()], - model_overrides, + model_overrides: None, + request_body_overlay: None, }], default_target: MakaDefaultTargetInit { slug: DEFAULT_EVAL_GATEWAY_SLUG.into(), @@ -147,6 +140,19 @@ impl MakaHostInitSettingsPayload { } } + pub fn with_endpoint_and_credential( + &self, + base_url: impl Into, + api_key: impl Into, + ) -> Self { + let mut cloned = self.clone(); + if let Some(conn) = cloned.connections.first_mut() { + conn.configuration.base_url = base_url.into(); + conn.api_key = api_key.into(); + } + cloned + } + pub fn to_json_value(&self) -> Value { serde_json::to_value(self).expect("MakaHostInitSettingsPayload serialization cannot fail") } @@ -169,6 +175,7 @@ pub async fn bootstrap_maka_host_root( .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) + .kill_on_drop(true) .spawn() .map_err(|err| format!("failed to spawn maka host init: {err}"))?; @@ -242,6 +249,22 @@ where Ok(items) } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UnattendedInteractionBlock { + pub kind: String, + pub request_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_use_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EnvironmentLimitation { + pub category: String, + pub detail: String, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum HostedExecutionProjection { @@ -251,13 +274,27 @@ pub enum HostedExecutionProjection { status: String, #[serde(rename = "failureReason")] failure_reason: Option, - usage: NormalizedUsage, + usage: Option, + #[serde(rename = "meteringState")] + metering_state: MeteringState, #[serde(rename = "costUsd")] cost_usd: Option, #[serde(rename = "operationId", default)] operation_id: Option, #[serde(rename = "invocationId", default)] invocation_id: Option, + #[serde( + rename = "blockedInteraction", + default, + skip_serializing_if = "Option::is_none" + )] + blocked_interaction: Option>, + #[serde( + rename = "environmentLimitation", + default, + skip_serializing_if = "Option::is_none" + )] + environment_limitation: Option>, }, Indeterminate { #[serde(rename = "executionId")] @@ -297,24 +334,42 @@ impl HostedExecutionProjection { } } +/// Public Maka subject configuration declared in an experiment specification. +/// +/// Every agent-facing setting maps 1:1 to a public Maka user interface (`maka host init`, +/// `settings.updateAssistantPreferences`, `agent.create`, and `agent.metadata`). +/// Benchmark-only profiles (`toolProfile`, `permissionMode: "bypass"`, synthetic model +/// overrides) are rejected at the contract boundary. +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaPublicSessionSettings { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_permissions: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_level: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub language: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_instructions: Option, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct MakaSubjectConfig { - pub maka_executable: Option, - pub node_path: String, - pub shim_path: String, + pub maka_executable: String, pub runtime_hosts_path: String, pub host_settlement_timeout_ms: u32, - pub base_url: String, + pub provider: String, pub connection_slug: String, + pub connection_name: String, + pub base_url: String, pub model: String, - pub provider_type: Option, - pub default_api_protocol: Option, - pub api_key_environment: Option, - pub thinking_level: Option, - pub permission_mode: String, - pub collaboration_mode: String, - pub orchestration_mode: String, - pub tool_profile: String, + pub api_key_environment: String, + pub model_override: Option, + pub request_headers: Option>, + pub request_body_overlay: Option, + pub session_settings: MakaPublicSessionSettings, } impl MakaSubjectConfig { @@ -323,151 +378,289 @@ impl MakaSubjectConfig { declared_credentials: &[String], ) -> Result { let val = Value::Object(config.clone()); - let mut required = vec![ - "runtimeHostsPath", - "baseUrl", - "connectionSlug", - "model", - "permissionMode", - "collaborationMode", - "orchestrationMode", - "hostSettlementTimeoutMs", - "toolProfile", - ]; - if config.contains_key("makaExecutable") { - required.push("makaExecutable"); - if config.contains_key("nodePath") { - required.push("nodePath"); - } - if config.contains_key("shimPath") { - required.push("shimPath"); - } - } else { - required.push("nodePath"); - required.push("shimPath"); - } - if config.contains_key("providerType") { - required.push("providerType"); - required.push("apiKeyEnvironment"); - } - if config.get("providerType").and_then(Value::as_str) == Some("custom") { - required.push("defaultApiProtocol"); - } - if config.contains_key("thinkingLevel") { - required.push("thinkingLevel"); - } + let obj = exact_object( + &val, + "Maka config", + &[ + "makaExecutable", + "runtimeHostsPath", + "hostSettlementTimeoutMs", + "baseUrl", + "model", + ], + &[ + "provider", + "connectionSlug", + "connectionName", + "apiKeyEnvironment", + "modelOverride", + "requestHeaders", + "requestBodyOverlay", + "sandboxMode", + "requestPermissions", + "thinkingLevel", + "language", + "workspaceInstructions", + ], + )?; - let obj = exact_object(&val, "Maka config", &required, &[])?; - for field in &required { - if *field == "hostSettlementTimeoutMs" { - continue; - } - let _ = nonempty_string(obj.get(*field).unwrap(), &format!("Maka config.{field}"))?; - } + let maka_executable = nonempty_string( + obj.get("makaExecutable").unwrap(), + "Maka config.makaExecutable", + )?; + let runtime_hosts_path = nonempty_string( + obj.get("runtimeHostsPath").unwrap(), + "Maka config.runtimeHostsPath", + )?; + let host_settlement_timeout_ms = positive_u32( + obj.get("hostSettlementTimeoutMs").unwrap(), + "Maka config.hostSettlementTimeoutMs", + )?; let base_url = nonempty_string(obj.get("baseUrl").unwrap(), "Maka config.baseUrl")?; - if url::Url::parse(&base_url).is_err() { + let parsed_url = url::Url::parse(&base_url) + .map_err(|_| ContractError::Invalid("Maka baseUrl is invalid".into()))?; + if !matches!(parsed_url.scheme(), "http" | "https") || parsed_url.host_str().is_none() { return Err(ContractError::Invalid("Maka baseUrl is invalid".into())); } - let host_settlement_timeout_ms = positive_u32( - obj.get("hostSettlementTimeoutMs").unwrap(), - "Maka config.hostSettlementTimeoutMs", - )?; + let model = nonempty_string(obj.get("model").unwrap(), "Maka config.model")?; - let api_key_environment = obj - .get("apiKeyEnvironment") - .and_then(Value::as_str) - .map(str::to_owned); - if let Some(env_name) = &api_key_environment + let provider = match obj.get("provider") { + Some(v) => nonempty_string(v, "Maka config.provider")?, + None => DEFAULT_EVAL_GATEWAY_PROVIDER.into(), + }; + + let connection_slug = match obj.get("connectionSlug") { + Some(v) => { + let s = nonempty_string(v, "Maka config.connectionSlug")?; + validate_connection_slug(&s)?; + s + } + None => DEFAULT_EVAL_GATEWAY_SLUG.into(), + }; + + let connection_name = match obj.get("connectionName") { + Some(v) => nonempty_string(v, "Maka config.connectionName")?, + None => DEFAULT_EVAL_GATEWAY_NAME.into(), + }; + + let explicit_api_key_env = match obj.get("apiKeyEnvironment") { + Some(v) => Some(nonempty_string(v, "Maka config.apiKeyEnvironment")?), + None => None, + }; + if let Some(env_name) = &explicit_api_key_env && !declared_credentials.iter().any(|c| c == env_name) { return Err(ContractError::Invalid( "Maka apiKeyEnvironment must name a declared subject credential".into(), )); } + let api_key_environment = match explicit_api_key_env { + Some(name) => name, + None if declared_credentials.len() == 1 => declared_credentials[0].clone(), + None => { + return Err(ContractError::Invalid( + "native Maka requires one credential or an explicit apiKeyEnvironment".into(), + )); + } + }; + + let model_override = match obj.get("modelOverride") { + Some(v) => Some(decode_json_object(v, "Maka config.modelOverride")?), + None => None, + }; + + let request_headers = match obj.get("requestHeaders") { + Some(v) => { + let map = v.as_object().ok_or_else(|| { + ContractError::Invalid("Maka config.requestHeaders must be an object".into()) + })?; + let mut headers = BTreeMap::new(); + for (k, hv) in map { + if k.trim().is_empty() { + return Err(ContractError::Invalid( + "Maka config.requestHeaders key must be non-empty".into(), + )); + } + let val_str = hv.as_str().ok_or_else(|| { + ContractError::Invalid( + "Maka config.requestHeaders values must be strings".into(), + ) + })?; + headers.insert(k.clone(), val_str.to_owned()); + } + Some(headers) + } + None => None, + }; + + let request_body_overlay = match obj.get("requestBodyOverlay") { + Some(v) => Some(decode_json_object(v, "Maka config.requestBodyOverlay")?), + None => None, + }; + + let sandbox_mode = match obj.get("sandboxMode") { + Some(v) => { + let mode = nonempty_string(v, "Maka config.sandboxMode")?; + if !matches!( + mode.as_str(), + "read-only" | "workspace-write" | "danger-full-access" + ) { + return Err(ContractError::Invalid( + "Maka config.sandboxMode must be read-only, workspace-write, or danger-full-access".into(), + )); + } + Some(mode) + } + None => None, + }; + + let request_permissions = match obj.get("requestPermissions") { + Some(v) => Some(v.as_bool().ok_or_else(|| { + ContractError::Invalid("Maka config.requestPermissions must be a boolean".into()) + })?), + None => None, + }; + + let thinking_level = match obj.get("thinkingLevel") { + Some(v) => { + let level = nonempty_string(v, "Maka config.thinkingLevel")?; + if !matches!( + level.as_str(), + "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "ultra" + ) { + return Err(ContractError::Invalid( + "Maka config.thinkingLevel is invalid".into(), + )); + } + Some(level) + } + None => None, + }; + + let language = match obj.get("language") { + Some(v) => { + let lang = nonempty_string(v, "Maka config.language")?; + if !matches!(lang.as_str(), "auto" | "en" | "zh-CN" | "zh-TW") { + return Err(ContractError::Invalid( + "Maka config.language must be auto, en, zh-CN, or zh-TW".into(), + )); + } + Some(lang) + } + None => None, + }; - let thinking_level = obj - .get("thinkingLevel") - .and_then(Value::as_str) - .map(str::to_owned); - if let Some(level) = &thinking_level - && !matches!( - level.as_str(), - "off" | "minimal" | "low" | "medium" | "high" | "max" - ) - { - return Err(ContractError::Invalid( - "Maka config.thinkingLevel is invalid".into(), - )); - } - - let tool_profile = - nonempty_string(obj.get("toolProfile").unwrap(), "Maka config.toolProfile")?; - if !matches!( - tool_profile.as_str(), - "default" | "headless-coding-v1" | "minimal" - ) { - return Err(ContractError::Invalid( - "Maka config.toolProfile is invalid".into(), - )); - } - - let maka_executable = obj - .get("makaExecutable") - .and_then(Value::as_str) - .map(str::to_owned); - let node_path = obj - .get("nodePath") - .and_then(Value::as_str) - .unwrap_or("node") - .to_owned(); - let shim_path = obj - .get("shimPath") - .and_then(Value::as_str) - .unwrap_or("") - .to_owned(); + let workspace_instructions = match obj.get("workspaceInstructions") { + Some(v) => Some(v.as_bool().ok_or_else(|| { + ContractError::Invalid("Maka config.workspaceInstructions must be a boolean".into()) + })?), + None => None, + }; Ok(Self { maka_executable, - node_path, - shim_path, - runtime_hosts_path: nonempty_string( - obj.get("runtimeHostsPath").unwrap(), - "Maka config.runtimeHostsPath", - )?, + runtime_hosts_path, host_settlement_timeout_ms, + provider, + connection_slug, + connection_name, base_url, - connection_slug: nonempty_string( - obj.get("connectionSlug").unwrap(), - "Maka config.connectionSlug", - )?, - model: nonempty_string(obj.get("model").unwrap(), "Maka config.model")?, - provider_type: obj - .get("providerType") - .and_then(Value::as_str) - .map(str::to_owned), - default_api_protocol: obj - .get("defaultApiProtocol") - .and_then(Value::as_str) - .map(str::to_owned), + model, api_key_environment, - thinking_level, - permission_mode: nonempty_string( - obj.get("permissionMode").unwrap(), - "Maka config.permissionMode", - )?, - collaboration_mode: nonempty_string( - obj.get("collaborationMode").unwrap(), - "Maka config.collaborationMode", - )?, - orchestration_mode: nonempty_string( - obj.get("orchestrationMode").unwrap(), - "Maka config.orchestrationMode", - )?, - tool_profile, + model_override, + request_headers, + request_body_overlay, + session_settings: MakaPublicSessionSettings { + sandbox_mode, + request_permissions, + thinking_level, + language, + workspace_instructions, + }, }) } + + pub fn host_init_settings( + &self, + base_url: impl Into, + api_key: impl Into, + ) -> MakaHostInitSettingsPayload { + let model_overrides = self.model_override.as_ref().map(|ov| { + let mut map = BTreeMap::new(); + map.insert(self.model.clone(), ov.clone()); + map + }); + MakaHostInitSettingsPayload { + connections: vec![MakaConnectionInit { + slug: self.connection_slug.clone(), + name: self.connection_name.clone(), + provider: self.provider.clone(), + configuration: MakaGatewayConfiguration { + base_url: base_url.into(), + }, + api_key: api_key.into(), + request_headers: self.request_headers.clone(), + enabled: true, + enabled_model_ids: vec![self.model.clone()], + model_overrides, + request_body_overlay: self.request_body_overlay.clone(), + }], + default_target: MakaDefaultTargetInit { + slug: self.connection_slug.clone(), + model_id: self.model.clone(), + }, + } + } + + /// Canonical digest over all public Maka settings that govern effective instructions, + /// model routing, tool exposure, and execution policies. + /// + /// Ephemeral loopback port numbers (`127.0.0.1:`, `localhost:`) are normalized + /// while canonical provider hosts and paths are preserved verbatim. + pub fn parity_config_sha256(&self) -> String { + let payload = serde_json::json!({ + "provider": self.provider, + "connectionSlug": self.connection_slug, + "connectionName": self.connection_name, + "routeSignature": url_route_signature(&self.base_url), + "model": self.model, + "modelOverride": self.model_override, + "requestHeaders": self.request_headers, + "requestBodyOverlay": self.request_body_overlay, + "sessionSettings": self.session_settings, + }); + sha256_hex(canonical_json_string(&payload).as_bytes()) + } +} + +fn validate_connection_slug(value: &str) -> Result<(), ContractError> { + let valid = |c: u8| c.is_ascii_lowercase() || c.is_ascii_digit(); + let b = value.as_bytes(); + if b.is_empty() + || !valid(b[0]) + || !valid(b[b.len() - 1]) + || !b.iter().all(|c| valid(*c) || *c == b'-') + || b.windows(2).any(|pair| pair == b"--") + { + return Err(ContractError::Invalid( + "Maka config.connectionSlug must be lowercase alphanumeric hyphen-separated".into(), + )); + } + Ok(()) +} + +pub fn url_route_signature(raw_url: &str) -> String { + let Ok(mut parsed) = url::Url::parse(raw_url) else { + return raw_url.to_owned(); + }; + if matches!(parsed.host_str(), Some("127.0.0.1" | "localhost")) { + let _ = parsed.set_port(None); + } + let trimmed_path = parsed.path().trim_end_matches('/').to_owned(); + parsed.set_path(&trimmed_path); + parsed.to_string() } #[derive(Debug, Default, Clone)] @@ -518,63 +711,29 @@ impl SubjectAdapter for RustHostSubject { let execution_id = Uuid::new_v4().to_string(); let root_path = format!("{}/{execution_id}", config.runtime_hosts_path); - let gateway_settings = MakaHostInitSettingsPayload::for_eval_gateway_base_url( - &config.base_url, - config - .api_key_environment - .as_deref() - .unwrap_or("EVAL_GATEWAY_API_KEY"), - &config.model, - ); - - let mut session_obj = serde_json::json!({ - "workspace": { "kind": "host_path", "path": context.cwd }, - "modelTarget": { - "kind": "explicit", - "connectionSlug": config.connection_slug, - "model": config.model, - }, - "permissionMode": config.permission_mode, - "collaborationMode": config.collaboration_mode, - "orchestrationMode": config.orchestration_mode, - "toolProfile": config.tool_profile, - }); - if let Some(level) = &config.thinking_level { - session_obj - .as_object_mut() - .unwrap() - .insert("thinkingLevel".into(), Value::String(level.clone())); - } + let host_init_settings = + config.host_init_settings(&config.base_url, &config.api_key_environment); + let artifact_root = context + .metadata + .get("artifactRoot") + .and_then(Value::as_str) + .unwrap_or(MAKA_RUNTIME_ARTIFACT_PATH); - let mut envelope = serde_json::json!({ + let envelope = serde_json::json!({ + "makaExecutable": config.maka_executable, "rootPath": root_path, - "artifactRoot": MAKA_RUNTIME_ARTIFACT_PATH, + "artifactRoot": artifact_root, "baseUrl": config.base_url, "hostSettlementTimeoutMs": config.host_settlement_timeout_ms, - "hostInitSettings": gateway_settings, + "hostInitSettings": host_init_settings, "execution": { "executionId": execution_id, - "session": session_obj, + "workspacePath": context.cwd, + "sessionSettings": config.session_settings, "content": { "text": context.task_input }, "maxSteps": max_steps, } }); - if let Some(provider_type) = &config.provider_type { - let mut conn = serde_json::json!({ - "providerType": provider_type, - "apiKeyEnvironment": config.api_key_environment, - }); - if let Some(proto) = &config.default_api_protocol { - conn.as_object_mut() - .unwrap() - .insert("defaultApiProtocol".into(), Value::String(proto.clone())); - } - envelope - .as_object_mut() - .unwrap() - .insert("connection".into(), conn); - } - let encoded_payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&envelope).unwrap()); let credential_environment: HashMap = cell .subject @@ -585,8 +744,8 @@ impl SubjectAdapter for RustHostSubject { let process = match context .execute(SubjectProcessRequest { - command: config.node_path.clone(), - args: vec![config.shim_path.clone(), encoded_payload], + command: resolve_maka_eval_executable(&config.maka_executable), + args: vec!["host-subject".into(), encoded_payload], environment: HashMap::new(), credential_environment, capture_stdout: true, @@ -612,14 +771,18 @@ impl SubjectAdapter for RustHostSubject { match projection { Some(HostedExecutionProjection::Settled { usage, + metering_state, cost_usd, operation_id, invocation_id, .. }) => ( - Some(usage.clone()), - MeteringState::Complete, - estimate_deepseek_cost(&usage, &config.model), + usage.clone(), + metering_state, + usage + .as_ref() + .filter(|_| metering_state == MeteringState::Complete) + .and_then(|u| estimate_deepseek_cost(u, &config.model)), cost_usd, operation_id, invocation_id, @@ -638,6 +801,8 @@ impl SubjectAdapter for RustHostSubject { Some(&execution_id), Some(&process), subject_reported_cost, + None, + None, ), operation_id: op_id, invocation_id: inv_id, @@ -697,6 +862,28 @@ impl SubjectAdapter for RustHostSubject { } } +fn resolve_maka_eval_executable(maka_executable: &str) -> String { + let eval_name = if cfg!(windows) { + "maka-eval.exe" + } else { + "maka-eval" + }; + let maka_path = Path::new(maka_executable); + let sibling = maka_path.with_file_name(eval_name); + if !maka_path.exists() || sibling.exists() { + return sibling.to_string_lossy().into_owned(); + } + if let Ok(current) = std::env::current_exe() + && current + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n == eval_name) + { + return current.to_string_lossy().into_owned(); + } + sibling.to_string_lossy().into_owned() +} + fn project_into_execution_result( projection: HostedExecutionProjection, output: Option, @@ -723,7 +910,7 @@ fn project_into_execution_result( &failure_reason, "Maka execution did not settle", )), - artifacts: maka_artifacts(Some(execution_id), process, None), + artifacts: maka_artifacts(Some(execution_id), process, None, None, None), operation_id, invocation_id, }, @@ -731,30 +918,46 @@ fn project_into_execution_result( status, failure_reason, usage, + metering_state: receipt_metering_state, cost_usd: subject_cost, operation_id, invocation_id, + blocked_interaction, + environment_limitation, .. } => { - let (reconciled_usage, metering_state, _) = - reconcile_host_and_proxy_usage(Some(&usage), true, proxy_counts); - let final_usage = reconciled_usage.unwrap_or(usage); - let cost_usd = if metering_state == MeteringState::Complete { - estimate_deepseek_cost(&final_usage, &config.model) + let (final_usage, metering_state, _) = if let Some(proxy) = proxy_counts { + reconcile_host_and_proxy_usage( + usage.as_ref(), + receipt_metering_state == MeteringState::Complete, + Some(proxy), + ) } else { - None + (usage, receipt_metering_state, None) }; + let cost_usd = final_usage + .as_ref() + .filter(|_| metering_state == MeteringState::Complete) + .and_then(|u| estimate_deepseek_cost(u, &config.model)); + + let artifacts = maka_artifacts( + Some(execution_id), + process, + subject_cost, + blocked_interaction.as_deref(), + environment_limitation.as_deref(), + ); if status == "cancelled" { return SubjectExecutionResult { output, - usage: Some(final_usage), + usage: final_usage, metering_state, cost_usd, duration_ms, outcome: ExecutionOutcome::Indeterminate, failure_reason: Some("Maka subject cancelled".into()), - artifacts: maka_artifacts(Some(execution_id), process, subject_cost), + artifacts, operation_id, invocation_id, }; @@ -762,26 +965,31 @@ fn project_into_execution_result( let outcome = if status == "completed" { ExecutionOutcome::Completed + } else if environment_limitation.is_some() { + ExecutionOutcome::InfraFailed } else { ExecutionOutcome::SubjectFailed }; let reason = if status == "completed" { None } else { - failure_reason - .as_deref() - .map(|r| safe_maka_failure_reason(r, "Maka execution failed")) + Some( + failure_reason + .as_deref() + .map(|r| safe_maka_failure_reason(r, "Maka execution failed")) + .unwrap_or_else(|| "Maka execution failed".into()), + ) }; SubjectExecutionResult { output, - usage: Some(final_usage), + usage: final_usage, metering_state, cost_usd, duration_ms, outcome, failure_reason: reason, - artifacts: maka_artifacts(Some(execution_id), process, subject_cost), + artifacts, operation_id, invocation_id, } @@ -812,10 +1020,38 @@ pub fn safe_maka_failure_reason(value: &str, fallback: &str) -> String { "Runtime Host connection failed before execution settlement", "Runtime Host could not settle execution", "Runtime Host did not exit cleanly", + "Maka execution failed", + "Maka step budget exhausted", ]; if SAFE_REASONS.contains(&value) { return value.to_owned(); } + if let Some(rest) = value.strip_prefix("Unattended execution blocked on required interaction: ") + && matches!( + rest, + "session_approval" + | "service_approval" + | "question" + | "permissions" + | "form" + | "client_capability" + | "computer_app" + | "system_permission" + ) + { + return value.to_owned(); + } + if let Some(rest) = value.strip_prefix("Environment limitation: ") + && matches!( + rest, + "authority_unavailable" + | "sandbox_unavailable" + | "workspace_unavailable" + | "host_startup_failed" + ) + { + return value.to_owned(); + } if let Some(rest) = value.strip_prefix("Runtime Host did not start: ") && matches!( rest, @@ -847,6 +1083,8 @@ pub fn maka_artifacts( execution_id: Option<&str>, process: Option<&SubjectProcessOutput>, subject_reported_cost_usd: Option, + blocked_interaction: Option<&UnattendedInteractionBlock>, + environment_limitation: Option<&EnvironmentLimitation>, ) -> Vec { let mut out = Vec::new(); if let Some(cost) = subject_reported_cost_usd { @@ -855,6 +1093,32 @@ pub fn maka_artifacts( obj.insert("costUsd".into(), Value::from(cost)); out.push(obj); } + if let Some(block) = blocked_interaction { + let mut obj = Map::new(); + obj.insert( + "kind".into(), + Value::String("unattended-interaction-blocked".into()), + ); + obj.insert("interactionKind".into(), Value::String(block.kind.clone())); + obj.insert("requestId".into(), Value::String(block.request_id.clone())); + if let Some(tool_use_id) = &block.tool_use_id { + obj.insert("toolUseId".into(), Value::String(tool_use_id.clone())); + } + out.push(obj); + } + if let Some(limitation) = environment_limitation { + let mut obj = Map::new(); + obj.insert( + "kind".into(), + Value::String("environment-limitation".into()), + ); + obj.insert( + "category".into(), + Value::String(limitation.category.clone()), + ); + obj.insert("detail".into(), Value::String(limitation.detail.clone())); + out.push(obj); + } let mut runtime_state = Map::new(); runtime_state.insert("kind".into(), Value::String("maka-runtime-state".into())); runtime_state.insert( @@ -893,7 +1157,7 @@ fn subject_failure( cancelled: bool, process: Option<&SubjectProcessOutput>, ) -> SubjectExecutionResult { - let mut artifacts = maka_artifacts(None, process, None); + let mut artifacts = maka_artifacts(None, process, None, None, None); let mut failure_art = Map::new(); failure_art.insert("kind".into(), Value::String("subject-failure".into())); failure_art.insert("stage".into(), Value::String(stage.into())); diff --git a/products/eval/src/subject/maka/native.rs b/products/eval/src/subject/maka/native.rs new file mode 100644 index 00000000000..0097a6bfae8 --- /dev/null +++ b/products/eval/src/subject/maka/native.rs @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Trial-local native Host launcher using Maka's public Settings, Session, and Draft/Receipt APIs. +//! +//! Preserves product parity by invoking the released `maka` composition without benchmark-specific +//! instructions, tool narrowing, synthetic model overrides, or silent approval bypasses. + +use super::{ + EnvironmentLimitation, HostedExecutionProjection, MakaHostInitSettingsPayload, + MakaPublicSessionSettings, UnattendedInteractionBlock, bootstrap_maka_host_root, +}; +use crate::measurement::{EvalMeteringGateway, GatewayConfig, MeteringState, derive_metering}; +use crate::store::write_atomic_bytes; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use maka_app_client::{Client, Journal, Mutation, SubscriptionEvent}; +use maka_host_runtime::application::host::Client as HostClient; +use maka_plugin_core::operation::OperationResult; +use maka_plugin_core::operation::{OperationCleanup, OperationOutcome, Receipt}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::{ + path::{Path, PathBuf}, + process::Stdio, + time::Duration, +}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::{Child, Command}, +}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +type Result = std::result::Result; + +/// Standard editor draft key used by the normal Maka Workbench composer. +const COMPOSER_DRAFT_KEY: &str = "composer"; +/// Standard user-created conversation placeholder title before task input. +const DEFAULT_SESSION_TITLE: &str = "Task"; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Request { + maka_executable: PathBuf, + root_path: PathBuf, + artifact_root: PathBuf, + base_url: String, + host_settlement_timeout_ms: u64, + host_init_settings: MakaHostInitSettingsPayload, + execution: Execution, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Execution { + execution_id: String, + workspace_path: String, + #[serde(default)] + session_settings: MakaPublicSessionSettings, + content: Content, + max_steps: u32, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Content { + text: String, +} + +struct SessionOutcome { + status: String, + failure_reason: Option, + operation_id: String, + run_id: String, + blocked_interaction: Option, + environment_limitation: Option, +} + +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| format!("native Host reply is missing {key}")) +} + +pub(crate) async fn commit_client_mutation(mutation: Mutation) -> Result { + mutation.prepare().await?; + match mutation.confirm().await?.receipt { + Receipt::Recorded { + outcome: + OperationOutcome::Returned { + result: OperationResult::Success { value }, + }, + cleanup: OperationCleanup::Confirmed, + } => Ok(value), + Receipt::Recorded { + outcome: + OperationOutcome::Returned { + result: OperationResult::Failure { diagnostic }, + }, + cleanup: OperationCleanup::Confirmed, + } => Err(diagnostic), + _ => Err("native Host mutation did not produce a confirmed receipt".into()), + } +} + +pub(crate) async fn start_host_client( + maka_executable: &Path, + root_path: &Path, + settlement_timeout_ms: u64, + journal: Journal, +) -> Result<(Child, Client)> { + let mut child = Command::new(maka_executable) + .args(["host", "serve", "--root"]) + .arg(root_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .map_err(|e| format!("native Host spawn failed: {e}"))?; + let mut stdout = BufReader::new( + child + .stdout + .take() + .ok_or("native Host stdout unavailable")?, + ); + let mut ready = String::new(); + tokio::time::timeout( + Duration::from_millis(settlement_timeout_ms), + stdout.read_line(&mut ready), + ) + .await + .map_err(|_| "native Host startup timed out")? + .map_err(|e| e.to_string())?; + let ready: Value = serde_json::from_str(&ready) + .map_err(|e| format!("invalid native Host ready message: {e}"))?; + if ready["kind"] != "ready" { + return Err("native Host did not become ready".into()); + } + let epoch = text(&ready, "hostEpoch")? + .parse() + .map_err(|_| "invalid native Host epoch")?; + let host = HostClient::connect(Path::new(text(&ready, "socketPath")?), Some(epoch)).await?; + let client = Client::new(host, journal, "receipts".into())?; + Ok((child, client)) +} + +pub(crate) async fn shutdown_host_client( + child: &mut Child, + client: &Client, + deadline: Duration, +) -> Result<()> { + tokio::time::timeout(deadline, async { + client.close().await?; + #[cfg(unix)] + if let Some(pid) = child.id() { + // Child retains the process handle and has not been reaped, so its PID + // cannot identify a replacement process during this signal delivery. + if unsafe { libc::kill(pid as i32, libc::SIGTERM) } != 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + } + #[cfg(not(unix))] + child.start_kill().map_err(|e| e.to_string())?; + let status = child.wait().await.map_err(|e| e.to_string())?; + if !status.success() { + return Err("native Host did not shut down cleanly".into()); + } + Ok(()) + }) + .await + .map_err(|_| "native Host cleanup timed out".to_owned())? +} + +async fn settings(client: &Client, method: &str, input: Value) -> Result { + commit_client_mutation( + client + .settings_mutation( + Uuid::new_v4(), + "settings", + method, + input, + Default::default(), + ) + .await?, + ) + .await +} + +/// Applies workspace root registration and any explicitly declared public Assistant Preferences. +pub(crate) async fn apply_public_host_preferences( + workspace_path: &str, + session_settings: &MakaPublicSessionSettings, + client: &Client, +) -> Result<()> { + let current = client + .query("settings", "workspaceLocations", json!({})) + .await?; + let source = current + .get("source") + .ok_or("native Host workspace source unavailable")?; + let changed = settings( + client, + "updateWorkspaceLocations", + json!({ + "source": source, + "roots": [{"label": "Workspace", "path": workspace_path}] + }), + ) + .await?; + if changed["status"] != "committed" { + return Err("native Host workspace configuration was not committed".into()); + } + + let mut mutations = Vec::new(); + if let Some(mode) = &session_settings.sandbox_mode { + mutations.push(json!({"kind": "default_sandbox", "mode": mode})); + } + if let Some(language) = &session_settings.language { + mutations.push(json!({"kind": "language", "preference": language})); + } + if let Some(enabled) = session_settings.workspace_instructions { + mutations.push(json!({"kind": "workspace_instructions", "enabled": enabled})); + } + + for mutation in mutations { + let preferences = client + .query("settings", "assistantPreferences", json!({})) + .await?; + let changed = settings( + client, + "updateAssistantPreferences", + json!({ + "expectedRevision": preferences["revision"], + "mutation": mutation, + }), + ) + .await?; + if changed["kind"] != "committed" { + return Err("native Host preferences were not committed".into()); + } + } + Ok(()) +} + +pub(crate) fn sandbox_mode_to_workspace_access(mode: &str) -> Result<&'static str> { + match mode { + "read-only" => Ok("read_only"), + "workspace-write" => Ok("workspace_write"), + "danger-full-access" => Ok("danger_full_access"), + _ => Err(format!("unsupported sandboxMode: {mode}")), + } +} + +async fn export_history(client: &Client, session: &str, path: &Path) -> Result<()> { + let mut through = Value::Null; + let mut position = json!({"kind":"start"}); + let mut cursors = std::collections::BTreeSet::new(); + let mut pages = Vec::new(); + loop { + let page = client + .query( + "agent", + "history", + json!({"sessionId":session,"through":through,"position":position}), + ) + .await?; + let fence = page + .get("through") + .ok_or("history fence unavailable")? + .clone(); + if !through.is_null() && through != fence { + return Err("history fence changed".into()); + } + through = fence; + let after = page + .get("after") + .ok_or("history continuation unavailable")? + .clone(); + pages.push(page); + if after.is_null() { + break; + } + if !cursors.insert(after.to_string()) { + return Err("history pagination cycle".into()); + } + position = json!({"kind":"after", "cursor":after}); + } + let bytes = serde_json::to_vec(&pages).map_err(|e| e.to_string())?; + let path = path.to_owned(); + tokio::task::spawn_blocking(move || write_atomic_bytes(&path, &bytes)) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()) +} + +async fn observe_pending_interaction( + client: &Client, + session: &str, + run: &str, + artifact_root: &Path, +) -> Result> { + let pending = client + .query( + "agent", + "pendingInteractions", + json!({"sessionId": session}), + ) + .await?; + let Some(entries) = pending.get("entries").and_then(Value::as_array) else { + return Ok(None); + }; + let Some(first) = entries.first() else { + return Ok(None); + }; + let request_id = text(first, "requestId")?.to_owned(); + let kind = text(first, "kind")?.to_owned(); + let tool_use_id = first + .get("toolUseId") + .and_then(Value::as_str) + .map(str::to_owned); + + let detail = client + .query( + "agent", + "interaction", + json!({"sessionId": session, "requestId": request_id}), + ) + .await + .unwrap_or(Value::Null); + + let evidence = json!({ + "sessionId": session, + "runId": run, + "entries": entries, + "firstInteraction": detail, + }); + if let Ok(bytes) = serde_json::to_vec(&evidence) { + let _ = write_atomic_bytes(&artifact_root.join("unattended-interactions.json"), &bytes); + } + + Ok(Some(UnattendedInteractionBlock { + kind, + request_id, + tool_use_id, + })) +} + +async fn execute( + request: &Request, + client: &Client, + stop: CancellationToken, +) -> Result { + let connections = client + .query("settings", "connections", json!({"kind":"start"})) + .await?; + let connection = connections["connections"] + .as_array() + .and_then(|items| { + items + .iter() + .find(|c| c["slug"] == request.host_init_settings.default_target.slug) + }) + .ok_or("native Host connection unavailable")?; + let backend = json!({ + "kind": "model", + "connectionId": connection["basis"]["connectionId"], + "modelId": request.host_init_settings.default_target.model_id, + "thinkingLevel": request.execution.session_settings.thinking_level, + }); + let created = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "create", + json!({ + "title": DEFAULT_SESSION_TITLE, + "automaticTitle": false, + "labels": [], + "backend": backend, + "workspace": { + "kind": "directory", + "path": request.execution.workspace_path, + }, + "tools": Value::Null, + }), + ) + .await?, + ) + .await?; + if created["kind"] != "created" { + let reason = created + .get("reason") + .and_then(Value::as_str) + .unwrap_or("rejected"); + if reason == "workspace_unavailable" { + return Ok(SessionOutcome { + status: "failed".into(), + failure_reason: Some("Environment limitation: workspace_unavailable".into()), + operation_id: String::new(), + run_id: String::new(), + blocked_interaction: None, + environment_limitation: Some(EnvironmentLimitation { + category: "workspace_unavailable".into(), + detail: format!( + "Host rejected workspace directory {}", + request.execution.workspace_path + ), + }), + }); + } + return Err(format!("native Host rejected session creation: {reason}")); + } + + let mut session_obj = created["session"].clone(); + let session = text(&session_obj, "id")?.to_owned(); + + if let Some(request_permissions) = request.execution.session_settings.request_permissions { + let access = match &request.execution.session_settings.sandbox_mode { + Some(mode) => sandbox_mode_to_workspace_access(mode)?.to_owned(), + None => session_obj["permissions"]["access"] + .as_str() + .unwrap_or("workspace_write") + .to_owned(), + }; + let revision = session_obj["revision"] + .as_u64() + .ok_or("session revision unavailable")?; + let updated = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "metadata", + json!({ + "sessionId": session, + "expectedRevision": revision, + "title": DEFAULT_SESSION_TITLE, + "labels": [], + "permissions": { + "settings": { + "access": access, + "requestPermissions": request_permissions, + }, + "revokeGrants": false, + }, + }), + ) + .await?, + ) + .await?; + if updated["kind"] != "committed" { + return Err("native Host rejected public session permission settings".into()); + } + session_obj = updated["session"].clone(); + let _ = session_obj; + } + + let draft = client + .save_draft( + "agent", + &session, + COMPOSER_DRAFT_KEY, + None, + request.execution.content.text.clone().into(), + ) + .await?; + let operation = Uuid::new_v4(); + let mutation = client + .reserve_send_draft( + operation, + "agent", + &session, + COMPOSER_DRAFT_KEY, + draft.revision, + json!({"backend": backend}), + ) + .await? + .open() + .await?; + let watch = client + .subscribe("agent", "watch", json!({"sessionId": session, "after": 0})) + .await?; + let accepted = commit_client_mutation(mutation).await?; + let run = match accepted["kind"].as_str() { + Some("started") => text(&accepted, "runId")?.to_owned(), + Some("queued") => { + let message = text(&accepted, "messageId")?; + let mut interval = tokio::time::interval(Duration::from_millis(100)); + loop { + tokio::select! { + _ = stop.cancelled() => return Err("native Host cancelled before Run admission".into()), + _ = interval.tick() => { + let work = client.query("agent", "work", json!({"sessionId": session, + "target": {"kind": "message", "messageId": message}, "cursor": null})).await?; + match work["kind"].as_str() { + Some("delivered") => break text(&work["execution"], "runId")?.to_owned(), + Some("pending") => {}, + _ => return Err("native Host could not resolve the original queued message".into()), + } + } + } + } + } + _ => { + return Err(format!( + "native Host rejected send: {}", + accepted.get("reason").unwrap_or(&Value::Null) + )); + } + }; + + let receipt = json!({ + "executionId": request.execution.execution_id, + "operationId": operation, + "runId": run, + "sessionId": session, + "rootPath": request.root_path, + }); + let bytes = serde_json::to_vec(&receipt).map_err(|e| e.to_string())?; + tokio::fs::write(request.artifact_root.join("execution-receipt.json"), bytes) + .await + .map_err(|e| e.to_string())?; + + let mut interval = tokio::time::interval(Duration::from_millis(100)); + let mut steps = std::collections::BTreeSet::new(); + let mut stopping = false; + let mut budget_exhausted = false; + let mut blocked_interaction: Option = None; + let mut environment_limitation: Option = None; + let mut next = Box::pin(watch.next()); + + let raw_status = loop { + tokio::select! { + _ = stop.cancelled(), if !stopping => { + commit_client_mutation(client.control(Uuid::new_v4(), "stop", json!({"sessionId": session, "runId": run})).await?).await?; + stopping = true; + } + event = &mut next => { + match event? { + Some(SubscriptionEvent::Item(page)) => { + if let Some(events) = page.get("events").and_then(Value::as_array) { + for event in events.iter().filter(|event| event["runId"] == run) { + if event["fact"]["kind"] == "step_ended" { + steps.insert(text(&event["fact"], "stepId")?.to_owned()); + } + } + } + if !stopping && steps.len() >= request.execution.max_steps as usize { + let execution = client.query("agent", "execution", json!({"sessionId": session, "runId": run})).await?; + if matches!(execution["execution"]["state"]["kind"].as_str(), Some("admitted" | "running" | "waiting")) { + commit_client_mutation(client.control(Uuid::new_v4(), "stop", json!({"sessionId": session, "runId": run})).await?).await?; + stopping = true; + budget_exhausted = true; + } + } + } + _ => return Err("native Host execution watch ended before settlement".into()), + } + next = Box::pin(watch.next()); + } + _ = interval.tick() => { + let execution = client.query("agent", "execution", json!({"sessionId": session, "runId": run})).await?; + if execution["kind"] != "found" { + return Err("native Host Run receipt unavailable".into()); + } + let state = &execution["execution"]["state"]; + match state["kind"].as_str() { + Some("completed") => break "completed", + Some("failed") => { + if let Some(reason_kind) = state.pointer("/reason/kind").and_then(Value::as_str) + && reason_kind == "current_authority_unavailable" + { + environment_limitation = Some(EnvironmentLimitation { + category: "authority_unavailable".into(), + detail: "Run failed because host execution authority became unavailable".into(), + }); + } + break "failed"; + } + Some("cancelled") => break "cancelled", + Some("waiting") => { + if !stopping { + if let Some(interaction) = observe_pending_interaction( + client, + &session, + &run, + &request.artifact_root, + ) + .await? + { + blocked_interaction = Some(interaction); + commit_client_mutation( + client + .control( + Uuid::new_v4(), + "stop", + json!({"sessionId": session, "runId": run}), + ) + .await?, + ) + .await?; + stopping = true; + } else if state["authorityUnavailable"].as_bool() == Some(true) { + environment_limitation = Some(EnvironmentLimitation { + category: "authority_unavailable".into(), + detail: "Run waiting on unavailable host authority in headless environment".into(), + }); + commit_client_mutation( + client + .control( + Uuid::new_v4(), + "stop", + json!({"sessionId": session, "runId": run}), + ) + .await?, + ) + .await?; + stopping = true; + } + } + } + Some("admitted" | "running") => { + if !stopping + && let Some(interaction) = observe_pending_interaction( + client, + &session, + &run, + &request.artifact_root, + ) + .await? + { + blocked_interaction = Some(interaction); + commit_client_mutation( + client + .control( + Uuid::new_v4(), + "stop", + json!({"sessionId": session, "runId": run}), + ) + .await?, + ) + .await?; + stopping = true; + } + } + _ => return Err("native Host returned an unknown Run state".into()), + } + } + } + }; + drop(next); + watch.close().await?; + + let (status, failure_reason) = if let Some(block) = &blocked_interaction { + ( + "failed".to_owned(), + Some(format!( + "Unattended execution blocked on required interaction: {}", + block.kind + )), + ) + } else if let Some(limitation) = &environment_limitation { + ( + "failed".to_owned(), + Some(format!("Environment limitation: {}", limitation.category)), + ) + } else if budget_exhausted { + ( + "failed".to_owned(), + Some("Maka step budget exhausted".into()), + ) + } else if raw_status != "completed" { + ( + raw_status.to_owned(), + (raw_status == "failed").then(|| "Maka execution failed".into()), + ) + } else { + ("completed".to_owned(), None) + }; + + export_history( + client, + &session, + &request.artifact_root.join("transcript.json"), + ) + .await?; + + Ok(SessionOutcome { + status, + failure_reason, + operation_id: operation.to_string(), + run_id: run, + blocked_interaction, + environment_limitation, + }) +} + +pub async fn run(payload: &str, stop: CancellationToken) -> Result { + let bytes = URL_SAFE_NO_PAD + .decode(payload) + .map_err(|_| "invalid native Host payload")?; + let request: Request = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + let deadline = Duration::from_millis(request.host_settlement_timeout_ms); + tokio::fs::create_dir_all(&request.artifact_root) + .await + .map_err(|e| e.to_string())?; + let credential_name = &request + .host_init_settings + .connections + .first() + .ok_or("Eval connection missing")? + .api_key; + let key = std::env::var(credential_name).map_err(|_| "Eval provider credential unavailable")?; + let credential = Uuid::new_v4().to_string(); + let gateway = EvalMeteringGateway::start(GatewayConfig { + upstream_base_url: request.base_url.clone(), + upstream_api_key: key, + profile: "maka".into(), + gateway_credential: credential.clone(), + checkpoint_path: Some(request.artifact_root.join("metering-checkpoint.json")), + checkpoint_secret: std::env::var("MAKA_EVAL_RESULT_TOKEN") + .unwrap_or_else(|_| Uuid::new_v4().to_string()), + }) + .await?; + let settings = request.host_init_settings.with_endpoint_and_credential( + gateway.base_url_matching_upstream(&request.base_url), + credential, + ); + tokio::time::timeout( + deadline, + bootstrap_maka_host_root(&request.maka_executable, &request.root_path, &settings), + ) + .await + .map_err(|_| "native Host initialization timed out")??; + let journal = Journal::open(&request.artifact_root.join("client-journal.sqlite")).await?; + let (mut child, client) = match start_host_client( + &request.maka_executable, + &request.root_path, + request.host_settlement_timeout_ms, + journal.clone(), + ) + .await + { + Ok(pair) => pair, + Err(err) => { + gateway.stop_accepting(); + let _ = gateway.close().await; + return Ok(HostedExecutionProjection::Settled { + execution_id: request.execution.execution_id, + status: "failed".into(), + failure_reason: Some("Environment limitation: host_startup_failed".into()), + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + operation_id: None, + invocation_id: None, + blocked_interaction: None, + environment_limitation: Some(Box::new(EnvironmentLimitation { + category: "host_startup_failed".into(), + detail: err, + })), + }); + } + }; + let configured = apply_public_host_preferences( + &request.execution.workspace_path, + &request.execution.session_settings, + &client, + ) + .await; + let cleanup = shutdown_host_client(&mut child, &client, deadline).await; + configured?; + cleanup?; + let (mut child, client) = start_host_client( + &request.maka_executable, + &request.root_path, + request.host_settlement_timeout_ms, + journal, + ) + .await?; + let result = { + let work = execute(&request, &client, stop.clone()); + tokio::pin!(work); + tokio::select! { + result = &mut work => result, + _ = stop.cancelled() => tokio::time::timeout(deadline, &mut work).await + .map_err(|_| "native Host cancellation did not settle".to_owned()).and_then(|result| result), + } + }; + gateway.stop_accepting(); + let cleanup = shutdown_host_client(&mut child, &client, deadline).await; + let counts = tokio::time::timeout(deadline, gateway.close()) + .await + .map_err(|_| "native Host metering did not settle")?; + cleanup?; + let outcome = result?; + let metering = derive_metering(&counts); + Ok(HostedExecutionProjection::Settled { + execution_id: request.execution.execution_id, + failure_reason: outcome.failure_reason, + status: outcome.status, + usage: counts.usage, + metering_state: metering.metering_state, + cost_usd: if metering.metering_state == MeteringState::Complete + && request.host_init_settings.default_target.model_id == "deepseek-v4-flash" + { + metering.cost_usd + } else { + None + }, + operation_id: (!outcome.operation_id.is_empty()).then_some(outcome.operation_id), + invocation_id: (!outcome.run_id.is_empty()).then_some(outcome.run_id), + blocked_interaction: outcome.blocked_interaction.map(Box::new), + environment_limitation: outcome.environment_limitation.map(Box::new), + }) +} diff --git a/products/eval/tests/integration.rs b/products/eval/tests/integration.rs index d4b486268e6..17c27c9a9e6 100644 --- a/products/eval/tests/integration.rs +++ b/products/eval/tests/integration.rs @@ -35,9 +35,13 @@ use maka_eval::measurement::{ costs_match_within_tolerance, deepseek_cost_usd, derive_metering, reconcile_host_and_proxy_usage, recover_external_metering_checkpoint, sign_metering_checkpoint, }; +use maka_eval::parity::{ + ParityCheckInput, check_declared_endpoint_route_parity, compare_parity_snapshots, + verify_product_parity, +}; use maka_eval::report::{ - build_cohort_summary, evaluate_differential_oracle, export_v1_canonical, export_v2_canonical, - rebuild_from_journal, + MilestoneGate, build_cohort_summary, evaluate_differential_oracle, evaluate_milestone_gate, + export_v1_canonical, export_v2_canonical, rebuild_from_journal, }; use maka_eval::runner::{ ConservativeCleanupProbe, RetryPolicy, RunExperimentInput, recover_unsettled_attempts, @@ -51,7 +55,7 @@ use maka_eval::subject::external_cli::{ ExternalCliSubject, encode_relay_result_frame, parse_relay_result_frame, }; use maka_eval::subject::maka::{ - HostedExecutionProjection, MakaHostInitSettingsPayload, RustHostSubject, + HostedExecutionProjection, MakaHostInitSettingsPayload, MakaSubjectConfig, RustHostSubject, bootstrap_maka_host_root, export_paginated_transcript, }; use serde_json::{Map, Value, json}; @@ -81,13 +85,7 @@ fn maka_host_init_settings_stdin_payload_matches_exact_session_contract() { }, "apiKey": "gw-secret-credential-99", "enabled": true, - "enabledModelIds": ["deepseek-v4-flash"], - "modelOverrides": { - "deepseek-v4-flash": { - "contextWindow": 200000, - "thinkingLevels": ["off"] - } - } + "enabledModelIds": ["deepseek-v4-flash"] } ], "defaultTarget": { @@ -96,6 +94,62 @@ fn maka_host_init_settings_stdin_payload_matches_exact_session_contract() { } }); assert_eq!(actual, expected); + + // Benchmark-only agent overrides must be rejected at the contract boundary. + for forbidden in [ + json!({"toolProfile": "minimal"}), + json!({"permissionMode": "bypass"}), + json!({"collaborationMode": "agent"}), + json!({"orchestrationMode": "default"}), + ] { + let mut cfg = json!({ + "makaExecutable": "/opt/maka-agent/maka", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 60000, + "baseUrl": "http://127.0.0.1:43120/v1", + "model": "deepseek-v4-flash" + }); + for (k, v) in forbidden.as_object().unwrap() { + cfg[k] = v.clone(); + } + assert!( + MakaSubjectConfig::decode(cfg.as_object().unwrap(), &["DEEPSEEK_API_KEY".into()]) + .is_err(), + "forbidden benchmark-only field must be rejected: {forbidden}" + ); + } + + // Public Maka configuration interfaces are accepted and preserved in hostInitSettings. + let public_cfg = json!({ + "makaExecutable": "/opt/maka-agent/maka", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 60000, + "provider": "openai-compatible", + "connectionSlug": "team-provider", + "connectionName": "Team Provider", + "baseUrl": "https://api.example.com/v1", + "model": "gpt-4.1", + "modelOverride": { + "contextWindow": 128000, + "thinkingLevels": ["off", "medium"] + }, + "sandboxMode": "read-only", + "requestPermissions": false, + "thinkingLevel": "medium", + "language": "en", + "workspaceInstructions": false + }); + let decoded = + MakaSubjectConfig::decode(public_cfg.as_object().unwrap(), &["OPENAI_API_KEY".into()]) + .expect("public Maka subject configuration must decode"); + let init = decoded + .host_init_settings("https://api.example.com/v1", "sk-test") + .to_json_value(); + assert_eq!(init["defaultTarget"]["slug"], json!("team-provider")); + assert_eq!( + init["connections"][0]["modelOverrides"]["gpt-4.1"]["contextWindow"], + json!(128000) + ); } #[cfg(unix)] @@ -313,6 +367,7 @@ fn all_five_semantic_difference_scenarios_and_three_verdict_differential_oracle( verifier_outcome: VerifierOutcome::Verified { score: 1.0 }, usage: Some(sample_usage.clone()), metering_state: MeteringState::LowerBound, + pricing: None, cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::NotRequired, critical_artifacts_state: CriticalArtifactsState::Present, @@ -341,6 +396,25 @@ fn all_five_semantic_difference_scenarios_and_three_verdict_differential_oracle( assert!(report_1.v1_parses); assert!(report_1.v1_matches_oracle); assert!(report_1.conditions_comparable); + for entry in registry.entries() { + let mut changed = v1_timeout_reward.clone(); + changed.result.score = Some(0.0); + changed.result.usage.as_mut().unwrap().input_tokens = 999_999; + let report = evaluate_differential_oracle( + &serde_json::to_value(changed).unwrap(), + &v1_timeout_reward, + &fp, + &fp, + Some(entry.id), + ®istry, + ); + assert!( + !report.v1_matches_oracle, + "scenario {:?} must not hide actual differences", + entry.id + ); + assert!(!report.conditions_comparable); + } // 2. Audit truncation vs. Missing audit log: let (trunc_state, trunc_ev) = describe_egress_audit( @@ -410,6 +484,32 @@ fn all_five_semantic_difference_scenarios_and_three_verdict_differential_oracle( let v1_cancel_after = v2_cancel_after.project_v1(); assert_eq!(v1_cancel_after.result.status, EvalResultStatus::Completed); assert_eq!(v1_cancel_after.result.score, Some(1.0)); + let mut legacy_cancel = v1_cancel_before.clone(); + legacy_cancel.result.failure_reason = + Some("executor cancelled before verification completed".into()); + let late = evaluate_differential_oracle( + &serde_json::to_value(&v1_cancel_after).unwrap(), + &legacy_cancel, + &fp, + &fp, + Some(DifferenceScenarioId::LateCancellation), + ®istry, + ); + assert!(!late.v1_matches_oracle); + assert!(late.adjudicated_difference); + assert!(late.conditions_comparable); + let mut altered = v1_cancel_after.clone(); + altered.result.usage.as_mut().unwrap().output_tokens += 1; + let altered = evaluate_differential_oracle( + &serde_json::to_value(altered).unwrap(), + &legacy_cancel, + &fp, + &fp, + Some(DifferenceScenarioId::LateCancellation), + ®istry, + ); + assert!(!altered.adjudicated_difference); + assert!(!altered.conditions_comparable); // 5. Missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0): let mut v2_missing_reward = v2_audit_trunc.clone(); @@ -509,6 +609,26 @@ fn store_journal_reconstruction_post_hoc_recovery_and_crash_recovery_boundaries( ExecutionOutcome::InfraFailed ); assert_eq!(recovered[0].cleanup_state, CleanupState::Confirmed); + let failed_store = maka_eval::store::FileAttemptStore::new(temp.path().join("failed")); + fs::create_dir_all(failed_store.journal_path()).unwrap(); + assert!(failed_store.append_v2(&recovered[0]).is_err()); + fs::remove_dir(failed_store.journal_path()).unwrap(); + assert!( + failed_store + .list_v1("task-alpha::1::maka") + .unwrap() + .is_empty() + ); + assert!( + export_v1_canonical(&spec, &failed_store) + .unwrap() + .is_empty() + ); + assert!( + export_v2_canonical(&spec, &failed_store) + .unwrap() + .is_empty() + ); // 3. Crash AFTER ResourceTagged with unconfirmed cleanup recovers as Indeterminate. store @@ -616,7 +736,6 @@ fn sqlite_runtime_artifact_capture_and_external_checkpoint_hmac_verification() { "inFlightRequests": 0, "admittedRequests": 2, "usageRequests": 2, - "removedWebTools": 1, "models": ["deepseek-v4-flash"], "toolNames": ["bash"] }); @@ -712,6 +831,10 @@ async fn runner_executes_task_groups_bounds_retries_and_exports_cohort_summary() assert_eq!(alpha_maka.sequence, 2); assert_eq!(alpha_maka.result.status, EvalResultStatus::Completed); assert_eq!(alpha_maka.result.score, Some(1.0)); + assert!(costs_match_within_tolerance( + alpha_maka.result.cost_usd, + Some(0.0001107) + )); // Verify all stored v1 files strictly decode under decode_v1_cell_attempt. let v1_export = export_v1_canonical(&spec, &dir.attempts).unwrap(); @@ -828,6 +951,8 @@ impl CommandRunner for FakeActiveTrial { }); } + assert_eq!(request.command, "/opt/maka-agent/maka-eval"); + assert_eq!(request.args[0], "host-subject"); // Maka subject invocation: decode the base64url envelope to extract executionId let call_num = self.maka_calls.fetch_add(1, Ordering::SeqCst) + 1; if self.cell_id == "task-alpha::1::maka" && call_num == 1 { @@ -836,7 +961,7 @@ impl CommandRunner for FakeActiveTrial { termination: SubjectProcessTermination::Exited, exit_code: 1, stdout: String::new(), - stderr: "transient shim crash".into(), + stderr: "transient native Host crash".into(), diagnostic: Some(SubjectProcessDiagnostic { category: SubjectProcessDiagnosticCategory::None, bytes: None, @@ -864,17 +989,20 @@ impl CommandRunner for FakeActiveTrial { execution_id, status: "completed".into(), failure_reason: None, - usage: NormalizedUsage { + usage: Some(NormalizedUsage { input_tokens: 300, output_tokens: 100, cache_read_tokens: 100, cache_write_tokens: 0, reasoning_tokens: 10, total_tokens: 400, - }, + }), + metering_state: MeteringState::Complete, cost_usd: Some(0.0001107), operation_id: Some("op-settled-1".into()), invocation_id: Some("inv-settled-1".into()), + blocked_interaction: None, + environment_limitation: None, }; Ok(SubjectProcessOutput { termination: SubjectProcessTermination::Exited, @@ -973,18 +1101,13 @@ fn sample_two_arm_spec_json() -> Value { "kind": "maka", "credentials": ["DEEPSEEK_API_KEY"], "config": { - "nodePath": "/opt/maka-node-toolchain/bin/node", - "shimPath": "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-maka-subject.js", + "makaExecutable": "/opt/maka-agent/maka", "runtimeHostsPath": "/tmp/maka-runtime-hosts", "hostSettlementTimeoutMs": 120000, "connectionSlug": "eval-gateway", "baseUrl": "http://127.0.0.1:43120/v1", "model": "deepseek-v4-flash", - "thinkingLevel": "off", - "permissionMode": "bypass", - "collaborationMode": "agent", - "orchestrationMode": "default", - "toolProfile": "headless-coding-v1" + "sandboxMode": "workspace-write" } }, { @@ -1124,6 +1247,16 @@ sys.exit(0) .await .unwrap(); + let mut unsupported_config = + HarborExecutorConfig::decode(&spec.executor.kind, &spec.executor.config).unwrap(); + unsupported_config.egress_proxy_required = true; + let unsupported = HarborTrialEnvironment::new(unsupported_config, temp.path().to_path_buf()); + let error = unsupported + .preflight(&[], CancellationToken::new()) + .await + .unwrap_err(); + assert!(error.to_string().contains("egressProxy is not supported")); + let cell_id = cell.id.clone(); let outcome = harbor_env .run_attempt( @@ -1164,6 +1297,7 @@ sys.exit(0) verifier_outcome: verified.to_verifier_outcome(), usage: None, metering_state: MeteringState::Unknown, + pricing: None, cleanup_state: CleanupState::Confirmed, audit_state: EgressAuditState::Clean, critical_artifacts_state: CriticalArtifactsState::Present, @@ -1211,3 +1345,868 @@ sys.exit(0) other => panic!("unexpected outcome: {other:?}"), } } + +#[test] +fn writer_lock_is_released_after_process_exit() { + const KEY: &str = "MAKA_EVAL_TEST_CRASH_LOCK"; + if let Some(path) = std::env::var_os(KEY) { + let store = maka_eval::store::FileAttemptStore::new(path); + let _guard = store.acquire_writer().unwrap(); + std::process::exit(0); + } + let temp = TempDir::new().unwrap(); + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "writer_lock_is_released_after_process_exit"]) + .env(KEY, temp.path()) + .status() + .unwrap(); + assert!(status.success()); + let store = maka_eval::store::FileAttemptStore::new(temp.path()); + let _guard = store + .acquire_writer() + .expect("crashed writer must release lock"); + assert!(store.acquire_writer().is_err()); +} + +#[tokio::test] +async fn runner_recovers_before_admission_and_blocks_unconfirmed_cleanup_across_runs() { + let temp = TempDir::new().unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let id = "task-alpha::1::external-cli"; + let calls = Arc::new(AtomicUsize::new(0)); + let env = Arc::new(FakeTrialEnvironment { + maka_calls: Arc::new(AtomicUsize::new(0)), + ext_calls: calls.clone(), + }); + let subjects: Vec> = vec![Arc::new(ExternalCliSubject::new())]; + let store = maka_eval::store::FileAttemptStore::new(temp.path().join("clean")); + store + .record_launch_intent(id, 1, 100, "interrupted") + .unwrap(); + let mut input = RunExperimentInput::new(&spec, &store, env.clone(), subjects.clone()); + input.cell_ids = Some(vec![id.into()]); + let result = run_experiment(input).await.unwrap(); + assert_eq!(result[id].sequence, 2); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let store = maka_eval::store::FileAttemptStore::new(temp.path().join("uncertain")); + store + .record_launch_intent(id, 1, 100, "interrupted") + .unwrap(); + store + .record_resource_tagged(id, 1, "harbor", "possibly-alive", None, None) + .unwrap(); + for _ in 0..2 { + let mut input = RunExperimentInput::new(&spec, &store, env.clone(), subjects.clone()); + input.cell_ids = Some(vec![id.into()]); + assert!(run_experiment(input).await.unwrap().is_empty()); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "must not repeat a possibly live operation" + ); + assert_eq!(store.next_sequence(id).unwrap(), 2); + store + .record_post_hoc_recovery(id, 1, 200, None, MeteringState::Unknown, true, None) + .unwrap(); + let mut input = RunExperimentInput::new(&spec, &store, env, subjects); + input.cell_ids = Some(vec![id.into()]); + assert_eq!(run_experiment(input).await.unwrap()[id].sequence, 2); + assert_eq!(calls.load(Ordering::SeqCst), 2); +} + +#[cfg(unix)] +#[tokio::test] +async fn trial_cancellation_terminates_execution_and_verification_waits() { + use std::os::unix::fs::PermissionsExt; + for verifying in [false, true] { + let temp = TempDir::new().unwrap(); + let notify = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let marker = temp.path().join("terminated"); + let script = format!( + r#"#!/usr/bin/env python3 +import json, pathlib, socket, sys, signal, os +pathlib.Path({marker:?} + '.pid').write_text(str(os.getpid())) +spec = json.loads(pathlib.Path(sys.argv[4]).read_text()) +k = spec['agent']['kwargs'] +def terminate(*args): + pathlib.Path({marker:?}).write_text('SIGTERM') + sys.exit(0) +signal.signal(signal.SIGTERM, terminate) +sock = socket.create_connection((k['relay_host'], k['relay_port'])) +r = sock.makefile('r') +w = sock.makefile('w') +w.write(json.dumps(dict(kind='ready', token=k['relay_token'], cwd='/workspace', instruction='task'))+'\n'); w.flush() +request = json.loads(r.readline()) +if {verifying}: + w.write(json.dumps(dict(kind='executed', token=k['relay_token'], termination='exited', exitCode=0, stdout='', diagnostic=dict(category='none')))+'\n'); w.flush() + json.loads(r.readline()) +notification = socket.create_connection(('127.0.0.1', {port})) +while True: signal.pause() +"#, + marker = marker.to_str().unwrap(), + verifying = if verifying { "True" } else { "False" }, + port = notify.local_addr().unwrap().port() + ); + let python = temp.path().join("trial.py"); + fs::write(&python, script).unwrap(); + fs::set_permissions(&python, fs::Permissions::from_mode(0o755)).unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let cells = expand_experiment(&spec); + let env = HarborTrialEnvironment::new( + HarborExecutorConfig::decode("harbor", &spec.executor.config).unwrap(), + temp.path().to_path_buf(), + ) + .with_env_overrides(std::collections::HashMap::from([ + ( + "MAKA_EVAL_HARBOR_PYTHON".into(), + python.to_string_lossy().into(), + ), + ( + "MAKA_EVAL_TRIALS_DIR".into(), + temp.path().join("trials").to_string_lossy().into(), + ), + ("DEEPSEEK_API_KEY".into(), "test-key".into()), + ])); + let cancel = CancellationToken::new(); + let run = env.run_attempt( + &cells[0], + 1, + &[], + cancel.clone(), + Box::new(move |active| { + Box::pin(async move { + let _ = active + .context() + .execute(SubjectProcessRequest { + command: "wait".into(), + args: vec![], + environment: Default::default(), + credential_environment: Default::default(), + capture_stdout: true, + }) + .await; + if verifying { + let _ = active.verify().await; + } + std::future::pending::().await + }) + }), + ); + let trigger = async { + notify.accept().await.unwrap(); + cancel.cancel(); + }; + let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::join!(run, trigger) + }) + .await; + if result.is_err() { + let pid = fs::read_to_string(format!("{}.pid", marker.display())).unwrap(); + std::process::Command::new("kill") + .args(["-KILL", pid.trim()]) + .status() + .unwrap(); + } + let (outcome, ()) = result.expect("cancellation must settle promptly"); + assert!(matches!( + outcome.unwrap(), + TrialEnvironmentOutcome::Indeterminate { .. } + )); + assert_eq!(fs::read_to_string(&marker).unwrap(), "SIGTERM"); + } +} + +#[tokio::test] +async fn unknown_model_cost_stays_unknown_in_both_reports() { + let temp = TempDir::new().unwrap(); + let mut json = sample_two_arm_spec_json(); + json["subjects"][0]["config"]["model"] = json!("unpriced-model"); + let spec = parse_experiment_spec(&json).unwrap(); + let store = maka_eval::store::FileAttemptStore::new(temp.path()); + let env = Arc::new(FakeTrialEnvironment { + maka_calls: Arc::new(AtomicUsize::new(0)), + ext_calls: Arc::new(AtomicUsize::new(0)), + }); + let mut input = + RunExperimentInput::new(&spec, &store, env, vec![Arc::new(RustHostSubject::new())]); + input.cell_ids = Some(vec!["task-beta::1::maka".into()]); + let results = run_experiment(input).await.unwrap(); + let result = &results["task-beta::1::maka"].result; + assert!(result.usage.is_some()); + assert_eq!(result.cost_usd, None); + assert_eq!( + export_v2_canonical(&spec, &store).unwrap()["task-beta::1::maka"] + .project_v1() + .result + .cost_usd, + None + ); +} + +#[cfg(unix)] +fn stage_real_maka_and_eval_binaries( + dir: &std::path::Path, +) -> (std::path::PathBuf, std::path::PathBuf) { + use std::os::unix::fs::symlink; + + let profile = std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .to_path_buf(); + let maka_src = std::env::var_os("CARGO_BIN_EXE_maka") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| profile.join("maka")); + assert!( + maka_src.is_file(), + "build maka-cli first, or supply CARGO_BIN_EXE_maka" + ); + let eval_src = std::env::var_os("CARGO_BIN_EXE_maka-eval") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| std::path::PathBuf::from(env!("CARGO_BIN_EXE_maka-eval"))); + assert!(eval_src.is_file(), "maka-eval binary must exist"); + + let bin_dir = dir.join("bin"); + fs::create_dir_all(&bin_dir).unwrap(); + let staged_maka = bin_dir.join("maka"); + let staged_eval = bin_dir.join("maka-eval"); + symlink(&maka_src, &staged_maka).unwrap(); + symlink(&eval_src, &staged_eval).unwrap(); + + if let Some(src_dir) = maka_src.parent() { + let worker = src_dir.join("maka-python-worker"); + if worker.is_file() { + let _ = symlink(&worker, bin_dir.join("maka-python-worker")); + } + } + let profile_worker = profile.join("maka-python-worker"); + if profile_worker.is_file() && !bin_dir.join("maka-python-worker").exists() { + let _ = symlink(&profile_worker, bin_dir.join("maka-python-worker")); + } + + (staged_maka, staged_eval) +} + +#[cfg(unix)] +#[tokio::test] +async fn native_host_executes_with_public_client_and_returns_authenticated_relay_frame() { + let temp = TempDir::new().unwrap(); + let (maka, eval) = stage_real_maka_and_eval_binaries(temp.path()); + let script = temp.path().join("native_host.py"); + fs::write(&script, include_str!("support/native_host.py")).unwrap(); + let output = tokio::process::Command::new("python3") + .arg(script) + .env("MAKA_TEST_MAKA", maka) + .env("MAKA_TEST_EVAL", eval) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn real_maka_product_parity_gate_verifies_instructions_tools_and_policies_for_m3_and_m5() { + let temp = TempDir::new().unwrap(); + let (maka, eval) = stage_real_maka_and_eval_binaries(temp.path()); + let workspace = temp.path().join("workspace"); + fs::create_dir_all(&workspace).unwrap(); + fs::write( + workspace.join("AGENTS.md"), + "Follow workspace engineering standards.\n", + ) + .unwrap(); + + // 1. Default released product composition parity (zero session overrides, full tool catalog). + let default_cfg_json = json!({ + "makaExecutable": maka.to_string_lossy(), + "runtimeHostsPath": temp.path().join("hosts-default").to_string_lossy(), + "hostSettlementTimeoutMs": 30000, + "provider": "openai-compatible", + "connectionSlug": "eval-gateway", + "baseUrl": "https://api.openai.com/v1", + "model": "gpt-4.1" + }); + let default_report = verify_product_parity(&ParityCheckInput { + subject_id: "maka".into(), + subject_config: default_cfg_json.as_object().unwrap().clone(), + subject_credentials: vec!["OPENAI_API_KEY".into()], + workspace_source: Some(workspace.clone()), + task_input: "Inspect the repository layout and reply Done.".into(), + max_steps: 10, + }) + .await + .expect("live parity verification against real Maka must succeed"); + assert!( + default_report.passed, + "default Maka composition parity failed: {:?}", + default_report.mismatches + ); + assert!(default_report.mismatches.is_empty()); + assert_eq!( + default_report.normal_tool_catalog_digest, + default_report.eval_tool_catalog_digest + ); + assert!( + !default_report.exposed_tool_names.is_empty(), + "released Maka composition must expose product tools without pruning" + ); + assert!( + default_report + .exposed_tool_names + .iter() + .any(|name| name == "AskUserQuestion"), + "interactive product tools must not be pruned: {:?}", + default_report.exposed_tool_names + ); + + // 2. Explicit public user configuration parity (read-only sandbox, disabled permission prompts, + // explicit reasoning level and modelOverride). + let configured_spec_json = json!({ + "schemaVersion": "maka.eval.v2", + "id": "parity-gated-exp", + "benchmark": { "id": "tb", "version": "1.0", "config": {} }, + "executor": { "kind": "harbor", "config": {} }, + "subjects": [ + { + "id": "maka", + "kind": "maka", + "credentials": ["OPENAI_API_KEY"], + "config": { + "makaExecutable": maka.to_string_lossy(), + "runtimeHostsPath": temp.path().join("hosts-configured").to_string_lossy(), + "hostSettlementTimeoutMs": 30000, + "provider": "openai-compatible", + "connectionSlug": "team-openai", + "connectionName": "Team OpenAI", + "baseUrl": "https://api.example.com/v1", + "model": "gpt-4.1", + "modelOverride": { + "contextWindow": 128000, + "thinkingLevels": ["off", "medium"] + }, + "sandboxMode": "read-only", + "requestPermissions": false, + "thinkingLevel": "medium", + "language": "en", + "workspaceInstructions": false + } + } + ], + "tasks": [ + { "id": "task-1", "input": "Verify parity.", "config": {} } + ], + "repetitions": 1, + "budget": { "maxSteps": 10 }, + "verifier": {} + }); + let spec_file = temp.path().join("spec.json"); + let parity_out = temp.path().join("parity-report.json"); + fs::write( + &spec_file, + serde_json::to_vec_pretty(&configured_spec_json).unwrap(), + ) + .unwrap(); + + // Exercise the `maka-eval parity` CLI subcommand end-to-end. + let cli_out = tokio::process::Command::new(&eval) + .args([ + "parity", + spec_file.to_str().unwrap(), + "--out", + parity_out.to_str().unwrap(), + ]) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + cli_out.status.success(), + "maka-eval parity CLI failed: stdout={} stderr={}", + String::from_utf8_lossy(&cli_out.stdout), + String::from_utf8_lossy(&cli_out.stderr) + ); + let reports: Vec = + serde_json::from_slice(&fs::read(&parity_out).unwrap()).unwrap(); + assert_eq!(reports.len(), 1); + assert!(reports[0].passed); + + // 3. M3 and M5 milestone acceptance gates. + let spec = parse_experiment_spec(&configured_spec_json).unwrap(); + let empty_canonical = std::collections::BTreeMap::new(); + + // Without a parity report, RunFingerprint alone MUST NOT pass M3 or M5. + let m3_missing = + evaluate_milestone_gate(MilestoneGate::M3Acceptance, &spec, &empty_canonical, &[]); + assert!(!m3_missing.passed); + assert!( + m3_missing.violations[0].contains("RunFingerprint documents conditions only"), + "unexpected violation: {:?}", + m3_missing.violations + ); + + // With matching parity report, M3 passes; M5 still requires complete settled cells. + let m3_passed = evaluate_milestone_gate( + MilestoneGate::M3Acceptance, + &spec, + &empty_canonical, + &reports, + ); + assert!(m3_passed.passed, "{:?}", m3_passed.violations); + assert_eq!(m3_passed.parity_verified_subjects, vec!["maka".to_owned()]); + + let m5_incomplete = evaluate_milestone_gate( + MilestoneGate::M5ProductionDelivery, + &spec, + &empty_canonical, + &reports, + ); + assert!(!m5_incomplete.passed); + assert!( + m5_incomplete + .violations + .iter() + .any(|v| v.contains("incomplete")) + ); + + // Mutating public subject settings after parity verification invalidates the gate. + let mut drifted_spec_json = configured_spec_json.clone(); + drifted_spec_json["subjects"][0]["config"]["sandboxMode"] = json!("danger-full-access"); + let drifted_spec = parse_experiment_spec(&drifted_spec_json).unwrap(); + let m3_drifted = evaluate_milestone_gate( + MilestoneGate::M3Acceptance, + &drifted_spec, + &empty_canonical, + &reports, + ); + assert!(!m3_drifted.passed); + assert!( + m3_drifted + .violations + .iter() + .any(|v| v.contains("does not match declared public configuration")) + ); + + // With all cells settled cleanly AND parity verified, M5 production delivery passes. + let mut settled_canonical = std::collections::BTreeMap::new(); + settled_canonical.insert( + "task-1::1::maka".to_owned(), + V1CellAttempt { + cell_id: "task-1::1::maka".into(), + sequence: 1, + started_at: 100, + completed_at: 200, + result: EvalResult { + status: EvalResultStatus::Completed, + score: Some(1.0), + duration_ms: 100, + usage: Some(NormalizedUsage { + input_tokens: 50, + output_tokens: 10, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 60, + }), + cost_usd: None, + failure_reason: None, + artifacts: vec![], + }, + }, + ); + let m5_passed = evaluate_milestone_gate( + MilestoneGate::M5ProductionDelivery, + &spec, + &settled_canonical, + &reports, + ); + assert!(m5_passed.passed, "{:?}", m5_passed.violations); +} + +#[test] +fn parity_gate_detects_instruction_tool_pruning_and_policy_drift() { + let baseline = maka_eval::parity::ParitySnapshot { + system_prompt_text: "You are Maka.".into(), + request_system_prompt: Some("You are Maka.".into()), + compaction_prompt: Some("Compact context.".into()), + language: json!("auto"), + dynamic_context: vec!["AGENTS.md".into()], + composition_sources: vec![json!({"plugin": "maka.agent.agent", "version": "1.0.0"})], + allowed_tools: None, + tool_discovery: true, + tool_selection: json!({"mode": "all"}), + tool_mode: json!("code_mode"), + code_language: json!("python"), + editing_tools: json!("replace"), + tool_catalog_digest: "digest-full".into(), + tools: vec![ + json!({"name": "AskUserQuestion"}), + json!({"name": "WebSearch"}), + ], + collaboration: json!("agent"), + approval_policy: json!("ask"), + sandbox_policy: json!({"access": "workspace-write"}), + thinking_level: json!(null), + provider_options: json!({}), + max_output_tokens: json!(16384), + context_policy: json!({"contextWindow": 200000}), + wire_path: "/v1/chat/completions".into(), + wire_model: Some("gpt-4.1".into()), + wire_messages: json!([ + {"role": "system", "content": "You are Maka."}, + {"role": "user", "content": "Fix the bug."} + ]), + wire_tools: json!([ + {"type": "function", "function": {"name": "AskUserQuestion"}}, + {"type": "function", "function": {"name": "WebSearch"}} + ]), + wire_options: json!({"stream": true}), + }; + + let ok_report = compare_parity_snapshots( + "maka", + "0.2.0", + "exe-sha", + "cfg-sha", + &baseline, + &baseline, + vec![], + ); + assert!(ok_report.passed); + assert!(ok_report.checks.iter().all(|c| c.passed)); + assert!(ok_report.mismatches.is_empty()); + + // Injecting benchmark hints into prompts fails effective_instructions. + let mut injected = baseline.clone(); + injected.wire_messages = json!([ + {"role": "system", "content": "You are Maka. Extra benchmark hint: run pytest first."}, + {"role": "user", "content": "Fix the bug."} + ]); + let report = compare_parity_snapshots( + "maka", + "0.2.0", + "exe-sha", + "cfg-sha", + &baseline, + &injected, + vec![], + ); + assert!(!report.passed); + assert!( + report + .mismatches + .iter() + .any(|m| m.dimension == "effective_instructions") + ); + + // Pruning web or interactive tools fails tool_exposure. + let mut pruned = baseline.clone(); + pruned.allowed_tools = Some(vec!["AskUserQuestion".into()]); + pruned.wire_tools = json!([ + {"type": "function", "function": {"name": "AskUserQuestion"}} + ]); + let report = compare_parity_snapshots( + "maka", + "0.2.0", + "exe-sha", + "cfg-sha", + &baseline, + &pruned, + vec![], + ); + assert!(!report.passed); + assert!( + report + .mismatches + .iter() + .any(|m| m.dimension == "tool_exposure" && m.field == "evalAllowedToolsUnpruned") + ); + assert!( + report + .mismatches + .iter() + .any(|m| m.dimension == "tool_exposure" && m.field == "wireTools") + ); + + // Silently bypassing approvals or changing sandbox/context policy fails execution_policies. + let mut bypassed = baseline.clone(); + bypassed.approval_policy = json!("never"); + bypassed.sandbox_policy = json!({"access": "danger-full-access"}); + let report = compare_parity_snapshots( + "maka", + "0.2.0", + "exe-sha", + "cfg-sha", + &baseline, + &bypassed, + vec![], + ); + assert!(!report.passed); + assert!( + report + .mismatches + .iter() + .any(|m| m.dimension == "execution_policies" && m.field == "approvalPolicy") + ); + + // Changing bundled plugin composition fails product_composition. + let mut custom_bundle = baseline.clone(); + custom_bundle.composition_sources = vec![json!({"plugin": "eval.custom.agent"})]; + let report = compare_parity_snapshots( + "maka", + "0.2.0", + "exe-sha", + "cfg-sha", + &baseline, + &custom_bundle, + vec![], + ); + assert!(!report.passed); + assert!( + report + .mismatches + .iter() + .any(|m| m.dimension == "product_composition") + ); + + // Declared endpoint route parity: DeepSeek host with custom model alias lacking "deepseek" + // must declare explicit applyPatch override so loopback metering gateway does not alter EditingTools. + let ambiguous_deepseek = MakaSubjectConfig::decode( + json!({ + "makaExecutable": "/opt/maka-agent/maka", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 60000, + "baseUrl": "https://api.deepseek.com/v1", + "model": "custom-coder-alias" + }) + .as_object() + .unwrap(), + &["DEEPSEEK_API_KEY".into()], + ) + .unwrap(); + let route_mismatches = check_declared_endpoint_route_parity(&ambiguous_deepseek); + assert_eq!(route_mismatches.len(), 1); + assert_eq!(route_mismatches[0].dimension, "tool_exposure"); + + let explicit_deepseek = MakaSubjectConfig::decode( + json!({ + "makaExecutable": "/opt/maka-agent/maka", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 60000, + "baseUrl": "https://api.deepseek.com/v1", + "model": "custom-coder-alias", + "modelOverride": { "applyPatch": true } + }) + .as_object() + .unwrap(), + &["DEEPSEEK_API_KEY".into()], + ) + .unwrap(); + assert!(check_declared_endpoint_route_parity(&explicit_deepseek).is_empty()); +} + +#[cfg(unix)] +#[tokio::test] +async fn unattended_maka_execution_preserves_interaction_semantics_without_silent_bypass() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + struct LocalCommandRunner(std::collections::HashMap); + impl CommandRunner for LocalCommandRunner { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut cmd = tokio::process::Command::new(&request.command); + cmd.args(&request.args) + .env_remove("MAKA_EVAL_RESULT_TOKEN") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + for (k, v) in &request.environment { + cmd.env(k, v); + } + for (target, source) in &request.credential_environment { + if let Some(val) = self.0.get(source) { + cmd.env(target, val); + } + } + let out = cmd.output().await.map_err(|e| e.to_string())?; + Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: out.status.code().unwrap_or(1), + stdout: String::from_utf8_lossy(&out.stdout).into_owned(), + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + diagnostic: None, + }) + }) + } + } + + let temp = TempDir::new().unwrap(); + let (maka, _eval) = stage_real_maka_and_eval_binaries(temp.path()); + let workspace = temp.path().join("workspace"); + let artifact_root = temp.path().join("artifacts"); + fs::create_dir_all(&workspace).unwrap(); + + // Start a local OpenAI-compatible SSE server that instructs Maka to invoke `AskUserQuestion`. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let shutdown = CancellationToken::new(); + let shutdown_bg = shutdown.clone(); + let server_task = tokio::spawn(async move { + loop { + let accepted = tokio::select! { + _ = shutdown_bg.cancelled() => break, + res = listener.accept() => match res { + Ok(v) => v, + Err(_) => break, + } + }; + let (mut stream, _) = accepted; + tokio::spawn(async move { + let mut buf = vec![0u8; 65536]; + let _ = stream.read(&mut buf).await; + let args_str = serde_json::to_string(&json!({ + "questions": [ + { + "question": "Should we deploy to production?", + "options": [ + { "label": "Yes" }, + { "label": "No" } + ] + } + ] + })) + .unwrap(); + let chunk1 = json!({ + "id": "chatcmpl-interact-1", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1", + "choices": [{ + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [{ + "index": 0, + "id": "call_question_1", + "type": "function", + "function": { + "name": "AskUserQuestion", + "arguments": args_str + } + }] + }, + "finish_reason": null + }] + }); + let chunk2 = json!({ + "id": "chatcmpl-interact-1", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1", + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "tool_calls" + }], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 12, + "total_tokens": 37 + } + }); + let body = format!("data: {chunk1}\n\ndata: {chunk2}\n\ndata: [DONE]\n\n"); + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(resp.as_bytes()).await; + }); + } + }); + + let spec = parse_experiment_spec(&json!({ + "schemaVersion": "maka.eval.v2", + "id": "unattended-interaction-test", + "benchmark": { "id": "tb", "version": "1.0", "config": {} }, + "executor": { "kind": "harbor", "config": {} }, + "subjects": [{ + "id": "maka", + "kind": "maka", + "credentials": ["OPENAI_API_KEY"], + "config": { + "makaExecutable": maka.to_string_lossy(), + "runtimeHostsPath": temp.path().join("hosts").to_string_lossy(), + "hostSettlementTimeoutMs": 20000, + "baseUrl": format!("http://127.0.0.1:{port}/v1"), + "model": "gpt-4.1" + } + }], + "tasks": [{ "id": "t1", "input": "Deploy the service.", "config": {} }], + "repetitions": 1, + "budget": { "maxSteps": 5 }, + "verifier": {} + })) + .unwrap(); + let cell = &expand_experiment(&spec)[0]; + + let mut metadata = Map::new(); + metadata.insert( + "artifactRoot".into(), + Value::String(artifact_root.to_string_lossy().into_owned()), + ); + let result = RustHostSubject::new() + .execute( + cell, + SubjectExecutionContext { + cwd: workspace.to_string_lossy().into_owned(), + task_input: cell.task.input.clone(), + metadata, + cancel: CancellationToken::new(), + runner: Arc::new(LocalCommandRunner(std::collections::HashMap::from([( + "OPENAI_API_KEY".into(), + "sk-test-unattended".into(), + )]))), + }, + ) + .await; + + shutdown.cancel(); + let _ = server_task.await; + + assert_eq!(result.outcome, ExecutionOutcome::SubjectFailed); + assert_eq!( + result.failure_reason.as_deref(), + Some("Unattended execution blocked on required interaction: question") + ); + assert!( + result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("unattended-interaction-blocked") + && a.get("interactionKind").and_then(Value::as_str) == Some("question") + }), + "expected structured unattended-interaction-blocked artifact, got {:?}", + result.artifacts + ); + let recorded_interactions: Value = serde_json::from_slice( + &fs::read(artifact_root.join("unattended-interactions.json")) + .expect("unattended-interactions.json artifact must be written"), + ) + .unwrap(); + assert_eq!( + recorded_interactions["entries"][0]["kind"], + json!("question") + ); +} diff --git a/products/eval/tests/support/native_host.py b/products/eval/tests/support/native_host.py new file mode 100644 index 00000000000..0207df01438 --- /dev/null +++ b/products/eval/tests/support/native_host.py @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import base64 +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +class Provider(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + body = json.dumps({ + "object": "list", + "data": [{"id": "deepseek-v4-flash", "object": "model"}], + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + + captured_requests = [] + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + raw_body = self.rfile.read(length) if length > 0 else b"{}" + Provider.captured_requests.append(json.loads(raw_body.decode())) + event = { + "id": "eval-test", "object": "chat.completion.chunk", + "created": 1, "model": "deepseek-v4-flash", + } + events = [ + {**event, "choices": [{"index": 0, "delta": { + "role": "assistant", "content": "Done.", + }, "finish_reason": None}]}, + {**event, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12, + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }}, + ] + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + for item in events: + self.wfile.write(("data: " + json.dumps(item) + "\n\n").encode()) + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + + +server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) +threading.Thread(target=server.serve_forever, daemon=True).start() +with tempfile.TemporaryDirectory(prefix="maka-native-eval-") as raw: + root = Path(raw) + workspace = root / "workspace" + workspace.mkdir() + request = { + "makaExecutable": os.environ["MAKA_TEST_MAKA"], + "rootPath": str(root / "host"), + "artifactRoot": str(root / "artifacts"), + "baseUrl": f"http://127.0.0.1:{server.server_port}/v1", + "hostSettlementTimeoutMs": 15000, + "hostInitSettings": { + "connections": [{ + "slug": "eval-gateway", "name": "Eval Gateway", + "provider": "openai-compatible", + "configuration": {"baseUrl": "http://placeholder/v1"}, + "apiKey": "EVAL_SMOKE_KEY", "enabled": True, + "enabledModelIds": ["deepseek-v4-flash"], + }], + "defaultTarget": {"slug": "eval-gateway", "modelId": "deepseek-v4-flash"}, + }, + "execution": { + "executionId": "native-smoke", + "workspacePath": str(workspace), + "sessionSettings": {}, + "content": {"text": "Reply Done."}, "maxSteps": 10, + }, + } + payload = base64.urlsafe_b64encode(json.dumps(request).encode()).decode().rstrip("=") + token = "a" * 32 + child = subprocess.Popen( + [os.environ["MAKA_TEST_EVAL"], "host-subject", payload], + env={**os.environ, "EVAL_SMOKE_KEY": "dummy-test-key", "MAKA_EVAL_RESULT_TOKEN": token}, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True, + ) + try: + stdout, stderr = child.communicate(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGTERM) + try: + child.communicate(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(child.pid, signal.SIGKILL) + child.communicate() + raise AssertionError("native Eval Host did not settle") from None + assert child.returncode == 0, stderr + prefix, framed_token, length, digest, encoded = stdout.strip().split(" ") + assert prefix == "MAKA-EVAL-RESULT-V1" and framed_token == token + body = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + assert len(body) == int(length) and hashlib.sha256(body).hexdigest() == digest + data = json.loads(body) + assert data["status"] == "completed", data + assert data["usage"] == dict( + inputTokens=10, outputTokens=2, cacheReadTokens=0, + cacheWriteTokens=0, reasoningTokens=0, totalTokens=12, + ), data + assert Provider.captured_requests, "provider should observe outbound request" + assert len(Provider.captured_requests[0].get("tools", [])) > 0, ( + f"default product tools must not be pruned: {list(Provider.captured_requests[0].keys())}" + ) + receipt = json.loads((root / "artifacts" / "execution-receipt.json").read_text()) + assert receipt["operationId"] == data["operationId"] + assert receipt["runId"] == data["invocationId"] + assert receipt["rootPath"] == str(root / "host") + assert receipt["sessionId"] + transcript = json.loads((root / "artifacts" / "transcript.json").read_text()) + assert any("Done." in chunk.get("text", "") for page in transcript for chunk in page["chunks"]) + +server.shutdown() diff --git a/scripts/ci.test.mjs b/scripts/ci.test.mjs index 5fb25089e8b..24d984dda5a 100644 --- a/scripts/ci.test.mjs +++ b/scripts/ci.test.mjs @@ -151,6 +151,7 @@ test('helper process changes reach integration consumers outside the Cargo depen const packages = [ ['maka-python-runtime', 'platform/python-runtime'], ['maka-cli', 'products/maka/cli'], + ['maka-eval', 'products/eval'], ['maka-agent-native', 'plugins/agent/native'], ['maka-computer-native', 'plugins/computer/native'], ].map(([name, path]) => ({ diff --git a/scripts/ci/suites.mjs b/scripts/ci/suites.mjs index 6864eac623d..80e476dfce6 100644 --- a/scripts/ci/suites.mjs +++ b/scripts/ci/suites.mjs @@ -20,6 +20,7 @@ import { readFileSync } from 'node:fs'; export const taskHelperConsumers = [ + 'maka-eval', 'maka-cli', 'maka-agent-native', 'maka-agent-engine', @@ -82,7 +83,12 @@ export const suites = { packages: 'workspace', cargoArgs: ['--locked', '--workspace'], profile: 'ci', - extraInputs: [...embeddedInputs, ...distributionInputs, 'products/maka/workbench/tests/fonts'], + extraInputs: [ + ...embeddedInputs, + ...distributionInputs, + 'products/maka/workbench/tests/fonts', + 'products/eval/tests/support', + ], affectedFilter: true, }, bridge: { From 4eddc488c70b912817a171e116c9eeeab66b1a3d Mon Sep 17 00:00:00 2001 From: colafornia Date: Fri, 13 Nov 2026 08:56:11 +0800 Subject: [PATCH 3/3] fix(eval): preserve model route resolution and enforce runtime binary identity - Resolve actual outbound model routes through Maka provider policy before attaching local metering gateway forwarding, and preserve untouched endpoint semantics on the parity reference path. - Record executed Maka binary digests in settled runtime artifacts and require confirmed runtime binary identity in M3 and M5 milestone gates. Generated-by: Google Antigravity --- Cargo.lock | 3 + products/eval/Cargo.toml | 3 + products/eval/README.md | 6 +- products/eval/README.zh-CN.md | 6 +- products/eval/src/measurement/gateway.rs | 33 +- products/eval/src/parity.rs | 532 ++++++++++++++++++- products/eval/src/report.rs | 93 +++- products/eval/src/subject/maka.rs | 220 ++++++-- products/eval/src/subject/maka/native.rs | 75 ++- products/eval/tests/integration.rs | 572 ++++++++++++++++++++- products/eval/tests/support/native_host.py | 4 + 11 files changed, 1451 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 344c222b1a3..8357c83ee0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6921,9 +6921,12 @@ dependencies = [ "hyper", "hyper-util", "libc", + "maka-agent-facts", + "maka-agent-ports", "maka-app-client", "maka-host-runtime", "maka-plugin-core", + "maka-providers", "regex", "reqwest", "rusqlite", diff --git a/products/eval/Cargo.toml b/products/eval/Cargo.toml index 3e3b555c696..6f867795cc5 100644 --- a/products/eval/Cargo.toml +++ b/products/eval/Cargo.toml @@ -27,9 +27,12 @@ name = "maka-eval" path = "src/main.rs" [dependencies] +maka-agent-facts.workspace = true +maka-agent-ports.workspace = true maka-app-client.workspace = true maka-host-runtime.workspace = true maka-plugin-core.workspace = true +maka-providers.workspace = true base64 = "0.23.1" clap.workspace = true futures-util.workspace = true diff --git a/products/eval/README.md b/products/eval/README.md index 0d6e0e903e0..7c3e5a27ea2 100644 --- a/products/eval/README.md +++ b/products/eval/README.md @@ -27,11 +27,11 @@ Measurement and control plane for multi-arm benchmark experiments (`maka.eval.v1 Install matching `maka` and `maka-eval` binaries side by side inside the Trial image. Set `subject.config.makaExecutable` to the native `maka` path; `nodePath`, `shimPath`, legacy provider configuration, and benchmark-only profiles (`toolProfile`, `permissionMode: "bypass"`, `collaborationMode`, `orchestrationMode`) are rejected. -Maka subjects are configured strictly through public product interfaces (`provider`, `connectionSlug`, `connectionName`, `baseUrl`, `model`, optional `modelOverride`, `requestHeaders`, `requestBodyOverlay`, `sandboxMode`, `requestPermissions`, `thinkingLevel`, `language`, `workspaceInstructions`, and `apiKeyEnvironment`). The adapter initializes an isolated Root via `maka host init --settings-stdin`, applies only declared public Settings/Session configurations, and submits tasks through the production composer draft/receipt client without prompt injection, tool pruning, or synthetic model overrides. A non-mutating local metering gateway records provider usage while forwarding outbound JSON bodies verbatim. Pending interactive approvals or questions are never auto-bypassed in unattended runs; they settle explicitly as `unattended-interaction-blocked`, and host/environment constraints are reported as `environment-limitation`. +Maka subjects are configured strictly through public product interfaces (`provider`, `connectionSlug`, `connectionName`, `baseUrl`, `model`, optional `modelOverride`, `requestHeaders`, `requestBodyOverlay`, `sandboxMode`, `requestPermissions`, `thinkingLevel`, `language`, `workspaceInstructions`, and `apiKeyEnvironment`). The adapter resolves Maka's actual outbound model route from the declared endpoint before attaching the local metering gateway, initializes an isolated Root via `maka host init --settings-stdin`, applies only declared public Settings/Session configurations, and submits tasks through the production composer draft/receipt client without prompt injection, tool pruning, or synthetic model overrides. The non-mutating local metering gateway records provider usage while forwarding outbound JSON bodies verbatim to the product-resolved route, and records the executed `maka` binary path and SHA-256 digest in settled runtime artifacts. Pending interactive approvals or questions are never auto-bypassed in unattended runs; they settle explicitly as `unattended-interaction-blocked`, and host/environment constraints are reported as `environment-limitation`. Product parity is a mandatory gate for `M3` acceptance and `M5` production delivery (`RunFingerprint` documents run conditions only and does not establish parity): -- `maka-eval parity [--subject ] [--out ]` runs both the normal Maka client path and the `maka-eval` entry point against a local probe under identical inputs and verifies committed `RunSpec`/`RequestSpec` facts plus wire payloads across product composition, effective instructions, tool exposure, and execution policies. -- `maka-eval report --milestone m3|m5 --parity-report ` enforces parity (and complete non-indeterminate settlement for `m5`). +- `maka-eval parity [--subject ] [--out ]` preserves original declared endpoint semantics on the reference path while driving `maka-eval`'s entry point against a local probe under identical inputs, verifying committed `RunSpec`/`RequestSpec` facts, resolved upstream target routes, and wire payloads across product composition, effective instructions, tool exposure, and execution policies. +- `maka-eval report --out --milestone m3|m5 --parity-report ` enforces parity, confirms the executed Maka binary identity from settled runtime execution evidence (supporting isolated container paths while failing closed when runtime evidence is absent or mismatched), and requires complete non-indeterminate settlement for `m5`. `egressProxy` currently fails preflight because the native executor does not provision the required sidecar and mounts. Unconfirmed cleanup blocks further admission across runs until a `PostHocRecovery` entry confirms disposal. Both report versions read committed journal records; result JSON files are no longer a separate authority. Records without pricing evidence retain unknown cost. diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md index a163b68d29d..4d82de49708 100644 --- a/products/eval/README.zh-CN.md +++ b/products/eval/README.zh-CN.md @@ -27,11 +27,11 @@ 在 Trial 镜像内将匹配版本的 `maka` 和 `maka-eval` 放在同一目录，`subject.config.makaExecutable` 指向原生 `maka`。拒绝 `nodePath`、`shimPath`、旧 provider 配置及仅面向评测的专用参数（`toolProfile`、`permissionMode: "bypass"`、`collaborationMode`、`orchestrationMode`）。 -Maka 被测对象仅通过公开产品接口配置（`provider`、`connectionSlug`、`connectionName`、`baseUrl`、`model`、可选 `modelOverride`、`requestHeaders`、`requestBodyOverlay`、`sandboxMode`、`requestPermissions`、`thinkingLevel`、`language`、`workspaceInstructions` 与 `apiKeyEnvironment`）。适配器通过 `maka host init --settings-stdin` 初始化隔离的 Root，仅应用显式声明的公开 Settings/Session 配置，并通过生产客户端的 composer 草稿与回执协议提交任务，不注入评测提示词、不裁剪或替换工具、不注入合成模型参数。本地计量网关以只读方式统计提供商用量，原样转发请求体。无人值守执行不会静默绕过审批或交互请求，遇到挂起交互时以 `unattended-interaction-blocked` 显式结算；工作区或环境受限时以 `environment-limitation` 显式上报。 +Maka 被测对象仅通过公开产品接口配置（`provider`、`connectionSlug`、`connectionName`、`baseUrl`、`model`、可选 `modelOverride`、`requestHeaders`、`requestBodyOverlay`、`sandboxMode`、`requestPermissions`、`thinkingLevel`、`language`、`workspaceInstructions` 与 `apiKeyEnvironment`）。适配器先按声明的原始端点解析产品实际模型路由再接入本地计量转发，通过 `maka host init --settings-stdin` 初始化隔离的 Root，仅应用显式声明的公开 Settings/Session 配置，并通过生产客户端的 composer 草稿与回执协议提交任务，不注入评测提示词、不裁剪或替换工具、不注入合成模型参数。本地计量网关以只读方式统计提供商用量并将请求体原样转发至产品解析出的目标路由，同时在结算产物中记录实际运行的 `maka` 二进制路径与 SHA-256 摘要。无人值守执行不会静默绕过审批或交互请求，遇到挂起交互时以 `unattended-interaction-blocked` 显式结算；工作区或环境受限时以 `environment-limitation` 显式上报。 产品等价性是 `M3` 验收与 `M5` 生产交付的前置准入门槛（`RunFingerprint` 仅记录运行条件，不构成产品等价性证明）： -- `maka-eval parity [--subject ] [--out ]` 在相同版本、公开配置、工作区与任务输入下同时驱动常规 Maka 客户端路径与 `maka-eval` 入口，比对落盘的 `RunSpec`/`RequestSpec` 事实与出站报文，验证产品组合、生效指令、工具暴露及执行策略等价。 -- `maka-eval report --milestone m3|m5 --parity-report ` 强制校验等价性报告（`m5` 同时要求全部单元完成确定性结算）。 +- `maka-eval parity [--subject ] [--out ]` 在相同版本、公开配置、工作区与任务输入下保留参考路径的原始端点语义，同时驱动 `maka-eval` 入口连接本地探针，比对落盘的 `RunSpec`/`RequestSpec` 事实、解析后的上游目标路由与出站报文，验证产品组合、生效指令、工具暴露及执行策略等价。 +- `maka-eval report --out --milestone m3|m5 --parity-report ` 强制校验等价性报告，并依据已结算的运行时执行证据确认实际 Maka 二进制身份（支持隔离容器路径，缺少或摘要不匹配时显式失败；`m5` 同时要求全部单元完成确定性结算）。 当前原生执行器尚未配置代理侧车与挂载，因此 `egressProxy` 会在预检失败。清理未确认的尝试会跨次运行阻断准入，直到 `PostHocRecovery` 记录确认资源已处置。两版报告均读取已提交日志，不再将结果 JSON 文件作为另一权威；缺少定价依据的记录保持成本未知。 diff --git a/products/eval/src/measurement/gateway.rs b/products/eval/src/measurement/gateway.rs index fc88a934ebe..f7477f94280 100644 --- a/products/eval/src/measurement/gateway.rs +++ b/products/eval/src/measurement/gateway.rs @@ -304,9 +304,22 @@ pub fn join_upstream_url(base_url: &str, request_uri: &str) -> Result Result { + let target = Url::parse(target_url).map_err(|e| format!("invalid target URL: {e}"))?; + let mut dest = + Url::parse(origin_override).map_err(|e| format!("invalid origin override: {e}"))?; + dest.set_path(target.path()); + dest.set_query(target.query()); + dest.set_fragment(None); + Ok(dest.to_string()) +} + +pub const UPSTREAM_TARGET_HEADER: &str = "x-maka-eval-upstream-target"; + #[derive(Debug, Clone)] pub struct GatewayConfig { pub upstream_base_url: String, + pub transport_origin_override: Option, pub upstream_api_key: String, pub profile: String, pub gateway_credential: String, @@ -632,10 +645,26 @@ async fn handle_gateway_request( )); } }; + let connect_url = match &state.config.transport_origin_override { + Some(origin) => match rewrite_url_origin(&target_url, origin) { + Ok(u) => u, + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "invalid gateway transport origin override", + )); + } + }, + None => target_url.clone(), + }; let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::POST); - let mut upstream_req = state.client.request(reqwest_method, &target_url); + let mut upstream_req = state.client.request(reqwest_method, &connect_url); + if state.config.transport_origin_override.is_some() { + upstream_req = upstream_req.header(UPSTREAM_TARGET_HEADER, &target_url); + } let uses_x_api_key = headers.contains_key("x-api-key") && !headers.contains_key(hyper::header::AUTHORIZATION); for (name, value) in &headers { @@ -648,6 +677,7 @@ async fn handle_gateway_request( | "connection" | "authorization" | "x-api-key" + | UPSTREAM_TARGET_HEADER ) { continue; } @@ -860,6 +890,7 @@ mod tests { let gateway = EvalMeteringGateway::start(GatewayConfig { upstream_base_url: format!("http://{upstream_addr}/v1"), + transport_origin_override: None, upstream_api_key: "real-upstream-secret".into(), profile: "maka".into(), gateway_credential: "attempt-gateway-token".into(), diff --git a/products/eval/src/parity.rs b/products/eval/src/parity.rs index 1b6b6a0a07b..8d7fe43a56f 100644 --- a/products/eval/src/parity.rs +++ b/products/eval/src/parity.rs @@ -39,12 +39,15 @@ use crate::environment::{ BoxFuture, CommandRunner, SubjectExecutionContext, SubjectProcessOutput, SubjectProcessRequest, SubjectProcessTermination, }; +use crate::measurement::{UPSTREAM_TARGET_HEADER, join_upstream_url}; use crate::subject::SubjectAdapter; use crate::subject::maka::native::{ apply_public_host_preferences, commit_client_mutation, sandbox_mode_to_workspace_access, - shutdown_host_client, start_host_client, + shutdown_host_client, start_host_client_with_env, +}; +use crate::subject::maka::{ + MakaSubjectConfig, RustHostSubject, bootstrap_maka_host_root, url_route_signature, }; -use crate::subject::maka::{MakaSubjectConfig, RustHostSubject, bootstrap_maka_host_root}; use http_body_util::{BodyExt, Full}; use hyper::body::{Bytes, Incoming}; use hyper::server::conn::http1; @@ -62,6 +65,7 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::process::Command; use tokio::sync::Mutex; @@ -157,6 +161,8 @@ pub struct ParitySnapshot { pub max_output_tokens: Value, pub context_policy: Value, pub wire_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upstream_target_url: Option, pub wire_model: Option, pub wire_messages: Value, pub wire_tools: Value, @@ -166,9 +172,70 @@ pub struct ParitySnapshot { #[derive(Debug, Clone)] struct CapturedWireRequest { path: String, + upstream_target: Option, body: Value, } +struct OfflineProxyGuard { + addr: std::net::SocketAddr, + shutdown: CancellationToken, + worker: Option>, +} + +impl OfflineProxyGuard { + async fn start() -> Result { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .map_err(|e| format!("failed to bind offline proxy guard: {e}"))?; + let addr = listener + .local_addr() + .map_err(|e| format!("failed to read offline proxy guard address: {e}"))?; + let shutdown = CancellationToken::new(); + let shutdown_clone = shutdown.clone(); + let worker = tokio::spawn(async move { + loop { + tokio::select! { + biased; + _ = shutdown_clone.cancelled() => break, + accepted = listener.accept() => { + let Ok((mut stream, _)) = accepted else { break; }; + tokio::spawn(async move { + let mut buf = [0u8; 4096]; + let _ = tokio::time::timeout( + Duration::from_millis(200), + stream.read(&mut buf), + ) + .await; + let _ = stream + .write_all( + b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await; + let _ = stream.shutdown().await; + }); + } + } + } + }); + Ok(Self { + addr, + shutdown, + worker: Some(worker), + }) + } + + fn url(&self) -> String { + format!("http://{}", self.addr) + } + + async fn close(mut self) { + self.shutdown.cancel(); + if let Some(w) = self.worker.take() { + w.abort(); + } + } +} + struct ParityProbeServer { addr: std::net::SocketAddr, captured: Arc>>, @@ -249,6 +316,11 @@ async fn handle_probe_request( ) -> Result>, Infallible> { let method = req.method().clone(); let path = req.uri().path().to_owned(); + let upstream_target = req + .headers() + .get(UPSTREAM_TARGET_HEADER) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); let bytes = req .into_body() .collect() @@ -287,6 +359,7 @@ async fn handle_probe_request( if method == hyper::Method::POST { store.lock().await.push(CapturedWireRequest { path: path.clone(), + upstream_target, body: body_json, }); } @@ -406,6 +479,7 @@ async fn handle_probe_request( struct LocalProcessRunner { work_dir: PathBuf, timeout: Duration, + extra_env: Vec<(String, String)>, } impl CommandRunner for LocalProcessRunner { @@ -418,10 +492,14 @@ impl CommandRunner for LocalProcessRunner { cmd.args(&request.args) .current_dir(&self.work_dir) .env_remove("MAKA_EVAL_RESULT_TOKEN") + .env_remove("REQUEST_METHOD") .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .kill_on_drop(true); + for (k, v) in &self.extra_env { + cmd.env(k, v); + } for (k, v) in &request.environment { cmd.env(k, v); } @@ -508,27 +586,51 @@ pub async fn verify_product_parity( let maka_executable_sha256 = sha256_hex(&maka_bytes); let maka_version = read_maka_version(&maka_path).await?; + let offline_proxy = OfflineProxyGuard::start().await?; + let offline_proxy_url = offline_proxy.url(); let probe = ParityProbeServer::start(config.model.clone()).await?; - let probe_base_url = probe.base_url_for(&config.base_url); + let probe_origin = format!("http://{}", probe.addr); let temp = tempfile::tempdir().map_err(|e| format!("tempdir failed: {e}"))?; let shared_workspace = temp.path().join("workspace"); prepare_shared_workspace(input.workspace_source.as_deref(), &shared_workspace)?; - let normal_root = temp.path().join("normal-root"); - let normal_journal_dir = temp.path().join("normal-client"); - fs::create_dir_all(&normal_journal_dir).map_err(|e| e.to_string())?; + let normal_origin_root = temp.path().join("normal-origin-root"); + let normal_origin_journal_dir = temp.path().join("normal-origin-client"); + fs::create_dir_all(&normal_origin_journal_dir).map_err(|e| e.to_string())?; - let normal_session_id = run_normal_product_client( + let origin_prep = run_normal_origin_preparation( &config, - &probe_base_url, - &normal_root, - &normal_journal_dir, + &offline_proxy_url, + &normal_origin_root, + &normal_origin_journal_dir, &shared_workspace, &input.task_input, ) .await; - let normal_requests = probe.take_requests().await; + + let (normal_wire_res, mut normal_requests) = match &origin_prep { + Ok((_, accepted_route_base_url)) => { + let normal_wire_root = temp.path().join("normal-wire-root"); + let normal_wire_journal_dir = temp.path().join("normal-wire-client"); + fs::create_dir_all(&normal_wire_journal_dir).map_err(|e| e.to_string())?; + let probe_base_url = probe.base_url_for(accepted_route_base_url); + let res = run_normal_product_client( + &config, + &probe_base_url, + &offline_proxy_url, + &normal_wire_root, + &normal_wire_journal_dir, + &shared_workspace, + &input.task_input, + ) + .await + .map(|sid| (normal_wire_root, sid)); + let reqs = probe.take_requests().await; + (res, reqs) + } + Err(err) => (Err(err.clone()), Vec::new()), + }; let eval_roots_dir = temp.path().join("eval-roots"); let eval_artifacts_dir = temp.path().join("eval-artifacts"); @@ -539,7 +641,8 @@ pub async fn verify_product_parity( &input.subject_id, &input.subject_config, &input.subject_credentials, - &probe_base_url, + &probe_origin, + &offline_proxy_url, &eval_roots_dir, &eval_artifacts_dir, &shared_workspace, @@ -554,27 +657,38 @@ pub async fn verify_product_parity( .await; let eval_requests = probe.take_requests().await; probe.close().await; + offline_proxy.close().await; - let normal_session_id = normal_session_id?; + let (normal_origin_session_id, accepted_route_base_url) = origin_prep?; + let (normal_wire_root, normal_wire_session_id) = normal_wire_res?; let (eval_root, eval_session_id) = eval_run?; - verify_maka_inspect(&maka_path, &normal_root, &normal_session_id).await?; + verify_maka_inspect(&maka_path, &normal_origin_root, &normal_origin_session_id).await?; + verify_maka_inspect(&maka_path, &normal_wire_root, &normal_wire_session_id).await?; verify_maka_inspect(&maka_path, &eval_root, &eval_session_id).await?; let normal_wire = normal_requests - .first() + .first_mut() .ok_or("normal Maka client did not emit a model request")?; + normal_wire.upstream_target = Some(join_upstream_url( + &accepted_route_base_url, + &normal_wire.path, + )?); let eval_wire = eval_requests .first() .ok_or("Eval Maka entry point did not emit a model request")?; let normal_snapshot = extract_committed_parity_snapshot( - &normal_root, - &normal_session_id, + &normal_origin_root, + &normal_origin_session_id, + &normal_wire_root, + &normal_wire_session_id, &shared_workspace, normal_wire, )?; let eval_snapshot = extract_committed_parity_snapshot( + &eval_root, + &eval_session_id, &eval_root, &eval_session_id, &shared_workspace, @@ -671,11 +785,320 @@ fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<(), String> { Ok(()) } +/// Runs Maka's normal public client flow with the untouched declared `config.base_url` +/// under an offline proxy guard so Maka commits `RunSpec`, `AcceptedModel` (with the true +/// product-resolved `route.base_url`), and `step_prepared` `RequestSpec` without external calls. +async fn run_normal_origin_preparation( + config: &MakaSubjectConfig, + offline_proxy_url: &str, + root_path: &Path, + journal_dir: &Path, + workspace_path: &Path, + task_input: &str, +) -> Result<(String, String), String> { + let deadline = Duration::from_millis(u64::from(config.host_settlement_timeout_ms)); + let maka_executable = Path::new(&config.maka_executable); + let api_key = std::env::var(&config.api_key_environment) + .unwrap_or_else(|_| "parity-probe-credential".into()); + let init_payload = config.host_init_settings(&config.base_url, api_key); + + tokio::time::timeout( + deadline, + bootstrap_maka_host_root(maka_executable, root_path, &init_payload), + ) + .await + .map_err(|_| "normal Maka origin Host initialization timed out".to_owned())??; + + let env_all_proxied = [ + ("ALL_PROXY", offline_proxy_url), + ("all_proxy", offline_proxy_url), + ("HTTP_PROXY", offline_proxy_url), + ("http_proxy", offline_proxy_url), + ("HTTPS_PROXY", offline_proxy_url), + ("https_proxy", offline_proxy_url), + ("NO_PROXY", ""), + ("no_proxy", ""), + ]; + + let journal = Journal::open(&journal_dir.join("client-journal.sqlite")).await?; + let (mut child, client) = start_host_client_with_env( + maka_executable, + root_path, + u64::from(config.host_settlement_timeout_ms), + journal.clone(), + &env_all_proxied, + ) + .await?; + + let workspace_str = workspace_path.to_string_lossy().into_owned(); + let configured = + apply_public_host_preferences(&workspace_str, &config.session_settings, &client).await; + let cleanup = shutdown_host_client(&mut child, &client, deadline).await; + configured?; + cleanup?; + + let (mut child, client) = start_host_client_with_env( + maka_executable, + root_path, + u64::from(config.host_settlement_timeout_ms), + journal, + &env_all_proxied, + ) + .await?; + + let session_res = async { + let connections = client + .query("settings", "connections", json!({"kind": "start"})) + .await?; + let connection = connections["connections"] + .as_array() + .and_then(|items| items.iter().find(|c| c["slug"] == config.connection_slug)) + .ok_or("normal Maka connection unavailable")?; + let backend = json!({ + "kind": "model", + "connectionId": connection["basis"]["connectionId"], + "modelId": config.model, + "thinkingLevel": config.session_settings.thinking_level, + }); + let created = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "create", + json!({ + "title": "Task", + "automaticTitle": false, + "labels": [], + "backend": backend, + "workspace": { + "kind": "directory", + "path": workspace_str, + }, + "tools": Value::Null, + }), + ) + .await?, + ) + .await?; + if created["kind"] != "created" { + return Err(format!("normal Maka session creation rejected: {created}")); + } + let session_obj = &created["session"]; + let session_id = session_obj["id"] + .as_str() + .ok_or("normal Maka session id missing")? + .to_owned(); + + if let Some(request_permissions) = config.session_settings.request_permissions { + let access = match &config.session_settings.sandbox_mode { + Some(mode) => sandbox_mode_to_workspace_access(mode)?.to_owned(), + None => session_obj["permissions"]["access"] + .as_str() + .unwrap_or("workspace_write") + .to_owned(), + }; + let revision = session_obj["revision"] + .as_u64() + .ok_or("normal session revision missing")?; + let updated = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "metadata", + json!({ + "sessionId": session_id, + "expectedRevision": revision, + "title": "Task", + "labels": [], + "permissions": { + "settings": { + "access": access, + "requestPermissions": request_permissions, + }, + "revokeGrants": false, + }, + }), + ) + .await?, + ) + .await?; + if updated["kind"] != "committed" { + return Err("normal Maka session permission update rejected".into()); + } + } + + let draft = client + .save_draft( + "agent", + &session_id, + "composer", + None, + task_input.to_owned().into(), + ) + .await?; + let mutation = client + .reserve_send_draft( + Uuid::new_v4(), + "agent", + &session_id, + "composer", + draft.revision, + json!({"backend": backend}), + ) + .await? + .open() + .await?; + let watch = client + .subscribe("agent", "watch", json!({"sessionId": session_id, "after": 0})) + .await?; + let accepted = commit_client_mutation(mutation).await?; + let run_id = match accepted["kind"].as_str() { + Some("started") => accepted["runId"] + .as_str() + .ok_or("normal Maka runId missing")? + .to_owned(), + Some("queued") => { + let msg_id = accepted["messageId"] + .as_str() + .ok_or("normal Maka queued messageId missing")?; + let mut wait_interval = tokio::time::interval(Duration::from_millis(50)); + loop { + wait_interval.tick().await; + let work = client + .query( + "agent", + "work", + json!({ + "sessionId": session_id, + "target": {"kind": "message", "messageId": msg_id}, + "cursor": null + }), + ) + .await?; + match work["kind"].as_str() { + Some("delivered") => { + break work["execution"]["runId"] + .as_str() + .ok_or("normal Maka delivered runId missing")? + .to_owned(); + } + Some("pending") => {} + _ => return Err(format!("normal Maka queued message failed: {work}")), + } + } + } + _ => return Err(format!("normal Maka send rejected: {accepted}")), + }; + + let mut prepared = false; + let mut stopping = false; + let mut interval = tokio::time::interval(Duration::from_millis(25)); + let mut next = Box::pin(watch.next()); + loop { + tokio::select! { + ev = &mut next => { + match ev? { + Some(SubscriptionEvent::Item(page)) => { + if let Some(events) = page.get("events").and_then(Value::as_array) { + for event in events.iter().filter(|e| e["runId"] == run_id) { + if event["fact"]["kind"] == "step_prepared" { + prepared = true; + } + } + } + if prepared && !stopping { + let exec = client + .query("agent", "execution", json!({"sessionId": session_id, "runId": run_id})) + .await?; + if matches!( + exec["execution"]["state"]["kind"].as_str(), + Some("admitted" | "running" | "waiting") + ) { + let _ = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "stop", + json!({"sessionId": session_id, "runId": run_id}), + ) + .await?, + ) + .await; + stopping = true; + } + } + next = Box::pin(watch.next()); + } + _ => return Err("normal Maka origin watch ended prematurely".into()), + } + } + _ = interval.tick() => { + let exec = client + .query("agent", "execution", json!({"sessionId": session_id, "runId": run_id})) + .await?; + match exec["execution"]["state"]["kind"].as_str() { + Some("completed" | "failed" | "cancelled") => break, + Some("admitted" | "running" | "waiting") if prepared && !stopping => { + let _ = commit_client_mutation( + client + .control( + Uuid::new_v4(), + "stop", + json!({"sessionId": session_id, "runId": run_id}), + ) + .await?, + ) + .await; + stopping = true; + } + _ => {} + } + } + } + } + drop(next); + watch.close().await?; + Ok::(session_id) + } + .await; + + let cleanup = shutdown_host_client(&mut child, &client, deadline).await; + let session_id = session_res?; + cleanup?; + + let accepted_route_base_url = read_accepted_route_base_url(root_path, &session_id)?; + Ok((session_id, accepted_route_base_url)) +} + +fn read_accepted_route_base_url(root_path: &Path, session_id: &str) -> Result { + let db_path = root_path.join(ROOT_SQLITE_FILENAME); + let conn = Connection::open_with_flags(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| format!("failed opening {}: {e}", db_path.display()))?; + let spec_bytes: Vec = conn + .query_row( + "SELECT c.payload FROM agent_run_specs s JOIN contents c ON c.id = s.spec_content_id WHERE s.session = ? ORDER BY s.rowid ASC LIMIT 1", + [session_id], + |row| row.get(0), + ) + .map_err(|e| format!("failed reading RunSpec for session {session_id}: {e}"))?; + let run_spec: Value = + serde_json::from_slice(&spec_bytes).map_err(|e| format!("invalid RunSpec JSON: {e}"))?; + let accepted_model = read_content_json(&conn, &run_spec["backend"]["descriptor"])?; + accepted_model + .pointer("/route/base_url") + .or_else(|| accepted_model.pointer("/route/baseUrl")) + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .ok_or_else(|| "AcceptedModel.route.base_url missing".into()) +} + /// Runs a session using Maka's normal public client flow directly against `probe_base_url` /// (with no `EvalMeteringGateway` and no `maka-eval` wrapper). async fn run_normal_product_client( config: &MakaSubjectConfig, probe_base_url: &str, + offline_proxy_url: &str, root_path: &Path, journal_dir: &Path, workspace_path: &Path, @@ -694,12 +1117,24 @@ async fn run_normal_product_client( .await .map_err(|_| "normal Maka Host initialization timed out".to_owned())??; + let env_loopback_direct = [ + ("ALL_PROXY", offline_proxy_url), + ("all_proxy", offline_proxy_url), + ("HTTP_PROXY", offline_proxy_url), + ("http_proxy", offline_proxy_url), + ("HTTPS_PROXY", offline_proxy_url), + ("https_proxy", offline_proxy_url), + ("NO_PROXY", "127.0.0.1,localhost"), + ("no_proxy", "127.0.0.1,localhost"), + ]; + let journal = Journal::open(&journal_dir.join("client-journal.sqlite")).await?; - let (mut child, client) = start_host_client( + let (mut child, client) = start_host_client_with_env( maka_executable, root_path, u64::from(config.host_settlement_timeout_ms), journal.clone(), + &env_loopback_direct, ) .await?; @@ -710,11 +1145,12 @@ async fn run_normal_product_client( configured?; cleanup?; - let (mut child, client) = start_host_client( + let (mut child, client) = start_host_client_with_env( maka_executable, root_path, u64::from(config.host_settlement_timeout_ms), journal, + &env_loopback_direct, ) .await?; @@ -902,13 +1338,14 @@ async fn run_normal_product_client( #[expect( clippy::too_many_arguments, - reason = "parity entry-point harness binds subject config, probe endpoint, isolated roots, artifacts, workspace, task input, step budget, and timeout explicitly" + reason = "parity entry-point harness binds subject config, probe origin, offline proxy, isolated roots, artifacts, workspace, task input, step budget, and timeout explicitly" )] async fn run_eval_subject_entry( subject_id: &str, subject_config: &JsonObject, subject_credentials: &[String], - probe_base_url: &str, + probe_origin: &str, + offline_proxy_url: &str, eval_roots_dir: &Path, eval_artifacts_dir: &Path, workspace_path: &Path, @@ -917,7 +1354,6 @@ async fn run_eval_subject_entry( timeout: Duration, ) -> Result<(PathBuf, String), String> { let mut cfg = subject_config.clone(); - cfg.insert("baseUrl".into(), Value::String(probe_base_url.to_owned())); cfg.insert( "runtimeHostsPath".into(), Value::String(eval_roots_dir.to_string_lossy().into_owned()), @@ -973,6 +1409,20 @@ async fn run_eval_subject_entry( let runner = Arc::new(LocalProcessRunner { work_dir: eval_artifacts_dir.to_path_buf(), timeout, + extra_env: vec![ + ( + "MAKA_EVAL_GATEWAY_TRANSPORT_ORIGIN".into(), + probe_origin.to_owned(), + ), + ("ALL_PROXY".into(), offline_proxy_url.to_owned()), + ("all_proxy".into(), offline_proxy_url.to_owned()), + ("HTTP_PROXY".into(), offline_proxy_url.to_owned()), + ("http_proxy".into(), offline_proxy_url.to_owned()), + ("HTTPS_PROXY".into(), offline_proxy_url.to_owned()), + ("https_proxy".into(), offline_proxy_url.to_owned()), + ("NO_PROXY".into(), "127.0.0.1,localhost".into()), + ("no_proxy".into(), "127.0.0.1,localhost".into()), + ], }); let context = SubjectExecutionContext { cwd: workspace_path.to_string_lossy().into_owned(), @@ -1027,12 +1477,11 @@ fn read_content_json(conn: &Connection, content_ref: &Value) -> Result Result { +) -> Result<(Connection, String, Value, Value, EphemeralNormalizer), String> { let db_path = root_path.join(ROOT_SQLITE_FILENAME); let conn = Connection::open_with_flags(&db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) .map_err(|e| format!("failed opening {}: {e}", db_path.display()))?; @@ -1069,6 +1518,30 @@ fn extract_committed_parity_snapshot( root_raw: root_path.to_string_lossy().into_owned(), root_canonical: canonical_root.to_string_lossy().into_owned(), }; + Ok((conn, run_id, run_spec, accepted_model, normalizer)) +} + +fn extract_committed_parity_snapshot( + facts_root_path: &Path, + facts_session_id: &str, + wire_root_path: &Path, + wire_session_id: &str, + workspace_path: &Path, + wire: &CapturedWireRequest, +) -> Result { + let (conn, run_id, run_spec, accepted_model, normalizer) = + build_ephemeral_normalizer_for_session(facts_root_path, facts_session_id, workspace_path)?; + let wire_normalizer = + if facts_root_path == wire_root_path && facts_session_id == wire_session_id { + None + } else { + let (_, _, _, _, wn) = build_ephemeral_normalizer_for_session( + wire_root_path, + wire_session_id, + workspace_path, + )?; + Some(wn) + }; let system_prompt_blob = read_content_json(&conn, &run_spec["systemPrompt"])?; let system_prompt_text = system_prompt_blob["text"] @@ -1195,7 +1668,8 @@ fn extract_committed_parity_snapshot( "bodyOverlay": accepted_model.get("body_overlay").cloned().unwrap_or(Value::Null), })); - let wire_normalized = normalizer.normalize_value(&wire.body); + let wire_norm = wire_normalizer.as_ref().unwrap_or(&normalizer); + let wire_normalized = wire_norm.normalize_value(&wire.body); let mut wire_rest = wire_normalized.as_object().cloned().unwrap_or_default(); let wire_model = wire_rest .remove("model") @@ -1254,6 +1728,7 @@ fn extract_committed_parity_snapshot( .unwrap_or(Value::Null), context_policy, wire_path: wire.path.clone(), + upstream_target_url: wire.upstream_target.as_deref().map(url_route_signature), wire_model, wire_messages, wire_tools, @@ -1595,6 +2070,11 @@ pub fn compare_parity_snapshots( Value::String(normal.wire_path.clone()), Value::String(eval.wire_path.clone()), ), + ( + "upstreamTargetUrl", + serde_json::to_value(&normal.upstream_target_url).unwrap(), + serde_json::to_value(&eval.upstream_target_url).unwrap(), + ), ( "wireModel", serde_json::to_value(&normal.wire_model).unwrap(), diff --git a/products/eval/src/report.rs b/products/eval/src/report.rs index 80c9d879cdd..0022c3d4f8b 100644 --- a/products/eval/src/report.rs +++ b/products/eval/src/report.rs @@ -498,20 +498,95 @@ pub fn evaluate_milestone_gate( )); continue; } - let exe_path = Path::new(&decoded.maka_executable); - if exe_path.is_file() - && let Ok(bytes) = std::fs::read(exe_path) - { - let actual_exe_sha = sha256_hex(&bytes); - if actual_exe_sha != report.maka_executable_sha256 { + let subject_cell_ids: Vec = expand_experiment(spec) + .into_iter() + .filter(|cell| cell.subject.id == subject.id) + .map(|cell| cell.id) + .collect(); + let settled_attempts: Vec<(&String, &V1CellAttempt)> = subject_cell_ids + .iter() + .filter_map(|cell_id| canonical_v1.get(cell_id).map(|att| (cell_id, att))) + .collect(); + if settled_attempts.is_empty() { + violations.push(format!( + "subject '{}' has no settled runtime execution evidence to confirm Maka binary identity", + subject.id + )); + continue; + } + let violations_before = violations.len(); + for (cell_id, attempt) in settled_attempts { + let runtime_state = + attempt.result.artifacts.iter().find(|art| { + art.get("kind").and_then(Value::as_str) == Some("maka-runtime-state") + }); + let Some(runtime_state) = runtime_state else { + violations.push(format!( + "subject '{}' cell '{}' is missing 'maka-runtime-state' runtime binary evidence", + subject.id, cell_id + )); + continue; + }; + let runtime_exe = runtime_state + .get("makaExecutable") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()); + let runtime_sha = runtime_state + .get("makaExecutableSha256") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()); + let Some(runtime_sha) = runtime_sha else { + violations.push(format!( + "subject '{}' cell '{}' is missing runtime makaExecutableSha256 evidence", + subject.id, cell_id + )); + continue; + }; + if runtime_exe != Some(decoded.maka_executable.as_str()) { + violations.push(format!( + "subject '{}' cell '{}' runtime makaExecutable ({:?}) does not match declared '{}'", + subject.id, cell_id, runtime_exe, decoded.maka_executable + )); + continue; + } + if runtime_sha != report.maka_executable_sha256 { violations.push(format!( - "subject '{}' Maka executable digest ({}) differs from parity-verified binary ({})", - subject.id, actual_exe_sha, report.maka_executable_sha256 + "subject '{}' cell '{}' runtime Maka binary digest ({}) differs from parity-verified binary ({})", + subject.id, cell_id, runtime_sha, report.maka_executable_sha256 )); continue; } } - parity_verified_subjects.push(subject.id.clone()); + let exe_path = Path::new(&decoded.maka_executable); + match std::fs::symlink_metadata(exe_path) { + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => match std::fs::read(exe_path) { + Ok(bytes) => { + let actual_exe_sha = sha256_hex(&bytes); + if actual_exe_sha != report.maka_executable_sha256 { + violations.push(format!( + "subject '{}' Maka executable digest ({}) differs from parity-verified binary ({})", + subject.id, actual_exe_sha, report.maka_executable_sha256 + )); + } + } + Err(err) => { + violations.push(format!( + "subject '{}' Maka executable '{}' could not be read to confirm binary identity: {err}", + subject.id, decoded.maka_executable + )); + } + }, + Err(err) => { + violations.push(format!( + "subject '{}' Maka executable '{}' metadata check failed: {err}", + subject.id, decoded.maka_executable + )); + } + } + if violations.len() == violations_before { + parity_verified_subjects.push(subject.id.clone()); + } } if milestone == MilestoneGate::M5ProductionDelivery { diff --git a/products/eval/src/subject/maka.rs b/products/eval/src/subject/maka.rs index 4d477e34b38..e0a00ba6321 100644 --- a/products/eval/src/subject/maka.rs +++ b/products/eval/src/subject/maka.rs @@ -41,9 +41,11 @@ use crate::subject::{SubjectAdapter, SubjectExecutionResult}; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; use std::path::Path; use std::process::Stdio; +use std::sync::{Arc, OnceLock}; use std::time::Instant; use tokio::io::AsyncWriteExt; use tokio::process::Command; @@ -283,6 +285,12 @@ pub enum HostedExecutionProjection { operation_id: Option, #[serde(rename = "invocationId", default)] invocation_id: Option, + #[serde( + rename = "makaExecutableSha256", + default, + skip_serializing_if = "Option::is_none" + )] + maka_executable_sha256: Option, #[serde( rename = "blockedInteraction", default, @@ -305,6 +313,12 @@ pub enum HostedExecutionProjection { operation_id: Option, #[serde(rename = "invocationId", default)] invocation_id: Option, + #[serde( + rename = "makaExecutableSha256", + default, + skip_serializing_if = "Option::is_none" + )] + maka_executable_sha256: Option, }, } @@ -663,6 +677,86 @@ pub fn url_route_signature(raw_url: &str) -> String { parsed.to_string() } +/// Resolves the actual outbound model route (`base_url`) using Maka's provider policy +/// against the declared connection endpoint before attaching local metering forwarding. +pub async fn resolve_product_route_base_url( + provider_name: &str, + connection_slug: &str, + declared_base_url: &str, + model_id: &str, + model_override: Option<&JsonObject>, + thinking_level: Option<&str>, +) -> Result { + static BUNDLED_CATALOG: OnceLock, String>> = + OnceLock::new(); + let snapshot = BUNDLED_CATALOG + .get_or_init(|| maka_providers::catalog::Snapshot::bundled().map(Arc::new)) + .as_ref() + .map_err(Clone::clone)?; + + let package = if provider_name == "chatgpt" { + maka_providers::codex::ID + } else { + maka_providers::api::ID + }; + let identity = maka_agent_facts::provider::Identity { + package_id: package.into(), + entry_id: package.into(), + scope: Default::default(), + name: provider_name.to_owned(), + }; + let definition = maka_providers::definition(&identity, snapshot.clone()) + .map_err(|e| format!("unavailable Maka provider '{provider_name}': {e}"))?; + let configuration = definition + .configure(serde_json::json!({ "baseUrl": declared_base_url })) + .map_err(|e| format!("invalid Maka provider configuration for '{provider_name}': {e}"))?; + + let overrides = model_override + .map(|ov| { + serde_json::from_value::(Value::Object( + ov.clone(), + )) + }) + .transpose() + .map_err(|e| format!("invalid Maka modelOverride for '{model_id}': {e}"))?; + let thinking = thinking_level + .map(|lvl| { + serde_json::from_value::(Value::String( + lvl.to_owned(), + )) + }) + .transpose() + .map_err(|e| format!("invalid Maka thinkingLevel '{thinking_level:?}': {e}"))?; + + let reference = overrides.as_ref().and_then(|v| v.catalog_model.as_ref()); + let public_model = snapshot + .model_base(provider_name, model_id, reference) + .map(Cow::into_owned); + let model_snapshot = Arc::new( + maka_providers::catalog::Snapshot::for_model(provider_name, model_id, public_model) + .map_err(|e| format!("invalid Maka catalog snapshot for '{model_id}': {e}"))?, + ); + let model_definition = maka_providers::definition(&identity, model_snapshot) + .map_err(|e| format!("unavailable Maka provider '{provider_name}': {e}"))?; + let resolved = model_definition + .prepare(maka_agent_ports::provider::Resolve { + connection: maka_agent_ports::provider::Connection { + id: connection_slug.to_owned(), + revision: 0, + configuration, + }, + model: maka_agent_facts::configuration::ModelInfo::new(model_id), + overrides, + thinking_level: thinking, + }) + .await + .map_err(|e| { + format!("failed resolving Maka model route for '{provider_name}/{model_id}': {e}") + })?; + + Ok(resolved.base_url) +} + #[derive(Debug, Default, Clone)] pub struct RustHostSubject; @@ -767,28 +861,54 @@ impl SubjectAdapter for RustHostSubject { if process.termination != SubjectProcessTermination::Exited { let projection = try_decode_matching_projection(&process.stdout, &execution_id); - let (usage, metering_state, cost_usd, subject_reported_cost, op_id, inv_id) = - match projection { - Some(HostedExecutionProjection::Settled { - usage, - metering_state, - cost_usd, - operation_id, - invocation_id, - .. - }) => ( - usage.clone(), - metering_state, - usage - .as_ref() - .filter(|_| metering_state == MeteringState::Complete) - .and_then(|u| estimate_deepseek_cost(u, &config.model)), - cost_usd, - operation_id, - invocation_id, - ), - _ => (None, MeteringState::Unknown, None, None, None, None), - }; + let ( + usage, + metering_state, + cost_usd, + subject_reported_cost, + op_id, + inv_id, + exe_sha, + ) = match projection { + Some(HostedExecutionProjection::Settled { + usage, + metering_state, + cost_usd, + operation_id, + invocation_id, + maka_executable_sha256, + .. + }) => ( + usage.clone(), + metering_state, + usage + .as_ref() + .filter(|_| metering_state == MeteringState::Complete) + .and_then(|u| estimate_deepseek_cost(u, &config.model)), + cost_usd, + operation_id, + invocation_id, + maka_executable_sha256, + ), + Some(HostedExecutionProjection::Indeterminate { + operation_id, + invocation_id, + maka_executable_sha256, + .. + }) => ( + None, + MeteringState::Unknown, + None, + None, + operation_id, + invocation_id, + maka_executable_sha256, + ), + None => (None, MeteringState::Unknown, None, None, None, None, None), + }; + let binary_identity = exe_sha + .as_deref() + .map(|sha| (config.maka_executable.as_str(), sha)); return SubjectExecutionResult { output: None, usage, @@ -799,6 +919,7 @@ impl SubjectAdapter for RustHostSubject { failure_reason: Some("Maka subject exceeded the framework timeout".into()), artifacts: maka_artifacts( Some(&execution_id), + binary_identity, Some(&process), subject_reported_cost, None, @@ -898,22 +1019,35 @@ fn project_into_execution_result( failure_reason, operation_id, invocation_id, + maka_executable_sha256, .. - } => SubjectExecutionResult { - output, - usage: None, - metering_state: MeteringState::Unknown, - cost_usd: None, - duration_ms, - outcome: ExecutionOutcome::Indeterminate, - failure_reason: Some(safe_maka_failure_reason( - &failure_reason, - "Maka execution did not settle", - )), - artifacts: maka_artifacts(Some(execution_id), process, None, None, None), - operation_id, - invocation_id, - }, + } => { + let binary_identity = maka_executable_sha256 + .as_deref() + .map(|sha| (config.maka_executable.as_str(), sha)); + SubjectExecutionResult { + output, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms, + outcome: ExecutionOutcome::Indeterminate, + failure_reason: Some(safe_maka_failure_reason( + &failure_reason, + "Maka execution did not settle", + )), + artifacts: maka_artifacts( + Some(execution_id), + binary_identity, + process, + None, + None, + None, + ), + operation_id, + invocation_id, + } + } HostedExecutionProjection::Settled { status, failure_reason, @@ -922,6 +1056,7 @@ fn project_into_execution_result( cost_usd: subject_cost, operation_id, invocation_id, + maka_executable_sha256, blocked_interaction, environment_limitation, .. @@ -940,8 +1075,12 @@ fn project_into_execution_result( .filter(|_| metering_state == MeteringState::Complete) .and_then(|u| estimate_deepseek_cost(u, &config.model)); + let binary_identity = maka_executable_sha256 + .as_deref() + .map(|sha| (config.maka_executable.as_str(), sha)); let artifacts = maka_artifacts( Some(execution_id), + binary_identity, process, subject_cost, blocked_interaction.as_deref(), @@ -1081,6 +1220,7 @@ pub fn safe_maka_failure_reason(value: &str, fallback: &str) -> String { pub fn maka_artifacts( execution_id: Option<&str>, + binary_identity: Option<(&str, &str)>, process: Option<&SubjectProcessOutput>, subject_reported_cost_usd: Option, blocked_interaction: Option<&UnattendedInteractionBlock>, @@ -1128,6 +1268,10 @@ pub fn maka_artifacts( if let Some(id) = execution_id { runtime_state.insert("executionId".into(), Value::String(id.into())); } + if let Some((exe, sha)) = binary_identity { + runtime_state.insert("makaExecutable".into(), Value::String(exe.into())); + runtime_state.insert("makaExecutableSha256".into(), Value::String(sha.into())); + } out.push(runtime_state); let mut stdout_art = Map::new(); @@ -1157,7 +1301,7 @@ fn subject_failure( cancelled: bool, process: Option<&SubjectProcessOutput>, ) -> SubjectExecutionResult { - let mut artifacts = maka_artifacts(None, process, None, None, None); + let mut artifacts = maka_artifacts(None, None, process, None, None, None); let mut failure_art = Map::new(); failure_art.insert("kind".into(), Value::String("subject-failure".into())); failure_art.insert("stage".into(), Value::String(stage.into())); diff --git a/products/eval/src/subject/maka/native.rs b/products/eval/src/subject/maka/native.rs index 0097a6bfae8..5a9d907dbfb 100644 --- a/products/eval/src/subject/maka/native.rs +++ b/products/eval/src/subject/maka/native.rs @@ -25,7 +25,9 @@ use super::{ EnvironmentLimitation, HostedExecutionProjection, MakaHostInitSettingsPayload, MakaPublicSessionSettings, UnattendedInteractionBlock, bootstrap_maka_host_root, + resolve_product_route_base_url, }; +use crate::contract::sha256_hex; use crate::measurement::{EvalMeteringGateway, GatewayConfig, MeteringState, derive_metering}; use crate::store::write_atomic_bytes; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; @@ -126,13 +128,37 @@ pub(crate) async fn start_host_client( settlement_timeout_ms: u64, journal: Journal, ) -> Result<(Child, Client)> { - let mut child = Command::new(maka_executable) - .args(["host", "serve", "--root"]) + start_host_client_with_env( + maka_executable, + root_path, + settlement_timeout_ms, + journal, + &[], + ) + .await +} + +pub(crate) async fn start_host_client_with_env( + maka_executable: &Path, + root_path: &Path, + settlement_timeout_ms: u64, + journal: Journal, + env_overrides: &[(&str, &str)], +) -> Result<(Child, Client)> { + let mut cmd = Command::new(maka_executable); + cmd.args(["host", "serve", "--root"]) .arg(root_path) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) - .kill_on_drop(true) + .kill_on_drop(true); + if !env_overrides.is_empty() { + cmd.env_remove("REQUEST_METHOD"); + } + for &(key, val) in env_overrides { + cmd.env(key, val); + } + let mut child = cmd .spawn() .map_err(|e| format!("native Host spawn failed: {e}"))?; let mut stdout = BufReader::new( @@ -365,6 +391,7 @@ async fn observe_pending_interaction( async fn execute( request: &Request, + maka_executable_sha256: Option<&str>, client: &Client, stop: CancellationToken, ) -> Result { @@ -532,6 +559,8 @@ async fn execute( "runId": run, "sessionId": session, "rootPath": request.root_path, + "makaExecutable": request.maka_executable, + "makaExecutableSha256": maka_executable_sha256, }); let bytes = serde_json::to_vec(&receipt).map_err(|e| e.to_string())?; tokio::fs::write(request.artifact_root.join("execution-receipt.json"), bytes) @@ -721,16 +750,37 @@ pub async fn run(payload: &str, stop: CancellationToken) -> Result Result Result Result result, @@ -827,6 +883,7 @@ pub async fn run(payload: &str, stop: CancellationToken) -> Result break, + accepted = offline_listener.accept() => { + let Ok((mut s, _)) = accepted else { break; }; + tokio::spawn(async move { + let mut b = [0u8; 2048]; + let _ = tokio::time::timeout( + std::time::Duration::from_millis(100), + s.read(&mut b), + ).await; + let _ = s.write_all( + b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ).await; + }); + } + } + } + }); + + // Local Anthropic SSE probe capturing exact request path and upstream target header. + let probe_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let probe_addr = probe_listener.local_addr().unwrap(); + let observed = Arc::new(tokio::sync::Mutex::new( + Vec::<(String, Option)>::new(), + )); + let observed_clone = observed.clone(); + let probe_shutdown = CancellationToken::new(); + let probe_shutdown_clone = probe_shutdown.clone(); + let probe_task = tokio::spawn(async move { + loop { + tokio::select! { + _ = probe_shutdown_clone.cancelled() => break, + accepted = probe_listener.accept() => { + let Ok((mut stream, _)) = accepted else { break; }; + let store = observed_clone.clone(); + tokio::spawn(async move { + let mut buf = Vec::new(); + let mut chunk = [0u8; 4096]; + let header_end = loop { + let Ok(n) = stream.read(&mut chunk).await else { return; }; + if n == 0 { return; } + buf.extend_from_slice(&chunk[..n]); + if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break pos + 4; + } + }; + let headers_str = String::from_utf8_lossy(&buf[..header_end]).into_owned(); + let mut lines = headers_str.lines(); + let req_line = lines.next().unwrap_or_default(); + let path = req_line.split_whitespace().nth(1).unwrap_or_default().to_owned(); + let mut target_hdr = None; + let mut content_len = 0usize; + for line in lines { + if let Some((k, v)) = line.split_once(':') { + if k.eq_ignore_ascii_case(maka_eval::measurement::UPSTREAM_TARGET_HEADER) { + target_hdr = Some(v.trim().to_owned()); + } else if k.eq_ignore_ascii_case("content-length") { + content_len = v.trim().parse().unwrap_or(0); + } + } + } + while buf.len() < header_end + content_len { + let Ok(n) = stream.read(&mut chunk).await else { break; }; + if n == 0 { break; } + buf.extend_from_slice(&chunk[..n]); + } + store.lock().await.push((path, target_hdr)); + let sse = concat!( + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_z\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"anthropic/claude-3.5-haiku\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":15,\"output_tokens\":0}}}\n\n", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"OK\"}}\n\n", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"output_tokens\":3}}\n\n", + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n" + ); + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + sse.len(), + sse + ); + let _ = stream.write_all(resp.as_bytes()).await; + }); + } + } + } + }); + + struct EnvLocalRunner(Vec<(String, String)>); + impl CommandRunner for EnvLocalRunner { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut cmd = tokio::process::Command::new(&request.command); + cmd.args(&request.args) + .env_remove("MAKA_EVAL_RESULT_TOKEN") + .env_remove("REQUEST_METHOD") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + for (k, v) in &self.0 { + cmd.env(k, v); + } + for (k, v) in &request.environment { + cmd.env(k, v); + } + for target in request.credential_environment.keys() { + cmd.env(target, "sk-zenmux-test"); + } + let out = cmd.output().await.map_err(|e| e.to_string())?; + Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: out.status.code().unwrap_or(1), + stdout: if request.capture_stdout { + String::from_utf8_lossy(&out.stdout).into_owned() + } else { + String::new() + }, + stderr: String::from_utf8_lossy(&out.stderr).into_owned(), + diagnostic: None, + }) + }) + } + } + + // 1. Live `RustHostSubject` execution with official Zenmux baseUrl + Claude model. + let spec_json = json!({ + "schemaVersion": "maka.eval.v2", + "id": "zenmux-route-exp", + "benchmark": { "id": "tb", "version": "1.0", "config": {} }, + "executor": { "kind": "local", "config": {} }, + "subjects": [ + { + "id": "maka-zenmux", + "kind": "maka", + "credentials": ["ZENMUX_API_KEY"], + "config": { + "makaExecutable": maka.to_string_lossy(), + "runtimeHostsPath": temp.path().join("hosts-eval").to_string_lossy(), + "hostSettlementTimeoutMs": 30000, + "provider": "zenmux", + "connectionSlug": "zenmux", + "baseUrl": "https://zenmux.ai/api/v1", + "model": "anthropic/claude-3.5-haiku" + } + } + ], + "tasks": [ + { "id": "t1", "input": "Reply with OK.", "config": {} } + ], + "repetitions": 1, + "budget": { "maxSteps": 5 }, + "verifier": {} + }); + let spec = parse_experiment_spec(&spec_json).unwrap(); + let cells = expand_experiment(&spec); + let artifact_root = temp.path().join("eval-artifacts"); + let mut metadata = Map::new(); + metadata.insert( + "artifactRoot".into(), + Value::String(artifact_root.to_string_lossy().into_owned()), + ); + let offline_url = format!("http://{offline_addr}"); + let exec_res = RustHostSubject::new() + .execute( + &cells[0], + SubjectExecutionContext { + cwd: workspace.to_string_lossy().into_owned(), + task_input: cells[0].task.input.clone(), + metadata, + cancel: CancellationToken::new(), + runner: Arc::new(EnvLocalRunner(vec![ + ( + "MAKA_EVAL_GATEWAY_TRANSPORT_ORIGIN".into(), + format!("http://{probe_addr}"), + ), + ("ALL_PROXY".into(), offline_url.clone()), + ("HTTP_PROXY".into(), offline_url.clone()), + ("HTTPS_PROXY".into(), offline_url.clone()), + ("NO_PROXY".into(), "127.0.0.1,localhost".into()), + ])), + }, + ) + .await; + + probe_shutdown.cancel(); + offline_shutdown.cancel(); + let _ = probe_task.await; + let _ = offline_task.await; + + assert_eq!( + exec_res.outcome, + ExecutionOutcome::Completed, + "zenmux claude Eval execution failed: {:?}", + exec_res.failure_reason + ); + let reqs = observed.lock().await.clone(); + assert_eq!(reqs.len(), 1); + assert_eq!(reqs[0].0, "/api/anthropic/v1/messages"); + assert_eq!( + reqs[0].1.as_deref(), + Some("https://zenmux.ai/api/anthropic/v1/messages") + ); + + // 2. End-to-end `verify_product_parity` on the exact reviewer scenario: + // `zenmux` + `anthropic/claude-3.5-haiku` + `https://zenmux.ai/api/v1`. + let parity_report = verify_product_parity(&ParityCheckInput { + subject_id: "maka-zenmux".into(), + subject_config: spec.subjects[0].config.clone(), + subject_credentials: spec.subjects[0].credentials.clone(), + workspace_source: Some(workspace), + task_input: "Reply with OK.".into(), + max_steps: 5, + }) + .await + .expect("zenmux claude parity verification must succeed"); + assert!( + parity_report.passed, + "zenmux claude parity failed: {:?}", + parity_report.mismatches + ); + assert!(parity_report.mismatches.is_empty()); +} + +#[cfg(unix)] +#[tokio::test] +async fn milestone_m3_cli_enforces_runtime_binary_identity_for_local_and_container_paths() { + let temp = TempDir::new().unwrap(); + let (maka, eval) = stage_real_maka_and_eval_binaries(temp.path()); + + let spec_json = json!({ + "schemaVersion": "maka.eval.v2", + "id": "m3-binary-identity-exp", + "benchmark": { "id": "tb", "version": "1.0", "config": {} }, + "executor": { "kind": "harbor", "config": {} }, + "subjects": [ + { + "id": "maka", + "kind": "maka", + "credentials": ["OPENAI_API_KEY"], + "config": { + "makaExecutable": maka.to_string_lossy(), + "runtimeHostsPath": temp.path().join("hosts").to_string_lossy(), + "hostSettlementTimeoutMs": 30000, + "provider": "openai-compatible", + "connectionSlug": "eval-gateway", + "baseUrl": "https://api.openai.com/v1", + "model": "gpt-4.1" + } + } + ], + "tasks": [ + { "id": "task-1", "input": "Reply with OK.", "config": {} } + ], + "repetitions": 1, + "budget": { "maxSteps": 5 }, + "verifier": {} + }); + + let parity_out = temp.path().join("parity.json"); + let orig_spec_path = temp.path().join("spec-orig.json"); + fs::write( + &orig_spec_path, + serde_json::to_vec_pretty(&spec_json).unwrap(), + ) + .unwrap(); + + let parity_cli = tokio::process::Command::new(&eval) + .args([ + "parity", + orig_spec_path.to_str().unwrap(), + "--out", + parity_out.to_str().unwrap(), + ]) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + parity_cli.status.success(), + "parity CLI failed: {}", + String::from_utf8_lossy(&parity_cli.stderr) + ); + let parity_reports: Vec = + serde_json::from_slice(&fs::read(&parity_out).unwrap()).unwrap(); + let verified_exe_sha = parity_reports[0].maka_executable_sha256.clone(); + + // Reviewer reproduction: mutate `makaExecutable` to a nonexistent path and run + // `report --milestone m3 --format v1` with no runtime execution evidence -> MUST fail! + let mut nonexistent_spec_json = spec_json.clone(); + nonexistent_spec_json["subjects"][0]["config"]["makaExecutable"] = + json!("/opt/isolated-container/maka"); + let nonexistent_spec_path = temp.path().join("spec-container.json"); + fs::write( + &nonexistent_spec_path, + serde_json::to_vec_pretty(&nonexistent_spec_json).unwrap(), + ) + .unwrap(); + + let empty_out_dir = temp.path().join("out-empty"); + let empty_m3 = tokio::process::Command::new(&eval) + .args([ + "report", + nonexistent_spec_path.to_str().unwrap(), + "--out", + empty_out_dir.to_str().unwrap(), + "--milestone", + "m3", + "--format", + "v1", + "--parity-report", + parity_out.to_str().unwrap(), + ]) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + !empty_m3.status.success(), + "report --milestone m3 --format v1 must fail when runtime binary identity evidence is absent" + ); + assert!( + String::from_utf8_lossy(&empty_m3.stderr).contains("no settled runtime execution evidence"), + "unexpected stderr: {}", + String::from_utf8_lossy(&empty_m3.stderr) + ); + + // Legitimate isolated container path: `/opt/isolated-container/maka` does not exist on the + // reporting host, but settled journal runtime evidence confirms both path and SHA-256 -> PASSES! + let container_spec = parse_experiment_spec(&nonexistent_spec_json).unwrap(); + let valid_container_out = temp.path().join("out-container-valid"); + let valid_dir = ExperimentDirectory::open(&valid_container_out, &container_spec).unwrap(); + valid_dir + .attempts + .append_v1(&V1CellAttempt { + cell_id: "task-1::1::maka".into(), + sequence: 1, + started_at: 10, + completed_at: 20, + result: EvalResult { + status: EvalResultStatus::Completed, + score: Some(1.0), + duration_ms: 10, + usage: None, + cost_usd: None, + failure_reason: None, + artifacts: vec![ + json!({ + "kind": "maka-runtime-state", + "makaExecutable": "/opt/isolated-container/maka", + "makaExecutableSha256": verified_exe_sha, + }) + .as_object() + .unwrap() + .clone(), + ], + }, + }) + .unwrap(); + + let valid_m3 = tokio::process::Command::new(&eval) + .args([ + "report", + nonexistent_spec_path.to_str().unwrap(), + "--out", + valid_container_out.to_str().unwrap(), + "--milestone", + "m3", + "--format", + "v1", + "--parity-report", + parity_out.to_str().unwrap(), + ]) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + valid_m3.status.success(), + "isolated container path with matching runtime binary digest must pass M3: {}", + String::from_utf8_lossy(&valid_m3.stderr) + ); + + // Tampering `spec.json`'s `makaExecutable` to a different nonexistent path after trials ran + // MUST fail M3 because runtime evidence recorded `/opt/isolated-container/maka`! + let mut tampered_path_spec = nonexistent_spec_json.clone(); + tampered_path_spec["subjects"][0]["config"]["makaExecutable"] = + json!("/opt/other-unverified/maka"); + let tampered_spec_path = temp.path().join("spec-tampered-path.json"); + fs::write( + &tampered_spec_path, + serde_json::to_vec_pretty(&tampered_path_spec).unwrap(), + ) + .unwrap(); + // Note: ExperimentDirectory verifies `experiment.json` matches `spec`, so evaluate against a + // store opened for `tampered_path_spec` carrying the runtime receipt from `/opt/isolated-container/maka` + // as well as one with a mismatched digest. + let mismatch_spec = parse_experiment_spec(&tampered_path_spec).unwrap(); + let mismatch_out = temp.path().join("out-container-mismatch"); + let mismatch_dir = ExperimentDirectory::open(&mismatch_out, &mismatch_spec).unwrap(); + mismatch_dir + .attempts + .append_v1(&V1CellAttempt { + cell_id: "task-1::1::maka".into(), + sequence: 1, + started_at: 10, + completed_at: 20, + result: EvalResult { + status: EvalResultStatus::Completed, + score: Some(1.0), + duration_ms: 10, + usage: None, + cost_usd: None, + failure_reason: None, + artifacts: vec![ + json!({ + "kind": "maka-runtime-state", + "makaExecutable": "/opt/other-unverified/maka", + "makaExecutableSha256": "0000000000000000000000000000000000000000000000000000000000000000", + }) + .as_object() + .unwrap() + .clone(), + ], + }, + }) + .unwrap(); + + let mismatch_m3 = tokio::process::Command::new(&eval) + .args([ + "report", + tampered_spec_path.to_str().unwrap(), + "--out", + mismatch_out.to_str().unwrap(), + "--milestone", + "m3", + "--format", + "v1", + "--parity-report", + parity_out.to_str().unwrap(), + ]) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + !mismatch_m3.status.success(), + "mismatched runtime binary digest must fail M3" + ); + assert!( + String::from_utf8_lossy(&mismatch_m3.stderr) + .contains("differs from parity-verified binary"), + "unexpected stderr: {}", + String::from_utf8_lossy(&mismatch_m3.stderr) + ); +} diff --git a/products/eval/tests/support/native_host.py b/products/eval/tests/support/native_host.py index 0207df01438..567a217fd42 100644 --- a/products/eval/tests/support/native_host.py +++ b/products/eval/tests/support/native_host.py @@ -132,10 +132,14 @@ def do_POST(self): assert len(Provider.captured_requests[0].get("tools", [])) > 0, ( f"default product tools must not be pruned: {list(Provider.captured_requests[0].keys())}" ) + expected_maka_sha = hashlib.sha256(Path(os.environ["MAKA_TEST_MAKA"]).read_bytes()).hexdigest() + assert data["makaExecutableSha256"] == expected_maka_sha, data receipt = json.loads((root / "artifacts" / "execution-receipt.json").read_text()) assert receipt["operationId"] == data["operationId"] assert receipt["runId"] == data["invocationId"] assert receipt["rootPath"] == str(root / "host") + assert receipt["makaExecutable"] == os.environ["MAKA_TEST_MAKA"] + assert receipt["makaExecutableSha256"] == expected_maka_sha assert receipt["sessionId"] transcript = json.loads((root / "artifacts" / "transcript.json").read_text()) assert any("Done." in chunk.get("text", "") for page in transcript for chunk in page["chunks"])