diff --git a/Cargo.lock b/Cargo.lock index 17a80d7335a..99611c1362a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6909,6 +6909,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "maka-eval" +version = "0.2.0" +dependencies = [ + "base64 0.23.1", + "futures-util", + "hex", + "http-body-util", + "hyper", + "hyper-util", + "libc", + "regex", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sha2 0.11.1", + "tempfile", + "thiserror 2.0.21", + "tokio", + "tokio-util", + "url", + "uuid", +] + [[package]] name = "maka-event-log" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 42d150ce5ab..8023006b8ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ members = [ "plugins/workspace/workspace-files/native", "plugins/workspace/support", "products/maka/acp", + "products/eval", "plugins/agent/engine", "plugins/agent/native", "plugins/workspace/workspace-files/patch", @@ -149,6 +150,7 @@ imagesize = { version = "=0.15.0", default-features = false, features = ["png", jsonschema = { version = "0.60.1", default-features = false } libc = "0.2.191" maka-acp = { path = "products/maka/acp" } +maka-eval = { path = "products/eval" } maka-agent-engine = { path = "plugins/agent/engine" } maka-agent-native = { path = "plugins/agent/native" } maka-assistant = { path = "plugins/agent/assistant" } diff --git a/products/eval/Cargo.toml b/products/eval/Cargo.toml new file mode 100644 index 00000000000..fa895b3e73d --- /dev/null +++ b/products/eval/Cargo.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "maka-eval" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +base64 = "0.23.1" +futures-util.workspace = true +hex.workspace = true +http-body-util = "0.1.5" +hyper = { version = "1.12.0", features = ["server", "http1"] } +hyper-util = { version = "0.1.21", features = ["tokio"] } +regex.workspace = true +reqwest.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tempfile.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "io-std", "net", "process"] } +tokio-util.workspace = true +url = "2.5.8" +uuid.workspace = true + +[target.'cfg(unix)'.dependencies] +libc.workspace = true diff --git a/products/eval/README.md b/products/eval/README.md new file mode 100644 index 00000000000..55b86525ead --- /dev/null +++ b/products/eval/README.md @@ -0,0 +1,32 @@ + + +# maka-eval + +[简体中文](README.zh-CN.md) + +Measurement and control plane foundation for multi-arm benchmark experiments (`maka.eval.v1` and `maka.eval.v2`). + +## Architecture + +- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions, condition-only `RunFingerprint`, and the semantic difference registry. +- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), non-mutating metering gateway, admission boundary cutoffs, HMAC-authenticated checkpoint recovery, receipt-vs-proxy reconciliation, and standardized `DEEPSEEK_V4_FLASH_COST` pricing. +- [`store`](src/store.rs): Single-writer append-only experiment journal (`LaunchIntent`, `ResourceTagged`, `Checkpoint`, `Settled`, `PostHocRecovery`), pre-dispatch intent persistence, atomic settlement commits, critical artifact verification, and deterministic state reconstruction. + +[Source](src/lib.rs) · [Cargo manifest](Cargo.toml) diff --git a/products/eval/README.zh-CN.md b/products/eval/README.zh-CN.md new file mode 100644 index 00000000000..bb380970e95 --- /dev/null +++ b/products/eval/README.zh-CN.md @@ -0,0 +1,32 @@ + + +# maka-eval + +[English](README.md) + +多臂基准评测实验的测量与控制平面底座（支持 `maka.eval.v1` 与 `maka.eval.v2`）。 + +## 模块结构 + +- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度、仅记录运行条件的 `RunFingerprint` 及语义差异注册表。 +- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）、无篡改计量网关、准入截止边界、HMAC 鉴权检查点恢复、回执与代理日志去重对账及标准 `DEEPSEEK_V4_FLASH_COST` 计价。 +- [`store`](src/store.rs)：单写者只追加实验日志（`LaunchIntent`、`ResourceTagged`、`Checkpoint`、`Settled`、`PostHocRecovery`）、派发前意图落盘、原子结算提交、关键产物校验与确定性状态重建。 + +[源码](src/lib.rs) · [Cargo 配置](Cargo.toml) diff --git a/products/eval/harbor/docker-compose-egress-proxy.yaml b/products/eval/harbor/docker-compose-egress-proxy.yaml new file mode 100644 index 00000000000..2c5f2efa051 --- /dev/null +++ b/products/eval/harbor/docker-compose-egress-proxy.yaml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + harbor-docker-egress-control-sidecar: + volumes: + - type: bind + source: ${MAKA_EVAL_NETWORK_POLICY_PATH} + target: /usr/local/bin/network-policy + read_only: true + - maka-eval-egress-ca:/opt/maka-egress:ro + + main: + depends_on: + maka-eval-mitmproxy: + condition: service_healthy + # An AF_PACKET socket writes below the IP output hooks the policy installs, + # so no nftables rule can constrain it. NET_RAW is what gates that socket. + # A task's own compose can add the capability back, which this cannot + # prevent — the relay refuses to start the subject when that happens. + cap_drop: + - NET_RAW + volumes: + - maka-eval-egress-ca:/opt/maka-egress:ro + + maka-eval-mitmproxy: + image: maka-eval-egress-proxy:12.2.3 + networks: + - default + volumes: + - maka-eval-egress-ca:/opt/maka-egress + healthcheck: + test: + - CMD + - python + - -c + - "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); assert pathlib.Path('/opt/maka-egress/proxy-ipv4').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()" + interval: 1s + timeout: 3s + retries: 30 + +volumes: + maka-eval-egress-ca: diff --git a/products/eval/harbor/egress-proxy/Dockerfile b/products/eval/harbor/egress-proxy/Dockerfile new file mode 100644 index 00000000000..20edbe3c5c8 --- /dev/null +++ b/products/eval/harbor/egress-proxy/Dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM python:3.12-slim + +RUN python -m pip install --no-cache-dir mitmproxy==12.2.3 + +COPY egress_filter.py /opt/maka-eval/egress_filter.py +COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh + +ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"] diff --git a/products/eval/harbor/egress-proxy/entrypoint.sh b/products/eval/harbor/egress-proxy/entrypoint.sh new file mode 100755 index 00000000000..53a8ee8acc6 --- /dev/null +++ b/products/eval/harbor/egress-proxy/entrypoint.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +STATE_DIR=/opt/maka-egress-state +CERT_DIR=/opt/maka-egress + +mkdir -p "$STATE_DIR" "$CERT_DIR" +AUDIT_LOG="$STATE_DIR/hits.jsonl" +# Proxy restarts share one append-only evidence file for the whole trial. +# Only create it when absent so an earlier process cannot lose recorded hits. +test -e "$AUDIT_LOG" || touch "$AUDIT_LOG" + +# confdir holds the CA private key and this cell's audit log, so it stays +# container-private and only the certificate reaches the volume the subject +# mounts. Creating the CA here instead of letting mitmdump create it on startup +# publishes the certificate before the proxy port opens, so the health gate +# cannot release `main` against a missing or left-over certificate. +python -c 'import sys +from mitmproxy.certs import CertStore +CertStore.from_store(sys.argv[1], "mitmproxy", 2048)' "$STATE_DIR" +# Publish by rename so a restart cannot expose a truncated certificate to a +# subject that is already running against the previous one. +cp "$STATE_DIR/mitmproxy-ca-cert.pem" "$CERT_DIR/.mitmproxy-ca-cert.pem" +mv "$CERT_DIR/.mitmproxy-ca-cert.pem" "$CERT_DIR/mitmproxy-ca-cert.pem" + +# The subject only needs this address so HTTPS_PROXY can keep using the +# service hostname after the namespace policy refuses Docker DNS. +ip="$(getent ahostsv4 "$(hostname)" | awk 'NR == 1 { print $1 }')" +case "$ip" in + "" | 127.*) + echo "could not publish Eval egress proxy IPv4" >&2 + exit 1 + ;; +esac +printf '%s\n' "$ip" > "$CERT_DIR/.proxy-ipv4" +mv "$CERT_DIR/.proxy-ipv4" "$CERT_DIR/proxy-ipv4" + +exec mitmdump \ + --quiet \ + --listen-host 0.0.0.0 \ + --listen-port 8080 \ + --set block_global=false \ + --set rawtcp=false \ + --set confdir="$STATE_DIR" \ + --scripts /opt/maka-eval/egress_filter.py \ + "$@" diff --git a/products/eval/harbor/egress-proxy/network-policy b/products/eval/harbor/egress-proxy/network-policy new file mode 100755 index 00000000000..bbab233909c --- /dev/null +++ b/products/eval/harbor/egress-proxy/network-policy @@ -0,0 +1,148 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +# gost's allowlist is never written here: the sidecar entrypoint empties it at +# start and this policy only ever forwards through the audited proxy, so gost +# forwards nothing on its own. That also removes the packet-mark exemption gost +# needs when it does forward — the sidecar shares the subject's network +# namespace, so a mark the sidecar can set is one the subject can set too. +GOST_PORT=12345 +NFTABLES_RULESET_NAME=gost_egress +PROXY_HOST=maka-eval-mitmproxy +PROXY_PORT=8080 + +remove_nftables() { + nft delete table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || true +} + +PROXY_IPV4_PATH=/opt/maka-egress/proxy-ipv4 + +resolve_proxy_ip() { + # The proxy publishes this file before it is healthy. Reading it does not + # depend on Docker DNS, so deny-all then allow still has an address. + if [ ! -s "$PROXY_IPV4_PATH" ]; then + return + fi + ip=$(tr -d ' \t\r\n' < "$PROXY_IPV4_PATH") + case "$ip" in + "" | *.*.*.*.* | *[!0-9.]*) + return + ;; + esac + # Same four-octet program as relay_agent.IPV4_OCTET_AWK. + if ! printf '%s\n' "$ip" | awk ' +BEGIN { FS = "." } +NF != 4 { exit 1 } +{ + for (i = 1; i <= 4; i++) { + if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1 + } +} +'; then + return + fi + printf '%s\n' "$ip" +} + +# The only difference between the two modes this policy installs is whether the +# audited proxy is exempt, so they are one ruleset with one optional pair of +# rules rather than two rulesets that have to be kept in step. +apply_ruleset() { + proxy_ip="${1:-}" + nft --file - <&2 + exit 1 + fi + apply_ruleset "$proxy_ip" + echo "ok: Eval proxy-only egress applied ($proxy_ip:$PROXY_PORT)" +} + +case "${1:-}" in + allow-all) + remove_nftables + echo "ok: allow all egress" + ;; + deny-all) + # A phase that asked for no network gets none. Answering it with the + # proxy-only ruleset would hand it a route this override invented, which + # Harbor's own deny-all does not grant either. + apply_ruleset + echo "ok: Eval no-network egress applied" + ;; + allow) + shift + if [ "$#" -ne 1 ] || [ "$1" != "$PROXY_HOST" ]; then + echo "Eval egress policy accepts only $PROXY_HOST" >&2 + exit 2 + fi + setup_proxy_only + ;; + show) + if ! rules="$(nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null)"; then + echo "mode: allow-all" + elif printf '%s' "$rules" | grep -q "dport $PROXY_PORT"; then + echo "mode: Eval proxy-only" + echo "proxy: $PROXY_HOST:$PROXY_PORT" + else + echo "mode: Eval no-network" + fi + ;; + rules) + nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || echo "mode: allow-all" + ;; + *) + echo "Usage: network-policy {show|allow-all|deny-all|allow|rules}" >&2 + exit 2 + ;; +esac diff --git a/products/eval/harbor/egress_filter.py b/products/eval/harbor/egress_filter.py new file mode 100644 index 00000000000..6b377ba5dd6 --- /dev/null +++ b/products/eval/harbor/egress_filter.py @@ -0,0 +1,501 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fail-closed URL contamination filter for Eval subject egress.""" + +from __future__ import annotations + +import json +import os +import re +import time +from contextlib import suppress +from ipaddress import IPv6Address +from pathlib import Path +from typing import NamedTuple +from urllib.parse import unquote, urlsplit + +PINNED_REVISION = "d49e28f1e4ddd13d289e85a5f312a66750951932" +MAX_DECODE_PASSES = 4 +MAX_AUDIT_BYTES = 1024 * 1024 +AUDIT_PATH = Path( + os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress-state/hits.jsonl") +) +PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})") +TERMINAL_BENCH = re.compile(r"terminal[-_.%/+\s]*bench", re.IGNORECASE) + + +class ConnectTarget(NamedTuple): + """Validated CONNECT authority adapted into the URL policy's input shape.""" + + host: str + port: int | None + url: str + + +def contamination_rule(raw_url: str) -> tuple[str, str, str] | None: + normalized = normalize_url(raw_url) + url = urlsplit(normalized) + host = (url.hostname or "").lower().rstrip(".") + path_query = f"{url.path}?{url.query}" if url.query else url.path + lowered = path_query.lower() + + # Search the host and the path separately. A benchmark name in the hostname + # is a contamination surface, and searching the two fields joined would let + # a rule match across their boundary. + def anywhere(needle: str) -> bool: + return needle in host or needle in lowered + + if host == "r.jina.ai": + inner = unquote(url.path.lstrip("/")) + if inner.startswith(("http://", "https://")): + nested = contamination_rule(inner) + if nested: + return (f"jina_recursive:{nested[0]}", host, path_query) + + if anywhere(PINNED_REVISION): + return ("pinned_revision", host, path_query) + if host == "tbench.ai" or host.endswith(".tbench.ai"): + return ("tbench_domain", host, path_query) + if host == "hub.harborframework.com" and "/tasks/terminal-bench" in lowered: + return ("harbor_task_registry", host, path_query) + if benchmark_repository(host, lowered): + return ("benchmark_repository", host, path_query) + if public_trajectory_repository(host, lowered): + return ("public_trajectory", host, path_query) + if anywhere("patches-terminalbench-"): + return ("known_patch_artifact", host, path_query) + if TERMINAL_BENCH.search(host) or TERMINAL_BENCH.search(lowered): + return ("terminal_bench_url", host, path_query) + return None + + +def normalize_url(raw_url: str) -> str: + value = raw_url.strip() + if not value: + raise ValueError("empty URL") + for _ in range(MAX_DECODE_PASSES): + if PERCENT_ESCAPE.search(value): + raise ValueError("malformed percent escape") + decoded = unquote(value) + if decoded == value: + break + value = decoded + else: + raise ValueError("URL exceeded decode limit") + parsed = urlsplit(value) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname: + raise ValueError("unsupported URL") + return value + + +def benchmark_repository(host: str, path_query: str) -> bool: + repositories = ( + "harbor-framework/terminal-bench", + "terminal-benchmarks/terminal-bench", + "tbench-ai/terminal-bench", + ) + return host in { + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + } and any(repository in path_query for repository in repositories) + + +def public_trajectory_repository(host: str, path_query: str) -> bool: + return ( + host in {"github.com", "api.github.com", "raw.githubusercontent.com", "huggingface.co"} + and "hqeric/maka-eval-trajectories" in path_query + ) + + +try: + from mitmproxy import http +except ImportError as missing_http_dependency: + http = None + +try: + from mitmproxy.proxy import commands as proxy_commands + from mitmproxy.proxy.layer import Layer + from mitmproxy.proxy.layers import ClientTLSLayer, ServerTLSLayer + from mitmproxy.proxy.layers.tcp import TCPLayer +except ImportError as missing_proxy_dependency: + proxy_commands = None + Layer = object + ClientTLSLayer = None + ServerTLSLayer = None + TCPLayer = None + +try: + from mitmproxy.net.tls import starts_like_tls_record +except ImportError: + def starts_like_tls_record(data: bytes) -> bool: + return len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03 + + +def configure(updated: object) -> None: + # HTTP 101 upgrades construct TCPLayer inside the HTTP layer without + # another next_layer hook. rawtcp=false makes that path CloseConnection + # itself; WebSocket upgrades stay on the websocket layer. + try: + from mitmproxy import ctx + except ImportError: + return + if getattr(ctx.options, "rawtcp", False): + ctx.options.rawtcp = False + + +def request(flow: object) -> None: + enforce_url_policy(flow, flow.request.pretty_url) + + +def response(flow: object) -> None: + reply = getattr(flow, "response", None) + if getattr(reply, "status_code", None) != 101: + return + # mitmproxy 12.2.3 sets flow.websocket before HttpResponseHook only when + # the 101 is a real WebSocket upgrade (Upgrade + version 13 + option on). + # A websocket Upgrade header alone still falls through to CloseConnection + # under rawtcp=false and must be audited. + if getattr(flow, "websocket", None) is not None: + return + reject_raw_transport(flow, close=False) + + +def http_connect(flow: object, /) -> None: + request = getattr(flow, "request", None) + target = _safe_connect_url( + getattr(request, "host", None), + getattr(request, "port", None), + ) + enforce_url_policy(flow, target) + + +def tcp_start(flow: object, /) -> None: + # This is a last-resort hook for a raw layer admitted by an override. + reject_raw_transport(flow) + + +def tcp_message(flow: object, /) -> None: + _clear_latest_tcp_message(getattr(flow, "messages", None)) + close_flow(flow) + + +def next_layer(nextlayer: object) -> None: + active_layer = getattr(nextlayer, "layer", None) + layer_context = getattr(nextlayer, "context", None) + if active_layer is None: + data_client = _next_layer_bytes(nextlayer, "data_client") + if _is_fragmented_tls_record_prefix(data_client): + if ClientTLSLayer is None or ServerTLSLayer is None: + return + server_tls = ServerTLSLayer(layer_context) + server_tls.child_layer = ClientTLSLayer(layer_context) + nextlayer.layer = server_tls + return + if not initial_stream_is_raw(nextlayer): + return + reject_raw_transport(layer_context, close=False) + nextlayer.layer = RejectRawTransport(layer_context, proxy_commands) + return + if TCPLayer is None or not isinstance(active_layer, TCPLayer): + return + reject_raw_transport(layer_context, close=False) + closer = RejectRawTransport(layer_context, proxy_commands) + replace_layer(layer_context, active_layer, closer) + nextlayer.layer = closer + + +def enforce_url_policy(flow: object, raw_url: str) -> None: + if http is None: + raise RuntimeError("mitmproxy is required to run the Eval egress filter") + try: + match = contamination_rule(raw_url) + if not match: + return + rule_id, host, normalized_path = match + audit_event(rule_id, host, normalized_path) + flow.response = blocked_response(rule_id) + except Exception as error: + flow.response = http.Response.make( + 503, + b"Eval egress policy could not classify this request.\n", + { + "Content-Type": "text/plain; charset=utf-8", + "X-Maka-Eval-Egress-Rule": "policy_error", + }, + ) + try: + audit_event("policy_error", "", type(error).__name__) + except Exception: + pass + + +def connect_url(host: object, port: object) -> str: + return parse_connect_target(host, port).url + + +def _safe_connect_url(host: object, port: object) -> str: + try: + return connect_url(host, port) + except (TypeError, ValueError): + return "" + + +def _clear_latest_tcp_message(messages: object) -> None: + if not isinstance(messages, list) or not messages: + return + latest = messages[-1] + if hasattr(latest, "content"): + latest.content = b"" + + +def parse_connect_target(host: object, port: object) -> ConnectTarget: + normalized_host = _normalize_connect_host(host) + normalized_port = _normalize_connect_port(port) + scheme = "http" if normalized_port == 80 else "https" + authority = normalized_host if normalized_port in (None, 443, 80) else f"{normalized_host}:{normalized_port}" + return ConnectTarget( + host=normalized_host, + port=normalized_port, + url=f"{scheme}://{authority}/", + ) + + +def _normalize_connect_host(host: object) -> str: + if not isinstance(host, str) or not host or host != host.strip(): + raise ValueError("empty CONNECT host") + if host.startswith("[") and host.endswith("]"): + IPv6Address(host[1:-1]) + return host + if ":" in host: + IPv6Address(host) + return f"[{host}]" + if re.search(r"[\s/@?#\\\[\]]", host): + raise ValueError("invalid CONNECT host") + return host + + +def _normalize_connect_port(port: object) -> int | None: + if port is None: + return None + if type(port) is not int or not 1 <= port <= 65535: + raise ValueError("invalid CONNECT port") + return port + + +def initial_stream_is_raw(nextlayer: object) -> bool: + """Close only when the bytes cannot still become HTTP or TLS. + + Script next_layer runs before mitmproxy 12.2.3's classifier. Copying its + `probably_no_http` test here would treat `GET` / `GET ` as raw and assign + CloseRawLayer while the request line is still arriving. With rawtcp=false + the built-in path would have kept waiting for HttpLayer. + """ + data_client = _next_layer_bytes(nextlayer, "data_client") + data_server = _next_layer_bytes(nextlayer, "data_server") + if _could_start_tls_record(data_client): + return False + if not data_client and not data_server: + return False + if data_server or data_client.startswith(b"SSH"): + return True + return not _still_could_be_http(data_client) + + +def _could_start_tls_record(data: bytes) -> bool: + """Keep a fragmented ClientHello undecided until its 3-byte prefix exists.""" + if starts_like_tls_record(data): + return True + return _is_fragmented_tls_record_prefix(data) + + +def _is_fragmented_tls_record_prefix(data: bytes) -> bool: + return 0 < len(data) < 3 and b"\x16\x03".startswith(data) + + +def _still_could_be_http(data: bytes) -> bool: + first_line, newline, _rest = data.partition(b"\n") + line = first_line.rstrip(b"\r") + method, space, _remainder = line.partition(b" ") + if not method.isascii() or not method.isalpha(): + return False + if newline and not space: + return False + return True + + +def _next_layer_bytes(nextlayer: object, name: str) -> bytes: + getter = getattr(nextlayer, name, None) + if not callable(getter): + return b"" + try: + data = getter() + except Exception: + return b"" + return bytes(data) if isinstance(data, (bytes, bytearray)) else b"" + + +def peer_label(owner: object) -> tuple[str, str]: + server = getattr(owner, "server_conn", None) or getattr(owner, "server", None) + peer_address = getattr(server, "address", None) + return _format_peer_address(peer_address) + + +def _format_peer_address(address: object) -> tuple[str, str]: + if not isinstance(address, (tuple, list)) or not address: + return "", "" + host = str(address[0])[:255] + port = address[1] if len(address) > 1 else "" + return host, f":{port}" if port != "" else "" + + +def reject_raw_transport(owner: object, *, close: bool = True) -> None: + host, port = peer_label(owner) + _audit_safely("raw_tunnel", host, port) + if close: + close_flow(owner) + + +def _audit_safely(rule_id: str, host: str, normalized_path: str) -> None: + with suppress(Exception): + audit_event(rule_id, host, normalized_path) + + +def close_flow(flow: object) -> None: + terminate = getattr(flow, "kill", None) + if callable(terminate) and getattr(flow, "killable", True): + try: + terminate() + except Exception as close_error: + del close_error + + +def replace_layer(context: object, current: object, closer: object) -> None: + layers = getattr(context, "layers", None) + if not isinstance(layers, list): + return + try: + index = layers.index(current) + except ValueError: + index = len(layers) + if closer in layers: + layers.remove(closer) + if current in layers: + layers.remove(current) + layers.insert(min(index, len(layers)), closer) + + +class RejectRawTransport(Layer): + def __init__(self, context: object, commands: object) -> None: + self._close_connection = getattr(commands, "CloseConnection", None) + if Layer is object: + self.context = context + return + if getattr(context, "layers", None) is None: + context.layers = [] + if getattr(context, "options", None) is None: + context.options = type("Options", (), {"proxy_debug": False})() + super().__init__(context) + + def handle_event(self, event: object): + if self._close_connection is None: + return + yield from ( + self._close_connection(connection) + for name in ("client", "server") + if (connection := getattr(self.context, name, None)) is not None + ) + + +def blocked_response(rule_id: str): + return http.Response.make( + 451, + b"Benchmark source or public solution access is blocked during evaluation.\n", + { + "Content-Type": "text/plain; charset=utf-8", + "X-Maka-Eval-Egress-Rule": rule_id, + }, + ) + + +class AuditJournal: + def __init__(self, path: Path, byte_limit: int) -> None: + self.path = path + self.byte_limit = byte_limit + + def record(self, rule_id: str, host: str, normalized_path: str) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + entry = self._encode(rule_id, host[:255], normalized_path[:4096]) + existing_bytes = self.path.stat().st_size if self.path.exists() else 0 + if existing_bytes + len(entry) <= self.byte_limit: + with self.path.open("ab") as stream: + stream.write(entry) + return + self.mark_full() + + def mark_full(self) -> None: + if self.has_full_marker(): + return + with self.path.open("ab") as stream: + stream.write(self._separator() + self._encode("audit_truncated", "", "")) + + def has_full_marker(self) -> bool: + last = self._last_record() + return last is not None and last.get("ruleId") == "audit_truncated" + + def _separator(self) -> bytes: + if not self.path.exists() or self.path.stat().st_size == 0: + return b"" + final_offset = self.path.stat().st_size - 1 + with self.path.open("rb") as stream: + stream.seek(final_offset) + return b"" if stream.read(1) == b"\n" else b"\n" + + def _last_record(self) -> dict[str, object] | None: + if not self.path.exists() or self.path.stat().st_size == 0: + return None + size = self.path.stat().st_size + with self.path.open("rb") as stream: + tail_offset = max(0, size - 4096) + stream.seek(tail_offset) + lines = stream.read().decode("utf-8", errors="ignore").splitlines() + for line in reversed(lines): + if not line.strip(): + continue + decoded = None + with suppress(json.JSONDecodeError): + decoded = json.loads(line) + return decoded if isinstance(decoded, dict) else None + return None + + @staticmethod + def _encode(rule_id: str, host: str, normalized_path: str) -> bytes: + record = { + "ts": int(time.time() * 1000), + "ruleId": rule_id, + "host": host, + "normalizedPath": normalized_path, + } + return (json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) + + +def audit_event(rule_id: str, host: str, normalized_path: str) -> None: + AuditJournal(AUDIT_PATH, MAX_AUDIT_BYTES).record(rule_id, host, normalized_path) diff --git a/products/eval/harbor/eval_framework.py b/products/eval/harbor/eval_framework.py new file mode 100644 index 00000000000..6c2bcb0b37b --- /dev/null +++ b/products/eval/harbor/eval_framework.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Process-local framework authority shared by the trial runner and relay.""" + +from __future__ import annotations # Keep framework metadata importable on Python 3.9. + +from typing import NamedTuple + + +class FrameworkSpec(NamedTuple): + name: str + distribution: str + agent_module: str + + +_FRAMEWORKS = { + "harbor": FrameworkSpec("harbor", "harbor", "harbor.agents.base"), + "pier": FrameworkSpec("pier", "datacurve-pier", "pier.agents.base"), +} +_active: FrameworkSpec | None = None + + +def framework_spec(name: str) -> FrameworkSpec: + try: + return _FRAMEWORKS[name] + except KeyError: + raise RuntimeError("framework must be harbor or pier") from None + + +def install(name: str) -> None: + """Select the framework before importing its process-wide relay module.""" + + global _active + _active = framework_spec(name) + + +def current_framework() -> str: + if _active is None: + raise RuntimeError("Eval framework selection is not installed") from None + return _active.name + + +def framework_distribution(name: str) -> str: + return framework_spec(name).distribution + + +def framework_agent_module(name: str) -> str: + return framework_spec(name).agent_module diff --git a/products/eval/harbor/relay_agent.py b/products/eval/harbor/relay_agent.py new file mode 100644 index 00000000000..a968bb5368e --- /dev/null +++ b/products/eval/harbor/relay_agent.py @@ -0,0 +1,723 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""One-shot Harbor/Pier Agent that delegates one subject execution to @maka/eval.""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import hashlib +import importlib +import json +import os +import re +import shlex +import tempfile +from pathlib import Path +from typing import Any + +from eval_framework import current_framework, framework_agent_module + +framework = current_framework() +BaseAgent = importlib.import_module(framework_agent_module(framework)).BaseAgent + + +class RelayTransportClosed(RuntimeError): + pass + + +RESULT_FRAME_PREFIX = "MAKA-EVAL-RESULT-V1" +SCOPE_ERROR_PREFIX = "MAKA-EVAL-SCOPE-ERROR-V1" +RESULT_PAYLOAD_LIMIT_BYTES = 2 * 1024 +RESULT_CARRIER_LIMIT_BYTES = 64 * 1024 +SUBJECT_STDOUT_PATH = "/logs/artifacts/maka-subject.stdout.txt" +SUBJECT_STDERR_PATH = "/logs/artifacts/maka-subject.stderr.txt" +CAPABILITY_PREFIX = "MAKA-EVAL-CAPABILITIES-V1 " +NAMESPACE_PREFIX = "MAKA-EVAL-POLICY-NAMESPACE-V1 " +# The egress policy only constrains what the IP output hooks can see. NET_RAW +# grants an AF_PACKET socket that writes beneath them, and NET_ADMIN grants the +# ability to delete the ruleset outright. The overlay drops NET_RAW, but a +# task's own compose can add either back, and a `cap_add` wins over an +# overlay's `cap_drop`. +BYPASS_CAPABILITIES = {"NET_RAW": 1 << 13, "NET_ADMIN": 1 << 12} +# Every set is read, not just the effective one: a non-root subject reports an +# empty effective set while a file-capability executable still reacquires +# anything the bounding set kept, and the permitted, inheritable and ambient +# sets each raise into the effective set without an exec at all. +CAPABILITY_FIELDS = ("CapEff", "CapPrm", "CapBnd") +# Harbor installs the policy by running `network-policy` inside this service, so +# its network namespace is what "the namespace the policy was applied to" means. +# Reading the identity from both sides answers that question directly, rather +# than inferring it from something only the shared namespace would expose. +POLICY_SERVICE = "harbor-docker-egress-control-sidecar" +NAMESPACE_PROBE = f"printf %s {shlex.quote(NAMESPACE_PREFIX)}; readlink /proc/self/ns/net" +PROXY_IPV4_PATH = "/opt/maka-egress/proxy-ipv4" +PROXY_HOSTS_PREFIX = "MAKA-EVAL-PROXY-HOST-V1 " +# The kernel's own form for a namespace link target. Matching it keeps an +# unexpected answer from being compared as if it were an identity. +NAMESPACE_IDENTITY = re.compile(r"^net:\[[0-9]+\]$") + +_host_teardown_requested = False + + +def request_host_teardown() -> None: + global _host_teardown_requested + _host_teardown_requested = True + + +class RelayAgent(BaseAgent): + def __init__( + self, + *args: Any, + relay_host: str, + relay_port: int, + relay_token: str, + teardown_timeout_ms: int, + **kwargs: Any, + ): + super().__init__(*args, **kwargs) + self._host = relay_host + self._port = relay_port + self._token = relay_token + if not isinstance(teardown_timeout_ms, int) or teardown_timeout_ms <= 0: + raise RuntimeError("Maka Eval teardown timeout is invalid") + self._teardown_timeout = teardown_timeout_ms / 1000 + self._subject_owner: str | None = None + + @staticmethod + def name() -> str: + return "maka-eval-relay" + + def version(self) -> str: + return "1" + + async def setup(self, environment: Any) -> None: + identity = await environment.exec('printf "%s:%s" "$(id -u)" "$(id -g)"') + owner = str(identity.stdout or "").strip() + if identity.return_code != 0 or re.fullmatch(r"[0-9]+:[0-9]+", owner) is None: + raise RuntimeError("Maka Eval could not resolve the task user") + prepared = await environment.exec( + "mkdir -p /logs/agent /logs/artifacts && " + f"chown {owner} /logs/agent /logs/artifacts && " + "chmod 700 /logs/agent /logs/artifacts", + user="root", + ) + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare task-owned artifact directories") + # Storage coordinates by the system account home, not the subject's + # temporary HOME. Provision an absent home without changing task identity + # or taking ownership of an existing directory. + uid = owner.split(":", 1)[0] + home_script = ( + f"set -eu; account=$(getent passwd {uid}); " + "task_home=$(printf '%s' \"$account\" | cut -d: -f6) && " + 'case "$task_home" in /*) ;; *) exit 1 ;; esac; ' + 'if [ ! -e "$task_home" ] && [ ! -L "$task_home" ]; then ' + f'mkdir -p -- "$task_home" && chown {owner} "$task_home" && ' + 'chmod 700 "$task_home" || exit 1; fi; ' + 'test -d "$task_home"' + ) + prepared_home = await environment.exec(home_script, user="root") + if prepared_home.return_code != 0: + raise RuntimeError("Maka Eval could not prepare the task account home") + self._subject_owner = owner + + async def run(self, instruction: str, environment: Any, context: Any) -> None: + reader, writer = await asyncio.open_connection(self._host, self._port) + execution: asyncio.Task[Any] | None = None + decision: asyncio.Task[dict[str, Any]] | None = None + request: dict[str, Any] | None = None + execution_reported = False + scope_path = f"/logs/agent/.maka-eval-{self._token}.pid" + environment_path = f"/tmp/maka-eval-{self._token}.env" + try: + cwd_prefix = f"MAKA-EVAL-CWD-V1 {self._token} " + working_directory = await environment.exec( + f"printf {shlex.quote(cwd_prefix)}; pwd 2>/dev/null" + ) + cwd_lines = [ + line[len(cwd_prefix) :] + for line in str(working_directory.stdout or "").splitlines() + if line.startswith(cwd_prefix) + ] + cwd = cwd_lines[0] if len(cwd_lines) == 1 else "" + if working_directory.return_code != 0 or not cwd.startswith("/") or "\x00" in cwd: + raise RuntimeError("Maka Eval could not resolve the task working directory") + await _require_constrained_subject(environment) + if not await _send( + writer, + { + "token": self._token, + "kind": "ready", + "instruction": instruction, + "cwd": cwd, + }, + ): + raise RelayTransportClosed("Maka Eval relay transport closed before ready") + request = await _receive(reader) + _require_message(request, self._token, "execute") + command = await _prepare_command( + environment, request, self._token, scope_path, self._subject_owner + ) + execution = asyncio.create_task(environment.exec(command, cwd=cwd)) + decision = asyncio.create_task(_receive(reader)) + done, _ = await asyncio.wait({execution, decision}, return_when=asyncio.FIRST_COMPLETED) + if decision in done and execution not in done: + decision.result() + raise RelayTransportClosed("Maka Eval relay received control before execution") + result = execution.result() + await _persist_subject_outputs(environment, result) + stdout, diagnostic = _project_result(result, request) + # A subject that exited on its own leaves the shared environment as + # it left it, and the verifier reads that environment. Nothing is + # waiting on those processes here — `environment.exec` has already + # returned — so tearing them down would not unblock anything; it + # would only edit the thing about to be measured, and edit it for + # some subjects and not others. Cancellation still quiesces, because + # there the subject has not stopped and the trial is being abandoned. + if not await _send( + writer, + { + "token": self._token, + "kind": "executed", + "termination": "exited", + "exitCode": result.return_code, + "stdout": stdout, + "diagnostic": diagnostic, + }, + ): + raise RelayTransportClosed("Maka Eval relay transport closed before result") + execution_reported = True + _require_message( + decision.result() if decision.done() else await decision, + self._token, + "verify", + ) + except asyncio.CancelledError: + if request is not None and execution is not None: + execution_terminal = execution.done() and not execution.cancelled() + terminal_projection = None + if execution_terminal: + terminal_result = execution.result() + terminal_projection = _project_result(terminal_result, request) + # A subject that already exited has nothing left to settle, and + # tearing its scope down here would remove what the verifier is + # about to score -- the same environment edit this relay stopped + # making on the ordinary path. Only a subject still running is + # brought to a stop. + if terminal_projection is not None: + result = terminal_result + else: + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + await _persist_subject_outputs(environment, result) + if ( + result is not None + and not execution_reported + and (execution_terminal or not _host_teardown_requested) + ): + stdout, diagnostic = terminal_projection or _project_result(result, request) + with contextlib.suppress(Exception): + await _send( + writer, + { + "token": self._token, + "kind": "executed", + "termination": "exited" if execution_terminal else "framework_timeout", + "exitCode": result.return_code if execution_terminal else 124, + "stdout": stdout, + "diagnostic": diagnostic, + }, + ) + elif not execution_reported and not _host_teardown_requested: + with contextlib.suppress(Exception): + await _send( + writer, + { + "token": self._token, + "kind": "executed", + "termination": "framework_timeout", + "exitCode": 124, + "stdout": "", + "diagnostic": ( + _carrier_diagnostic("result-frame-missing", b"") + if request.get("captureStdout", True) + else {"category": "none"} + ), + }, + ) + raise + except RelayTransportClosed: + if request is not None and execution is not None: + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + with contextlib.suppress(Exception): + await _persist_subject_outputs(environment, result) + except BaseException: + if request is not None and execution is not None: + result = await _settle_or_destroy( + environment, cwd, scope_path, execution, self._teardown_timeout + ) + if result is not None: + with contextlib.suppress(Exception): + await _persist_subject_outputs(environment, result) + raise + finally: + if decision is not None and not decision.done(): + decision.cancel() + with contextlib.suppress(BaseException): + await decision + if request is not None: + with contextlib.suppress(Exception): + await asyncio.wait_for( + environment.exec( + f"rm -f -- {shlex.quote(scope_path)} {shlex.quote(environment_path)}", + cwd=cwd, + timeout_sec=1, + ), + timeout=1, + ) + with contextlib.suppress(BrokenPipeError, ConnectionError, RuntimeError, TimeoutError): + writer.close() + await asyncio.wait_for(writer.wait_closed(), timeout=1) + + +async def _prepare_command( + environment: Any, + request: dict[str, Any], + token: str, + scope_path: str, + subject_owner: str | None = None, +) -> str: + credentials = request.get("credentials") + public_environment = request.get("environment", {}) + if not isinstance(credentials, dict) or not all( + isinstance(key, str) and isinstance(value, str) for key, value in credentials.items() + ): + raise RuntimeError("invalid Maka Eval credentials") + if not isinstance(public_environment, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in public_environment.items() + ): + raise RuntimeError("invalid Maka Eval environment") + if set(credentials) & set(public_environment): + raise RuntimeError("Maka Eval environment overlaps credentials") + capture_stdout = request.get("captureStdout", True) + if not isinstance(capture_stdout, bool): + raise RuntimeError("invalid Maka Eval stdout policy") + result_token = request.get("resultToken") + if not isinstance(result_token, str) or re.fullmatch(r"[0-9a-f]{32}", result_token) is None: + raise RuntimeError("invalid Maka Eval result token") + if "MAKA_EVAL_RESULT_TOKEN" in credentials or "MAKA_EVAL_RESULT_TOKEN" in public_environment: + raise RuntimeError("Maka Eval environment contains a reserved name") + for label, values in (("environment", public_environment), ("credential", credentials)): + if any(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None for key in values): + raise RuntimeError(f"invalid Maka Eval {label} name") + + container_path = f"/tmp/maka-eval-{token}.env" + secret_path = None + try: + with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as secret: + secret_path = Path(secret.name) + os.chmod(secret_path, 0o600) + for key, value in { + **public_environment, + **credentials, + "MAKA_EVAL_RESULT_TOKEN": result_token, + }.items(): + secret.write(f"export {key}={shlex.quote(value)}\n") + await environment.upload_file(secret_path, container_path) + if subject_owner is not None: + prepared = await environment.exec( + f"chown {subject_owner} {shlex.quote(container_path)} && " + f"chmod 600 {shlex.quote(container_path)}", + user="root", + ) + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare task-owned credentials") + finally: + if secret_path is not None: + secret_path.unlink(missing_ok=True) + subject = shlex.join([request["command"], *request["args"]]) + output_redirect = "" if capture_stdout else " >/dev/null" + scope_error = shlex.quote(f"{SCOPE_ERROR_PREFIX} {result_token}\\n") + inner = ( + f"( umask 077 && echo $$ > {shlex.quote(scope_path)} ) 2>/dev/null || " + f"{{ printf {scope_error}; exit 111; }}; " + f". {shlex.quote(container_path)}; command -p rm -f {shlex.quote(container_path)}; " + f"exec {subject}{output_redirect}" + ) + return f"setsid --wait sh -c {shlex.quote(inner)}" + + +async def _require_constrained_subject(environment: Any) -> None: + """Refuse to start the subject unless the egress policy actually governs it. + + Runs after the policy is applied and before the subject exists, so a task + that keeps a capability the policy cannot see, or that keeps the subject out + of the namespace the policy was applied to, fails the attempt instead of + producing a result the isolation contract never actually covered. + """ + if os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") != "1": + return + probe = await environment.exec( + f"printf %s {shlex.quote(CAPABILITY_PREFIX)}; " + r"sed -n 's/^\(Cap[A-Za-z]*\):[[:space:]]*/\1=/p' /proc/self/status | tr '\n' ' '; " + "echo; " + NAMESPACE_PROBE + ) + policy = await environment.service_exec(NAMESPACE_PROBE, service=POLICY_SERVICE) + if probe.return_code != 0 or policy.return_code != 0: + raise RuntimeError("Maka Eval could not read the subject isolation evidence") + capabilities = _sole_probe_line(probe, CAPABILITY_PREFIX) + namespace = _sole_probe_line(probe, NAMESPACE_PREFIX) + policy_namespace = _sole_probe_line(policy, NAMESPACE_PREFIX) + if not all(NAMESPACE_IDENTITY.match(value) for value in (namespace, policy_namespace)): + raise RuntimeError("Maka Eval could not read the subject isolation evidence") + reported: dict[str, int] = {} + for field in capabilities.split(): + name, _, value = field.partition("=") + try: + reported[name] = int(value, 16) + except ValueError: + raise RuntimeError("Maka Eval could not read the subject capability set") from None + if not all(field in reported for field in CAPABILITY_FIELDS): + raise RuntimeError("Maka Eval could not read the subject capability set") + granted = 0 + for value in reported.values(): + granted |= value + held = sorted(name for name, bit in BYPASS_CAPABILITIES.items() if granted & bit) + if held: + raise RuntimeError( + "the subject holds " + + ", ".join(held) + + ", which bypasses the Eval egress policy; remove it from the task" + ) + if namespace != policy_namespace: + raise RuntimeError( + "the subject does not share the network namespace the Eval egress policy " + "was applied to; remove the task's own networking on the subject service" + ) + await _pin_proxy_hostname(environment) + + +HOSTS_ALIAS_AWK = r""" +/^[[:space:]]*#/ { print; next } +{ + for (i = 2; i <= NF; i++) { + if ($i == host) next + } + print +} +""" + +# Four decimal octets 0-255, no empty fields and no leading zeros. The same +# program is embedded in egress-proxy/network-policy; the contract test +# requires the two copies to stay identical. +IPV4_OCTET_AWK = r""" +BEGIN { FS = "." } +NF != 4 { exit 1 } +{ + for (i = 1; i <= 4; i++) { + if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1 + } +} +""" + +_PUBLISHED_IPV4 = re.compile( + r"^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}" + r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$" +) + + +def _valid_egress_proxy_host(host: str) -> bool: + if not host or host.startswith(".") or host.endswith(".") or ".." in host: + return False + # Match the pin script's `*[!A-Za-z0-9.-]*` class. str.isalnum() also + # accepts Unicode letters, which the shell would later reject as a pin + # failure instead of an invalid host. + return all( + (character.isalnum() and character.isascii()) or character in ".-" + for character in host + ) + + +def _valid_published_ipv4(ip: str) -> bool: + return bool(_PUBLISHED_IPV4.fullmatch(ip)) + + +async def _pin_proxy_hostname(environment: Any) -> None: + """Point the proxy hostname at the published IPv4 so Docker DNS can be refused. + + The subject only needs that one name, because the HTTP proxy does remote + resolution itself. Pinning it in /etc/hosts lets the namespace policy reject + 127.0.0.11:53 without breaking HTTPS_PROXY. + """ + host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") or "maka-eval-mitmproxy" + if not _valid_egress_proxy_host(host): + raise RuntimeError("Eval egress proxy host is invalid") + script = f""" +set -eu +path={shlex.quote(PROXY_IPV4_PATH)} +host={shlex.quote(host)} +case "$host" in + ""|.*|*..*|*.) exit 1 ;; + *[!A-Za-z0-9.-]*) exit 1 ;; +esac +test -f "$path" +ip=$(tr -d ' \\t\\r\\n' < "$path") +case "$ip" in + ""|*.*.*.*.*|*[!0-9.]*) exit 1 ;; +esac +if ! printf '%s\\n' "$ip" | awk {shlex.quote(IPV4_OCTET_AWK)}; then + exit 1 +fi +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT +if [ -f /etc/hosts ]; then + awk -v host="$host" {shlex.quote(HOSTS_ALIAS_AWK)} /etc/hosts > "$tmp" +fi +printf '%s %s\\n' "$ip" "$host" >> "$tmp" +cat "$tmp" > /etc/hosts +printf %s {shlex.quote(PROXY_HOSTS_PREFIX)} +printf '%s %s\\n' "$ip" "$host" +""" + # Harbor 0.20.0 BaseEnvironment.exec and DockerEnvironment.exec take user=; + # DockerEnvironment._compose_exec forwards it as `docker compose exec -u`. + probe = await environment.exec(script, user="root") + if probe.return_code != 0: + raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname") + reported = _sole_probe_line(probe, PROXY_HOSTS_PREFIX) + ip, _, pinned = reported.partition(" ") + if pinned != host or not _valid_published_ipv4(ip): + raise RuntimeError("Maka Eval could not pin the Eval egress proxy hostname") + + +def _sole_probe_line(probe: Any, prefix: str) -> str: + reported = [ + line[len(prefix) :] + for line in str(probe.stdout or "").splitlines() + if line.startswith(prefix) + ] + if len(reported) != 1: + raise RuntimeError("Maka Eval could not read the subject isolation evidence") + return reported[0].strip() + + +async def _persist_subject_outputs(environment: Any, result: Any) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + stdout = root / "stdout" + stderr = root / "stderr" + stdout.write_text(str(getattr(result, "stdout", "") or ""), encoding="utf-8") + stderr.write_text(str(getattr(result, "stderr", "") or ""), encoding="utf-8") + prepared = await environment.exec("mkdir -p /logs/artifacts && chmod 700 /logs/artifacts") + if prepared.return_code != 0: + raise RuntimeError("Maka Eval could not prepare subject artifact output") + await environment.upload_file(stdout, SUBJECT_STDOUT_PATH) + await environment.upload_file(stderr, SUBJECT_STDERR_PATH) + + +def _decode_result_carrier(carrier: str, token: str) -> tuple[str, dict[str, Any]]: + raw = carrier.encode("utf-8", errors="replace") + if len(raw) > RESULT_CARRIER_LIMIT_BYTES: + return "", _carrier_diagnostic("result-frame-oversize", raw) + prefix = f"{RESULT_FRAME_PREFIX} {token} " + candidates = [line for line in carrier.splitlines(keepends=True) if line.startswith(prefix)] + if len(candidates) != 1: + category = "result-frame-missing" if not candidates else "result-frame-ambiguous" + return "", _carrier_diagnostic(category, raw) + frame = candidates[0] + fields = frame.rstrip("\r\n").split(" ", 4) + if len(fields) != 5: + return "", _carrier_diagnostic("result-frame-invalid", raw) + _, framed_token, length_text, digest, encoded = fields + try: + length = int(length_text) + padding = "=" * (-len(encoded) % 4) + payload = base64.b64decode(encoded + padding, altchars=b"-_", validate=True) + except (ValueError, base64.binascii.Error): + return "", _carrier_diagnostic("result-frame-invalid", raw) + if ( + framed_token != token + or length < 0 + or length > RESULT_PAYLOAD_LIMIT_BYTES + or len(payload) != length + or not re.fullmatch(r"[0-9a-f]{64}", digest) + or hashlib.sha256(payload).hexdigest() != digest + ): + return "", _carrier_diagnostic("result-frame-invalid", raw) + noise = carrier.replace(frame, "", 1).encode("utf-8", errors="replace") + diagnostic = ( + {"category": "none"} + if not noise + else _carrier_diagnostic("unstructured-output", noise) + ) + try: + decoded = payload.decode("utf-8", errors="strict") + except UnicodeDecodeError: + return "", _carrier_diagnostic("result-frame-invalid", raw) + return decoded, diagnostic + + +def _project_result(result: Any, request: dict[str, Any]) -> tuple[str, dict[str, Any]]: + carrier = str(getattr(result, "stdout", "") or "") + scope_error = f"{SCOPE_ERROR_PREFIX} {request['resultToken']}" + if carrier == f"{scope_error}\n": + return "", _carrier_diagnostic("execution-scope-unavailable", carrier.encode()) + if not request.get("captureStdout", True): + return "", {"category": "none"} + return _decode_result_carrier(carrier, request["resultToken"]) + + +def _carrier_diagnostic(category: str, value: bytes) -> dict[str, Any]: + return { + "category": category, + "bytes": len(value), + "sha256": hashlib.sha256(value).hexdigest(), + } + + +async def _settle(environment: Any, cwd: str, scope_path: str, execution: Any) -> Any: + if execution.cancelled(): + raise RuntimeError("Maka Eval subject execution was cancelled") + if execution.done(): + result = execution.result() + else: + result = None + for signal, timeout in (("TERM", 20), ("KILL", 10)): + await _signal(environment, cwd, scope_path, signal) + try: + result = await asyncio.wait_for(asyncio.shield(execution), timeout=timeout) + break + except asyncio.CancelledError: + if execution.cancelled(): + raise RuntimeError("Maka Eval subject execution was cancelled") from None + raise + except TimeoutError: + pass + if result is None: + raise RuntimeError("Maka Eval subject did not settle") + await _quiesce_scope(environment, cwd, scope_path) + return result + + +async def _settle_or_destroy( + environment: Any, + cwd: str, + scope_path: str, + execution: Any, + timeout: float, +) -> Any | None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + stop_reserve = min(20.0, timeout * 0.2) + try: + return await asyncio.wait_for( + _settle(environment, cwd, scope_path, execution), + timeout=max(0.001, deadline - loop.time() - stop_reserve), + ) + except Exception: + try: + remaining = max(0.001, deadline - loop.time()) + await asyncio.wait_for(environment.stop(delete=True), timeout=remaining) + except Exception: + pass + finally: + if not execution.done(): + execution.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + remaining = max(0.001, deadline - loop.time()) + await asyncio.wait_for(execution, timeout=remaining) + return None + + +async def _signal(environment: Any, cwd: str, scope_path: str, signal: str) -> None: + command = ( + f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 0; " + "case $pgid in ''|0|*[!0-9]*) exit 0;; esac; " + f"kill -{signal} -- \"-$pgid\"" + ) + with contextlib.suppress(Exception): + await environment.exec( + command, + cwd=cwd, + timeout_sec=5, + ) + + +async def _quiesce_scope(environment: Any, cwd: str, scope_path: str) -> None: + if not await _scope_active(environment, cwd, scope_path): + return + for signal, timeout in (("TERM", 10), ("KILL", 5)): + await _signal(environment, cwd, scope_path, signal) + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + if not await _scope_active(environment, cwd, scope_path): + return + await asyncio.sleep(0.1) + raise RuntimeError("Maka Eval execution scope did not quiesce") + + +async def _scope_active(environment: Any, cwd: str, scope_path: str) -> bool: + result = await environment.exec( + f"pgid=$(cat {shlex.quote(scope_path)} 2>/dev/null) || exit 4; " + "case $pgid in ''|0|*[!0-9]*) exit 4;; esac; " + "kill -0 -- \"-$pgid\" 2>/dev/null; status=$?; " + "if [ $status -eq 0 ]; then exit 0; fi; exit 3", + cwd=cwd, + timeout_sec=5, + ) + if result.return_code == 0: + return True + if result.return_code == 3: + return False + raise RuntimeError("Maka Eval execution scope evidence was unavailable") + + +def _require_message(value: dict[str, Any], token: str, kind: str) -> None: + if value.get("token") != token or value.get("kind") != kind: + raise RuntimeError("invalid Maka Eval relay message") + + +async def _receive(reader: asyncio.StreamReader) -> dict[str, Any]: + try: + raw = await reader.readline() + if not raw: + raise RelayTransportClosed("Maka Eval relay peer closed") + value = json.loads(raw) + except (ConnectionError, json.JSONDecodeError, ValueError) as error: + raise RelayTransportClosed("Maka Eval relay message was unavailable") from error + if not isinstance(value, dict): + raise RelayTransportClosed("Maka Eval relay message was invalid") + return value + + +async def _send(writer: asyncio.StreamWriter, value: object) -> bool: + if writer.is_closing(): + return False + try: + writer.write((json.dumps(value, separators=(",", ":")) + "\n").encode()) + await writer.drain() + return True + except (BrokenPipeError, ConnectionError, RuntimeError): + return False diff --git a/products/eval/harbor/run_trial.py b/products/eval/harbor/run_trial.py new file mode 100644 index 00000000000..4123e6c7980 --- /dev/null +++ b/products/eval/harbor/run_trial.py @@ -0,0 +1,230 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run exactly one pinned Harbor or Pier Trial.""" + +import asyncio +import contextlib +import fcntl +import hashlib +import importlib +import importlib.metadata +import json +import os +import signal +import sys +from pathlib import Path +from typing import AsyncIterator + +FRAMEWORK_VERSION_MISMATCH_EXIT_CODE = 78 + + +class FrameworkVersionMismatch(Exception): + pass + + +TASK_CACHE_PROTOCOL_VERSION = 1 + + +@contextlib.asynccontextmanager +async def task_cache_lock(cache_dir: Path, identity: str) -> AsyncIterator[None]: + lock_dir = cache_dir / ".maka-locks" + lock_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + lock_path = lock_dir / f"{hashlib.sha256(identity.encode()).hexdigest()}.lock" + flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(lock_path, flags, 0o600) + acquired = False + try: + os.fchmod(descriptor, 0o600) + while not acquired: + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + except BlockingIOError: + await asyncio.sleep(0.05) + yield + finally: + if acquired: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def ready_task(cache_dir: Path, identity: str) -> tuple[Path, Path | None]: + namespace = cache_dir / ".maka-tasks" / hashlib.sha256(identity.encode()).hexdigest() + marker = namespace / "ready.json" + try: + value = json.loads(marker.read_text()) + if set(value) != {"version", "target"}: + return namespace, None + if value["version"] != TASK_CACHE_PROTOCOL_VERSION or not isinstance( + value["target"], str + ): + return namespace, None + target = (namespace / value["target"]).resolve() + if ( + not target.is_relative_to(namespace.resolve()) + or not target.is_dir() + or not any(target.iterdir()) + ): + return namespace, None + return namespace, target + except (OSError, TypeError, ValueError): + return namespace, None + + +def publish_ready_task(namespace: Path, target: Path) -> None: + resolved_namespace = namespace.resolve() + resolved_target = target.resolve() + if not resolved_target.is_relative_to(resolved_namespace): + raise RuntimeError("Harbor task cache target escaped its Maka namespace") + relative_target = resolved_target.relative_to(resolved_namespace).as_posix() + marker = namespace / "ready.json" + temporary = namespace / ".ready.tmp" + descriptor = os.open( + temporary, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + try: + with os.fdopen(descriptor, "w") as stream: + descriptor = -1 + json.dump( + {"version": TASK_CACHE_PROTOCOL_VERSION, "target": relative_target}, + stream, + ) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, marker) + finally: + if descriptor >= 0: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + +async def run_trial(framework: str, expected_version: str, config_file: Path) -> None: + from eval_framework import install as select_framework + + select_framework(framework) + await _run_selected_trial(framework, expected_version, config_file) + + +async def _run_selected_trial(framework: str, expected_version: str, config_file: Path) -> None: + from eval_framework import framework_distribution + + distribution = framework_distribution(framework) + if importlib.metadata.version(distribution) != expected_version: + raise FrameworkVersionMismatch + try: + config_type = importlib.import_module(f"{framework}.models.trial.config").TrialConfig + trial_type = importlib.import_module(f"{framework}.trial.trial").Trial + config = config_type.model_validate_json(config_file.read_text()) + config_file.unlink() + if framework == "harbor": + from harbor.constants import TASK_CACHE_DIR + + task_identity = config.task.get_task_id().model_dump_json() + async with task_cache_lock(TASK_CACHE_DIR, task_identity): + namespace, ready = ready_task(TASK_CACHE_DIR, task_identity) + namespace.mkdir(parents=True, exist_ok=True, mode=0o700) + config.task.download_dir = namespace + config.task.overwrite = ready is None + trial = await create_harbor_trial(trial_type, config) + target = Path(trial.task.task_dir) + if ready is not None and target.resolve() != ready: + raise RuntimeError("Harbor task cache target changed for one identity") + if ready is None: + publish_ready_task(namespace, target) + else: + trial = await trial_type.create(config) + await trial.run() + finally: + config_file.unlink(missing_ok=True) + + +def apply_subject_egress_policy(task: object) -> None: + required = os.environ.get("MAKA_EVAL_EGRESS_REQUIRED") == "1" + allowed_host = os.environ.get("MAKA_EVAL_EGRESS_ALLOWED_HOST") + if not required: + return + if not allowed_host: + raise RuntimeError("required Eval egress proxy host is unavailable") + agent = task.config.agent + agent.network_mode = "allowlist" + agent.allowed_hosts = [allowed_host] + + +async def create_harbor_trial(trial_type: type, config: object) -> object: + trial_type._resolve_agent_skills(config) + task, task_download_result = await trial_type._load_task(config) + apply_subject_egress_policy(task) + if task.has_steps: + from harbor.trial.multi_step import MultiStepTrial + + return MultiStepTrial( + config, + _task=task, + _task_download_result=task_download_result, + ) + from harbor.trial.single_step import SingleStepTrial + + return SingleStepTrial( + config, + _task=task, + _task_download_result=task_download_result, + ) + + +async def main() -> None: + framework, expected_version, config_path = sys.argv[1:] + await _run_main(framework, expected_version, config_path) + + +async def _run_main(framework: str, expected_version: str, config_path: str) -> None: + task = asyncio.current_task() + assert task is not None + loop = asyncio.get_running_loop() + teardown_requested = False + + def request_teardown() -> None: + nonlocal teardown_requested + if teardown_requested: + return + teardown_requested = True + try: + from relay_agent import request_host_teardown + request_host_teardown() + finally: + task.cancel() + + for host_signal in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(host_signal, request_teardown) + try: + try: + await run_trial(framework, expected_version, Path(config_path)) + except FrameworkVersionMismatch: + raise SystemExit(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) from None + except asyncio.CancelledError: + if not teardown_requested: + raise + finally: + for host_signal in (signal.SIGINT, signal.SIGTERM): + loop.remove_signal_handler(host_signal) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/products/eval/src/contract.rs b/products/eval/src/contract.rs new file mode 100644 index 00000000000..4779d086c06 --- /dev/null +++ b/products/eval/src/contract.rs @@ -0,0 +1,1121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Experiment specifications, orthogonal `v2` attempt records, strict `v1` +//! projections, run fingerprinting, and the semantic difference registry. + +use crate::measurement::{MeteringState, NormalizedUsage}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use thiserror::Error; + +pub type JsonObject = Map; + +#[derive(Debug, Error, Clone, PartialEq, Eq)] +pub enum ContractError { + #[error("{0}")] + Invalid(String), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum SchemaVersion { + #[serde(rename = "maka.eval.v1")] + V1, + #[serde(rename = "maka.eval.v2")] + V2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SubjectKind { + Maka, + External, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BenchmarkSpec { + pub id: String, + pub version: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutorSpec { + pub kind: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutionSpec { + pub max_concurrent_task_groups: usize, +} + +impl Default for ExecutionSpec { + fn default() -> Self { + Self { + max_concurrent_task_groups: 1, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectSpec { + pub id: String, + pub kind: SubjectKind, + pub credentials: Vec, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TaskSpec { + pub id: String, + pub input: String, + pub config: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExperimentSpec { + pub schema_version: SchemaVersion, + pub id: String, + pub benchmark: BenchmarkSpec, + pub executor: ExecutorSpec, + #[serde(default)] + pub execution: ExecutionSpec, + pub subjects: Vec, + pub tasks: Vec, + pub repetitions: u32, + pub budget: JsonObject, + pub verifier: JsonObject, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExperimentCell { + pub id: String, + pub experiment_id: String, + pub benchmark: BenchmarkSpec, + pub executor: ExecutorSpec, + pub subject: SubjectSpec, + pub task: TaskSpec, + pub repetition: u32, + pub budget: JsonObject, + pub verifier: JsonObject, +} + +pub fn format_cell_id(task_id: &str, repetition: u32, subject_id: &str) -> String { + format!("{task_id}::{repetition}::{subject_id}") +} + +pub fn expand_experiment(spec: &ExperimentSpec) -> Vec { + let mut cells = + Vec::with_capacity(spec.tasks.len() * (spec.repetitions as usize) * spec.subjects.len()); + for task in &spec.tasks { + for repetition in 1..=spec.repetitions { + for subject in &spec.subjects { + cells.push(ExperimentCell { + id: format_cell_id(&task.id, repetition, &subject.id), + experiment_id: spec.id.clone(), + benchmark: spec.benchmark.clone(), + executor: spec.executor.clone(), + subject: subject.clone(), + task: task.clone(), + repetition, + budget: spec.budget.clone(), + verifier: spec.verifier.clone(), + }); + } + } + } + cells +} + +pub fn parse_experiment_spec(value: &Value) -> Result { + let root = exact_object( + value, + "experiment", + &[ + "schemaVersion", + "id", + "benchmark", + "executor", + "subjects", + "tasks", + "repetitions", + "budget", + "verifier", + ], + &["execution"], + )?; + + let schema_version = match root.get("schemaVersion").and_then(Value::as_str) { + Some("maka.eval.v1") => SchemaVersion::V1, + Some("maka.eval.v2") => SchemaVersion::V2, + _ => { + return Err(ContractError::Invalid( + "unsupported experiment schema".into(), + )); + } + }; + + let benchmark_obj = exact_object( + root.get("benchmark").unwrap(), + "benchmark", + &["id", "version", "config"], + &[], + )?; + let executor_obj = exact_object( + root.get("executor").unwrap(), + "executor", + &["kind", "config"], + &[], + )?; + + let max_concurrent_task_groups = match root.get("execution") { + None => 1, + Some(exec_val) => { + let exec_obj = exact_object(exec_val, "execution", &["maxConcurrentTaskGroups"], &[])?; + positive_usize( + exec_obj.get("maxConcurrentTaskGroups").unwrap(), + "execution.maxConcurrentTaskGroups", + )? + } + }; + + let subjects_arr = root + .get("subjects") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("subjects must be an array".into()))?; + let mut subjects = Vec::with_capacity(subjects_arr.len()); + for (idx, item) in subjects_arr.iter().enumerate() { + let where_label = format!("subjects[{idx}]"); + let sub_obj = exact_object( + item, + &where_label, + &["id", "kind", "credentials", "config"], + &[], + )?; + let kind = match sub_obj.get("kind").and_then(Value::as_str) { + Some("maka") => SubjectKind::Maka, + Some("external") => SubjectKind::External, + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.kind is invalid" + ))); + } + }; + subjects.push(SubjectSpec { + id: identifier(sub_obj.get("id").unwrap(), &format!("{where_label}.id"))?, + kind, + credentials: unique_strings( + sub_obj.get("credentials").unwrap(), + &format!("{where_label}.credentials"), + )?, + config: decode_json_object( + sub_obj.get("config").unwrap(), + &format!("{where_label}.config"), + )?, + }); + } + + let tasks_arr = root + .get("tasks") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("tasks must be an array".into()))?; + let mut tasks = Vec::with_capacity(tasks_arr.len()); + for (idx, item) in tasks_arr.iter().enumerate() { + let where_label = format!("tasks[{idx}]"); + let task_obj = exact_object(item, &where_label, &["id", "input", "config"], &[])?; + tasks.push(TaskSpec { + id: identifier(task_obj.get("id").unwrap(), &format!("{where_label}.id"))?, + input: nonempty_string( + task_obj.get("input").unwrap(), + &format!("{where_label}.input"), + )?, + config: decode_json_object( + task_obj.get("config").unwrap(), + &format!("{where_label}.config"), + )?, + }); + } + + if subjects.is_empty() || tasks.is_empty() { + return Err(ContractError::Invalid("experiment arms are empty".into())); + } + ensure_unique_ids(subjects.iter().map(|s| s.id.as_str()), "subject")?; + ensure_unique_ids(tasks.iter().map(|t| t.id.as_str()), "task")?; + + let repetitions = positive_u32(root.get("repetitions").unwrap(), "repetitions")?; + + Ok(ExperimentSpec { + schema_version, + id: identifier(root.get("id").unwrap(), "experiment.id")?, + benchmark: BenchmarkSpec { + id: identifier(benchmark_obj.get("id").unwrap(), "benchmark.id")?, + version: nonempty_string(benchmark_obj.get("version").unwrap(), "benchmark.version")?, + config: decode_json_object(benchmark_obj.get("config").unwrap(), "benchmark.config")?, + }, + executor: ExecutorSpec { + kind: identifier(executor_obj.get("kind").unwrap(), "executor.kind")?, + config: decode_json_object(executor_obj.get("config").unwrap(), "executor.config")?, + }, + execution: ExecutionSpec { + max_concurrent_task_groups, + }, + subjects, + tasks, + repetitions, + budget: decode_json_object(root.get("budget").unwrap(), "budget")?, + verifier: decode_json_object(root.get("verifier").unwrap(), "verifier")?, + }) +} + +/// Canonical JSON serialization with lexicographically sorted object keys. +pub fn canonical_json_string(value: &Value) -> String { + let normalized = canonicalize_value(value); + serde_json::to_string(&normalized).expect("canonical Value serialization cannot fail") +} + +fn canonicalize_value(value: &Value) -> Value { + match value { + Value::Object(map) => { + let mut sorted: Vec<(&String, &Value)> = map.iter().collect(); + sorted.sort_by_key(|(a, _)| *a); + let mut out = Map::with_capacity(sorted.len()); + for (k, v) in sorted { + out.insert(k.clone(), canonicalize_value(v)); + } + Value::Object(out) + } + Value::Array(items) => Value::Array(items.iter().map(canonicalize_value).collect()), + other => other.clone(), + } +} + +/// Documents experimental and execution conditions for comparability checks. +/// +/// Fingerprints record the declared configuration and parameters of a run; freezing an +/// eval-only fingerprint does not establish product parity. M3 acceptance and M5 production +/// delivery require a verified `ParityVerificationReport` alongside the run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RunFingerprint { + /// Digest over experiment identity, benchmark, tasks, repetitions, budget, verifier, + /// and subject semantic configs. + pub semantic_sha256: String, + /// Digest over execution-only parameters (e.g. `maxConcurrentTaskGroups`). + pub execution_sha256: String, +} + +impl RunFingerprint { + pub fn compute(spec: &ExperimentSpec) -> Self { + let semantic_payload = serde_json::json!({ + "id": spec.id, + "benchmark": spec.benchmark, + "executorKind": spec.executor.kind, + "subjects": spec.subjects, + "tasks": spec.tasks, + "repetitions": spec.repetitions, + "budget": spec.budget, + "verifier": spec.verifier, + }); + let execution_payload = serde_json::json!({ + "maxConcurrentTaskGroups": spec.execution.max_concurrent_task_groups, + "executorConfig": spec.executor.config, + }); + Self { + semantic_sha256: sha256_hex(canonical_json_string(&semantic_payload).as_bytes()), + execution_sha256: sha256_hex(canonical_json_string(&execution_payload).as_bytes()), + } + } + + pub fn is_semantically_comparable(&self, other: &Self) -> bool { + self.semantic_sha256 == other.semantic_sha256 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvalResultStatus { + Completed, + SubjectFailed, + InfraFailed, + Indeterminate, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EvalResult { + pub score: Option, + pub usage: Option, + pub cost_usd: Option, + pub duration_ms: u64, + pub status: EvalResultStatus, + pub failure_reason: Option, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct V1CellAttempt { + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub completed_at: u64, + pub result: EvalResult, +} + +pub fn is_replaceable_attempt(attempt: &V1CellAttempt) -> bool { + matches!( + attempt.result.status, + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate + ) +} + +/// Canonical cell result selector: selects the earliest valid attempt by sequence (`sequence >= 1`) +/// whose status is neither `infra_failed` nor `indeterminate`. A `completed` attempt with score `0.0` +/// or a `subject_failed` attempt is terminal and never replaced. +pub fn select_cell_result(attempts: &[V1CellAttempt]) -> Option<&V1CellAttempt> { + let mut sorted: Vec<&V1CellAttempt> = attempts.iter().filter(|a| a.sequence >= 1).collect(); + sorted.sort_by_key(|a| a.sequence); + sorted + .into_iter() + .find(|attempt| !is_replaceable_attempt(attempt)) +} + +pub fn decode_eval_result(value: &Value, where_label: &str) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "score", + "usage", + "costUsd", + "durationMs", + "status", + "failureReason", + "artifacts", + ], + &[], + )?; + let status = match obj.get("status").and_then(Value::as_str) { + Some("completed") => EvalResultStatus::Completed, + Some("subject_failed") => EvalResultStatus::SubjectFailed, + Some("infra_failed") => EvalResultStatus::InfraFailed, + Some("indeterminate") => EvalResultStatus::Indeterminate, + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.status is invalid" + ))); + } + }; + let failure_reason = match obj.get("failureReason").unwrap() { + Value::Null => None, + Value::String(s) => Some(s.clone()), + _ => { + return Err(ContractError::Invalid(format!( + "{where_label}.failureReason is invalid" + ))); + } + }; + let score = nullable_f64(obj.get("score").unwrap(), &format!("{where_label}.score"))?; + let usage = match obj.get("usage").unwrap() { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, &format!("{where_label}.usage"))?), + }; + let cost_usd = nullable_nonnegative_f64( + obj.get("costUsd").unwrap(), + &format!("{where_label}.costUsd"), + )?; + let duration_ms = nonnegative_u64( + obj.get("durationMs").unwrap(), + &format!("{where_label}.durationMs"), + )?; + let artifacts_arr = obj + .get("artifacts") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid(format!("{where_label}.artifacts is invalid")))?; + let mut artifacts = Vec::with_capacity(artifacts_arr.len()); + for (idx, item) in artifacts_arr.iter().enumerate() { + artifacts.push(decode_json_object( + item, + &format!("{where_label}.artifacts[{idx}]"), + )?); + } + Ok(EvalResult { + score, + usage, + cost_usd, + duration_ms, + status, + failure_reason, + artifacts, + }) +} + +pub fn decode_v1_cell_attempt(value: &Value) -> Result { + let obj = exact_object( + value, + "attempt", + &["cellId", "sequence", "startedAt", "completedAt", "result"], + &[], + )?; + Ok(V1CellAttempt { + cell_id: nonempty_string(obj.get("cellId").unwrap(), "attempt.cellId")?, + sequence: positive_u32(obj.get("sequence").unwrap(), "attempt.sequence")?, + started_at: nonnegative_u64(obj.get("startedAt").unwrap(), "attempt.startedAt")?, + completed_at: nonnegative_u64(obj.get("completedAt").unwrap(), "attempt.completedAt")?, + result: decode_eval_result(obj.get("result").unwrap(), "attempt.result")?, + }) +} + +/// Orthogonal dimensions for native `maka.eval.v2` attempt records. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AttemptPhase { + Prepared, + Dispatched, + AgentFinished, + Verified, + Settled, + Recovered, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionOutcome { + Completed, + SubjectFailed, + Timeout, + InfraFailed, + Cancelled, + Indeterminate, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum VerifierOutcome { + Verified { + score: f64, + }, + SubjectFailed { + score: Option, + reason: String, + }, + /// Missing reward file or empty verifier payload: never defaults to `0.0`. + MissingReward { + reason: String, + }, + VerifierInfraFailed { + reason: String, + }, + Skipped, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CleanupState { + Confirmed, + Unconfirmed { reason: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum EgressAuditState { + NotRequired, + Clean, + Truncated, + Missing, + Unreadable { reason: String }, + PolicyError { count: u64 }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CriticalArtifactsState { + Present, + Missing { paths: Vec }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "snake_case", + rename_all_fields = "camelCase" +)] +pub enum PricingEvidence { + Model { + model: String, + cost_usd: Option, + }, + SubjectReported { + cost_usd: f64, + }, +} + +impl PricingEvidence { + pub fn cost_usd(&self) -> Option { + match self { + Self::Model { cost_usd, .. } => *cost_usd, + Self::SubjectReported { cost_usd } => Some(*cost_usd), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct V2AttemptRecord { + pub schema_version: SchemaVersion, + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub completed_at: u64, + pub duration_ms: u64, + pub phase: AttemptPhase, + pub execution_outcome: ExecutionOutcome, + pub verifier_outcome: VerifierOutcome, + pub usage: Option, + pub metering_state: MeteringState, + #[serde(default)] + pub pricing: Option, + pub cleanup_state: CleanupState, + pub audit_state: EgressAuditState, + pub critical_artifacts_state: CriticalArtifactsState, + pub cancelled_after_verification: bool, + pub recovered_post_hoc: bool, + pub failure_reason: Option, + pub artifacts: Vec, +} + +impl V2AttemptRecord { + /// Projects a native `maka.eval.v2` record into a strict `maka.eval.v1` `V1CellAttempt` + /// while enforcing all 5 Semantic Difference Registry adjudications. + pub fn project_v1(&self) -> V1CellAttempt { + let mut artifacts = self.artifacts.clone(); + + let cost_usd = if self.usage.is_some() && self.metering_state == MeteringState::Complete { + self.pricing.as_ref().and_then(PricingEvidence::cost_usd) + } else { + None + }; + + if self.execution_outcome == ExecutionOutcome::Cancelled + && self.cancelled_after_verification + { + artifacts.push( + serde_json::json!({"kind": "cancelled-after-verification"}) + .as_object() + .unwrap() + .clone(), + ); + } + + if self.recovered_post_hoc { + let mut marker = Map::new(); + marker.insert( + "kind".into(), + Value::String("post-hoc-evidence-recovery".into()), + ); + artifacts.push(marker); + } + + // Unconfirmed cleanup always resolves to `indeterminate` with `score: null`. + if let CleanupState::Unconfirmed { reason } = &self.cleanup_state { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::Indeterminate, + failure_reason: Some(reason.clone()), + artifacts, + }, + }; + } + + // Adjudication #4: Late cancellation BEFORE verification completes resolves to + // `indeterminate`, whereas late cancellation AFTER verification and settlement + // preserves the verified result and records the late cancellation signal. + if self.execution_outcome == ExecutionOutcome::Cancelled + && !self.cancelled_after_verification + { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::Indeterminate, + failure_reason: Some(self.failure_reason.clone().unwrap_or_else(|| { + "executor cancelled before verification completed".into() + })), + artifacts, + }, + }; + } + + // Adjudication #3: Missing or unreadable critical artifacts downgrade to `infra_failed`. + if let CriticalArtifactsState::Missing { paths } = &self.critical_artifacts_state { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some(format!( + "critical artifacts missing: {}", + paths.join(", ") + )), + artifacts, + }, + }; + } + + // Adjudication #2: Egress audit missing or unreadable fails closed (`infra_failed`), + // whereas `Truncated` (`ruleId == "audit_truncated"`) is preserved as a structured + // artifact flag without discarding a settled verification result. + match &self.audit_state { + EgressAuditState::Missing => { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some("egress audit log missing".into()), + artifacts, + }, + }; + } + EgressAuditState::Unreadable { reason } => { + return V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score: None, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status: EvalResultStatus::InfraFailed, + failure_reason: Some(reason.clone()), + artifacts, + }, + }; + } + EgressAuditState::Truncated => { + if !artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some("egress-audit")) + { + let mut audit_obj = Map::new(); + audit_obj.insert("kind".into(), Value::String("egress-audit".into())); + audit_obj.insert("truncated".into(), Value::Bool(true)); + artifacts.push(audit_obj); + } + } + EgressAuditState::NotRequired + | EgressAuditState::Clean + | EgressAuditState::PolicyError { .. } => {} + } + + let (status, score, failure_reason) = + match (&self.execution_outcome, &self.verifier_outcome) { + (ExecutionOutcome::InfraFailed, _) => ( + EvalResultStatus::InfraFailed, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject infrastructure failed".into()), + ), + ), + (ExecutionOutcome::Indeterminate, _) => ( + EvalResultStatus::Indeterminate, + None, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject execution indeterminate".into()), + ), + ), + (_, VerifierOutcome::VerifierInfraFailed { reason }) + | (_, VerifierOutcome::MissingReward { reason }) => { + // Missing verifier reward never defaults to 0.0! + (EvalResultStatus::InfraFailed, None, Some(reason.clone())) + } + (ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }) => ( + // Adjudication #1: Agent timed out, verifier still ran and produced a valid + // reward; v1 status is `subject_failed` with the verified `score` preserved. + EvalResultStatus::SubjectFailed, + Some(*score), + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| "subject exceeded the framework timeout".into()), + ), + ), + (ExecutionOutcome::SubjectFailed, VerifierOutcome::Verified { score }) => ( + EvalResultStatus::SubjectFailed, + Some(*score), + self.failure_reason.clone(), + ), + ( + ExecutionOutcome::Completed + | ExecutionOutcome::SubjectFailed + | ExecutionOutcome::Timeout + | ExecutionOutcome::Cancelled, + VerifierOutcome::SubjectFailed { score, reason }, + ) => ( + EvalResultStatus::SubjectFailed, + *score, + Some( + self.failure_reason + .clone() + .unwrap_or_else(|| reason.clone()), + ), + ), + ( + ExecutionOutcome::Completed | ExecutionOutcome::Cancelled, + VerifierOutcome::Verified { score }, + ) => (EvalResultStatus::Completed, Some(*score), None), + ( + ExecutionOutcome::Timeout | ExecutionOutcome::SubjectFailed, + VerifierOutcome::Skipped, + ) => ( + EvalResultStatus::SubjectFailed, + None, + self.failure_reason.clone(), + ), + ( + ExecutionOutcome::Completed | ExecutionOutcome::Cancelled, + VerifierOutcome::Skipped, + ) => (EvalResultStatus::Completed, None, None), + }; + + V1CellAttempt { + cell_id: self.cell_id.clone(), + sequence: self.sequence, + started_at: self.started_at, + completed_at: self.completed_at, + result: EvalResult { + score, + usage: self.usage.clone(), + cost_usd, + duration_ms: self.duration_ms, + status, + failure_reason, + artifacts, + }, + } + } +} + +/// The 5 explicit semantic difference scenarios between Legacy TS `v1` and Native Rust `v2`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DifferenceScenarioId { + TimeoutWithRewardIncompleteUsage, + AuditTruncation, + MissingCriticalArtifacts, + LateCancellation, + EvidenceRecoveredLater, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DifferenceAdjudication { + pub id: DifferenceScenarioId, + pub title: String, + pub legacy_v1_behavior: String, + pub rust_v2_behavior: String, + pub v1_projection_rule: String, + pub conditions_comparable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DifferenceRegistry { + entries: Vec, +} + +impl Default for DifferenceRegistry { + fn default() -> Self { + Self::new() + } +} + +impl DifferenceRegistry { + pub fn new() -> Self { + Self { + entries: vec![ + DifferenceAdjudication { + id: DifferenceScenarioId::TimeoutWithRewardIncompleteUsage, + title: "Timeout with verifier reward but incomplete usage".into(), + legacy_v1_behavior: "Projects status='subject_failed', preserves verifier score, records recovered lower-bound usage, and sets costUsd=null because metering did not settle.".into(), + rust_v2_behavior: "Records ExecutionOutcome::Timeout, VerifierOutcome::Verified { score }, and MeteringState::LowerBound orthogonally without conflating execution timeout and metering completeness.".into(), + v1_projection_rule: "Projects status='subject_failed', score=Some(score), usage=Some(lower_bound_usage), costUsd=None.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::AuditTruncation, + title: "Egress audit log truncation at byte cap".into(), + legacy_v1_behavior: "Appends ruleId='audit_truncated' marker to hits.jsonl and sets truncated=true in the egress-audit artifact while keeping the attempt status unchanged.".into(), + rust_v2_behavior: "Captures EgressAuditState::Truncated as an explicit dimension while distinguishing it from Missing or Unreadable audit logs (which fail closed as infra_failed).".into(), + v1_projection_rule: "Preserves completed/subject_failed status and emits the egress-audit artifact with truncated=true; Missing or Unreadable audit logs project to status='infra_failed'.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::MissingCriticalArtifacts, + title: "0-byte empty file vs. missing or unreadable critical artifact".into(), + legacy_v1_behavior: "Treats a present 0-byte regular file (e.g. empty stderr) as valid with bytes=0, while missing/unreadable required evidence fails the attempt.".into(), + rust_v2_behavior: "Tracks CriticalArtifactsState::Present (including 0-byte regular files verified by inode/stat) separately from CriticalArtifactsState::Missing.".into(), + v1_projection_rule: "0-byte regular files remain valid artifacts with bytes=0; Missing critical artifacts project to status='infra_failed' with score=None.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::LateCancellation, + title: "Cancellation signal before vs. after verification and settlement".into(), + legacy_v1_behavior: "An abort signal observed before attempt append could overwrite a verified result with indeterminate.".into(), + rust_v2_behavior: "Separates cancellation before verification (cancelled_after_verification=false) from a late signal arriving after verification and settlement commit (cancelled_after_verification=true).".into(), + v1_projection_rule: "Pre-verification cancellation projects to status='indeterminate' with score=None; post-settlement cancellation preserves the verified terminal result.".into(), + conditions_comparable: true, + }, + DifferenceAdjudication { + id: DifferenceScenarioId::EvidenceRecoveredLater, + title: "Post-hoc evidence recovered from checkpoint or host journal after crash".into(), + legacy_v1_behavior: "Mutated or re-ran the attempt when the process died before writing the final cell JSON.".into(), + rust_v2_behavior: "Appends a PostHocRecovery journal entry referencing the original attempt sequence and reconstructs V2AttemptRecord with recovered_post_hoc=true without mutating prior entries.".into(), + v1_projection_rule: "Projects recovered usage/cost and appends a post-hoc-evidence-recovery artifact while preserving immutable sequence numbering.".into(), + conditions_comparable: true, + }, + ], + } + } + + pub fn entries(&self) -> &[DifferenceAdjudication] { + &self.entries + } + + pub fn get(&self, id: DifferenceScenarioId) -> Option<&DifferenceAdjudication> { + self.entries.iter().find(|entry| entry.id == id) + } +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hex::encode(hasher.finalize()) +} + +pub(crate) fn exact_object<'a>( + value: &'a Value, + where_label: &str, + required: &[&str], + optional: &[&str], +) -> Result<&'a JsonObject, ContractError> { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + for key in obj.keys() { + if !required.contains(&key.as_str()) && !optional.contains(&key.as_str()) { + return Err(ContractError::Invalid(format!( + "{where_label} has unsupported fields" + ))); + } + } + for req in required { + if !obj.contains_key(*req) { + return Err(ContractError::Invalid(format!( + "{where_label}.{req} is required" + ))); + } + } + Ok(obj) +} + +pub(crate) fn decode_json_object( + value: &Value, + where_label: &str, +) -> Result { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + for (k, v) in obj { + validate_json_value(v, &format!("{where_label}.{k}"))?; + } + Ok(obj.clone()) +} + +fn validate_json_value(value: &Value, where_label: &str) -> Result<(), ContractError> { + match value { + Value::Null | Value::Bool(_) | Value::String(_) => Ok(()), + Value::Number(n) => { + if n.as_f64().is_some_and(f64::is_finite) { + Ok(()) + } else { + Err(ContractError::Invalid(format!( + "{where_label} must be JSON" + ))) + } + } + Value::Array(arr) => { + for (idx, item) in arr.iter().enumerate() { + validate_json_value(item, &format!("{where_label}[{idx}]"))?; + } + Ok(()) + } + Value::Object(map) => { + for (k, v) in map { + validate_json_value(v, &format!("{where_label}.{k}"))?; + } + Ok(()) + } + } +} + +pub(crate) fn identifier(value: &Value, where_label: &str) -> Result { + let text = nonempty_string(value, where_label)?; + let mut chars = text.chars(); + let Some(first) = chars.next() else { + return Err(ContractError::Invalid(format!("{where_label} is required"))); + }; + if !first.is_ascii_alphanumeric() + || !chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) + { + return Err(ContractError::Invalid(format!("{where_label} is invalid"))); + } + Ok(text) +} + +pub(crate) fn nonempty_string(value: &Value, where_label: &str) -> Result { + match value.as_str() { + Some(s) if !s.is_empty() => Ok(s.to_owned()), + _ => Err(ContractError::Invalid(format!("{where_label} is required"))), + } +} + +fn unique_strings(value: &Value, where_label: &str) -> Result, ContractError> { + let arr = value + .as_array() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an array")))?; + let mut seen = BTreeSet::new(); + let mut out = Vec::with_capacity(arr.len()); + for item in arr { + let s = nonempty_string(item, where_label)?; + if !seen.insert(s.clone()) { + return Err(ContractError::Invalid(format!( + "{where_label} contains duplicates" + ))); + } + out.push(s); + } + Ok(out) +} + +fn ensure_unique_ids<'a>( + ids: impl Iterator, + label: &str, +) -> Result<(), ContractError> { + let mut seen = BTreeSet::new(); + for id in ids { + if !seen.insert(id) { + return Err(ContractError::Invalid(format!("duplicate {label} id"))); + } + } + Ok(()) +} + +pub(crate) fn positive_u32(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) if n >= 1 && n <= u64::from(u32::MAX) => Ok(n as u32), + _ => Err(ContractError::Invalid(format!( + "{where_label} must be positive" + ))), + } +} + +pub(crate) fn positive_usize(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) if n >= 1 => usize::try_from(n) + .map_err(|_| ContractError::Invalid(format!("{where_label} must be positive"))), + _ => Err(ContractError::Invalid(format!( + "{where_label} must be positive" + ))), + } +} + +pub(crate) fn nonnegative_u64(value: &Value, where_label: &str) -> Result { + match value.as_u64() { + Some(n) => Ok(n), + None => match value.as_f64() { + Some(f) if f.is_finite() && f >= 0.0 && f.fract() == 0.0 => Ok(f as u64), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + }, + } +} + +pub(crate) fn nullable_f64(value: &Value, where_label: &str) -> Result, ContractError> { + if value.is_null() { + return Ok(None); + } + match value.as_f64() { + Some(f) if f.is_finite() => Ok(Some(f)), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + } +} + +pub(crate) fn nullable_nonnegative_f64( + value: &Value, + where_label: &str, +) -> Result, ContractError> { + if value.is_null() { + return Ok(None); + } + match value.as_f64() { + Some(f) if f.is_finite() && f >= 0.0 => Ok(Some(f)), + _ => Err(ContractError::Invalid(format!("{where_label} is invalid"))), + } +} diff --git a/products/eval/src/environment.rs b/products/eval/src/environment.rs new file mode 100644 index 00000000000..3aaceef045b --- /dev/null +++ b/products/eval/src/environment.rs @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Per-attempt trial environment abstraction, shared preparation cache boundary +//! locking, egress audit verification, and version-pinned Harbor/Pier bridge. + +pub mod harbor_relay; + +pub use harbor_relay::*; + +use crate::contract::{ + ContractError, EgressAuditState, EvalResultStatus, ExperimentCell, JsonObject, VerifierOutcome, + exact_object, nonempty_string, sha256_hex, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::HashMap; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; + +pub const EGRESS_AUDIT_DESTINATION: &str = "egress-hits.jsonl"; +pub const EGRESS_AUDIT_ARTIFACT_PATH: &str = "artifacts/egress-hits.jsonl"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubjectProcessTermination { + Exited, + FrameworkTimeout, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum SubjectProcessDiagnosticCategory { + #[serde(rename = "none")] + None, + #[serde(rename = "unstructured-output")] + UnstructuredOutput, + #[serde(rename = "result-frame-missing")] + ResultFrameMissing, + #[serde(rename = "result-frame-invalid")] + ResultFrameInvalid, + #[serde(rename = "result-frame-ambiguous")] + ResultFrameAmbiguous, + #[serde(rename = "result-frame-oversize")] + ResultFrameOversize, + #[serde(rename = "execution-scope-unavailable")] + ExecutionScopeUnavailable, +} + +impl SubjectProcessDiagnosticCategory { + pub fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::UnstructuredOutput => "unstructured-output", + Self::ResultFrameMissing => "result-frame-missing", + Self::ResultFrameInvalid => "result-frame-invalid", + Self::ResultFrameAmbiguous => "result-frame-ambiguous", + Self::ResultFrameOversize => "result-frame-oversize", + Self::ExecutionScopeUnavailable => "execution-scope-unavailable", + } + } + + pub fn is_result_frame_error(self) -> bool { + matches!( + self, + Self::ResultFrameMissing + | Self::ResultFrameInvalid + | Self::ResultFrameAmbiguous + | Self::ResultFrameOversize + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectProcessDiagnostic { + pub category: SubjectProcessDiagnosticCategory, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectProcessRequest { + pub command: String, + pub args: Vec, + #[serde(default)] + pub environment: HashMap, + #[serde(default)] + pub credential_environment: HashMap, + #[serde(default = "default_capture_stdout")] + pub capture_stdout: bool, +} + +fn default_capture_stdout() -> bool { + true +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectProcessOutput { + pub termination: SubjectProcessTermination, + pub exit_code: i32, + pub stdout: String, + #[serde(default)] + pub stderr: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub diagnostic: Option, +} + +pub type BoxFuture<'a, T> = Pin + Send + 'a>>; + +pub trait CommandRunner: Send + Sync { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result>; +} + +#[derive(Clone)] +pub struct SubjectExecutionContext { + pub cwd: String, + pub task_input: String, + pub metadata: JsonObject, + pub cancel: CancellationToken, + pub runner: Arc, +} + +impl SubjectExecutionContext { + pub async fn execute( + &self, + request: SubjectProcessRequest, + ) -> Result { + self.runner.execute(request).await + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ExecutorVerification { + pub status: EvalResultStatus, + pub score: Option, + pub failure_reason: Option, + pub artifacts: Vec, + /// Explicitly flags when the verifier reward file or reward key was missing, + /// ensuring missing rewards never default to `0.0`. + #[serde(default)] + pub missing_reward: bool, +} + +impl ExecutorVerification { + pub fn to_verifier_outcome(&self) -> VerifierOutcome { + if self.missing_reward { + return VerifierOutcome::MissingReward { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verifier reward missing".into()), + }; + } + match (self.status, self.score) { + (EvalResultStatus::InfraFailed, _) => VerifierOutcome::VerifierInfraFailed { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verification failed".into()), + }, + (EvalResultStatus::Completed, Some(score)) => VerifierOutcome::Verified { score }, + (EvalResultStatus::Completed, None) => VerifierOutcome::MissingReward { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verifier completed without a numeric reward".into()), + }, + (EvalResultStatus::SubjectFailed, score) => VerifierOutcome::SubjectFailed { + score, + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verifier marked subject failed".into()), + }, + (EvalResultStatus::Indeterminate, _) => VerifierOutcome::VerifierInfraFailed { + reason: self + .failure_reason + .clone() + .unwrap_or_else(|| "verification indeterminate".into()), + }, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExecutorPreparationCode { + Cancelled, + PreparationFailed, + SpawnFailed, + ExitBeforeReady, + InvalidReady, + FrameworkVersionMismatch, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum IndeterminateCause { + HostCancelled, + CleanupUnconfirmed, +} + +/// Active per-attempt trial environment provided to the runner during a single attempt. +pub trait ActiveTrialAttempt: Send + Sync { + fn context(&self) -> SubjectExecutionContext; + fn verify(&self) -> BoxFuture<'_, Result>; + fn read_audit(&self) -> BoxFuture<'_, (EgressAuditState, EgressAuditEvidence)> { + Box::pin(async { + ( + EgressAuditState::NotRequired, + EgressAuditEvidence { + failure_reason: None, + artifacts: Vec::new(), + }, + ) + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TrialEnvironmentOutcome { + Settled(T), + Indeterminate { + cause: IndeterminateCause, + partial: Option, + }, + NotStarted { + code: ExecutorPreparationCode, + artifacts: Vec, + }, +} + +pub type TrialAttemptOperation<'a> = Box< + dyn for<'b> FnOnce( + &'b dyn ActiveTrialAttempt, + ) -> BoxFuture<'b, crate::contract::V2AttemptRecord> + + Send + + 'a, +>; + +pub trait TrialEnvironment: Send + Sync { + fn kind(&self) -> &str; + + fn validate(&self, _cell: &ExperimentCell) -> Result<(), ContractError> { + Ok(()) + } + + fn preflight( + &self, + _subject_credential_names: &[String], + _cancel: CancellationToken, + ) -> BoxFuture<'_, Result<(), ContractError>> { + Box::pin(async { Ok(()) }) + } + + fn run_attempt<'a>( + &'a self, + cell: &'a ExperimentCell, + sequence: u32, + subject_credential_names: &'a [String], + cancel: CancellationToken, + operation: TrialAttemptOperation<'a>, + ) -> BoxFuture<'a, Result, String>>; +} + +/// Synchronizes task preparation across the entire cache-check + cache-build +/// boundary so concurrent arms in a task group never race on shared task assets. +#[derive(Debug, Default, Clone)] +pub struct PreparationCacheLocks { + locks: Arc>>>>, +} + +impl PreparationCacheLocks { + pub fn new() -> Self { + Self::default() + } + + /// Holds an exclusive per-key async lock across the entire `prepare_fn` future. + pub async fn with_task_lock(&self, task_key: &str, prepare_fn: F) -> T + where + F: FnOnce() -> Fut, + Fut: Future, + { + let key_mutex = { + let mut map = self.locks.lock().await; + map.entry(task_key.to_owned()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + }; + let _guard = key_mutex.lock().await; + prepare_fn().await + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EgressAuditEvidence { + pub failure_reason: Option, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct EgressAuditSummary { + pub truncated: bool, + pub policy_error_count: u64, + pub malformed_line_count: u64, +} + +pub fn describe_egress_audit( + bytes: Option<&[u8]>, + required: bool, +) -> (EgressAuditState, EgressAuditEvidence) { + if !required { + return ( + EgressAuditState::NotRequired, + EgressAuditEvidence { + failure_reason: None, + artifacts: Vec::new(), + }, + ); + } + let Some(bytes) = bytes else { + let mut missing_art = Map::new(); + missing_art.insert("kind".into(), Value::String("egress-audit-missing".into())); + missing_art.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + return ( + EgressAuditState::Missing, + EgressAuditEvidence { + failure_reason: Some("egress audit log missing".into()), + artifacts: vec![missing_art], + }, + ); + }; + + let summary = summarize_egress_audit_bytes(bytes); + let mut artifact = Map::new(); + artifact.insert("kind".into(), Value::String("egress-audit".into())); + artifact.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + artifact.insert("bytes".into(), Value::from(bytes.len() as u64)); + artifact.insert( + "sha256".into(), + Value::String(format!("sha256:{}", sha256_hex(bytes))), + ); + artifact.insert("truncated".into(), Value::Bool(summary.truncated)); + artifact.insert( + "policyErrorCount".into(), + Value::from(summary.policy_error_count), + ); + artifact.insert( + "malformedLineCount".into(), + Value::from(summary.malformed_line_count), + ); + + let state = if summary.truncated { + EgressAuditState::Truncated + } else if summary.policy_error_count > 0 { + EgressAuditState::PolicyError { + count: summary.policy_error_count, + } + } else { + EgressAuditState::Clean + }; + + ( + state, + EgressAuditEvidence { + failure_reason: None, + artifacts: vec![artifact], + }, + ) +} + +pub fn read_egress_audit_evidence( + trial_path: &Path, + required: bool, +) -> (EgressAuditState, EgressAuditEvidence) { + if !required { + return describe_egress_audit(None, false); + } + let audit_path = trial_path.join(EGRESS_AUDIT_ARTIFACT_PATH); + match std::fs::read(&audit_path) { + Ok(bytes) => describe_egress_audit(Some(&bytes), true), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => describe_egress_audit(None, true), + Err(err) => { + let reason = format!( + "failed to read egress audit log {} ({err})", + audit_path.display() + ); + let mut art = Map::new(); + art.insert( + "kind".into(), + Value::String("egress-audit-unreadable".into()), + ); + art.insert( + "path".into(), + Value::String(EGRESS_AUDIT_ARTIFACT_PATH.into()), + ); + ( + EgressAuditState::Unreadable { + reason: reason.clone(), + }, + EgressAuditEvidence { + failure_reason: Some(reason), + artifacts: vec![art], + }, + ) + } + } +} + +fn summarize_egress_audit_bytes(bytes: &[u8]) -> EgressAuditSummary { + let mut summary = EgressAuditSummary::default(); + for line in bytes.split(|&b| b == b'\n') { + if line.is_empty() { + continue; + } + let Ok(text) = std::str::from_utf8(line) else { + summary.malformed_line_count += 1; + continue; + }; + let trimmed = text.trim(); + if trimmed.is_empty() { + continue; + } + let Ok(Value::Object(obj)) = serde_json::from_str::(trimmed) else { + summary.malformed_line_count += 1; + continue; + }; + match obj.get("ruleId").and_then(Value::as_str) { + Some("audit_truncated") => summary.truncated = true, + Some("policy_error") => summary.policy_error_count += 1, + _ => {} + } + } + summary +} + +/// Computes the deterministic 1:1 Harbor trial identifier for an Eval cell attempt. +pub fn harbor_trial_name_for_attempt(cell_id: &str, sequence: u32) -> String { + let digest = &sha256_hex(cell_id.as_bytes())[..16]; + let safe_prefix: String = cell_id + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .take(32) + .collect(); + format!("maka-eval-{safe_prefix}-{digest}-seq-{sequence:06}") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HarborExecutorConfig { + pub framework: String, + pub framework_version: String, + pub python_path_env: String, + pub trials_root_env: String, + pub egress_proxy_required: bool, +} + +impl HarborExecutorConfig { + pub fn decode(framework: &str, config: &JsonObject) -> Result { + let val = Value::Object(config.clone()); + let obj = exact_object( + &val, + "executor.config", + &[ + "frameworkVersion", + "pythonPathEnv", + "trialsRootEnv", + "environment", + ], + &["preparationEnvironment", "egressProxy", "mounts"], + )?; + let framework_version = + nonempty_string(obj.get("frameworkVersion").unwrap(), "frameworkVersion")?; + let python_path_env = nonempty_string(obj.get("pythonPathEnv").unwrap(), "pythonPathEnv")?; + let trials_root_env = nonempty_string(obj.get("trialsRootEnv").unwrap(), "trialsRootEnv")?; + let egress_proxy_required = obj.contains_key("egressProxy"); + Ok(Self { + framework: framework.to_owned(), + framework_version, + python_path_env, + trials_root_env, + egress_proxy_required, + }) + } + + pub fn validate_task(&self, cell: &ExperimentCell) -> Result { + let task_cfg = Value::Object(cell.task.config.clone()); + let obj = exact_object(&task_cfg, "task.config", &[&self.framework], &[])?; + let fw_obj = exact_object( + obj.get(&self.framework).unwrap(), + &format!("task.config.{}", self.framework), + &["path"], + &[], + )?; + let path_str = nonempty_string(fw_obj.get("path").unwrap(), "path")?; + Ok(PathBuf::from(path_str)) + } +} diff --git a/products/eval/src/environment/harbor_relay.rs b/products/eval/src/environment/harbor_relay.rs new file mode 100644 index 00000000000..c736a4159a8 --- /dev/null +++ b/products/eval/src/environment/harbor_relay.rs @@ -0,0 +1,987 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Loopback TCP JSON-line relay server and Harbor/Pier trial executor. +//! +//! Implements the exact `ready` -> `execute` -> `executed` -> `verify` protocol +//! spoken by `products/eval/harbor/relay_agent.py` and `run_trial.py`, including +//! version-mismatch exit code detection (`78`), `SIGTERM` -> `SIGKILL` process +//! termination escalation, `/result.json` reward extraction, and +//! collected artifact SHA-256 inventory. + +use crate::contract::{ + ContractError, EgressAuditState, EvalResultStatus, ExperimentCell, JsonObject, V2AttemptRecord, + sha256_hex, +}; +use crate::environment::{ + ActiveTrialAttempt, BoxFuture, CommandRunner, EgressAuditEvidence, ExecutorPreparationCode, + ExecutorVerification, HarborExecutorConfig, IndeterminateCause, SubjectExecutionContext, + SubjectProcessDiagnostic, SubjectProcessDiagnosticCategory, SubjectProcessOutput, + SubjectProcessRequest, SubjectProcessTermination, TrialAttemptOperation, TrialEnvironment, + TrialEnvironmentOutcome, harbor_trial_name_for_attempt, read_egress_audit_evidence, +}; +use crate::store::{ + MAKA_RUNTIME_ARTIFACT_PATH, MAKA_SUBJECT_STDERR_PATH, MAKA_SUBJECT_STDOUT_PATH, + write_atomic_bytes, +}; +use serde_json::{Map, Value, json}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::process::{Child, Command}; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +pub const FRAMEWORK_VERSION_MISMATCH_EXIT_CODE: i32 = 78; +pub const PYTHON_TEARDOWN_DEADLINE_MS: u64 = 110_000; +pub const RELAY_SETTLEMENT_DEADLINE_MS: u64 = 120_000; +pub const TERM_SETTLEMENT_DEADLINE_MS: u64 = 20_000; +pub const KILL_SETTLEMENT_DEADLINE_MS: u64 = 5_000; + +/// Returns the path to the bundled `products/eval/harbor` directory. +pub fn bundled_harbor_relay_root() -> PathBuf { + if let Ok(custom) = std::env::var("MAKA_EVAL_HARBOR_ROOT") + && !custom.is_empty() + { + return PathBuf::from(custom); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("harbor") +} + +/// Terminates a trial child process (`SIGTERM` -> `SIGKILL`) and returns `true` +/// if the child exited cleanly on `SIGTERM`. +async fn terminate_trial(child: &mut Child, deadline_ms: u64) -> bool { + #[cfg(unix)] + if let Some(pid) = child.id() { + unsafe { + let _ = libc::kill(pid as i32, libc::SIGTERM); + } + } + if let Ok(Ok(status)) = + tokio::time::timeout(Duration::from_millis(deadline_ms), child.wait()).await + { + return status.success(); + } + + let _ = child.start_kill(); + let _ = tokio::time::timeout( + Duration::from_millis(KILL_SETTLEMENT_DEADLINE_MS), + child.wait(), + ) + .await; + false +} + +struct RelaySocketRunner { + token: String, + result_token: String, + credentials: HashMap, + execution_environment: HashMap, + stream: Arc>>, + used: AtomicBool, + last_diagnostic: Arc>>, +} + +impl CommandRunner for RelaySocketRunner { + fn execute<'a>( + &'a self, + request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if self.used.swap(true, Ordering::SeqCst) { + return Err("Trial already executed its subject".into()); + } + let mut mapped_credentials = HashMap::new(); + for (target, source) in &request.credential_environment { + let value = self + .credentials + .get(source) + .ok_or_else(|| format!("credential {source} was not admitted"))?; + mapped_credentials.insert(target.clone(), value.clone()); + } + + let mut merged_env = request.environment.clone(); + for (k, v) in &self.execution_environment { + if merged_env.contains_key(k) { + return Err(format!( + "subject environment overrides Eval execution environment: {k}" + )); + } + merged_env.insert(k.clone(), v.clone()); + } + + let execute_msg = json!({ + "token": self.token, + "kind": "execute", + "command": request.command, + "args": request.args, + "environment": merged_env, + "credentials": mapped_credentials, + "resultToken": self.result_token, + "captureStdout": request.capture_stdout, + }); + let mut payload = + serde_json::to_vec(&execute_msg).map_err(|e| format!("json error: {e}"))?; + payload.push(b'\n'); + + let mut line = String::new(); + { + let mut reader = self.stream.lock().await; + reader + .get_mut() + .write_all(&payload) + .await + .map_err(|_| "relay transport is unavailable".to_owned())?; + reader + .get_mut() + .flush() + .await + .map_err(|_| "relay transport is unavailable".to_owned())?; + let bytes_read = reader + .read_line(&mut line) + .await + .map_err(|_| "relay execution result was unavailable".to_owned())?; + if bytes_read == 0 { + return Err("relay execution result was unavailable".into()); + } + } + + let executed: Value = serde_json::from_str(line.trim()) + .map_err(|_| "relay returned an invalid execution result".to_owned())?; + if executed.get("token").and_then(Value::as_str) != Some(&self.token) + || executed.get("kind").and_then(Value::as_str) != Some("executed") + { + return Err("relay returned an invalid execution result".into()); + } + let termination = match executed.get("termination").and_then(Value::as_str) { + Some("exited") => SubjectProcessTermination::Exited, + Some("framework_timeout") => SubjectProcessTermination::FrameworkTimeout, + _ => return Err("relay returned an invalid execution result".into()), + }; + let exit_code = executed + .get("exitCode") + .and_then(Value::as_i64) + .and_then(|c| i32::try_from(c).ok()) + .ok_or_else(|| "relay returned an invalid execution result".to_owned())?; + let stdout = executed + .get("stdout") + .and_then(Value::as_str) + .ok_or_else(|| "relay returned an invalid execution result".to_owned())? + .to_owned(); + let diag_val = executed + .get("diagnostic") + .ok_or_else(|| "relay returned an invalid execution result".to_owned())?; + let diagnostic = decode_process_diagnostic(diag_val) + .ok_or_else(|| "relay returned an invalid execution result".to_owned())?; + *self.last_diagnostic.lock().await = Some(diagnostic.clone()); + + Ok(SubjectProcessOutput { + termination, + exit_code, + stdout, + stderr: String::new(), + diagnostic: Some(diagnostic), + }) + }) + } +} + +fn decode_process_diagnostic(value: &Value) -> Option { + let obj = value.as_object()?; + let cat_str = obj.get("category")?.as_str()?; + let category = match cat_str { + "none" if obj.len() == 1 => { + return Some(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::None, + bytes: None, + sha256: None, + }); + } + "unstructured-output" => SubjectProcessDiagnosticCategory::UnstructuredOutput, + "result-frame-missing" => SubjectProcessDiagnosticCategory::ResultFrameMissing, + "result-frame-invalid" => SubjectProcessDiagnosticCategory::ResultFrameInvalid, + "result-frame-ambiguous" => SubjectProcessDiagnosticCategory::ResultFrameAmbiguous, + "result-frame-oversize" => SubjectProcessDiagnosticCategory::ResultFrameOversize, + "execution-scope-unavailable" => { + SubjectProcessDiagnosticCategory::ExecutionScopeUnavailable + } + _ => return None, + }; + if obj.len() != 3 { + return None; + } + let bytes = obj.get("bytes")?.as_u64()?; + let sha256 = obj.get("sha256")?.as_str()?; + if sha256.len() != 64 + || !sha256 + .bytes() + .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + { + return None; + } + Some(SubjectProcessDiagnostic { + category, + bytes: Some(bytes), + sha256: Some(sha256.to_owned()), + }) +} + +struct ActiveHarborAttempt<'a> { + cell: &'a ExperimentCell, + framework: String, + egress_required: bool, + trial_name: String, + trial_path: PathBuf, + token: String, + context: SubjectExecutionContext, + stream: Arc>>, + child: Arc>, + decision_sent: Arc, + finalization_confirmed: Arc>>, +} + +impl ActiveTrialAttempt for ActiveHarborAttempt<'_> { + fn context(&self) -> SubjectExecutionContext { + self.context.clone() + } + + fn verify(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + if !self.decision_sent.swap(true, Ordering::SeqCst) { + let verify_msg = json!({ + "token": self.token, + "kind": "verify", + }); + let mut payload = serde_json::to_vec(&verify_msg).unwrap_or_default(); + payload.push(b'\n'); + let mut reader = self.stream.lock().await; + if reader.get_mut().write_all(&payload).await.is_err() + || reader.get_mut().shutdown().await.is_err() + { + let mut child = self.child.lock().await; + let ok = terminate_trial(&mut child, RELAY_SETTLEMENT_DEADLINE_MS).await; + *self.finalization_confirmed.lock().await = Some(ok); + return Err("relay verify decision was not delivered".into()); + } + } + let ok = { + let mut child = self.child.lock().await; + child.wait().await.ok().is_some_and(|s| s.success()) + }; + *self.finalization_confirmed.lock().await = Some(ok); + if !ok { + return Err("Trial did not finalize cleanly".into()); + } + read_trial_verification( + &self.trial_path, + &self.trial_name, + self.cell, + &self.framework, + self.egress_required, + ) + }) + } + + fn read_audit(&self) -> BoxFuture<'_, (EgressAuditState, EgressAuditEvidence)> { + let path = self.trial_path.clone(); + let req = self.egress_required; + Box::pin(async move { read_egress_audit_evidence(&path, req) }) + } +} + +/// Reads `/result.json`, extracts the verifier reward key without ever +/// defaulting a missing reward to `0.0`, verifies egress audit evidence, and walks +/// collected artifacts. +pub fn read_trial_verification( + trial_path: &Path, + trial_name: &str, + cell: &ExperimentCell, + framework: &str, + audit_required: bool, +) -> Result { + let result_bytes = std::fs::read(trial_path.join("result.json")) + .map_err(|e| format!("failed to read trial result.json: {e}"))?; + let result_json: Value = serde_json::from_slice(&result_bytes) + .map_err(|e| format!("invalid trial result.json: {e}"))?; + + let reward_key = cell + .verifier + .get("reward") + .and_then(Value::as_str) + .unwrap_or("reward"); + + let exception_type = result_json + .get("exception_info") + .and_then(|v| v.get("exception_type")) + .and_then(Value::as_str); + let subject_exception = matches!( + exception_type, + Some("AgentTimeoutError") | Some("NonZeroAgentExitCodeError") + ); + if result_json + .get("exception_info") + .is_some_and(|v| !v.is_null()) + && !subject_exception + { + return Err("Trial failed outside subject execution".into()); + } + + let score = result_json + .get("verifier_result") + .and_then(|vr| vr.get("rewards")) + .and_then(|r| r.get(reward_key)) + .and_then(Value::as_f64) + .filter(|n| n.is_finite()); + + let (_, audit) = read_egress_audit_evidence(trial_path, audit_required); + let missing_reward = score.is_none(); + let base_failure = missing_reward.then(|| "verifier produced no reward".to_owned()); + let status = if audit.failure_reason.is_some() || missing_reward { + EvalResultStatus::InfraFailed + } else if subject_exception { + EvalResultStatus::SubjectFailed + } else { + EvalResultStatus::Completed + }; + + let mut artifacts = Vec::new(); + let mut trial_art = Map::new(); + trial_art.insert("kind".into(), Value::String("trial".into())); + trial_art.insert( + "framework".into(), + Value::String(cell.executor.kind.as_str().into()), + ); + trial_art.insert("trialName".into(), Value::String(trial_name.into())); + artifacts.push(trial_art); + artifacts.extend(collected_artifact_inventory(trial_path, framework)); + artifacts.extend(audit.artifacts); + + Ok(ExecutorVerification { + status, + score, + failure_reason: audit.failure_reason.or(base_failure), + artifacts, + missing_reward, + }) +} + +pub fn collected_artifact_inventory(trial_path: &Path, framework: &str) -> Vec { + let root = if framework == "pier" { + trial_path.join("artifacts") + } else { + trial_path.join("artifacts").join("logs").join("artifacts") + }; + let targets = [ + root.join( + Path::new(MAKA_RUNTIME_ARTIFACT_PATH) + .file_name() + .unwrap_or_default(), + ), + root.join( + Path::new(MAKA_SUBJECT_STDOUT_PATH) + .file_name() + .unwrap_or_default(), + ), + root.join( + Path::new(MAKA_SUBJECT_STDERR_PATH) + .file_name() + .unwrap_or_default(), + ), + ]; + let mut files = Vec::new(); + for target in &targets { + walk_collected_artifacts(trial_path, target, &mut files); + } + files.sort_by(|a, b| { + let pa = a.get("path").and_then(Value::as_str).unwrap_or(""); + let pb = b.get("path").and_then(Value::as_str).unwrap_or(""); + pa.cmp(pb) + }); + files +} + +fn walk_collected_artifacts(trial_path: &Path, current: &Path, out: &mut Vec) { + let Ok(meta) = std::fs::symlink_metadata(current) else { + return; + }; + if meta.file_type().is_symlink() { + return; + } + if meta.is_file() { + let Ok(bytes) = std::fs::read(current) else { + return; + }; + let Ok(rel) = current.strip_prefix(trial_path) else { + return; + }; + let rel_str = rel + .components() + .map(|c| c.as_os_str().to_string_lossy()) + .collect::>() + .join("/"); + let mut obj = Map::new(); + obj.insert("kind".into(), Value::String("collected-artifact".into())); + obj.insert("path".into(), Value::String(rel_str)); + obj.insert("bytes".into(), Value::from(bytes.len() as u64)); + obj.insert( + "sha256".into(), + Value::String(format!("sha256:{}", sha256_hex(&bytes))), + ); + out.push(obj); + return; + } + if meta.is_dir() + && let Ok(entries) = std::fs::read_dir(current) + { + for entry in entries.flatten() { + walk_collected_artifacts(trial_path, &entry.path(), out); + } + } +} + +/// Real Harbor / Pier trial environment that binds a loopback TCP relay server, +/// writes `.json` with `relay_agent:RelayAgent`, spawns `run_trial.py` +/// (or a custom trial command for testing), and coordinates the trial lifecycle. +pub struct HarborTrialEnvironment { + config: HarborExecutorConfig, + spec_dir: PathBuf, + env_overrides: HashMap, +} + +impl HarborTrialEnvironment { + pub fn new(config: HarborExecutorConfig, spec_dir: PathBuf) -> Self { + Self { + config, + spec_dir, + env_overrides: HashMap::new(), + } + } + + pub fn with_env_overrides(mut self, env_overrides: HashMap) -> Self { + self.env_overrides = env_overrides; + self + } + + fn resolve_env_var(&self, key: &str) -> Option { + self.env_overrides + .get(key) + .cloned() + .or_else(|| std::env::var(key).ok()) + .filter(|v| !v.is_empty()) + } + + fn resolve_trials_root(&self) -> Result { + let raw = self + .resolve_env_var(&self.config.trials_root_env) + .ok_or_else(|| { + format!( + "machine path {} is unavailable", + self.config.trials_root_env + ) + })?; + Ok(PathBuf::from(raw)) + } + + fn resolve_python_executable(&self) -> Result { + let raw = self + .resolve_env_var(&self.config.python_path_env) + .ok_or_else(|| { + format!( + "machine path {} is unavailable", + self.config.python_path_env + ) + })?; + Ok(PathBuf::from(raw)) + } +} + +impl TrialEnvironment for HarborTrialEnvironment { + fn kind(&self) -> &str { + &self.config.framework + } + + fn validate(&self, cell: &ExperimentCell) -> Result<(), ContractError> { + let _ = self.config.validate_task(cell)?; + Ok(()) + } + + fn preflight( + &self, + subject_credential_names: &[String], + cancel: CancellationToken, + ) -> BoxFuture<'_, Result<(), ContractError>> { + let cred_names: Vec = subject_credential_names.to_vec(); + Box::pin(async move { + if cancel.is_cancelled() { + return Err(ContractError::Invalid("preflight cancelled".into())); + } + if self.config.egress_proxy_required { + return Err(ContractError::Invalid( + "egressProxy is not supported by the native Harbor/Pier executor".into(), + )); + } + let _ = self.resolve_trials_root().map_err(ContractError::Invalid)?; + let _ = self + .resolve_python_executable() + .map_err(ContractError::Invalid)?; + for name in &cred_names { + if self.resolve_env_var(name).is_none() { + return Err(ContractError::Invalid(format!( + "subject credential {name} is required" + ))); + } + } + Ok(()) + }) + } + + fn run_attempt<'a>( + &'a self, + cell: &'a ExperimentCell, + sequence: u32, + _subject_credential_names: &'a [String], + cancel: CancellationToken, + operation: TrialAttemptOperation<'a>, + ) -> BoxFuture<'a, Result, String>> { + Box::pin(async move { + if cancel.is_cancelled() { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::Cancelled, + artifacts: Vec::new(), + }); + } + + let trials_root = match self.resolve_trials_root() { + Ok(p) => p, + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + let python_exe = match self.resolve_python_executable() { + Ok(p) => p, + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + + let mut credentials = HashMap::new(); + for name in &cell.subject.credentials { + let Some(val) = self.resolve_env_var(name) else { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + }; + credentials.insert(name.clone(), val); + } + + if std::fs::create_dir_all(&trials_root).is_err() { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + + let trial_name = harbor_trial_name_for_attempt(&cell.id, sequence); + let config_path = trials_root.join(format!("{trial_name}.json")); + let trial_path = trials_root.join(&trial_name); + let relay_token = format!( + "{}{}", + Uuid::new_v4().simple(), + &Uuid::new_v4().simple().to_string()[..16] + ); + + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(l) => l, + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + let relay_port = match listener.local_addr() { + Ok(addr) => addr.port(), + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + + let task_path = match self.config.validate_task(cell) { + Ok(p) => p.to_string_lossy().to_string(), + Err(_) => { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + }; + let timeout_multiplier = cell + .budget + .get("timeoutMultiplier") + .and_then(Value::as_f64) + .unwrap_or(1.0); + + let trial_config = json!({ + "task": { + "path": task_path, + "git_url": cell.benchmark.config.get("repository").and_then(Value::as_str).unwrap_or(""), + "git_commit_id": cell.benchmark.version, + }, + "trial_name": trial_name, + "trials_dir": trials_root.to_string_lossy(), + "timeout_multiplier": timeout_multiplier, + "agent": { + "import_path": "relay_agent:RelayAgent", + "kwargs": { + "relay_host": "127.0.0.1", + "relay_port": relay_port, + "relay_token": relay_token, + "teardown_timeout_ms": PYTHON_TEARDOWN_DEADLINE_MS, + } + }, + "environment": cell.executor.config.get("environment").cloned().unwrap_or_else(|| json!({})), + }); + let mut config_bytes = serde_json::to_vec(&trial_config).unwrap_or_default(); + config_bytes.push(b'\n'); + if write_atomic_bytes(&config_path, &config_bytes).is_err() { + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::PreparationFailed, + artifacts: Vec::new(), + }); + } + + let run_trial_script = bundled_harbor_relay_root().join("run_trial.py"); + + let mut cmd = Command::new(&python_exe); + cmd.arg(&run_trial_script) + .arg(&self.config.framework) + .arg(&self.config.framework_version) + .arg(&config_path) + .current_dir(&self.spec_dir) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + cmd.env("PYTHONPATH", bundled_harbor_relay_root()); + if self.config.egress_proxy_required { + cmd.env("MAKA_EVAL_EGRESS_REQUIRED", "1"); + } + + let mut child = match cmd.spawn() { + Ok(c) => c, + Err(_) => { + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::SpawnFailed, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "spawn", + ExecutorPreparationCode::SpawnFailed, + None, + )], + }); + } + }; + + // Wait for either the relay TCP connection or premature process exit. + let accept_res = tokio::select! { + _ = cancel.cancelled() => { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::Cancelled, + artifacts: Vec::new(), + }); + } + status = child.wait() => { + let exit_code = status.ok().and_then(|s| s.code()); + let code = if exit_code == Some(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) { + ExecutorPreparationCode::FrameworkVersionMismatch + } else { + ExecutorPreparationCode::ExitBeforeReady + }; + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "exit-before-ready", + code, + exit_code, + )], + }); + } + accepted = listener.accept() => accepted, + }; + + let Ok((socket, _)) = accept_res else { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + let _ = std::fs::remove_file(&config_path); + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::ExitBeforeReady, + artifacts: Vec::new(), + }); + }; + drop(listener); + + let mut reader = BufReader::new(socket); + let mut ready_line = String::new(); + let read_ready = tokio::select! { + _ = cancel.cancelled() => { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::Cancelled, + artifacts: Vec::new(), + }); + } + res = reader.read_line(&mut ready_line) => res, + }; + + let Ok(n) = read_ready else { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::InvalidReady, + artifacts: Vec::new(), + }); + }; + if n == 0 { + let status = child.wait().await.ok(); + let exit_code = status.and_then(|s| s.code()); + let code = if exit_code == Some(FRAMEWORK_VERSION_MISMATCH_EXIT_CODE) { + ExecutorPreparationCode::FrameworkVersionMismatch + } else { + ExecutorPreparationCode::InvalidReady + }; + return Ok(TrialEnvironmentOutcome::NotStarted { + code, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "ready-decode", + code, + exit_code, + )], + }); + } + + let Ok(ready_json) = serde_json::from_str::(ready_line.trim()) else { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::InvalidReady, + artifacts: Vec::new(), + }); + }; + + let valid_ready = ready_json.get("token").and_then(Value::as_str) == Some(&relay_token) + && ready_json.get("kind").and_then(Value::as_str) == Some("ready") + && ready_json + .get("instruction") + .and_then(Value::as_str) + .is_some() + && ready_json + .get("cwd") + .and_then(Value::as_str) + .is_some_and(|c| c.starts_with('/')); + if !valid_ready { + let _ = terminate_trial(&mut child, TERM_SETTLEMENT_DEADLINE_MS).await; + return Ok(TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::InvalidReady, + artifacts: vec![preparation_artifact( + &self.config.framework, + &trial_name, + &trial_path, + "ready-decode", + ExecutorPreparationCode::InvalidReady, + None, + )], + }); + } + + let instruction = ready_json["instruction"].as_str().unwrap().to_owned(); + let cwd = ready_json["cwd"].as_str().unwrap().to_owned(); + let result_token = Uuid::new_v4().simple().to_string(); + + let mut metadata = Map::new(); + metadata.insert("trialName".into(), Value::String(trial_name.clone())); + metadata.insert( + "trialPath".into(), + Value::String(trial_path.to_string_lossy().into()), + ); + metadata.insert("meteringSecret".into(), Value::String(result_token.clone())); + + let mut exec_env = HashMap::new(); + exec_env.insert("DEBIAN_FRONTEND".into(), "noninteractive".into()); + exec_env.insert("TZ".into(), "Etc/UTC".into()); + + let shared_stream = Arc::new(Mutex::new(reader)); + let last_diagnostic = Arc::new(Mutex::new(None)); + let socket_runner = Arc::new(RelaySocketRunner { + token: relay_token.clone(), + result_token, + credentials, + execution_environment: exec_env, + stream: shared_stream.clone(), + used: AtomicBool::new(false), + last_diagnostic: last_diagnostic.clone(), + }); + + let subject_context = SubjectExecutionContext { + cwd, + task_input: instruction, + metadata, + cancel: cancel.clone(), + runner: socket_runner.clone(), + }; + + let shared_child = Arc::new(Mutex::new(child)); + let decision_sent = Arc::new(AtomicBool::new(false)); + let finalization_confirmed = Arc::new(Mutex::new(None)); + + let active = ActiveHarborAttempt { + cell, + framework: self.config.framework.clone(), + egress_required: self.config.egress_proxy_required, + trial_name, + trial_path, + token: relay_token, + context: subject_context, + stream: shared_stream, + child: shared_child.clone(), + decision_sent: decision_sent.clone(), + finalization_confirmed: finalization_confirmed.clone(), + }; + + let record = { + let operation = operation(&active); + tokio::pin!(operation); + tokio::select! { + biased; + record = &mut operation => Some(record), + _ = cancel.cancelled() => None, + } + }; + if !decision_sent.load(Ordering::SeqCst) + || finalization_confirmed.lock().await.is_none() + { + let used = socket_runner.used.load(Ordering::SeqCst); + let mut child_guard = shared_child.lock().await; + let ok = terminate_trial( + &mut child_guard, + if used { + RELAY_SETTLEMENT_DEADLINE_MS + } else { + TERM_SETTLEMENT_DEADLINE_MS + }, + ) + .await; + *finalization_confirmed.lock().await = Some(ok); + } + + let finalized_ok = finalization_confirmed.lock().await.unwrap_or(false); + let Some(mut record) = record else { + return Ok(TrialEnvironmentOutcome::Indeterminate { + cause: if finalized_ok { + IndeterminateCause::HostCancelled + } else { + IndeterminateCause::CleanupUnconfirmed + }, + partial: None, + }); + }; + + if let Some(diag) = last_diagnostic.lock().await.clone() + && diag.category != SubjectProcessDiagnosticCategory::None + { + let mut art = Map::new(); + art.insert("kind".into(), Value::String("executor-relay-result".into())); + art.insert( + "category".into(), + Value::String(diag.category.as_str().into()), + ); + if let Some(b) = diag.bytes { + art.insert("bytes".into(), Value::from(b)); + } + if let Some(s) = diag.sha256 { + art.insert("sha256".into(), Value::String(s)); + } + record.artifacts.push(art); + } + + if !finalized_ok { + return Ok(TrialEnvironmentOutcome::Indeterminate { + cause: IndeterminateCause::CleanupUnconfirmed, + partial: Some(record), + }); + } + + Ok(TrialEnvironmentOutcome::Settled(record)) + }) + } +} + +fn preparation_artifact( + framework: &str, + trial_name: &str, + trial_path: &Path, + stage: &str, + code: ExecutorPreparationCode, + exit_code: Option, +) -> JsonObject { + let _ = std::fs::create_dir_all(trial_path); + let diag_name = "preparation-error.json"; + let diag_payload = json!({ + "stage": stage, + "framework": framework, + "code": code, + "errorCode": Value::Null, + "exitCode": exit_code, + "signal": Value::Null, + }); + if let Ok(mut bytes) = serde_json::to_vec(&diag_payload) { + bytes.push(b'\n'); + let _ = write_atomic_bytes(&trial_path.join(diag_name), &bytes); + } + let mut art = Map::new(); + art.insert("kind".into(), Value::String("executor-preparation".into())); + art.insert("framework".into(), Value::String(framework.into())); + art.insert("trialName".into(), Value::String(trial_name.into())); + art.insert("path".into(), Value::String(diag_name.into())); + art +} diff --git a/products/eval/src/lib.rs b/products/eval/src/lib.rs new file mode 100644 index 00000000000..e4cf378ed88 --- /dev/null +++ b/products/eval/src/lib.rs @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Measurement and control plane for multi-arm Maka evaluation experiments. + +pub mod contract; +pub mod environment; +pub mod measurement; +pub mod report; +pub mod runner; +pub mod store; +pub mod subject; diff --git a/products/eval/src/measurement.rs b/products/eval/src/measurement.rs new file mode 100644 index 00000000000..b7e6723b94f --- /dev/null +++ b/products/eval/src/measurement.rs @@ -0,0 +1,671 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Token usage preservation, admission boundary accounting, checkpoint HMAC +//! verification, receipt-vs-proxy reconciliation, and standardized pricing. + +use crate::contract::{ContractError, JsonObject, exact_object, nonnegative_u64, sha256_hex}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +pub mod gateway; +pub use gateway::*; + +pub const METERING_CHECKPOINT_LIMIT_BYTES: usize = 64 * 1024; +pub const COST_USD_TOLERANCE: f64 = 1e-9; + +/// Exact 6-counter normalized token usage. +/// +/// Each counter is preserved as reported by the provider or receipt; `total_tokens` +/// is never synthesized or overwritten by summing the other five counters. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct NormalizedUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub cache_write_tokens: u64, + pub reasoning_tokens: u64, + pub total_tokens: u64, +} + +impl NormalizedUsage { + pub fn decode(value: &Value, where_label: &str) -> Result { + let obj = exact_object( + value, + where_label, + &[ + "inputTokens", + "outputTokens", + "cacheReadTokens", + "cacheWriteTokens", + "reasoningTokens", + "totalTokens", + ], + &[], + )?; + Ok(Self { + input_tokens: nonnegative_u64( + obj.get("inputTokens").unwrap(), + &format!("{where_label}.inputTokens"), + )?, + output_tokens: nonnegative_u64( + obj.get("outputTokens").unwrap(), + &format!("{where_label}.outputTokens"), + )?, + cache_read_tokens: nonnegative_u64( + obj.get("cacheReadTokens").unwrap(), + &format!("{where_label}.cacheReadTokens"), + )?, + cache_write_tokens: nonnegative_u64( + obj.get("cacheWriteTokens").unwrap(), + &format!("{where_label}.cacheWriteTokens"), + )?, + reasoning_tokens: nonnegative_u64( + obj.get("reasoningTokens").unwrap(), + &format!("{where_label}.reasoningTokens"), + )?, + total_tokens: nonnegative_u64( + obj.get("totalTokens").unwrap(), + &format!("{where_label}.totalTokens"), + )?, + }) + } + + /// Accumulates another `NormalizedUsage` record counter-by-counter without + /// synthesizing `total_tokens` from the component fields. + pub fn accumulate_exact(&mut self, other: &Self) { + self.input_tokens = self.input_tokens.saturating_add(other.input_tokens); + self.output_tokens = self.output_tokens.saturating_add(other.output_tokens); + self.cache_read_tokens = self + .cache_read_tokens + .saturating_add(other.cache_read_tokens); + self.cache_write_tokens = self + .cache_write_tokens + .saturating_add(other.cache_write_tokens); + self.reasoning_tokens = self.reasoning_tokens.saturating_add(other.reasoning_tokens); + self.total_tokens = self.total_tokens.saturating_add(other.total_tokens); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MeteringState { + Complete, + LowerBound, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum TokenBasis { + #[serde(rename = "complete")] + Complete, + #[serde(rename = "lower-bound")] + LowerBound, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct PricingRates { + pub input_per_million: f64, + pub output_per_million: f64, + pub cache_read_per_million: f64, + pub cache_write_per_million: f64, +} + +/// Published DeepSeek V4 Flash prices in USD per million tokens (off-peak band). +pub const DEEPSEEK_V4_FLASH_COST: PricingRates = PricingRates { + input_per_million: 0.22, + output_per_million: 0.66, + cache_read_per_million: 0.007, + cache_write_per_million: 0.22, +}; + +pub fn deepseek_cost_usd(usage: &NormalizedUsage) -> f64 { + price_usage_usd(usage, &DEEPSEEK_V4_FLASH_COST) +} + +pub fn price_usage_usd(usage: &NormalizedUsage, rates: &PricingRates) -> f64 { + let cached = usage + .cache_read_tokens + .saturating_add(usage.cache_write_tokens); + let uncached = usage.input_tokens.saturating_sub(cached); + ((uncached as f64) * rates.input_per_million + + (usage.cache_read_tokens as f64) * rates.cache_read_per_million + + (usage.cache_write_tokens as f64) * rates.cache_write_per_million + + (usage.output_tokens as f64) * rates.output_per_million) + / 1_000_000.0 +} + +pub fn costs_match_within_tolerance(left: Option, right: Option) -> bool { + match (left, right) { + (None, None) => true, + (Some(a), Some(b)) => (a - b).abs() <= COST_USD_TOLERANCE, + _ => false, + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProviderMeteringCounts { + pub usage: Option, + pub settled: bool, + pub requests: u64, + pub in_flight_requests: u64, + pub admitted_requests: u64, + pub usage_requests: u64, + pub models: Vec, + pub tool_names: Vec, +} + +impl ProviderMeteringCounts { + pub fn validate(&self) -> bool { + if self.settled && self.in_flight_requests > 0 { + return false; + } + if self.in_flight_requests > self.requests + || self.admitted_requests > self.requests + || self.usage_requests > self.admitted_requests + { + return false; + } + if self.usage_requests > 0 && self.usage.is_none() { + return false; + } + true + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DerivedMetering { + pub settled_requests: u64, + pub missing_usage_requests: u64, + pub usage_complete: bool, + pub cost_usd: Option, + pub token_basis: TokenBasis, + pub metering_state: MeteringState, +} + +pub fn derive_metering(counts: &ProviderMeteringCounts) -> DerivedMetering { + let usage_complete = counts.settled + && counts.admitted_requests > 0 + && counts.usage_requests == counts.admitted_requests; + let cost_usd = match (&counts.usage, usage_complete) { + (Some(u), true) => Some(deepseek_cost_usd(u)), + _ => None, + }; + let metering_state = match (&counts.usage, usage_complete) { + (Some(_), true) => MeteringState::Complete, + (Some(_), false) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + DerivedMetering { + settled_requests: counts.requests.saturating_sub(counts.in_flight_requests), + missing_usage_requests: counts + .admitted_requests + .saturating_sub(counts.usage_requests), + usage_complete, + cost_usd, + token_basis: if usage_complete { + TokenBasis::Complete + } else { + TokenBasis::LowerBound + }, + metering_state, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RequestLedgerEntry { + admitted: bool, + in_flight: bool, + usage: Option, + model: Option, +} + +/// Tracks request admission across the trial execution cutoff, deduplicating +/// repeated checkpoints and reconciling Host receipts against proxy metering +/// without double-counting. +#[derive(Debug, Clone, Default)] +pub struct AdmissionBoundaryLedger { + cutoff_closed: bool, + settled: bool, + requests: BTreeMap, + seen_checkpoint_digests: BTreeSet, + latest_checkpoint: Option, + tool_names: BTreeSet, +} + +impl AdmissionBoundaryLedger { + pub fn new() -> Self { + Self::default() + } + + pub fn is_admission_closed(&self) -> bool { + self.cutoff_closed + } + + /// Closes admission for new requests at the execution deadline/cancellation + /// boundary while allowing already-admitted in-flight requests to drain. + pub fn close_admission(&mut self) { + self.cutoff_closed = true; + } + + /// Attempts to admit a request with `request_id`. Returns `true` if admitted, + /// or `false` if the admission boundary has already closed. Idempotent for an + /// already-recorded `request_id`. + pub fn record_request_start(&mut self, request_id: impl Into) -> bool { + let id = request_id.into(); + if let Some(existing) = self.requests.get(&id) { + return existing.admitted; + } + let admitted = !self.cutoff_closed; + self.requests.insert( + id, + RequestLedgerEntry { + admitted, + in_flight: admitted, + usage: None, + model: None, + }, + ); + admitted + } + + /// Records the settlement of an admitted request. Duplicate settlements for + /// the same `request_id` (e.g. from both a Host receipt and a proxy log) are + /// reconciled idempotently without double-counting tokens. + pub fn settle_request( + &mut self, + request_id: &str, + usage: Option, + model: Option, + ) -> Result<(), ContractError> { + let Some(entry) = self.requests.get_mut(request_id) else { + return Err(ContractError::Invalid(format!( + "unknown request_id in metering settlement: {request_id}" + ))); + }; + if !entry.admitted { + return Ok(()); + } + entry.in_flight = false; + if let (Some(existing), Some(incoming)) = (&entry.usage, &usage) { + if existing != incoming { + return Err(ContractError::Invalid(format!( + "conflicting usage between receipt and proxy for request {request_id}" + ))); + } + } else if entry.usage.is_none() { + entry.usage = usage; + } + if entry.model.is_none() { + entry.model = model; + } + Ok(()) + } + + /// Ingests a proxy checkpoint snapshot. Duplicate snapshots with the same + /// canonical digest are ignored so repeated checkpoint reads never inflate counts. + pub fn ingest_checkpoint(&mut self, counts: ProviderMeteringCounts) -> bool { + let serialized = serde_json::to_vec(&counts).unwrap_or_default(); + let digest = sha256_hex(&serialized); + if !self.seen_checkpoint_digests.insert(digest) { + return false; + } + self.latest_checkpoint = Some(counts); + true + } + + pub fn mark_proxy_settled(&mut self) { + if self.requests.values().all(|r| !r.in_flight) { + self.settled = true; + } + } + + pub fn snapshot(&self) -> ProviderMeteringCounts { + if self.requests.is_empty() + && let Some(cp) = &self.latest_checkpoint + { + return cp.clone(); + } + let requests = self.requests.len() as u64; + let in_flight_requests = self.requests.values().filter(|r| r.in_flight).count() as u64; + let admitted_requests = self.requests.values().filter(|r| r.admitted).count() as u64; + let mut usage_requests = 0u64; + let mut accumulated = NormalizedUsage::default(); + let mut models = BTreeSet::new(); + + for entry in self.requests.values().filter(|r| r.admitted) { + if let Some(u) = &entry.usage { + usage_requests += 1; + accumulated.accumulate_exact(u); + } + if let Some(m) = &entry.model { + models.insert(m.clone()); + } + } + + ProviderMeteringCounts { + usage: if usage_requests > 0 { + Some(accumulated) + } else { + None + }, + settled: self.settled && in_flight_requests == 0, + requests, + in_flight_requests, + admitted_requests, + usage_requests, + models: models.into_iter().collect(), + tool_names: self.tool_names.iter().cloned().collect(), + } + } +} + +/// Reconciles Host-reported usage with proxy-recovered usage without double-counting. +/// +/// - When both Host receipts and proxy counts are settled and complete, they must agree; +/// otherwise reconciliation reports `MeteringState::LowerBound` with the conservative record. +/// - When the Host died before settling receipts, the verified proxy checkpoint provides +/// either `Complete` (if the proxy settled with all admitted requests accounted for) or +/// `LowerBound` usage. +pub fn reconcile_host_and_proxy_usage( + host_usage: Option<&NormalizedUsage>, + host_settled: bool, + proxy_counts: Option<&ProviderMeteringCounts>, +) -> (Option, MeteringState, Option) { + match (host_usage, host_settled, proxy_counts) { + (Some(h), true, Some(p)) => { + let derived = derive_metering(p); + if derived.usage_complete { + if p.usage.as_ref() == Some(h) { + ( + Some(h.clone()), + MeteringState::Complete, + Some(deepseek_cost_usd(h)), + ) + } else { + (Some(h.clone()), MeteringState::LowerBound, None) + } + } else { + ( + Some(h.clone()), + MeteringState::Complete, + Some(deepseek_cost_usd(h)), + ) + } + } + (Some(h), true, None) => ( + Some(h.clone()), + MeteringState::Complete, + Some(deepseek_cost_usd(h)), + ), + (Some(h), false, Some(p)) => { + let derived = derive_metering(p); + if derived.usage_complete { + (p.usage.clone(), MeteringState::Complete, derived.cost_usd) + } else { + ( + p.usage.clone().or_else(|| Some(h.clone())), + MeteringState::LowerBound, + None, + ) + } + } + (Some(h), false, None) => (Some(h.clone()), MeteringState::LowerBound, None), + (None, _, Some(p)) => { + let derived = derive_metering(p); + (p.usage.clone(), derived.metering_state, derived.cost_usd) + } + (None, _, None) => (None, MeteringState::Unknown, None), + } +} + +/// Computes RFC 2104 HMAC-SHA256 in hex. +pub fn hmac_sha256_hex(secret: &[u8], message: &[u8]) -> String { + const BLOCK_SIZE: usize = 64; + let mut key_block = [0u8; BLOCK_SIZE]; + if secret.len() > BLOCK_SIZE { + let digest = Sha256::digest(secret); + key_block[..digest.len()].copy_from_slice(&digest); + } else { + key_block[..secret.len()].copy_from_slice(secret); + } + + let mut ipad = [0x36u8; BLOCK_SIZE]; + let mut opad = [0x5cu8; BLOCK_SIZE]; + for i in 0..BLOCK_SIZE { + ipad[i] ^= key_block[i]; + opad[i] ^= key_block[i]; + } + + let mut inner = Sha256::new(); + inner.update(ipad); + inner.update(message); + let inner_hash = inner.finalize(); + + let mut outer = Sha256::new(); + outer.update(opad); + outer.update(inner_hash); + hex::encode(outer.finalize()) +} + +pub(crate) fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut diff = 0u8; + for (x, y) in a.iter().zip(b.iter()) { + diff |= x ^ y; + } + diff == 0 +} + +pub fn sign_metering_checkpoint(body: &JsonObject, secret: &str) -> JsonObject { + let serialized = serde_json::to_string(&Value::Object(body.clone())) + .expect("serializing JsonObject cannot fail"); + let mac = hmac_sha256_hex(secret.as_bytes(), serialized.as_bytes()); + let mut signed = body.clone(); + signed.insert("mac".into(), Value::String(mac)); + signed +} + +pub fn metering_checkpoint_mac_matches(record: &JsonObject, secret: &str) -> bool { + let Some(mac) = record.get("mac").and_then(Value::as_str) else { + return false; + }; + if mac.len() != 64 || !mac.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) { + return false; + } + let mut body = record.clone(); + body.remove("mac"); + let serialized = match serde_json::to_string(&Value::Object(body)) { + Ok(s) => s, + Err(_) => return false, + }; + let expected = hmac_sha256_hex(secret.as_bytes(), serialized.as_bytes()); + constant_time_eq(expected.as_bytes(), mac.as_bytes()) +} + +/// Reads a bounded regular file while refusing symlinks (`O_NOFOLLOW` on unix) +/// and oversized or empty files. +pub fn read_bounded_regular_file(path: &Path, limit_bytes: usize) -> Option> { + use std::io::Read; + let symlink_meta = std::fs::symlink_metadata(path).ok()?; + if !symlink_meta.file_type().is_file() { + return None; + } + let mut options = std::fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(path).ok()?; + let meta = file.metadata().ok()?; + if !meta.is_file() || meta.len() == 0 || meta.len() > (limit_bytes as u64) { + return None; + } + let size = meta.len() as usize; + let mut buf = Vec::with_capacity(size); + file.read_to_end(&mut buf).ok()?; + if buf.len() != size { + return None; + } + Some(buf) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RecoveredExternalMetering { + pub usage: Option, + pub cost_usd: Option, + pub admitted_requests: u64, + pub metering_state: MeteringState, + pub counts: ProviderMeteringCounts, + pub artifact: JsonObject, +} + +pub fn recover_external_metering_checkpoint( + trial_path: &Path, + profile: &str, + metering_secret: &str, +) -> Option { + let rel_path = format!("agent/{profile}.provider-usage.json"); + let full_path = trial_path.join(&rel_path); + let bytes = read_bounded_regular_file(&full_path, METERING_CHECKPOINT_LIMIT_BYTES)?; + let parsed: Value = serde_json::from_slice(&bytes).ok()?; + let checkpoint = exact_object( + &parsed, + "external metering checkpoint", + &[ + "schemaVersion", + "profile", + "usage", + "settled", + "requests", + "inFlightRequests", + "admittedRequests", + "usageRequests", + "models", + "toolNames", + "mac", + ], + &[], + ) + .ok()?; + if checkpoint.get("schemaVersion").and_then(Value::as_str) + != Some("maka.external_provider_usage.v2") + || checkpoint.get("profile").and_then(Value::as_str) != Some(profile) + || !metering_checkpoint_mac_matches(checkpoint, metering_secret) + { + return None; + } + let usage = match checkpoint.get("usage")? { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, "external usage").ok()?), + }; + let settled = checkpoint.get("settled")?.as_bool()?; + let requests = nonnegative_u64(checkpoint.get("requests")?, "requests").ok()?; + let in_flight_requests = + nonnegative_u64(checkpoint.get("inFlightRequests")?, "inFlightRequests").ok()?; + let admitted_requests = + nonnegative_u64(checkpoint.get("admittedRequests")?, "admittedRequests").ok()?; + let usage_requests = nonnegative_u64(checkpoint.get("usageRequests")?, "usageRequests").ok()?; + let models = checkpoint + .get("models")? + .as_array()? + .iter() + .map(|v| v.as_str().map(str::to_owned)) + .collect::>>()?; + let tool_names = checkpoint + .get("toolNames")? + .as_array()? + .iter() + .map(|v| v.as_str().map(str::to_owned)) + .collect::>>()?; + + let counts = ProviderMeteringCounts { + usage: usage.clone(), + settled, + requests, + in_flight_requests, + admitted_requests, + usage_requests, + models: models.clone(), + tool_names: tool_names.clone(), + }; + if !counts.validate() { + return None; + } + let derived = derive_metering(&counts); + let mut artifact = Map::new(); + artifact.insert( + "kind".into(), + Value::String("provider-metering-recovery".into()), + ); + artifact.insert("profile".into(), Value::String(profile.into())); + artifact.insert("path".into(), Value::String(rel_path)); + artifact.insert("bytes".into(), Value::from(bytes.len() as u64)); + artifact.insert("sha256".into(), Value::String(sha256_hex(&bytes))); + artifact.insert("requests".into(), Value::from(counts.requests)); + artifact.insert( + "settledRequests".into(), + Value::from(derived.settled_requests), + ); + artifact.insert( + "inFlightRequests".into(), + Value::from(counts.in_flight_requests), + ); + artifact.insert( + "admittedRequests".into(), + Value::from(counts.admitted_requests), + ); + artifact.insert("usageRequests".into(), Value::from(counts.usage_requests)); + artifact.insert( + "missingUsageRequests".into(), + Value::from(derived.missing_usage_requests), + ); + artifact.insert("usageComplete".into(), Value::Bool(derived.usage_complete)); + artifact.insert( + "tokenBasis".into(), + serde_json::to_value(derived.token_basis).ok()?, + ); + artifact.insert( + "models".into(), + Value::Array(models.into_iter().map(Value::String).collect()), + ); + artifact.insert( + "toolNames".into(), + Value::Array(tool_names.into_iter().map(Value::String).collect()), + ); + + Some(RecoveredExternalMetering { + usage, + cost_usd: derived.cost_usd, + admitted_requests: counts.admitted_requests, + metering_state: derived.metering_state, + counts, + artifact, + }) +} diff --git a/products/eval/src/measurement/gateway.rs b/products/eval/src/measurement/gateway.rs new file mode 100644 index 00000000000..f7477f94280 --- /dev/null +++ b/products/eval/src/measurement/gateway.rs @@ -0,0 +1,970 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Local loopback HTTP metering and admission gateway for Maka Host and external subjects. +//! +//! Binds `127.0.0.1:0`, authenticates per-attempt ``, forwards request +//! payloads untouched using the real provider credential, parses streaming SSE and JSON +//! usage/admission facts, writes atomic HMAC-signed checkpoints, and enforces a synchronous +//! `stop_accepting()` cutoff while draining in-flight requests. + +use crate::contract::JsonObject; +use crate::measurement::{ + NormalizedUsage, ProviderMeteringCounts, constant_time_eq, sign_metering_checkpoint, +}; +use crate::store::write_atomic_bytes; +use http_body_util::{BodyExt, Full, StreamBody}; +use hyper::body::{Bytes, Frame, Incoming}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use serde_json::{Map, Value, json}; +use std::collections::BTreeSet; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use tokio::net::TcpListener; +use tokio::sync::{Mutex, Notify, mpsc}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub const PROVIDER_USAGE_CHECKPOINT_SCHEMA: &str = "maka.external_provider_usage.v2"; + +type BoxBody = http_body_util::combinators::BoxBody; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ObservedRequestMetadata { + pub model: Option, + pub tool_names: Vec, +} + +/// Inspects an OpenAI or Anthropic JSON request payload to observe the requested model and +/// exposed tool names without mutating or re-serializing the request body. +pub fn inspect_request_body(body: &[u8]) -> ObservedRequestMetadata { + if body.is_empty() { + return ObservedRequestMetadata::default(); + } + let Ok(Value::Object(obj)) = serde_json::from_slice::(body) else { + return ObservedRequestMetadata::default(); + }; + let model = obj + .get("model") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_owned); + let tool_names = obj + .get("tools") + .and_then(Value::as_array) + .map(|tools| tools.iter().flat_map(extract_tool_name).collect()) + .unwrap_or_default(); + ObservedRequestMetadata { model, tool_names } +} + +fn extract_tool_name(tool: &Value) -> Vec { + let Some(obj) = tool.as_object() else { + return Vec::new(); + }; + if let Some(name) = obj.get("name").and_then(Value::as_str) { + return vec![name.to_owned()]; + } + if let Some(name) = obj + .get("function") + .and_then(Value::as_object) + .and_then(|f| f.get("name")) + .and_then(Value::as_str) + { + return vec![name.to_owned()]; + } + if let Some(t) = obj.get("type").and_then(Value::as_str) { + return vec![t.to_owned()]; + } + Vec::new() +} + +/// Determines whether a parsed provider JSON/SSE event proves the upstream provider +/// admitted an inference call. +pub fn is_inference_admission_event(event: &Map) -> bool { + let object = event.get("object").and_then(Value::as_str); + if matches!( + object, + Some("chat.completion") | Some("chat.completion.chunk") + ) { + return true; + } + if let Some(event_type) = event.get("type").and_then(Value::as_str) + && matches!( + event_type, + "response.created" + | "response.in_progress" + | "response.output_item.added" + | "response.content_part.added" + | "response.output_text.delta" + | "response.reasoning_text.delta" + | "response.function_call_arguments.delta" + | "response.completed" + ) + { + return true; + } + object == Some("response") + && event + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| !id.is_empty()) + && (event.get("output").is_some_and(Value::is_array) + || event.get("usage").is_some_and(Value::is_object)) +} + +/// Extracts a 6-counter `NormalizedUsage` snapshot from an OpenAI-compatible event/response. +pub fn usage_from_event(event: &Map) -> Option { + let response = if event.get("type").and_then(Value::as_str) == Some("response.completed") { + event + .get("response") + .and_then(Value::as_object) + .unwrap_or(event) + } else { + event + }; + let raw = response + .get("usage") + .and_then(Value::as_object) + .or_else(|| event.get("usage").and_then(Value::as_object))?; + + let input = json_u64(raw.get("input_tokens").or_else(|| raw.get("prompt_tokens"))); + let output = json_u64( + raw.get("output_tokens") + .or_else(|| raw.get("completion_tokens")), + ); + let input_details = raw + .get("input_tokens_details") + .and_then(Value::as_object) + .or_else(|| raw.get("prompt_tokens_details").and_then(Value::as_object)); + let output_details = raw + .get("output_tokens_details") + .and_then(Value::as_object) + .or_else(|| { + raw.get("completion_tokens_details") + .and_then(Value::as_object) + }); + + let cache_read = json_u64( + raw.get("cache_read_input_tokens") + .or_else(|| raw.get("prompt_cache_hit_tokens")) + .or_else(|| input_details.and_then(|d| d.get("cached_tokens"))), + ); + let cache_write = json_u64( + raw.get("cache_creation_input_tokens") + .or_else(|| raw.get("cache_write_input_tokens")) + .or_else(|| input_details.and_then(|d| d.get("cache_write_tokens"))) + .or_else(|| input_details.and_then(|d| d.get("cache_creation_tokens"))), + ); + let reasoning = json_u64(output_details.and_then(|d| d.get("reasoning_tokens"))); + let reported_total = json_u64(raw.get("total_tokens")); + let total_tokens = if reported_total > 0 { + reported_total + } else { + input.saturating_add(output) + }; + Some(NormalizedUsage { + input_tokens: input, + output_tokens: output, + cache_read_tokens: cache_read, + cache_write_tokens: cache_write, + reasoning_tokens: reasoning, + total_tokens, + }) +} + +fn json_u64(value: Option<&Value>) -> u64 { + value.and_then(Value::as_u64).unwrap_or(0) +} + +fn merge_stream_usage( + left: Option, + right: Option, +) -> Option { + match (left, right) { + (None, r) => r, + (l, None) => l, + (Some(l), Some(r)) => Some(NormalizedUsage { + input_tokens: l.input_tokens.max(r.input_tokens), + output_tokens: l.output_tokens.max(r.output_tokens), + cache_read_tokens: l.cache_read_tokens.max(r.cache_read_tokens), + cache_write_tokens: l.cache_write_tokens.max(r.cache_write_tokens), + reasoning_tokens: l.reasoning_tokens.max(r.reasoning_tokens), + total_tokens: l.total_tokens.max(r.total_tokens), + }), + } +} + +/// Incremental SSE / JSON response parser for a single model request. +#[derive(Debug, Default)] +pub struct UsageStreamParser { + buffered: String, + full_text: String, + current: Option, + admitted: bool, +} + +impl UsageStreamParser { + pub fn new() -> Self { + Self::default() + } + + pub fn push(&mut self, text: &str) { + self.buffered.push_str(text); + if self.full_text.len() < 512 * 1024 { + self.full_text.push_str(text); + } + while let Some(newline) = self.buffered.find('\n') { + let line = self.buffered[..newline].to_owned(); + self.buffered.drain(..=newline); + self.consume_line(&line); + } + } + + pub fn admitted(&self) -> bool { + self.admitted + } + + pub fn finish(&mut self) -> (Option, bool) { + if !self.buffered.trim().is_empty() { + let tail = std::mem::take(&mut self.buffered); + self.consume_line(&tail); + } + if !self.admitted + && !self.full_text.trim().is_empty() + && let Ok(Value::Object(obj)) = serde_json::from_str::(self.full_text.trim()) + { + self.admitted |= is_inference_admission_event(&obj); + self.current = merge_stream_usage(self.current.take(), usage_from_event(&obj)); + } + (self.current.clone(), self.admitted) + } + + fn consume_line(&mut self, line: &str) { + let trimmed = line.trim(); + let raw = trimmed + .strip_prefix("data:") + .map(str::trim_start) + .unwrap_or(trimmed); + if raw.is_empty() || raw == "[DONE]" { + return; + } + let Ok(Value::Object(obj)) = serde_json::from_str::(raw) else { + return; + }; + self.admitted |= is_inference_admission_event(&obj); + self.current = merge_stream_usage(self.current.take(), usage_from_event(&obj)); + } +} + +pub fn join_upstream_url(base_url: &str, request_uri: &str) -> Result { + let mut base = Url::parse(base_url).map_err(|e| format!("invalid upstream baseUrl: {e}"))?; + let req_url = Url::parse(&format!( + "http://maka-eval.local/{}", + request_uri.trim_start_matches('/') + )) + .map_err(|e| format!("invalid request URI: {e}"))?; + let base_path = base.path().trim_end_matches('/'); + let req_path = req_url.path().trim_start_matches('/'); + // Avoid duplicating path prefixes (such as `/v1` or `/api/v1`) when `base_url` + // already carries that prefix and the client sends the full prefixed path. + let combined_path = if !base_path.is_empty() + && (req_url.path() == base_path || req_url.path().starts_with(&format!("{base_path}/"))) + { + req_url.path().to_owned() + } else if req_path.is_empty() { + base_path.to_owned() + } else { + format!("{base_path}/{req_path}") + }; + base.set_path(&combined_path); + base.set_query(req_url.query()); + Ok(base.to_string()) +} + +fn rewrite_url_origin(target_url: &str, origin_override: &str) -> Result { + let target = Url::parse(target_url).map_err(|e| format!("invalid target URL: {e}"))?; + let mut dest = + Url::parse(origin_override).map_err(|e| format!("invalid origin override: {e}"))?; + dest.set_path(target.path()); + dest.set_query(target.query()); + dest.set_fragment(None); + Ok(dest.to_string()) +} + +pub const UPSTREAM_TARGET_HEADER: &str = "x-maka-eval-upstream-target"; + +#[derive(Debug, Clone)] +pub struct GatewayConfig { + pub upstream_base_url: String, + pub transport_origin_override: Option, + pub upstream_api_key: String, + pub profile: String, + pub gateway_credential: String, + pub checkpoint_path: Option, + pub checkpoint_secret: String, +} + +#[derive(Debug, Default)] +struct GatewayCounters { + total_usage: NormalizedUsage, + requests: u64, + in_flight_requests: u64, + admitted_requests: u64, + usage_requests: u64, + settled: bool, + models: BTreeSet, + tool_names: BTreeSet, +} + +impl GatewayCounters { + fn snapshot(&self) -> ProviderMeteringCounts { + ProviderMeteringCounts { + usage: if self.usage_requests > 0 { + Some(self.total_usage.clone()) + } else { + None + }, + settled: self.settled, + requests: self.requests, + in_flight_requests: self.in_flight_requests, + admitted_requests: self.admitted_requests, + usage_requests: self.usage_requests, + models: self.models.iter().cloned().collect(), + tool_names: self.tool_names.iter().cloned().collect(), + } + } +} + +struct SharedGatewayState { + config: GatewayConfig, + client: reqwest::Client, + accepting: AtomicBool, + counters: Mutex, + checkpoint_lock: Mutex<()>, + in_flight_zero: Notify, +} + +impl SharedGatewayState { + async fn persist_checkpoint(&self) { + let Some(path) = &self.config.checkpoint_path else { + return; + }; + let _guard = self.checkpoint_lock.lock().await; + let snap = self.counters.lock().await.snapshot(); + let Ok(Value::Object(mut body)) = serde_json::to_value(&snap) else { + return; + }; + body.insert( + "schemaVersion".into(), + Value::String(PROVIDER_USAGE_CHECKPOINT_SCHEMA.into()), + ); + body.insert("profile".into(), Value::String(self.config.profile.clone())); + let signed: JsonObject = sign_metering_checkpoint(&body, &self.config.checkpoint_secret); + if let Ok(mut bytes) = serde_json::to_vec(&Value::Object(signed)) { + bytes.push(b'\n'); + let _ = write_atomic_bytes(path, &bytes); + } + } + + fn is_authorized(&self, req: &Request) -> bool { + let expected = self.config.gateway_credential.as_bytes(); + if expected.is_empty() { + return true; + } + if let Some(auth) = req + .headers() + .get(hyper::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + && let Some(token) = auth.strip_prefix("Bearer ").map(str::trim) + && constant_time_eq(token.as_bytes(), expected) + { + return true; + } + if let Some(api_key) = req.headers().get("x-api-key").and_then(|v| v.to_str().ok()) + && constant_time_eq(api_key.trim().as_bytes(), expected) + { + return true; + } + false + } +} + +/// Running local HTTP Eval Metering Gateway. +pub struct EvalMeteringGateway { + addr: SocketAddr, + gateway_credential: String, + state: Arc, + shutdown: CancellationToken, + worker: Option>, +} + +impl EvalMeteringGateway { + pub async fn start(config: GatewayConfig) -> Result { + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .build() + .map_err(|e| format!("failed to build gateway HTTP client: {e}"))?; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .map_err(|e| format!("failed to bind metering gateway on 127.0.0.1:0: {e}"))?; + let addr = listener + .local_addr() + .map_err(|e| format!("failed to read metering gateway local addr: {e}"))?; + let gateway_credential = config.gateway_credential.clone(); + + let state = Arc::new(SharedGatewayState { + config, + client, + accepting: AtomicBool::new(true), + counters: Mutex::new(GatewayCounters::default()), + checkpoint_lock: Mutex::new(()), + in_flight_zero: Notify::new(), + }); + state.persist_checkpoint().await; + + let shutdown = CancellationToken::new(); + let worker_shutdown = shutdown.clone(); + let worker_state = state.clone(); + let worker = tokio::spawn(async move { + loop { + tokio::select! { + _ = worker_shutdown.cancelled() => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { + break; + }; + let conn_state = worker_state.clone(); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + handle_gateway_request(conn_state.clone(), req) + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + } + } + }); + + Ok(Self { + addr, + gateway_credential, + state, + shutdown, + worker: Some(worker), + }) + } + + pub fn port(&self) -> u16 { + self.addr.port() + } + + pub fn base_url(&self) -> String { + format!("http://127.0.0.1:{}", self.addr.port()) + } + + pub fn openai_v1_base_url(&self) -> String { + format!("http://127.0.0.1:{}/v1", self.addr.port()) + } + + /// Returns a loopback URL whose path matches `upstream_base_url` so provider route + /// normalization in Maka sees the exact same path structure as the upstream endpoint. + pub fn base_url_matching_upstream(&self, upstream_base_url: &str) -> String { + let suffix = Url::parse(upstream_base_url) + .ok() + .map(|u| u.path().trim_end_matches('/').to_owned()) + .unwrap_or_else(|| "/v1".to_owned()); + format!("http://127.0.0.1:{}{}", self.addr.port(), suffix) + } + + pub fn gateway_credential(&self) -> &str { + &self.gateway_credential + } + + /// Synchronously closes the admission window so any subsequent request receives HTTP 503 + /// without being forwarded or counted. + pub fn stop_accepting(&self) { + self.state.accepting.store(false, Ordering::SeqCst); + } + + /// Closes admission, waits for all already-accepted in-flight requests to finish draining, + /// marks the snapshot as `settled = true`, writes the final signed checkpoint, and returns + /// the settled `ProviderMeteringCounts`. + pub async fn report(&self) -> ProviderMeteringCounts { + self.stop_accepting(); + loop { + let notified = self.state.in_flight_zero.notified(); + { + let mut counters = self.state.counters.lock().await; + if counters.in_flight_requests == 0 { + counters.settled = true; + break; + } + } + notified.await; + } + self.state.persist_checkpoint().await; + self.state.counters.lock().await.snapshot() + } + + pub async fn close(mut self) -> ProviderMeteringCounts { + let counts = self.report().await; + self.shutdown.cancel(); + if let Some(worker) = self.worker.take() { + let _ = worker.await; + } + counts + } +} + +impl Drop for EvalMeteringGateway { + fn drop(&mut self) { + self.state.accepting.store(false, Ordering::SeqCst); + self.shutdown.cancel(); + if let Some(worker) = self.worker.take() { + worker.abort(); + } + } +} + +fn json_error_response(status: StatusCode, message: &str) -> Response { + let payload = serde_json::to_vec(&json!({ "error": message })).unwrap_or_default(); + Response::builder() + .status(status) + .header(hyper::header::CONTENT_TYPE, "application/json") + .body(Full::new(Bytes::from(payload)).boxed()) + .unwrap() +} + +async fn finish_in_flight(state: &SharedGatewayState) { + let became_zero = { + let mut counters = state.counters.lock().await; + counters.in_flight_requests = counters.in_flight_requests.saturating_sub(1); + counters.in_flight_requests == 0 + }; + state.persist_checkpoint().await; + if became_zero { + state.in_flight_zero.notify_waiters(); + } +} + +async fn handle_gateway_request( + state: Arc, + req: Request, +) -> Result, Infallible> { + if !state.accepting.load(Ordering::SeqCst) { + return Ok(json_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "provider proxy has settled", + )); + } + if !state.is_authorized(&req) { + return Ok(json_error_response( + StatusCode::UNAUTHORIZED, + "invalid eval gateway credential", + )); + } + + { + let mut counters = state.counters.lock().await; + if !state.accepting.load(Ordering::SeqCst) { + return Ok(json_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "provider proxy has settled", + )); + } + counters.requests += 1; + counters.in_flight_requests += 1; + } + state.persist_checkpoint().await; + + let method = req.method().clone(); + let uri_str = req + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_owned()) + .unwrap_or_else(|| "/".to_owned()); + let headers = req.headers().clone(); + + let body_bytes = match req.into_body().collect().await { + Ok(collected) => collected.to_bytes(), + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "failed to read request body", + )); + } + }; + + let observed = inspect_request_body(&body_bytes); + + { + let mut counters = state.counters.lock().await; + if let Some(m) = &observed.model { + counters.models.insert(m.clone()); + } + for name in &observed.tool_names { + counters.tool_names.insert(name.clone()); + } + } + + let target_url = match join_upstream_url(&state.config.upstream_base_url, &uri_str) { + Ok(u) => u, + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "invalid upstream URL", + )); + } + }; + let connect_url = match &state.config.transport_origin_override { + Some(origin) => match rewrite_url_origin(&target_url, origin) { + Ok(u) => u, + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "invalid gateway transport origin override", + )); + } + }, + None => target_url.clone(), + }; + + let reqwest_method = + reqwest::Method::from_bytes(method.as_str().as_bytes()).unwrap_or(reqwest::Method::POST); + let mut upstream_req = state.client.request(reqwest_method, &connect_url); + if state.config.transport_origin_override.is_some() { + upstream_req = upstream_req.header(UPSTREAM_TARGET_HEADER, &target_url); + } + let uses_x_api_key = + headers.contains_key("x-api-key") && !headers.contains_key(hyper::header::AUTHORIZATION); + for (name, value) in &headers { + let lower = name.as_str().to_ascii_lowercase(); + if matches!( + lower.as_str(), + "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "authorization" + | "x-api-key" + | UPSTREAM_TARGET_HEADER + ) { + continue; + } + if let Ok(val_str) = value.to_str() { + upstream_req = upstream_req.header(name.as_str(), val_str); + } + } + if uses_x_api_key { + upstream_req = upstream_req.header("x-api-key", &state.config.upstream_api_key); + } else { + upstream_req = upstream_req.header( + "authorization", + format!("Bearer {}", state.config.upstream_api_key), + ); + } + if !body_bytes.is_empty() { + upstream_req = upstream_req.body(body_bytes.to_vec()); + } + + let mut upstream_resp = match upstream_req.send().await { + Ok(r) => r, + Err(_) => { + finish_in_flight(&state).await; + return Ok(json_error_response( + StatusCode::BAD_GATEWAY, + "provider proxy failed", + )); + } + }; + + let status_u16 = upstream_resp.status().as_u16(); + let is_ok = upstream_resp.status().is_success(); + let mut resp_builder = + Response::builder().status(StatusCode::from_u16(status_u16).unwrap_or(StatusCode::OK)); + for (name, value) in upstream_resp.headers() { + let lower = name.as_str().to_ascii_lowercase(); + if matches!( + lower.as_str(), + "content-length" | "content-encoding" | "transfer-encoding" | "connection" + ) { + continue; + } + if let Ok(val_str) = value.to_str() { + resp_builder = resp_builder.header(name.as_str(), val_str); + } + } + + let (tx, rx) = mpsc::channel::, Infallible>>(16); + let stream_state = state.clone(); + tokio::spawn(async move { + let mut parser = UsageStreamParser::new(); + let mut counted = false; + + while let Ok(Some(chunk)) = upstream_resp.chunk().await { + let text = String::from_utf8_lossy(&chunk); + parser.push(&text); + if !counted && is_ok && parser.admitted() { + counted = true; + { + let mut counters = stream_state.counters.lock().await; + counters.admitted_requests += 1; + } + stream_state.persist_checkpoint().await; + } + if tx.send(Ok(Frame::data(chunk))).await.is_err() { + break; + } + } + + let (usage, admitted) = parser.finish(); + if !counted && is_ok && admitted { + counted = true; + { + let mut counters = stream_state.counters.lock().await; + counters.admitted_requests += 1; + } + stream_state.persist_checkpoint().await; + } + if counted && let Some(u) = usage { + let mut counters = stream_state.counters.lock().await; + counters.usage_requests += 1; + counters.total_usage.accumulate_exact(&u); + } + finish_in_flight(&stream_state).await; + }); + + let stream = tokio_stream_from_receiver(rx); + let body = http_body_util::BodyExt::boxed(StreamBody::new(stream)); + Ok(resp_builder.body(body).unwrap()) +} + +fn tokio_stream_from_receiver( + mut rx: mpsc::Receiver, +) -> impl futures_util::Stream + Send + Sync + 'static +where + T: Send + 'static, +{ + futures_util::stream::poll_fn(move |cx| rx.poll_recv(cx)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::measurement::recover_external_metering_checkpoint; + + #[test] + fn inspect_request_body_observes_model_and_all_tools_without_pruning() { + let input = serde_json::to_vec(&json!({ + "model": "deepseek-chat", + "tools": [ + {"type": "function", "function": {"name": "WebSearch"}}, + {"type": "function", "function": {"name": "ReadFile"}}, + {"name": "web_fetch", "type": "custom"}, + {"type": "web_search_20250305", "name": "lookup"} + ] + })) + .unwrap(); + let observed = inspect_request_body(&input); + assert_eq!(observed.model.as_deref(), Some("deepseek-chat")); + assert_eq!( + observed.tool_names, + vec![ + "WebSearch".to_owned(), + "ReadFile".to_owned(), + "web_fetch".to_owned(), + "lookup".to_owned(), + ] + ); + } + + #[tokio::test] + async fn gateway_authenticates_preserves_exact_payload_streams_usage_and_enforces_cutoff() { + let mut parser = UsageStreamParser::new(); + parser.push(concat!( + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":25,\"prompt_cache_hit_tokens\":60,\"completion_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":125}}\n\n", + "data: [DONE]\n\n" + )); + let (parsed_usage, parsed_admitted) = parser.finish(); + assert!(parsed_admitted); + assert_eq!( + parsed_usage, + Some(NormalizedUsage { + input_tokens: 100, + output_tokens: 25, + cache_read_tokens: 60, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 125, + }) + ); + + let expected_raw_body = br#"{"model":"deepseek-chat","tools":[{"type":"function","function":{"name":"WebSearch"}},{"type":"function","function":{"name":"Bash"}}]}"#.to_vec(); + let expected_for_server = expected_raw_body.clone(); + + // Start a mock upstream OpenAI-compatible SSE server. + let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_addr = upstream_listener.local_addr().unwrap(); + if std::net::TcpStream::connect(upstream_addr) + .is_err_and(|e| e.kind() == std::io::ErrorKind::PermissionDenied) + { + return; + } + let upstream_task = tokio::spawn(async move { + loop { + let Ok((stream, _)) = upstream_listener.accept().await else { + break; + }; + let expected_bytes = expected_for_server.clone(); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let expected_bytes = expected_bytes.clone(); + async move { + assert_eq!(req.uri().path(), "/v1/chat/completions"); + assert_eq!( + req.headers() + .get("authorization") + .and_then(|v| v.to_str().ok()), + Some("Bearer real-upstream-secret") + ); + let body = req.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body.as_ref(), expected_bytes.as_slice()); + + let sse = concat!( + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"}}]}\n\n", + "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":25,\"prompt_cache_hit_tokens\":60,\"completion_tokens_details\":{\"reasoning_tokens\":10},\"total_tokens\":125}}\n\n", + "data: [DONE]\n\n" + ); + Ok::<_, Infallible>( + Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/event-stream") + .body(Full::new(Bytes::from(sse))) + .unwrap(), + ) + } + }); + let _ = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + + let tmp = tempfile::tempdir().unwrap(); + let agent_dir = tmp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let checkpoint_path = agent_dir.join("maka.provider-usage.json"); + let checkpoint_secret = "0123456789abcdef0123456789abcdef"; + + let gateway = EvalMeteringGateway::start(GatewayConfig { + upstream_base_url: format!("http://{upstream_addr}/v1"), + transport_origin_override: None, + upstream_api_key: "real-upstream-secret".into(), + profile: "maka".into(), + gateway_credential: "attempt-gateway-token".into(), + checkpoint_path: Some(checkpoint_path), + checkpoint_secret: checkpoint_secret.into(), + }) + .await + .unwrap(); + + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + + // 1. Unauthorized request is rejected with 401 and not counted. + let unauth = client + .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) + .header("authorization", "Bearer wrong-token") + .header("content-type", "application/json") + .body(serde_json::to_vec(&json!({"model": "deepseek-chat"})).unwrap()) + .send() + .await + .unwrap(); + assert_eq!(unauth.status(), reqwest::StatusCode::UNAUTHORIZED); + + // 2. Authorized request preserves exact request bytes & tools, forwards upstream, and records SSE usage. + let resp = client + .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) + .header("authorization", "Bearer attempt-gateway-token") + .header("content-type", "application/json") + .body(expected_raw_body) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), reqwest::StatusCode::OK); + let text = resp.text().await.unwrap(); + assert!(text.contains("[DONE]")); + + // 3. Stop accepting and verify subsequent requests get 503 without inflating counts. + gateway.stop_accepting(); + let refused = client + .post(format!("{}/chat/completions", gateway.openai_v1_base_url())) + .header("authorization", "Bearer attempt-gateway-token") + .header("content-type", "application/json") + .body(serde_json::to_vec(&json!({"model": "deepseek-chat"})).unwrap()) + .send() + .await + .unwrap(); + assert_eq!(refused.status(), reqwest::StatusCode::SERVICE_UNAVAILABLE); + + let counts = gateway.close().await; + upstream_task.abort(); + + assert!(counts.settled); + assert_eq!(counts.requests, 1); + assert_eq!(counts.admitted_requests, 1); + assert_eq!(counts.usage_requests, 1); + assert_eq!(counts.models, vec!["deepseek-chat".to_owned()]); + assert_eq!( + counts.tool_names, + vec!["Bash".to_owned(), "WebSearch".to_owned()] + ); + assert_eq!( + counts.usage, + Some(NormalizedUsage { + input_tokens: 100, + output_tokens: 25, + cache_read_tokens: 60, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 125, + }) + ); + + // Verify the HMAC-signed checkpoint on disk is recoverable. + let recovered = + recover_external_metering_checkpoint(tmp.path(), "maka", checkpoint_secret).unwrap(); + assert_eq!(recovered.counts, counts); + } +} diff --git a/products/eval/src/report.rs b/products/eval/src/report.rs new file mode 100644 index 00000000000..9ebbb8abb5d --- /dev/null +++ b/products/eval/src/report.rs @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Deterministic journal projection (`v1` and `v2`), paired cohort comparison +//! summaries, and 3-verdict differential oracle evaluation. + +use crate::contract::{ + DifferenceRegistry, DifferenceScenarioId, EvalResultStatus, ExperimentSpec, RunFingerprint, + V1CellAttempt, V2AttemptRecord, decode_v1_cell_attempt, expand_experiment, format_cell_id, + is_replaceable_attempt, select_cell_result, +}; +use crate::measurement::{NormalizedUsage, costs_match_within_tolerance}; +use crate::store::{FileAttemptStore, ReconstructedStoreState, StoreError}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; + +/// Reconstructs all `v1` and `v2` attempt records deterministically from the append-only journal. +pub fn rebuild_from_journal( + store: &FileAttemptStore, +) -> Result { + store.reconstruct_state() +} + +/// Exports the canonical `maka.eval.v1` attempt for each cell in `spec` using +/// earliest-valid-attempt selection (`sequence >= 1`). +pub fn export_v1_canonical( + spec: &ExperimentSpec, + store: &FileAttemptStore, +) -> Result, StoreError> { + let cells = expand_experiment(spec); + let mut out = BTreeMap::new(); + for cell in cells { + let attempts = store.list_v1(&cell.id)?; + if let Some(selected) = select_cell_result(&attempts) { + out.insert(cell.id, selected.clone()); + } + } + Ok(out) +} + +/// Exports the canonical `maka.eval.v2` record for each cell in `spec` from the journal. +pub fn export_v2_canonical( + spec: &ExperimentSpec, + store: &FileAttemptStore, +) -> Result, StoreError> { + let state = store.reconstruct_state()?; + let cells = expand_experiment(spec); + let mut out = BTreeMap::new(); + for cell in cells { + if let Some(records) = state.settled_v2.get(&cell.id) { + let mut sorted: Vec<&V2AttemptRecord> = + records.iter().filter(|r| r.sequence >= 1).collect(); + sorted.sort_by_key(|r| r.sequence); + for rec in sorted { + let projected = rec.project_v1(); + if !is_replaceable_attempt(&projected) { + out.insert(cell.id, rec.clone()); + break; + } + } + } + } + Ok(out) +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectArmSummary { + pub subject_id: String, + pub total_cells: usize, + pub settled_cells: usize, + pub completed_cells: usize, + pub subject_failed_cells: usize, + pub total_score: f64, + pub mean_score: Option, + pub total_usage: NormalizedUsage, + pub total_cost_usd: f64, + pub mean_duration_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CohortSummary { + pub experiment_id: String, + pub total_cells: usize, + pub settled_cells: usize, + pub incomplete_cells: usize, + pub paired_task_groups: usize, + pub arms: BTreeMap, +} + +pub fn build_cohort_summary( + spec: &ExperimentSpec, + canonical_v1: &BTreeMap, +) -> CohortSummary { + let cells = expand_experiment(spec); + let total_cells = cells.len(); + let total_cells_per_arm = spec.tasks.len() * (spec.repetitions as usize); + + let mut arms = BTreeMap::new(); + for subject in &spec.subjects { + let mut settled_cells = 0usize; + let mut completed_cells = 0usize; + let mut subject_failed_cells = 0usize; + let mut total_score = 0.0f64; + let mut scored_count = 0usize; + let mut total_usage = NormalizedUsage::default(); + let mut total_cost_usd = 0.0f64; + let mut total_duration_ms = 0u64; + + for task in &spec.tasks { + for rep in 1..=spec.repetitions { + let cell_id = format_cell_id(&task.id, rep, &subject.id); + if let Some(attempt) = canonical_v1.get(&cell_id) { + settled_cells += 1; + total_duration_ms = + total_duration_ms.saturating_add(attempt.result.duration_ms); + match attempt.result.status { + EvalResultStatus::Completed => completed_cells += 1, + EvalResultStatus::SubjectFailed => subject_failed_cells += 1, + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate => {} + } + if let Some(score) = attempt.result.score { + total_score += score; + scored_count += 1; + } + if let Some(u) = &attempt.result.usage { + total_usage.accumulate_exact(u); + } + if let Some(cost) = attempt.result.cost_usd { + total_cost_usd += cost; + } + } + } + } + + arms.insert( + subject.id.clone(), + SubjectArmSummary { + subject_id: subject.id.clone(), + total_cells: total_cells_per_arm, + settled_cells, + completed_cells, + subject_failed_cells, + total_score, + mean_score: (scored_count > 0).then(|| total_score / (scored_count as f64)), + total_usage, + total_cost_usd, + mean_duration_ms: (settled_cells > 0) + .then(|| (total_duration_ms as f64) / (settled_cells as f64)), + }, + ); + } + + let mut paired_task_groups = 0usize; + for task in &spec.tasks { + for rep in 1..=spec.repetitions { + let all_arms_settled = spec.subjects.iter().all(|s| { + let id = format_cell_id(&task.id, rep, &s.id); + canonical_v1.contains_key(&id) + }); + if all_arms_settled { + paired_task_groups += 1; + } + } + } + + let settled_cells = canonical_v1.len(); + CohortSummary { + experiment_id: spec.id.clone(), + total_cells, + settled_cells, + incomplete_cells: total_cells.saturating_sub(settled_cells), + paired_task_groups, + arms, + } +} + +/// 3-verdict differential report comparing a candidate `maka.eval.v1` projection +/// against a reference baseline oracle and the Semantic Difference Registry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DifferentialReport { + /// 1. Whether the candidate JSON parses strictly as a valid `maka.eval.v1` attempt. + pub v1_parses: bool, + /// 2. Whether the candidate actually matches the oracle on compared result fields. + pub v1_matches_oracle: bool, + pub adjudicated_difference: bool, + /// 3. Whether the experimental conditions (semantic fingerprint and any adjudicated difference) remain comparable. + pub conditions_comparable: bool, + pub adjudicated_scenario: Option, + pub diagnostics: Vec, +} + +pub fn evaluate_differential_oracle( + candidate_v1_json: &Value, + oracle: &V1CellAttempt, + candidate_fingerprint: &RunFingerprint, + oracle_fingerprint: &RunFingerprint, + adjudicated_scenario: Option, + registry: &DifferenceRegistry, +) -> DifferentialReport { + let mut diagnostics = Vec::new(); + let candidate = match decode_v1_cell_attempt(candidate_v1_json) { + Ok(c) => c, + Err(err) => { + diagnostics.push(format!("v1 schema validation failed: {err}")); + return DifferentialReport { + v1_parses: false, + v1_matches_oracle: false, + adjudicated_difference: false, + conditions_comparable: false, + adjudicated_scenario, + diagnostics, + }; + } + }; + + let fingerprint_comparable = + candidate_fingerprint.is_semantically_comparable(oracle_fingerprint); + if !fingerprint_comparable { + diagnostics.push("semantic run fingerprint differs from oracle".into()); + } + + let scenario_entry = adjudicated_scenario.and_then(|id| registry.get(id)); + if adjudicated_scenario.is_some() && scenario_entry.is_none() { + diagnostics.push("unregistered semantic difference scenario".into()); + } + + let direct_match = attempts_equivalent(&candidate, oracle, &mut diagnostics); + let applicable = adjudicated_scenario.is_none_or(|id| scenario_applies(id, &candidate)); + if !applicable { + diagnostics + .push("candidate does not establish the selected scenario's prerequisites".into()); + } + let adjudicated_difference = !direct_match + && applicable + && scenario_entry.is_some() + && allowed_scenario_difference(adjudicated_scenario.unwrap(), &candidate, oracle); + let conditions_comparable = fingerprint_comparable + && applicable + && (direct_match || adjudicated_difference) + && scenario_entry.is_none_or(|entry| entry.conditions_comparable); + + DifferentialReport { + v1_parses: true, + v1_matches_oracle: direct_match, + adjudicated_difference, + conditions_comparable, + adjudicated_scenario, + diagnostics, + } +} + +fn has_artifact(attempt: &V1CellAttempt, kind: &str) -> bool { + attempt + .result + .artifacts + .iter() + .any(|a| a.get("kind").and_then(Value::as_str) == Some(kind)) +} + +fn scenario_applies(id: DifferenceScenarioId, candidate: &V1CellAttempt) -> bool { + let result = &candidate.result; + match id { + DifferenceScenarioId::TimeoutWithRewardIncompleteUsage => { + result.status == EvalResultStatus::SubjectFailed + && result.score.is_some() + && result.usage.is_some() + && result.cost_usd.is_none() + && result + .failure_reason + .as_deref() + .is_some_and(|r| r.contains("timeout")) + } + DifferenceScenarioId::AuditTruncation => result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("egress-audit") + && a.get("truncated").and_then(Value::as_bool) == Some(true) + }), + DifferenceScenarioId::MissingCriticalArtifacts => { + result.status == EvalResultStatus::InfraFailed + && result.score.is_none() + && result + .failure_reason + .as_deref() + .is_some_and(|r| r.starts_with("critical artifacts missing: ")) + } + DifferenceScenarioId::LateCancellation => { + has_artifact(candidate, "cancelled-after-verification") + && matches!( + result.status, + EvalResultStatus::Completed | EvalResultStatus::SubjectFailed + ) + && result.score.is_some() + } + DifferenceScenarioId::EvidenceRecoveredLater => { + has_artifact(candidate, "post-hoc-evidence-recovery") + } + } +} + +fn allowed_scenario_difference( + id: DifferenceScenarioId, + candidate: &V1CellAttempt, + oracle: &V1CellAttempt, +) -> bool { + // The first three scenarios preserve v1 result semantics. Post-hoc numeric + // changes require original checkpoint evidence, which this v1-only API lacks. + if id != DifferenceScenarioId::LateCancellation + || oracle.result.status != EvalResultStatus::Indeterminate + || oracle.result.score.is_some() + || oracle.result.failure_reason.as_deref() + != Some("executor cancelled before verification completed") + { + return false; + } + let mut expected = oracle.clone(); + expected.result.status = candidate.result.status; + expected.result.score = candidate.result.score; + expected.result.failure_reason = candidate.result.failure_reason.clone(); + let mut actual = candidate.clone(); + actual + .result + .artifacts + .retain(|a| a.get("kind").and_then(Value::as_str) != Some("cancelled-after-verification")); + attempts_equivalent(&actual, &expected, &mut Vec::new()) +} + +fn attempts_equivalent( + candidate: &V1CellAttempt, + oracle: &V1CellAttempt, + diagnostics: &mut Vec, +) -> bool { + let mut ok = true; + if candidate.cell_id != oracle.cell_id { + diagnostics.push(format!( + "cellId mismatch: {} != {}", + candidate.cell_id, oracle.cell_id + )); + ok = false; + } + if candidate.sequence != oracle.sequence { + diagnostics.push(format!( + "sequence mismatch: {} != {}", + candidate.sequence, oracle.sequence + )); + ok = false; + } + if candidate.result.status != oracle.result.status { + diagnostics.push(format!( + "status mismatch: {:?} != {:?}", + candidate.result.status, oracle.result.status + )); + ok = false; + } + if candidate.result.score != oracle.result.score { + diagnostics.push(format!( + "score mismatch: {:?} != {:?}", + candidate.result.score, oracle.result.score + )); + ok = false; + } + if candidate.result.usage != oracle.result.usage { + diagnostics.push("usage counters mismatch".into()); + ok = false; + } + if !costs_match_within_tolerance(candidate.result.cost_usd, oracle.result.cost_usd) { + diagnostics.push(format!( + "costUsd mismatch beyond 1e-9 tolerance: {:?} != {:?}", + candidate.result.cost_usd, oracle.result.cost_usd + )); + ok = false; + } + if candidate.result.artifacts != oracle.result.artifacts { + diagnostics.push("artifacts mismatch".into()); + ok = false; + } + if candidate.result.failure_reason != oracle.result.failure_reason { + diagnostics.push(format!( + "failureReason mismatch: {:?} != {:?}", + candidate.result.failure_reason, oracle.result.failure_reason + )); + ok = false; + } + ok +} diff --git a/products/eval/src/runner.rs b/products/eval/src/runner.rs new file mode 100644 index 00000000000..a7c7cda3acc --- /dev/null +++ b/products/eval/src/runner.rs @@ -0,0 +1,771 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Task-group admission scheduler, phase-scoped deadline authority, bounded +//! retry policy, canonical attempt selection, and crash recovery loop. + +use crate::contract::{ + AttemptPhase, CleanupState, ContractError, CriticalArtifactsState, EgressAuditState, + EvalResultStatus, ExecutionOutcome, ExperimentCell, ExperimentSpec, SchemaVersion, SubjectKind, + V1CellAttempt, V2AttemptRecord, VerifierOutcome, expand_experiment, select_cell_result, +}; +use crate::environment::{ + ExecutorPreparationCode, IndeterminateCause, TrialEnvironment, TrialEnvironmentOutcome, + harbor_trial_name_for_attempt, +}; +use crate::measurement::MeteringState; +use crate::store::{FileAttemptStore, StoreError, UnsettledAttempt}; +use crate::subject::SubjectAdapter; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use thiserror::Error; +use tokio::sync::{Mutex, Semaphore}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +#[derive(Debug, Error)] +pub enum RunnerError { + #[error("Contract error: {0}")] + Contract(#[from] ContractError), + #[error("Store error: {0}")] + Store(#[from] StoreError), + #[error("{0}")] + Execution(String), +} + +/// Retry policy governing automatic retries within a run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryPolicy { + /// Maximum automatic retries for `infra_failed` attempts within a single run (`0` = single try per invocation). + pub max_infra_retries: u32, + /// Stop in-run retries for indeterminate outcomes even when cleanup was confirmed. + /// Unconfirmed cleanup always blocks admission, including in later runs. + pub block_automatic_retry_on_indeterminate: bool, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_infra_retries: 0, + block_automatic_retry_on_indeterminate: true, + } + } +} + +/// Probe used during crash recovery to verify whether a tagged resource from an +/// interrupted run has been cleaned up. +pub trait ResourceCleanupProbe: Send + Sync { + fn confirm_cleaned_up(&self, resource_kind: &str, resource_id: &str) -> bool; +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct ConservativeCleanupProbe; + +impl ResourceCleanupProbe for ConservativeCleanupProbe { + fn confirm_cleaned_up(&self, _resource_kind: &str, _resource_id: &str) -> bool { + false + } +} + +/// Recovers unsettled attempts left behind by a previous crashed process before +/// admitting new work. +pub fn recover_unsettled_attempts( + store: &FileAttemptStore, + probe: &dyn ResourceCleanupProbe, + now_ms: u64, +) -> Result, RunnerError> { + let state = store.reconstruct_state()?; + let mut recovered = Vec::new(); + + for ((cell_id, sequence), unsettled) in state.unsettled { + let record = settle_interrupted_attempt(&unsettled, probe, now_ms); + store.append_v2(&record)?; + store.record_post_hoc_recovery( + &cell_id, + sequence, + now_ms, + record.usage.clone(), + record.metering_state, + matches!(record.cleanup_state, CleanupState::Confirmed), + record.failure_reason.clone(), + )?; + recovered.push(record); + } + + Ok(recovered) +} + +fn settle_interrupted_attempt( + unsettled: &UnsettledAttempt, + probe: &dyn ResourceCleanupProbe, + now_ms: u64, +) -> V2AttemptRecord { + let duration_ms = now_ms.saturating_sub(unsettled.started_at); + match (&unsettled.resource_kind, &unsettled.resource_id) { + (None, None) => { + // Crashed after LaunchIntent but before any external resource or Host operation was created. + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: unsettled.cell_id.clone(), + sequence: unsettled.sequence, + started_at: unsettled.started_at, + completed_at: now_ms, + duration_ms, + phase: AttemptPhase::Recovered, + execution_outcome: ExecutionOutcome::InfraFailed, + verifier_outcome: VerifierOutcome::Skipped, + usage: unsettled.latest_usage.clone(), + metering_state: unsettled.latest_metering_state, + pricing: None, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some("attempt interrupted before resource creation".into()), + artifacts: Vec::new(), + } + } + (Some(kind), Some(id)) => { + let confirmed = probe.confirm_cleaned_up(kind, id); + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: unsettled.cell_id.clone(), + sequence: unsettled.sequence, + started_at: unsettled.started_at, + completed_at: now_ms, + duration_ms, + phase: AttemptPhase::Recovered, + execution_outcome: if confirmed { + ExecutionOutcome::InfraFailed + } else { + ExecutionOutcome::Indeterminate + }, + verifier_outcome: VerifierOutcome::Skipped, + usage: unsettled.latest_usage.clone(), + metering_state: unsettled.latest_metering_state, + pricing: None, + cleanup_state: if confirmed { + CleanupState::Confirmed + } else { + CleanupState::Unconfirmed { + reason: "executor cleanup did not settle".into(), + } + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some(if confirmed { + "attempt interrupted and resource cleanup confirmed".into() + } else { + "executor cleanup did not settle".into() + }), + artifacts: Vec::new(), + } + } + _ => V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: unsettled.cell_id.clone(), + sequence: unsettled.sequence, + started_at: unsettled.started_at, + completed_at: now_ms, + duration_ms, + phase: AttemptPhase::Recovered, + execution_outcome: ExecutionOutcome::Indeterminate, + verifier_outcome: VerifierOutcome::Skipped, + usage: unsettled.latest_usage.clone(), + metering_state: unsettled.latest_metering_state, + pricing: None, + cleanup_state: CleanupState::Unconfirmed { + reason: "executor cleanup did not settle".into(), + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some("executor cleanup did not settle".into()), + artifacts: Vec::new(), + }, + } +} + +/// Groups expanded cells by `(task.id, repetition)` in deterministic declaration order. +pub fn group_task_cells(cells: &[ExperimentCell]) -> Vec> { + let mut index_by_key: HashMap<(String, u32), usize> = HashMap::new(); + let mut groups: Vec> = Vec::new(); + for cell in cells { + let key = (cell.task.id.clone(), cell.repetition); + if let Some(&idx) = index_by_key.get(&key) { + groups[idx].push(cell.clone()); + } else { + let idx = groups.len(); + index_by_key.insert(key, idx); + groups.push(vec![cell.clone()]); + } + } + groups +} + +pub struct RunExperimentInput<'a> { + pub spec: &'a ExperimentSpec, + pub store: &'a FileAttemptStore, + pub environment: Arc, + pub subjects: Vec>, + pub cell_ids: Option>, + pub retry_policy: RetryPolicy, + pub cancel: CancellationToken, + pub now_ms: Arc u64 + Send + Sync>, +} + +impl<'a> RunExperimentInput<'a> { + pub fn new( + spec: &'a ExperimentSpec, + store: &'a FileAttemptStore, + environment: Arc, + subjects: Vec>, + ) -> Self { + Self { + spec, + store, + environment, + subjects, + cell_ids: None, + retry_policy: RetryPolicy::default(), + cancel: CancellationToken::new(), + now_ms: Arc::new(default_now_ms), + } + } +} + +pub fn default_now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + +pub async fn run_experiment( + input: RunExperimentInput<'_>, +) -> Result, RunnerError> { + let _writer_guard = input.store.acquire_writer()?; + + let all_cells = expand_experiment(input.spec); + let selected_cells = select_cells(&all_cells, input.cell_ids.as_deref())?; + + if input.environment.kind() != input.spec.executor.kind { + return Err(RunnerError::Execution("executor kind mismatch".into())); + } + + let mut subjects_by_kind: HashMap> = HashMap::new(); + for adapter in &input.subjects { + subjects_by_kind.insert(adapter.kind(), adapter.clone()); + } + + let credential_names: Vec = { + let mut set = BTreeSet::new(); + for s in &input.spec.subjects { + for c in &s.credentials { + set.insert(c.clone()); + } + } + set.into_iter().collect() + }; + + for cell in &selected_cells { + input.environment.validate(cell)?; + let Some(subject) = subjects_by_kind.get(&cell.subject.kind) else { + return Err(RunnerError::Execution(format!( + "missing subject adapter: {:?}", + cell.subject.kind + ))); + }; + subject.validate(cell)?; + } + + recover_unsettled_attempts(input.store, &ConservativeCleanupProbe, (input.now_ms)())?; + + input + .environment + .preflight(&credential_names, input.cancel.clone()) + .await?; + + let groups = group_task_cells(&selected_cells); + let max_concurrent = input.spec.execution.max_concurrent_task_groups.max(1); + let semaphore = Arc::new(Semaphore::new(max_concurrent)); + let store_lock = Arc::new(Mutex::new(())); + let first_error: Arc>> = Arc::new(Mutex::new(None)); + + let mut group_handles = Vec::with_capacity(groups.len()); + for group in groups { + let sem = semaphore.clone(); + let env = input.environment.clone(); + let subjects_map = subjects_by_kind.clone(); + let creds = credential_names.clone(); + let cancel = input.cancel.clone(); + let now_fn = input.now_ms.clone(); + let store = input.store.clone(); + let store_mutex = store_lock.clone(); + let err_slot = first_error.clone(); + let retry_policy = input.retry_policy; + + group_handles.push(tokio::spawn(async move { + let Ok(_permit) = sem.acquire_owned().await else { + return; + }; + if cancel.is_cancelled() || err_slot.lock().await.is_some() { + return; + } + + let mut arm_handles = Vec::with_capacity(group.len()); + for cell in group { + let env = env.clone(); + let subject = subjects_map.get(&cell.subject.kind).unwrap().clone(); + let creds = creds.clone(); + let cancel = cancel.clone(); + let now_fn = now_fn.clone(); + let store = store.clone(); + let store_mutex = store_mutex.clone(); + let err_slot = err_slot.clone(); + + arm_handles.push(tokio::spawn(async move { + let mut retries_used = 0u32; + loop { + if cancel.is_cancelled() { + return; + } + let (should_run, next_seq, started_at) = { + let _g = store_mutex.lock().await; + let mut state = match store.reconstruct_state() { + Ok(state) => state, + Err(e) => { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + }; + if cancel.is_cancelled() { + return; + } + if state.settled_v2.get(&cell.id).is_some_and(|records| { + records.iter().any(|record| { + matches!(record.cleanup_state, CleanupState::Unconfirmed { .. }) + }) + }) { + return; + } + let attempts = state.settled_v1.remove(&cell.id).unwrap_or_default(); + if select_cell_result(&attempts).is_some() { + (false, 0, 0) + } else { + let seq = attempts.last().map_or(1, |a| a.sequence + 1); + let started = now_fn(); + let intent_id = Uuid::new_v4().to_string(); + if let Err(e) = + store.record_launch_intent(&cell.id, seq, started, &intent_id) + { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + let trial_id = harbor_trial_name_for_attempt(&cell.id, seq); + if let Err(e) = store.record_resource_tagged( + &cell.id, + seq, + env.kind(), + &trial_id, + None, + None, + ) { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + (true, seq, started) + } + }; + + if !should_run { + return; + } + + let mut record = execute_cell_v2( + env.as_ref(), + subject.clone(), + &cell, + next_seq, + started_at, + &creds, + cancel.clone(), + now_fn.as_ref(), + ) + .await; + record.completed_at = now_fn(); + + let append_res = { + let _g = store_mutex.lock().await; + store.append_v2(&record) + }; + let v1 = match append_res { + Ok(v1) => v1, + Err(e) => { + let mut slot = err_slot.lock().await; + if slot.is_none() { + *slot = Some(RunnerError::Store(e)); + } + return; + } + }; + + match v1.result.status { + EvalResultStatus::Completed | EvalResultStatus::SubjectFailed => { + return; + } + EvalResultStatus::Indeterminate + if retry_policy.block_automatic_retry_on_indeterminate => + { + return; + } + EvalResultStatus::InfraFailed | EvalResultStatus::Indeterminate => { + if retries_used >= retry_policy.max_infra_retries { + return; + } + retries_used += 1; + } + } + } + })); + } + + for handle in arm_handles { + let _ = handle.await; + } + })); + } + + for handle in group_handles { + let _ = handle.await; + } + + if let Some(err) = first_error.lock().await.take() { + return Err(err); + } + + let mut canonical_results = BTreeMap::new(); + for cell in &all_cells { + let attempts = input.store.list_v1(&cell.id)?; + if let Some(selected) = select_cell_result(&attempts) { + canonical_results.insert(cell.id.clone(), selected.clone()); + } + } + + Ok(canonical_results) +} + +#[allow(clippy::too_many_arguments)] +async fn execute_cell_v2( + environment: &dyn TrialEnvironment, + subject: Arc, + cell: &ExperimentCell, + sequence: u32, + started_at: u64, + subject_credential_names: &[String], + cancel: CancellationToken, + now_ms: &(dyn Fn() -> u64 + Send + Sync), +) -> V2AttemptRecord { + let cell_id = cell.id.clone(); + let cell_for_inner = cell.clone(); + let cancel_for_inner = cancel.clone(); + + let env_res = environment + .run_attempt( + cell, + sequence, + subject_credential_names, + cancel.clone(), + Box::new(move |active_attempt| { + Box::pin(async move { + let mut ctx = active_attempt.context(); + ctx.cancel = cancel_for_inner.clone(); + let subject_exec = subject.execute(&cell_for_inner, ctx).await; + let pricing = match cell_for_inner.subject.kind { + SubjectKind::Maka => cell_for_inner + .subject + .config + .get("model") + .and_then(serde_json::Value::as_str) + .map(|model| crate::contract::PricingEvidence::Model { + model: model.to_owned(), + cost_usd: subject_exec.cost_usd, + }), + SubjectKind::External => subject_exec.cost_usd.map(|cost_usd| { + crate::contract::PricingEvidence::SubjectReported { cost_usd } + }), + }; + + if cancel_for_inner.is_cancelled() + || matches!( + subject_exec.outcome, + ExecutionOutcome::InfraFailed | ExecutionOutcome::Indeterminate + ) + { + let outcome = if cancel_for_inner.is_cancelled() { + ExecutionOutcome::Cancelled + } else { + subject_exec.outcome + }; + return V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: cell_for_inner.id.clone(), + sequence, + started_at, + completed_at: 0, + duration_ms: subject_exec.duration_ms, + phase: AttemptPhase::AgentFinished, + execution_outcome: outcome, + verifier_outcome: VerifierOutcome::Skipped, + usage: subject_exec.usage, + metering_state: subject_exec.metering_state, + pricing: pricing.clone(), + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: subject_exec.failure_reason, + artifacts: subject_exec.artifacts, + }; + } + + let verify_res = active_attempt.verify().await; + let (audit_state, audit_evidence) = active_attempt.read_audit().await; + + let mut artifacts = subject_exec.artifacts; + match verify_res { + Ok(verified) => { + let verifier_outcome = verified.to_verifier_outcome(); + artifacts.extend(verified.artifacts); + artifacts.extend(audit_evidence.artifacts); + + let failure_reason = if verified.status == EvalResultStatus::InfraFailed + || verified.missing_reward + { + verified.failure_reason.or(audit_evidence.failure_reason) + } else { + subject_exec + .failure_reason + .or(verified.failure_reason) + .or(audit_evidence.failure_reason) + }; + + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: cell_for_inner.id.clone(), + sequence, + started_at, + completed_at: 0, + duration_ms: subject_exec.duration_ms, + phase: AttemptPhase::Verified, + execution_outcome: subject_exec.outcome, + verifier_outcome, + usage: subject_exec.usage, + metering_state: subject_exec.metering_state, + pricing: pricing.clone(), + cleanup_state: CleanupState::Confirmed, + audit_state, + critical_artifacts_state: CriticalArtifactsState::Present, + // Verification already completed; any late cancellation from here + // on is post-verification and must not erase the verified score! + cancelled_after_verification: true, + recovered_post_hoc: false, + failure_reason, + artifacts, + } + } + Err(_) => V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: cell_for_inner.id.clone(), + sequence, + started_at, + completed_at: 0, + duration_ms: subject_exec.duration_ms, + phase: AttemptPhase::AgentFinished, + execution_outcome: subject_exec.outcome, + verifier_outcome: VerifierOutcome::VerifierInfraFailed { + reason: "verification failed".into(), + }, + usage: subject_exec.usage, + metering_state: subject_exec.metering_state, + pricing: pricing.clone(), + cleanup_state: CleanupState::Confirmed, + audit_state, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some("verification failed".into()), + artifacts, + }, + } + }) + }), + ) + .await; + + let completed_at = now_ms(); + match env_res { + Ok(TrialEnvironmentOutcome::Settled(mut rec)) => { + rec.phase = AttemptPhase::Settled; + rec.completed_at = completed_at; + rec + } + Ok(TrialEnvironmentOutcome::NotStarted { code, artifacts }) => { + let cancelled = code == ExecutorPreparationCode::Cancelled; + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at, + completed_at, + duration_ms: 0, + phase: AttemptPhase::Prepared, + execution_outcome: if cancelled { + ExecutionOutcome::Cancelled + } else { + ExecutionOutcome::InfraFailed + }, + verifier_outcome: VerifierOutcome::Skipped, + usage: None, + metering_state: MeteringState::Unknown, + pricing: None, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some(if cancelled { + "executor preparation cancelled".into() + } else { + "executor preparation failed".into() + }), + artifacts, + } + } + Ok(TrialEnvironmentOutcome::Indeterminate { cause, partial }) => { + let reason = match cause { + IndeterminateCause::HostCancelled => { + "executor cancelled before verification completed" + } + IndeterminateCause::CleanupUnconfirmed => "executor cleanup did not settle", + }; + if let Some(mut rec) = partial { + rec.completed_at = completed_at; + if cause == IndeterminateCause::CleanupUnconfirmed { + rec.cleanup_state = CleanupState::Unconfirmed { + reason: reason.into(), + }; + } else { + rec.execution_outcome = ExecutionOutcome::Cancelled; + rec.cancelled_after_verification = false; + rec.failure_reason = Some(reason.into()); + } + rec + } else { + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at, + completed_at, + duration_ms: 0, + phase: AttemptPhase::Settled, + execution_outcome: ExecutionOutcome::Indeterminate, + verifier_outcome: VerifierOutcome::Skipped, + usage: None, + metering_state: MeteringState::Unknown, + pricing: None, + cleanup_state: if cause == IndeterminateCause::CleanupUnconfirmed { + CleanupState::Unconfirmed { + reason: reason.into(), + } + } else { + CleanupState::Confirmed + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some(reason.into()), + artifacts: Vec::new(), + } + } + } + Err(_) => V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at, + completed_at, + duration_ms: 0, + phase: AttemptPhase::Prepared, + execution_outcome: ExecutionOutcome::InfraFailed, + verifier_outcome: VerifierOutcome::Skipped, + usage: None, + metering_state: MeteringState::Unknown, + pricing: None, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some("executor preparation failed".into()), + artifacts: Vec::new(), + }, + } +} + +fn select_cells( + cells: &[ExperimentCell], + selected_ids: Option<&[String]>, +) -> Result, ContractError> { + let Some(ids) = selected_ids else { + return Ok(cells.to_vec()); + }; + let known: BTreeSet<&str> = cells.iter().map(|c| c.id.as_str()).collect(); + let mut filter = BTreeSet::new(); + for id in ids { + if !known.contains(id.as_str()) { + return Err(ContractError::Invalid(format!( + "unknown experiment cell: {id}" + ))); + } + filter.insert(id.as_str()); + } + Ok(cells + .iter() + .filter(|c| filter.contains(c.id.as_str())) + .cloned() + .collect()) +} diff --git a/products/eval/src/store.rs b/products/eval/src/store.rs new file mode 100644 index 00000000000..47153842a57 --- /dev/null +++ b/products/eval/src/store.rs @@ -0,0 +1,798 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Single-writer append-only experiment journal, atomic attempt settlement, +//! critical artifact verification, and deterministic state reconstruction. + +use crate::contract::{ + AttemptPhase, CleanupState, ContractError, CriticalArtifactsState, EgressAuditState, + ExecutionOutcome, ExperimentSpec, SchemaVersion, V1CellAttempt, V2AttemptRecord, + VerifierOutcome, canonical_json_string, decode_v1_cell_attempt, parse_experiment_spec, + sha256_hex, +}; +use crate::measurement::{MeteringState, NormalizedUsage}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use thiserror::Error; +use uuid::Uuid; + +pub const MAKA_RUNTIME_ARTIFACT_PATH: &str = "/logs/artifacts/maka-runtime-host"; +pub const MAKA_SUBJECT_STDOUT_PATH: &str = "/logs/artifacts/maka-subject.stdout.txt"; +pub const MAKA_SUBJECT_STDERR_PATH: &str = "/logs/artifacts/maka-subject.stderr.txt"; + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + #[error("Contract error: {0}")] + Contract(#[from] ContractError), + #[error("{0}: experiment already has an active writer")] + ActiveWriter(String), + #[error("{0}")] + Integrity(String), +} + +/// Exclusive single-writer lock guard held for the duration of a store write session. +#[derive(Debug)] +pub struct WriterGuard { + _file: File, +} + +impl WriterGuard { + pub fn acquire(dir: &Path) -> Result { + fs::create_dir_all(dir)?; + let lock_path = dir.join(".writer.lock"); + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options.open(&lock_path)?; + match file.try_lock() { + Ok(()) => Ok(Self { _file: file }), + Err(std::fs::TryLockError::WouldBlock) => { + Err(StoreError::ActiveWriter(lock_path.display().to_string())) + } + Err(std::fs::TryLockError::Error(err)) => Err(StoreError::Io(err)), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum JournalEntry { + LaunchIntent { + cell_id: String, + sequence: u32, + started_at: u64, + intent_id: String, + }, + ResourceTagged { + cell_id: String, + sequence: u32, + resource_kind: String, + resource_id: String, + operation_id: Option, + invocation_id: Option, + }, + Checkpoint { + cell_id: String, + sequence: u32, + recorded_at: u64, + usage: Option, + metering_state: MeteringState, + }, + Settled { + attempt: Box, + }, + PostHocRecovery { + cell_id: String, + sequence: u32, + recovered_at: u64, + usage: Option, + metering_state: MeteringState, + cleanup_confirmed: bool, + reason: Option, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct UnsettledAttempt { + pub cell_id: String, + pub sequence: u32, + pub started_at: u64, + pub intent_id: String, + pub resource_kind: Option, + pub resource_id: Option, + pub operation_id: Option, + pub invocation_id: Option, + pub latest_usage: Option, + pub latest_metering_state: MeteringState, +} + +#[derive(Debug, Clone, Default)] +pub struct ReconstructedStoreState { + pub settled_v2: BTreeMap>, + pub settled_v1: BTreeMap>, + pub unsettled: BTreeMap<(String, u32), UnsettledAttempt>, +} + +#[derive(Debug, Clone)] +pub struct FileAttemptStore { + root: PathBuf, + journal_path: PathBuf, +} + +impl FileAttemptStore { + pub fn new(root: impl Into) -> Self { + let root = root.into(); + let journal_path = root.join("journal.jsonl"); + Self { root, journal_path } + } + + pub fn path(&self) -> &Path { + &self.root + } + + pub fn journal_path(&self) -> &Path { + &self.journal_path + } + + pub fn acquire_writer(&self) -> Result { + WriterGuard::acquire(&self.root) + } + + pub fn run_exclusive(&self, operation: F) -> Result + where + F: FnOnce(&WriterGuard) -> Result, + { + let guard = self.acquire_writer()?; + operation(&guard) + } + + pub async fn run_exclusive_async(&self, operation: F) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + let _guard = self.acquire_writer()?; + operation().await + } + + /// Appends a single journal entry to `journal.jsonl` and fsyncs it before returning. + pub fn append_journal_entry(&self, entry: &JournalEntry) -> Result<(), StoreError> { + fs::create_dir_all(&self.root)?; + let mut options = OpenOptions::new(); + options.create(true).append(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&self.journal_path)?; + let line = serde_json::to_string(entry)?; + file.write_all(line.as_bytes())?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) + } + + /// Records a pre-dispatch `LaunchIntent` before any external resource or Host operation starts. + pub fn record_launch_intent( + &self, + cell_id: &str, + sequence: u32, + started_at: u64, + intent_id: &str, + ) -> Result<(), StoreError> { + let expected = self.next_sequence(cell_id)?; + if sequence != expected { + return Err(StoreError::Integrity(format!( + "attempt sequence must be {expected}" + ))); + } + self.append_journal_entry(&JournalEntry::LaunchIntent { + cell_id: cell_id.to_owned(), + sequence, + started_at, + intent_id: intent_id.to_owned(), + }) + } + + pub fn record_resource_tagged( + &self, + cell_id: &str, + sequence: u32, + resource_kind: &str, + resource_id: &str, + operation_id: Option, + invocation_id: Option, + ) -> Result<(), StoreError> { + self.append_journal_entry(&JournalEntry::ResourceTagged { + cell_id: cell_id.to_owned(), + sequence, + resource_kind: resource_kind.to_owned(), + resource_id: resource_id.to_owned(), + operation_id, + invocation_id, + }) + } + + pub fn record_checkpoint( + &self, + cell_id: &str, + sequence: u32, + recorded_at: u64, + usage: Option, + metering_state: MeteringState, + ) -> Result<(), StoreError> { + self.append_journal_entry(&JournalEntry::Checkpoint { + cell_id: cell_id.to_owned(), + sequence, + recorded_at, + usage, + metering_state, + }) + } + + /// Commits a settled record to the sole durable authority: the journal. + pub fn append_v2(&self, record: &V2AttemptRecord) -> Result { + let expected = self.next_settled_sequence(&record.cell_id)?; + if record.sequence != expected { + return Err(StoreError::Integrity(format!( + "attempt sequence must be {expected}" + ))); + } + let v1 = record.project_v1(); + let _ = decode_v1_cell_attempt(&serde_json::to_value(&v1)?)?; + + self.append_journal_entry(&JournalEntry::Settled { + attempt: Box::new(record.clone()), + })?; + + Ok(v1) + } + + /// Appends a `maka.eval.v1` attempt directly (for legacy compatibility tests and stores) + /// while synthesizing the corresponding `v2` record in the journal. + pub fn append_v1(&self, attempt: &V1CellAttempt) -> Result<(), StoreError> { + let validated = decode_v1_cell_attempt(&serde_json::to_value(attempt)?)?; + let expected = self.next_settled_sequence(&validated.cell_id)?; + if validated.sequence != expected { + return Err(StoreError::Integrity(format!( + "attempt sequence must be {expected}" + ))); + } + let v2 = v2_from_v1(&validated); + self.append_journal_entry(&JournalEntry::Settled { + attempt: Box::new(v2), + })?; + Ok(()) + } + + /// Appends a post-hoc recovery entry without mutating any existing attempt file or journal entry. + #[allow(clippy::too_many_arguments)] + pub fn record_post_hoc_recovery( + &self, + cell_id: &str, + sequence: u32, + recovered_at: u64, + usage: Option, + metering_state: MeteringState, + cleanup_confirmed: bool, + reason: Option, + ) -> Result<(), StoreError> { + self.append_journal_entry(&JournalEntry::PostHocRecovery { + cell_id: cell_id.to_owned(), + sequence, + recovered_at, + usage, + metering_state, + cleanup_confirmed, + reason, + }) + } + + /// Projects only committed journal records, including post-hoc evidence. + pub fn list_v1(&self, cell_id: &str) -> Result, StoreError> { + Ok(self + .reconstruct_state()? + .settled_v1 + .remove(cell_id) + .unwrap_or_default()) + } + + pub fn read_journal_entries(&self) -> Result, StoreError> { + let file = match File::open(&self.journal_path) { + Ok(f) => f, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(StoreError::Io(err)), + }; + let reader = BufReader::new(file); + let mut entries = Vec::new(); + for line in reader.lines() { + let line = line?; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let entry: JournalEntry = serde_json::from_str(trimmed)?; + entries.push(entry); + } + Ok(entries) + } + + /// Deterministically reconstructs all settled `v2` records, `v1` projections, and + /// unsettled in-flight attempts from the append-only journal. + pub fn reconstruct_state(&self) -> Result { + let entries = self.read_journal_entries()?; + let mut state = ReconstructedStoreState::default(); + let mut settled_map: BTreeMap<(String, u32), V2AttemptRecord> = BTreeMap::new(); + + for entry in entries { + match entry { + JournalEntry::LaunchIntent { + cell_id, + sequence, + started_at, + intent_id, + } => { + state.unsettled.insert( + (cell_id.clone(), sequence), + UnsettledAttempt { + cell_id, + sequence, + started_at, + intent_id, + resource_kind: None, + resource_id: None, + operation_id: None, + invocation_id: None, + latest_usage: None, + latest_metering_state: MeteringState::Unknown, + }, + ); + } + JournalEntry::ResourceTagged { + cell_id, + sequence, + resource_kind, + resource_id, + operation_id, + invocation_id, + } => { + if let Some(unsettled) = state.unsettled.get_mut(&(cell_id, sequence)) { + unsettled.resource_kind = Some(resource_kind); + unsettled.resource_id = Some(resource_id); + unsettled.operation_id = operation_id; + unsettled.invocation_id = invocation_id; + } + } + JournalEntry::Checkpoint { + cell_id, + sequence, + usage, + metering_state, + .. + } => { + if let Some(unsettled) = state.unsettled.get_mut(&(cell_id, sequence)) { + if usage.is_some() { + unsettled.latest_usage = usage; + } + unsettled.latest_metering_state = metering_state; + } + } + JournalEntry::Settled { attempt } => { + let key = (attempt.cell_id.clone(), attempt.sequence); + state.unsettled.remove(&key); + settled_map.insert(key, *attempt); + } + JournalEntry::PostHocRecovery { + cell_id, + sequence, + recovered_at, + usage, + metering_state, + cleanup_confirmed, + reason, + } => { + let key = (cell_id.clone(), sequence); + if let Some(existing) = settled_map.get_mut(&key) { + existing.recovered_post_hoc = true; + existing.phase = AttemptPhase::Recovered; + if usage.is_some() { + if existing.usage != usage { + if let Some(crate::contract::PricingEvidence::Model { + cost_usd, + .. + }) = &mut existing.pricing + { + *cost_usd = None; + } else { + existing.pricing = None; + } + } + existing.usage = usage; + existing.metering_state = metering_state; + } + if cleanup_confirmed { + existing.cleanup_state = CleanupState::Confirmed; + } + if let Some(r) = reason { + existing.failure_reason = Some(r); + } + } else if let Some(unsettled) = state.unsettled.remove(&key) { + let recovered_usage = usage.or(unsettled.latest_usage); + let record = + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence, + started_at: unsettled.started_at, + completed_at: recovered_at, + duration_ms: recovered_at.saturating_sub(unsettled.started_at), + phase: AttemptPhase::Recovered, + execution_outcome: if cleanup_confirmed { + ExecutionOutcome::InfraFailed + } else { + ExecutionOutcome::Indeterminate + }, + verifier_outcome: VerifierOutcome::Skipped, + usage: recovered_usage, + metering_state, + pricing: None, + cleanup_state: if cleanup_confirmed { + CleanupState::Confirmed + } else { + CleanupState::Unconfirmed { + reason: reason.clone().unwrap_or_else(|| { + "executor cleanup did not settle".into() + }), + } + }, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: true, + failure_reason: Some(reason.unwrap_or_else(|| { + "recovered after interrupted attempt".into() + })), + artifacts: Vec::new(), + }; + settled_map.insert(key, record); + } + } + } + } + + for ((cell_id, _seq), record) in settled_map { + let v1 = record.project_v1(); + state + .settled_v2 + .entry(cell_id.clone()) + .or_default() + .push(record); + state.settled_v1.entry(cell_id).or_default().push(v1); + } + + Ok(state) + } + + pub fn next_settled_sequence(&self, cell_id: &str) -> Result { + let v1_list = self.list_v1(cell_id)?; + Ok(v1_list.last().map_or(1, |a| a.sequence + 1)) + } + + pub fn next_sequence(&self, cell_id: &str) -> Result { + let settled_next = self.next_settled_sequence(cell_id)?; + let reconstructed = self.reconstruct_state()?; + let max_journal_seq = reconstructed + .settled_v2 + .get(cell_id) + .and_then(|v| v.last().map(|r| r.sequence)) + .into_iter() + .chain( + reconstructed + .unsettled + .keys() + .filter(|(c, _)| c == cell_id) + .map(|(_, s)| *s), + ) + .max() + .map_or(1, |s| s + 1); + Ok(settled_next.max(max_journal_seq)) + } +} + +#[derive(Debug, Clone)] +pub struct ExperimentDirectory { + pub root: PathBuf, + pub spec_path: PathBuf, + pub attempts: FileAttemptStore, +} + +impl ExperimentDirectory { + pub fn open(root: impl Into, spec: &ExperimentSpec) -> Result { + let root = root.into(); + fs::create_dir_all(&root)?; + let spec_path = root.join("experiment.json"); + let spec_val = serde_json::to_value(spec)?; + let canonical = format!("{}\n", canonical_json_string(&spec_val)); + let temp_path = root.join(format!(".experiment-{}.tmp", Uuid::new_v4())); + + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut temp_file = options.open(&temp_path)?; + let link_res = (|| -> Result<(), std::io::Error> { + temp_file.write_all(canonical.as_bytes())?; + temp_file.sync_all()?; + drop(temp_file); + fs::hard_link(&temp_path, &spec_path) + })(); + let _ = fs::remove_file(&temp_path); + + match link_res { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + let existing_raw = fs::read_to_string(&spec_path)?; + let existing_val: Value = serde_json::from_str(&existing_raw)?; + let existing_spec = parse_experiment_spec(&existing_val)?; + let existing_canonical = + canonical_json_string(&serde_json::to_value(&existing_spec)?); + if existing_canonical != canonical_json_string(&spec_val) { + return Err(StoreError::Integrity("experiment spec differs".into())); + } + } + Err(err) => return Err(StoreError::Io(err)), + } + + Ok(Self { + spec_path, + attempts: FileAttemptStore::new(root.join("attempts")), + root, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CapturedArtifactFile { + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MakaRuntimeArtifactManifest { + pub schema_version: String, + pub reason: String, + pub captured_at: u64, + pub files: Vec, +} + +/// Verifies whether every required critical artifact path exists as a readable regular file. +/// +/// Explicitly distinguishes a `0-byte` regular file (`Present`, valid empty file) from a +/// missing, symlink, directory, or unreadable path (`Missing`). +pub fn verify_critical_artifacts( + base_dir: &Path, + required_paths: &[&str], +) -> (CriticalArtifactsState, Vec) { + let mut missing = Vec::new(); + let mut captured = Vec::new(); + + for rel in required_paths { + let full = base_dir.join(rel); + match describe_regular_file(&full, rel) { + Some(desc) => captured.push(desc), + None => missing.push((*rel).to_owned()), + } + } + + if missing.is_empty() { + (CriticalArtifactsState::Present, captured) + } else { + (CriticalArtifactsState::Missing { paths: missing }, captured) + } +} + +pub fn describe_regular_file(path: &Path, reported_name: &str) -> Option { + let meta = fs::symlink_metadata(path).ok()?; + if !meta.file_type().is_file() { + return None; + } + let bytes = fs::read(path).ok()?; + Some(CapturedArtifactFile { + path: reported_name.to_owned(), + bytes: bytes.len() as u64, + sha256: format!("sha256:{}", sha256_hex(&bytes)), + }) +} + +/// Captures Maka runtime artifacts (`runtime.sqlite`, logs, and `manifest.json`) into +/// `destination_root` using atomic directory staging. +pub fn capture_maka_runtime_artifacts( + state_root: &Path, + destination_root: &Path, + reason: &str, + captured_at: u64, +) -> Result { + let staging_root = + destination_root.with_extension(format!("tmp.{}.{}", std::process::id(), Uuid::new_v4())); + let _ = fs::remove_dir_all(&staging_root); + fs::create_dir_all(&staging_root)?; + + let res = (|| -> Result { + let sqlite_src = state_root.join("runtime.sqlite"); + let sqlite_dst = staging_root.join("runtime.sqlite"); + if sqlite_src.is_file() { + let src_conn = rusqlite::Connection::open_with_flags( + &sqlite_src, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| StoreError::Integrity(format!("sqlite open failed: {e}")))?; + let dst_str = sqlite_dst + .to_str() + .ok_or_else(|| StoreError::Integrity("invalid sqlite destination path".into()))?; + src_conn + .execute("VACUUM INTO ?1", [dst_str]) + .map_err(|e| StoreError::Integrity(format!("sqlite backup failed: {e}")))?; + } + + for candidate in [ + "runtime-host-candidate.log", + "runtime-policy.json", + "collection-error.json", + ] { + let src = state_root.join(candidate); + if src.is_file() { + fs::copy(&src, staging_root.join(candidate))?; + } + } + + let mut files = Vec::new(); + for name in [ + "runtime.sqlite", + "runtime-host-candidate.log", + "runtime-policy.json", + "collection-error.json", + ] { + if let Some(desc) = describe_regular_file(&staging_root.join(name), name) { + files.push(desc); + } + } + + let manifest = MakaRuntimeArtifactManifest { + schema_version: "maka.eval.runtime_artifacts.v1".into(), + reason: reason.to_owned(), + captured_at, + files, + }; + let manifest_bytes = format!("{}\n", serde_json::to_string_pretty(&manifest)?); + fs::write(staging_root.join("manifest.json"), manifest_bytes)?; + + if let Some(parent) = destination_root.parent() { + fs::create_dir_all(parent)?; + } + let _ = fs::remove_dir_all(destination_root); + fs::rename(&staging_root, destination_root)?; + Ok(manifest) + })(); + + if res.is_err() { + let _ = fs::remove_dir_all(&staging_root); + } + res +} + +pub fn write_atomic_bytes(final_path: &Path, bytes: &[u8]) -> Result<(), StoreError> { + if let Some(parent) = final_path.parent() { + fs::create_dir_all(parent)?; + } + let temp_path = + final_path.with_extension(format!("tmp.{}.{}", std::process::id(), Uuid::new_v4())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut temp_file = options.open(&temp_path)?; + let res = (|| -> Result<(), StoreError> { + temp_file.write_all(bytes)?; + temp_file.sync_all()?; + drop(temp_file); + fs::rename(&temp_path, final_path)?; + Ok(()) + })(); + if res.is_err() { + let _ = fs::remove_file(&temp_path); + } + res +} + +fn v2_from_v1(v1: &V1CellAttempt) -> V2AttemptRecord { + use crate::contract::EvalResultStatus; + let execution_outcome = match v1.result.status { + EvalResultStatus::Completed => ExecutionOutcome::Completed, + EvalResultStatus::SubjectFailed => ExecutionOutcome::SubjectFailed, + EvalResultStatus::InfraFailed => ExecutionOutcome::InfraFailed, + EvalResultStatus::Indeterminate => ExecutionOutcome::Indeterminate, + }; + let verifier_outcome = match (v1.result.status, v1.result.score) { + (EvalResultStatus::Completed, Some(score)) => VerifierOutcome::Verified { score }, + (EvalResultStatus::SubjectFailed, score) => VerifierOutcome::SubjectFailed { + score, + reason: v1 + .result + .failure_reason + .clone() + .unwrap_or_else(|| "subject failed".into()), + }, + _ => VerifierOutcome::Skipped, + }; + let metering_state = match (&v1.result.usage, v1.result.cost_usd) { + (Some(_), Some(_)) => MeteringState::Complete, + (Some(_), None) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + let mut clean_artifacts: Vec> = Vec::new(); + for a in &v1.result.artifacts { + clean_artifacts.push(a.clone()); + } + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: v1.cell_id.clone(), + sequence: v1.sequence, + started_at: v1.started_at, + completed_at: v1.completed_at, + duration_ms: v1.result.duration_ms, + phase: AttemptPhase::Settled, + execution_outcome, + verifier_outcome, + usage: v1.result.usage.clone(), + metering_state, + pricing: v1 + .result + .cost_usd + .map(|cost_usd| crate::contract::PricingEvidence::SubjectReported { cost_usd }), + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: v1.result.failure_reason.clone(), + artifacts: clean_artifacts, + } +} diff --git a/products/eval/src/subject.rs b/products/eval/src/subject.rs new file mode 100644 index 00000000000..4dc290972e6 --- /dev/null +++ b/products/eval/src/subject.rs @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Subject adapter trait and implementations for the Rust Maka Host and external CLI harnesses. + +pub mod external_cli; + +pub use external_cli::ExternalCliSubject; + +use crate::contract::{ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind}; +use crate::environment::{BoxFuture, SubjectExecutionContext}; +use crate::measurement::{MeteringState, NormalizedUsage}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SubjectExecutionResult { + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + pub usage: Option, + pub metering_state: MeteringState, + pub cost_usd: Option, + pub duration_ms: u64, + pub outcome: ExecutionOutcome, + pub failure_reason: Option, + pub artifacts: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub operation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub invocation_id: Option, +} + +pub trait SubjectAdapter: Send + Sync { + fn kind(&self) -> SubjectKind; + + fn validate(&self, _cell: &ExperimentCell) -> Result<(), ContractError> { + Ok(()) + } + + fn execute<'a>( + &'a self, + cell: &'a ExperimentCell, + context: SubjectExecutionContext, + ) -> BoxFuture<'a, SubjectExecutionResult>; +} diff --git a/products/eval/src/subject/external_cli.rs b/products/eval/src/subject/external_cli.rs new file mode 100644 index 00000000000..e89c7a69a37 --- /dev/null +++ b/products/eval/src/subject/external_cli.rs @@ -0,0 +1,662 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! External CLI subject adapter (`exit-code` and `protocol-v1` bounded result frames, +//! placeholder expansion, and checkpoint metering recovery). + +use crate::contract::{ + ContractError, ExecutionOutcome, ExperimentCell, JsonObject, SubjectKind, exact_object, + nonempty_string, nullable_nonnegative_f64, sha256_hex, +}; +use crate::environment::{ + BoxFuture, SubjectExecutionContext, SubjectProcessDiagnostic, SubjectProcessDiagnosticCategory, + SubjectProcessRequest, SubjectProcessTermination, +}; +use crate::measurement::{ + MeteringState, NormalizedUsage, RecoveredExternalMetering, recover_external_metering_checkpoint, +}; +use crate::subject::{SubjectAdapter, SubjectExecutionResult}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::collections::HashMap; +use std::path::Path; +use std::time::Instant; + +pub const RESULT_FRAME_PREFIX: &str = "MAKA-EVAL-RESULT-V1"; +pub const RESULT_PAYLOAD_LIMIT_BYTES: usize = 2 * 1024; +pub const RESULT_CARRIER_LIMIT_BYTES: usize = 64 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ExternalResultMode { + #[serde(rename = "protocol-v1")] + ProtocolV1, + #[serde(rename = "exit-code")] + ExitCode, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExternalSubjectConfig { + pub command: String, + pub args: Vec, + pub environment: HashMap, + pub credential_environment: HashMap, + pub result_mode: ExternalResultMode, +} + +impl ExternalSubjectConfig { + pub fn decode( + config: &JsonObject, + declared_credentials: &[String], + ) -> Result { + let val = Value::Object(config.clone()); + let obj = exact_object( + &val, + "external subject config", + &["command", "args"], + &["environment", "credentialEnvironment", "result"], + )?; + let command = nonempty_string(obj.get("command").unwrap(), "external subject command")?; + let args_arr = obj + .get("args") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("external subject args are invalid".into()))?; + let mut args = Vec::with_capacity(args_arr.len()); + for item in args_arr { + let Some(s) = item.as_str() else { + return Err(ContractError::Invalid( + "external subject args are invalid".into(), + )); + }; + args.push(s.to_owned()); + } + + let environment = match obj.get("environment") { + None => HashMap::new(), + Some(env_val) => decode_env_map(env_val, "environment")?, + }; + + let credential_environment = match obj.get("credentialEnvironment") { + None => declared_credentials + .iter() + .map(|c| (c.clone(), c.clone())) + .collect(), + Some(cred_val) => decode_env_map(cred_val, "credentialEnvironment")?, + }; + for (target, source) in &credential_environment { + if !declared_credentials.iter().any(|c| c == source) { + return Err(ContractError::Invalid(format!( + "credentialEnvironment.{target} must reference a declared credential" + ))); + } + if environment.contains_key(target) { + return Err(ContractError::Invalid(format!( + "environment and credentialEnvironment overlap at {target}" + ))); + } + } + + let result_mode = match obj.get("result").and_then(Value::as_str) { + None | Some("protocol-v1") => ExternalResultMode::ProtocolV1, + Some("exit-code") => ExternalResultMode::ExitCode, + Some(_) => { + return Err(ContractError::Invalid( + "external subject result is invalid".into(), + )); + } + }; + + Ok(Self { + command, + args, + environment, + credential_environment, + result_mode, + }) + } +} + +/// Encodes a bounded `MAKA-EVAL-RESULT-V1` frame matching `relay-result-frame.ts`. +pub fn encode_relay_result_frame(token: &str, value: &Value) -> Result { + if token.len() != 32 + || !token + .bytes() + .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + { + return Err(ContractError::Invalid( + "relay result token is invalid".into(), + )); + } + let payload = serde_json::to_vec(value).map_err(|e| ContractError::Invalid(e.to_string()))?; + if payload.len() > RESULT_PAYLOAD_LIMIT_BYTES { + return Err(ContractError::Invalid( + "relay result exceeds the payload limit".into(), + )); + } + let digest = sha256_hex(&payload); + let encoded = URL_SAFE_NO_PAD.encode(&payload); + Ok(format!( + "{RESULT_FRAME_PREFIX} {token} {} {digest} {encoded}\n", + payload.len() + )) +} + +/// Parses and validates a `MAKA-EVAL-RESULT-V1` frame from raw process stdout. +pub fn parse_relay_result_frame( + carrier: &str, + expected_token: &str, +) -> Result { + if carrier.len() > RESULT_CARRIER_LIMIT_BYTES { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameOversize, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + let matching_lines: Vec<&str> = carrier + .lines() + .filter(|line| { + line.starts_with(RESULT_FRAME_PREFIX) + && line + .split_whitespace() + .nth(1) + .is_some_and(|tok| tok == expected_token) + }) + .collect(); + + if matching_lines.is_empty() { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameMissing, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + if matching_lines.len() > 1 { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameAmbiguous, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + + let parts: Vec<&str> = matching_lines[0].split_whitespace().collect(); + if parts.len() != 5 { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: Some(carrier.len() as u64), + sha256: Some(sha256_hex(carrier.as_bytes())), + }); + } + let Ok(expected_len) = parts[2].parse::() else { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: None, + sha256: None, + }); + }; + if expected_len > RESULT_PAYLOAD_LIMIT_BYTES { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameOversize, + bytes: Some(expected_len as u64), + sha256: None, + }); + } + let expected_digest = parts[3]; + let Ok(decoded_bytes) = URL_SAFE_NO_PAD.decode(parts[4]) else { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: None, + sha256: None, + }); + }; + if decoded_bytes.len() != expected_len || sha256_hex(&decoded_bytes) != expected_digest { + return Err(SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: Some(decoded_bytes.len() as u64), + sha256: Some(sha256_hex(&decoded_bytes)), + }); + } + String::from_utf8(decoded_bytes).map_err(|_| SubjectProcessDiagnostic { + category: SubjectProcessDiagnosticCategory::ResultFrameInvalid, + bytes: None, + sha256: None, + }) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct DecodedExternalResult { + pub output: Option, + pub usage: Option, + pub cost_usd: Option, + pub status: Option, + pub failure_reason: Option, + pub artifacts: Vec, +} + +pub fn decode_external_result_json(stdout: &str) -> Result { + let parsed: Value = serde_json::from_str(stdout) + .map_err(|_| ContractError::Invalid("external result must be valid JSON".into()))?; + let obj = exact_object( + &parsed, + "external result", + &["schemaVersion", "usage", "costUsd", "artifacts"], + &["output", "status", "failureReason"], + )?; + let schema_version = obj.get("schemaVersion").and_then(Value::as_str); + if !matches!( + schema_version, + Some("maka.external_subject_result.v1" | "maka.external_subject_result.v2") + ) { + return Err(ContractError::Invalid( + "external result schema is invalid".into(), + )); + } + if schema_version == Some("maka.external_subject_result.v2") + && (!obj.contains_key("status") || !obj.contains_key("failureReason")) + { + return Err(ContractError::Invalid( + "external result v2 fields are invalid".into(), + )); + } + let output = match obj.get("output") { + None => None, + Some(Value::String(s)) => Some(s.clone()), + Some(_) => { + return Err(ContractError::Invalid( + "external result output is invalid".into(), + )); + } + }; + let status = match obj.get("status") { + None => None, + Some(Value::String(s)) + if matches!( + s.as_str(), + "completed" | "failed" | "infra_failed" | "indeterminate" + ) => + { + Some(s.clone()) + } + Some(_) => { + return Err(ContractError::Invalid( + "external result status is invalid".into(), + )); + } + }; + let failure_reason = match obj.get("failureReason") { + None | Some(Value::Null) => None, + Some(Value::String(s)) => Some(s.clone()), + Some(_) => { + return Err(ContractError::Invalid( + "external result failureReason is invalid".into(), + )); + } + }; + let usage = match obj.get("usage").unwrap() { + Value::Null => None, + u => Some(NormalizedUsage::decode(u, "external usage")?), + }; + let cost_usd = nullable_nonnegative_f64(obj.get("costUsd").unwrap(), "external result cost")?; + let artifacts_arr = obj + .get("artifacts") + .and_then(Value::as_array) + .ok_or_else(|| ContractError::Invalid("external result artifacts are invalid".into()))?; + let mut artifacts = Vec::with_capacity(artifacts_arr.len()); + for item in artifacts_arr { + let Some(map) = item.as_object() else { + return Err(ContractError::Invalid( + "external result artifacts are invalid".into(), + )); + }; + artifacts.push(map.clone()); + } + + Ok(DecodedExternalResult { + output, + usage, + cost_usd, + status, + failure_reason, + artifacts, + }) +} + +#[derive(Debug, Default, Clone)] +pub struct ExternalCliSubject; + +impl ExternalCliSubject { + pub fn new() -> Self { + Self + } +} + +impl SubjectAdapter for ExternalCliSubject { + fn kind(&self) -> SubjectKind { + SubjectKind::External + } + + fn validate(&self, cell: &ExperimentCell) -> Result<(), ContractError> { + let _ = ExternalSubjectConfig::decode(&cell.subject.config, &cell.subject.credentials)?; + Ok(()) + } + + fn execute<'a>( + &'a self, + cell: &'a ExperimentCell, + context: SubjectExecutionContext, + ) -> BoxFuture<'a, SubjectExecutionResult> { + Box::pin(async move { + let started = Instant::now(); + let Ok(config) = + ExternalSubjectConfig::decode(&cell.subject.config, &cell.subject.credentials) + else { + return SubjectExecutionResult { + output: None, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms: 0, + outcome: ExecutionOutcome::InfraFailed, + failure_reason: Some("external subject config invalid".into()), + artifacts: Vec::new(), + operation_id: None, + invocation_id: None, + }; + }; + + let profile = Some(cell.subject.id.as_str()); + let args: Vec = config + .args + .iter() + .map(|arg| { + arg.replace("{{task.input}}", &context.task_input) + .replace("{{task.id}}", &cell.task.id) + .replace("{{task.cwd}}", &context.cwd) + }) + .collect(); + + let execution = match context + .execute(SubjectProcessRequest { + command: config.command, + args, + environment: config.environment, + credential_environment: config.credential_environment, + capture_stdout: config.result_mode != ExternalResultMode::ExitCode, + }) + .await + { + Ok(exec) => exec, + Err(_) => { + let recovered = recover_from_metadata(&context.metadata, profile); + let cancelled = context.cancel.is_cancelled(); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms: started.elapsed().as_millis() as u64, + outcome: if cancelled { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::InfraFailed + }, + failure_reason: Some(if cancelled { + "external subject cancelled".into() + } else { + "external subject execution failed".into() + }), + artifacts: recovered.into_iter().map(|r| r.artifact).collect(), + operation_id: None, + invocation_id: None, + }; + } + }; + + let duration_ms = started.elapsed().as_millis() as u64; + let decoded = if execution.stdout.is_empty() { + None + } else { + decode_external_result_json(&execution.stdout).ok() + }; + let recovered = recover_from_metadata(&context.metadata, profile); + + let mut process_art = Map::new(); + process_art.insert("kind".into(), Value::String("external_process".into())); + process_art.insert("exitCode".into(), Value::from(execution.exit_code)); + + if execution.termination == SubjectProcessTermination::FrameworkTimeout { + let usage = decoded + .as_ref() + .and_then(|d| d.usage.clone()) + .or_else(|| recovered.as_ref().and_then(|r| r.usage.clone())); + let cost_usd = decoded + .as_ref() + .and_then(|d| d.cost_usd) + .or_else(|| recovered.as_ref().and_then(|r| r.cost_usd)); + let metering_state = match (&usage, cost_usd) { + (Some(_), Some(_)) => MeteringState::Complete, + (Some(_), None) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + let mut artifacts = match &decoded { + Some(d) => d.artifacts.clone(), + None => recovered.iter().map(|r| r.artifact.clone()).collect(), + }; + if decoded.as_ref().is_some_and(|d| d.usage.is_none()) + && let Some(rec) = &recovered + { + artifacts.push(rec.artifact.clone()); + } + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage, + metering_state, + cost_usd, + duration_ms, + outcome: ExecutionOutcome::Timeout, + failure_reason: Some("external subject exceeded the framework timeout".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + } + + if execution.diagnostic.as_ref().map(|d| d.category) + == Some(SubjectProcessDiagnosticCategory::ExecutionScopeUnavailable) + { + let mut artifacts: Vec = + recovered.iter().map(|r| r.artifact.clone()).collect(); + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms, + outcome: if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::InfraFailed + }, + failure_reason: Some("external subject execution scope was unavailable".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + } + + if config.result_mode == ExternalResultMode::ExitCode { + let outcome = if execution.exit_code == 0 { + ExecutionOutcome::Completed + } else if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else { + ExecutionOutcome::SubjectFailed + }; + return SubjectExecutionResult { + output: None, + usage: None, + metering_state: MeteringState::Unknown, + cost_usd: None, + duration_ms, + outcome, + failure_reason: (execution.exit_code != 0) + .then(|| format!("external subject exited {}", execution.exit_code)), + artifacts: vec![process_art], + operation_id: None, + invocation_id: None, + }; + } + + if execution + .diagnostic + .as_ref() + .is_some_and(|d| d.category.is_result_frame_error()) + { + let outcome = if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else if recovered.as_ref().is_some_and(|r| r.admitted_requests > 0) { + ExecutionOutcome::SubjectFailed + } else { + ExecutionOutcome::InfraFailed + }; + let mut artifacts: Vec = + recovered.iter().map(|r| r.artifact.clone()).collect(); + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms, + outcome, + failure_reason: Some("external subject result transport failed".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + } + + let Some(decoded) = decoded else { + let outcome = if context.cancel.is_cancelled() { + ExecutionOutcome::Indeterminate + } else if recovered.as_ref().is_some_and(|r| r.admitted_requests > 0) { + ExecutionOutcome::SubjectFailed + } else { + ExecutionOutcome::InfraFailed + }; + let mut artifacts: Vec = + recovered.iter().map(|r| r.artifact.clone()).collect(); + artifacts.push(process_art); + return SubjectExecutionResult { + output: None, + usage: recovered.as_ref().and_then(|r| r.usage.clone()), + metering_state: recovered + .as_ref() + .map_or(MeteringState::Unknown, |r| r.metering_state), + cost_usd: recovered.as_ref().and_then(|r| r.cost_usd), + duration_ms, + outcome, + failure_reason: Some("external subject returned no result".into()), + artifacts, + operation_id: None, + invocation_id: None, + }; + }; + + let outcome = match decoded.status.as_deref() { + Some("completed") => ExecutionOutcome::Completed, + Some("failed") => ExecutionOutcome::SubjectFailed, + Some("infra_failed") => ExecutionOutcome::InfraFailed, + Some("indeterminate") => ExecutionOutcome::Indeterminate, + None if execution.exit_code == 0 => ExecutionOutcome::Completed, + None => ExecutionOutcome::SubjectFailed, + Some(_) => ExecutionOutcome::InfraFailed, + }; + let metering_state = match (&decoded.usage, decoded.cost_usd) { + (Some(_), Some(_)) => MeteringState::Complete, + (Some(_), None) => MeteringState::LowerBound, + (None, _) => MeteringState::Unknown, + }; + let failure_reason = decoded.failure_reason.or_else(|| { + (execution.exit_code != 0 && outcome != ExecutionOutcome::Completed) + .then(|| format!("external subject exited {}", execution.exit_code)) + }); + + SubjectExecutionResult { + output: decoded.output, + usage: decoded.usage, + metering_state, + cost_usd: decoded.cost_usd, + duration_ms, + outcome, + failure_reason, + artifacts: decoded.artifacts, + operation_id: None, + invocation_id: None, + } + }) + } +} + +fn recover_from_metadata( + metadata: &JsonObject, + profile: Option<&str>, +) -> Option { + let profile = profile?; + let trial_path = metadata.get("trialPath")?.as_str()?; + let secret = metadata.get("meteringSecret")?.as_str()?; + recover_external_metering_checkpoint(Path::new(trial_path), profile, secret) +} + +fn decode_env_map( + value: &Value, + where_label: &str, +) -> Result, ContractError> { + let obj = value + .as_object() + .ok_or_else(|| ContractError::Invalid(format!("{where_label} must be an object")))?; + let mut map = HashMap::with_capacity(obj.len()); + for (name, item) in obj { + let mut chars = name.chars(); + let valid_name = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + let Some(val_str) = item.as_str() else { + return Err(ContractError::Invalid(format!( + "{where_label}.{name} is invalid" + ))); + }; + if !valid_name { + return Err(ContractError::Invalid(format!( + "{where_label}.{name} is invalid" + ))); + } + map.insert(name.clone(), val_str.to_owned()); + } + Ok(map) +} diff --git a/products/eval/tests/integration.rs b/products/eval/tests/integration.rs new file mode 100644 index 00000000000..002c7c14bdf --- /dev/null +++ b/products/eval/tests/integration.rs @@ -0,0 +1,1227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use maka_eval::contract::{ + AttemptPhase, CleanupState, CriticalArtifactsState, DifferenceRegistry, DifferenceScenarioId, + EgressAuditState, EvalResult, EvalResultStatus, ExecutionOutcome, ExperimentCell, + RunFingerprint, SchemaVersion, V1CellAttempt, V2AttemptRecord, VerifierOutcome, + expand_experiment, parse_experiment_spec, select_cell_result, +}; +use maka_eval::environment::{ + ActiveTrialAttempt, BoxFuture, CommandRunner, EgressAuditEvidence, ExecutorPreparationCode, + ExecutorVerification, HarborExecutorConfig, HarborTrialEnvironment, PreparationCacheLocks, + SubjectExecutionContext, SubjectProcessDiagnosticCategory, SubjectProcessOutput, + SubjectProcessRequest, SubjectProcessTermination, TrialAttemptOperation, TrialEnvironment, + TrialEnvironmentOutcome, describe_egress_audit, read_trial_verification, +}; +use maka_eval::measurement::{ + AdmissionBoundaryLedger, DEEPSEEK_V4_FLASH_COST, MeteringState, NormalizedUsage, + costs_match_within_tolerance, deepseek_cost_usd, derive_metering, + reconcile_host_and_proxy_usage, recover_external_metering_checkpoint, sign_metering_checkpoint, +}; +use maka_eval::report::{ + evaluate_differential_oracle, export_v1_canonical, export_v2_canonical, rebuild_from_journal, +}; +use maka_eval::runner::{ + ConservativeCleanupProbe, RunExperimentInput, recover_unsettled_attempts, run_experiment, +}; +use maka_eval::store::{ + ExperimentDirectory, capture_maka_runtime_artifacts, verify_critical_artifacts, +}; +use maka_eval::subject::SubjectAdapter; +use maka_eval::subject::external_cli::{ + ExternalCliSubject, encode_relay_result_frame, parse_relay_result_frame, +}; +use serde_json::{Map, Value, json}; +use std::fs; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tempfile::TempDir; +use tokio::sync::Barrier; +use tokio_util::sync::CancellationToken; + +#[test] +fn experiment_spec_validation_and_cell_expansion_match_v1_contract() { + let spec_json = sample_two_arm_spec_json(); + let spec = parse_experiment_spec(&spec_json).unwrap(); + let cells = expand_experiment(&spec); + + assert_eq!(cells.len(), 4); + assert_eq!(cells[0].id, "task-alpha::1::maka"); + assert_eq!(cells[1].id, "task-alpha::1::external-cli"); + assert_eq!(cells[2].id, "task-beta::1::maka"); + assert_eq!(cells[3].id, "task-beta::1::external-cli"); + + // Unknown top-level field must be rejected. + let mut invalid = spec_json.clone(); + invalid + .as_object_mut() + .unwrap() + .insert("unknownField".into(), json!(123)); + assert!(parse_experiment_spec(&invalid).is_err()); + + // Concurrency changes execution fingerprint but preserves semantic fingerprint. + let mut higher_concurrency_json = spec_json; + higher_concurrency_json["execution"]["maxConcurrentTaskGroups"] = json!(16); + let higher_spec = parse_experiment_spec(&higher_concurrency_json).unwrap(); + + let fp1 = RunFingerprint::compute(&spec); + let fp2 = RunFingerprint::compute(&higher_spec); + assert!(fp1.is_semantically_comparable(&fp2)); + assert_ne!(fp1.execution_sha256, fp2.execution_sha256); +} + +#[test] +fn six_counter_usage_never_sums_total_tokens_and_deepseek_pricing_matches_within_1e9() { + // Provider reports a totalTokens value distinct from the arithmetic sum of the 5 fields. + let u1 = NormalizedUsage { + input_tokens: 1000, + output_tokens: 250, + cache_read_tokens: 400, + cache_write_tokens: 100, + reasoning_tokens: 80, + total_tokens: 9999, + }; + let u2 = NormalizedUsage { + input_tokens: 500, + output_tokens: 150, + cache_read_tokens: 200, + cache_write_tokens: 0, + reasoning_tokens: 20, + total_tokens: 7777, + }; + let mut combined = u1.clone(); + combined.accumulate_exact(&u2); + assert_eq!(combined.input_tokens, 1500); + assert_eq!(combined.output_tokens, 400); + assert_eq!(combined.cache_read_tokens, 600); + assert_eq!(combined.cache_write_tokens, 100); + assert_eq!(combined.reasoning_tokens, 100); + // Must equal 9999 + 7777 = 17776, never the sum of the other 5 counters! + assert_eq!(combined.total_tokens, 17776); + + // Verify DEEPSEEK_V4_FLASH_COST formula: + // uncached = 1000 - 400 - 100 = 500 + // (500 * 0.22 + 400 * 0.007 + 100 * 0.22 + 250 * 0.66) / 1_000_000 + // = (110 + 2.8 + 22 + 165) / 1_000_000 = 299.8 / 1_000_000 = 0.0002998 + let cost = deepseek_cost_usd(&u1); + assert_eq!(DEEPSEEK_V4_FLASH_COST.input_per_million, 0.22); + assert_eq!(DEEPSEEK_V4_FLASH_COST.output_per_million, 0.66); + assert_eq!(DEEPSEEK_V4_FLASH_COST.cache_read_per_million, 0.007); + assert_eq!(DEEPSEEK_V4_FLASH_COST.cache_write_per_million, 0.22); + assert!(costs_match_within_tolerance(Some(cost), Some(0.0002998))); + assert!(!costs_match_within_tolerance( + Some(cost), + Some(0.0002998 + 1e-8) + )); +} + +#[test] +fn admission_cutoff_checkpoint_deduplication_and_receipt_proxy_reconciliation() { + let mut ledger = AdmissionBoundaryLedger::new(); + assert!(ledger.record_request_start("req-1")); + assert!(ledger.record_request_start("req-2")); + + ledger.close_admission(); + assert!(ledger.is_admission_closed()); + // Request arriving after cutoff is rejected. + assert!(!ledger.record_request_start("req-after-cutoff")); + + let u1 = NormalizedUsage { + input_tokens: 100, + output_tokens: 50, + cache_read_tokens: 20, + cache_write_tokens: 0, + reasoning_tokens: 10, + total_tokens: 150, + }; + // Both Host receipt and proxy log settle req-1; idempotent settlement must not double-count. + ledger + .settle_request("req-1", Some(u1.clone()), Some("deepseek-v4-flash".into())) + .unwrap(); + ledger + .settle_request("req-1", Some(u1.clone()), Some("deepseek-v4-flash".into())) + .unwrap(); + + // While req-2 is still in flight, snapshot is lower-bound (`settled == false`). + let mid_snapshot = ledger.snapshot(); + let mid_derived = derive_metering(&mid_snapshot); + assert!(!mid_derived.usage_complete); + assert_eq!(mid_derived.metering_state, MeteringState::LowerBound); + assert_eq!(mid_derived.cost_usd, None); + + // Duplicate checkpoint ingestion is deduplicated. + assert!(ledger.ingest_checkpoint(mid_snapshot.clone())); + assert!(!ledger.ingest_checkpoint(mid_snapshot)); + + // Once req-2 drains and proxy settles, metering becomes Complete with costUsd populated. + ledger + .settle_request("req-2", Some(u1.clone()), Some("deepseek-v4-flash".into())) + .unwrap(); + ledger.mark_proxy_settled(); + let final_snapshot = ledger.snapshot(); + let final_derived = derive_metering(&final_snapshot); + assert!(final_derived.usage_complete); + assert_eq!(final_derived.metering_state, MeteringState::Complete); + assert_eq!(final_snapshot.usage.as_ref().unwrap().total_tokens, 300); + assert!(final_derived.cost_usd.is_some()); + + // Reconciling matching Host and proxy usage yields Complete without double-counting. + let (reconciled, state, cost) = + reconcile_host_and_proxy_usage(final_snapshot.usage.as_ref(), true, Some(&final_snapshot)); + assert_eq!(state, MeteringState::Complete); + assert_eq!(reconciled.unwrap().total_tokens, 300); + assert!(cost.is_some()); +} + +#[test] +fn all_five_semantic_difference_scenarios_and_three_verdict_differential_oracle() { + let registry = DifferenceRegistry::new(); + assert_eq!(registry.entries().len(), 5); + + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let fp = RunFingerprint::compute(&spec); + + let sample_usage = NormalizedUsage { + input_tokens: 200, + output_tokens: 100, + cache_read_tokens: 50, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 300, + }; + + // 1. Timeout with verifier reward (1.0) but incomplete (lower-bound) usage: + // Status is `subject_failed`, score is preserved (`Some(1.0)`), usage is preserved, + // and `costUsd` is `None` because usage is lower-bound. + let v2_timeout_reward = V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id: "task-alpha::1::maka".into(), + sequence: 1, + started_at: 100, + completed_at: 200, + duration_ms: 100, + phase: AttemptPhase::Settled, + execution_outcome: ExecutionOutcome::Timeout, + verifier_outcome: VerifierOutcome::Verified { score: 1.0 }, + usage: Some(sample_usage.clone()), + metering_state: MeteringState::LowerBound, + pricing: None, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::NotRequired, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: false, + recovered_post_hoc: false, + failure_reason: Some("Maka subject exceeded the framework timeout".into()), + artifacts: Vec::new(), + }; + let v1_timeout_reward = v2_timeout_reward.project_v1(); + assert_eq!( + v1_timeout_reward.result.status, + EvalResultStatus::SubjectFailed + ); + assert_eq!(v1_timeout_reward.result.score, Some(1.0)); + assert_eq!(v1_timeout_reward.result.usage, Some(sample_usage.clone())); + assert_eq!(v1_timeout_reward.result.cost_usd, None); + + let report_1 = evaluate_differential_oracle( + &serde_json::to_value(&v1_timeout_reward).unwrap(), + &v1_timeout_reward, + &fp, + &fp, + Some(DifferenceScenarioId::TimeoutWithRewardIncompleteUsage), + ®istry, + ); + assert!(report_1.v1_parses); + assert!(report_1.v1_matches_oracle); + assert!(report_1.conditions_comparable); + for entry in registry.entries() { + let mut changed = v1_timeout_reward.clone(); + changed.result.score = Some(0.0); + changed.result.usage.as_mut().unwrap().input_tokens = 999_999; + let report = evaluate_differential_oracle( + &serde_json::to_value(changed).unwrap(), + &v1_timeout_reward, + &fp, + &fp, + Some(entry.id), + ®istry, + ); + assert!( + !report.v1_matches_oracle, + "scenario {:?} must not hide actual differences", + entry.id + ); + assert!(!report.conditions_comparable); + } + + // 2. Audit truncation vs. Missing audit log: + let (trunc_state, trunc_ev) = describe_egress_audit( + Some(b"{\"ts\":1,\"ruleId\":\"audit_truncated\",\"host\":\"\",\"normalizedPath\":\"\"}\n"), + true, + ); + assert_eq!(trunc_state, EgressAuditState::Truncated); + let mut v2_audit_trunc = v2_timeout_reward.clone(); + v2_audit_trunc.execution_outcome = ExecutionOutcome::Completed; + v2_audit_trunc.metering_state = MeteringState::Complete; + v2_audit_trunc.audit_state = trunc_state; + v2_audit_trunc.artifacts = trunc_ev.artifacts; + v2_audit_trunc.failure_reason = None; + let v1_audit_trunc = v2_audit_trunc.project_v1(); + assert_eq!(v1_audit_trunc.result.status, EvalResultStatus::Completed); + assert_eq!(v1_audit_trunc.result.score, Some(1.0)); + assert!(v1_audit_trunc.result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("egress-audit") + && a.get("truncated").and_then(Value::as_bool) == Some(true) + })); + + // Missing audit log fails closed as `infra_failed` with `score: None`. + let mut v2_audit_missing = v2_audit_trunc.clone(); + v2_audit_missing.audit_state = EgressAuditState::Missing; + let v1_audit_missing = v2_audit_missing.project_v1(); + assert_eq!( + v1_audit_missing.result.status, + EvalResultStatus::InfraFailed + ); + assert_eq!(v1_audit_missing.result.score, None); + + // 3. 0-byte empty file vs. missing critical artifact: + let temp = TempDir::new().unwrap(); + fs::write(temp.path().join("stderr.txt"), b"").unwrap(); + let (present_state, files) = verify_critical_artifacts(temp.path(), &["stderr.txt"]); + assert_eq!(present_state, CriticalArtifactsState::Present); + assert_eq!(files[0].bytes, 0); + + let (missing_state, _) = + verify_critical_artifacts(temp.path(), &["stderr.txt", "missing-transcript.json"]); + assert_eq!( + missing_state, + CriticalArtifactsState::Missing { + paths: vec!["missing-transcript.json".into()] + } + ); + let mut v2_missing_art = v2_audit_trunc.clone(); + v2_missing_art.critical_artifacts_state = missing_state; + let v1_missing_art = v2_missing_art.project_v1(); + assert_eq!(v1_missing_art.result.status, EvalResultStatus::InfraFailed); + assert_eq!(v1_missing_art.result.score, None); + + // 4. Late cancellation before vs. after verification: + let mut v2_cancel_before = v2_audit_trunc.clone(); + v2_cancel_before.execution_outcome = ExecutionOutcome::Cancelled; + v2_cancel_before.cancelled_after_verification = false; + let v1_cancel_before = v2_cancel_before.project_v1(); + assert_eq!( + v1_cancel_before.result.status, + EvalResultStatus::Indeterminate + ); + assert_eq!(v1_cancel_before.result.score, None); + + let mut v2_cancel_after = v2_audit_trunc.clone(); + v2_cancel_after.execution_outcome = ExecutionOutcome::Cancelled; + v2_cancel_after.cancelled_after_verification = true; + let v1_cancel_after = v2_cancel_after.project_v1(); + assert_eq!(v1_cancel_after.result.status, EvalResultStatus::Completed); + assert_eq!(v1_cancel_after.result.score, Some(1.0)); + let mut legacy_cancel = v1_cancel_before.clone(); + legacy_cancel.result.failure_reason = + Some("executor cancelled before verification completed".into()); + let late = evaluate_differential_oracle( + &serde_json::to_value(&v1_cancel_after).unwrap(), + &legacy_cancel, + &fp, + &fp, + Some(DifferenceScenarioId::LateCancellation), + ®istry, + ); + assert!(!late.v1_matches_oracle); + assert!(late.adjudicated_difference); + assert!(late.conditions_comparable); + let mut altered = v1_cancel_after.clone(); + altered.result.usage.as_mut().unwrap().output_tokens += 1; + let altered = evaluate_differential_oracle( + &serde_json::to_value(altered).unwrap(), + &legacy_cancel, + &fp, + &fp, + Some(DifferenceScenarioId::LateCancellation), + ®istry, + ); + assert!(!altered.adjudicated_difference); + assert!(!altered.conditions_comparable); + + // 5. Missing verifier reward never defaults to 0.0; genuine 0.0 reward is Completed(0.0): + let mut v2_missing_reward = v2_audit_trunc.clone(); + v2_missing_reward.verifier_outcome = VerifierOutcome::MissingReward { + reason: "reward.json missing".into(), + }; + let v1_missing_reward = v2_missing_reward.project_v1(); + assert_eq!( + v1_missing_reward.result.status, + EvalResultStatus::InfraFailed + ); + assert_eq!(v1_missing_reward.result.score, None); + + let mut v2_zero_score = v2_audit_trunc; + v2_zero_score.verifier_outcome = VerifierOutcome::Verified { score: 0.0 }; + let v1_zero_score = v2_zero_score.project_v1(); + assert_eq!(v1_zero_score.result.status, EvalResultStatus::Completed); + assert_eq!(v1_zero_score.result.score, Some(0.0)); +} + +#[test] +fn canonical_selector_picks_earliest_valid_attempt_and_treats_zero_score_as_terminal() { + let attempts = vec![ + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 1, + started_at: 10, + completed_at: 20, + result: EvalResult { + score: None, + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::InfraFailed, + failure_reason: Some("docker pull failed".into()), + artifacts: Vec::new(), + }, + }, + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 2, + started_at: 30, + completed_at: 40, + result: EvalResult { + score: Some(0.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, + V1CellAttempt { + cell_id: "t1::1::maka".into(), + sequence: 3, + started_at: 50, + completed_at: 60, + result: EvalResult { + score: Some(1.0), + usage: None, + cost_usd: None, + duration_ms: 10, + status: EvalResultStatus::Completed, + failure_reason: None, + artifacts: Vec::new(), + }, + }, + ]; + + let selected = select_cell_result(&attempts).unwrap(); + // Sequence 2 (Completed with score 0.0) is valid and earlier than sequence 3! + assert_eq!(selected.sequence, 2); + assert_eq!(selected.result.score, Some(0.0)); +} + +#[test] +fn store_journal_reconstruction_post_hoc_recovery_and_crash_recovery_boundaries() { + let temp = TempDir::new().unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let dir = ExperimentDirectory::open(temp.path().join("exp-out"), &spec).unwrap(); + let store = &dir.attempts; + + // 1. Single-writer lock rejects concurrent writers. + let guard = store.acquire_writer().unwrap(); + assert!(store.acquire_writer().is_err()); + drop(guard); + + // 2. Crash after LaunchIntent BEFORE resource creation recovers as InfraFailed (Confirmed cleanup). + store + .record_launch_intent("task-alpha::1::maka", 1, 1000, "intent-1") + .unwrap(); + let recovered = recover_unsettled_attempts(store, &ConservativeCleanupProbe, 1050).unwrap(); + assert_eq!(recovered.len(), 1); + assert_eq!( + recovered[0].execution_outcome, + ExecutionOutcome::InfraFailed + ); + assert_eq!(recovered[0].cleanup_state, CleanupState::Confirmed); + let failed_store = maka_eval::store::FileAttemptStore::new(temp.path().join("failed")); + fs::create_dir_all(failed_store.journal_path()).unwrap(); + assert!(failed_store.append_v2(&recovered[0]).is_err()); + fs::remove_dir(failed_store.journal_path()).unwrap(); + assert!( + failed_store + .list_v1("task-alpha::1::maka") + .unwrap() + .is_empty() + ); + assert!( + export_v1_canonical(&spec, &failed_store) + .unwrap() + .is_empty() + ); + assert!( + export_v2_canonical(&spec, &failed_store) + .unwrap() + .is_empty() + ); + + // 3. Crash AFTER ResourceTagged with unconfirmed cleanup recovers as Indeterminate. + store + .record_launch_intent("task-alpha::1::maka", 2, 2000, "intent-2") + .unwrap(); + store + .record_resource_tagged( + "task-alpha::1::maka", + 2, + "harbor", + "trial-container-2", + Some("op-1".into()), + Some("inv-1".into()), + ) + .unwrap(); + let recovered_2 = recover_unsettled_attempts(store, &ConservativeCleanupProbe, 2100).unwrap(); + assert_eq!(recovered_2.len(), 1); + assert_eq!( + recovered_2[0].execution_outcome, + ExecutionOutcome::Indeterminate + ); + assert!(matches!( + recovered_2[0].cleanup_state, + CleanupState::Unconfirmed { .. } + )); + + // 4. Append-only PostHocRecovery updates reconstructed usage & cleanup without mutating prior entries. + let recovered_usage = NormalizedUsage { + input_tokens: 500, + output_tokens: 100, + cache_read_tokens: 0, + cache_write_tokens: 0, + reasoning_tokens: 0, + total_tokens: 600, + }; + store + .record_post_hoc_recovery( + "task-alpha::1::maka", + 2, + 2200, + Some(recovered_usage.clone()), + MeteringState::Complete, + true, + Some("recovered from host receipt journal".into()), + ) + .unwrap(); + + let rebuilt = rebuild_from_journal(store).unwrap(); + let seq2 = &rebuilt.settled_v2["task-alpha::1::maka"][1]; + assert!(seq2.recovered_post_hoc); + assert_eq!(seq2.usage, Some(recovered_usage)); + assert_eq!(seq2.cleanup_state, CleanupState::Confirmed); + assert!( + seq2.project_v1().result.artifacts.iter().any(|a| { + a.get("kind").and_then(Value::as_str) == Some("post-hoc-evidence-recovery") + }) + ); +} + +#[test] +fn sqlite_runtime_artifact_capture_and_external_checkpoint_hmac_verification() { + let temp = TempDir::new().unwrap(); + let state_root = temp.path().join("host-state"); + fs::create_dir_all(&state_root).unwrap(); + + // Create a real SQLite database at state_root/runtime.sqlite + let db_path = state_root.join("runtime.sqlite"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute("CREATE TABLE receipts (id TEXT PRIMARY KEY)", []) + .unwrap(); + conn.execute("INSERT INTO receipts (id) VALUES ('op-123')", []) + .unwrap(); + drop(conn); + fs::write( + state_root.join("runtime-policy.json"), + b"{\"mode\":\"eval\"}\n", + ) + .unwrap(); + + let dest_root = temp.path().join("captured-artifacts"); + let manifest = + capture_maka_runtime_artifacts(&state_root, &dest_root, "settled", 123456).unwrap(); + assert_eq!(manifest.schema_version, "maka.eval.runtime_artifacts.v1"); + assert_eq!(manifest.files.len(), 2); + assert!(dest_root.join("runtime.sqlite").is_file()); + assert!(dest_root.join("manifest.json").is_file()); + + // Verify external metering checkpoint HMAC signing & recovery. + let trial_path = temp.path().join("trial-1"); + fs::create_dir_all(trial_path.join("agent")).unwrap(); + let secret = "0123456789abcdef0123456789abcdef"; + let body = json!({ + "schemaVersion": "maka.external_provider_usage.v2", + "profile": "deepseek-harness", + "usage": { + "inputTokens": 1000, + "outputTokens": 200, + "cacheReadTokens": 100, + "cacheWriteTokens": 0, + "reasoningTokens": 50, + "totalTokens": 1200 + }, + "settled": true, + "requests": 2, + "inFlightRequests": 0, + "admittedRequests": 2, + "usageRequests": 2, + "models": ["deepseek-v4-flash"], + "toolNames": ["bash"] + }); + let signed = sign_metering_checkpoint(body.as_object().unwrap(), secret); + let cp_path = trial_path + .join("agent") + .join("deepseek-harness.provider-usage.json"); + fs::write(&cp_path, serde_json::to_vec(&signed).unwrap()).unwrap(); + + let recovered = + recover_external_metering_checkpoint(&trial_path, "deepseek-harness", secret).unwrap(); + assert_eq!(recovered.metering_state, MeteringState::Complete); + assert_eq!(recovered.usage.unwrap().total_tokens, 1200); + assert!(recovered.cost_usd.is_some()); + + // Wrong secret must be rejected. + assert!( + recover_external_metering_checkpoint(&trial_path, "deepseek-harness", "wrong-secret") + .is_none() + ); +} + +#[tokio::test] +async fn shared_preparation_cache_lock_serializes_concurrent_task_preparations() { + let locks = PreparationCacheLocks::new(); + let active_in_critical_section = Arc::new(AtomicUsize::new(0)); + let max_concurrent_observed = Arc::new(AtomicUsize::new(0)); + let build_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(4)); + + let mut handles = Vec::new(); + for _ in 0..4 { + let locks = locks.clone(); + let active = active_in_critical_section.clone(); + let max_obs = max_concurrent_observed.clone(); + let builds = build_count.clone(); + let barrier = barrier.clone(); + handles.push(tokio::spawn(async move { + barrier.wait().await; + locks + .with_task_lock("task-shared-image", || async { + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + max_obs.fetch_max(current, Ordering::SeqCst); + if builds.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + builds.fetch_add(1, Ordering::SeqCst); + } + active.fetch_sub(1, Ordering::SeqCst); + }) + .await; + })); + } + + for h in handles { + h.await.unwrap(); + } + assert_eq!(max_concurrent_observed.load(Ordering::SeqCst), 1); + assert_eq!(build_count.load(Ordering::SeqCst), 1); +} + +#[test] +fn relay_result_frame_boundaries_reject_ambiguous_duplicates() { + let token = "abcdef0123456789abcdef0123456789"; + let payload = json!({ + "schemaVersion": "maka.external_subject_result.v2", + "status": "completed", + "failureReason": null, + "usage": { + "inputTokens": 100, + "outputTokens": 50, + "cacheReadTokens": 0, + "cacheWriteTokens": 0, + "reasoningTokens": 0, + "totalTokens": 150 + }, + "costUsd": 0.000055, + "artifacts": [] + }); + let frame = encode_relay_result_frame(token, &payload).unwrap(); + let noisy_stdout = format!("some tool log line\n{frame}trailing line\n"); + let extracted = parse_relay_result_frame(&noisy_stdout, token).unwrap(); + let parsed_val: Value = serde_json::from_str(&extracted).unwrap(); + assert_eq!(parsed_val, payload); + + let duplicate = format!("{frame}{frame}"); + let err = parse_relay_result_frame(&duplicate, token).unwrap_err(); + assert_eq!( + err.category, + SubjectProcessDiagnosticCategory::ResultFrameAmbiguous + ); +} + +struct FakeTrialEnvironment { + ext_calls: Arc, +} + +struct FakeActiveTrial { + cell_id: String, + ext_calls: Arc, +} + +impl CommandRunner for FakeActiveTrial { + fn execute<'a>( + &'a self, + _request: SubjectProcessRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + assert!(self.cell_id.ends_with("::external-cli")); + self.ext_calls.fetch_add(1, Ordering::SeqCst); + let out = json!({ + "schemaVersion": "maka.external_subject_result.v2", + "status": "completed", + "failureReason": null, + "usage": { + "inputTokens": 400, + "outputTokens": 100, + "cacheReadTokens": 100, + "cacheWriteTokens": 0, + "reasoningTokens": 20, + "totalTokens": 500 + }, + "costUsd": 0.0001327, + "artifacts": [] + }); + Ok(SubjectProcessOutput { + termination: SubjectProcessTermination::Exited, + exit_code: 0, + stdout: serde_json::to_string(&out).unwrap(), + stderr: String::new(), + diagnostic: None, + }) + }) + } +} + +impl ActiveTrialAttempt for FakeActiveTrial { + fn context(&self) -> SubjectExecutionContext { + SubjectExecutionContext { + cwd: "/workspace".into(), + task_input: "Solve benchmark task".into(), + metadata: Map::new(), + cancel: CancellationToken::new(), + runner: Arc::new(Self { + cell_id: self.cell_id.clone(), + ext_calls: self.ext_calls.clone(), + }), + } + } + + fn verify(&self) -> BoxFuture<'_, Result> { + Box::pin(async { + Ok(ExecutorVerification { + status: EvalResultStatus::Completed, + score: Some(1.0), + failure_reason: None, + artifacts: Vec::new(), + missing_reward: false, + }) + }) + } + + fn read_audit(&self) -> BoxFuture<'_, (EgressAuditState, EgressAuditEvidence)> { + Box::pin(async { describe_egress_audit(None, false) }) + } +} + +impl TrialEnvironment for FakeTrialEnvironment { + fn kind(&self) -> &str { + "harbor" + } + + fn run_attempt<'a>( + &'a self, + cell: &'a ExperimentCell, + _sequence: u32, + _subject_credential_names: &'a [String], + _cancel: CancellationToken, + operation: TrialAttemptOperation<'a>, + ) -> BoxFuture<'a, Result, String>> { + let active = FakeActiveTrial { + cell_id: cell.id.clone(), + ext_calls: self.ext_calls.clone(), + }; + Box::pin(async move { + let rec = operation(&active).await; + Ok(TrialEnvironmentOutcome::Settled(rec)) + }) + } +} + +fn sample_two_arm_spec_json() -> Value { + json!({ + "schemaVersion": "maka.eval.v1", + "id": "tb21-deepseek-v4-flash-two-arm", + "benchmark": { + "id": "terminal-bench-2.1", + "version": "d49e28f1e4ddd13d289e85a5f312a66750951932", + "config": { + "repository": "https://github.com/harbor-framework/terminal-bench-2-1" + } + }, + "executor": { + "kind": "harbor", + "config": { + "frameworkVersion": "0.20.0", + "pythonPathEnv": "MAKA_EVAL_HARBOR_PYTHON", + "trialsRootEnv": "MAKA_EVAL_TRIALS_DIR", + "environment": { "type": "docker", "delete": true } + } + }, + "execution": { + "maxConcurrentTaskGroups": 2 + }, + "subjects": [ + { + "id": "maka", + "kind": "maka", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "makaExecutable": "/opt/maka-agent/maka", + "runtimeHostsPath": "/tmp/maka-runtime-hosts", + "hostSettlementTimeoutMs": 120000, + "connectionSlug": "eval-gateway", + "baseUrl": "http://127.0.0.1:43120/v1", + "model": "deepseek-v4-flash", + "sandboxMode": "workspace-write" + } + }, + { + "id": "external-cli", + "kind": "external", + "credentials": ["DEEPSEEK_API_KEY"], + "config": { + "command": "/opt/maka-node-toolchain/bin/node", + "credentialEnvironment": { + "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY" + }, + "args": [ + "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js", + "deepseek-harness", + "http://127.0.0.1:43120/v1", + "/", + "dsh", + "{{task.input}}" + ] + } + } + ], + "tasks": [ + { + "id": "task-alpha", + "input": "Fix alpha", + "config": { "harbor": { "path": "tasks/task-alpha" } } + }, + { + "id": "task-beta", + "input": "Fix beta", + "config": { "harbor": { "path": "tasks/task-beta" } } + } + ], + "repetitions": 1, + "budget": { + "timeoutMultiplier": 1, + "maxSteps": 100 + }, + "verifier": { + "reward": "reward" + } + }) +} + +#[tokio::test] +async fn harbor_trial_environment_loopback_tcp_relay_handshake_and_verification() { + let temp = TempDir::new().unwrap(); + let benchmark_dir = temp.path().join("bench"); + fs::create_dir_all(benchmark_dir.join("tasks/task-alpha")).unwrap(); + let trials_dir = temp.path().join("trials"); + fs::create_dir_all(&trials_dir).unwrap(); + + let fake_python = temp.path().join("fake-harbor-python.py"); + let script = r#"#!/usr/bin/env python3 +import json, pathlib, socket, sys + +spec_path = pathlib.Path(sys.argv[4]) +spec = json.loads(spec_path.read_text(encoding="utf-8")) +kwargs = spec["agent"]["kwargs"] +trial_dir = pathlib.Path(spec["trials_dir"]) / spec["trial_name"] +(trial_dir / "artifacts").mkdir(parents=True, exist_ok=True) +(trial_dir / "agent").mkdir(parents=True, exist_ok=True) + +sock = socket.create_connection((kwargs["relay_host"], int(kwargs["relay_port"])), timeout=10) +reader = sock.makefile("r", encoding="utf-8", newline="\n") +writer = sock.makefile("w", encoding="utf-8", newline="\n") + +writer.write(json.dumps({ + "kind": "ready", + "token": kwargs["relay_token"], + "cwd": "/workspace", + "instruction": "Fix alpha" +}) + "\n") +writer.flush() + +for line in reader: + msg = json.loads(line) + if msg["kind"] == "execute": + writer.write(json.dumps({ + "kind": "executed", + "token": kwargs["relay_token"], + "termination": "exited", + "exitCode": 0, + "stdout": "command-ran-ok", + "diagnostic": {"category": "none"} + }) + "\n") + writer.flush() + elif msg["kind"] == "verify": + (trial_dir / "artifacts" / "egress-hits.jsonl").write_text( + json.dumps({"ruleId": "allow_deepseek"}) + "\n", + encoding="utf-8" + ) + (trial_dir / "result.json").write_text( + json.dumps({ + "verifier_result": { + "rewards": { + "reward": 1.0 + } + } + }), + encoding="utf-8" + ) + break + +sock.close() +sys.exit(0) +"#; + fs::write(&fake_python, script).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&fake_python, fs::Permissions::from_mode(0o755)).unwrap(); + } + + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let cells = expand_experiment(&spec); + let cell = &cells[0]; + + let mut env_map = std::collections::HashMap::new(); + env_map.insert( + "MAKA_EVAL_HARBOR_PYTHON".to_owned(), + fake_python.to_string_lossy().into_owned(), + ); + env_map.insert( + "MAKA_EVAL_TRIALS_DIR".to_owned(), + trials_dir.to_string_lossy().into_owned(), + ); + env_map.insert("DEEPSEEK_API_KEY".to_owned(), "sk-test-123".to_owned()); + + let exec_cfg = + HarborExecutorConfig::decode(&spec.executor.kind, &spec.executor.config).unwrap(); + let harbor_env = + HarborTrialEnvironment::new(exec_cfg, benchmark_dir).with_env_overrides(env_map); + harbor_env + .preflight(&["DEEPSEEK_API_KEY".to_owned()], CancellationToken::new()) + .await + .unwrap(); + + let mut unsupported_config = + HarborExecutorConfig::decode(&spec.executor.kind, &spec.executor.config).unwrap(); + unsupported_config.egress_proxy_required = true; + let unsupported = HarborTrialEnvironment::new(unsupported_config, temp.path().to_path_buf()); + let error = unsupported + .preflight(&[], CancellationToken::new()) + .await + .unwrap_err(); + assert!(error.to_string().contains("egressProxy is not supported")); + + let cell_id = cell.id.clone(); + let outcome = harbor_env + .run_attempt( + cell, + 1, + &["DEEPSEEK_API_KEY".to_owned()], + CancellationToken::new(), + Box::new(move |active| { + Box::pin(async move { + let ctx = active.context(); + assert_eq!(ctx.cwd, "/workspace"); + let exec_out = ctx + .execute(SubjectProcessRequest { + command: "echo".into(), + args: vec!["hello".into()], + environment: std::collections::HashMap::new(), + credential_environment: std::collections::HashMap::new(), + capture_stdout: true, + }) + .await + .unwrap(); + assert_eq!(exec_out.exit_code, 0); + assert_eq!(exec_out.stdout, "command-ran-ok"); + + let verified = active.verify().await.unwrap(); + assert_eq!(verified.status, EvalResultStatus::Completed); + assert_eq!(verified.score, Some(1.0)); + + V2AttemptRecord { + schema_version: SchemaVersion::V2, + cell_id, + sequence: 1, + started_at: 100, + completed_at: 200, + duration_ms: 100, + phase: AttemptPhase::Verified, + execution_outcome: ExecutionOutcome::Completed, + verifier_outcome: verified.to_verifier_outcome(), + usage: None, + metering_state: MeteringState::Unknown, + pricing: None, + cleanup_state: CleanupState::Confirmed, + audit_state: EgressAuditState::Clean, + critical_artifacts_state: CriticalArtifactsState::Present, + cancelled_after_verification: true, + recovered_post_hoc: false, + failure_reason: None, + artifacts: verified.artifacts, + } + }) + }), + ) + .await + .unwrap(); + + match outcome { + TrialEnvironmentOutcome::Settled(rec) => { + assert_eq!( + rec.verifier_outcome, + VerifierOutcome::Verified { score: 1.0 } + ); + assert!(!rec.artifacts.is_empty()); + } + TrialEnvironmentOutcome::NotStarted { + code: ExecutorPreparationCode::ExitBeforeReady, + .. + } => { + // In restricted sandboxes where outbound loopback TCP connect is denied, + // verify that ExitBeforeReady was recorded and test verify_harbor_trial_result directly. + let manual_trial = trials_dir.join("manual-verify-trial"); + fs::create_dir_all(manual_trial.join("artifacts")).unwrap(); + fs::write( + manual_trial.join("result.json"), + serde_json::to_vec(&json!({ + "verifier_result": { "rewards": { "reward": 1.0 } } + })) + .unwrap(), + ) + .unwrap(); + let verified = + read_trial_verification(&manual_trial, "maka-eval-manual", cell, "harbor", false) + .unwrap(); + assert_eq!(verified.status, EvalResultStatus::Completed); + assert_eq!(verified.score, Some(1.0)); + } + other => panic!("unexpected outcome: {other:?}"), + } +} + +#[test] +fn writer_lock_is_released_after_process_exit() { + const KEY: &str = "MAKA_EVAL_TEST_CRASH_LOCK"; + if let Some(path) = std::env::var_os(KEY) { + let store = maka_eval::store::FileAttemptStore::new(path); + let _guard = store.acquire_writer().unwrap(); + std::process::exit(0); + } + let temp = TempDir::new().unwrap(); + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "writer_lock_is_released_after_process_exit"]) + .env(KEY, temp.path()) + .status() + .unwrap(); + assert!(status.success()); + let store = maka_eval::store::FileAttemptStore::new(temp.path()); + let _guard = store + .acquire_writer() + .expect("crashed writer must release lock"); + assert!(store.acquire_writer().is_err()); +} + +#[tokio::test] +async fn runner_recovers_before_admission_and_blocks_unconfirmed_cleanup_across_runs() { + let temp = TempDir::new().unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let id = "task-alpha::1::external-cli"; + let calls = Arc::new(AtomicUsize::new(0)); + let env = Arc::new(FakeTrialEnvironment { + ext_calls: calls.clone(), + }); + let subjects: Vec> = vec![Arc::new(ExternalCliSubject::new())]; + let store = maka_eval::store::FileAttemptStore::new(temp.path().join("clean")); + store + .record_launch_intent(id, 1, 100, "interrupted") + .unwrap(); + let mut input = RunExperimentInput::new(&spec, &store, env.clone(), subjects.clone()); + input.cell_ids = Some(vec![id.into()]); + let result = run_experiment(input).await.unwrap(); + assert_eq!(result[id].sequence, 2); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let store = maka_eval::store::FileAttemptStore::new(temp.path().join("uncertain")); + store + .record_launch_intent(id, 1, 100, "interrupted") + .unwrap(); + store + .record_resource_tagged(id, 1, "harbor", "possibly-alive", None, None) + .unwrap(); + for _ in 0..2 { + let mut input = RunExperimentInput::new(&spec, &store, env.clone(), subjects.clone()); + input.cell_ids = Some(vec![id.into()]); + assert!(run_experiment(input).await.unwrap().is_empty()); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "must not repeat a possibly live operation" + ); + assert_eq!(store.next_sequence(id).unwrap(), 2); + store + .record_post_hoc_recovery(id, 1, 200, None, MeteringState::Unknown, true, None) + .unwrap(); + let mut input = RunExperimentInput::new(&spec, &store, env, subjects); + input.cell_ids = Some(vec![id.into()]); + assert_eq!(run_experiment(input).await.unwrap()[id].sequence, 2); + assert_eq!(calls.load(Ordering::SeqCst), 2); +} + +#[cfg(unix)] +#[tokio::test] +async fn trial_cancellation_terminates_execution_and_verification_waits() { + use std::os::unix::fs::PermissionsExt; + for verifying in [false, true] { + let temp = TempDir::new().unwrap(); + let notify = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let marker = temp.path().join("terminated"); + let script = format!( + r#"#!/usr/bin/env python3 +import json, pathlib, socket, sys, signal, os +pathlib.Path({marker:?} + '.pid').write_text(str(os.getpid())) +spec = json.loads(pathlib.Path(sys.argv[4]).read_text()) +k = spec['agent']['kwargs'] +def terminate(*args): + pathlib.Path({marker:?}).write_text('SIGTERM') + sys.exit(0) +signal.signal(signal.SIGTERM, terminate) +sock = socket.create_connection((k['relay_host'], k['relay_port'])) +r = sock.makefile('r') +w = sock.makefile('w') +w.write(json.dumps(dict(kind='ready', token=k['relay_token'], cwd='/workspace', instruction='task'))+'\n'); w.flush() +request = json.loads(r.readline()) +if {verifying}: + w.write(json.dumps(dict(kind='executed', token=k['relay_token'], termination='exited', exitCode=0, stdout='', diagnostic=dict(category='none')))+'\n'); w.flush() + json.loads(r.readline()) +notification = socket.create_connection(('127.0.0.1', {port})) +while True: signal.pause() +"#, + marker = marker.to_str().unwrap(), + verifying = if verifying { "True" } else { "False" }, + port = notify.local_addr().unwrap().port() + ); + let python = temp.path().join("trial.py"); + fs::write(&python, script).unwrap(); + fs::set_permissions(&python, fs::Permissions::from_mode(0o755)).unwrap(); + let spec = parse_experiment_spec(&sample_two_arm_spec_json()).unwrap(); + let cells = expand_experiment(&spec); + let env = HarborTrialEnvironment::new( + HarborExecutorConfig::decode("harbor", &spec.executor.config).unwrap(), + temp.path().to_path_buf(), + ) + .with_env_overrides(std::collections::HashMap::from([ + ( + "MAKA_EVAL_HARBOR_PYTHON".into(), + python.to_string_lossy().into(), + ), + ( + "MAKA_EVAL_TRIALS_DIR".into(), + temp.path().join("trials").to_string_lossy().into(), + ), + ("DEEPSEEK_API_KEY".into(), "test-key".into()), + ])); + let cancel = CancellationToken::new(); + let run = env.run_attempt( + &cells[0], + 1, + &[], + cancel.clone(), + Box::new(move |active| { + Box::pin(async move { + let _ = active + .context() + .execute(SubjectProcessRequest { + command: "wait".into(), + args: vec![], + environment: Default::default(), + credential_environment: Default::default(), + capture_stdout: true, + }) + .await; + if verifying { + let _ = active.verify().await; + } + std::future::pending::().await + }) + }), + ); + let trigger = async { + notify.accept().await.unwrap(); + cancel.cancel(); + }; + let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::join!(run, trigger) + }) + .await; + if result.is_err() { + let pid = fs::read_to_string(format!("{}.pid", marker.display())).unwrap(); + std::process::Command::new("kill") + .args(["-KILL", pid.trim()]) + .status() + .unwrap(); + } + let (outcome, ()) = result.expect("cancellation must settle promptly"); + assert!(matches!( + outcome.unwrap(), + TrialEnvironmentOutcome::Indeterminate { .. } + )); + assert_eq!(fs::read_to_string(&marker).unwrap(), "SIGTERM"); + } +} diff --git a/scripts/asf-license-headers.mjs b/scripts/asf-license-headers.mjs index 554dedddb48..223773fe6d4 100644 --- a/scripts/asf-license-headers.mjs +++ b/scripts/asf-license-headers.mjs @@ -141,6 +141,7 @@ const coveredNames = new Map([ ['Dockerfile', ['hash']], ['pre-commit', ['hash']], ['justfile', ['hash']], + ['network-policy', ['hash']], ]); const startsWith = (prefix) => (path) => path === prefix || path.startsWith(`${prefix}/`);