feat(eval): enforce product parity and runtime binary identity gates

Cargo.toml

@@ -34,6 +34,7 @@ members = [
     "plugins/workspace/workspace-files/native",
     "plugins/workspace/support",
     "products/maka/acp",
+    "products/eval",
     "plugins/agent/engine",
     "plugins/agent/native",
     "plugins/workspace/workspace-files/patch",
@@ -149,6 +150,7 @@ imagesize = { version = "=0.15.0", default-features = false, features = ["png",
 jsonschema = { version = "0.60.1", default-features = false }
 libc = "0.2.191"
 maka-acp = { path = "products/maka/acp" }
+maka-eval = { path = "products/eval" }
 maka-agent-engine = { path = "plugins/agent/engine" }
 maka-agent-native = { path = "plugins/agent/native" }
 maka-assistant = { path = "plugins/agent/assistant" }

products/eval/Cargo.toml

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "maka-eval"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[[bin]]
+name = "maka-eval"
+path = "src/main.rs"
+
+[dependencies]
+maka-agent-facts.workspace = true
+maka-agent-ports.workspace = true
+maka-app-client.workspace = true
+maka-host-runtime.workspace = true
+maka-plugin-core.workspace = true
+maka-providers.workspace = true
+base64 = "0.23.1"
+clap.workspace = true
+futures-util.workspace = true
+hex.workspace = true
+http-body-util = "0.1.5"
+hyper = { version = "1.12.0", features = ["server", "http1"] }
+hyper-util = { version = "0.1.21", features = ["tokio"] }
+regex.workspace = true
+reqwest.workspace = true
+rusqlite.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+tempfile.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["fs", "io-std", "net", "process"] }
+tokio-util.workspace = true
+url = "2.5.8"
+uuid.workspace = true
+
+[target.'cfg(unix)'.dependencies]
+libc.workspace = true

products/eval/README.md

@@ -0,0 +1,51 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# maka-eval
+
+[简体中文](README.zh-CN.md)
+
+Measurement and control plane for multi-arm benchmark experiments (`maka.eval.v1` and `maka.eval.v2`). `maka-eval` owns experiment decoding, task-group scheduling, deadline authority, durable append-only attempt journaling, token/cost metering reconciliation, trial environment orchestration, and deterministic cohort reporting.
+
+## Native Host & Product Parity
+
+Install matching `maka` and `maka-eval` binaries side by side inside the Trial image. Set `subject.config.makaExecutable` to the native `maka` path; `nodePath`, `shimPath`, legacy provider configuration, and benchmark-only profiles (`toolProfile`, `permissionMode: "bypass"`, `collaborationMode`, `orchestrationMode`) are rejected.
+
+Maka subjects are configured strictly through public product interfaces (`provider`, `connectionSlug`, `connectionName`, `baseUrl`, `model`, optional `modelOverride`, `requestHeaders`, `requestBodyOverlay`, `sandboxMode`, `requestPermissions`, `thinkingLevel`, `language`, `workspaceInstructions`, and `apiKeyEnvironment`). The adapter resolves Maka's actual outbound model route from the declared endpoint before attaching the local metering gateway, initializes an isolated Root via `maka host init --settings-stdin`, applies only declared public Settings/Session configurations, and submits tasks through the production composer draft/receipt client without prompt injection, tool pruning, or synthetic model overrides. The non-mutating local metering gateway records provider usage while forwarding outbound JSON bodies verbatim to the product-resolved route, and records the executed `maka` binary path and SHA-256 digest in settled runtime artifacts. Pending interactive approvals or questions are never auto-bypassed in unattended runs; they settle explicitly as `unattended-interaction-blocked`, and host/environment constraints are reported as `environment-limitation`.
+
+Product parity is a mandatory gate for `M3` acceptance and `M5` production delivery (`RunFingerprint` documents run conditions only and does not establish parity):
+- `maka-eval parity <spec.json> [--subject <id>] [--out <report.json>]` preserves original declared endpoint semantics on the reference path while driving `maka-eval`'s entry point against a local probe under identical inputs, verifying committed `RunSpec`/`RequestSpec` facts, resolved upstream target routes, and wire payloads across product composition, effective instructions, tool exposure, and execution policies.
+- `maka-eval report <spec.json> --out <dir> --milestone m3|m5 --parity-report <report.json>` enforces parity, confirms the executed Maka binary identity from settled runtime execution evidence (supporting isolated container paths while failing closed when runtime evidence is absent or mismatched), and requires complete non-indeterminate settlement for `m5`.
+
+`egressProxy` currently fails preflight because the native executor does not provision the required sidecar and mounts. Unconfirmed cleanup blocks further admission across runs until a `PostHocRecovery` entry confirms disposal. Both report versions read committed journal records; result JSON files are no longer a separate authority. Records without pricing evidence retain unknown cost.
+
+For focused verification, build `maka-cli` first and supply its path as runtime `CARGO_BIN_EXE_maka` when running `cargo test -p maka-eval`. CI supplies and archives the native helper. The real Host and parity tests use local fake providers and no paid model calls.
+
+## Architecture
+
+- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions (lifecycle phase, execution outcome, metering state, cleanup state), condition-only `RunFingerprint`, and the semantic difference registry.
+- [`parity`](src/parity.rs): End-to-end Product Parity verification comparing normal Maka client execution against `maka-eval` across product composition, effective instructions, tool exposure (Code Mode, skills, schemas), and execution policies.
+- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), non-mutating metering gateway, admission boundary cutoffs, HMAC-authenticated checkpoint recovery, receipt-vs-proxy reconciliation without double-counting, and standardized `DEEPSEEK_V4_FLASH_COST` pricing.
+- [`store`](src/store.rs): Single-writer append-only experiment journal (`LaunchIntent`, `ResourceTagged`, `Checkpoint`, `Settled`, `PostHocRecovery`), pre-dispatch intent persistence, atomic settlement commits, critical artifact verification, and deterministic state reconstruction.
+- [`runner`](src/runner.rs): Task-group admission (`maxConcurrentTaskGroups`), shared preparation cache boundary lock, phase-scoped deadline authority, bounded retry policy, earliest-valid-attempt canonical selection, and crash recovery.
+- [`environment`](src/environment.rs): Per-attempt `TrialEnvironment` abstraction and version-pinned Harbor / Pier bridge with 1:1 attempt-to-trial mapping.
+- [`subject`](src/subject.rs): Subject adapters for the Rust Maka Host (`maka host init --settings-stdin --root <ROOT_DIR>` isolated root bootstrap, public Settings/Session client execution, explicit unattended interaction & environment limitation reporting, durable receipt journal, and transcript/usage settlement) and external CLI harnesses (`exit-code` and `protocol-v1` bounded result frames).
+- [`report`](src/report.rs): Deterministic journal projection to `v1` and `v2`, paired cohort summaries, `M3`/`M5` milestone parity acceptance gates, and 3-verdict differential oracle evaluation (`v1_parses`, `v1_matches_oracle`, `conditions_comparable`).
+
+[Source](src/lib.rs) · [Cargo manifest](Cargo.toml)

products/eval/README.zh-CN.md

@@ -0,0 +1,51 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# maka-eval
+
+[English](README.md)
+
+多臂基准评测实验的测量与控制平面（支持 `maka.eval.v1` 与 `maka.eval.v2`）。`maka-eval` 负责实验契约解析、任务组并发调度、阶段截止时间权威、持久化只追加（append-only）尝试日志、Token 与成本计量对账、Trial 环境编排及确定性对照报告生成。
+
+## 原生 Host 与产品等价性（Product Parity）
+
+在 Trial 镜像内将匹配版本的 `maka` 和 `maka-eval` 放在同一目录，`subject.config.makaExecutable` 指向原生 `maka`。拒绝 `nodePath`、`shimPath`、旧 provider 配置及仅面向评测的专用参数（`toolProfile`、`permissionMode: "bypass"`、`collaborationMode`、`orchestrationMode`）。
+
+Maka 被测对象仅通过公开产品接口配置（`provider`、`connectionSlug`、`connectionName`、`baseUrl`、`model`、可选 `modelOverride`、`requestHeaders`、`requestBodyOverlay`、`sandboxMode`、`requestPermissions`、`thinkingLevel`、`language`、`workspaceInstructions` 与 `apiKeyEnvironment`）。适配器先按声明的原始端点解析产品实际模型路由再接入本地计量转发，通过 `maka host init --settings-stdin` 初始化隔离的 Root，仅应用显式声明的公开 Settings/Session 配置，并通过生产客户端的 composer 草稿与回执协议提交任务，不注入评测提示词、不裁剪或替换工具、不注入合成模型参数。本地计量网关以只读方式统计提供商用量并将请求体原样转发至产品解析出的目标路由，同时在结算产物中记录实际运行的 `maka` 二进制路径与 SHA-256 摘要。无人值守执行不会静默绕过审批或交互请求，遇到挂起交互时以 `unattended-interaction-blocked` 显式结算；工作区或环境受限时以 `environment-limitation` 显式上报。
+
+产品等价性是 `M3` 验收与 `M5` 生产交付的前置准入门槛（`RunFingerprint` 仅记录运行条件，不构成产品等价性证明）：
+- `maka-eval parity <spec.json> [--subject <id>] [--out <report.json>]` 在相同版本、公开配置、工作区与任务输入下保留参考路径的原始端点语义，同时驱动 `maka-eval` 入口连接本地探针，比对落盘的 `RunSpec`/`RequestSpec` 事实、解析后的上游目标路由与出站报文，验证产品组合、生效指令、工具暴露及执行策略等价。
+- `maka-eval report <spec.json> --out <dir> --milestone m3|m5 --parity-report <report.json>` 强制校验等价性报告，并依据已结算的运行时执行证据确认实际 Maka 二进制身份（支持隔离容器路径，缺少或摘要不匹配时显式失败；`m5` 同时要求全部单元完成确定性结算）。
+
+当前原生执行器尚未配置代理侧车与挂载，因此 `egressProxy` 会在预检失败。清理未确认的尝试会跨次运行阻断准入，直到 `PostHocRecovery` 记录确认资源已处置。两版报告均读取已提交日志，不再将结果 JSON 文件作为另一权威；缺少定价依据的记录保持成本未知。
+
+定向验证前先构建 `maka-cli`，运行 `cargo test -p maka-eval` 时通过运行时环境变量 `CARGO_BIN_EXE_maka` 指定其路径。CI 负责提供和归档原生辅助程序。真实 Host 与等价性测试使用本地假提供商，不产生付费模型调用。
+
+## 模块结构
+
+- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度（生命周期阶段、执行结果、计量完整性、环境清理状态）、仅记录运行条件的 `RunFingerprint` 及语义差异注册表。
+- [`parity`](src/parity.rs)：端到端产品等价性验证，对比常规 Maka 客户端与 `maka-eval` 在产品组合、生效指令、工具暴露（Code Mode、内置 skill、工具 schema）及执行策略四项维度的等价性。
+- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）、无篡改计量网关、准入截止边界、HMAC 鉴权检查点恢复、回执与代理日志去重对账及标准 `DEEPSEEK_V4_FLASH_COST` 计价。
+- [`store`](src/store.rs)：单写者只追加实验日志（`LaunchIntent`、`ResourceTagged`、`Checkpoint`、`Settled`、`PostHocRecovery`）、派发前意图落盘、原子结算提交、关键产物校验与确定性状态重建。
+- [`runner`](src/runner.rs)：任务组并发准入（`maxConcurrentTaskGroups`）、共享预备缓存边界锁、分阶段截止时间控制、有界重试策略、最早有效尝试（`sequence >= 1`）规范结果选择及崩溃恢复。
+- [`environment`](src/environment.rs)：单次尝试级 `TrialEnvironment` 抽象及版本锁定的 Harbor / Pier 桥接（1:1 尝试与 Trial 映射）。
+- [`subject`](src/subject.rs)：被测系统适配器，包含 Rust Maka Host 适配器（通过 `maka host init --settings-stdin --root <ROOT_DIR>` 引导隔离的 Host Root、公开 Settings/Session 客户端执行、无人值守交互阻断与环境限制显式上报、持久化回执日志与历史/用量结算）和外部 CLI 适配器（`exit-code` 与 `protocol-v1` 有界结果帧）。
+- [`report`](src/report.rs)：从日志确定性重建 `v1` 与 `v2` 结果、生成成对队列汇总报告、`M3`/`M5` 里程碑等价性准入校验及三项裁决差分对比报告（`v1_parses`、`v1_matches_oracle`、`conditions_comparable`）。
+
+[源码](src/lib.rs) · [Cargo 配置](Cargo.toml)

products/eval/harbor/docker-compose-egress-proxy.yaml

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+services:
+  harbor-docker-egress-control-sidecar:
+    volumes:
+      - type: bind
+        source: ${MAKA_EVAL_NETWORK_POLICY_PATH}
+        target: /usr/local/bin/network-policy
+        read_only: true
+      - maka-eval-egress-ca:/opt/maka-egress:ro
+
+  main:
+    depends_on:
+      maka-eval-mitmproxy:
+        condition: service_healthy
+    # An AF_PACKET socket writes below the IP output hooks the policy installs,
+    # so no nftables rule can constrain it. NET_RAW is what gates that socket.
+    # A task's own compose can add the capability back, which this cannot
+    # prevent — the relay refuses to start the subject when that happens.
+    cap_drop:
+      - NET_RAW
+    volumes:
+      - maka-eval-egress-ca:/opt/maka-egress:ro
+
+  maka-eval-mitmproxy:
+    image: maka-eval-egress-proxy:12.2.3
+    networks:
+      - default
+    volumes:
+      - maka-eval-egress-ca:/opt/maka-egress
+    healthcheck:
+      test:
+        - CMD
+        - python
+        - -c
+        - "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); assert pathlib.Path('/opt/maka-egress/proxy-ipv4').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()"
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+volumes:
+  maka-eval-egress-ca:

products/eval/harbor/egress-proxy/Dockerfile

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM python:3.12-slim
+
+RUN python -m pip install --no-cache-dir mitmproxy==12.2.3
+
+COPY egress_filter.py /opt/maka-eval/egress_filter.py
+COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh
+
+ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"]