feat(eval): add Harbor trial environment, experiment runner, and canonical reporting

Cargo.toml

@@ -34,6 +34,7 @@ members = [
     "plugins/workspace/workspace-files/native",
     "plugins/workspace/support",
     "products/maka/acp",
+    "products/eval",
     "plugins/agent/engine",
     "plugins/agent/native",
     "plugins/workspace/workspace-files/patch",
@@ -149,6 +150,7 @@ imagesize = { version = "=0.15.0", default-features = false, features = ["png",
 jsonschema = { version = "0.60.1", default-features = false }
 libc = "0.2.191"
 maka-acp = { path = "products/maka/acp" }
+maka-eval = { path = "products/eval" }
 maka-agent-engine = { path = "plugins/agent/engine" }
 maka-agent-native = { path = "plugins/agent/native" }
 maka-assistant = { path = "plugins/agent/assistant" }

products/eval/Cargo.toml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "maka-eval"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[dependencies]
+base64 = "0.23.1"
+futures-util.workspace = true
+hex.workspace = true
+http-body-util = "0.1.5"
+hyper = { version = "1.12.0", features = ["server", "http1"] }
+hyper-util = { version = "0.1.21", features = ["tokio"] }
+regex.workspace = true
+reqwest.workspace = true
+rusqlite.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+tempfile.workspace = true
+thiserror.workspace = true
+tokio = { workspace = true, features = ["fs", "io-std", "net", "process"] }
+tokio-util.workspace = true
+url = "2.5.8"
+uuid.workspace = true
+
+[target.'cfg(unix)'.dependencies]
+libc.workspace = true

products/eval/README.md

@@ -0,0 +1,32 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# maka-eval
+
+[简体中文](README.zh-CN.md)
+
+Measurement and control plane foundation for multi-arm benchmark experiments (`maka.eval.v1` and `maka.eval.v2`).
+
+## Architecture
+
+- [`contract`](src/contract.rs): Strict `maka.eval.v1` experiment and attempt projection contracts, native `maka.eval.v2` orthogonal attempt dimensions, condition-only `RunFingerprint`, and the semantic difference registry.
+- [`measurement`](src/measurement.rs): Exact 6-counter token accounting (`inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWriteTokens`, `reasoningTokens`, `totalTokens` preserved without summation), non-mutating metering gateway, admission boundary cutoffs, HMAC-authenticated checkpoint recovery, receipt-vs-proxy reconciliation, and standardized `DEEPSEEK_V4_FLASH_COST` pricing.
+- [`store`](src/store.rs): Single-writer append-only experiment journal (`LaunchIntent`, `ResourceTagged`, `Checkpoint`, `Settled`, `PostHocRecovery`), pre-dispatch intent persistence, atomic settlement commits, critical artifact verification, and deterministic state reconstruction.
+
+[Source](src/lib.rs) · [Cargo manifest](Cargo.toml)

products/eval/README.zh-CN.md

@@ -0,0 +1,32 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# maka-eval
+
+[English](README.md)
+
+多臂基准评测实验的测量与控制平面底座（支持 `maka.eval.v1` 与 `maka.eval.v2`）。
+
+## 模块结构
+
+- [`contract`](src/contract.rs)：严格的 `maka.eval.v1` 实验与结果投影契约、原生的 `maka.eval.v2` 正交尝试维度、仅记录运行条件的 `RunFingerprint` 及语义差异注册表。
+- [`measurement`](src/measurement.rs)：精确的 6 计数器 Token 核算（保留 `inputTokens`、`outputTokens`、`cacheReadTokens`、`cacheWriteTokens`、`reasoningTokens`、`totalTokens`，不重算求和）、无篡改计量网关、准入截止边界、HMAC 鉴权检查点恢复、回执与代理日志去重对账及标准 `DEEPSEEK_V4_FLASH_COST` 计价。
+- [`store`](src/store.rs)：单写者只追加实验日志（`LaunchIntent`、`ResourceTagged`、`Checkpoint`、`Settled`、`PostHocRecovery`）、派发前意图落盘、原子结算提交、关键产物校验与确定性状态重建。
+
+[源码](src/lib.rs) · [Cargo 配置](Cargo.toml)

products/eval/harbor/docker-compose-egress-proxy.yaml

@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+services:
+  harbor-docker-egress-control-sidecar:
+    volumes:
+      - type: bind
+        source: ${MAKA_EVAL_NETWORK_POLICY_PATH}
+        target: /usr/local/bin/network-policy
+        read_only: true
+      - maka-eval-egress-ca:/opt/maka-egress:ro
+
+  main:
+    depends_on:
+      maka-eval-mitmproxy:
+        condition: service_healthy
+    # An AF_PACKET socket writes below the IP output hooks the policy installs,
+    # so no nftables rule can constrain it. NET_RAW is what gates that socket.
+    # A task's own compose can add the capability back, which this cannot
+    # prevent — the relay refuses to start the subject when that happens.
+    cap_drop:
+      - NET_RAW
+    volumes:
+      - maka-eval-egress-ca:/opt/maka-egress:ro
+
+  maka-eval-mitmproxy:
+    image: maka-eval-egress-proxy:12.2.3
+    networks:
+      - default
+    volumes:
+      - maka-eval-egress-ca:/opt/maka-egress
+    healthcheck:
+      test:
+        - CMD
+        - python
+        - -c
+        - "import pathlib,socket; assert pathlib.Path('/opt/maka-egress/mitmproxy-ca-cert.pem').is_file(); assert pathlib.Path('/opt/maka-egress/proxy-ipv4').is_file(); socket.create_connection(('127.0.0.1',8080),2).close()"
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+volumes:
+  maka-eval-egress-ca:

products/eval/harbor/egress-proxy/Dockerfile

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM python:3.12-slim
+
+RUN python -m pip install --no-cache-dir mitmproxy==12.2.3
+
+COPY egress_filter.py /opt/maka-eval/egress_filter.py
+COPY egress-proxy/entrypoint.sh /opt/maka-eval/entrypoint.sh
+
+ENTRYPOINT ["/opt/maka-eval/entrypoint.sh"]

products/eval/harbor/egress-proxy/entrypoint.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+STATE_DIR=/opt/maka-egress-state
+CERT_DIR=/opt/maka-egress
+
+mkdir -p "$STATE_DIR" "$CERT_DIR"
+AUDIT_LOG="$STATE_DIR/hits.jsonl"
+# Proxy restarts share one append-only evidence file for the whole trial.
+# Only create it when absent so an earlier process cannot lose recorded hits.
+test -e "$AUDIT_LOG" || touch "$AUDIT_LOG"
+
+# confdir holds the CA private key and this cell's audit log, so it stays
+# container-private and only the certificate reaches the volume the subject
+# mounts. Creating the CA here instead of letting mitmdump create it on startup
+# publishes the certificate before the proxy port opens, so the health gate
+# cannot release `main` against a missing or left-over certificate.
+python -c 'import sys
+from mitmproxy.certs import CertStore
+CertStore.from_store(sys.argv[1], "mitmproxy", 2048)' "$STATE_DIR"
+# Publish by rename so a restart cannot expose a truncated certificate to a
+# subject that is already running against the previous one.
+cp "$STATE_DIR/mitmproxy-ca-cert.pem" "$CERT_DIR/.mitmproxy-ca-cert.pem"
+mv "$CERT_DIR/.mitmproxy-ca-cert.pem" "$CERT_DIR/mitmproxy-ca-cert.pem"
+
+# The subject only needs this address so HTTPS_PROXY can keep using the
+# service hostname after the namespace policy refuses Docker DNS.
+ip="$(getent ahostsv4 "$(hostname)" | awk 'NR == 1 { print $1 }')"
+case "$ip" in
+  "" | 127.*)
+    echo "could not publish Eval egress proxy IPv4" >&2
+    exit 1
+    ;;
+esac
+printf '%s\n' "$ip" > "$CERT_DIR/.proxy-ipv4"
+mv "$CERT_DIR/.proxy-ipv4" "$CERT_DIR/proxy-ipv4"
+
+exec mitmdump \
+  --quiet \
+  --listen-host 0.0.0.0 \
+  --listen-port 8080 \
+  --set block_global=false \
+  --set rawtcp=false \
+  --set confdir="$STATE_DIR" \
+  --scripts /opt/maka-eval/egress_filter.py \
+  "$@"