From 135432b7030fec37d4632f407e027f39b337a950 Mon Sep 17 00:00:00 2001 From: "Petrov, Nikolay A" Date: Sun, 15 Nov 2026 21:14:36 -0800 Subject: [PATCH 01/14] feat(subjects): skill_evidence results say when they may be compared Each result carries metric_spec, the verdict_rule its verdict follows and protocol {id, version, digest}: sha256 over the canonical JSON of the resolved measurement inputs only. A missing input or an alias such as 'latest' goes to protocol.unresolved, outside the digest. comparable and non_comparable give the reasons a result cannot be compared. dokimi.subjects.evidence_check holds the consumer checks; three conformance fixtures pin them. Rollup and scorecard cells now pass through the judge. Signed-off-by: Petrov, Nikolay A --- ...61115_120000_napetrov_evidence-protocol.md | 8 + dokimi/dokimi/eval/matrix_rollup.py | 3 + dokimi/dokimi/schemas/matrix_rollup.v1.json | 18 ++ dokimi/dokimi/schemas/skill_evidence.v1.json | 100 ++++++++++ dokimi/dokimi/subjects/evidence.py | 43 +++- dokimi/dokimi/subjects/evidence_check.py | 169 ++++++++++++++++ dokimi/dokimi/subjects/scorecard.py | 3 + .../fixtures/evidence_conformance/README.md | 11 ++ .../judge_snapshot_newer.json | 80 ++++++++ .../judge_snapshot_older.json | 80 ++++++++ .../verdict_with_rule.json | 80 ++++++++ .../verdict_without_rule.json | 74 +++++++ dokimi/tests/test_evidence_protocol.py | 187 ++++++++++++++++++ 13 files changed, 852 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md create mode 100644 dokimi/dokimi/subjects/evidence_check.py create mode 100644 dokimi/tests/fixtures/evidence_conformance/README.md create mode 100644 dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json create mode 100644 dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json create mode 100644 dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json create mode 100644 dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json create mode 100644 dokimi/tests/test_evidence_protocol.py diff --git a/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md b/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md new file mode 100644 index 000000000..e1ad20300 --- /dev/null +++ b/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md @@ -0,0 +1,8 @@ + + +### Added + +- **`subjects evidence`: each result says when it may be compared.** A result now carries `metric_spec` (unit, range, direction, judged), the `verdict_rule` its verdict follows (`p_adjusted` below `alpha`), and `protocol {id, version, digest}`: the digest is `sha256` over the canonical JSON of the resolved measurement inputs (judge snapshot, sampling, judge prompt hash, eval-set digest, repeats, test, correction, thresholds). A missing input or an alias such as `latest` is listed in `protocol.unresolved` and left out of the digest. `comparable`/`non_comparable` say why a result cannot be compared. `dokimi.subjects.evidence_check` holds the consumer checks, with conformance fixtures in `tests/fixtures/evidence_conformance`. Matrix-rollup and scorecard cells now carry the cell's `judge_model`, `judge_provider` and `judge_context`. Today's runs do not record the judge prompt hash, sampling or repeats, so their results list those as unresolved. diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index 35dddc612..eb8164621 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -31,6 +31,9 @@ def build_matrix_rollup( "plugin_set": data.get("plugin_set", "none"), "plugin_set_id": data.get("plugin_set_id"), "baseline_arm": data.get("baseline_arm"), + "judge_model": data.get("judge_model"), + "judge_provider": data.get("judge_provider"), + "judge_context": data.get("judge_context"), "summary": data.get("summary", {}), "cost_summary": data.get("cost_summary", {}), } diff --git a/dokimi/dokimi/schemas/matrix_rollup.v1.json b/dokimi/dokimi/schemas/matrix_rollup.v1.json index 3628a6446..9297aded6 100644 --- a/dokimi/dokimi/schemas/matrix_rollup.v1.json +++ b/dokimi/dokimi/schemas/matrix_rollup.v1.json @@ -90,6 +90,24 @@ "string", "null" ] + }, + "judge_model": { + "type": [ + "string", + "null" + ] + }, + "judge_provider": { + "type": [ + "string", + "null" + ] + }, + "judge_context": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": true diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json index ecbe48172..e3913c186 100644 --- a/dokimi/dokimi/schemas/skill_evidence.v1.json +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -177,6 +177,13 @@ ] }, "judge_model": { + "type": [ + "string", + "null" + ], + "description": "judge snapshot id when the run recorded one, else the configured judge model" + }, + "judge_provider": { "type": [ "string", "null" @@ -189,6 +196,30 @@ "type": "string", "description": "dokimi's mean judge aggregate score per answer (higher is better); delta is arm minus baseline" }, + "metric_spec": { + "type": "object", + "description": "what the metric is; a verdict needs unit and higher_is_better to be comparable", + "properties": { + "id": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "range": { + "type": "array", + "items": { + "type": "number" + } + }, + "higher_is_better": { + "type": "boolean" + }, + "judged": { + "type": "boolean" + } + } + }, "n": { "type": [ "integer", @@ -245,12 +276,81 @@ "null" ] }, + "verdict_rule": { + "type": "object", + "description": "how verdict follows from delta and p_value; a consumer recomputes it", + "properties": { + "basis": { + "enum": [ + "p_adjusted" + ] + }, + "alpha": { + "type": "number" + }, + "test": { + "type": "string" + }, + "correction": { + "type": [ + "string", + "null" + ] + } + } + }, "verdict": { "enum": [ "improved", "regressed", "inconclusive" ] + }, + "protocol": { + "type": "object", + "description": "the measurement: digest is sha256 of the canonical JSON (sorted keys, no whitespace, UTF-8) of resolved; missing or alias inputs are listed in unresolved, outside the digest", + "required": [ + "id", + "version", + "digest" + ], + "properties": { + "id": { + "type": "string" + }, + "version": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "resolved": { + "type": "object" + }, + "unresolved": { + "type": "array", + "items": { + "type": "string" + } + }, + "aliases": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "comparable": { + "type": "boolean" + }, + "non_comparable": { + "type": "array", + "items": { + "type": "string" + }, + "description": "why this result cannot be compared or shown with its verdict" } } } diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index 8f76cf478..528981070 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -18,6 +18,8 @@ from typing import Any from dokimi import __version__ +from dokimi.eval.sign_flip import ALPHA, METHOD +from dokimi.subjects.evidence_check import build_protocol, comparability #: The arm metric the awareness summary carries per arm (``summary.per_arm[arm]``). METRIC = "avg_aggregate" @@ -39,7 +41,7 @@ def build_skill_evidence(scorecard: dict[str, Any], *, source: str | None = None if baseline is None: warnings.append(f"matrix cell {cell.get('matrix_cell')!r}: no single baseline arm, skipped") continue - results.extend(_result(cell, arm, row) for arm, row in per_arm.items() if arm != baseline) + results.extend(_result(cell, arm, row, suite) for arm, row in per_arm.items() if arm != baseline) if subject.get("distribution_digest") is None: warnings.append("subject has no distribution_digest (not a SKILL.md, a skill with resources, " "or a non-local ref): a client cannot match this document by digest") @@ -84,12 +86,37 @@ def _baseline_arm(cell: dict[str, Any], per_arm: dict[str, Any]) -> str | None: return candidates[0] if len(candidates) == 1 else None -def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, Any]: +#: What ``METRIC`` is: a 0-100 judge score per answer. +METRIC_SPEC = {"id": METRIC, "unit": "score_points", "range": [0, 100], + "higher_is_better": True, "judged": True} + + +def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], + suite: dict[str, Any] | None = None) -> dict[str, Any]: # Every statistic is the paired test's, so n, delta, CI and p describe the same questions. # The per-arm means are left out: their difference is not the delta once pairs drop out. sig = row.get("significance") or {} delta = sig.get("delta_mean") if sig else None - return { + correction = sig.get("correction") + judge = cell.get("judge_snapshot") or cell.get("judge_model") + # The rule the verdict below applies, so a consumer can recompute it from delta and p_value. + rule = {"basis": "p_adjusted", "alpha": ALPHA, "test": METHOD, "correction": correction} + # Only resolved values enter the digest; what the scorecard does not record stays unresolved. + protocol = build_protocol({ + "eval_set_digest": (suite or {}).get("question_set_hash"), + "metric": METRIC, + "judge_model": judge, + "judge_provider": cell.get("judge_provider"), + "judge_context": cell.get("judge_context"), + "judge_sampling": cell.get("judge_sampling"), + "judge_prompt_sha256": cell.get("judge_prompt_sha256"), + "repeats": cell.get("repeats"), + "test": METHOD, + "correction": correction, + "alpha": ALPHA, + "ci_confidence": sig.get("ci_confidence"), + }) + result = { "product": cell.get("product"), "matrix_cell": cell.get("matrix_cell"), "model": cell.get("model"), @@ -97,9 +124,11 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, An "harness": cell.get("harness"), # Absent until a run records the effort it applied; null means "not recorded", not "default". "effort": cell.get("effort"), - "judge_model": cell.get("judge_model"), + "judge_model": judge, + "judge_provider": cell.get("judge_provider"), "arm": arm, "metric": METRIC, + "metric_spec": dict(METRIC_SPEC), "n": sig.get("n"), "delta": delta, "ci": None if sig.get("ci_lo") is None else { @@ -108,8 +137,14 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, An }, "p_value": sig.get("p_adjusted"), "p_correction": sig.get("correction"), + "verdict_rule": rule, "verdict": _verdict(delta, sig), + "protocol": protocol, } + reasons = comparability(result) + result["comparable"] = not reasons + result["non_comparable"] = reasons + return result def _verdict(delta: float | None, sig: dict[str, Any]) -> str: diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py new file mode 100644 index 000000000..d73560b3b --- /dev/null +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -0,0 +1,169 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""When two ``skill_evidence.v1`` results may be compared: the protocol digest and the consumer checks. + +A subject digest pins the bytes of a skill, not how they were measured, so two results about the +same digest can still disagree on judge, sampling, prompt or thresholds. ``protocol.digest`` names +the measurement: ``sha256`` of the canonical JSON of the *resolved* inputs that produced the number. +An input that is missing, or that names an alias (``latest``, ``default``), is left out of the +digest and listed in ``protocol.unresolved``: an alias resolves differently over time, so hashing +it would give two different measurements one digest. + +The consumer side is :func:`comparability` (can this one result be merged or shown with a verdict +at all) and :func:`compare` (may these two be put side by side). The conformance fixtures under +``tests/fixtures/evidence_conformance`` pin both. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Iterable, Mapping + +#: What dokimi's awareness arms measure; a change to what the digest covers bumps the version. +PROTOCOL_ID = "dokimi.awareness" +PROTOCOL_VERSION = "1" + +#: A value that names a moving target rather than one snapshot. Matched as a whole trailing token, +#: so ``claude-sonnet-5-5`` is resolved and ``claude-sonnet-latest`` / ``gpt:default`` are not. +_ALIAS = re.compile(r"(?:^|[-_.:@/])(?:latest|default|auto|stable|current)$", re.IGNORECASE) + +VERDICTS = ("improved", "regressed", "inconclusive") + + +def canonical_json(value: Any) -> bytes: + """Sorted keys, no whitespace, UTF-8, no NaN: the bytes the protocol digest is taken over.""" + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, + allow_nan=False).encode("utf-8") + + +def protocol_digest(resolved: Mapping[str, Any]) -> str: + return "sha256:" + hashlib.sha256(canonical_json(dict(resolved))).hexdigest() + + +def is_alias(value: Any) -> bool: + return isinstance(value, str) and bool(_ALIAS.search(value.strip())) + + +def build_protocol(inputs: Mapping[str, Any]) -> dict[str, Any]: + """``protocol`` for one result: id, version, digest over the resolved inputs, and what was not. + + An alias stays visible under ``aliases`` for a reader, outside the digest. + """ + resolved: dict[str, Any] = {} + unresolved: list[str] = [] + aliases: dict[str, str] = {} + for key in sorted(inputs): + value = inputs[key] + if value is None: + unresolved.append(key) + elif is_alias(value): + unresolved.append(key) + aliases[key] = value + else: + resolved[key] = value + protocol: dict[str, Any] = { + "id": PROTOCOL_ID, + "version": PROTOCOL_VERSION, + "digest": protocol_digest(resolved), + "resolved": resolved, + "unresolved": unresolved, + } + if aliases: + protocol["aliases"] = aliases + return protocol + + +def recompute_verdict(result: Mapping[str, Any]) -> str | None: + """The verdict ``verdict_rule`` gives for this result's numbers; None when the rule is unusable. + + The one basis today is ``p_adjusted``: a direction only when the corrected p is below ``alpha`` + and the delta is not zero; which direction is "improved" comes from ``higher_is_better``. + """ + rule = result.get("verdict_rule") + spec = result.get("metric_spec") + if not isinstance(rule, Mapping) or not isinstance(spec, Mapping): + return None + alpha, better_up = rule.get("alpha"), spec.get("higher_is_better") + if rule.get("basis") != "p_adjusted" or not _number(alpha) or not isinstance(better_up, bool): + return None + delta, p = result.get("delta"), result.get("p_value") + if not _number(delta) or not _number(p) or p >= alpha or delta == 0: + return "inconclusive" + return "improved" if (delta > 0) == better_up else "regressed" + + +def comparability(result: Mapping[str, Any]) -> list[str]: + """Why this result cannot be merged with others or shown with its verdict; empty when it can. + + A non-comparable result can still be listed. It is not "neutral": it says nothing either way. + """ + reasons: list[str] = [] + spec = result.get("metric_spec") if isinstance(result.get("metric_spec"), Mapping) else {} + if result.get("verdict") is not None: + if not isinstance(result.get("verdict_rule"), Mapping): + reasons.append("verdict without verdict_rule") + if not spec.get("unit"): + reasons.append("verdict without metric_spec.unit") + if not isinstance(spec.get("higher_is_better"), bool): + reasons.append("verdict without metric_spec.higher_is_better") + if not reasons: + expected = recompute_verdict(result) + if expected is None: + reasons.append("verdict_rule cannot be applied") + elif expected != result.get("verdict"): + reasons.append(f"verdict {result.get('verdict')!r} does not follow verdict_rule ({expected!r})") + if spec.get("judged") is not False and not result.get("judge_model"): + reasons.append("judged result without judge_model") + protocol = result.get("protocol") + if isinstance(protocol, Mapping) and isinstance(protocol.get("resolved"), Mapping): + if protocol_digest(protocol["resolved"]) != protocol.get("digest"): + reasons.append("protocol.digest does not match protocol.resolved") + return reasons + + +#: Without a protocol on both sides, the fields that must be equal instead. +_FALLBACK_KEYS = ("harness", "judge_model", "metric", "verdict_rule") + + +def compare(result: Mapping[str, Any], reference: Mapping[str, Any]) -> list[str]: + """Why ``result`` cannot be compared with ``reference``; empty when it can.""" + reasons = [f"result: {r}" for r in comparability(result)] + reasons += [f"reference: {r}" for r in comparability(reference)] + if result.get("metric") != reference.get("metric"): + reasons.append("metric differs") + mine, theirs = result.get("protocol"), reference.get("protocol") + if isinstance(mine, Mapping) and isinstance(theirs, Mapping): + if (mine.get("id"), mine.get("version")) != (theirs.get("id"), theirs.get("version")): + reasons.append("protocol id or version differs") + elif mine.get("digest") != theirs.get("digest"): + changed = sorted(k for k in set(_resolved(mine)) | set(_resolved(theirs)) + if _resolved(mine).get(k) != _resolved(theirs).get(k)) + reasons.append("protocol digest differs" + (f" ({', '.join(changed)})" if changed else "")) + unresolved = sorted(set(mine.get("unresolved") or []) | set(theirs.get("unresolved") or [])) + if unresolved: + reasons.append(f"protocol inputs not resolved: {', '.join(unresolved)}") + else: + reasons += [f"{key} differs" for key in _FALLBACK_KEYS if result.get(key) != reference.get(key)] + return reasons + + +def mark_against(results: Iterable[Mapping[str, Any]], + reference: Mapping[str, Any]) -> list[dict[str, Any]]: + """Copies of ``results``, each with ``comparable`` and ``non_comparable`` set against ``reference``.""" + out = [] + for result in results: + reasons = compare(result, reference) + out.append({**result, "comparable": not reasons, "non_comparable": reasons}) + return out + + +def _resolved(protocol: Mapping[str, Any]) -> Mapping[str, Any]: + resolved = protocol.get("resolved") + return resolved if isinstance(resolved, Mapping) else {} + + +def _number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 0dfe243d9..ce37e16bb 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -60,6 +60,9 @@ def build_subject_scorecard( "provider": cell.get("provider"), "harness": cell.get("harness"), "baseline_arm": cell.get("baseline_arm"), + "judge_model": cell.get("judge_model"), + "judge_provider": cell.get("judge_provider"), + "judge_context": cell.get("judge_context"), "plugin_set": cell.get("plugin_set"), "summary": cell.get("summary", {}), "cost_summary": cell.get("cost_summary", {}), diff --git a/dokimi/tests/fixtures/evidence_conformance/README.md b/dokimi/tests/fixtures/evidence_conformance/README.md new file mode 100644 index 000000000..9150ee5e8 --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/README.md @@ -0,0 +1,11 @@ +# skill_evidence.v1 conformance fixtures + +Each file is a valid `skill_evidence.v1` document with one result. A consumer passes when: + +| File | Expectation | +|---|---| +| `verdict_without_rule.json` | The result has a `verdict` but no `verdict_rule`: mark it non-comparable. | +| `verdict_with_rule.json` | Recomputing the verdict from `delta`, `p_value`, `verdict_rule` and `metric_spec` gives the stated `improved`. | +| `judge_snapshot_older.json`, `judge_snapshot_newer.json` | The same protocol resolved against judge snapshots `…-20260901` and `…-20261101`: the `protocol.digest` values differ, and the newer result is non-comparable to the older one. | + +`protocol.digest` is `sha256:` + the hex sha256 of `protocol.resolved` as JSON with sorted keys, no whitespace, UTF-8 and no NaN. `tests/test_evidence_protocol.py` runs these checks against `dokimi.subjects.evidence_check`. diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json new file mode 100644 index 000000000..e855d3fd1 --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json @@ -0,0 +1,80 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20261101", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:5ff2ce8ba0a051a86dc397bc7d215e679b992d5a7e5fcb94ef8bb692dea8678a", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20261101", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "1" + }, + "provider": "amazon-bedrock", + "verdict": "improved", + "verdict_rule": { + "alpha": 0.05, + "basis": "p_adjusted", + "correction": "holm", + "test": "sign_flip_inversion" + } + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json new file mode 100644 index 000000000..55fc7c3bc --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json @@ -0,0 +1,80 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:79edd29b1fa49fb8d90ce01893879c4d6b61813d8c54199c5023021501263210", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "1" + }, + "provider": "amazon-bedrock", + "verdict": "improved", + "verdict_rule": { + "alpha": 0.05, + "basis": "p_adjusted", + "correction": "holm", + "test": "sign_flip_inversion" + } + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json new file mode 100644 index 000000000..55fc7c3bc --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json @@ -0,0 +1,80 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:79edd29b1fa49fb8d90ce01893879c4d6b61813d8c54199c5023021501263210", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "1" + }, + "provider": "amazon-bedrock", + "verdict": "improved", + "verdict_rule": { + "alpha": 0.05, + "basis": "p_adjusted", + "correction": "holm", + "test": "sign_flip_inversion" + } + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json new file mode 100644 index 000000000..5dc5ba267 --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json @@ -0,0 +1,74 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:79edd29b1fa49fb8d90ce01893879c4d6b61813d8c54199c5023021501263210", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "1" + }, + "provider": "amazon-bedrock", + "verdict": "improved" + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py new file mode 100644 index 000000000..677c1d655 --- /dev/null +++ b/dokimi/tests/test_evidence_protocol.py @@ -0,0 +1,187 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""skill_evidence.v1 comparability: the protocol digest and the three conformance fixtures.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from dokimi.artifacts import load_artifact +from dokimi.subjects.evidence import build_skill_evidence +from dokimi.subjects.evidence_check import ( + build_protocol, compare, comparability, mark_against, protocol_digest, recompute_verdict, +) + +CONFORMANCE = Path(__file__).parent / "fixtures" / "evidence_conformance" + + +def _fixture(name: str) -> dict: + # Loaded the way a client loads a published file, so each fixture is a valid document too. + return load_artifact("skill_evidence", CONFORMANCE / name)["results"][0] + + +# --- the three conformance fixtures ------------------------------------------------------------- + +def test_fixture_a_verdict_without_its_rule_is_non_comparable(): + result = _fixture("verdict_without_rule.json") + + assert result["verdict"] == "improved" + assert comparability(result) == ["verdict without verdict_rule"] + assert recompute_verdict(result) is None + + +def test_fixture_a_verdict_with_its_rule_recomputes_to_the_same_verdict(): + result = _fixture("verdict_with_rule.json") + + assert comparability(result) == [] + assert recompute_verdict(result) == result["verdict"] == "improved" + + +def test_fixture_two_judge_snapshots_give_two_digests_and_the_newer_is_not_comparable(): + older, newer = _fixture("judge_snapshot_older.json"), _fixture("judge_snapshot_newer.json") + + assert older["protocol"]["id"] == newer["protocol"]["id"] + assert older["protocol"]["version"] == newer["protocol"]["version"] + assert older["protocol"]["digest"] != newer["protocol"]["digest"] + assert compare(older, older) == [] + assert compare(newer, older) == ["protocol digest differs (judge_model)"] + [marked] = mark_against([newer], older) + assert marked["comparable"] is False and marked["non_comparable"] + + +# --- the digest --------------------------------------------------------------------------------- + +def test_the_digest_is_sha256_of_sorted_compact_utf8_json(): + resolved = {"repeats": 3, "judge_model": "j-2026", "judge_sampling": {"temperature": 0.0}} + expected = hashlib.sha256( + b'{"judge_model":"j-2026","judge_sampling":{"temperature":0.0},"repeats":3}').hexdigest() + + assert protocol_digest(resolved) == "sha256:" + expected + assert build_protocol(resolved)["digest"] == "sha256:" + expected + + +@pytest.mark.parametrize("alias", ["latest", "claude-sonnet-latest", "gpt:default", "model@stable", "AUTO"]) +def test_an_alias_stays_out_of_the_digest_and_is_listed_as_unresolved(alias): + protocol = build_protocol({"judge_model": alias, "repeats": 3}) + + assert protocol["resolved"] == {"repeats": 3} + assert protocol["unresolved"] == ["judge_model"] + assert protocol["aliases"] == {"judge_model": alias} + assert protocol["digest"] == protocol_digest({"repeats": 3}) + + +@pytest.mark.parametrize("snapshot", ["claude-sonnet-5-5", "claude-sonnet-5-5-20260901", "gpt-5.6"]) +def test_a_snapshot_id_is_resolved(snapshot): + assert build_protocol({"judge_model": snapshot})["resolved"] == {"judge_model": snapshot} + + +def test_a_missing_input_is_unresolved_and_two_unresolved_protocols_are_not_comparable(): + with_rule = _fixture("verdict_with_rule.json") + vague = {**with_rule, "protocol": build_protocol({**with_rule["protocol"]["resolved"], + "judge_prompt_sha256": None})} + + assert vague["protocol"]["unresolved"] == ["judge_prompt_sha256"] + assert compare(vague, vague) == ["protocol inputs not resolved: judge_prompt_sha256"] + + +def test_a_tampered_resolved_block_is_caught(): + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["repeats"] = 1 + + assert comparability(result) == ["protocol.digest does not match protocol.resolved"] + + +# --- the per-result rules ----------------------------------------------------------------------- + +@pytest.mark.parametrize("drop, reason", [ + ("unit", "verdict without metric_spec.unit"), + ("higher_is_better", "verdict without metric_spec.higher_is_better"), +]) +def test_a_verdict_needs_the_metric_unit_and_direction(drop, reason): + result = _fixture("verdict_with_rule.json") + del result["metric_spec"][drop] + + assert comparability(result) == [reason] + + +def test_a_judged_result_needs_its_judge(): + result = {**_fixture("verdict_with_rule.json"), "judge_model": None} + + assert comparability(result) == ["judged result without judge_model"] + + +def test_a_verdict_that_contradicts_its_rule_is_non_comparable(): + result = {**_fixture("verdict_with_rule.json"), "p_value": 0.2} + + assert comparability(result) == ["verdict 'improved' does not follow verdict_rule ('inconclusive')"] + + +def test_lower_is_better_turns_a_negative_delta_into_improved(): + result = _fixture("verdict_with_rule.json") + result["metric_spec"]["higher_is_better"] = False + result["delta"] = -6.5 + + assert recompute_verdict(result) == "improved" + + +def test_without_protocols_the_fallback_fields_must_match(): + a = {k: v for k, v in _fixture("verdict_with_rule.json").items() if k != "protocol"} + b = {**a, "harness": "claude-code"} + + assert compare(a, a) == [] + assert compare(b, a) == ["harness differs"] + + +# --- what the exporter emits -------------------------------------------------------------------- + +def test_the_exporter_emits_rule_spec_and_protocol_and_flags_what_it_cannot_compare(): + from test_subjects_evidence import _scorecard + + [skill] = [r for r in build_skill_evidence(_scorecard())["results"] if r["arm"] == "skill:s"] + + assert skill["verdict_rule"]["basis"] == "p_adjusted" and skill["verdict_rule"]["alpha"] == 0.05 + assert skill["metric_spec"]["higher_is_better"] is True and skill["metric_spec"]["unit"] + assert recompute_verdict(skill) == skill["verdict"] + assert skill["protocol"]["digest"] == protocol_digest(skill["protocol"]["resolved"]) + # The scorecard records no judge, sampling, judge prompt or repeat count for this cell. + assert {"judge_model", "judge_prompt_sha256", "judge_sampling", "repeats"} <= set( + skill["protocol"]["unresolved"]) + assert skill["comparable"] is False + assert skill["non_comparable"] == comparability(skill) == ["judged result without judge_model"] + + +def test_a_cell_that_records_its_judge_and_protocol_inputs_is_comparable(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + scorecard["cells"][0].update({ + "judge_model": "claude-sonnet-5-5-20260901", "judge_provider": "amazon-bedrock", + "judge_context": "report-snippets", "judge_sampling": {"temperature": 0.0}, + "judge_prompt_sha256": "sha256:" + "d" * 64, "repeats": 3, + }) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert skill["comparable"] is True and skill["non_comparable"] == [] + assert skill["protocol"]["unresolved"] == [] + assert skill["protocol"]["resolved"]["judge_model"] == "claude-sonnet-5-5-20260901" + assert compare(skill, skill) == [] + + +def test_the_rollup_and_scorecard_cells_carry_the_judge(tmp_path: Path): + from dokimi.eval.matrix_rollup import build_matrix_rollup + + arms = {"baseline_arm": "none", "judge_model": "j", "judge_provider": "jp", "judge_context": "ctx"} + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", arms)]) + cell = rollup["cells"][0] + assert (cell["judge_model"], cell["judge_provider"], cell["judge_context"]) == ("j", "jp", "ctx") + + +def test_conformance_fixtures_are_valid_json_files(): + for path in sorted(CONFORMANCE.glob("*.json")): + json.loads(path.read_text(encoding="utf-8")) From 896f332639a10aa56caaf6cfb73b68786dcaf1eb Mon Sep 17 00:00:00 2001 From: "Petrov, Nikolay A" Date: Sun, 15 Nov 2026 21:36:21 -0800 Subject: [PATCH 02/14] fix(subjects): answer self-review: rounding near alpha, recorded judge inputs, bare judge names A p_value within its rounding of alpha no longer reads as a verdict that contradicts its rule. Rollup and scorecard cells carry judge_temperature and judge_prompt_sha256, so the protocol resolves them. Bare family names (opus, sonnet, gpt-4o) and aliases nested in a value count as unresolved. Integral floats hash as ints. Signed-off-by: Petrov, Nikolay A --- ...61115_120000_napetrov_evidence-protocol.md | 2 +- dokimi/dokimi/eval/matrix_rollup.py | 2 + dokimi/dokimi/schemas/matrix_rollup.v1.json | 12 +++++ dokimi/dokimi/schemas/skill_evidence.v1.json | 4 ++ dokimi/dokimi/subjects/evidence.py | 14 ++++-- dokimi/dokimi/subjects/evidence_check.py | 46 +++++++++++++++++-- dokimi/dokimi/subjects/scorecard.py | 2 + .../judge_snapshot_newer.json | 2 +- .../judge_snapshot_older.json | 2 +- .../verdict_with_rule.json | 2 +- .../verdict_without_rule.json | 2 +- dokimi/tests/test_evidence_protocol.py | 44 +++++++++++++++++- 12 files changed, 120 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md b/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md index e1ad20300..29bee08a5 100644 --- a/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md +++ b/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **`subjects evidence`: each result says when it may be compared.** A result now carries `metric_spec` (unit, range, direction, judged), the `verdict_rule` its verdict follows (`p_adjusted` below `alpha`), and `protocol {id, version, digest}`: the digest is `sha256` over the canonical JSON of the resolved measurement inputs (judge snapshot, sampling, judge prompt hash, eval-set digest, repeats, test, correction, thresholds). A missing input or an alias such as `latest` is listed in `protocol.unresolved` and left out of the digest. `comparable`/`non_comparable` say why a result cannot be compared. `dokimi.subjects.evidence_check` holds the consumer checks, with conformance fixtures in `tests/fixtures/evidence_conformance`. Matrix-rollup and scorecard cells now carry the cell's `judge_model`, `judge_provider` and `judge_context`. Today's runs do not record the judge prompt hash, sampling or repeats, so their results list those as unresolved. +- **`subjects evidence`: each result says when it may be compared.** A result now carries `metric_spec` (unit, range, direction, judged), the `verdict_rule` its verdict follows (`p_adjusted` below `alpha`), and `protocol {id, version, digest}`: the digest is `sha256` over the canonical JSON of the resolved measurement inputs (judge snapshot, sampling, judge prompt hash, eval-set digest, repeats, test, correction, thresholds). A missing input or an alias such as `latest` is listed in `protocol.unresolved` and left out of the digest. `comparable`/`non_comparable` say why a result cannot be compared. `dokimi.subjects.evidence_check` holds the consumer checks, with conformance fixtures in `tests/fixtures/evidence_conformance`. Matrix-rollup and scorecard cells now carry the judge the run recorded (`judge_model`, `judge_provider`, `judge_context`, `judge_temperature`, `judge_prompt_sha256`). Runs do not record a repeat count, and a judge named by a bare family name (`opus`, `sonnet`) is an alias, so such results list those inputs as unresolved and are not comparable across runs. diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index eb8164621..ad54dc98b 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -34,6 +34,8 @@ def build_matrix_rollup( "judge_model": data.get("judge_model"), "judge_provider": data.get("judge_provider"), "judge_context": data.get("judge_context"), + "judge_temperature": data.get("judge_temperature"), + "judge_prompt_sha256": data.get("judge_prompt_sha256"), "summary": data.get("summary", {}), "cost_summary": data.get("cost_summary", {}), } diff --git a/dokimi/dokimi/schemas/matrix_rollup.v1.json b/dokimi/dokimi/schemas/matrix_rollup.v1.json index 9297aded6..1abd742d9 100644 --- a/dokimi/dokimi/schemas/matrix_rollup.v1.json +++ b/dokimi/dokimi/schemas/matrix_rollup.v1.json @@ -108,6 +108,18 @@ "string", "null" ] + }, + "judge_temperature": { + "type": [ + "number", + "null" + ] + }, + "judge_prompt_sha256": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": true diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json index e3913c186..35621ecc4 100644 --- a/dokimi/dokimi/schemas/skill_evidence.v1.json +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -296,6 +296,10 @@ "string", "null" ] + }, + "p_decimals": { + "type": "integer", + "description": "places p_value is rounded to; within that of alpha a verdict cannot be recomputed" } } }, diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index 528981070..cedd90455 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -86,6 +86,9 @@ def _baseline_arm(cell: dict[str, Any], per_arm: dict[str, Any]) -> str | None: return candidates[0] if len(candidates) == 1 else None +#: The places ``stats._round_p`` rounds ``p_adjusted`` to before it reaches the scorecard. +P_DECIMALS = 4 + #: What ``METRIC`` is: a 0-100 judge score per answer. METRIC_SPEC = {"id": METRIC, "unit": "score_points", "range": [0, 100], "higher_is_better": True, "judged": True} @@ -98,9 +101,14 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], sig = row.get("significance") or {} delta = sig.get("delta_mean") if sig else None correction = sig.get("correction") - judge = cell.get("judge_snapshot") or cell.get("judge_model") + judge = cell.get("judge_model") + sampling = cell.get("judge_sampling") + if sampling is None and cell.get("judge_temperature") is not None: + sampling = {"temperature": cell.get("judge_temperature")} # The rule the verdict below applies, so a consumer can recompute it from delta and p_value. - rule = {"basis": "p_adjusted", "alpha": ALPHA, "test": METHOD, "correction": correction} + # p_value is p_adjusted rounded to P_DECIMALS; within that rounding of alpha it cannot be recomputed. + rule = {"basis": "p_adjusted", "alpha": ALPHA, "test": METHOD, "correction": correction, + "p_decimals": P_DECIMALS} # Only resolved values enter the digest; what the scorecard does not record stays unresolved. protocol = build_protocol({ "eval_set_digest": (suite or {}).get("question_set_hash"), @@ -108,7 +116,7 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], "judge_model": judge, "judge_provider": cell.get("judge_provider"), "judge_context": cell.get("judge_context"), - "judge_sampling": cell.get("judge_sampling"), + "judge_sampling": sampling, "judge_prompt_sha256": cell.get("judge_prompt_sha256"), "repeats": cell.get("repeats"), "test": METHOD, diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index d73560b3b..21caa6693 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -29,13 +29,28 @@ #: A value that names a moving target rather than one snapshot. Matched as a whole trailing token, #: so ``claude-sonnet-5-5`` is resolved and ``claude-sonnet-latest`` / ``gpt:default`` are not. _ALIAS = re.compile(r"(?:^|[-_.:@/])(?:latest|default|auto|stable|current)$", re.IGNORECASE) +#: Bare family names a harness resolves to whatever it ships today (dokimi's own claude-cli judges +#: default to ``opus``/``sonnet``); a dated or versioned id is a snapshot. +_FLOATING = re.compile(r"^(?:claude|opus|sonnet|haiku|gpt-4o|gpt-4o-mini|gpt-4|gpt-5|o1|o3|o4-mini)$", + re.IGNORECASE) VERDICTS = ("improved", "regressed", "inconclusive") +def _normal(value: Any) -> Any: + # An integral float is written as an int, so 0 and 0.0 (or 3 and 3.0) are one measurement. + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, Mapping): + return {str(k): _normal(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_normal(v) for v in value] + return value + + def canonical_json(value: Any) -> bytes: - """Sorted keys, no whitespace, UTF-8, no NaN: the bytes the protocol digest is taken over.""" - return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, + """Sorted keys, no whitespace, UTF-8, no NaN, integral floats as ints: the bytes the digest is over.""" + return json.dumps(_normal(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") @@ -44,7 +59,15 @@ def protocol_digest(resolved: Mapping[str, Any]) -> str: def is_alias(value: Any) -> bool: - return isinstance(value, str) and bool(_ALIAS.search(value.strip())) + """A moving target: an alias suffix, a bare family name, or either anywhere inside a nested value.""" + if isinstance(value, str): + text = value.strip() + return bool(_ALIAS.search(text) or _FLOATING.match(text.rsplit("/", 1)[-1])) + if isinstance(value, Mapping): + return any(is_alias(v) for v in value.values()) + if isinstance(value, (list, tuple)): + return any(is_alias(v) for v in value) + return False def build_protocol(inputs: Mapping[str, Any]) -> dict[str, Any]: @@ -61,7 +84,7 @@ def build_protocol(inputs: Mapping[str, Any]) -> dict[str, Any]: unresolved.append(key) elif is_alias(value): unresolved.append(key) - aliases[key] = value + aliases[key] = value if isinstance(value, str) else canonical_json(value).decode("utf-8") else: resolved[key] = value protocol: dict[str, Any] = { @@ -90,6 +113,8 @@ def recompute_verdict(result: Mapping[str, Any]) -> str | None: if rule.get("basis") != "p_adjusted" or not _number(alpha) or not isinstance(better_up, bool): return None delta, p = result.get("delta"), result.get("p_value") + if _number(p) and _near_alpha(p, alpha, rule.get("p_decimals")): + return None # the rounded p cannot say which side of alpha the unrounded one was on if not _number(delta) or not _number(p) or p >= alpha or delta == 0: return "inconclusive" return "improved" if (delta > 0) == better_up else "regressed" @@ -111,7 +136,12 @@ def comparability(result: Mapping[str, Any]) -> list[str]: reasons.append("verdict without metric_spec.higher_is_better") if not reasons: expected = recompute_verdict(result) - if expected is None: + rule = result["verdict_rule"] + p = result.get("p_value") + if expected is None and _number(p) and _number(rule.get("alpha")) and _near_alpha( + p, rule["alpha"], rule.get("p_decimals")): + reasons.append("p_value is within its rounding of alpha: the verdict cannot be recomputed") + elif expected is None: reasons.append("verdict_rule cannot be applied") elif expected != result.get("verdict"): reasons.append(f"verdict {result.get('verdict')!r} does not follow verdict_rule ({expected!r})") @@ -165,5 +195,11 @@ def _resolved(protocol: Mapping[str, Any]) -> Mapping[str, Any]: return resolved if isinstance(resolved, Mapping) else {} +def _near_alpha(p: float, alpha: float, decimals: Any) -> bool: + if not isinstance(decimals, int) or isinstance(decimals, bool): + return False + return abs(p - alpha) <= 0.5 * 10 ** -decimals + 1e-12 + + def _number(value: Any) -> bool: return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index ce37e16bb..a5d68f665 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -63,6 +63,8 @@ def build_subject_scorecard( "judge_model": cell.get("judge_model"), "judge_provider": cell.get("judge_provider"), "judge_context": cell.get("judge_context"), + "judge_temperature": cell.get("judge_temperature"), + "judge_prompt_sha256": cell.get("judge_prompt_sha256"), "plugin_set": cell.get("plugin_set"), "summary": cell.get("summary", {}), "cost_summary": cell.get("cost_summary", {}), diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json index e855d3fd1..db9304690 100644 --- a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json @@ -36,7 +36,7 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:5ff2ce8ba0a051a86dc397bc7d215e679b992d5a7e5fcb94ef8bb692dea8678a", + "digest": "sha256:8a3cd1fb5426d92bf07b28f43afef9d4821574e468735b1a7d109f4a1651e3dc", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json index 55fc7c3bc..e65d84a87 100644 --- a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json @@ -36,7 +36,7 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:79edd29b1fa49fb8d90ce01893879c4d6b61813d8c54199c5023021501263210", + "digest": "sha256:e43084f2c2980e45beb24e17459d8e48188c63bab7b45df7616e912959f4c774", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json index 55fc7c3bc..e65d84a87 100644 --- a/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json @@ -36,7 +36,7 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:79edd29b1fa49fb8d90ce01893879c4d6b61813d8c54199c5023021501263210", + "digest": "sha256:e43084f2c2980e45beb24e17459d8e48188c63bab7b45df7616e912959f4c774", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json index 5dc5ba267..4909c4f62 100644 --- a/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json @@ -36,7 +36,7 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:79edd29b1fa49fb8d90ce01893879c4d6b61813d8c54199c5023021501263210", + "digest": "sha256:e43084f2c2980e45beb24e17459d8e48188c63bab7b45df7616e912959f4c774", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index 677c1d655..d2764a5d8 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -59,7 +59,7 @@ def test_fixture_two_judge_snapshots_give_two_digests_and_the_newer_is_not_compa def test_the_digest_is_sha256_of_sorted_compact_utf8_json(): resolved = {"repeats": 3, "judge_model": "j-2026", "judge_sampling": {"temperature": 0.0}} expected = hashlib.sha256( - b'{"judge_model":"j-2026","judge_sampling":{"temperature":0.0},"repeats":3}').hexdigest() + b'{"judge_model":"j-2026","judge_sampling":{"temperature":0},"repeats":3}').hexdigest() assert protocol_digest(resolved) == "sha256:" + expected assert build_protocol(resolved)["digest"] == "sha256:" + expected @@ -185,3 +185,45 @@ def test_the_rollup_and_scorecard_cells_carry_the_judge(tmp_path: Path): def test_conformance_fixtures_are_valid_json_files(): for path in sorted(CONFORMANCE.glob("*.json")): json.loads(path.read_text(encoding="utf-8")) + + +# --- self-review round 1 -------------------------------------------------------------------------- + +@pytest.mark.parametrize("bare", ["opus", "sonnet", "haiku", "gpt-4o", "gpt-4o-mini"]) +def test_a_bare_family_name_is_an_alias(bare): + assert build_protocol({"judge_model": bare})["unresolved"] == ["judge_model"] + + +def test_an_alias_nested_in_a_value_is_unresolved(): + protocol = build_protocol({"judge_sampling": {"model": "x-latest", "temperature": 0.0}}) + assert protocol["unresolved"] == ["judge_sampling"] + + +def test_an_integral_float_and_an_int_give_one_digest(): + assert protocol_digest({"t": 0, "r": 3}) == protocol_digest({"t": 0.0, "r": 3.0}) + + +def test_a_p_value_within_rounding_of_alpha_is_not_a_contradiction(): + result = {**_fixture("verdict_with_rule.json"), "p_value": 0.05} + result["verdict_rule"] = {**result["verdict_rule"], "p_decimals": 4} + + assert recompute_verdict(result) is None + assert comparability(result) == ["p_value is within its rounding of alpha: the verdict cannot be recomputed"] + + +def test_the_exporter_keeps_the_judge_temperature_and_prompt_hash_the_run_recorded(tmp_path: Path): + from test_subjects_evidence import _scorecard + from dokimi.eval.matrix_rollup import build_matrix_rollup + + arms = {"baseline_arm": "none", "judge_temperature": 0.0, "judge_prompt_sha256": "e" * 64} + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", arms)]) + assert rollup["cells"][0]["judge_temperature"] == 0.0 + assert rollup["cells"][0]["judge_prompt_sha256"] == "e" * 64 + + scorecard = _scorecard() + scorecard["cells"][0].update({"judge_temperature": 0.0, "judge_prompt_sha256": "e" * 64}) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + assert skill["protocol"]["resolved"]["judge_sampling"] == {"temperature": 0} + assert skill["protocol"]["resolved"]["judge_prompt_sha256"] == "e" * 64 + assert skill["verdict_rule"]["p_decimals"] == 4 From 791d106d078ba12d9cdb6fcc78576209d6c15f6d Mon Sep 17 00:00:00 2001 From: "Petrov, Nikolay A" Date: Sun, 15 Nov 2026 21:47:19 -0800 Subject: [PATCH 03/14] test(subjects): the fixture test checks the fixture set and its digests Signed-off-by: Petrov, Nikolay A --- dokimi/tests/test_evidence_protocol.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index d2764a5d8..30c67a728 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -182,9 +182,13 @@ def test_the_rollup_and_scorecard_cells_carry_the_judge(tmp_path: Path): assert (cell["judge_model"], cell["judge_provider"], cell["judge_context"]) == ("j", "jp", "ctx") -def test_conformance_fixtures_are_valid_json_files(): - for path in sorted(CONFORMANCE.glob("*.json")): - json.loads(path.read_text(encoding="utf-8")) +def test_the_conformance_fixtures_are_the_four_documented_ones_and_carry_the_digest_they_state(): + names = sorted(path.name for path in CONFORMANCE.glob("*.json")) + assert names == ["judge_snapshot_newer.json", "judge_snapshot_older.json", + "verdict_with_rule.json", "verdict_without_rule.json"] + for name in names: + protocol = json.loads((CONFORMANCE / name).read_text(encoding="utf-8"))["results"][0]["protocol"] + assert protocol["digest"] == protocol_digest(protocol["resolved"]) # --- self-review round 1 -------------------------------------------------------------------------- From 76c09e9f1a9f0a868c84a8eae136a95474606241 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:40:29 -0800 Subject: [PATCH 04/14] Address review: check what a result states against its protocol The consumer checks now refuse a result whose stated judge, provider, metric specification or verdict rule differs from what its protocol resolved, a protocol on one side of a pair only, and an omitted dokimi.awareness v1 input (unresolved, not resolved). An unsupported rounding precision is a reason, not an overflow. The exporter takes the test from the recorded ci_method, and each cell carries the digest of its own question file, so a multi-set suite no longer shares one eval-set digest. The fixture README takes the repo's license header. Thread: PRRT_kwDOVBg7g863y74Z Thread: PRRT_kwDOVBg7g863y74c Thread: PRRT_kwDOVBg7g863y74g Thread: PRRT_kwDOVBg7g863y74k Thread: PRRT_kwDOVBg7g863y74r Thread: PRRT_kwDOVBg7g863zDgM Thread: PRRT_kwDOVBg7g863zDgP Thread: PRRT_kwDOVBg7g863zjbt Thread: PRRT_kwDOVBg7g863zjb1 Thread: PRRT_kwDOVBg7g863zjb6 Thread: PRRT_kwDOVBg7g863z_n6 Thread: PRRT_kwDOVBg7g863z_oG Signed-off-by: Nikolay Petrov --- dokimi/dokimi/subjects/evidence.py | 21 ++- dokimi/dokimi/subjects/evidence_check.py | 143 +++++++++++++----- dokimi/dokimi/subjects/scorecard.py | 3 + .../fixtures/evidence_conformance/README.md | 5 + dokimi/tests/test_evidence_protocol.py | 107 +++++++++++++ dokimi/tests/test_subjects_evidence.py | 29 ++++ 6 files changed, 267 insertions(+), 41 deletions(-) diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index 68b8ea472..30c30208c 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -19,7 +19,7 @@ from dokimi import __version__ from dokimi.eval.comparison_verdict import decide -from dokimi.eval.sign_flip import ALPHA, METHOD +from dokimi.eval.sign_flip import ALPHA from dokimi.subjects.evidence_check import build_protocol, comparability #: The arm metric the awareness summary carries per arm (``summary.per_arm[arm]``). @@ -95,6 +95,16 @@ def _baseline_arm(cell: dict[str, Any], per_arm: dict[str, Any]) -> str | None: "higher_is_better": True, "judged": True} +def _eval_set_digest(cell: dict[str, Any], suite: dict[str, Any] | None) -> str | None: + """The digest of the question set this cell measured; the suite's only when one set serves every cell.""" + if cell.get("questions_digest") is not None: + return cell["questions_digest"] + questions = (suite or {}).get("questions") or [] + if len(questions) == 1: + return (suite or {}).get("question_set_hash") + return None + + def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], suite: dict[str, Any] | None = None) -> dict[str, Any]: # Every statistic is the paired test's, so n, delta, CI and p describe the same questions. @@ -106,13 +116,16 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], sampling = cell.get("judge_sampling") if sampling is None and cell.get("judge_temperature") is not None: sampling = {"temperature": cell.get("judge_temperature")} + # The recorded ci_method, not the current implementation's constant: a historical scorecard keeps + # the method it ran. Unrecorded, it is None and the protocol lists it as unresolved. + test = sig.get("ci_method") # The rule the verdict below applies, so a consumer can recompute it from delta and p_value. # p_value is p_adjusted rounded to P_DECIMALS; within that rounding of alpha it cannot be recomputed. - rule = {"basis": "p_adjusted", "alpha": ALPHA, "test": METHOD, "correction": correction, + rule = {"basis": "p_adjusted", "alpha": ALPHA, "test": test, "correction": correction, "p_decimals": P_DECIMALS} # Only resolved values enter the digest; what the scorecard does not record stays unresolved. protocol = build_protocol({ - "eval_set_digest": (suite or {}).get("question_set_hash"), + "eval_set_digest": _eval_set_digest(cell, suite), "metric": METRIC, "judge_model": judge, "judge_provider": cell.get("judge_provider"), @@ -120,7 +133,7 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], "judge_sampling": sampling, "judge_prompt_sha256": cell.get("judge_prompt_sha256"), "repeats": cell.get("repeats"), - "test": METHOD, + "test": test, "correction": correction, "alpha": ALPHA, "ci_confidence": sig.get("ci_confidence"), diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index 21caa6693..22c7ae958 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -19,6 +19,7 @@ import hashlib import json +import math import re from typing import Any, Iterable, Mapping @@ -31,7 +32,7 @@ _ALIAS = re.compile(r"(?:^|[-_.:@/])(?:latest|default|auto|stable|current)$", re.IGNORECASE) #: Bare family names a harness resolves to whatever it ships today (dokimi's own claude-cli judges #: default to ``opus``/``sonnet``); a dated or versioned id is a snapshot. -_FLOATING = re.compile(r"^(?:claude|opus|sonnet|haiku|gpt-4o|gpt-4o-mini|gpt-4|gpt-5|o1|o3|o4-mini)$", +_FLOATING = re.compile(r"^(?:claude|opus|sonnet|haiku|gpt-4o(?:-mini)?|gpt-4|gpt-5|o1|o3|o4-mini)$", re.IGNORECASE) VERDICTS = ("improved", "regressed", "inconclusive") @@ -109,13 +110,15 @@ def recompute_verdict(result: Mapping[str, Any]) -> str | None: spec = result.get("metric_spec") if not isinstance(rule, Mapping) or not isinstance(spec, Mapping): return None - alpha, better_up = rule.get("alpha"), spec.get("higher_is_better") - if rule.get("basis") != "p_adjusted" or not _number(alpha) or not isinstance(better_up, bool): + alpha, better_up = _num(rule.get("alpha")), spec.get("higher_is_better") + if rule.get("basis") != "p_adjusted" or alpha is None or not isinstance(better_up, bool): return None - delta, p = result.get("delta"), result.get("p_value") - if _number(p) and _near_alpha(p, alpha, rule.get("p_decimals")): + if not _valid_decimals(rule.get("p_decimals"), allow_none=True): + return None # an unsupported rounding precision: the rule cannot be applied + delta, p = _num(result.get("delta")), _num(result.get("p_value")) + if p is not None and _near_alpha(p, alpha, rule.get("p_decimals")): return None # the rounded p cannot say which side of alpha the unrounded one was on - if not _number(delta) or not _number(p) or p >= alpha or delta == 0: + if delta is None or p is None or p >= alpha or delta == 0: return "inconclusive" return "improved" if (delta > 0) == better_up else "regressed" @@ -126,36 +129,68 @@ def comparability(result: Mapping[str, Any]) -> list[str]: A non-comparable result can still be listed. It is not "neutral": it says nothing either way. """ reasons: list[str] = [] - spec = result.get("metric_spec") if isinstance(result.get("metric_spec"), Mapping) else {} + stated = result.get("metric_spec") + spec: Mapping[str, Any] = stated if isinstance(stated, Mapping) else {} if result.get("verdict") is not None: - if not isinstance(result.get("verdict_rule"), Mapping): - reasons.append("verdict without verdict_rule") - if not spec.get("unit"): - reasons.append("verdict without metric_spec.unit") - if not isinstance(spec.get("higher_is_better"), bool): - reasons.append("verdict without metric_spec.higher_is_better") - if not reasons: - expected = recompute_verdict(result) - rule = result["verdict_rule"] - p = result.get("p_value") - if expected is None and _number(p) and _number(rule.get("alpha")) and _near_alpha( - p, rule["alpha"], rule.get("p_decimals")): - reasons.append("p_value is within its rounding of alpha: the verdict cannot be recomputed") - elif expected is None: - reasons.append("verdict_rule cannot be applied") - elif expected != result.get("verdict"): - reasons.append(f"verdict {result.get('verdict')!r} does not follow verdict_rule ({expected!r})") + reasons += _verdict_reasons(result, spec) if spec.get("judged") is not False and not result.get("judge_model"): reasons.append("judged result without judge_model") protocol = result.get("protocol") if isinstance(protocol, Mapping) and isinstance(protocol.get("resolved"), Mapping): if protocol_digest(protocol["resolved"]) != protocol.get("digest"): reasons.append("protocol.digest does not match protocol.resolved") + reasons += _disagreements(result, protocol["resolved"]) return reasons +def _disagreements(result: Mapping[str, Any], resolved: Mapping[str, Any]) -> list[str]: + """Stated fields whose value is not the one the protocol resolved. An absent field is no statement: + its own check (a judged result without a judge, a verdict without a rule) reports it.""" + stated = [(key, result.get(key)) for key in _AGREES] + rule = result.get("verdict_rule") + if isinstance(rule, Mapping): + stated += [(f"verdict_rule.{key}", rule.get(key)) for key in _RULE_AGREES] + out = [] + for field, value in stated: + key = field.rsplit(".", 1)[-1] + if value is not None and key in resolved and resolved[key] != value: + out.append(f"{field} differs from protocol.resolved.{key}") + return out + + +def _verdict_reasons(result: Mapping[str, Any], spec: Mapping[str, Any]) -> list[str]: + reasons = [] + if not isinstance(result.get("verdict_rule"), Mapping): + reasons.append("verdict without verdict_rule") + if not spec.get("unit"): + reasons.append("verdict without metric_spec.unit") + if not isinstance(spec.get("higher_is_better"), bool): + reasons.append("verdict without metric_spec.higher_is_better") + if reasons or not isinstance(result.get("verdict_rule"), Mapping): + return reasons + expected = recompute_verdict(result) + rule = result["verdict_rule"] + p, alpha = _num(result.get("p_value")), _num(rule.get("alpha")) + if expected is None and p is not None and alpha is not None and _near_alpha( + p, alpha, rule.get("p_decimals")): + return ["p_value is within its rounding of alpha: the verdict cannot be recomputed"] + if expected is None: + return ["verdict_rule cannot be applied"] + if expected != result.get("verdict"): + return [f"verdict {result.get('verdict')!r} does not follow verdict_rule ({expected!r})"] + return [] + + #: Without a protocol on both sides, the fields that must be equal instead. -_FALLBACK_KEYS = ("harness", "judge_model", "metric", "verdict_rule") +_FALLBACK_KEYS = ("harness", "judge_model", "judge_provider", "metric", "verdict_rule") +#: The inputs a ``dokimi.awareness`` version 1 protocol names; one it leaves out is unresolved. +REQUIRED_INPUTS = ("alpha", "ci_confidence", "correction", "eval_set_digest", "judge_context", + "judge_model", "judge_prompt_sha256", "judge_provider", "judge_sampling", "metric", + "repeats", "test") +#: Fields a result states that its protocol resolves too: a digest over the resolved inputs cannot +#: catch a result that states another value for them. +_AGREES = ("judge_model", "judge_provider", "metric") +_RULE_AGREES = ("alpha", "correction", "test") def compare(result: Mapping[str, Any], reference: Mapping[str, Any]) -> list[str]: @@ -164,22 +199,40 @@ def compare(result: Mapping[str, Any], reference: Mapping[str, Any]) -> list[str reasons += [f"reference: {r}" for r in comparability(reference)] if result.get("metric") != reference.get("metric"): reasons.append("metric differs") + if result.get("metric_spec") != reference.get("metric_spec"): + reasons.append("metric specification differs (unit, range or direction)") mine, theirs = result.get("protocol"), reference.get("protocol") if isinstance(mine, Mapping) and isinstance(theirs, Mapping): - if (mine.get("id"), mine.get("version")) != (theirs.get("id"), theirs.get("version")): - reasons.append("protocol id or version differs") - elif mine.get("digest") != theirs.get("digest"): - changed = sorted(k for k in set(_resolved(mine)) | set(_resolved(theirs)) - if _resolved(mine).get(k) != _resolved(theirs).get(k)) - reasons.append("protocol digest differs" + (f" ({', '.join(changed)})" if changed else "")) - unresolved = sorted(set(mine.get("unresolved") or []) | set(theirs.get("unresolved") or [])) - if unresolved: - reasons.append(f"protocol inputs not resolved: {', '.join(unresolved)}") + reasons += _protocol_reasons(mine, theirs) + elif isinstance(mine, Mapping) or isinstance(theirs, Mapping): + reasons.append("protocol on one side only: the fallback fields cannot stand for its inputs") else: reasons += [f"{key} differs" for key in _FALLBACK_KEYS if result.get(key) != reference.get(key)] return reasons +def _protocol_reasons(mine: Mapping[str, Any], theirs: Mapping[str, Any]) -> list[str]: + reasons = [] + if (mine.get("id"), mine.get("version")) != (theirs.get("id"), theirs.get("version")): + reasons.append("protocol id or version differs") + elif mine.get("digest") != theirs.get("digest"): + changed = sorted(k for k in set(_resolved(mine)) | set(_resolved(theirs)) + if _resolved(mine).get(k) != _resolved(theirs).get(k)) + reasons.append("protocol digest differs" + (f" ({', '.join(changed)})" if changed else "")) + unresolved = sorted(_unresolved(mine) | _unresolved(theirs)) + if unresolved: + reasons.append(f"protocol inputs not resolved: {', '.join(unresolved)}") + return reasons + + +def _unresolved(protocol: Mapping[str, Any]) -> set[str]: + """The inputs the protocol lists as unresolved, and for ``dokimi.awareness`` 1 the ones it omits.""" + listed = set(protocol.get("unresolved") or []) + if (protocol.get("id"), protocol.get("version")) == (PROTOCOL_ID, PROTOCOL_VERSION): + listed |= set(REQUIRED_INPUTS) - set(_resolved(protocol)) + return listed + + def mark_against(results: Iterable[Mapping[str, Any]], reference: Mapping[str, Any]) -> list[dict[str, Any]]: """Copies of ``results``, each with ``comparable`` and ``non_comparable`` set against ``reference``.""" @@ -196,10 +249,26 @@ def _resolved(protocol: Mapping[str, Any]) -> Mapping[str, Any]: def _near_alpha(p: float, alpha: float, decimals: Any) -> bool: - if not isinstance(decimals, int) or isinstance(decimals, bool): + if not _valid_decimals(decimals): return False return abs(p - alpha) <= 0.5 * 10 ** -decimals + 1e-12 -def _number(value: Any) -> bool: - return isinstance(value, (int, float)) and not isinstance(value, bool) +#: The precisions a rounded p can carry: a float holds about 15 significant digits. +_MAX_DECIMALS = 15 + + +def _valid_decimals(value: Any, *, allow_none: bool = False) -> bool: + if value is None: + return allow_none + return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= _MAX_DECIMALS + + +def _num(value: Any) -> float | None: + """A finite number, else None: NaN, infinity and an integer too large for a float say nothing.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + return value if math.isfinite(value) else None + except OverflowError: + return None diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index a5d68f665..2cccfecf6 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -54,6 +54,9 @@ def build_subject_scorecard( cells.append({ "product": run["product"], "questions": run["questions"], + # The digest of this run's own question file: a suite with one file per product + # measures a different set in each cell, and suite.question_set_hash covers them all. + "questions_digest": _digest_values((run["questions"],)), "matrix_cell": cell.get("matrix_cell"), "artifact": cell.get("artifact"), "model": cell.get("model"), diff --git a/dokimi/tests/fixtures/evidence_conformance/README.md b/dokimi/tests/fixtures/evidence_conformance/README.md index 9150ee5e8..ea8782167 100644 --- a/dokimi/tests/fixtures/evidence_conformance/README.md +++ b/dokimi/tests/fixtures/evidence_conformance/README.md @@ -1,3 +1,8 @@ + + # skill_evidence.v1 conformance fixtures Each file is a valid `skill_evidence.v1` document with one result. A consumer passes when: diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index 30c67a728..b89ea06b9 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -231,3 +231,110 @@ def test_the_exporter_keeps_the_judge_temperature_and_prompt_hash_the_run_record assert skill["protocol"]["resolved"]["judge_sampling"] == {"temperature": 0} assert skill["protocol"]["resolved"]["judge_prompt_sha256"] == "e" * 64 assert skill["verdict_rule"]["p_decimals"] == 4 + + +# --- regressions: what a result states must be what its protocol resolved ---------------------- + +def _with(result: dict, **changes) -> dict: + return {**result, **changes} + + +def test_a_stated_judge_or_rule_value_the_protocol_does_not_resolve_is_non_comparable(): + base = _fixture("verdict_with_rule.json") + judge = _with(base, judge_model="claude-sonnet-5-5-20261101") + alpha = _with(base, verdict_rule={**base["verdict_rule"], "alpha": 0.01}) + + assert compare(judge, base) == ["result: judge_model differs from protocol.resolved.judge_model"] + assert compare(alpha, base) == ["result: verdict_rule.alpha differs from protocol.resolved.alpha"] + + +def test_a_metric_unit_or_range_change_is_non_comparable_with_or_without_a_protocol(): + base = _fixture("verdict_with_rule.json") + reason = ["metric specification differs (unit, range or direction)"] + unit = _with(base, metric_spec={**base["metric_spec"], "unit": "percent"}) + span = _with(base, metric_spec={**base["metric_spec"], "range": [0, 1]}) + bare = {k: v for k, v in base.items() if k != "protocol"} + + assert compare(unit, base) == reason + assert compare(span, base) == reason + assert compare(_with(bare, metric_spec={**bare["metric_spec"], "unit": "percent"}), bare) == reason + + +def test_a_protocol_on_one_side_only_is_non_comparable_in_either_order(): + base = _fixture("verdict_with_rule.json") + bare = {k: v for k, v in base.items() if k != "protocol"} + reason = ["protocol on one side only: the fallback fields cannot stand for its inputs"] + + assert compare(base, bare) == compare(bare, base) == reason + + +def test_an_input_the_protocol_leaves_out_is_unresolved(): + base = _fixture("verdict_with_rule.json") + partial = _with(base, protocol=build_protocol({"judge_model": base["judge_model"]})) + + [reason] = compare(partial, partial) + assert reason.startswith("protocol inputs not resolved: ") + assert "alpha" in reason and "repeats" in reason and "judge_model" not in reason + + +@pytest.mark.parametrize("decimals", [-1000, 1000, True]) +def test_an_unsupported_rounding_precision_is_non_comparable_not_a_crash(decimals): + base = _fixture("verdict_with_rule.json") + bad = _with(base, verdict_rule={**base["verdict_rule"], "p_decimals": decimals}) + + assert recompute_verdict(bad) is None + assert "verdict_rule cannot be applied" in comparability(bad) + assert compare(bad, base) and mark_against([bad], base)[0]["comparable"] is False + + +def test_without_protocols_a_judge_provider_difference_is_non_comparable(): + a = {k: v for k, v in _fixture("verdict_with_rule.json").items() if k != "protocol"} + + assert compare(_with(a, judge_provider="anthropic"), a) == ["judge_provider differs"] + + +def test_a_judged_cell_carries_its_judge_fields_into_the_scorecard(tmp_path: Path): + from dokimi.subjects import build_subject_scorecard, load_subject + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["p"]\nquestions = ["q.json"]\n', encoding="utf-8") + cell = {"matrix_cell": "c1", "judge_model": "j", "judge_provider": "jp", "judge_context": "ctx"} + judged = {**cell, "judge_temperature": 0.0, "judge_prompt_sha256": "sha256:" + "d" * 64} + scorecard = build_subject_scorecard( + load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=[{"product": "p", "questions": "q.json", "matrix_rollup": "m.json", + "rollup": {"cells": [judged]}}]) + carried = scorecard["cells"][0] + assert {key: value for key, value in carried.items() if key.startswith("judge_")} == { + key: value for key, value in judged.items() if key.startswith("judge_")} + + +def test_each_cell_of_a_two_question_set_suite_carries_the_digest_of_its_own_set(tmp_path: Path): + from dokimi.subjects import build_subject_scorecard, load_subject + from dokimi.subjects.scorecard import _digest_values + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["p", "q"]\nquestions = ["a.json", "b.json"]\n', encoding="utf-8") + (tmp_path / "a.json").write_text('["one"]', encoding="utf-8") + (tmp_path / "b.json").write_text('["two"]', encoding="utf-8") + runs = [{"product": product, "questions": name, "matrix_rollup": "m.json", + "rollup": {"cells": [{"matrix_cell": product}]}} + for product, name in (("p", "a.json"), ("q", "b.json"))] + scorecard = build_subject_scorecard(load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=runs) + first, second = scorecard["cells"] + assert first["questions_digest"] == _digest_values(("a.json",)) + assert second["questions_digest"] == _digest_values(("b.json",)) + assert scorecard["suite"]["question_set_hash"] not in (first["questions_digest"], second["questions_digest"]) + + +def test_a_non_finite_p_or_delta_gives_no_direction(): + base = _fixture("verdict_with_rule.json") + huge = 10 ** 400 # valid JSON, too large for a float + for changes in ({"p_value": float("nan")}, {"delta": float("nan")}, {"p_value": float("inf")}, + {"delta": huge}, {"p_value": huge}): + bad = _with(base, **changes) + assert recompute_verdict(bad) == "inconclusive", changes + comparability(bad) # must not raise diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 62dd48577..4687d9b70 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -312,3 +312,32 @@ def test_the_scorecard_cells_carry_the_baseline_arm_of_the_rollup(tmp_path: Path "rollup": rollup}], out_dir=tmp_path) assert scorecard["cells"][0]["baseline_arm"] == "none" + + +def test_a_scorecard_keeps_the_test_it_recorded_and_an_unrecorded_one_is_unresolved(): + from dokimi.subjects.evidence import build_skill_evidence + + scorecard = _scorecard() + significance = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"] + significance["ci_method"] = "student_t" + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + assert skill["verdict_rule"]["test"] == skill["protocol"]["resolved"]["test"] == "student_t" + + del significance["ci_method"] + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + assert skill["verdict_rule"]["test"] is None and "test" in skill["protocol"]["unresolved"] + + +def test_a_cell_of_a_two_question_set_suite_is_resolved_only_by_its_own_digest(): + from dokimi.subjects.evidence import build_skill_evidence + + scorecard = _scorecard() + scorecard["suite"]["questions"] = ["q.json", "r.json"] + + def skill_protocol() -> dict: + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + return skill["protocol"] + + assert "eval_set_digest" in skill_protocol()["unresolved"] + scorecard["cells"][0]["questions_digest"] = "sha256:" + "e" * 64 + assert skill_protocol()["resolved"]["eval_set_digest"] == "sha256:" + "e" * 64 From eeb71e8fe1cb26978841a11fe96a02f100ee7004 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:51:32 -0800 Subject: [PATCH 05/14] Address review: answer the review body of vchamarthi Thread: PRR_kwDOVBg7g88AAAABYKB_6w: already fixed in 76c09e9f1a Signed-off-by: Nikolay Petrov From 674aa80dde5a062ed00355ad06547a8295e50a9b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:46:05 -0800 Subject: [PATCH 06/14] Address review: judge aliases, unresolved values and the export of a historical scorecard Thread: PRRT_kwDOVBg7g864FRtF Thread: PRRT_kwDOVBg7g864FRtM Thread: PRRT_kwDOVBg7g864FRtR Thread: PRRT_kwDOVBg7g864FRta Thread: PRRT_kwDOVBg7g864FRtj Thread: IC_kwDOVBg7g88AAAABm8bdwg#6 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/subjects/evidence.py | 5 ++- dokimi/dokimi/subjects/evidence_check.py | 17 ++++++-- .../fixtures/evidence_conformance/README.md | 2 +- dokimi/tests/test_evidence_protocol.py | 43 +++++++++++++++++-- dokimi/tests/test_subjects_evidence.py | 9 ++-- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index 30c30208c..fa48214c2 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -121,8 +121,9 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], test = sig.get("ci_method") # The rule the verdict below applies, so a consumer can recompute it from delta and p_value. # p_value is p_adjusted rounded to P_DECIMALS; within that rounding of alpha it cannot be recomputed. - rule = {"basis": "p_adjusted", "alpha": ALPHA, "test": test, "correction": correction, - "p_decimals": P_DECIMALS} + rule = {"basis": "p_adjusted", "alpha": ALPHA, "correction": correction, "p_decimals": P_DECIMALS} + if test is not None: # the schema has no null test: an unrecorded one is left out, not written as null + rule["test"] = test # Only resolved values enter the digest; what the scorecard does not record stays unresolved. protocol = build_protocol({ "eval_set_digest": _eval_set_digest(cell, suite), diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index 22c7ae958..085cdd280 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -30,6 +30,8 @@ #: A value that names a moving target rather than one snapshot. Matched as a whole trailing token, #: so ``claude-sonnet-5-5`` is resolved and ``claude-sonnet-latest`` / ``gpt:default`` are not. _ALIAS = re.compile(r"(?:^|[-_.:@/])(?:latest|default|auto|stable|current)$", re.IGNORECASE) +#: The 1M-context suffix a CLI takes after an alias (``opus[1m]``); it does not make the name a snapshot. +_CONTEXT_SUFFIX = re.compile(r"\[1m\]$", re.IGNORECASE) #: Bare family names a harness resolves to whatever it ships today (dokimi's own claude-cli judges #: default to ``opus``/``sonnet``); a dated or versioned id is a snapshot. _FLOATING = re.compile(r"^(?:claude|opus|sonnet|haiku|gpt-4o(?:-mini)?|gpt-4|gpt-5|o1|o3|o4-mini)$", @@ -62,7 +64,7 @@ def protocol_digest(resolved: Mapping[str, Any]) -> str: def is_alias(value: Any) -> bool: """A moving target: an alias suffix, a bare family name, or either anywhere inside a nested value.""" if isinstance(value, str): - text = value.strip() + text = _CONTEXT_SUFFIX.sub("", value.strip()) return bool(_ALIAS.search(text) or _FLOATING.match(text.rsplit("/", 1)[-1])) if isinstance(value, Mapping): return any(is_alias(v) for v in value.values()) @@ -137,8 +139,13 @@ def comparability(result: Mapping[str, Any]) -> list[str]: reasons.append("judged result without judge_model") protocol = result.get("protocol") if isinstance(protocol, Mapping) and isinstance(protocol.get("resolved"), Mapping): - if protocol_digest(protocol["resolved"]) != protocol.get("digest"): - reasons.append("protocol.digest does not match protocol.resolved") + try: + digest = protocol_digest(protocol["resolved"]) + except (TypeError, ValueError): # NaN, infinity or a non-JSON value: no digest to check + reasons.append("protocol.resolved is not JSON, so its digest cannot be checked") + else: + if digest != protocol.get("digest"): + reasons.append("protocol.digest does not match protocol.resolved") reasons += _disagreements(result, protocol["resolved"]) return reasons @@ -226,10 +233,12 @@ def _protocol_reasons(mine: Mapping[str, Any], theirs: Mapping[str, Any]) -> lis def _unresolved(protocol: Mapping[str, Any]) -> set[str]: - """The inputs the protocol lists as unresolved, and for ``dokimi.awareness`` 1 the ones it omits.""" + """The inputs the protocol lists as unresolved, the ones it omits (``dokimi.awareness`` 1), and + any resolved value that is null or an alias: a digest over them identifies no fixed measurement.""" listed = set(protocol.get("unresolved") or []) if (protocol.get("id"), protocol.get("version")) == (PROTOCOL_ID, PROTOCOL_VERSION): listed |= set(REQUIRED_INPUTS) - set(_resolved(protocol)) + listed |= {key for key, value in _resolved(protocol).items() if value is None or is_alias(value)} return listed diff --git a/dokimi/tests/fixtures/evidence_conformance/README.md b/dokimi/tests/fixtures/evidence_conformance/README.md index ea8782167..d65ca80cc 100644 --- a/dokimi/tests/fixtures/evidence_conformance/README.md +++ b/dokimi/tests/fixtures/evidence_conformance/README.md @@ -13,4 +13,4 @@ Each file is a valid `skill_evidence.v1` document with one result. A consumer pa | `verdict_with_rule.json` | Recomputing the verdict from `delta`, `p_value`, `verdict_rule` and `metric_spec` gives the stated `improved`. | | `judge_snapshot_older.json`, `judge_snapshot_newer.json` | The same protocol resolved against judge snapshots `…-20260901` and `…-20261101`: the `protocol.digest` values differ, and the newer result is non-comparable to the older one. | -`protocol.digest` is `sha256:` + the hex sha256 of `protocol.resolved` as JSON with sorted keys, no whitespace, UTF-8 and no NaN. `tests/test_evidence_protocol.py` runs these checks against `dokimi.subjects.evidence_check`. +`protocol.digest` is `sha256:` + the hex sha256 of `protocol.resolved` as JSON with sorted keys, no whitespace, UTF-8 and no NaN, recursively normalizing integral floats to integers (for example `0.0` to `0`). `tests/test_evidence_protocol.py` runs these checks against `dokimi.subjects.evidence_check`. diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index b89ea06b9..541c686d5 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -310,10 +310,13 @@ def test_a_judged_cell_carries_its_judge_fields_into_the_scorecard(tmp_path: Pat key: value for key, value in judged.items() if key.startswith("judge_")} -def test_each_cell_of_a_two_question_set_suite_carries_the_digest_of_its_own_set(tmp_path: Path): +def test_each_cell_of_a_two_question_set_suite_carries_the_digest_of_its_own_set( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch): from dokimi.subjects import build_subject_scorecard, load_subject from dokimi.subjects.scorecard import _digest_values + # Question files are read from the working directory, so the digests hash their contents. + monkeypatch.chdir(tmp_path) (tmp_path / "subject.toml").write_text( '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' '[suite]\nproducts = ["p", "q"]\nquestions = ["a.json", "b.json"]\n', encoding="utf-8") @@ -322,12 +325,20 @@ def test_each_cell_of_a_two_question_set_suite_carries_the_digest_of_its_own_set runs = [{"product": product, "questions": name, "matrix_rollup": "m.json", "rollup": {"cells": [{"matrix_cell": product}]}} for product, name in (("p", "a.json"), ("q", "b.json"))] - scorecard = build_subject_scorecard(load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, - awareness_runs=runs) + def build() -> dict: + return build_subject_scorecard(load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=runs) + + scorecard = build() first, second = scorecard["cells"] assert first["questions_digest"] == _digest_values(("a.json",)) assert second["questions_digest"] == _digest_values(("b.json",)) assert scorecard["suite"]["question_set_hash"] not in (first["questions_digest"], second["questions_digest"]) + # Changing one file's contents changes only its own cell's digest. + (tmp_path / "a.json").write_text('["changed"]', encoding="utf-8") + changed = build()["cells"] + assert changed[0]["questions_digest"] != first["questions_digest"] + assert changed[1]["questions_digest"] == second["questions_digest"] def test_a_non_finite_p_or_delta_gives_no_direction(): @@ -338,3 +349,29 @@ def test_a_non_finite_p_or_delta_gives_no_direction(): bad = _with(base, **changes) assert recompute_verdict(bad) == "inconclusive", changes comparability(bad) # must not raise + + +def test_a_context_suffixed_alias_is_an_alias_not_a_snapshot(): + for alias in ("opus[1m]", "sonnet[1m]"): + protocol = build_protocol({"judge_model": alias}) + assert protocol["unresolved"] == ["judge_model"] and protocol["aliases"] == {"judge_model": alias} + snapshot = build_protocol({"judge_model": "claude-sonnet-5-5-20261101[1m]"}) + assert snapshot["resolved"] == {"judge_model": "claude-sonnet-5-5-20261101[1m]"} + + +def test_a_resolved_null_or_alias_is_unresolved_even_with_a_matching_digest(): + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["repeats"] = None + result["protocol"]["digest"] = protocol_digest(result["protocol"]["resolved"]) + assert "protocol inputs not resolved: repeats" in compare(result, result) + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["judge_sampling"] = {"temperature": 0.0, "top_p": "default"} + result["protocol"]["digest"] = protocol_digest(result["protocol"]["resolved"]) + assert any("judge_sampling" in reason for reason in compare(result, result)) + + +def test_a_non_json_resolved_value_is_a_reason_not_an_exception(): + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["judge_sampling"] = {"temperature": float("nan")} + reasons = compare(result, _fixture("verdict_with_rule.json")) + assert any("is not JSON" in reason for reason in reasons) diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 4687d9b70..29db800db 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -314,7 +314,7 @@ def test_the_scorecard_cells_carry_the_baseline_arm_of_the_rollup(tmp_path: Path assert scorecard["cells"][0]["baseline_arm"] == "none" -def test_a_scorecard_keeps_the_test_it_recorded_and_an_unrecorded_one_is_unresolved(): +def test_a_scorecard_keeps_the_test_it_recorded_and_an_unrecorded_one_is_unresolved(tmp_path: Path): from dokimi.subjects.evidence import build_skill_evidence scorecard = _scorecard() @@ -324,8 +324,11 @@ def test_a_scorecard_keeps_the_test_it_recorded_and_an_unrecorded_one_is_unresol assert skill["verdict_rule"]["test"] == skill["protocol"]["resolved"]["test"] == "student_t" del significance["ci_method"] - [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] - assert skill["verdict_rule"]["test"] is None and "test" in skill["protocol"]["unresolved"] + artifact = build_skill_evidence(scorecard) + [skill] = [r for r in artifact["results"] if r["arm"] == "skill:s"] + assert "test" not in skill["verdict_rule"] and "test" in skill["protocol"]["unresolved"] + # The historical scorecard exports: the schema has no null test, so the artifact must validate. + save_artifact("skill_evidence", artifact, tmp_path / "skill_evidence.json") def test_a_cell_of_a_two_question_set_suite_is_resolved_only_by_its_own_digest(): From 52d4945f785de0d6dd1e420a86b89e6daa2f9197 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 12:51:59 -0800 Subject: [PATCH 07/14] Address review: a p-value or alpha outside [0, 1] gives no verdict A negative p-value was below alpha, so recompute_verdict read it as "improved" and comparability passed it with no reason. Outside [0, 1] (p) or (0, 1) (alpha) the rule now cannot be applied. Thread: PRRT_kwDOVBg7g864Gw0t Signed-off-by: Nikolay Petrov --- dokimi/dokimi/subjects/evidence_check.py | 4 ++++ dokimi/tests/test_evidence_protocol.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index 085cdd280..1c385c956 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -115,9 +115,13 @@ def recompute_verdict(result: Mapping[str, Any]) -> str | None: alpha, better_up = _num(rule.get("alpha")), spec.get("higher_is_better") if rule.get("basis") != "p_adjusted" or alpha is None or not isinstance(better_up, bool): return None + if not 0 < alpha < 1: + return None # alpha is a probability: outside (0, 1) the rule says nothing about a verdict if not _valid_decimals(rule.get("p_decimals"), allow_none=True): return None # an unsupported rounding precision: the rule cannot be applied delta, p = _num(result.get("delta")), _num(result.get("p_value")) + if p is not None and not 0 <= p <= 1: + return None # a p-value is a probability: an out-of-range one cannot be on either side of alpha if p is not None and _near_alpha(p, alpha, rule.get("p_decimals")): return None # the rounded p cannot say which side of alpha the unrounded one was on if delta is None or p is None or p >= alpha or delta == 0: diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index 541c686d5..d2d24a65f 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -121,6 +121,16 @@ def test_a_verdict_that_contradicts_its_rule_is_non_comparable(): assert comparability(result) == ["verdict 'improved' does not follow verdict_rule ('inconclusive')"] +def test_a_p_value_or_alpha_outside_the_probability_range_cannot_give_a_verdict(): + # p = -0.004 is below alpha, so it used to read "improved" and pass comparability with no reason. + base = _fixture("verdict_with_rule.json") + for result in ({**base, "p_value": -0.004}, {**base, "p_value": 1.5}, + {**base, "verdict_rule": {**base["verdict_rule"], "alpha": 1.5}}): + assert recompute_verdict(result) is None + assert "verdict_rule cannot be applied" in comparability(result) + assert compare(result, base) + + def test_lower_is_better_turns_a_negative_delta_into_improved(): result = _fixture("verdict_with_rule.json") result["metric_spec"]["higher_is_better"] = False From 1cd1ebb674d5b5ead5167bfdd2911518eaee1eb8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 23:31:36 -0800 Subject: [PATCH 08/14] subjects: a harness change is not comparable when both results carry a protocol The protocol digest does not cover harness, and the fallback check that compared it only ran without protocols, so two results from different harnesses compared as equal. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/subjects/evidence_check.py | 3 +++ dokimi/tests/test_evidence_protocol.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index 1c385c956..c5b369f54 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -215,6 +215,9 @@ def compare(result: Mapping[str, Any], reference: Mapping[str, Any]) -> list[str mine, theirs = result.get("protocol"), reference.get("protocol") if isinstance(mine, Mapping) and isinstance(theirs, Mapping): reasons += _protocol_reasons(mine, theirs) + # harness is not a protocol input, so a digest over the inputs cannot tell two harnesses apart + if result.get("harness") != reference.get("harness"): + reasons.append("harness differs") elif isinstance(mine, Mapping) or isinstance(theirs, Mapping): reasons.append("protocol on one side only: the fallback fields cannot stand for its inputs") else: diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index d2d24a65f..0856262a8 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -385,3 +385,9 @@ def test_a_non_json_resolved_value_is_a_reason_not_an_exception(): result["protocol"]["resolved"]["judge_sampling"] = {"temperature": float("nan")} reasons = compare(result, _fixture("verdict_with_rule.json")) assert any("is not JSON" in reason for reason in reasons) + + +def test_a_harness_change_is_a_reason_when_both_sides_carry_a_protocol(): + # harness is not a protocol input, so the digest cannot tell two harnesses apart + base = _fixture("verdict_with_rule.json") + assert compare(_with(base, harness="another-harness"), base) == ["harness differs"] From a9046108915cf9aa57d0a6c42d003e3b23f749c9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:51:12 -0800 Subject: [PATCH 09/14] Address review: a questions file not found from the cwd leaves its digest unresolved Thread: PRRT_kwDOVBg7g864IfKs Signed-off-by: Nikolay Petrov --- dokimi/dokimi/schemas/subject_scorecard.v2.json | 2 +- dokimi/dokimi/subjects/scorecard.py | 9 +++++---- dokimi/tests/test_evidence_protocol.py | 6 ++++++ dokimi/tests/test_subjects_digest.py | 6 +++--- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/schemas/subject_scorecard.v2.json b/dokimi/dokimi/schemas/subject_scorecard.v2.json index b054ac2b8..48393803e 100644 --- a/dokimi/dokimi/schemas/subject_scorecard.v2.json +++ b/dokimi/dokimi/schemas/subject_scorecard.v2.json @@ -90,7 +90,7 @@ } }, "question_set_hash": { - "type": "string" + "type": ["string", "null"] }, "tasks": { "type": "array", diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 2cccfecf6..48bc07f8b 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -140,14 +140,15 @@ def build_subject_scorecard( } -def _digest_values(values: tuple[str, ...]) -> str: +def _digest_values(values: tuple[str, ...]) -> str | None: h = hashlib.sha256() for value in values: + path = Path(value) + if not path.is_file(): # the name alone is no identity: the input stays unresolved + return None h.update(os.fsencode(value)) h.update(b"\0") - path = Path(value) - if path.is_file(): - h.update(path.read_bytes()) + h.update(path.read_bytes()) return "sha256:" + h.hexdigest() diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index 0856262a8..452023de5 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -391,3 +391,9 @@ def test_a_harness_change_is_a_reason_when_both_sides_carry_a_protocol(): # harness is not a protocol input, so the digest cannot tell two harnesses apart base = _fixture("verdict_with_rule.json") assert compare(_with(base, harness="another-harness"), base) == ["harness differs"] + + +def test_a_questions_file_not_found_from_the_cwd_leaves_its_digest_unresolved(): + # the name alone is no identity: two sets named q.json would share one digest, so the input stays unknown + from dokimi.subjects.scorecard import _digest_values + assert _digest_values(("questions/q.json",)) is None diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 99e400e9b..2c4aebf2c 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -100,9 +100,9 @@ def test_a_ref_that_is_no_path_and_holds_a_lone_surrogate_is_digested(tmp_path: @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") -def test_a_question_value_with_a_lone_surrogate_is_digested(): - digest = _digest_values(("q\udcff",)) - assert digest.startswith("sha256:") and digest != _digest_values(("q\udcfe",)) +def test_a_question_value_with_a_lone_surrogate_that_is_no_file_stays_unresolved(): + # a name the cwd does not show is no identity, so the digest is None and the name is not hashed + assert _digest_values(("q\udcff",)) is None @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") From dfd982ededdd7f4e863e521159603e353eb7c54b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:51:38 -0800 Subject: [PATCH 10/14] Address review: a mixed judge context is an unresolved protocol input Thread: PRRT_kwDOVBg7g864Z2pL Signed-off-by: Nikolay Petrov --- dokimi/dokimi/subjects/evidence_check.py | 4 +++- dokimi/tests/test_evidence_protocol.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index c5b369f54..6dffa07d1 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -23,6 +23,8 @@ import re from typing import Any, Iterable, Mapping +from dokimi.eval.judge_context import PROTOCOL_MIXED + #: What dokimi's awareness arms measure; a change to what the digest covers bumps the version. PROTOCOL_ID = "dokimi.awareness" PROTOCOL_VERSION = "1" @@ -83,7 +85,7 @@ def build_protocol(inputs: Mapping[str, Any]) -> dict[str, Any]: aliases: dict[str, str] = {} for key in sorted(inputs): value = inputs[key] - if value is None: + if value is None or value == PROTOCOL_MIXED: # mixed: some scores saw other inputs than the rest unresolved.append(key) elif is_alias(value): unresolved.append(key) diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index 452023de5..8ec1c699f 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -397,3 +397,10 @@ def test_a_questions_file_not_found_from_the_cwd_leaves_its_digest_unresolved(): # the name alone is no identity: two sets named q.json would share one digest, so the input stays unknown from dokimi.subjects.scorecard import _digest_values assert _digest_values(("questions/q.json",)) is None + + +def test_a_mixed_judge_context_is_an_unresolved_input_not_a_measurement_identity(): + # some scores saw the stored material and some only the snippets: two such runs must not compare clean + from dokimi.eval.judge_context import PROTOCOL_MIXED + protocol = build_protocol({"judge_context": PROTOCOL_MIXED, "repeats": 3}) + assert "judge_context" in protocol["unresolved"] and "judge_context" not in protocol["resolved"] From 0da9b1bda4b6c1a5e371ec3700c61792f9399279 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:21:30 -0800 Subject: [PATCH 11/14] Address review: the missing-file digest is unresolved (item b) The change is in a904610891 (Thread: PRRT_kwDOVBg7g864IfKs); this commit names item b of that thread. Thread: PRRT_kwDOVBg7g864IfKs.b Signed-off-by: Nikolay Petrov From fb83e2820151393bd101085bbddcf1445dd4578e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:23:18 -0800 Subject: [PATCH 12/14] Address review: narrow the lone-surrogate digest test to the missing-file contract A question file that is not found now has no digest, so the test that expected a hash of the name is replaced. Test-narrowed: test_a_question_value_with_a_lone_surrogate_is_digested: a missing file is unresolved (None), not hashed by name; the name-hash contract is gone. Thread: PRRT_kwDOVBg7g864IfKs.b Signed-off-by: Nikolay Petrov From 14af859ee4a78bddae7101eb6fd7a33f0c7951b3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:24:20 -0800 Subject: [PATCH 13/14] Address review: say why the lone-surrogate digest test was narrowed Test-narrowed: test_a_question_value_with_a_lone_surrogate_is_digested now asserts None: a missing file has no digest, the name is not hashed. Thread: PRRT_kwDOVBg7g864IfKs.b Signed-off-by: Nikolay Petrov --- dokimi/tests/test_subjects_digest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 2c4aebf2c..719a1ea03 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -100,6 +100,7 @@ def test_a_ref_that_is_no_path_and_holds_a_lone_surrogate_is_digested(tmp_path: @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") +# Test-narrowed: test_a_question_value_with_a_lone_surrogate_is_digested now asserts None: a missing file has no digest, the name is not hashed. def test_a_question_value_with_a_lone_surrogate_that_is_no_file_stays_unresolved(): # a name the cwd does not show is no identity, so the digest is None and the name is not hashed assert _digest_values(("q\udcff",)) is None From 46f07a2a4df051a4535ccbe0904fe90e6015faa2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 16:03:57 -0800 Subject: [PATCH 14/14] Address review: family_tested is a protocol input; a recorded judge snapshot resolves an alias Protocol version 2: family_tested joins the resolved inputs, so a result corrected over a different number of tests is not comparable with one corrected over another number. Version 1 digests are not comparable with version 2. A claude-cli judge records the snapshot it ran (judge_resolved_models). When the stated judge is an alias and exactly one snapshot was recorded, the protocol resolves the judge to it; two or none leave it unresolved. The rollup and scorecard cells carry judge_resolved_models when the run recorded it, and the skill_evidence and cell schemas describe it. Thread: PRRT_kwDOVBg7g864Z2pR Thread: PRRT_kwDOVBg7g864Z2pW Signed-off-by: Nikolay Petrov --- ...1117_170000_napetrov_protocol-version-2.md | 12 +++ dokimi/dokimi/eval/matrix_rollup.py | 2 + dokimi/dokimi/schemas/arms.v1.json | 7 ++ dokimi/dokimi/schemas/matrix_rollup.v1.json | 6 ++ dokimi/dokimi/schemas/skill_evidence.v1.json | 2 +- .../dokimi/schemas/subject_scorecard.v2.json | 6 ++ dokimi/dokimi/subjects/evidence.py | 8 +- dokimi/dokimi/subjects/evidence_check.py | 6 +- dokimi/dokimi/subjects/scorecard.py | 2 + .../fixtures/evidence_conformance/README.md | 2 +- .../judge_snapshot_newer.json | 5 +- .../judge_snapshot_older.json | 5 +- .../verdict_with_rule.json | 5 +- .../verdict_without_rule.json | 5 +- dokimi/tests/test_evidence_protocol.py | 84 +++++++++++++++++++ dokimi/tests/test_subjects_evidence.py | 3 +- 16 files changed, 145 insertions(+), 15 deletions(-) create mode 100644 dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md diff --git a/dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md b/dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md new file mode 100644 index 000000000..55554fda8 --- /dev/null +++ b/dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md @@ -0,0 +1,12 @@ + + +### Changed + +- **`dokimi.awareness` protocol is version 2**: `family_tested` (the multiplier the correction ran over) + is a protocol input, so a result corrected over a different number of tests is no longer comparable + with one that was not. Version 1 digests cannot be compared with version 2. A judge that + recorded exactly one snapshot resolves an alias judge (`opus`) to it; the rollup and scorecard cells + carry `judge_resolved_models`. diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index ad54dc98b..8c15a8736 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -36,6 +36,8 @@ def build_matrix_rollup( "judge_context": data.get("judge_context"), "judge_temperature": data.get("judge_temperature"), "judge_prompt_sha256": data.get("judge_prompt_sha256"), + **({"judge_resolved_models": data["judge_resolved_models"]} + if "judge_resolved_models" in data else {}), "summary": data.get("summary", {}), "cost_summary": data.get("cost_summary", {}), } diff --git a/dokimi/dokimi/schemas/arms.v1.json b/dokimi/dokimi/schemas/arms.v1.json index ca58c79f8..25f1ffb47 100644 --- a/dokimi/dokimi/schemas/arms.v1.json +++ b/dokimi/dokimi/schemas/arms.v1.json @@ -57,6 +57,13 @@ "null" ] }, + "judge_resolved_models": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the snapshot ids the judge reported, sorted; present when the run recorded them" + }, "judge_context": { "description": "Protocol for what the judge was shown about each arm's treatment material. treatment_material.v1 when every score saw the stored material; treatment_material.v1+snippet_fallback when only some did (pools with nothing). Absent when every score used the 200-char report snippets — the protocol of runs judged before the stamp existed.", "type": "string" diff --git a/dokimi/dokimi/schemas/matrix_rollup.v1.json b/dokimi/dokimi/schemas/matrix_rollup.v1.json index 1abd742d9..761367e58 100644 --- a/dokimi/dokimi/schemas/matrix_rollup.v1.json +++ b/dokimi/dokimi/schemas/matrix_rollup.v1.json @@ -97,6 +97,12 @@ "null" ] }, + "judge_resolved_models": { + "type": "array", + "items": { + "type": "string" + } + }, "judge_provider": { "type": [ "string", diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json index 35621ecc4..41bfa44a3 100644 --- a/dokimi/dokimi/schemas/skill_evidence.v1.json +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -181,7 +181,7 @@ "string", "null" ], - "description": "judge snapshot id when the run recorded one, else the configured judge model" + "description": "judge snapshot id when the run recorded exactly one for an alias judge, else the configured judge model" }, "judge_provider": { "type": [ diff --git a/dokimi/dokimi/schemas/subject_scorecard.v2.json b/dokimi/dokimi/schemas/subject_scorecard.v2.json index 48393803e..480531ff8 100644 --- a/dokimi/dokimi/schemas/subject_scorecard.v2.json +++ b/dokimi/dokimi/schemas/subject_scorecard.v2.json @@ -236,6 +236,12 @@ "null" ] }, + "judge_resolved_models": { + "type": "array", + "items": { + "type": "string" + } + }, "atoms": { "type": "object" }, diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index fa48214c2..62234e6eb 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -20,7 +20,7 @@ from dokimi import __version__ from dokimi.eval.comparison_verdict import decide from dokimi.eval.sign_flip import ALPHA -from dokimi.subjects.evidence_check import build_protocol, comparability +from dokimi.subjects.evidence_check import build_protocol, comparability, is_alias #: The arm metric the awareness summary carries per arm (``summary.per_arm[arm]``). METRIC = "avg_aggregate" @@ -113,6 +113,11 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], delta = sig.get("delta_mean") if sig else None correction = sig.get("correction") judge = cell.get("judge_model") + # A claude-cli judge records the snapshot it ran; one recorded id resolves an alias judge (``opus``). + # Two ids, or none, leave the alias unresolved: which one scored is not known. + snapshots = cell.get("judge_resolved_models") + if is_alias(judge) and isinstance(snapshots, list) and len(snapshots) == 1 and isinstance(snapshots[0], str): + judge = snapshots[0] sampling = cell.get("judge_sampling") if sampling is None and cell.get("judge_temperature") is not None: sampling = {"temperature": cell.get("judge_temperature")} @@ -136,6 +141,7 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], "repeats": cell.get("repeats"), "test": test, "correction": correction, + "family_tested": sig.get("family_tested"), "alpha": ALPHA, "ci_confidence": sig.get("ci_confidence"), }) diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py index 6dffa07d1..603f45e21 100644 --- a/dokimi/dokimi/subjects/evidence_check.py +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -27,7 +27,7 @@ #: What dokimi's awareness arms measure; a change to what the digest covers bumps the version. PROTOCOL_ID = "dokimi.awareness" -PROTOCOL_VERSION = "1" +PROTOCOL_VERSION = "2" #: A value that names a moving target rather than one snapshot. Matched as a whole trailing token, #: so ``claude-sonnet-5-5`` is resolved and ``claude-sonnet-latest`` / ``gpt:default`` are not. @@ -196,8 +196,8 @@ def _verdict_reasons(result: Mapping[str, Any], spec: Mapping[str, Any]) -> list #: Without a protocol on both sides, the fields that must be equal instead. _FALLBACK_KEYS = ("harness", "judge_model", "judge_provider", "metric", "verdict_rule") -#: The inputs a ``dokimi.awareness`` version 1 protocol names; one it leaves out is unresolved. -REQUIRED_INPUTS = ("alpha", "ci_confidence", "correction", "eval_set_digest", "judge_context", +#: The inputs a ``dokimi.awareness`` version 2 protocol names; one it leaves out is unresolved. +REQUIRED_INPUTS = ("alpha", "ci_confidence", "correction", "eval_set_digest", "family_tested", "judge_context", "judge_model", "judge_prompt_sha256", "judge_provider", "judge_sampling", "metric", "repeats", "test") #: Fields a result states that its protocol resolves too: a digest over the resolved inputs cannot diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 48bc07f8b..0c63c4fdd 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -68,6 +68,8 @@ def build_subject_scorecard( "judge_context": cell.get("judge_context"), "judge_temperature": cell.get("judge_temperature"), "judge_prompt_sha256": cell.get("judge_prompt_sha256"), + **({"judge_resolved_models": cell["judge_resolved_models"]} + if "judge_resolved_models" in cell else {}), "plugin_set": cell.get("plugin_set"), "summary": cell.get("summary", {}), "cost_summary": cell.get("cost_summary", {}), diff --git a/dokimi/tests/fixtures/evidence_conformance/README.md b/dokimi/tests/fixtures/evidence_conformance/README.md index d65ca80cc..c1325a563 100644 --- a/dokimi/tests/fixtures/evidence_conformance/README.md +++ b/dokimi/tests/fixtures/evidence_conformance/README.md @@ -13,4 +13,4 @@ Each file is a valid `skill_evidence.v1` document with one result. A consumer pa | `verdict_with_rule.json` | Recomputing the verdict from `delta`, `p_value`, `verdict_rule` and `metric_spec` gives the stated `improved`. | | `judge_snapshot_older.json`, `judge_snapshot_newer.json` | The same protocol resolved against judge snapshots `…-20260901` and `…-20261101`: the `protocol.digest` values differ, and the newer result is non-comparable to the older one. | -`protocol.digest` is `sha256:` + the hex sha256 of `protocol.resolved` as JSON with sorted keys, no whitespace, UTF-8 and no NaN, recursively normalizing integral floats to integers (for example `0.0` to `0`). `tests/test_evidence_protocol.py` runs these checks against `dokimi.subjects.evidence_check`. +`protocol.digest` is `sha256:` + the hex sha256 of `protocol.resolved` as JSON with sorted keys, no whitespace, UTF-8 and no NaN, recursively normalizing integral floats to integers (for example `0.0` to `0`). Protocol version 2 resolves `family_tested`, the number of results the multiple-comparison correction ran over; the version-1 digests are not comparable with these. `tests/test_evidence_protocol.py` runs these checks against `dokimi.subjects.evidence_check`. diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json index db9304690..9048e6ed4 100644 --- a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json @@ -36,13 +36,14 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:8a3cd1fb5426d92bf07b28f43afef9d4821574e468735b1a7d109f4a1651e3dc", + "digest": "sha256:872a6210214f38ed9e5eb5a789c1c9f712d3dbcf952387625323d56c88547730", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, "ci_confidence": 0.95, "correction": "holm", "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, "judge_context": "report-snippets", "judge_model": "claude-sonnet-5-5-20261101", "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", @@ -56,7 +57,7 @@ "test": "sign_flip_inversion" }, "unresolved": [], - "version": "1" + "version": "2" }, "provider": "amazon-bedrock", "verdict": "improved", diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json index e65d84a87..19ffa446b 100644 --- a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json @@ -36,13 +36,14 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:e43084f2c2980e45beb24e17459d8e48188c63bab7b45df7616e912959f4c774", + "digest": "sha256:2b8f9912f6deb3e504f2456964ed5a722aabc1eee99f4345a33e40e014ec5cb6", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, "ci_confidence": 0.95, "correction": "holm", "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, "judge_context": "report-snippets", "judge_model": "claude-sonnet-5-5-20260901", "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", @@ -56,7 +57,7 @@ "test": "sign_flip_inversion" }, "unresolved": [], - "version": "1" + "version": "2" }, "provider": "amazon-bedrock", "verdict": "improved", diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json index e65d84a87..19ffa446b 100644 --- a/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json @@ -36,13 +36,14 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:e43084f2c2980e45beb24e17459d8e48188c63bab7b45df7616e912959f4c774", + "digest": "sha256:2b8f9912f6deb3e504f2456964ed5a722aabc1eee99f4345a33e40e014ec5cb6", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, "ci_confidence": 0.95, "correction": "holm", "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, "judge_context": "report-snippets", "judge_model": "claude-sonnet-5-5-20260901", "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", @@ -56,7 +57,7 @@ "test": "sign_flip_inversion" }, "unresolved": [], - "version": "1" + "version": "2" }, "provider": "amazon-bedrock", "verdict": "improved", diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json index 4909c4f62..509e1b3b0 100644 --- a/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json @@ -36,13 +36,14 @@ "p_value": 0.004, "product": "oneTBB", "protocol": { - "digest": "sha256:e43084f2c2980e45beb24e17459d8e48188c63bab7b45df7616e912959f4c774", + "digest": "sha256:2b8f9912f6deb3e504f2456964ed5a722aabc1eee99f4345a33e40e014ec5cb6", "id": "dokimi.awareness", "resolved": { "alpha": 0.05, "ci_confidence": 0.95, "correction": "holm", "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, "judge_context": "report-snippets", "judge_model": "claude-sonnet-5-5-20260901", "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", @@ -56,7 +57,7 @@ "test": "sign_flip_inversion" }, "unresolved": [], - "version": "1" + "version": "2" }, "provider": "amazon-bedrock", "verdict": "improved" diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py index 8ec1c699f..fd423194d 100644 --- a/dokimi/tests/test_evidence_protocol.py +++ b/dokimi/tests/test_evidence_protocol.py @@ -404,3 +404,87 @@ def test_a_mixed_judge_context_is_an_unresolved_input_not_a_measurement_identity from dokimi.eval.judge_context import PROTOCOL_MIXED protocol = build_protocol({"judge_context": PROTOCOL_MIXED, "repeats": 3}) assert "judge_context" in protocol["unresolved"] and "judge_context" not in protocol["resolved"] + + +# --- the family multiplier and the recorded judge snapshot (self-review round 10) ---------------- + +def test_the_family_tested_multiplier_is_a_protocol_input_of_version_2(): + from dokimi.subjects.evidence_check import PROTOCOL_VERSION, REQUIRED_INPUTS + + assert PROTOCOL_VERSION == "2" + assert "family_tested" in REQUIRED_INPUTS + + +def test_a_result_corrected_over_a_different_number_of_tests_is_not_comparable(): + from test_subjects_evidence import _scorecard + + def skill_result(tested: int) -> dict: + scorecard = _scorecard() + scorecard["cells"][0].update({ + "judge_model": "claude-sonnet-5-5-20260901", "judge_provider": "amazon-bedrock", + "judge_context": "report-snippets", "judge_sampling": {"temperature": 0.0}, + "judge_prompt_sha256": "sha256:" + "d" * 64, "repeats": 3}) + scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"]["family_tested"] = tested + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + return skill + + two, six = skill_result(2), skill_result(6) + + assert two["protocol"]["resolved"]["family_tested"] == 2 + assert two["protocol"]["digest"] != six["protocol"]["digest"] + assert compare(six, two) == ["protocol digest differs (family_tested)"] + + +def test_a_family_tested_the_scorecard_does_not_record_is_unresolved(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + del scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"]["family_tested"] + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert "family_tested" in skill["protocol"]["unresolved"] + assert any("family_tested" in reason for reason in compare(skill, skill)) + + +def test_a_judge_alias_is_resolved_by_the_one_snapshot_the_run_recorded(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + scorecard["cells"][0].update({"judge_model": "opus", "judge_resolved_models": ["claude-opus-5-5-20261101"]}) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert skill["judge_model"] == "claude-opus-5-5-20261101" + assert skill["protocol"]["resolved"]["judge_model"] == "claude-opus-5-5-20261101" + assert "judge_model" not in skill["protocol"]["unresolved"] + + +def test_a_judge_alias_with_two_recorded_snapshots_stays_unresolved(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + scorecard["cells"][0].update({"judge_model": "opus", "judge_resolved_models": [ + "claude-opus-5-5-20260901", "claude-opus-5-5-20261101"]}) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert skill["judge_model"] == "opus" + assert "judge_model" in skill["protocol"]["unresolved"] + assert any("judge_model" in reason for reason in compare(skill, skill)) + + +def test_the_rollup_and_the_scorecard_cell_carry_the_recorded_judge_snapshots(tmp_path: Path): + from dokimi.eval.matrix_rollup import build_matrix_rollup + from dokimi.subjects import build_subject_scorecard, load_subject + + arms = {"baseline_arm": "none", "judge_model": "opus", "judge_resolved_models": ["claude-opus-5-5-20261101"]} + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", arms)]) + assert rollup["cells"][0]["judge_resolved_models"] == ["claude-opus-5-5-20261101"] + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["p"]\nquestions = ["q.json"]\n', encoding="utf-8") + scorecard = build_subject_scorecard( + load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=[{"product": "p", "questions": "q.json", "matrix_rollup": "m.json", + "rollup": {"cells": [rollup["cells"][0]]}}]) + assert scorecard["cells"][0]["judge_resolved_models"] == ["claude-opus-5-5-20261101"] diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 29db800db..7bcdd9900 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -21,7 +21,8 @@ def _significance(delta: float, *, significant: bool) -> dict: return {"ci_confidence": 0.95, "ci_lo": delta - 1.5, "ci_hi": delta + 1.5, "ci_method": "student_t", - "correction": "holm", "delta_mean": delta, "n": 6, "p_adjusted": 0.001 if significant else 0.4, + "correction": "holm", "delta_mean": delta, "n": 6, "family_tested": 2, + "p_adjusted": 0.001 if significant else 0.4, "significant_adjusted": significant}