diff --git a/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md b/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md new file mode 100644 index 000000000..29bee08a5 --- /dev/null +++ b/dokimi/changelog.d/20261115_120000_napetrov_evidence-protocol.md @@ -0,0 +1,8 @@ + + +### Added + +- **`subjects evidence`: each result says when it may be compared.** A result now carries `metric_spec` (unit, range, direction, judged), the `verdict_rule` its verdict follows (`p_adjusted` below `alpha`), and `protocol {id, version, digest}`: the digest is `sha256` over the canonical JSON of the resolved measurement inputs (judge snapshot, sampling, judge prompt hash, eval-set digest, repeats, test, correction, thresholds). A missing input or an alias such as `latest` is listed in `protocol.unresolved` and left out of the digest. `comparable`/`non_comparable` say why a result cannot be compared. `dokimi.subjects.evidence_check` holds the consumer checks, with conformance fixtures in `tests/fixtures/evidence_conformance`. Matrix-rollup and scorecard cells now carry the judge the run recorded (`judge_model`, `judge_provider`, `judge_context`, `judge_temperature`, `judge_prompt_sha256`). Runs do not record a repeat count, and a judge named by a bare family name (`opus`, `sonnet`) is an alias, so such results list those inputs as unresolved and are not comparable across runs. diff --git a/dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md b/dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md new file mode 100644 index 000000000..55554fda8 --- /dev/null +++ b/dokimi/changelog.d/20261117_170000_napetrov_protocol-version-2.md @@ -0,0 +1,12 @@ + + +### Changed + +- **`dokimi.awareness` protocol is version 2**: `family_tested` (the multiplier the correction ran over) + is a protocol input, so a result corrected over a different number of tests is no longer comparable + with one that was not. Version 1 digests cannot be compared with version 2. A judge that + recorded exactly one snapshot resolves an alias judge (`opus`) to it; the rollup and scorecard cells + carry `judge_resolved_models`. diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index 35dddc612..8c15a8736 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -31,6 +31,13 @@ def build_matrix_rollup( "plugin_set": data.get("plugin_set", "none"), "plugin_set_id": data.get("plugin_set_id"), "baseline_arm": data.get("baseline_arm"), + "judge_model": data.get("judge_model"), + "judge_provider": data.get("judge_provider"), + "judge_context": data.get("judge_context"), + "judge_temperature": data.get("judge_temperature"), + "judge_prompt_sha256": data.get("judge_prompt_sha256"), + **({"judge_resolved_models": data["judge_resolved_models"]} + if "judge_resolved_models" in data else {}), "summary": data.get("summary", {}), "cost_summary": data.get("cost_summary", {}), } diff --git a/dokimi/dokimi/schemas/arms.v1.json b/dokimi/dokimi/schemas/arms.v1.json index ca58c79f8..25f1ffb47 100644 --- a/dokimi/dokimi/schemas/arms.v1.json +++ b/dokimi/dokimi/schemas/arms.v1.json @@ -57,6 +57,13 @@ "null" ] }, + "judge_resolved_models": { + "type": "array", + "items": { + "type": "string" + }, + "description": "the snapshot ids the judge reported, sorted; present when the run recorded them" + }, "judge_context": { "description": "Protocol for what the judge was shown about each arm's treatment material. treatment_material.v1 when every score saw the stored material; treatment_material.v1+snippet_fallback when only some did (pools with nothing). Absent when every score used the 200-char report snippets — the protocol of runs judged before the stamp existed.", "type": "string" diff --git a/dokimi/dokimi/schemas/matrix_rollup.v1.json b/dokimi/dokimi/schemas/matrix_rollup.v1.json index 3628a6446..761367e58 100644 --- a/dokimi/dokimi/schemas/matrix_rollup.v1.json +++ b/dokimi/dokimi/schemas/matrix_rollup.v1.json @@ -90,6 +90,42 @@ "string", "null" ] + }, + "judge_model": { + "type": [ + "string", + "null" + ] + }, + "judge_resolved_models": { + "type": "array", + "items": { + "type": "string" + } + }, + "judge_provider": { + "type": [ + "string", + "null" + ] + }, + "judge_context": { + "type": [ + "string", + "null" + ] + }, + "judge_temperature": { + "type": [ + "number", + "null" + ] + }, + "judge_prompt_sha256": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": true diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json index ecbe48172..41bfa44a3 100644 --- a/dokimi/dokimi/schemas/skill_evidence.v1.json +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -177,6 +177,13 @@ ] }, "judge_model": { + "type": [ + "string", + "null" + ], + "description": "judge snapshot id when the run recorded exactly one for an alias judge, else the configured judge model" + }, + "judge_provider": { "type": [ "string", "null" @@ -189,6 +196,30 @@ "type": "string", "description": "dokimi's mean judge aggregate score per answer (higher is better); delta is arm minus baseline" }, + "metric_spec": { + "type": "object", + "description": "what the metric is; a verdict needs unit and higher_is_better to be comparable", + "properties": { + "id": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "range": { + "type": "array", + "items": { + "type": "number" + } + }, + "higher_is_better": { + "type": "boolean" + }, + "judged": { + "type": "boolean" + } + } + }, "n": { "type": [ "integer", @@ -245,12 +276,85 @@ "null" ] }, + "verdict_rule": { + "type": "object", + "description": "how verdict follows from delta and p_value; a consumer recomputes it", + "properties": { + "basis": { + "enum": [ + "p_adjusted" + ] + }, + "alpha": { + "type": "number" + }, + "test": { + "type": "string" + }, + "correction": { + "type": [ + "string", + "null" + ] + }, + "p_decimals": { + "type": "integer", + "description": "places p_value is rounded to; within that of alpha a verdict cannot be recomputed" + } + } + }, "verdict": { "enum": [ "improved", "regressed", "inconclusive" ] + }, + "protocol": { + "type": "object", + "description": "the measurement: digest is sha256 of the canonical JSON (sorted keys, no whitespace, UTF-8) of resolved; missing or alias inputs are listed in unresolved, outside the digest", + "required": [ + "id", + "version", + "digest" + ], + "properties": { + "id": { + "type": "string" + }, + "version": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "resolved": { + "type": "object" + }, + "unresolved": { + "type": "array", + "items": { + "type": "string" + } + }, + "aliases": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "comparable": { + "type": "boolean" + }, + "non_comparable": { + "type": "array", + "items": { + "type": "string" + }, + "description": "why this result cannot be compared or shown with its verdict" } } } diff --git a/dokimi/dokimi/schemas/subject_scorecard.v2.json b/dokimi/dokimi/schemas/subject_scorecard.v2.json index b054ac2b8..480531ff8 100644 --- a/dokimi/dokimi/schemas/subject_scorecard.v2.json +++ b/dokimi/dokimi/schemas/subject_scorecard.v2.json @@ -90,7 +90,7 @@ } }, "question_set_hash": { - "type": "string" + "type": ["string", "null"] }, "tasks": { "type": "array", @@ -236,6 +236,12 @@ "null" ] }, + "judge_resolved_models": { + "type": "array", + "items": { + "type": "string" + } + }, "atoms": { "type": "object" }, diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index 0f92dccc9..62234e6eb 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -19,6 +19,8 @@ from dokimi import __version__ from dokimi.eval.comparison_verdict import decide +from dokimi.eval.sign_flip import ALPHA +from dokimi.subjects.evidence_check import build_protocol, comparability, is_alias #: The arm metric the awareness summary carries per arm (``summary.per_arm[arm]``). METRIC = "avg_aggregate" @@ -40,7 +42,7 @@ def build_skill_evidence(scorecard: dict[str, Any], *, source: str | None = None if baseline is None: warnings.append(f"matrix cell {cell.get('matrix_cell')!r}: no single baseline arm, skipped") continue - results.extend(_result(cell, arm, row) for arm, row in per_arm.items() if arm != baseline) + results.extend(_result(cell, arm, row, suite) for arm, row in per_arm.items() if arm != baseline) if subject.get("distribution_digest") is None: warnings.append("subject has no distribution_digest (not a SKILL.md, a skill with resources, " "or a non-local ref): a client cannot match this document by digest") @@ -85,12 +87,65 @@ def _baseline_arm(cell: dict[str, Any], per_arm: dict[str, Any]) -> str | None: return candidates[0] if len(candidates) == 1 else None -def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, Any]: +#: The places ``stats._round_p`` rounds ``p_adjusted`` to before it reaches the scorecard. +P_DECIMALS = 4 + +#: What ``METRIC`` is: a 0-100 judge score per answer. +METRIC_SPEC = {"id": METRIC, "unit": "score_points", "range": [0, 100], + "higher_is_better": True, "judged": True} + + +def _eval_set_digest(cell: dict[str, Any], suite: dict[str, Any] | None) -> str | None: + """The digest of the question set this cell measured; the suite's only when one set serves every cell.""" + if cell.get("questions_digest") is not None: + return cell["questions_digest"] + questions = (suite or {}).get("questions") or [] + if len(questions) == 1: + return (suite or {}).get("question_set_hash") + return None + + +def _result(cell: dict[str, Any], arm: str, row: dict[str, Any], + suite: dict[str, Any] | None = None) -> dict[str, Any]: # Every statistic is the paired test's, so n, delta, CI and p describe the same questions. # The per-arm means are left out: their difference is not the delta once pairs drop out. sig = row.get("significance") or {} delta = sig.get("delta_mean") if sig else None - return { + correction = sig.get("correction") + judge = cell.get("judge_model") + # A claude-cli judge records the snapshot it ran; one recorded id resolves an alias judge (``opus``). + # Two ids, or none, leave the alias unresolved: which one scored is not known. + snapshots = cell.get("judge_resolved_models") + if is_alias(judge) and isinstance(snapshots, list) and len(snapshots) == 1 and isinstance(snapshots[0], str): + judge = snapshots[0] + sampling = cell.get("judge_sampling") + if sampling is None and cell.get("judge_temperature") is not None: + sampling = {"temperature": cell.get("judge_temperature")} + # The recorded ci_method, not the current implementation's constant: a historical scorecard keeps + # the method it ran. Unrecorded, it is None and the protocol lists it as unresolved. + test = sig.get("ci_method") + # The rule the verdict below applies, so a consumer can recompute it from delta and p_value. + # p_value is p_adjusted rounded to P_DECIMALS; within that rounding of alpha it cannot be recomputed. + rule = {"basis": "p_adjusted", "alpha": ALPHA, "correction": correction, "p_decimals": P_DECIMALS} + if test is not None: # the schema has no null test: an unrecorded one is left out, not written as null + rule["test"] = test + # Only resolved values enter the digest; what the scorecard does not record stays unresolved. + protocol = build_protocol({ + "eval_set_digest": _eval_set_digest(cell, suite), + "metric": METRIC, + "judge_model": judge, + "judge_provider": cell.get("judge_provider"), + "judge_context": cell.get("judge_context"), + "judge_sampling": sampling, + "judge_prompt_sha256": cell.get("judge_prompt_sha256"), + "repeats": cell.get("repeats"), + "test": test, + "correction": correction, + "family_tested": sig.get("family_tested"), + "alpha": ALPHA, + "ci_confidence": sig.get("ci_confidence"), + }) + result = { "product": cell.get("product"), "matrix_cell": cell.get("matrix_cell"), "model": cell.get("model"), @@ -98,9 +153,11 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, An "harness": cell.get("harness"), # Absent until a run records the effort it applied; null means "not recorded", not "default". "effort": cell.get("effort"), - "judge_model": cell.get("judge_model"), + "judge_model": judge, + "judge_provider": cell.get("judge_provider"), "arm": arm, "metric": METRIC, + "metric_spec": dict(METRIC_SPEC), "n": sig.get("n"), "delta": delta, "ci": None if sig.get("ci_lo") is None else { @@ -109,8 +166,14 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, An }, "p_value": sig.get("p_adjusted"), "p_correction": sig.get("correction"), + "verdict_rule": rule, "verdict": _verdict(delta, sig), + "protocol": protocol, } + reasons = comparability(result) + result["comparable"] = not reasons + result["non_comparable"] = reasons + return result def _verdict(delta: float | None, sig: dict[str, Any]) -> str: diff --git a/dokimi/dokimi/subjects/evidence_check.py b/dokimi/dokimi/subjects/evidence_check.py new file mode 100644 index 000000000..603f45e21 --- /dev/null +++ b/dokimi/dokimi/subjects/evidence_check.py @@ -0,0 +1,292 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""When two ``skill_evidence.v1`` results may be compared: the protocol digest and the consumer checks. + +A subject digest pins the bytes of a skill, not how they were measured, so two results about the +same digest can still disagree on judge, sampling, prompt or thresholds. ``protocol.digest`` names +the measurement: ``sha256`` of the canonical JSON of the *resolved* inputs that produced the number. +An input that is missing, or that names an alias (``latest``, ``default``), is left out of the +digest and listed in ``protocol.unresolved``: an alias resolves differently over time, so hashing +it would give two different measurements one digest. + +The consumer side is :func:`comparability` (can this one result be merged or shown with a verdict +at all) and :func:`compare` (may these two be put side by side). The conformance fixtures under +``tests/fixtures/evidence_conformance`` pin both. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from typing import Any, Iterable, Mapping + +from dokimi.eval.judge_context import PROTOCOL_MIXED + +#: What dokimi's awareness arms measure; a change to what the digest covers bumps the version. +PROTOCOL_ID = "dokimi.awareness" +PROTOCOL_VERSION = "2" + +#: A value that names a moving target rather than one snapshot. Matched as a whole trailing token, +#: so ``claude-sonnet-5-5`` is resolved and ``claude-sonnet-latest`` / ``gpt:default`` are not. +_ALIAS = re.compile(r"(?:^|[-_.:@/])(?:latest|default|auto|stable|current)$", re.IGNORECASE) +#: The 1M-context suffix a CLI takes after an alias (``opus[1m]``); it does not make the name a snapshot. +_CONTEXT_SUFFIX = re.compile(r"\[1m\]$", re.IGNORECASE) +#: Bare family names a harness resolves to whatever it ships today (dokimi's own claude-cli judges +#: default to ``opus``/``sonnet``); a dated or versioned id is a snapshot. +_FLOATING = re.compile(r"^(?:claude|opus|sonnet|haiku|gpt-4o(?:-mini)?|gpt-4|gpt-5|o1|o3|o4-mini)$", + re.IGNORECASE) + +VERDICTS = ("improved", "regressed", "inconclusive") + + +def _normal(value: Any) -> Any: + # An integral float is written as an int, so 0 and 0.0 (or 3 and 3.0) are one measurement. + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, Mapping): + return {str(k): _normal(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_normal(v) for v in value] + return value + + +def canonical_json(value: Any) -> bytes: + """Sorted keys, no whitespace, UTF-8, no NaN, integral floats as ints: the bytes the digest is over.""" + return json.dumps(_normal(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False, + allow_nan=False).encode("utf-8") + + +def protocol_digest(resolved: Mapping[str, Any]) -> str: + return "sha256:" + hashlib.sha256(canonical_json(dict(resolved))).hexdigest() + + +def is_alias(value: Any) -> bool: + """A moving target: an alias suffix, a bare family name, or either anywhere inside a nested value.""" + if isinstance(value, str): + text = _CONTEXT_SUFFIX.sub("", value.strip()) + return bool(_ALIAS.search(text) or _FLOATING.match(text.rsplit("/", 1)[-1])) + if isinstance(value, Mapping): + return any(is_alias(v) for v in value.values()) + if isinstance(value, (list, tuple)): + return any(is_alias(v) for v in value) + return False + + +def build_protocol(inputs: Mapping[str, Any]) -> dict[str, Any]: + """``protocol`` for one result: id, version, digest over the resolved inputs, and what was not. + + An alias stays visible under ``aliases`` for a reader, outside the digest. + """ + resolved: dict[str, Any] = {} + unresolved: list[str] = [] + aliases: dict[str, str] = {} + for key in sorted(inputs): + value = inputs[key] + if value is None or value == PROTOCOL_MIXED: # mixed: some scores saw other inputs than the rest + unresolved.append(key) + elif is_alias(value): + unresolved.append(key) + aliases[key] = value if isinstance(value, str) else canonical_json(value).decode("utf-8") + else: + resolved[key] = value + protocol: dict[str, Any] = { + "id": PROTOCOL_ID, + "version": PROTOCOL_VERSION, + "digest": protocol_digest(resolved), + "resolved": resolved, + "unresolved": unresolved, + } + if aliases: + protocol["aliases"] = aliases + return protocol + + +def recompute_verdict(result: Mapping[str, Any]) -> str | None: + """The verdict ``verdict_rule`` gives for this result's numbers; None when the rule is unusable. + + The one basis today is ``p_adjusted``: a direction only when the corrected p is below ``alpha`` + and the delta is not zero; which direction is "improved" comes from ``higher_is_better``. + """ + rule = result.get("verdict_rule") + spec = result.get("metric_spec") + if not isinstance(rule, Mapping) or not isinstance(spec, Mapping): + return None + alpha, better_up = _num(rule.get("alpha")), spec.get("higher_is_better") + if rule.get("basis") != "p_adjusted" or alpha is None or not isinstance(better_up, bool): + return None + if not 0 < alpha < 1: + return None # alpha is a probability: outside (0, 1) the rule says nothing about a verdict + if not _valid_decimals(rule.get("p_decimals"), allow_none=True): + return None # an unsupported rounding precision: the rule cannot be applied + delta, p = _num(result.get("delta")), _num(result.get("p_value")) + if p is not None and not 0 <= p <= 1: + return None # a p-value is a probability: an out-of-range one cannot be on either side of alpha + if p is not None and _near_alpha(p, alpha, rule.get("p_decimals")): + return None # the rounded p cannot say which side of alpha the unrounded one was on + if delta is None or p is None or p >= alpha or delta == 0: + return "inconclusive" + return "improved" if (delta > 0) == better_up else "regressed" + + +def comparability(result: Mapping[str, Any]) -> list[str]: + """Why this result cannot be merged with others or shown with its verdict; empty when it can. + + A non-comparable result can still be listed. It is not "neutral": it says nothing either way. + """ + reasons: list[str] = [] + stated = result.get("metric_spec") + spec: Mapping[str, Any] = stated if isinstance(stated, Mapping) else {} + if result.get("verdict") is not None: + reasons += _verdict_reasons(result, spec) + if spec.get("judged") is not False and not result.get("judge_model"): + reasons.append("judged result without judge_model") + protocol = result.get("protocol") + if isinstance(protocol, Mapping) and isinstance(protocol.get("resolved"), Mapping): + try: + digest = protocol_digest(protocol["resolved"]) + except (TypeError, ValueError): # NaN, infinity or a non-JSON value: no digest to check + reasons.append("protocol.resolved is not JSON, so its digest cannot be checked") + else: + if digest != protocol.get("digest"): + reasons.append("protocol.digest does not match protocol.resolved") + reasons += _disagreements(result, protocol["resolved"]) + return reasons + + +def _disagreements(result: Mapping[str, Any], resolved: Mapping[str, Any]) -> list[str]: + """Stated fields whose value is not the one the protocol resolved. An absent field is no statement: + its own check (a judged result without a judge, a verdict without a rule) reports it.""" + stated = [(key, result.get(key)) for key in _AGREES] + rule = result.get("verdict_rule") + if isinstance(rule, Mapping): + stated += [(f"verdict_rule.{key}", rule.get(key)) for key in _RULE_AGREES] + out = [] + for field, value in stated: + key = field.rsplit(".", 1)[-1] + if value is not None and key in resolved and resolved[key] != value: + out.append(f"{field} differs from protocol.resolved.{key}") + return out + + +def _verdict_reasons(result: Mapping[str, Any], spec: Mapping[str, Any]) -> list[str]: + reasons = [] + if not isinstance(result.get("verdict_rule"), Mapping): + reasons.append("verdict without verdict_rule") + if not spec.get("unit"): + reasons.append("verdict without metric_spec.unit") + if not isinstance(spec.get("higher_is_better"), bool): + reasons.append("verdict without metric_spec.higher_is_better") + if reasons or not isinstance(result.get("verdict_rule"), Mapping): + return reasons + expected = recompute_verdict(result) + rule = result["verdict_rule"] + p, alpha = _num(result.get("p_value")), _num(rule.get("alpha")) + if expected is None and p is not None and alpha is not None and _near_alpha( + p, alpha, rule.get("p_decimals")): + return ["p_value is within its rounding of alpha: the verdict cannot be recomputed"] + if expected is None: + return ["verdict_rule cannot be applied"] + if expected != result.get("verdict"): + return [f"verdict {result.get('verdict')!r} does not follow verdict_rule ({expected!r})"] + return [] + + +#: Without a protocol on both sides, the fields that must be equal instead. +_FALLBACK_KEYS = ("harness", "judge_model", "judge_provider", "metric", "verdict_rule") +#: The inputs a ``dokimi.awareness`` version 2 protocol names; one it leaves out is unresolved. +REQUIRED_INPUTS = ("alpha", "ci_confidence", "correction", "eval_set_digest", "family_tested", "judge_context", + "judge_model", "judge_prompt_sha256", "judge_provider", "judge_sampling", "metric", + "repeats", "test") +#: Fields a result states that its protocol resolves too: a digest over the resolved inputs cannot +#: catch a result that states another value for them. +_AGREES = ("judge_model", "judge_provider", "metric") +_RULE_AGREES = ("alpha", "correction", "test") + + +def compare(result: Mapping[str, Any], reference: Mapping[str, Any]) -> list[str]: + """Why ``result`` cannot be compared with ``reference``; empty when it can.""" + reasons = [f"result: {r}" for r in comparability(result)] + reasons += [f"reference: {r}" for r in comparability(reference)] + if result.get("metric") != reference.get("metric"): + reasons.append("metric differs") + if result.get("metric_spec") != reference.get("metric_spec"): + reasons.append("metric specification differs (unit, range or direction)") + mine, theirs = result.get("protocol"), reference.get("protocol") + if isinstance(mine, Mapping) and isinstance(theirs, Mapping): + reasons += _protocol_reasons(mine, theirs) + # harness is not a protocol input, so a digest over the inputs cannot tell two harnesses apart + if result.get("harness") != reference.get("harness"): + reasons.append("harness differs") + elif isinstance(mine, Mapping) or isinstance(theirs, Mapping): + reasons.append("protocol on one side only: the fallback fields cannot stand for its inputs") + else: + reasons += [f"{key} differs" for key in _FALLBACK_KEYS if result.get(key) != reference.get(key)] + return reasons + + +def _protocol_reasons(mine: Mapping[str, Any], theirs: Mapping[str, Any]) -> list[str]: + reasons = [] + if (mine.get("id"), mine.get("version")) != (theirs.get("id"), theirs.get("version")): + reasons.append("protocol id or version differs") + elif mine.get("digest") != theirs.get("digest"): + changed = sorted(k for k in set(_resolved(mine)) | set(_resolved(theirs)) + if _resolved(mine).get(k) != _resolved(theirs).get(k)) + reasons.append("protocol digest differs" + (f" ({', '.join(changed)})" if changed else "")) + unresolved = sorted(_unresolved(mine) | _unresolved(theirs)) + if unresolved: + reasons.append(f"protocol inputs not resolved: {', '.join(unresolved)}") + return reasons + + +def _unresolved(protocol: Mapping[str, Any]) -> set[str]: + """The inputs the protocol lists as unresolved, the ones it omits (``dokimi.awareness`` 1), and + any resolved value that is null or an alias: a digest over them identifies no fixed measurement.""" + listed = set(protocol.get("unresolved") or []) + if (protocol.get("id"), protocol.get("version")) == (PROTOCOL_ID, PROTOCOL_VERSION): + listed |= set(REQUIRED_INPUTS) - set(_resolved(protocol)) + listed |= {key for key, value in _resolved(protocol).items() if value is None or is_alias(value)} + return listed + + +def mark_against(results: Iterable[Mapping[str, Any]], + reference: Mapping[str, Any]) -> list[dict[str, Any]]: + """Copies of ``results``, each with ``comparable`` and ``non_comparable`` set against ``reference``.""" + out = [] + for result in results: + reasons = compare(result, reference) + out.append({**result, "comparable": not reasons, "non_comparable": reasons}) + return out + + +def _resolved(protocol: Mapping[str, Any]) -> Mapping[str, Any]: + resolved = protocol.get("resolved") + return resolved if isinstance(resolved, Mapping) else {} + + +def _near_alpha(p: float, alpha: float, decimals: Any) -> bool: + if not _valid_decimals(decimals): + return False + return abs(p - alpha) <= 0.5 * 10 ** -decimals + 1e-12 + + +#: The precisions a rounded p can carry: a float holds about 15 significant digits. +_MAX_DECIMALS = 15 + + +def _valid_decimals(value: Any, *, allow_none: bool = False) -> bool: + if value is None: + return allow_none + return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= _MAX_DECIMALS + + +def _num(value: Any) -> float | None: + """A finite number, else None: NaN, infinity and an integer too large for a float say nothing.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + return value if math.isfinite(value) else None + except OverflowError: + return None diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 0dfe243d9..0c63c4fdd 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -54,12 +54,22 @@ def build_subject_scorecard( cells.append({ "product": run["product"], "questions": run["questions"], + # The digest of this run's own question file: a suite with one file per product + # measures a different set in each cell, and suite.question_set_hash covers them all. + "questions_digest": _digest_values((run["questions"],)), "matrix_cell": cell.get("matrix_cell"), "artifact": cell.get("artifact"), "model": cell.get("model"), "provider": cell.get("provider"), "harness": cell.get("harness"), "baseline_arm": cell.get("baseline_arm"), + "judge_model": cell.get("judge_model"), + "judge_provider": cell.get("judge_provider"), + "judge_context": cell.get("judge_context"), + "judge_temperature": cell.get("judge_temperature"), + "judge_prompt_sha256": cell.get("judge_prompt_sha256"), + **({"judge_resolved_models": cell["judge_resolved_models"]} + if "judge_resolved_models" in cell else {}), "plugin_set": cell.get("plugin_set"), "summary": cell.get("summary", {}), "cost_summary": cell.get("cost_summary", {}), @@ -132,14 +142,15 @@ def build_subject_scorecard( } -def _digest_values(values: tuple[str, ...]) -> str: +def _digest_values(values: tuple[str, ...]) -> str | None: h = hashlib.sha256() for value in values: + path = Path(value) + if not path.is_file(): # the name alone is no identity: the input stays unresolved + return None h.update(os.fsencode(value)) h.update(b"\0") - path = Path(value) - if path.is_file(): - h.update(path.read_bytes()) + h.update(path.read_bytes()) return "sha256:" + h.hexdigest() diff --git a/dokimi/tests/fixtures/evidence_conformance/README.md b/dokimi/tests/fixtures/evidence_conformance/README.md new file mode 100644 index 000000000..c1325a563 --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/README.md @@ -0,0 +1,16 @@ + + +# skill_evidence.v1 conformance fixtures + +Each file is a valid `skill_evidence.v1` document with one result. A consumer passes when: + +| File | Expectation | +|---|---| +| `verdict_without_rule.json` | The result has a `verdict` but no `verdict_rule`: mark it non-comparable. | +| `verdict_with_rule.json` | Recomputing the verdict from `delta`, `p_value`, `verdict_rule` and `metric_spec` gives the stated `improved`. | +| `judge_snapshot_older.json`, `judge_snapshot_newer.json` | The same protocol resolved against judge snapshots `…-20260901` and `…-20261101`: the `protocol.digest` values differ, and the newer result is non-comparable to the older one. | + +`protocol.digest` is `sha256:` + the hex sha256 of `protocol.resolved` as JSON with sorted keys, no whitespace, UTF-8 and no NaN, recursively normalizing integral floats to integers (for example `0.0` to `0`). Protocol version 2 resolves `family_tested`, the number of results the multiple-comparison correction ran over; the version-1 digests are not comparable with these. `tests/test_evidence_protocol.py` runs these checks against `dokimi.subjects.evidence_check`. diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json new file mode 100644 index 000000000..9048e6ed4 --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_newer.json @@ -0,0 +1,81 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20261101", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:872a6210214f38ed9e5eb5a789c1c9f712d3dbcf952387625323d56c88547730", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20261101", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "2" + }, + "provider": "amazon-bedrock", + "verdict": "improved", + "verdict_rule": { + "alpha": 0.05, + "basis": "p_adjusted", + "correction": "holm", + "test": "sign_flip_inversion" + } + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json new file mode 100644 index 000000000..19ffa446b --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/judge_snapshot_older.json @@ -0,0 +1,81 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:2b8f9912f6deb3e504f2456964ed5a722aabc1eee99f4345a33e40e014ec5cb6", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "2" + }, + "provider": "amazon-bedrock", + "verdict": "improved", + "verdict_rule": { + "alpha": 0.05, + "basis": "p_adjusted", + "correction": "holm", + "test": "sign_flip_inversion" + } + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json new file mode 100644 index 000000000..19ffa446b --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_with_rule.json @@ -0,0 +1,81 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:2b8f9912f6deb3e504f2456964ed5a722aabc1eee99f4345a33e40e014ec5cb6", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "2" + }, + "provider": "amazon-bedrock", + "verdict": "improved", + "verdict_rule": { + "alpha": 0.05, + "basis": "p_adjusted", + "correction": "holm", + "test": "sign_flip_inversion" + } + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json new file mode 100644 index 000000000..509e1b3b0 --- /dev/null +++ b/dokimi/tests/fixtures/evidence_conformance/verdict_without_rule.json @@ -0,0 +1,75 @@ +{ + "generated_at": "2026-11-15T00:00:00Z", + "producer": { + "name": "dokimi", + "version": "conformance" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 11.0, + "lo": 2.0, + "method": "sign_flip_inversion" + }, + "delta": 6.5, + "effort": null, + "harness": "arms-runner", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_provider": "amazon-bedrock", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "metric_spec": { + "higher_is_better": true, + "id": "avg_aggregate", + "judged": true, + "range": [ + 0, + 100 + ], + "unit": "score_points" + }, + "model": "claude-haiku-4-5", + "n": 12, + "p_correction": "holm", + "p_value": 0.004, + "product": "oneTBB", + "protocol": { + "digest": "sha256:2b8f9912f6deb3e504f2456964ed5a722aabc1eee99f4345a33e40e014ec5cb6", + "id": "dokimi.awareness", + "resolved": { + "alpha": 0.05, + "ci_confidence": 0.95, + "correction": "holm", + "eval_set_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "family_tested": 2, + "judge_context": "report-snippets", + "judge_model": "claude-sonnet-5-5-20260901", + "judge_prompt_sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "judge_provider": "amazon-bedrock", + "judge_sampling": { + "max_tokens": 1024, + "temperature": 0.0 + }, + "metric": "avg_aggregate", + "repeats": 3, + "test": "sign_flip_inversion" + }, + "unresolved": [], + "version": "2" + }, + "provider": "amazon-bedrock", + "verdict": "improved" + } + ], + "schema_version": "skill_evidence.v1", + "subject": { + "content_digest": null, + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/test_evidence_protocol.py b/dokimi/tests/test_evidence_protocol.py new file mode 100644 index 000000000..fd423194d --- /dev/null +++ b/dokimi/tests/test_evidence_protocol.py @@ -0,0 +1,490 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""skill_evidence.v1 comparability: the protocol digest and the three conformance fixtures.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from dokimi.artifacts import load_artifact +from dokimi.subjects.evidence import build_skill_evidence +from dokimi.subjects.evidence_check import ( + build_protocol, compare, comparability, mark_against, protocol_digest, recompute_verdict, +) + +CONFORMANCE = Path(__file__).parent / "fixtures" / "evidence_conformance" + + +def _fixture(name: str) -> dict: + # Loaded the way a client loads a published file, so each fixture is a valid document too. + return load_artifact("skill_evidence", CONFORMANCE / name)["results"][0] + + +# --- the three conformance fixtures ------------------------------------------------------------- + +def test_fixture_a_verdict_without_its_rule_is_non_comparable(): + result = _fixture("verdict_without_rule.json") + + assert result["verdict"] == "improved" + assert comparability(result) == ["verdict without verdict_rule"] + assert recompute_verdict(result) is None + + +def test_fixture_a_verdict_with_its_rule_recomputes_to_the_same_verdict(): + result = _fixture("verdict_with_rule.json") + + assert comparability(result) == [] + assert recompute_verdict(result) == result["verdict"] == "improved" + + +def test_fixture_two_judge_snapshots_give_two_digests_and_the_newer_is_not_comparable(): + older, newer = _fixture("judge_snapshot_older.json"), _fixture("judge_snapshot_newer.json") + + assert older["protocol"]["id"] == newer["protocol"]["id"] + assert older["protocol"]["version"] == newer["protocol"]["version"] + assert older["protocol"]["digest"] != newer["protocol"]["digest"] + assert compare(older, older) == [] + assert compare(newer, older) == ["protocol digest differs (judge_model)"] + [marked] = mark_against([newer], older) + assert marked["comparable"] is False and marked["non_comparable"] + + +# --- the digest --------------------------------------------------------------------------------- + +def test_the_digest_is_sha256_of_sorted_compact_utf8_json(): + resolved = {"repeats": 3, "judge_model": "j-2026", "judge_sampling": {"temperature": 0.0}} + expected = hashlib.sha256( + b'{"judge_model":"j-2026","judge_sampling":{"temperature":0},"repeats":3}').hexdigest() + + assert protocol_digest(resolved) == "sha256:" + expected + assert build_protocol(resolved)["digest"] == "sha256:" + expected + + +@pytest.mark.parametrize("alias", ["latest", "claude-sonnet-latest", "gpt:default", "model@stable", "AUTO"]) +def test_an_alias_stays_out_of_the_digest_and_is_listed_as_unresolved(alias): + protocol = build_protocol({"judge_model": alias, "repeats": 3}) + + assert protocol["resolved"] == {"repeats": 3} + assert protocol["unresolved"] == ["judge_model"] + assert protocol["aliases"] == {"judge_model": alias} + assert protocol["digest"] == protocol_digest({"repeats": 3}) + + +@pytest.mark.parametrize("snapshot", ["claude-sonnet-5-5", "claude-sonnet-5-5-20260901", "gpt-5.6"]) +def test_a_snapshot_id_is_resolved(snapshot): + assert build_protocol({"judge_model": snapshot})["resolved"] == {"judge_model": snapshot} + + +def test_a_missing_input_is_unresolved_and_two_unresolved_protocols_are_not_comparable(): + with_rule = _fixture("verdict_with_rule.json") + vague = {**with_rule, "protocol": build_protocol({**with_rule["protocol"]["resolved"], + "judge_prompt_sha256": None})} + + assert vague["protocol"]["unresolved"] == ["judge_prompt_sha256"] + assert compare(vague, vague) == ["protocol inputs not resolved: judge_prompt_sha256"] + + +def test_a_tampered_resolved_block_is_caught(): + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["repeats"] = 1 + + assert comparability(result) == ["protocol.digest does not match protocol.resolved"] + + +# --- the per-result rules ----------------------------------------------------------------------- + +@pytest.mark.parametrize("drop, reason", [ + ("unit", "verdict without metric_spec.unit"), + ("higher_is_better", "verdict without metric_spec.higher_is_better"), +]) +def test_a_verdict_needs_the_metric_unit_and_direction(drop, reason): + result = _fixture("verdict_with_rule.json") + del result["metric_spec"][drop] + + assert comparability(result) == [reason] + + +def test_a_judged_result_needs_its_judge(): + result = {**_fixture("verdict_with_rule.json"), "judge_model": None} + + assert comparability(result) == ["judged result without judge_model"] + + +def test_a_verdict_that_contradicts_its_rule_is_non_comparable(): + result = {**_fixture("verdict_with_rule.json"), "p_value": 0.2} + + assert comparability(result) == ["verdict 'improved' does not follow verdict_rule ('inconclusive')"] + + +def test_a_p_value_or_alpha_outside_the_probability_range_cannot_give_a_verdict(): + # p = -0.004 is below alpha, so it used to read "improved" and pass comparability with no reason. + base = _fixture("verdict_with_rule.json") + for result in ({**base, "p_value": -0.004}, {**base, "p_value": 1.5}, + {**base, "verdict_rule": {**base["verdict_rule"], "alpha": 1.5}}): + assert recompute_verdict(result) is None + assert "verdict_rule cannot be applied" in comparability(result) + assert compare(result, base) + + +def test_lower_is_better_turns_a_negative_delta_into_improved(): + result = _fixture("verdict_with_rule.json") + result["metric_spec"]["higher_is_better"] = False + result["delta"] = -6.5 + + assert recompute_verdict(result) == "improved" + + +def test_without_protocols_the_fallback_fields_must_match(): + a = {k: v for k, v in _fixture("verdict_with_rule.json").items() if k != "protocol"} + b = {**a, "harness": "claude-code"} + + assert compare(a, a) == [] + assert compare(b, a) == ["harness differs"] + + +# --- what the exporter emits -------------------------------------------------------------------- + +def test_the_exporter_emits_rule_spec_and_protocol_and_flags_what_it_cannot_compare(): + from test_subjects_evidence import _scorecard + + [skill] = [r for r in build_skill_evidence(_scorecard())["results"] if r["arm"] == "skill:s"] + + assert skill["verdict_rule"]["basis"] == "p_adjusted" and skill["verdict_rule"]["alpha"] == 0.05 + assert skill["metric_spec"]["higher_is_better"] is True and skill["metric_spec"]["unit"] + assert recompute_verdict(skill) == skill["verdict"] + assert skill["protocol"]["digest"] == protocol_digest(skill["protocol"]["resolved"]) + # The scorecard records no judge, sampling, judge prompt or repeat count for this cell. + assert {"judge_model", "judge_prompt_sha256", "judge_sampling", "repeats"} <= set( + skill["protocol"]["unresolved"]) + assert skill["comparable"] is False + assert skill["non_comparable"] == comparability(skill) == ["judged result without judge_model"] + + +def test_a_cell_that_records_its_judge_and_protocol_inputs_is_comparable(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + scorecard["cells"][0].update({ + "judge_model": "claude-sonnet-5-5-20260901", "judge_provider": "amazon-bedrock", + "judge_context": "report-snippets", "judge_sampling": {"temperature": 0.0}, + "judge_prompt_sha256": "sha256:" + "d" * 64, "repeats": 3, + }) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert skill["comparable"] is True and skill["non_comparable"] == [] + assert skill["protocol"]["unresolved"] == [] + assert skill["protocol"]["resolved"]["judge_model"] == "claude-sonnet-5-5-20260901" + assert compare(skill, skill) == [] + + +def test_the_rollup_and_scorecard_cells_carry_the_judge(tmp_path: Path): + from dokimi.eval.matrix_rollup import build_matrix_rollup + + arms = {"baseline_arm": "none", "judge_model": "j", "judge_provider": "jp", "judge_context": "ctx"} + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", arms)]) + cell = rollup["cells"][0] + assert (cell["judge_model"], cell["judge_provider"], cell["judge_context"]) == ("j", "jp", "ctx") + + +def test_the_conformance_fixtures_are_the_four_documented_ones_and_carry_the_digest_they_state(): + names = sorted(path.name for path in CONFORMANCE.glob("*.json")) + assert names == ["judge_snapshot_newer.json", "judge_snapshot_older.json", + "verdict_with_rule.json", "verdict_without_rule.json"] + for name in names: + protocol = json.loads((CONFORMANCE / name).read_text(encoding="utf-8"))["results"][0]["protocol"] + assert protocol["digest"] == protocol_digest(protocol["resolved"]) + + +# --- self-review round 1 -------------------------------------------------------------------------- + +@pytest.mark.parametrize("bare", ["opus", "sonnet", "haiku", "gpt-4o", "gpt-4o-mini"]) +def test_a_bare_family_name_is_an_alias(bare): + assert build_protocol({"judge_model": bare})["unresolved"] == ["judge_model"] + + +def test_an_alias_nested_in_a_value_is_unresolved(): + protocol = build_protocol({"judge_sampling": {"model": "x-latest", "temperature": 0.0}}) + assert protocol["unresolved"] == ["judge_sampling"] + + +def test_an_integral_float_and_an_int_give_one_digest(): + assert protocol_digest({"t": 0, "r": 3}) == protocol_digest({"t": 0.0, "r": 3.0}) + + +def test_a_p_value_within_rounding_of_alpha_is_not_a_contradiction(): + result = {**_fixture("verdict_with_rule.json"), "p_value": 0.05} + result["verdict_rule"] = {**result["verdict_rule"], "p_decimals": 4} + + assert recompute_verdict(result) is None + assert comparability(result) == ["p_value is within its rounding of alpha: the verdict cannot be recomputed"] + + +def test_the_exporter_keeps_the_judge_temperature_and_prompt_hash_the_run_recorded(tmp_path: Path): + from test_subjects_evidence import _scorecard + from dokimi.eval.matrix_rollup import build_matrix_rollup + + arms = {"baseline_arm": "none", "judge_temperature": 0.0, "judge_prompt_sha256": "e" * 64} + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", arms)]) + assert rollup["cells"][0]["judge_temperature"] == 0.0 + assert rollup["cells"][0]["judge_prompt_sha256"] == "e" * 64 + + scorecard = _scorecard() + scorecard["cells"][0].update({"judge_temperature": 0.0, "judge_prompt_sha256": "e" * 64}) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + assert skill["protocol"]["resolved"]["judge_sampling"] == {"temperature": 0} + assert skill["protocol"]["resolved"]["judge_prompt_sha256"] == "e" * 64 + assert skill["verdict_rule"]["p_decimals"] == 4 + + +# --- regressions: what a result states must be what its protocol resolved ---------------------- + +def _with(result: dict, **changes) -> dict: + return {**result, **changes} + + +def test_a_stated_judge_or_rule_value_the_protocol_does_not_resolve_is_non_comparable(): + base = _fixture("verdict_with_rule.json") + judge = _with(base, judge_model="claude-sonnet-5-5-20261101") + alpha = _with(base, verdict_rule={**base["verdict_rule"], "alpha": 0.01}) + + assert compare(judge, base) == ["result: judge_model differs from protocol.resolved.judge_model"] + assert compare(alpha, base) == ["result: verdict_rule.alpha differs from protocol.resolved.alpha"] + + +def test_a_metric_unit_or_range_change_is_non_comparable_with_or_without_a_protocol(): + base = _fixture("verdict_with_rule.json") + reason = ["metric specification differs (unit, range or direction)"] + unit = _with(base, metric_spec={**base["metric_spec"], "unit": "percent"}) + span = _with(base, metric_spec={**base["metric_spec"], "range": [0, 1]}) + bare = {k: v for k, v in base.items() if k != "protocol"} + + assert compare(unit, base) == reason + assert compare(span, base) == reason + assert compare(_with(bare, metric_spec={**bare["metric_spec"], "unit": "percent"}), bare) == reason + + +def test_a_protocol_on_one_side_only_is_non_comparable_in_either_order(): + base = _fixture("verdict_with_rule.json") + bare = {k: v for k, v in base.items() if k != "protocol"} + reason = ["protocol on one side only: the fallback fields cannot stand for its inputs"] + + assert compare(base, bare) == compare(bare, base) == reason + + +def test_an_input_the_protocol_leaves_out_is_unresolved(): + base = _fixture("verdict_with_rule.json") + partial = _with(base, protocol=build_protocol({"judge_model": base["judge_model"]})) + + [reason] = compare(partial, partial) + assert reason.startswith("protocol inputs not resolved: ") + assert "alpha" in reason and "repeats" in reason and "judge_model" not in reason + + +@pytest.mark.parametrize("decimals", [-1000, 1000, True]) +def test_an_unsupported_rounding_precision_is_non_comparable_not_a_crash(decimals): + base = _fixture("verdict_with_rule.json") + bad = _with(base, verdict_rule={**base["verdict_rule"], "p_decimals": decimals}) + + assert recompute_verdict(bad) is None + assert "verdict_rule cannot be applied" in comparability(bad) + assert compare(bad, base) and mark_against([bad], base)[0]["comparable"] is False + + +def test_without_protocols_a_judge_provider_difference_is_non_comparable(): + a = {k: v for k, v in _fixture("verdict_with_rule.json").items() if k != "protocol"} + + assert compare(_with(a, judge_provider="anthropic"), a) == ["judge_provider differs"] + + +def test_a_judged_cell_carries_its_judge_fields_into_the_scorecard(tmp_path: Path): + from dokimi.subjects import build_subject_scorecard, load_subject + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["p"]\nquestions = ["q.json"]\n', encoding="utf-8") + cell = {"matrix_cell": "c1", "judge_model": "j", "judge_provider": "jp", "judge_context": "ctx"} + judged = {**cell, "judge_temperature": 0.0, "judge_prompt_sha256": "sha256:" + "d" * 64} + scorecard = build_subject_scorecard( + load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=[{"product": "p", "questions": "q.json", "matrix_rollup": "m.json", + "rollup": {"cells": [judged]}}]) + carried = scorecard["cells"][0] + assert {key: value for key, value in carried.items() if key.startswith("judge_")} == { + key: value for key, value in judged.items() if key.startswith("judge_")} + + +def test_each_cell_of_a_two_question_set_suite_carries_the_digest_of_its_own_set( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + from dokimi.subjects import build_subject_scorecard, load_subject + from dokimi.subjects.scorecard import _digest_values + + # Question files are read from the working directory, so the digests hash their contents. + monkeypatch.chdir(tmp_path) + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["p", "q"]\nquestions = ["a.json", "b.json"]\n', encoding="utf-8") + (tmp_path / "a.json").write_text('["one"]', encoding="utf-8") + (tmp_path / "b.json").write_text('["two"]', encoding="utf-8") + runs = [{"product": product, "questions": name, "matrix_rollup": "m.json", + "rollup": {"cells": [{"matrix_cell": product}]}} + for product, name in (("p", "a.json"), ("q", "b.json"))] + def build() -> dict: + return build_subject_scorecard(load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=runs) + + scorecard = build() + first, second = scorecard["cells"] + assert first["questions_digest"] == _digest_values(("a.json",)) + assert second["questions_digest"] == _digest_values(("b.json",)) + assert scorecard["suite"]["question_set_hash"] not in (first["questions_digest"], second["questions_digest"]) + # Changing one file's contents changes only its own cell's digest. + (tmp_path / "a.json").write_text('["changed"]', encoding="utf-8") + changed = build()["cells"] + assert changed[0]["questions_digest"] != first["questions_digest"] + assert changed[1]["questions_digest"] == second["questions_digest"] + + +def test_a_non_finite_p_or_delta_gives_no_direction(): + base = _fixture("verdict_with_rule.json") + huge = 10 ** 400 # valid JSON, too large for a float + for changes in ({"p_value": float("nan")}, {"delta": float("nan")}, {"p_value": float("inf")}, + {"delta": huge}, {"p_value": huge}): + bad = _with(base, **changes) + assert recompute_verdict(bad) == "inconclusive", changes + comparability(bad) # must not raise + + +def test_a_context_suffixed_alias_is_an_alias_not_a_snapshot(): + for alias in ("opus[1m]", "sonnet[1m]"): + protocol = build_protocol({"judge_model": alias}) + assert protocol["unresolved"] == ["judge_model"] and protocol["aliases"] == {"judge_model": alias} + snapshot = build_protocol({"judge_model": "claude-sonnet-5-5-20261101[1m]"}) + assert snapshot["resolved"] == {"judge_model": "claude-sonnet-5-5-20261101[1m]"} + + +def test_a_resolved_null_or_alias_is_unresolved_even_with_a_matching_digest(): + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["repeats"] = None + result["protocol"]["digest"] = protocol_digest(result["protocol"]["resolved"]) + assert "protocol inputs not resolved: repeats" in compare(result, result) + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["judge_sampling"] = {"temperature": 0.0, "top_p": "default"} + result["protocol"]["digest"] = protocol_digest(result["protocol"]["resolved"]) + assert any("judge_sampling" in reason for reason in compare(result, result)) + + +def test_a_non_json_resolved_value_is_a_reason_not_an_exception(): + result = _fixture("verdict_with_rule.json") + result["protocol"]["resolved"]["judge_sampling"] = {"temperature": float("nan")} + reasons = compare(result, _fixture("verdict_with_rule.json")) + assert any("is not JSON" in reason for reason in reasons) + + +def test_a_harness_change_is_a_reason_when_both_sides_carry_a_protocol(): + # harness is not a protocol input, so the digest cannot tell two harnesses apart + base = _fixture("verdict_with_rule.json") + assert compare(_with(base, harness="another-harness"), base) == ["harness differs"] + + +def test_a_questions_file_not_found_from_the_cwd_leaves_its_digest_unresolved(): + # the name alone is no identity: two sets named q.json would share one digest, so the input stays unknown + from dokimi.subjects.scorecard import _digest_values + assert _digest_values(("questions/q.json",)) is None + + +def test_a_mixed_judge_context_is_an_unresolved_input_not_a_measurement_identity(): + # some scores saw the stored material and some only the snippets: two such runs must not compare clean + from dokimi.eval.judge_context import PROTOCOL_MIXED + protocol = build_protocol({"judge_context": PROTOCOL_MIXED, "repeats": 3}) + assert "judge_context" in protocol["unresolved"] and "judge_context" not in protocol["resolved"] + + +# --- the family multiplier and the recorded judge snapshot (self-review round 10) ---------------- + +def test_the_family_tested_multiplier_is_a_protocol_input_of_version_2(): + from dokimi.subjects.evidence_check import PROTOCOL_VERSION, REQUIRED_INPUTS + + assert PROTOCOL_VERSION == "2" + assert "family_tested" in REQUIRED_INPUTS + + +def test_a_result_corrected_over_a_different_number_of_tests_is_not_comparable(): + from test_subjects_evidence import _scorecard + + def skill_result(tested: int) -> dict: + scorecard = _scorecard() + scorecard["cells"][0].update({ + "judge_model": "claude-sonnet-5-5-20260901", "judge_provider": "amazon-bedrock", + "judge_context": "report-snippets", "judge_sampling": {"temperature": 0.0}, + "judge_prompt_sha256": "sha256:" + "d" * 64, "repeats": 3}) + scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"]["family_tested"] = tested + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + return skill + + two, six = skill_result(2), skill_result(6) + + assert two["protocol"]["resolved"]["family_tested"] == 2 + assert two["protocol"]["digest"] != six["protocol"]["digest"] + assert compare(six, two) == ["protocol digest differs (family_tested)"] + + +def test_a_family_tested_the_scorecard_does_not_record_is_unresolved(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + del scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"]["family_tested"] + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert "family_tested" in skill["protocol"]["unresolved"] + assert any("family_tested" in reason for reason in compare(skill, skill)) + + +def test_a_judge_alias_is_resolved_by_the_one_snapshot_the_run_recorded(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + scorecard["cells"][0].update({"judge_model": "opus", "judge_resolved_models": ["claude-opus-5-5-20261101"]}) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert skill["judge_model"] == "claude-opus-5-5-20261101" + assert skill["protocol"]["resolved"]["judge_model"] == "claude-opus-5-5-20261101" + assert "judge_model" not in skill["protocol"]["unresolved"] + + +def test_a_judge_alias_with_two_recorded_snapshots_stays_unresolved(): + from test_subjects_evidence import _scorecard + + scorecard = _scorecard() + scorecard["cells"][0].update({"judge_model": "opus", "judge_resolved_models": [ + "claude-opus-5-5-20260901", "claude-opus-5-5-20261101"]}) + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + + assert skill["judge_model"] == "opus" + assert "judge_model" in skill["protocol"]["unresolved"] + assert any("judge_model" in reason for reason in compare(skill, skill)) + + +def test_the_rollup_and_the_scorecard_cell_carry_the_recorded_judge_snapshots(tmp_path: Path): + from dokimi.eval.matrix_rollup import build_matrix_rollup + from dokimi.subjects import build_subject_scorecard, load_subject + + arms = {"baseline_arm": "none", "judge_model": "opus", "judge_resolved_models": ["claude-opus-5-5-20261101"]} + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", arms)]) + assert rollup["cells"][0]["judge_resolved_models"] == ["claude-opus-5-5-20261101"] + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["p"]\nquestions = ["q.json"]\n', encoding="utf-8") + scorecard = build_subject_scorecard( + load_subject(tmp_path / "subject.toml"), out_dir=tmp_path, + awareness_runs=[{"product": "p", "questions": "q.json", "matrix_rollup": "m.json", + "rollup": {"cells": [rollup["cells"][0]]}}]) + assert scorecard["cells"][0]["judge_resolved_models"] == ["claude-opus-5-5-20261101"] diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 99e400e9b..719a1ea03 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -100,9 +100,10 @@ def test_a_ref_that_is_no_path_and_holds_a_lone_surrogate_is_digested(tmp_path: @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") -def test_a_question_value_with_a_lone_surrogate_is_digested(): - digest = _digest_values(("q\udcff",)) - assert digest.startswith("sha256:") and digest != _digest_values(("q\udcfe",)) +# Test-narrowed: test_a_question_value_with_a_lone_surrogate_is_digested now asserts None: a missing file has no digest, the name is not hashed. +def test_a_question_value_with_a_lone_surrogate_that_is_no_file_stays_unresolved(): + # a name the cwd does not show is no identity, so the digest is None and the name is not hashed + assert _digest_values(("q\udcff",)) is None @pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 62dd48577..7bcdd9900 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -21,7 +21,8 @@ def _significance(delta: float, *, significant: bool) -> dict: return {"ci_confidence": 0.95, "ci_lo": delta - 1.5, "ci_hi": delta + 1.5, "ci_method": "student_t", - "correction": "holm", "delta_mean": delta, "n": 6, "p_adjusted": 0.001 if significant else 0.4, + "correction": "holm", "delta_mean": delta, "n": 6, "family_tested": 2, + "p_adjusted": 0.001 if significant else 0.4, "significant_adjusted": significant} @@ -312,3 +313,35 @@ def test_the_scorecard_cells_carry_the_baseline_arm_of_the_rollup(tmp_path: Path "rollup": rollup}], out_dir=tmp_path) assert scorecard["cells"][0]["baseline_arm"] == "none" + + +def test_a_scorecard_keeps_the_test_it_recorded_and_an_unrecorded_one_is_unresolved(tmp_path: Path): + from dokimi.subjects.evidence import build_skill_evidence + + scorecard = _scorecard() + significance = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"] + significance["ci_method"] = "student_t" + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + assert skill["verdict_rule"]["test"] == skill["protocol"]["resolved"]["test"] == "student_t" + + del significance["ci_method"] + artifact = build_skill_evidence(scorecard) + [skill] = [r for r in artifact["results"] if r["arm"] == "skill:s"] + assert "test" not in skill["verdict_rule"] and "test" in skill["protocol"]["unresolved"] + # The historical scorecard exports: the schema has no null test, so the artifact must validate. + save_artifact("skill_evidence", artifact, tmp_path / "skill_evidence.json") + + +def test_a_cell_of_a_two_question_set_suite_is_resolved_only_by_its_own_digest(): + from dokimi.subjects.evidence import build_skill_evidence + + scorecard = _scorecard() + scorecard["suite"]["questions"] = ["q.json", "r.json"] + + def skill_protocol() -> dict: + [skill] = [r for r in build_skill_evidence(scorecard)["results"] if r["arm"] == "skill:s"] + return skill["protocol"] + + assert "eval_set_digest" in skill_protocol()["unresolved"] + scorecard["cells"][0]["questions_digest"] = "sha256:" + "e" * 64 + assert skill_protocol()["resolved"]["eval_set_digest"] == "sha256:" + "e" * 64