diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index b1204d950..a18fd6f01 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -62,6 +62,9 @@ header: # written verbatim as the agent's answer (.agent-eval/run.py). - '.agent-eval/tasks/**/*.patch' - '.agent-eval/tasks/**/oracle_answer.md' + # Generated study tables and figures (the judge-panel study's CSV and SVG, written by + # research/judge-panel): a header would break the CSV readers and is lost on regeneration. + - 'dokimi/docs/assets/judge-panel/**' # Changelog fragments (__.md) and the template # `scriv create` copies into each: `scriv collect` moves their text into # CHANGELOG.md, which carries the header itself. diff --git a/dokimi/changelog.d/20261108_120000_napetrov_judge-panel-cli.md b/dokimi/changelog.d/20261108_120000_napetrov_judge-panel-cli.md new file mode 100644 index 000000000..6b736fb8a --- /dev/null +++ b/dokimi/changelog.d/20261108_120000_napetrov_judge-panel-cli.md @@ -0,0 +1,3 @@ +### Added + +- **Judge panel for skill novelty**: `skill novelty analyze` and `probe` now take `--judge PROVIDER/MODEL`. Repeat the flag to form a panel. `--judge-aggregate {median,majority,unanimous}` (default `median`) turns the panel's verdicts into one per answer. The record gets a `judge_panel` block: each judge's own paired 2×2, the judge range of the added share, and a `judge_sensitive` flag set when that range exceeds 0.10. Each run keeps every judge's verdict (`analyze` writes them to `.judge_verdicts.json`), and the report names the panel. A warning is printed when two judges share a vendor family. The `--estimate-only` dollar figure and call counts include the extra judges. A model given twice, under any spelling, is refused. Background: `docs/judge-panel-study.md`. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index fe220c6ea..89da517a2 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1611,6 +1611,8 @@ {"key": "--extract-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--include-resources", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--judge", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-aggregate", "action": "_StoreAction", "choices": ["'majority'", "'median'", "'unanimous'"], "const": null, "default": "median", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -1707,6 +1709,8 @@ {"key": "--extract-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--extract-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--judge", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-aggregate", "action": "_StoreAction", "choices": ["'majority'", "'median'", "'unanimous'"], "const": null, "default": "median", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 1f3bfbaf2..ddefde067 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -103,20 +103,24 @@ be right or wrong) and `harmful_conflicting` (below), which overrides the other `redundant` or `degraded` means known. An atom that was never probed counts as new, which is the direction that does not claim the model knew something nobody asked it. The model is never asked whether it knew something. + - **Needed vs inert is judged.** The extractor model sees the task and answers, per atom, *"can acting on this task go wrong, or be much slower, without knowing it?"* The task is built from the artifact's `name` and `description` plus the need graph's `core` and `rare_critical` nodes, never from the body, so an artifact cannot make itself relevant by containing a sentence. Eval files are not part of it either. + - **Reliability.** In the study both axes were judged, and an independent reviewer agreed with 71.6% of assignments (853 atoms), with useful-vs-inert at ±3 points and known-vs-novel at ±2; `scaffolding` was the most stable share ([§4](skill-novelty-evidence.md)). Here only the ±3 carries over. Known vs new is read off the paired probes instead, so its error is the probe grading's: two judges agreed on 85% of verdicts on the correct-ish binary collapse (above). + - **Inert is not "safe to delete".** The `drop-inert 0.60` candidate, which cuts inert text under a 0.60 token budget (the suffix is a budget, as for every §10.6 candidate, not "all inert atoms at once"), was non-inferior on pooled data (+0.005, CI [−0.045, +0.057]) and *worse* on `nemotron-nano-12b` for `dpnp-quickstart` ([§10.6](skill-novelty-evidence.md)). + - An atom the value judge did not answer for gets no category. The artifact says how many (`N of M atoms got no need/scaffolding verdict`, split into calls that failed, rows the reply skipped, malformed rows, and replies with no usable verdict row; an empty list for a batch is @@ -131,6 +135,7 @@ marked. Read it as a review queue: - **Internal only.** It says the two statements disagree, not which is wrong. That needs a primary source, which this layer does not consult. + - **Lower bound.** Only atoms in the same call are compared. `analyze` sends the whole probed atom set (at most `max_atoms`, 120 by default) in one call, so every pair is compared unless a call fails or its reply has to be split. The artifact records the share as @@ -146,6 +151,7 @@ marked. Read it as a review queue: `harmful_conflicting` row reads `not measured (0 found)`. An artifact written before `conflict_coverage` existed has no such field and keeps the old reading, a 0 that counts as cleared, unless its atom set was capped; re-run `analyze` to check it. + - **Unchecked atoms.** A call whose reply does not parse, or whose rows name no pair of the batch (an empty list is fine: it means no conflicts), is split and re-asked only while both halves keep two atoms, since one statement has no pair to look at; a chunk of two or three whose @@ -153,6 +159,7 @@ marked. Read it as a review queue: a chunk (the 41st, say), are counted in `N of M atoms were not checked for conflicts`, split by cause (`K in calls that failed (last error: …)`, `K whose reply named no usable pair`, `K alone in a batch, with no pair to check`). Their ids are not written to the artifact. + - **False positives (anecdotal, not a rate).** Spot-checking 7 of the 31 pairs one run reported ([mattpocock/skills](https://github.com/mattpocock/skills) @ `49dd158`) found 4 real, 1 ambiguous and 2 false positives: steps of one procedure read as alternatives ("watch it fail" @@ -205,6 +212,7 @@ Consequences for authoring: - A skill that looks bloated against a frontier model may be correctly sized for a small one. Decide which tier you are writing for, and record it. + - `degraded` rises as the model weakens (0.019 → 0.076). Weak models copy your errors; strong models overrule them. A stale line in a skill is a bigger risk to a small model — **except** for reversed or out-of-date claims, where the opposite holds: strong models resisted planted inversions *less* @@ -274,7 +282,7 @@ Anthropic's). model that wrote the saved answer key (the atoms file's `extractor`) against the judge. The same model, or two models of one vendor family (the vendor is read off the model id: `claude`, `sonnet`, `haiku`, `opus` and `anthropic` are one family; `gpt` and `openai` another; `gemini` and -`mistral` likewise), prints a `WARNING` on stderr. The vendor is read off the name, so a model +`mistral` likewise; `grok` and `xai` another), prints a `WARNING` on stderr. The vendor is read off the name, so a model served locally under a vendor's name (`gpt-oss-120b` is OpenAI's) counts as that vendor; ids that name no known vendor never match. `analyze` records the result in the artifact's `run_config.extractor_judge` (`extractor`, `judge`, `same_model`, `same_vendor`); `probe` @@ -298,6 +306,50 @@ family with one of them, and the warning does not check that; say so when you qu A third family keeps all three apart. Changing the extractor changes the atoms, so compare only cells that name the same extractor. +## A judge panel + +One judge is one opinion. The [judge-panel study](judge-panel-study.md) graded the same answers with +seven judges. The added share of one answer model moved by up to 0.23 with the judge. The +original single judge was the most lenient of all, so its added shares sat at or near the +bottom of the panel's range. `analyze` and `probe` take a panel: + +```bash +python cli.py skill novelty probe --artifact skill:path/to/skill --atoms atoms.json --model grok-4.6 --provider openrouter --judge openrouter/openai/gpt-6-luna --judge openrouter/anthropic/claude-sonnet-5 --judge openrouter/moonshotai/kimi-k3 --judge-aggregate median +``` + +- `--judge PROVIDER/MODEL`, repeated, is the panel. It replaces `--judge-model` and + `--judge-provider`, and the two forms cannot be mixed. The first judge is the default + extractor. Every judge is checked against the answerer like a single judge. + +- `--judge-aggregate` turns the verdicts into one per answer: + + - `median` (default): the median of the verdict scale, rounding toward wrong on an even + panel. + - `majority`: correct when more than half the judges say so. + - `unanimous`: correct only when all of them say so. + + In the study, median and majority matched Dawid-Skene on 99.3% of rows. Any-correct + moved the headline by up to 0.20, and unanimous raised the added share by up to 0.09 over + majority. A judge whose grade fails is left out of the vote, except for a permanent provider + error (a bad key, an unknown model), which stops the run and keeps the partial record. + +- The record gets a `judge_panel` block with: + + - the judges and the rule; + - each judge's own paired 2×2 (`per_judge_paired`); + - the judge range of the added share (`added_share_range`); + - `judge_sensitive`, set when the lenient range exceeds 0.10 (the study's noise floor). + + Each run keeps every judge's verdict in `judge_verdicts`. `analyze` writes them beside + the record as `.judge_verdicts.json`; the report names the panel instead of one judge. + +- Two judges of one vendor family (the families named above) print a warning. Same-family judges agree with each other + more than with the rest, so the panel then has fewer independent votes than judges. Three + judges from three families cut the spread across panels from SD 0.039 to 0.025. + +Each extra judge grades every answer again, so it adds the judge's share of the cost. +`--estimate-only` counts it, and `--max-usd` caps the whole panel. + ## Using it as an authoring tool The commands in order, with the costs and the two free steps that come first, are in @@ -370,6 +422,7 @@ Two derived views an author acts on directly: - **Dead weight**: the dim spans, sorted by token cost. The largest single win in most artifacts. Unread spans are not listed. + - **Sole carriers**: spans that are the only source of some atom. These are what a careless edit silently deletes; the shortener pins them. @@ -385,9 +438,11 @@ the highlight is load-bearing. Three measured constraints ship with it: - **Do not attribute by lexical overlap** with the atom statement. It misses negation — negation words are function words — so its extra loss on `negative-constraint` atoms is +0.143 against an LLM attributor's +0.392. The cheap attributor is weakest on exactly the knowledge that matters most. + - **Carry the heading path** wherever a span is checked. Atom recovery from a bare span is 0.833 and rises to 0.907 with the heading path prepended, for about 16 tokens: a span reading "Use 4 for most workloads" is not a bad citation, it is a sentence whose subject lives in its heading. + - **Render an unlocated state.** 19% of atoms are not recoverable from their cited *file* at all — syntheses inferred while reading the whole document, which the faithfulness check calls supported. Their span pointer is a fiction, and colouring a span for them is worse than saying nothing. @@ -463,6 +518,7 @@ Two limits on that rule, both measured: - **Per artifact it is unproven.** With 7–8 tasks per (artifact, model) cell the 95% interval is ~0.32 task-score points wide, so 44 of 60 individual cells were inconclusive. Pooled evidence licenses a shortening *policy*; it does not license a specific cut without running that cut. + - **The verdict does not transfer across model tiers.** Of 20 (artifact, candidate) pairs measured on three models each, only 9 agreed; the smallest model was the worst-hit in 9 of 20, and the same candidate was non-inferior on a frontier model and *worse* on a small one. Small models need the @@ -477,6 +533,7 @@ Two content patterns were measured as actively harmful, independent of topic: entirely: it replied with a preamble promising to look things up and scored 0.047, against 0.537 with no skill at all. Skill bodies should be answerable standalone; progressive disclosure is a loading strategy, not a sentence to write. + - **Investment in prose order.** The study's shuffled arm scored *higher* than the intact artifact on four of five models, but it injected more files than the artifact did, so that result does not carry over to this measurement. Ordering, transitions and narrative framing carried no @@ -525,6 +582,7 @@ the artifact's contribution runs the other way. The weakest model knew *no* loca - **Conceptual prose is the first thing to cut.** It is the half the model already has. Explaining what a cache miss is spends tokens; stating this stack's exact flag, threshold or ordering does not. + - **Write the procedure down, precisely.** `motif` and `contract` atoms — gates, orderings, flags, output formats — are where the artifact is least replaceable, and they are also the atoms a greedy token-budget shortener discards first (their value is conditional on a failure the sentence does @@ -567,6 +625,7 @@ Things to know before reading a bundle result: - **It is an upper bound.** Injecting the whole bundle measures what the artifact *could* deliver if the agent read all of it. Whether the agent chooses to load `references/x.md` at runtime is the agentic track's question (`skill-agent:`), not this one. + - **The two modes are different cells.** `include_resources` is recorded in `run_config`; an entry-point run and a bundle run of the same skill are not comparable numbers. A run that read half or less of the readable words says so on stdout (the `coverage:` line @@ -577,11 +636,13 @@ Things to know before reading a bundle result: `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share` to 4 places, which nets out the role decisions below; `outside_share` is the on-disk figure, and the two differ for a skill whose only resources are its evals). + - **A skill's own `evals/` are not content.** They are the question/answer keys for testing the skill, so extracting atoms from them would credit the skill with delivering its own test set. Excluded by default (10 of the 33 corpus skills ship them), with the reason naming the opt-in; `.git/`, `__pycache__/`, `venv/`, hidden files and friends are never walked at all. + - **Exclusions are reported, never silent.** A file over `--max-file-words` is skipped whole rather than truncated, because a cut reference would be attributed as if complete; every skip, role filter, budget overrun and non-text file appears in `bundle.excluded` @@ -591,6 +652,7 @@ Things to know before reading a bundle result: A file that is mostly printable text but does not decode as UTF-8 (latin-1, cp1252, UTF-16) is reported `unreadable` whatever its suffix, and the coverage check then says `measured=False`; only a NUL-bearing non-UTF-16 file, or a blob that is not mostly printable, is silently "not text". + - **One rule decides what an exclusion means.** Every row's `kind` has one class (`metrics/bundle_coverage.py`, `EXCLUSION_CLASS`): `budget` and `not_requested` are a *gap* — the words in `unmeasured_words`, the files a warning calls "excluded"; `policy` is a *decision*, never @@ -598,6 +660,7 @@ Things to know before reading a bundle result: exclusion is a *file* the run did not select; text it selected that no extraction call could read is an *unread span* (`spans.n_unread`, below), which narrows the dead-token figure and not coverage. + - **Coverage has one basis, and the record says which.** The CLI note, the analyzer's warnings, the gate's `artifact_coverage` and the report all read the record through the same `Coverage`, over *readable* words: role decisions are left out of the denominator, so for an @@ -745,6 +808,7 @@ Two numbers, reported side by side and never merged: - **Lexical** — BM25 over each candidate's `name + description`, which is exactly the text a tool schema shows. Free, deterministic, and a floor. + - **Model** (`--model`, optional) — a model picks from the same cards, with the card order a **seeded random permutation per query** (`--seed`), so a model that always picks the first option tends to `1/N` rather than to a figure set by how the query count lines up @@ -890,6 +954,7 @@ Three properties worth knowing before quoting either: says why, and still writes the result with `"status": "no_queries"`. Every rate line prints its `n_queries` (and `triage: self-derived` when it is self-derived), and fewer than five scored queries draw a warning. + - **The model figure is an upper bound on harness behaviour.** A harness whose own skill listing truncates or omits descriptions selects on less text than this command shows — which has been observed. Only a run inside the harness settles it. @@ -968,6 +1033,7 @@ run, so on a two-arm run too `ci95_added` changes (and is null below 5 atoms). - **Placebo-gated `added`** (`added_over_placebo`, needs the `placebo` arm). This counts a probe only when it is right with the artifact, wrong without it, *and* wrong with the placebo text. A gain the placebo also gets is a context effect, not knowledge. + - **Noise floor** (`noise_floor`, needs the `no_artifact_repeat` arm). That arm asks every baseline question again as a new sample (a new cache `draw`). Its flips against the baseline are what `added` and `degraded` would be if the artifact did nothing. @@ -1053,6 +1119,7 @@ The hybrid ships. Two consequences worth remembering when reading a `knowledge` - **Never accept an embedding-only retention score.** Both cheap matchers scored 0.00 recall on the contradiction label: they cannot tell *restates* from *reverses*, so a rewrite that inverted a rule would be reported as faithfully retained. + - **The oracle fails safe.** With no adjudicator configured, the ambiguous band resolves to *absent*, so reported `knowledge` is a floor — a shortening can look worse than it is, never better. @@ -1068,6 +1135,7 @@ reverses to +0.047. Two things not to conclude from that: adjudications, ≤0.0072 even at 0.80 agreement, and replacing per-pair verdicts with a calibrated aggregate collapses to a constant. The escalation *is* the oracle. Shortcut only at `hhem ≥ 0.995 AND top_sim ≥ 0.6 AND the atom carries no number or version`, and never shortcut the "no" end. + - **No cheap matcher may score a protected atom.** Agreement against the band mean falls −0.18 to −0.25 on numeric atoms, −0.07 to −0.10 on version-bound ones, −0.03 to −0.05 on negations, because a similarity score reads topic and not value: `_mm512_reduce_add_epi64` scores mid-scale against an atom @@ -1416,6 +1484,7 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. far from 1 (0.629 one way, 0.851 the other, over 1134 oracle comparisons, and 0.733/0.876 with a third extractor that never saw the reference set). **An atom count is extractor-conditional; an `added` rate is a ratio over whatever inventory the extractor produced and is far more robust.** Quote counts with the extractor named. + - **Tag labelling reproduces; the `type` vocabulary does not.** Holding extraction fixed and having two other models re-label the same 2640 atoms under the same glossary, six of nine tags reach κ 0.52–0.73 and the knowledge-type table above re-derives to within 0.08 under every labeller — so the @@ -1425,6 +1494,7 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. with both labellers moving `contract` and `edge` atoms into `motif`. Gate on the protection tags (`routing`, `negative-constraint`, `version-bound` — the top three by κ); report shape slices without gating on them. + - **Judge calibration — and a 0.15 band on `added` differences.** Signs and rankings, not second decimals. Quantified: the two graders' fine-scale κ is 0.576 (the 0.664 figure is raw agreement), and they differ by +7.7pp under strict grading and −11.6pp under lenient — opposite directions, which is the bias shape @@ -1432,22 +1502,27 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. [0.106, 0.194]**, so **no `added` difference below 0.15 may be quoted**, and for one model that band covers almost the entire between-artifact spread. Narrowing it needs more dual grading (only 25 probes have both arms dual-graded), not more probes. + - **Retention is not monotone in the text you supply.** The matcher retrieves its top chunks before it entails, so a longer text can score *lower* than a subset of itself: atom recovery from a cited span is 0.833, rises to 0.907 with the heading path, then falls to 0.809 when the whole file is supplied. Compare retention only between texts of similar length — which the shortening candidates are, by construction — and never read a retention figure across a large length gap. + - **The need graph has a 14% noise floor** (author/critic path-label agreement was 86%), which propagates into relevance-weighted `value`. + - **Need frequency is assumed uniform.** Nothing here measures how often each need actually arises, so `value` ranks atoms *within* one artifact and must not be compared across artifacts. Real query or issue traffic is what closes that, not another judge. + - **Category boundaries are partly judged.** In the study, where both axes were judged, an independent reviewer agreed with the five-category assignment on 72% of atoms; the disagreements concentrate on useful-vs-inert (±3 points) and known-vs-novel (±2 points). The scaffolding share is the most robust of them. Here useful-vs-inert is still judged and the ±3 applies; known-vs-novel is measured by the paired probes, so it carries the probe grading's error instead. + - **Per-artifact `added` is a profile, not a ranking.** Paired probes per artifact were 12–29 in the study, and subsampling shows the cross-artifact ranking destabilizing below a 0.75 probe budget (Spearman ρ 0.43–0.63 at 0.10). Correlating `added` against the measured task delta says the same @@ -1463,10 +1538,12 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. soft. Two validity checks come with it: `K0` and `degraded` both predict the delta negatively in every group, as designed. All of it under **strict** grading — lenient `added` correlates at 0.126 pooled and 0.000 within-artifact, so partial credit does not survive to task outcomes. + - **A topic-level verdict is a distribution, never a label.** No whole artifact in the study had unanimous atom outcomes under strict grading, on any model; at heading level unanimity was 0.39–0.44 and roughly one atom in four disagreed with its own section's majority. "This section is redundant" is not a statement this instrument can make. + - **Loading a correct artifact raises resistance to a reversed premise.** An earlier reading of the study (45 probes worse, 14 better) had the sign inverted by its analysis script; recomputed on the planted-inversion probes, paired by probe and model, 49 got worse and 367 better @@ -1476,6 +1553,7 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. the artifact, a planted wrong value is stated 0.942 of the time when plausible, 0.846 when absurd, and flagged 3% of the time. `degraded` is the visible tip of that; the channel underneath it transmits at 85–94%. + - **A cell can stop being re-measurable.** Two of the models behind most of these numbers no longer resolve, and one that does no longer accepts `temperature=0`. A novelty figure is quotable only with the served model named and the date attached; a cell whose model is gone is a dead cell, not a cell @@ -1485,4 +1563,5 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. - [skill-novelty-evidence.md](skill-novelty-evidence.md) — the 33-skill study these numbers come from, with its method and its caveats. + - [evaluating-treatments.md](evaluating-treatments.md) — the arm framework this layer consumes. diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index ab3985679..7edb0d3ef 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -85,6 +85,8 @@ class Concept: canonical=( Flag("--judge", "PROVIDER/MODEL", _MODEL_REF, repeatable=True, meaning="the judge; repeated, it is a panel"), + Flag("--judge-aggregate", "{median,majority,unanimous}", "choice", + meaning="how a judge panel's verdicts become one"), Flag("--extract-model", "PROVIDER/MODEL", _MODEL_REF), Flag("--escalate-model", "PROVIDER/MODEL", _MODEL_REF), Flag("--fill-model", "PROVIDER/MODEL", _MODEL_REF), diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 11bba7747..9b7bf529e 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -18,8 +18,10 @@ from typing import TYPE_CHECKING, NoReturn from dokimi.commands.inputs import placebo_text_for_arms_or_exit -from dokimi.commands.novelty_run import (add_judge2_args, add_spend_cap, judge2_client, - exit_if_analyze_partial, stamp, usable_cache_dir) +from dokimi.commands.novelty_run import (add_judge2_args, add_panel_args, add_spend_cap, + attach_panel, exit_if_analyze_partial, + judge2_client, panel_clients, save_panel_verdicts, stamp, + usable_cache_dir) from dokimi.commands.novelty_routing_inputs import ( checked_out_path, load_all_vs_all_field, @@ -114,7 +116,8 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) extractor, answerer, judge = cost.clients(args) - judge2 = judge2_client(args, answerer, judge, cost.client) + panel = panel_clients(args, cost.client) + judge2 = judge2_client(args, answerer, judge, cost.client, panel) config = AnalyzeConfig( artifact_ref=args.artifact, artifact_kind=artifact_bundle.kind, @@ -135,7 +138,8 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: try: result = analyze(artifact_bundle.text, config, extractor=extractor, answerer=answerer, judge=judge, - artifact_bundle=artifact_bundle) + artifact_bundle=artifact_bundle, + panel=panel, aggregate=args.judge_aggregate) except SelfJudgeError as exc: # the judge is the answering model: the flags contradict raise InputError(str(exc)) from exc except (ExtractionError, ValueError) as exc: # a model failure under way @@ -145,6 +149,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: exit_if_analyze_partial(result, out_json) # a run the cap stopped keeps what it paid for stamp(result.artifact, args, result.measure_result, answerer=answerer, judge=judge, judge2=judge2) save_artifact("novelty", result.artifact, out_json) + verdicts_path = save_panel_verdicts(result, out_json) out_md = out_json.with_suffix(".md") out_md.write_text(render_novelty_markdown(result.artifact), encoding="utf-8") title = f"{artifact_bundle.name or args.artifact} × {answerer.model}" @@ -156,6 +161,8 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: print(f"OK Saved novelty artifact: {out_json}") print(f"OK Saved novelty report: {out_md}") print(f"OK Saved annotated view: {view_path}") + if verdicts_path: + print(f"OK Saved judge verdicts: {verdicts_path}") _print_headline(result.artifact) @@ -219,10 +226,12 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: settings = _settings(args) placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) extractor, answerer, judge = cost.clients(args) + panel = panel_clients(args, cost.client) warn_answer_key(saved, answerer, judge) try: # before the paraphrase calls below, which are paid - check_self_judge(judge, answerer, args.allow_self_judge) - judge2 = judge2_client(args, answerer, judge, cost.client) + for member in (judge, *panel): + check_self_judge(member, answerer, args.allow_self_judge) + judge2 = judge2_client(args, answerer, judge, cost.client, panel) except SelfJudgeError as exc: raise InputError(str(exc)) from exc paraphrases, reversals = settings.values["paraphrases"], settings.values["reversals"] @@ -242,6 +251,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: arms=_arms(args.arms), concurrency=args.concurrency, allow_self_judge=args.allow_self_judge, seed=args.seed, placebo_text=placebo_text, framing=args.context_framing, + panel=panel, aggregate=args.judge_aggregate, ) except ValueError as exc: # a model failure under way raise SystemExit(str(exc)) from None @@ -252,6 +262,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: probe_set=probe_set, measure_result=result, sampling=sampling_record(args, atoms, saved), framing=args.context_framing, ) + attach_panel(payload, result, args.judge_aggregate, answerer.model) stamp(payload, args, result, answerer=answerer, judge=judge, judge2=judge2) out = Path(args.out) if args.out else _default_dir(args.artifact) / "probes.json" # A run the cap stopped keeps the answers it paid for, as a partial result (and exits). @@ -623,6 +634,7 @@ def _add_model_args(parser: argparse.ArgumentParser, positive_int) -> None: help="Provider for the judge. Without --judge-model, the default judge " "that runs on this provider; if none does, the run exits asking " "for --judge-model") + add_panel_args(parser) parser.add_argument("--extract-model", default=None, dest="extract_model", help="Extractor/adjudicator model (defaults to --judge-model, so the " "answering model never writes its own answer key)") diff --git a/dokimi/dokimi/commands/novelty_budget.py b/dokimi/dokimi/commands/novelty_budget.py index cbe5e4a44..5aa443575 100644 --- a/dokimi/dokimi/commands/novelty_budget.py +++ b/dokimi/dokimi/commands/novelty_budget.py @@ -109,7 +109,7 @@ def write_run(payload: dict, out: Path, partial: dict | None = None) -> Path: #: shared cap refuses the second judge, so its block is a stop record or empty. PARTIAL_SUMMARIES = ("paired", "arms", "abstentions", "reversal_resisted", "reversal_resisted_by_arm", "reversal_paired", "confidence_correlation", - "judge2") + "judge2", "judge_panel") def save_probe_run(payload: dict, out: Path, *, done: int, not_run: int, @@ -131,9 +131,12 @@ def save_probe_run(payload: dict, out: Path, *, done: int, not_run: int, "message": message} # The jobs run arm by arm, so a cut leaves the arms uneven: the summaries over the answers # kept would read as results next to status "partial". The runs stay; the summaries go. + panel = payload.get("judge_panel") for block in PARTIAL_SUMMARIES: if block in payload: payload[block] = None + if panel: + payload["judge_panel"] = {k: panel[k] for k in ("judges", "rule") if k in panel} exit_partial(message, write_run(payload, out, partial)) @@ -182,6 +185,7 @@ def run_estimate(args: argparse.Namespace, settings, text: str, estimate_usd) -> ``estimate_usd`` is the command's pricing (``dokimi.novelty.cost.estimate_usd``), looked up by the caller at call time, so a test that patches ``cost.estimate_usd`` takes effect. """ + from dokimi.novelty import cost from dokimi.novelty.measure import parse_arms from dokimi.novelty.spans import segment from dokimi.utils import InputError @@ -200,6 +204,7 @@ def run_estimate(args: argparse.Namespace, settings, text: str, estimate_usd) -> args, n_tokens, estimate["estimate"]["by_role"], settings.tier, context_share=(n_arms - 1 - ("no_artifact_repeat" in args.arms)) / n_arms, ) + cost.add_panel_calls(args, estimate["estimate"]) # after the price, which counts each judge itself return estimate, n_tokens diff --git a/dokimi/dokimi/commands/novelty_run.py b/dokimi/dokimi/commands/novelty_run.py index f8c55c01d..0b03371b2 100644 --- a/dokimi/dokimi/commands/novelty_run.py +++ b/dokimi/dokimi/commands/novelty_run.py @@ -10,6 +10,7 @@ import argparse import functools import math +from pathlib import Path import sys from dokimi.commands.cache_dir import usable_cache_dir # noqa: F401 - re-exported for the novelty commands @@ -70,14 +71,14 @@ def add_judge2_args(parser: argparse.ArgumentParser, providers) -> None: help="Provider for --judge2 (default: the judge's provider)") -def judge2_client(args: argparse.Namespace, answerer, judge, make_client): - """The ``--judge2`` client, checked against the answerer before any paid call; ``None`` without it.""" +def judge2_client(args: argparse.Namespace, answerer, judge, make_client, panel=()): + """The ``--judge2`` client, checked against the answerer and every panel judge before any paid call; ``None`` without it.""" from dokimi.novelty.judge2 import check_judge2 if not getattr(args, "judge2", None): return None client = make_client(args.judge2, args.judge2_provider or judge.provider, "judge2", args) - check_judge2(client, answerer, args.judge2_frac, args.allow_self_judge, judge) + check_judge2(client, answerer, args.judge2_frac, args.allow_self_judge, judge, panel) return client @@ -106,3 +107,58 @@ def stamp(payload: dict, args: argparse.Namespace, measure_result, *, answerer, payload["judge2"] = {"model": judge2.model, "stopped": str(exc)} print(f"WARNING second judge stopped: {exc}", file=sys.stderr) payload["judge2"]["cost"] = judge2.cost_summary() + + +def add_panel_args(parser: argparse.ArgumentParser) -> None: + """``--judge`` (repeated, it is a panel) and ``--judge-aggregate``.""" + parser.add_argument("--judge", action="append", default=None, dest="judges", + metavar="PROVIDER/MODEL", + help="A judge, as PROVIDER/MODEL; repeated, it is a panel: every judge " + "grades every answer and --judge-aggregate decides. Pick judges " + "from different vendors (docs/judge-panel-study.md: three from " + "three families cut the judge spread by a third). Replaces " + "--judge-model/--judge-provider; the first judge is the default " + "extractor") + parser.add_argument("--judge-aggregate", default="median", dest="judge_aggregate", + choices=("median", "majority", "unanimous"), + help="How a panel's verdicts become one: median of the verdict scale " + "(default), lenient majority, or unanimous (every judge must agree; " + "not conservative, it can raise the added share)") + + +def panel_clients(args: argparse.Namespace, make_client) -> tuple: + """Clients for the judges after the first ``--judge`` (the first is the judge client).""" + from dokimi.novelty.cost import judge_refs + from dokimi.utils import InputError + + try: + refs = judge_refs(args) + except ValueError as exc: + raise InputError(str(exc)) from exc + return tuple(make_client(model, provider, "judge", args) for model, provider in refs[1:]) + + +def attach_panel(payload: dict, result, aggregate: str, answerer_model: str | None = None) -> None: + """Add the ``judge_panel`` block to a written record and print its warnings.""" + from dokimi.novelty.panel_record import panel_record + from dokimi.novelty.panel import panel_warnings + + judge_panel = panel_record(result, aggregate, answerer_model) + if judge_panel: + payload["judge_panel"] = judge_panel + for note in panel_warnings(judge_panel): + print(f"WARNING: {note}", file=sys.stderr) + + +def save_panel_verdicts(result, out: Path) -> Path | None: + """Write each judge's verdict on every answer beside ``out``, so a panel run can be audited or + re-aggregated without grading again; ``None`` for a single-judge run.""" + import json + + runs = [run.as_dict() for run in result.measure_result.runs if run.judge_verdicts] + dest = out.with_name(f"{out.stem}.judge_verdicts.json") + if not runs: # a single-judge run into the same output: the panel's file is not this run's + dest.unlink(missing_ok=True) + return None + dest.write_text(json.dumps(runs, indent=2), encoding="utf-8") + return dest diff --git a/dokimi/dokimi/novelty/analyze.py b/dokimi/dokimi/novelty/analyze.py index 3b08bd3c4..e514b4607 100644 --- a/dokimi/dokimi/novelty/analyze.py +++ b/dokimi/dokimi/novelty/analyze.py @@ -24,9 +24,12 @@ from . import attribute, bundle as bundle_mod from . import classify, items as items_mod from . import measure, needs, probes, provenance, report, retention, shorten, spans +from . import panel as panel_mod +from . import panel_grade +from . import panel_record as panel_record_mod from .artifact import load_artifact_text # noqa: F401 (public path: novelty.analyze) from .analyze_partial import Progress, stopped_record -from .client import LLMClient, SpendCapReached, roll_up_cost, same_client, warn_extractor_judge +from .client import LLMClient, SpendCapReached, roll_up_cost, warn_extractor_judge from .retention import RetentionOracle from .tiers import DEFAULT_SETTINGS, RunSettings @@ -176,6 +179,8 @@ def analyze( ref_digest: str | None = None, artifact_bundle: "bundle_mod.Bundle | None" = None, tier: dict | None = None, + panel: tuple[LLMClient, ...] = (), + aggregate: str = "median", ) -> AnalyzeResult: """Run the full pipeline over one artifact's text. @@ -203,23 +208,27 @@ def analyze( per-file provenance: every span, atom and outcome can then be traced to the file it came from, which is the cut an author needs to decide what belongs in the body. Without it the run is entry-point-only and says so. + panel: More judges besides ``judge`` (see :func:`measure.run_measurement`). + aggregate: The panel's rule for one verdict per answer. """ progress = Progress() try: return _analyze(text, config, extractor=extractor, answerer=answerer, judge=judge, ref_digest=ref_digest, artifact_bundle=artifact_bundle, tier=tier, - progress=progress) + progress=progress, panel=panel, aggregate=aggregate) except SpendCapReached as stop: if ref_digest is None and artifact_bundle is not None: ref_digest = artifact_bundle.measured_digest() record = stopped_record( progress, stop, artifact_kind=config.artifact_kind, artifact_ref=config.artifact_ref, ref_digest=ref_digest, model=answerer.model, provider=answerer.provider, - judge_model=judge.model, clients=[extractor, answerer, judge], + judge_model=judge.model, clients=[extractor, answerer, judge, *panel], provenance=provenance.provenance( artifact_digest=ref_digest, answerer_model=answerer.model, judge_model=judge.model, context_framing=config.context_framing), - bundle=artifact_bundle.summary() if artifact_bundle is not None else {}) + bundle=artifact_bundle.summary() if artifact_bundle is not None else {}, + judge_panel={"judges": [judge.model, *(m.model for m in panel)], "rule": aggregate} + if panel else None) return AnalyzeResult( artifact=record, atoms=progress.atoms, spans=progress.spans, measure_result=progress.measure_result, extracted_atoms=progress.extracted, @@ -237,10 +246,14 @@ def _analyze( artifact_bundle: "bundle_mod.Bundle | None", tier: dict | None, progress: Progress, + panel: tuple[LLMClient, ...], + aggregate: str, ) -> AnalyzeResult: """The pipeline of :func:`analyze`; ``progress`` records each stage as it starts and what it produced, so a spend stop anywhere can be saved.""" - measure.check_self_judge(judge, answerer, config.allow_self_judge) + panel_grade.check_judges( + (judge, *panel), aggregate, + lambda member: measure.check_self_judge(member, answerer, config.allow_self_judge)) relation = warn_extractor_judge(extractor.model, judge, extractor.provider) warnings: list[str] = progress.warnings span_list = progress.spans = spans.segment(text) @@ -327,10 +340,13 @@ def _analyze( answerer=answerer, judge=judge, arms=config.arms, concurrency=config.concurrency, allow_self_judge=config.allow_self_judge, placebo_text=config.placebo_text, seed=config.seed, framing=config.context_framing, + panel=panel, aggregate=aggregate, ) _refuse_cut_short(measure_result) + judge_panel = panel_record_mod.panel_record(measure_result, aggregate, answerer.model) + warnings.extend(panel_mod.panel_warnings(judge_panel)) progress.stage = "classify" - if config.allow_self_judge and same_client(judge, answerer): + if config.allow_self_judge and panel_mod.same_as_answerer((judge, *panel), answerer): warnings.append( "judge and answerer are the same model (--allow-self-judge): scores are inflated" ) @@ -471,7 +487,7 @@ def _analyze( conflicts=conflicts, conflict_coverage=conflict_coverage, extraction=extraction.as_dict(), - cost=roll_up_cost([extractor, answerer, judge]), + cost=roll_up_cost([extractor, answerer, judge, *panel]), warnings=warnings, ) payload["run_config"] = _run_config(config) @@ -482,6 +498,7 @@ def _analyze( provenance.same_vendor_warning(payload["provenance"])])}) payload["run_config"]["tier"] = settings.tier_record(sampled=capped) payload["run_config"]["extractor_judge"] = relation + payload.update(panel_mod.panel_block(judge_panel)) return AnalyzeResult( artifact=payload, attributions=attributions, diff --git a/dokimi/dokimi/novelty/analyze_partial.py b/dokimi/dokimi/novelty/analyze_partial.py index 2f54c9af8..797284835 100644 --- a/dokimi/dokimi/novelty/analyze_partial.py +++ b/dokimi/dokimi/novelty/analyze_partial.py @@ -75,7 +75,8 @@ def stop_block(progress: Progress, stop: SpendCapReached) -> dict: def stopped_record(progress: Progress, stop: SpendCapReached, *, artifact_kind: str, artifact_ref: str, ref_digest: str | None, model: str, provider: str, - judge_model: str, clients: list, provenance: dict, bundle: dict) -> dict: + judge_model: str, clients: list, provenance: dict, bundle: dict, + judge_panel: dict | None = None) -> dict: """The partial record: it validates as ``novelty.v1`` and says ``status: partial``.""" result = progress.measure_result extraction = progress.extraction @@ -89,6 +90,7 @@ def stopped_record(progress: Progress, stop: SpendCapReached, *, artifact_kind: "model": model, "provider": provider, "judge_model": judge_model, + **({"judge_panel": judge_panel} if judge_panel else {}), "atoms": { # The probed sample once it is chosen, as a finished record counts; before, all extracted. "n": len(progress.extracted if progress.sampling == "none" else progress.atoms), diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index e625d0b5a..335429d75 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -71,6 +71,8 @@ class SpendCapReached(BaseException): reply: str | None = None truncated: bool = False + #: With a judge panel: the verdicts of the judges graded before this stop, by model. + panel_votes: dict[str, str] | None = None def __init__(self, message: str, *, reason: str = "spend_cap") -> None: super().__init__(message) @@ -335,6 +337,7 @@ def same_client(a: LLMClient, b: LLMClient) -> bool: ("claude", "anthropic"), ("anthropic", "anthropic"), ("sonnet", "anthropic"), ("opus", "anthropic"), ("haiku", "anthropic"), ("gpt", "openai"), ("openai", "openai"), ("gemini", "google"), ("mistral", "mistral"), + ("grok", "xai"), ("xai", "xai"), ) diff --git a/dokimi/dokimi/novelty/cost.py b/dokimi/dokimi/novelty/cost.py index f0b7f300e..12260e36a 100644 --- a/dokimi/dokimi/novelty/cost.py +++ b/dokimi/dokimi/novelty/cost.py @@ -16,11 +16,50 @@ from dokimi.utils import InputError +def judge_refs(args: argparse.Namespace) -> list[tuple[str, str]]: + """The ``--judge PROVIDER/MODEL`` list as (model, provider) pairs; empty without the flag.""" + from dokimi.llm import ANSWER_PROVIDERS as PROVIDERS + from dokimi.utils import same_model + + refs = getattr(args, "judges", None) or [] + if refs and (getattr(args, "judge_model", None) or getattr(args, "judge_provider", None)): + raise ValueError("--judge replaces --judge-model and --judge-provider; pass one form") + out: list[tuple[str, str]] = [] + for ref in refs: + provider, sep, model = ref.partition("/") + if not sep or not model or provider not in PROVIDERS: + raise ValueError(f"--judge {ref!r} is not PROVIDER/MODEL with PROVIDER one of " + f"{', '.join(PROVIDERS)}") + if any(same_model(m, model) for m, _ in out): + # one model would vote twice, and the panel keys each judge's verdicts by model name + raise ValueError(f"--judge {ref!r}: model {model!r} is given twice") + out.append((model, provider)) + return out + + +def add_panel_usd(args: argparse.Namespace, usd, by_role, n_tokens, tier, context_share, models): + """The USD estimate with each panel judge after the first grading every answer again.""" + from dokimi.novelty.tiers import estimate_usd + + for model, provider in judge_refs(args)[1:]: + if usd is None: + break + extra = estimate_usd({"judge": by_role.get("judge", 0)}, n_tokens, + {"judge": (model, provider)}, tier, context_share) + if extra is None: + models[f"judge {model}"] = (model, provider) + usd = None if extra is None else round(usd + extra, 6) + return usd + + def role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: """The (model, provider) of each role, defaulting so roles never silently collapse.""" from dokimi.novelty.client import default_judge - if args.judge_model: + panel = judge_refs(args) + if panel: + judge_model, judge_provider = panel[0] + elif args.judge_model: judge_model, judge_provider = args.judge_model, args.judge_provider or args.provider else: # a CLI answerer with no judge provider judges on the same CLI, another family @@ -38,20 +77,29 @@ def estimate_usd( args: argparse.Namespace, n_tokens: int, by_role: dict[str, int], tier: str | None, context_share: float = 0.5, ) -> float | None: - """The estimate's dollar figure; ``None`` when a role's model cannot be resolved or priced. + """The estimate's dollar figure; ``None`` when a role's model cannot be priced. The reason goes to stderr: a role whose model does not resolve, or a role with no list - price, makes the figure unavailable, and the note names it. + price, makes the figure unavailable, and the note names it. Each panel judge after the + first is priced as grading every answer again. + + Raises: + InputError: A ``--judge`` that is not a usable PROVIDER/MODEL, as the run itself refuses it. """ from dokimi.llm_pricing import list_cost_usd from dokimi.novelty.tiers import estimate_usd as tier_estimate_usd + try: + judge_refs(args) + except ValueError as exc: + raise InputError(str(exc)) from exc try: models = role_models(args) except ValueError as exc: print(f"The USD estimate is unavailable: {exc}", file=sys.stderr) return None - usd = tier_estimate_usd(by_role, n_tokens, models, tier, context_share) + usd = add_panel_usd(args, tier_estimate_usd(by_role, n_tokens, models, tier, context_share), + by_role, n_tokens, tier, context_share, models) if usd is None: unpriced = sorted( f"{role} ({model})" @@ -66,6 +114,22 @@ def estimate_usd( return usd +def add_panel_calls(args: argparse.Namespace, estimate: dict) -> None: + """Count the panel's grading calls in a run estimate: each judge after the first grades every answer again.""" + + try: + extra_judges = max(0, len(judge_refs(args)) - 1) + except ValueError as exc: + raise InputError(str(exc)) from exc + extra = extra_judges * estimate["by_role"]["judge"] + if not extra: + return + estimate["by_role"]["judge"] += extra + for key in ("probe_calls", "total_calls", "total_calls_if_every_pair_escalates", + "expected_calls"): + estimate[key] += extra + + def clients(args: argparse.Namespace): """Build the three role clients, defaulting so roles never silently collapse.""" try: diff --git a/dokimi/dokimi/novelty/judge2.py b/dokimi/dokimi/novelty/judge2.py index 418127e94..14a68be1f 100644 --- a/dokimi/dokimi/novelty/judge2.py +++ b/dokimi/dokimi/novelty/judge2.py @@ -22,8 +22,8 @@ def check_judge2(judge2: LLMClient, answerer: LLMClient, frac: float, allow_self_judge: bool, - judge: LLMClient | None = None) -> None: - """Refuse a second judge that is the answering model or the first judge, or a fraction outside (0, 1]. + judge: LLMClient | None = None, panel: tuple[LLMClient, ...] = ()) -> None: + """Refuse a second judge that is the answering model, the first judge or a panel judge, or a fraction outside (0, 1]. The model test is the repo's model-identity rule (``same_client``, which reads ``same_model``): an alias, an equivalent API id and a Bedrock id of one model are one model. @@ -43,11 +43,12 @@ def check_judge2(judge2: LLMClient, answerer: LLMClient, frac: float, allow_self f"--judge2 {judge2.model!r} is the answering model; grading a response with the " "model that produced it inflates the score. Pass another --judge2, or " "--allow-self-judge to do it deliberately.") - if judge is not None and same_client(judge2, judge): - raise InputError( - f"--judge2 {judge2.model!r} is the same model as the first judge {judge.model!r}; " - "its agreement with that judge is not an independent second opinion. Pass another " - "--judge2, or --allow-self-judge to do it deliberately.") + for role, member in (("first judge", judge), *(("panel judge", m) for m in panel)): + if member is not None and same_client(judge2, member): + raise InputError( + f"--judge2 {judge2.model!r} is the same model as the {role} {member.model!r}; " + "its agreement with that judge is not an independent second opinion. Pass another " + "--judge2, or --allow-self-judge to do it deliberately.") def cohen_kappa(a: list[bool], b: list[bool]) -> float | None: @@ -62,7 +63,12 @@ def cohen_kappa(a: list[bool], b: list[bool]) -> float | None: def first_verdict(run) -> str: - """The first judge's verdict. A check's veto may overrule it in ``run.verdict``, not here.""" + """The first judge's verdict. A check's veto may overrule it in ``run.verdict``, not here. + + With a panel the first judge is the first entry of ``judge_verdicts``, not the aggregate. + """ + if run.judge_verdicts: + return next(iter(run.judge_verdicts.values())) return run.judge_verdict if run.judge_verdict is not None else run.verdict diff --git a/dokimi/dokimi/novelty/measure.py b/dokimi/dokimi/novelty/measure.py index c58feeb8d..1e490b960 100644 --- a/dokimi/dokimi/novelty/measure.py +++ b/dokimi/dokimi/novelty/measure.py @@ -34,6 +34,7 @@ from .client import LLMClient, SpendCapReached, parse_json_object, same_client from .controls import (REPEAT_ARM, clustered_ci, control_fields, reversal_by_arm, reversal_paired) +from .panel_grade import check_judges, grade_panel from .probes import Probe from .routing_stop import PERMANENT_REASON, stop_if_run_wide @@ -135,6 +136,7 @@ class ProbeRun: response: str verdict: str confidence: float | None = None # recorded, never scored + judge_verdicts: dict[str, str] | None = field(default=None, kw_only=True) # panel only: each judge's verdict by model #: Set only when the probe carries ``grading``: the judge's own verdict, which ``verdict`` #: may overrule, where the verdict came from (``judge`` or ``check``) and the check results. judge_verdict: str | None = None @@ -161,6 +163,7 @@ def as_dict(self) -> dict: "response": self.response, "verdict": self.verdict, "confidence": self.confidence, + **({"judge_verdicts": self.judge_verdicts} if self.judge_verdicts else {}), **({"judge_verdict": self.judge_verdict, "verdict_source": self.verdict_source, "checks": self.checks} if self.checks else {}), } @@ -175,13 +178,13 @@ class MeasureResult: with_arm: str = "full" # (probe, arm) jobs the spend cap stopped before they were answered; 0 for a whole run. not_run: int = 0 - # Answers the cap paid for but the judge call was refused: kept in ``runs`` (no datum), - # and counted here so the run is not taken as finished. + # Answers the cap paid for but the judge refused: kept in ``runs`` (no datum), counted here. ungraded: int = 0 # (reason, message) of a call the cap refused; None when nothing was stopped. stop: tuple[str, str] | None = None # The probes answered, kept so a second judge can regrade a subset (``judge2``). probes: list[Probe] = field(default_factory=list) + judges: tuple[str, ...] = () # configured judges, judge first (a stopped panel may not vote all) def paired(self, scoring: str) -> dict: """The 2×2 for one scoring (see :func:`paired_outcomes`).""" @@ -334,6 +337,8 @@ def run_measurement( placebo_text: str | None = None, seed: int = 0, framing: str = "reference", + panel: tuple[LLMClient, ...] = (), + aggregate: str = "median", ) -> MeasureResult: """Answer every probe in every arm and grade the answers. @@ -352,8 +357,11 @@ def run_measurement( irrelevant context actively harms them. seed: Seed for the ``shuffled`` control, so the control is reproducible. framing: One of :data:`FRAMINGS`; how the context is introduced in the prompt. + panel: More judges besides ``judge``, each grading every answer. + aggregate: The rule that makes one verdict of the panel's (:func:`~.panel.aggregate`). """ - check_self_judge(judge, answerer, allow_self_judge) + judges = (judge, *panel) + check_judges((judge, *panel), aggregate, lambda m: check_self_judge(m, answerer, allow_self_judge)) check_arms(arms, placebo_text) if framing not in FRAMINGS: raise InputError(f"unknown context framing {framing!r}; known: {list(FRAMINGS)}") @@ -372,6 +380,7 @@ def run_measurement( def one(job: tuple[Probe, str]) -> ProbeRun: probe, arm = job + by_judge: dict[str, str] | None = None answered = True try: response, confidence = _answer( @@ -387,13 +396,14 @@ def one(job: tuple[Probe, str]) -> ProbeRun: answered = False else: try: - verdict = _grade_unless_stopped(stops, probe, response, judge) + verdict, by_judge = grade_panel( + lambda m: _grade_unless_stopped(stops, probe, response, m), judges, aggregate, ERRORED) except SpendCapReached as exc: return _capped_grade_run(probe, arm, response, confidence, exc, stops, ungraded) run = ProbeRun( atom_id=probe.atom_id, probe_kind=probe.kind, arm=arm, question=probe.question, response=response, verdict=verdict, - confidence=confidence, + confidence=confidence, judge_verdicts=by_judge, ) if probe.grading and answered: _apply_checks(run, probe, response, verdict) @@ -412,7 +422,7 @@ def guarded(job: tuple[Probe, str]) -> ProbeRun | None: stop = (stops[0].reason, str(stops[0])) if stops else None return MeasureResult(runs=runs, arms=tuple(arms), with_arm="full", not_run=len(jobs) - len(runs), ungraded=len(ungraded), stop=stop, - probes=list(probes)) + probes=list(probes), judges=tuple(j.model for j in judges)) def _held_permanent_stop(stops: list[SpendCapReached]) -> SpendCapReached | None: @@ -441,14 +451,15 @@ def _capped_grade_run(probe: Probe, arm: str, response: str, confidence: float | exc: SpendCapReached, stops: list[SpendCapReached], ungraded: list[ProbeRun]) -> ProbeRun: """The answer was paid for and the judge hit the cap: keep the verdict a refused judge reply - names, else keep the run ungraded (no datum). The checks grade the run either way, and the + names (not a truncated one, not one judge's of a panel), else keep the run ungraded. The checks grade the run either way, and the measurement stops here.""" stops.append(exc) - named = _reply_verdict(exc.reply) + panel_votes = getattr(exc, "panel_votes", None) + named = None if exc.truncated or panel_votes is not None else _reply_verdict(exc.reply) run = ProbeRun( atom_id=probe.atom_id, probe_kind=probe.kind, arm=arm, question=probe.question, response=response, verdict=named or ERRORED, - confidence=confidence, + confidence=confidence, judge_verdicts=panel_votes or None, ) if probe.grading: # the veto runs on the answer whatever the refused judge reply said _apply_checks(run, probe, response, named or ERRORED) @@ -787,4 +798,3 @@ def _pearson(xs: list[float], ys: list[float]) -> float | None: if den == 0: return None return round(sum(a * b for a, b in zip(dx, dy, strict=True)) / den, 4) - diff --git a/dokimi/dokimi/novelty/panel.py b/dokimi/dokimi/novelty/panel.py new file mode 100644 index 000000000..26bc56753 --- /dev/null +++ b/dokimi/dokimi/novelty/panel.py @@ -0,0 +1,117 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A judge panel for the paired probes: several judges grade each answer, one rule decides. + +The judge-panel study (docs/judge-panel-study.md) re-graded the skill-novelty sample with +seven judges from seven families. The paired added share of one answer model moved by up to +0.23 with the judge, while the order of the extreme models did not move. Majority, median +and Dawid-Skene agreed on 99.3% of rows; any-correct moved the headline by up to 0.20, and +unanimous raised the added share by up to 0.09 over majority. Three judges from three families cut the spread across panels from SD 0.039 to +0.025. Hence the defaults here: the median rule, a warning when two judges share a vendor, +and a per-judge 2×2 next to the panel's, so a judge-sensitive headline is visible. +""" + +from __future__ import annotations + +from collections import Counter + +from .client import same_client, vendor_of + +# The order the median walks: best first. The three wrong verdicts score 0 and tie. +_SCORE = { + "correct_complete": 1.0, + "correct_missing_qualifier": 0.7, + "partially_correct": 0.4, + "wrong_other": 0.0, + "wrong_reversed": 0.0, + "abstained": 0.0, +} +LENIENT = frozenset({"correct_complete", "correct_missing_qualifier", "partially_correct"}) +RULES = ("median", "majority", "unanimous") +# A judge range of the lenient added share above this marks the headline judge-sensitive. +# The study's noise floor (a fresh no-skill draw against the first) was 0.00-0.10. +SENSITIVE_RANGE = 0.10 + + +def _most_common(votes: list[str]) -> str: + """The most common verdict; a tie goes to the lower score, then to the scale's order.""" + counts = Counter(votes) + order = list(_SCORE) + return min(counts, key=lambda v: (-counts[v], _SCORE[v], order.index(v))) + + +def aggregate(votes: list[str], rule: str) -> str | None: + """One verdict from a panel's verdicts, or ``None`` when no judge returned one. + + ``votes`` holds only verdicts on the scale (failed grades are left out by the caller). + + * ``median``: the lower median of the verdict scores (an even panel rounds toward + wrong); among the 0-score verdicts the most common one, so ``wrong_reversed`` survives. + * ``majority``: correct (lenient) when more than half the judges say so, and then the + median of the correct votes; otherwise the most common of the other votes. + * ``unanimous``: correct only when every judge says so, then the lowest correct vote. + """ + if rule not in RULES: + raise ValueError(f"unknown judge aggregation rule {rule!r}; known: {list(RULES)}") + if not votes: + return None + good = [v for v in votes if v in LENIENT] + bad = [v for v in votes if v not in LENIENT] + if rule == "median": + ranked = sorted(votes, key=lambda v: -_SCORE[v]) + pick = ranked[len(ranked) // 2] + if _SCORE[pick] == 0.0: + return _most_common([v for v in votes if _SCORE[v] == 0.0]) + return pick + if rule == "majority": + if len(good) > len(votes) / 2: + return aggregate(good, "median") + return _most_common(bad) if bad else _most_common(good) + # unanimous + if not bad: + return min(good, key=lambda v: _SCORE[v]) + return _most_common(bad) + + +def shared_vendors(models: list[str]) -> dict[str, list[str]]: + """Vendor -> the panel's judges from it, for every vendor with more than one judge.""" + by_vendor: dict[str, list[str]] = {} + for model in models: + vendor = vendor_of(model) + if vendor: + by_vendor.setdefault(vendor, []).append(model) + return {v: ms for v, ms in by_vendor.items() if len(ms) > 1} + + +def panel_warnings(record: dict | None) -> list[str]: + """Warnings a panel record carries into the artifact's ``warnings``.""" + if not record: + return [] + out = [] + for vendor, models in sorted(record["shared_vendors"].items()): + out.append(f"judge panel has {len(models)} judges from one vendor ({vendor}: " + f"{', '.join(models)}); judges of one family agree with each other more " + "than with the rest, so the panel counts them nearly once") + for model in record.get("same_vendor_as_answerer", []): + out.append(f"judge {model} is of the answerer's vendor: a judge can favour answers of its " + "own vendor; pass a --judge of another vendor") + for model, failed in sorted(record.get("failed_grades", {}).items()): + if failed: + out.append(f"judge {model} failed to grade {failed} answer(s); its vote is left out " + "of those verdicts") + if record["judge_sensitive"]: + out.append(f"judge-sensitive: the lenient added share varies by " + f"{record['added_share_range']['lenient']:.2f} across the panel's judges " + f"(more than {SENSITIVE_RANGE:.2f}); report the range, not one judge's number") + return out + + +def same_as_answerer(judges, answerer) -> bool: + """Whether any judge is the answerer's model (the self-judge warning).""" + return any(same_client(j, answerer) for j in judges) + + +def panel_block(judge_panel: dict | None) -> dict: + """The ``judge_panel`` key of a record: empty for a single-judge run.""" + return {"judge_panel": judge_panel} if judge_panel else {} diff --git a/dokimi/dokimi/novelty/panel_grade.py b/dokimi/dokimi/novelty/panel_grade.py new file mode 100644 index 000000000..1f5f848b0 --- /dev/null +++ b/dokimi/dokimi/novelty/panel_grade.py @@ -0,0 +1,39 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Grading with a judge panel: the checks made before a paid call and one answer's votes.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence + +from dokimi.utils import InputError +from .client import LLMClient, SpendCapReached, same_client +from .panel import RULES, aggregate as aggregate_votes + + +def check_judges(members: Sequence[LLMClient], aggregate: str, + check_one: Callable[[LLMClient], None]) -> None: + """Refuse an unknown rule, a judge ``check_one`` refuses, or one model twice (it would vote twice).""" + if aggregate not in RULES: + raise InputError(f"unknown judge aggregation rule {aggregate!r}; known: {list(RULES)}") + for i, member in enumerate(members): + check_one(member) + if any(same_client(member, other) for other in members[:i]): + raise InputError(f"judge {member.model!r} is the same model as another judge") + + +def grade_panel(grade: Callable[[LLMClient], str], judges: Sequence[LLMClient], aggregate: str, + errored: str) -> tuple[str, dict[str, str] | None]: + """One answer's verdict and, with a panel, each judge's verdict by model; a failed judge is no vote.""" + if len(judges) == 1: + return grade(judges[0]), None + by_judge: dict[str, str] = {} + try: + for m in judges: + by_judge[m.model] = grade(m) + except SpendCapReached as exc: + exc.panel_votes = by_judge + raise + votes = [v for v in by_judge.values() if v != errored] + return aggregate_votes(votes, aggregate) or errored, by_judge diff --git a/dokimi/dokimi/novelty/panel_record.py b/dokimi/dokimi/novelty/panel_record.py new file mode 100644 index 000000000..0f4b34e48 --- /dev/null +++ b/dokimi/dokimi/novelty/panel_record.py @@ -0,0 +1,100 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The ``judge_panel`` block of a probe or novelty record, built from a panel's measured runs.""" + +from __future__ import annotations + +from .measure import BASELINE_ARM, ERRORED, ProbeRun, paired_outcomes, resolve_verdict +from .client import vendor_of +from .panel import SENSITIVE_RANGE, shared_vendors + +SCORINGS = ("lenient", "strict") + + +def _own_verdict(run: ProbeRun, model: str) -> str: + """The verdict ``model`` gave ``run``, with the run's program checks applied as the headline + applies them; a judge that cast no vote (a spend stop, a failed call) is a missing datum.""" + raw = (run.judge_verdicts or {}).get(model) + if raw is None or raw == ERRORED: + return ERRORED + return resolve_verdict(raw, run.checks)[0] if run.checks else raw + + +def _own_runs(runs: list[ProbeRun], model: str) -> list[ProbeRun]: + """``runs`` with each verdict replaced by the verdict ``model`` gave it.""" + return [ + ProbeRun(atom_id=r.atom_id, probe_kind=r.probe_kind, arm=r.arm, question=r.question, + response=r.response, verdict=_own_verdict(r, model)) + for r in runs + ] + + +def _graded_pairs(own: list[ProbeRun], with_arm: str) -> set[tuple[str, str, str]]: + """The probes one judge graded in both the baseline and ``with_arm``.""" + def keys(arm: str) -> set[tuple[str, str, str]]: + return {(r.atom_id, r.probe_kind, r.question) for r in own + if r.arm == arm and r.probe_kind != "reversal" and not r.errored} + return keys(BASELINE_ARM) & keys(with_arm) + + +def _added_range(per_judge_runs: dict[str, list[ProbeRun]], with_arm: str, + scoring: str) -> float | None: + """The spread of the added share across judges on the pairs every judge graded. + + A judge whose grade failed on one arm of a pair drops that pair from its own 2×2; the + range is taken on the pairs graded by all, so a missing grade is not read as a judge + that disagrees. A judge that graded no pair is left out. ``None`` when fewer than two judges + graded a pair, or when no pair is graded by every one of them. + """ + graded = [own for own in per_judge_runs.values() if _graded_pairs(own, with_arm)] + if len(graded) < 2: + return None + shared = set.intersection(*(_graded_pairs(own, with_arm) for own in graded)) + if not shared: + return None + shares = [] + for own in graded: + kept = [r for r in own + if r.probe_kind == "reversal" or (r.atom_id, r.probe_kind, r.question) in shared] + shares.append(paired_outcomes(kept, with_arm=with_arm, scoring=scoring)["added"]) + return round(max(shares) - min(shares), 4) + + +def panel_record(measure_result, rule: str, answerer_model: str | None = None) -> dict | None: + """The ``judge_panel`` block of a probe or novelty record: who judged, the rule, each + judge's own paired 2×2, the judge range of the added share, and a sensitivity flag. + + The range is over the pairs every judge graded in both arms; each judge's own 2×2 keeps + its own sample (``n_unpaired_dropped`` says what it lost). ``None`` for a single-judge + run, which records no panel. + """ + runs = measure_result.runs + judges: list[str] = list(measure_result.judges) + for run in runs: + for model in run.judge_verdicts or {}: + if model not in judges: + judges.append(model) + if len(judges) < 2: + return None + own_runs = {model: _own_runs(runs, model) for model in judges} + with_arm = measure_result.with_arm + per_judge = { + model: {s: paired_outcomes(own, with_arm=with_arm, scoring=s) for s in SCORINGS} + for model, own in own_runs.items() + } + added_range = {s: _added_range(own_runs, with_arm, s) for s in SCORINGS} + lenient_range = added_range["lenient"] + return { + "judges": judges, + "rule": rule, + "shared_vendors": shared_vendors(judges), + "same_vendor_as_answerer": [m for m in judges if answerer_model and vendor_of(m) + and vendor_of(m) == vendor_of(answerer_model)], + "failed_grades": {m: sum((r.judge_verdicts or {}).get(m) == ERRORED for r in runs) + for m in judges}, + "per_judge_paired": per_judge, + "added_share_range": added_range, + "judge_sensitive": lenient_range is not None and lenient_range > SENSITIVE_RANGE, + "sensitive_threshold": SENSITIVE_RANGE, + } diff --git a/dokimi/dokimi/novelty/report.py b/dokimi/dokimi/novelty/report.py index e96f810b1..5eb233d85 100644 --- a/dokimi/dokimi/novelty/report.py +++ b/dokimi/dokimi/novelty/report.py @@ -496,6 +496,15 @@ def _render_categories(artifact: dict, uncleared: str | None) -> list[str]: return [*lines, ""] +def _judged_by(artifact: dict) -> str: + """The judge line: the one judge, or a panel's judges and its rule.""" + panel = artifact.get("judge_panel") + if not panel: + return f"**{artifact['judge_model']}**" + return (f"a panel of {len(panel['judges'])} judges (**{'**, **'.join(panel['judges'])}**; " + f"{panel['rule']} rule)") + + def render_novelty_markdown(artifact: dict) -> str: """Render the summary report, in the order an author should act on it.""" lines: list[str] = [] @@ -504,7 +513,7 @@ def render_novelty_markdown(artifact: dict) -> str: lines.append("") lines.append( f"Model **{artifact['model']}** ({artifact['provider']}), judged by " - f"**{artifact['judge_model']}**. Artifact: {artifact['artifact']['tokens']} tokens, " + f"{_judged_by(artifact)}. Artifact: {artifact['artifact']['tokens']} tokens, " f"{artifact['atoms']['n']} atoms ({artifact['atoms']['sampling']}), " f"{artifact['atoms']['n_probed']} probed.{_capped_at(artifact)}" ) diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index 24af02e9c..e87240073 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -519,6 +519,10 @@ "type": "object", "description": "With --judge2: a seeded --judge2-frac subset of the graded runs regraded by a second judge; lenient_agreement, kappa_lenient, exact_agreement over n_compared, and that judge's cost." }, + "judge_panel": { + "type": "object", + "description": "With two or more --judge: judges, rule, shared_vendors, same_vendor_as_answerer, failed_grades, each judge's own paired 2x2 (per_judge_paired), the spread of the added share across judges (added_share_range) and judge_sensitive." + }, "status": { "const": "partial", "description": "Present only on a partial record: `skill novelty analyze` stopped by --max-usd (or a call with no known price) keeps what it paid for in .partial.json. Every reader of a finished novelty.json refuses it." diff --git a/dokimi/dokimi/subjects/novelty_track.py b/dokimi/dokimi/subjects/novelty_track.py index 32aaabc9e..255329f2d 100644 --- a/dokimi/dokimi/subjects/novelty_track.py +++ b/dokimi/dokimi/subjects/novelty_track.py @@ -74,6 +74,8 @@ def novelty_cell(artifact: dict, *, source: str | None = None) -> dict[str, Any] } if artifact.get("run_config"): cell["run_config"] = dict(artifact["run_config"]) + if artifact.get("judge_panel"): + cell["judge_panel"] = dict(artifact["judge_panel"]) if source: cell["source"] = source return cell diff --git a/dokimi/tests/test_novelty_panel.py b/dokimi/tests/test_novelty_panel.py new file mode 100644 index 000000000..31ec064c2 --- /dev/null +++ b/dokimi/tests/test_novelty_panel.py @@ -0,0 +1,526 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The judge panel: aggregation rules, per-judge 2×2, vendor and sensitivity warnings, CLI.""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from dokimi.novelty.atoms import Atom +from dokimi.novelty.client import LLMClient +from dokimi.novelty.cost import judge_refs as _judge_refs +from dokimi.novelty.cost import role_models as _role_models +from dokimi.novelty.judge2 import check_judge2 +from dokimi.novelty.measure import ( + BASELINE_ARM, + ERRORED, + MeasureResult, + ProbeRun, + SelfJudgeError, + run_measurement, +) +from dokimi.novelty.panel import aggregate, panel_warnings, shared_vendors +from dokimi.novelty.panel_record import panel_record +from dokimi.novelty.probes import Probe +from dokimi.utils import InputError + +CC, CMQ, PC = "correct_complete", "correct_missing_qualifier", "partially_correct" +WO, WR, AB = "wrong_other", "wrong_reversed", "abstained" + + +@pytest.mark.parametrize("votes, rule, want", [ + ([CC, CMQ, WO], "median", CMQ), + ([CC, WO, WO], "median", WO), + ([CC, CC, WO, WO], "median", WO), # an even panel rounds toward wrong + ([WR, WR, WO, CC, CC], "median", WR), # the most common wrong verdict survives + ([CC, PC, WO], "majority", PC), # correct majority: lower median of the correct votes + ([CC, WO, AB, AB], "majority", AB), + ([CC, WO], "majority", WO), # an exact split is not more than half + ([CC, CMQ, CC], "unanimous", CMQ), # all correct: the lowest correct vote + ([CC, CC, WR], "unanimous", WR), + ([], "median", None), +]) +def test_aggregate_rules(votes, rule, want): + assert aggregate(votes, rule) == want + + +def test_aggregate_rejects_unknown_rule(): + with pytest.raises(ValueError, match="rule"): + aggregate([CC], "mean") + + +def test_shared_vendors_names_only_repeated_families(): + assert shared_vendors(["gpt-6-luna", "gpt-6-sol", "claude-sonnet-5", "kimi-k3"]) == { + "openai": ["gpt-6-luna", "gpt-6-sol"]} + + +def _judge(model: str, verdict_for): + def transport(prompt, system): + return json.dumps({"verdict": verdict_for(prompt)}), None + return LLMClient(model=model, transport=transport) + + +def _setup(): + atoms = [Atom(statement=f"fact {i}", question=f"Q{i}?", answer="yes", span_ids=("s0001",)) + for i in range(4)] + probes = [Probe(atom_id=a.id, kind="direct", question=a.question, expected=a.answer) + for a in atoms] + + def answer(prompt, system): + return ("ARTIFACT" if "Reference material" in prompt else "BARE") + "\nconfidence: 0.5", None + + return atoms, probes, LLMClient(model="answerer", transport=answer) + + +def test_panel_grades_with_every_judge_and_keeps_each_verdict(): + atoms, probes, answerer = _setup() + # lenient judge: everything correct; strict judges: only the with-artifact answer + lenient = _judge("gpt-x", lambda p: CC) + strict_a = _judge("claude-y", lambda p: CC if "RESPONSE: ARTIFACT" in p else WO) + strict_b = _judge("kimi-z", lambda p: CC if "RESPONSE: ARTIFACT" in p else WO) + result = run_measurement(atoms, probes, "the artifact", answerer=answerer, judge=lenient, + panel=(strict_a, strict_b), aggregate="median", concurrency=1) + assert all(set(r.judge_verdicts) == {"gpt-x", "claude-y", "kimi-z"} for r in result.runs) + assert result.paired("lenient")["added"] == 1.0 # the panel sides with the two strict judges + rec = panel_record(result, "median") + assert rec["judges"] == ["gpt-x", "claude-y", "kimi-z"] + assert rec["per_judge_paired"]["gpt-x"]["lenient"]["added"] == 0.0 + assert rec["per_judge_paired"]["kimi-z"]["lenient"]["added"] == 1.0 + assert rec["added_share_range"]["lenient"] == 1.0 + assert rec["judge_sensitive"] is True + assert any("judge-sensitive" in w for w in panel_warnings(rec)) + assert "judge_verdicts" in result.runs[0].as_dict() + + +def test_single_judge_run_records_no_panel(): + atoms, probes, answerer = _setup() + result = run_measurement(atoms, probes, "t", answerer=answerer, + judge=_judge("j", lambda p: CC), concurrency=1) + assert result.runs[0].judge_verdicts is None + assert "judge_verdicts" not in result.runs[0].as_dict() + assert panel_record(result, "median") is None + + +def test_a_judge_that_fails_is_left_out_of_the_vote(): + atoms, probes, answerer = _setup() + broken = _judge("broken", lambda p: None) + result = run_measurement(atoms[:1], probes[:1], "t", answerer=answerer, + judge=_judge("a", lambda p: WO), panel=(broken, _judge("b", lambda p: WO)), + concurrency=1) + base = next(r for r in result.runs if r.arm == BASELINE_ARM) + assert base.verdict == WO + assert base.judge_verdicts["broken"] == "error" + + +def test_a_failed_vote_does_not_count_as_a_wrong_one(): + atoms, probes, answerer = _setup() + broken = _judge("broken", lambda p: None) + result = run_measurement(atoms[:1], probes[:1], "t", answerer=answerer, + judge=_judge("a", lambda p: CC), + panel=(broken, _judge("b", lambda p: CC)), + aggregate="unanimous", concurrency=1) + base = next(r for r in result.runs if r.arm == BASELINE_ARM) + assert base.verdict == CC + + +def test_every_panel_judge_is_checked_against_the_answerer(): + atoms, probes, answerer = _setup() + with pytest.raises(SelfJudgeError): + run_measurement(atoms, probes, "t", answerer=answerer, judge=_judge("j", lambda p: CC), + panel=(answerer,)) + + +def _args(**kw): + base = dict(judges=None, judge_model=None, judge_provider=None, model="m", provider="litellm", + extract_model=None, extract_provider=None) + base.update(kw) + return argparse.Namespace(**base) + + +def test_judge_flag_parses_provider_model_and_sets_the_extractor(): + args = _args(judges=["openrouter/gpt-6-luna", "openrouter/claude-sonnet-5"]) + assert _judge_refs(args) == [("gpt-6-luna", "openrouter"), ("claude-sonnet-5", "openrouter")] + roles = _role_models(args) + assert roles["judge"] == ("gpt-6-luna", "openrouter") + assert roles["extractor"] == ("gpt-6-luna", "openrouter") + + +@pytest.mark.parametrize("kw, match", [ + ({"judges": ["gpt-6-luna"]}, "PROVIDER/MODEL"), + ({"judges": ["nope/x"]}, "PROVIDER/MODEL"), + ({"judges": ["openrouter/a", "openrouter/a"]}, "twice"), + ({"judges": ["openrouter/a", "openai/a"]}, "twice"), + ({"judges": ["openrouter/a"], "judge_model": "b"}, "one form"), +]) +def test_judge_flag_rejects_bad_refs(kw, match): + with pytest.raises(ValueError, match=match): + _judge_refs(_args(**kw)) + + +def test_two_judges_of_one_vendor_the_vendor_map_lacks_are_one_family(): + grok = ["x-ai/grok-4.6", "x-ai/grok-4"] + assert shared_vendors(grok) == {"xai": grok} + assert shared_vendors(["acme/one", "acme/two"]) == {} # two unrecognised judges share nothing + + +def test_judge_flag_rejects_one_model_under_two_spellings(): + args = _args(judges=["openai/gpt-4o", "openrouter/openai/gpt-4o", "anthropic/claude-sonnet-5"]) + with pytest.raises(ValueError, match="twice"): + _judge_refs(args) + + +def test_run_measurement_rejects_one_model_given_twice_before_any_call(): + atoms, probes, answerer = _setup() + calls = [] + a = _judge("gpt-4o", lambda p: calls.append(p) or CC) + b = _judge("openai/gpt-4o", lambda p: calls.append(p) or CC) + with pytest.raises(InputError, match="same model as another judge"): + run_measurement(atoms, probes, "t", answerer=answerer, judge=a, panel=(b,)) + assert calls == [] + + +def test_judge2_is_checked_against_every_panel_judge(): + _, _, answerer = _setup() + first, second = _judge("j-one", lambda p: CC), _judge("j-two", lambda p: CC) + with pytest.raises(InputError, match="panel judge"): + check_judge2(_judge("j-two", lambda p: CC), answerer, 0.5, False, first, (second,)) + check_judge2(_judge("j-three", lambda p: CC), answerer, 0.5, False, first, (second,)) + + +def _run(atom: str, arm: str, verdicts: dict[str, str]) -> ProbeRun: + return ProbeRun(atom_id=atom, probe_kind="direct", arm=arm, question=f"Q{atom}?", + response="r", verdict=next(iter(verdicts.values())), judge_verdicts=verdicts) + + +def test_added_range_is_taken_on_the_pairs_every_judge_graded(): + err = "error" + runs = [ + _run("a", BASELINE_ARM, {"j1": WO, "j2": WO}), _run("a", "full", {"j1": CC, "j2": err}), + _run("b", BASELINE_ARM, {"j1": CC, "j2": CC}), _run("b", "full", {"j1": CC, "j2": CC}), + ] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["per_judge_paired"]["j1"]["lenient"]["added"] == 0.5 + assert rec["per_judge_paired"]["j2"]["lenient"]["added"] == 0.0 # its own sample lost pair a + assert rec["added_share_range"]["lenient"] == 0.0 + assert rec["judge_sensitive"] is False + + +def test_a_judge_sees_the_veto_its_own_verdict_meets_and_a_missing_vote_is_no_datum(): + veto = [{"role": "veto", "passed": False}] + runs = [ + _run("a", BASELINE_ARM, {"j1": WO, "j2": WO}), + _run("a", "full", {"j1": CC, "j2": CC}), # the veto fails: both judges' CC is wrong_other + _run("b", BASELINE_ARM, {"j1": WO, "j2": WO}), + _run("b", "full", {"j1": CC}), # j2 cast no vote on this answer (a spend stop) + ] + runs[1].checks = veto + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["per_judge_paired"]["j1"]["lenient"]["added"] == 0.5 # pair a: the veto makes it redundant + j2 = rec["per_judge_paired"]["j2"]["lenient"] + assert (j2["n_paired"], j2["n_unpaired_dropped"], j2["added"]) == (1, 1, 0.0) + + +def test_panel_warns_for_each_judge_of_the_answerers_vendor_and_for_a_failing_judge(): + err = "error" + runs = [_run("a", BASELINE_ARM, {"claude-opus-5": WO, "gpt-6-luna": err}), + _run("a", "full", {"claude-opus-5": CC, "gpt-6-luna": err})] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median", + "claude-sonnet-5") + assert rec["same_vendor_as_answerer"] == ["claude-opus-5"] + assert rec["failed_grades"] == {"claude-opus-5": 0, "gpt-6-luna": 2} + warnings = panel_warnings(rec) + assert any("claude-opus-5 is of the answerer's vendor" in w for w in warnings) + assert any("gpt-6-luna failed to grade 2" in w for w in warnings) + + +def test_a_range_of_exactly_the_threshold_is_not_judge_sensitive(): + runs = [] + for i in range(10): # j1 sees one probe gained, j2 none: a range of exactly 0.10 + runs += [_run(str(i), BASELINE_ARM, {"j1": WO, "j2": WO}), + _run(str(i), "full", {"j1": CC if i == 0 else WO, "j2": WO})] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["added_share_range"]["lenient"] == 0.1 + assert rec["judge_sensitive"] is False + + +def test_estimate_counts_each_panel_judges_grading_calls(): + from dokimi.novelty.cost import add_panel_calls + + estimate = {"by_role": {"extractor": 3, "answerer": 8, "judge": 8}, "probe_calls": 32, + "total_calls": 40, "total_calls_if_every_pair_escalates": 44, + "expected_calls": 19} + add_panel_calls(_args(judges=["openai/a", "openai/b", "anthropic/c"]), estimate) + assert estimate["by_role"]["judge"] == 24 + assert (estimate["probe_calls"], estimate["total_calls"], estimate["expected_calls"]) == (48, 56, 35) + assert estimate["total_calls_if_every_pair_escalates"] == 60 + single = dict(estimate, by_role=dict(estimate["by_role"])) + add_panel_calls(_args(judges=["openai/a"]), single) + assert single == estimate + + +def test_estimate_only_counts_each_panel_judge_at_the_command(tmp_path, capsys): + from dokimi.cli import build_parser + + (tmp_path / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + def estimate(*judges): + flags = [f for j in judges for f in ("--judge", j)] + args = build_parser().parse_args(["skill", "novelty", "analyze", "--artifact", + f"skill:{tmp_path}", "--estimate-only", *flags]) + args.func(args) + out = json.loads(capsys.readouterr().out) + return dict(out["estimate"], expected_usd=out["expected_usd"]) + one = estimate("openai/gpt-5.6-luna") + three = estimate("openai/gpt-5.6-luna", "anthropic/claude-sonnet-5", "openai/gpt-6-luna") + assert three["by_role"]["judge"] == 3 * one["by_role"]["judge"] + assert three["probe_calls"] > one["probe_calls"] + assert three["expected_usd"] > one["expected_usd"] + + +def test_estimate_only_refuses_an_unusable_judge(tmp_path): + from dokimi.cli import build_parser + + (tmp_path / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + args = build_parser().parse_args(["skill", "novelty", "analyze", "--artifact", + f"skill:{tmp_path}", "--estimate-only", "--judge", "nope/x"]) + with pytest.raises(InputError, match="PROVIDER/MODEL"): + args.func(args) + + +def test_the_report_names_the_panel_and_analyze_saves_each_verdict(tmp_path): + from dokimi.commands.novelty_run import save_panel_verdicts + from dokimi.novelty.report import _judged_by + + runs = [_run("a", BASELINE_ARM, {"j1": WO, "j2": CC}), _run("a", "full", {"j1": CC, "j2": CC})] + result = type("R", (), {"measure_result": MeasureResult(runs=runs)})() + out = tmp_path / "novelty.json" + dest = save_panel_verdicts(result, out) + assert dest == tmp_path / "novelty.judge_verdicts.json" + assert json.loads(dest.read_text())[0]["judge_verdicts"] == {"j1": WO, "j2": CC} + assert "panel of 2 judges" in _judged_by({"judge_model": "j1", "judge_panel": + {"judges": ["j1", "j2"], "rule": "median"}}) + assert _judged_by({"judge_model": "j1"}) == "**j1**" + single = type("R", (), {"measure_result": MeasureResult( + runs=[ProbeRun("a", "direct", "full", "q", "r", WO)])})() + assert save_panel_verdicts(single, out) is None + assert not dest.exists() # a one-judge run into the same output does not leave the panel's file + + +def _capped_judge(model: str, verdict: str, *, priced: bool, truncated: bool = False): + from dokimi.metrics.usage import UsageRecord + + def transport(prompt, system): + usage = UsageRecord(cost_usd=1.0 if priced else None, truncated=truncated) + return json.dumps({"verdict": verdict}), usage + + from dokimi.novelty.client import SpendCap + return LLMClient(model=model, transport=transport, cap=SpendCap(limit_usd=100.0)) + + +def _one_probe(): + atoms, probes, answerer = _setup() + return atoms[:1], probes[:1], answerer + + +def test_the_novelty_schema_declares_the_judge_panel_block(): + import json + from pathlib import Path + + schema = Path(__file__).resolve().parents[1] / "dokimi" / "schemas" / "novelty.v1.json" + assert "judge_panel" in json.loads(schema.read_text(encoding="utf-8"))["properties"] + + +def test_a_panel_stopped_before_every_judge_voted_still_records_the_panel(): + atoms, probes, answerer = _one_probe() + first = _capped_judge("j1", WO, priced=True) + second = _capped_judge("j2", CC, priced=False) + result = run_measurement(atoms, probes, "t", answerer=answerer, judge=first, panel=(second,), + aggregate="unanimous", concurrency=1) + assert result.stop is not None + rec = panel_record(result, "unanimous") + assert rec is not None + assert rec["judges"] == ["j1", "j2"] and rec["rule"] == "unanimous" + + +def test_an_unpriced_judge_stopping_a_panel_does_not_decide_the_verdict(): + atoms, probes, answerer = _one_probe() + first = _capped_judge("j1", WO, priced=True) + second = _capped_judge("j2", CC, priced=False) + result = run_measurement(atoms, probes, "t", answerer=answerer, judge=first, panel=(second,), + aggregate="median", concurrency=1) + stopped = [r for r in result.runs if r.judge_verdicts and "j2" not in r.judge_verdicts] + assert stopped and all(r.verdict == "error" for r in stopped) + assert result.ungraded == len(stopped) + assert result.paired("lenient")["added"] == 0.0 + + +def test_a_truncated_judge_reply_under_a_cap_is_a_failed_grade(): + atoms, probes, answerer = _one_probe() + judge = _capped_judge("j", CC, priced=False, truncated=True) + result = run_measurement(atoms, probes, "t", answerer=answerer, judge=judge, concurrency=1) + assert all(r.verdict == "error" for r in result.runs) + assert result.ungraded == len(result.runs) + + +def test_analyze_refuses_a_panel_member_that_answers_before_any_paid_call(tmp_path, monkeypatch): + from dokimi.cli import build_parser + from dokimi.novelty import cost + from dokimi.novelty.client import LLMClient + + (tmp_path / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + base = ["skill", "novelty", "analyze", "--artifact", f"skill:{tmp_path}"] + answer_model, answer_provider = cost.role_models(build_parser().parse_args(base))["answerer"] + calls: list[str] = [] + + def transport(prompt, system): + calls.append(prompt) # any call here is a paid one that the refusal should have stopped + raise AssertionError("a paid call happened before the self-judge refusal") + + monkeypatch.setattr(cost, "client", + lambda model, provider, role, args: LLMClient(model=model, transport=transport)) + args = build_parser().parse_args( + [*base, "--judge", f"{answer_provider}/{answer_model}", "--judge", "openai/gpt-6-luna"]) + with pytest.raises(InputError, match=answer_model): + args.func(args) + assert calls == [] + + +@pytest.mark.parametrize("sub", ["probe", "analyze"]) +def test_the_answerer_reaches_the_panel_record_so_the_vendor_warning_shows( + sub, tmp_path, monkeypatch, capsys): + from dokimi.cli import build_parser + from dokimi.novelty import cost + + monkeypatch.chdir(tmp_path) # analyze also writes its report under ./results + skill = tmp_path / "s" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text("---\nname: s\ndescription: d\n---\n\n# H\n\nDo not enable the legacy driver.\n", + encoding="utf-8") + rows = [{"statement": "The legacy driver must not be enabled.", "question": "Enable it?", + "answer": "No.", "kind": "contract", "tags": [], "span_ids": ["s0002"]}] + + def grader(prompt, system=None): + if "Extract the checkable knowledge" in prompt: + return json.dumps(rows), None + return json.dumps({"verdict": CC}), None + + def answer(prompt, system=None): + return "No.\nconfidence: 0.5", None + + monkeypatch.setattr(cost, "client", lambda model, provider, role, args: LLMClient( + model=model, transport=answer if role == "answerer" else grader)) + atoms, out = tmp_path / "atoms.json", tmp_path / "p.json" + run = [sub, "--out", str(out), "--no-paraphrase", "--no-reversal", "--model", "claude-sonnet-5", + "--judge", "anthropic/claude-opus-5", "--judge", "openai/gpt-6-luna"] + for argv in ([["atoms", "--out", str(atoms)], [*run, "--atoms", str(atoms)]] if sub == "probe" + else [run]): + args = build_parser().parse_args(["skill", "novelty", argv[0], "--artifact", f"skill:{skill}", + *argv[1:]]) + args.func(args) + assert json.loads(out.read_text())["judge_panel"]["same_vendor_as_answerer"] == ["claude-opus-5"] + assert "claude-opus-5" in capsys.readouterr().err + + +def test_an_errored_vote_under_a_failed_veto_is_no_datum(): + veto = [{"role": "veto", "passed": False}] + runs = [ + _run("a", BASELINE_ARM, {"j1": WO, "j2": ERRORED}), + _run("a", "full", {"j1": CC, "j2": WO}), + ] + runs[0].checks = veto + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + j2 = rec["per_judge_paired"]["j2"]["lenient"] + assert (j2["n_paired"], j2["n_unpaired_dropped"]) == (0, 1) + + +def test_probe_run_keeps_its_positional_fields_after_confidence(): + run = ProbeRun("a", "direct", "full", "q", "r", WO, 0.5, CC, "check", + [{"role": "veto", "passed": False}]) + assert (run.judge_verdict, run.verdict_source, run.judge_verdicts) == (CC, "check", None) + assert run.as_dict()["checks"] == [{"role": "veto", "passed": False}] + + +def test_a_stopped_panel_analysis_keeps_its_judges_and_rule(tmp_path): + from dokimi.novelty.analyze import AnalyzeConfig, analyze, load_artifact_text + from dokimi.novelty.client import SpendCap + from tests.test_novelty_analyze_cap_partial import _after, _priced, _skill + + path = _skill(tmp_path) + kind, name, description, frontmatter, body = load_artifact_text(f"skill:{path}") + config = AnalyzeConfig(artifact_ref=f"skill:{path}", artifact_kind=kind, name=name, + description=description, frontmatter=frontmatter, concurrency=1) + cap = SpendCap(limit_usd=1.0) + transport = _priced(_after("Rewrite each statement below")) + extractor, answerer, judge, second = ( + LLMClient(model=f"{role}-m", role="judge" if role == "second" else role, + transport=transport, cap=cap) + for role in ("extractor", "answerer", "judge", "second")) + result = analyze(body, config, extractor=extractor, answerer=answerer, judge=judge, + panel=(second,), aggregate="unanimous") + assert result.partial + assert result.artifact["judge_panel"] == {"judges": ["judge-m", "second-m"], "rule": "unanimous"} + + +def test_a_judge_that_grades_nothing_does_not_hide_a_split_between_the_others(): + runs = [ + _run("p1", BASELINE_ARM, {"A": WO, "B": WO, "C": ERRORED}), + _run("p1", "full", {"A": CC, "B": WO, "C": ERRORED}), + ] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["per_judge_paired"]["A"]["lenient"]["added"] == 1.0 + assert rec["per_judge_paired"]["B"]["lenient"]["added"] == 0.0 + assert rec["judge_sensitive"] is True + + +def test_a_partial_probe_keeps_no_panel_numbers(tmp_path): + import re + from pathlib import Path + + from dokimi.commands.novelty_budget import save_probe_run + + payload = { + "paired": {"lenient": {"added": 0.5, "n_paired": 2}}, + "judge_panel": { + "judges": ["a", "b"], + "rule": "unanimous", + "per_judge_paired": {"a": {"lenient": {"added": 0.5}}}, + "added_share_range": {"lenient": 0.0, "strict": None}, + "judge_sensitive": False, + }, + } + with pytest.raises(SystemExit) as stopped: + save_probe_run(payload, tmp_path / "probes.json", done=1, not_run=1, + stop=("spend_cap", "cap reached")) + partial = Path(re.search(r"partial result is in (\S+)", str(stopped.value)).group(1)) + saved = json.loads(partial.read_text(encoding="utf-8")) + assert saved["paired"] is None + panel = saved.get("judge_panel") or {} + assert panel.get("judges") == ["a", "b"] + assert panel.get("rule") == "unanimous" + assert not {"per_judge_paired", "added_share_range", "judge_sensitive"} & set(panel) + + +def test_a_panel_artifact_keeps_its_panel_in_the_scorecard_cell(): + from dokimi.subjects.novelty_track import novelty_cell + + panel = {"judges": ["first", "second", "third"], "rule": "unanimous"} + artifact = {"model": "answerer", "judge_model": "first", "judge_panel": panel, + "paired": {"lenient": {"added": 0.3}}, "atoms": {}, "spans": {}, "k0": {}, + "warnings": []} + assert novelty_cell(artifact).get("judge_panel") == panel + + +def test_one_judge_grading_a_pair_gives_no_range(): + runs = [ + _run("a", BASELINE_ARM, {"j1": WO, "j2": ERRORED}), + _run("a", "full", {"j1": CC, "j2": ERRORED}), + ] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["added_share_range"] == {"lenient": None, "strict": None} + assert rec["judge_sensitive"] is False diff --git a/dokimi/tests/test_novelty_provenance.py b/dokimi/tests/test_novelty_provenance.py index 975f466e9..39a2f1929 100644 --- a/dokimi/tests/test_novelty_provenance.py +++ b/dokimi/tests/test_novelty_provenance.py @@ -364,3 +364,16 @@ def test_judge2_frac_of_one_is_allowed(): check_judge2(judge2, answerer, 1.0, False) with pytest.raises(InputError, match="judge2-frac"): check_judge2(judge2, answerer, 1.5, False) + + +def test_regrade_of_a_panel_run_compares_with_the_first_judge_not_the_aggregate(): + # The first judge said correct, the panel's median is wrong; judge2 agrees with the first judge. + # A run whose first judge failed is left out, though the other judges voted. + measured = _measured(2) + for run in measured.runs: + run.verdict = run.judge_verdict = WRONG + run.judge_verdicts = {"first": RIGHT, "second": WRONG, "third": WRONG} + measured.runs[1].judge_verdicts = {"first": ERRORED, "second": WRONG, "third": WRONG} + judge2 = LLMClient(model="gpt-6", transport=lambda p, s: (json.dumps({"verdict": RIGHT}), None)) + out = regrade(measured, judge2, frac=1.0, concurrency=1) + assert (out["n_sampled"], out["n_compared"], out["lenient_agreement"]) == (1, 1, 1.0)