diff --git a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md new file mode 100644 index 000000000..6bd9cbd04 --- /dev/null +++ b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md @@ -0,0 +1,17 @@ + + +### Added + +- **`catalog novelty ROOT`: the corpus novelty sweep under one budget.** Runs + `skill novelty analyze` on every skill under ROOT, with `--max-usd` as the whole-run budget + (required), `--max-usd-per-skill`, `--jobs`, and JSONL output with one row per skill as it + finishes. `--resume` keeps the measured rows whose skill digest and settings are unchanged, and + an existing `--jsonl` is not overwritten without it. `--strata CSV` adds bootstrap 95% + intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. + Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish and keeps the + rows done; a second signal kills the subprocess calls in flight and starts no further call. Identical + skill directories (SKILL.md and bundled files) of one skill count once. + See `docs/skill-novelty.md`, Corpus sweep. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a9b377d5b..a36b7f5a9 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -354,6 +354,56 @@ "subcommand_required": null, "subcommands": {} }, + "novelty": { + "allow_abbrev": true, + "params": [ + {"key": "root", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": null}, + {"key": "--allow-self-judge", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--arms", "action": "_StoreAction", "choices": null, "const": null, "default": "no_artifact,full", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--bootstrap-resamples", "action": "_StoreAction", "choices": null, "const": null, "default": 2000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--budgets", "action": "_StoreAction", "choices": null, "const": null, "default": [0.75, 0.5], "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": "+", "position": null, "required": false, "type": "float"}, + {"key": "--cache-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--concurrency", "action": "_StoreAction", "choices": null, "const": null, "default": 5, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--context-framing", "action": "_StoreAction", "choices": ["'neutral'", "'reference'"], "const": null, "default": "reference", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--extract-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--extract-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--include-resources", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--jobs", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--jsonl", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-aggregate", "action": "_StoreAction", "choices": ["'majority'", "'median'", "'unanimous'"], "const": null, "default": "median", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge2-frac", "action": "_StoreAction", "choices": null, "const": null, "default": 0.15, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "float"}, + {"key": "--judge2-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--limit", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-atoms", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-file-words", "action": "_StoreAction", "choices": null, "const": null, "default": 4000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-resource-words", "action": "_StoreAction", "choices": null, "const": null, "default": 20000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-tokens", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli_types.positive_usd#3f7e5c8902a3"}, + {"key": "--max-usd-per-skill", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli_types.positive_usd#3f7e5c8902a3"}, + {"key": "--model", "action": "_StoreAction", "choices": null, "const": null, "default": "gpt-6.1-sol", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--no-paraphrase", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--no-reversal", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, + {"key": "--placebo-text", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": "openai", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--resource-roles", "action": "_StoreAction", "choices": ["'data'", "'eval'", "'other'", "'pattern'", "'reference'", "'script'", "'template'", "'top'", "'trigger'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": "+", "position": null, "required": false, "type": null}, + {"key": "--resume", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--seed", "action": "_StoreAction", "choices": null, "const": null, "default": 0, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "int"}, + {"key": "--strata", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--stratum-column", "action": "_StoreAction", "choices": null, "const": null, "default": "stratum", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--summary-only", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--summary-out", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--tier", "action": "_StoreAction", "choices": ["'standard'", "'study'", "'triage'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} + ], + "path": "dokimi catalog novelty", + "subcommand_required": null, + "subcommands": {} + }, "routing": { "allow_abbrev": true, "params": [ diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index e4f45a847..14a3f1df8 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1178,6 +1178,54 @@ A cell with no check measured at all is `NOT MEASURED`, not `PASS`. Failing `warn`-severity checks and the artifact's own `warnings` also surface in the scorecard's top-level `warnings`. +## Corpus sweep: `catalog novelty` + +`catalog novelty ROOT --out-dir DIR --max-usd USD` runs `skill novelty analyze` on every +`SKILL.md` under ROOT and writes one row per skill to `--jsonl` (default `DIR/sweep.jsonl`) as +each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets +`DIR//novelty.json` and `novelty.md`. The calls are paid: `--max-usd` is required. + +- **Budget.** `--max-usd` is the whole-run budget, shared by the skills analysed at once + (`--jobs`, threads). `--max-usd-per-skill` caps one skill inside it; a skill that hits either + cap is a `partial` row with no novelty file, and the sweep goes on. Once the whole-run budget + is spent, no further skill starts: it is not in the JSONL, and the run says how many were not run. +- **Resume.** `--resume` keeps the `ok` rows whose skill directory digest, measurement settings + and dokimi version are unchanged, rewrites the JSONL with them and with the rows of skills this + run does not select (a narrower `--limit`), and runs the other skills. Set + `DOKIMI_RESPONSE_CACHE` so the calls already paid replay at $0. Without `--resume`, an existing + `--jsonl` is refused rather than overwritten. +- **Settings.** The sweep takes the measurement flags it needs (`--model`, `--judge-model`, + `--tier`, `--max-atoms`, `--budgets`, `--arms`, `--include-resources`, ...); the rest keep their + `skill novelty analyze` defaults. `--judge2` is not part of the sweep: run it on one skill. +- **Summary.** The first line of the summary is the share the model already knew without the + skill, `k0.lenient`, and the share it did not know (`1 - known`): the new-to-the-model share. + Each interval is a 95% percentile bootstrap over the skills (2000 resamples, + `--bootstrap-resamples`). `--strata CSV` adds one line per stratum: the CSV has a `skill` + column (the skill's directory under ROOT, or its name) and a stratum column (`--stratum-column`, + default `stratum`); a skill the CSV does not name is `(unassigned)`. +- **Summary only.** `--summary-only --jsonl FILE` prints the summary of a finished or partial + sweep with no model call. `--summary-out FILE` also writes it as JSON. +- **Identical copies.** Directories under ROOT whose files (`SKILL.md` and everything bundled with it) are + byte-identical and name the same skill (the frontmatter `name`, or the directory name when it has none) + are one skill: the shallowest path is kept, the other copies get no row and no call, and stderr names + them (not an error). Two directories with the same `SKILL.md` and different bundled files are two skills. +- **Interrupt.** Ctrl-C or SIGTERM stops the sweep without new calls: the skills not yet started + are cancelled, the skills in flight finish and keep their rows, and the run exits nonzero (130 + for SIGINT, 143 for SIGTERM). A SIGTERM therefore waits for the calls in flight. The rows already + written stay in the JSONL, so `--resume` continues from them. A second signal kills the subprocess calls in + flight (their paid work is lost, so set `DOKIMI_RESPONSE_CACHE` to replay what was done) and stops + at once. An API call in flight cannot be killed: its row is not written, and the process waits for it to end + before it exits, so its paid call is lost unless `DOKIMI_RESPONSE_CACHE` is set. No call after it starts. + A skill that is measured again (`--resume`) first loses its earlier `novelty.json` and `novelty.md`, so a + `partial` or `error` row never sits beside a file from an older run. `--summary-out` may not be the + `--jsonl` file. + +```bash +dokimi catalog novelty skills/ --out-dir out/sweep --max-usd 20 --max-usd-per-skill 1 \ + --jobs 4 --model --judge-model --strata strata.csv +dokimi catalog novelty skills/ --out-dir out/sweep --summary-only +``` + ## Cost, and how it is kept down The measurement is LLM-call bound, and the probe term is predictable: **every probe costs an diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 515b7fc55..b39d90a1d 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -131,6 +131,7 @@ class Tombstone: Leaf(("skill", "check"), FREE), Leaf(("skill", "routing"), PAID), Leaf(("catalog", "routing"), PAID), + Leaf(("catalog", "novelty"), PAID), Leaf(("repo", "validate"), FREE), Leaf(("repo", "scan"), FREE), Leaf(("repo", "inspect"), FREE), diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index d122702cd..3cc643881 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -506,6 +506,10 @@ def register(sub, positive_int, eval_sub) -> None: add_catalog_routing(catalog_sub, positive_int) # catalog routing: all-vs-all (#94 M3) + from dokimi.commands import catalog_novelty + + catalog_novelty.register(catalog_sub, positive_int) # catalog novelty: the corpus sweep + kinds = ["product", "subject"] list_p = catalog_sub.add_parser( "list", help="List products (--kind product) or subject descriptors (--kind subject)" diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py new file mode 100644 index 000000000..65ab88634 --- /dev/null +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -0,0 +1,598 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``catalog novelty ROOT``: the corpus novelty sweep, one ``skill novelty analyze`` per skill. + +Every ``SKILL.md`` under ROOT is analysed with the flags of ``skill novelty analyze``, and the +model calls are paid. ``--max-usd`` is one budget for the whole sweep and ``--max-usd-per-skill`` +a cap on each skill; ``--jobs`` runs skills in parallel threads that share the budget. Each skill +writes one row to ``--jsonl`` as it finishes, so a killed sweep keeps its rows, and ``--resume`` +runs only the skills without a measured row (same directory digest, same settings). +``--summary-only`` prints the summary of a JSONL file without any model call. Directories with the +same contents (``SKILL.md`` and bundled files) that name the same skill count as one (the shallowest path +is kept, the others are named on stderr). Ctrl-C or SIGTERM +cancels the skills not yet started, lets the running ones finish, keeps the rows done and exits nonzero +(a second signal kills the subprocess calls in flight and stops at once; an API call in flight cannot be killed but no call after it starts, and its row is not written). + +A skill the budget stops mid-analysis is a ``partial`` row and leaves no novelty file. Set +``DOKIMI_RESPONSE_CACHE`` so a resumed run replays the calls already paid for at $0. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import math +import re +import signal +import sys +import threading +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Optional + +from dokimi import __version__ +from dokimi.atomic_write import write_text_atomic +from dokimi.commands.novelty_run import stops_at_cap +from dokimi.novelty.sweep import read_rows, resumable, summarize, summary_lines +from dokimi.utils import InputError + +if TYPE_CHECKING: + from dokimi.novelty.client import SpendCap + +#: The JSONL file a sweep writes unless ``--jsonl`` names another. +DEFAULT_JSONL = "sweep.jsonl" + + +class _Stop: + """A stop request: the signal (SIGINT or SIGTERM) that asked the sweep to stop, once it has.""" + + def __init__(self) -> None: + self.signum: Optional[int] = None + self.hard = threading.Event() # set by a second signal: no skill in flight starts another call + + @property + def requested(self) -> bool: + return self.signum is not None + + @property + def exit_code(self) -> int: + return 128 + (self.signum or signal.SIGINT) # the shell's code for a process a signal stopped + + +@contextmanager +def _interruptible(stop: _Stop): + """While the sweep runs, the first SIGINT or SIGTERM only sets *stop*: the sweep cancels the skills + not started, and the skills in flight finish. A second signal kills the subprocess calls in flight + (``process.kill_children``), sets ``stop.hard`` so no skill in flight starts another call, and raises + KeyboardInterrupt. The handlers are restored after. + Only the main thread can install them.""" + if threading.current_thread() is not threading.main_thread(): + yield + return + + def on_signal(signum: int, frame: object) -> None: + if stop.requested: # a second signal: stop at once, the calls in flight too + from dokimi.process import kill_children + + stop.hard.set() # the API calls in flight end on their own, but the next one never starts + kill_children() + raise KeyboardInterrupt + stop.signum = signum + + previous = {signum: signal.signal(signum, on_signal) for signum in (signal.SIGINT, signal.SIGTERM)} + try: + yield + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler if handler is not None else signal.SIG_DFL) + + +@dataclass(frozen=True) +class _Item: + """One skill of the root: its key (its directory relative to ROOT), SKILL.md, digest and load error.""" + + key: str + name: str + skill_md: Path + sha256: Optional[str] + error: Optional[str] = None # a SKILL.md that does not load: its row is an error, and no call is made + + @property + def directory(self) -> Path: + return self.skill_md.parent + + +def _skill_digest(skill_md: Path) -> str: + """The digest a resumed run compares: the directory tree, and the entry point's loaded content + (a symlinked SKILL.md changes its target's text without changing the link the tree digest reads).""" + from dokimi.utils import tree_digest + + entry = hashlib.sha256(skill_md.read_bytes()).hexdigest() + return hashlib.sha256(f"{tree_digest(skill_md.parent, tracked={})}:{entry}".encode("utf-8")).hexdigest() + + +def _items(root: Path, limit: Optional[int]) -> list[_Item]: + """The skills under *root* in path order, the first *limit* of them. + + A bundle identical to a shallower one (same name, same files) is left out and named on stderr; + two bundles with the same SKILL.md and different bundled files are two skills. + """ + from dokimi.skills.loader import TEMPLATE_DIR_NAMES, load_skill, walk_skill_files + + candidates: list[_Item] = [] + for skill_md in walk_skill_files(root): + directory = skill_md.parent + if not skill_md.is_file() or TEMPLATE_DIR_NAMES.intersection(directory.relative_to(root).parts): + continue + error: Optional[str] = None + try: + name = load_skill(skill_md).name + except (OSError, ValueError, RecursionError) as exc: + name, error = directory.name, str(exc) + try: + sha: Optional[str] = _skill_digest(skill_md) + except OSError: + sha = None + candidates.append(_Item(key=directory.relative_to(root).as_posix(), name=name, skill_md=skill_md, + sha256=sha, error=error)) + keeper: dict[tuple[str, str], _Item] = {} + for item in sorted(candidates, key=lambda i: (len(i.skill_md.parts), i.skill_md.as_posix())): + if item.sha256 is not None and item.error is None: + keeper.setdefault((item.name, item.sha256), item) + items: list[_Item] = [] + for item in candidates: + kept = keeper.get((item.name, item.sha256 or "")) if item.sha256 is not None and item.error is None else item + if kept is not None and kept is not item: + print(f"# left out {item.skill_md}: identical to {kept.skill_md}", file=sys.stderr) + continue + items.append(item) + if limit is not None and len(items) == limit: + break + return items + + +#: The most characters of a skill's key kept in its output directory name (the hash of the whole key follows). +DIRNAME_CHARS = 80 + + +def _dirname(key: str) -> str: + """A directory name for one skill's outputs: readable, and unique even when two keys read alike.""" + readable = re.sub(r"[^A-Za-z0-9._-]+", "_", key).strip("_")[-DIRNAME_CHARS:].strip("_") or "root" + return f"{readable}-{hashlib.sha256(key.encode('utf-8')).hexdigest()[:8]}" + + +def _fingerprint(args: argparse.Namespace, placebo_text: Optional[str]) -> dict[str, Any]: + """The settings a measured row was made under; a resumed run keeps a row only when these are unchanged.""" + from dokimi.commands.novelty import _settings + from dokimi.novelty import cost + + fingerprint = { + "roles": {role: list(pair) for role, pair in cost.role_models(args).items()}, + "settings": dict(_settings(args).values), + "arms": args.arms, + "budgets": list(args.budgets), + "max_atoms": args.max_atoms, + "seed": args.seed, + "max_tokens": args.max_tokens, + "context_framing": args.context_framing, + "include_resources": args.include_resources, + "resource_roles": args.resource_roles, + "no_paraphrase": args.no_paraphrase, + "no_reversal": args.no_reversal, + "allow_self_judge": args.allow_self_judge, + "max_file_words": args.max_file_words, + "max_resource_words": args.max_resource_words, + "placebo_text": hashlib.sha256(placebo_text.encode("utf-8")).hexdigest() if placebo_text else None, + } + return json.loads(json.dumps(fingerprint, default=str)) + + +def _analyze_skill(args: argparse.Namespace, placebo_text: Optional[str], cache_dir: Optional[Path]): + """One skill through the analyze pipeline, as ``cmd_novelty_analyze`` runs it, without the writes. + + Returns the :class:`~dokimi.novelty.analyze.AnalyzeResult`; the caller decides what to save. + """ + from dokimi.commands.novelty import _arms, _bundle_from_args, _settings, stamp + from dokimi.novelty import cost + from dokimi.novelty.analyze import AnalyzeConfig, analyze + + bundle = _bundle_from_args(args) + settings = _settings(args) + extractor, answerer, judge = cost.clients(args) + config = AnalyzeConfig( + artifact_ref=args.artifact, artifact_kind=bundle.kind, name=bundle.name, + description=bundle.description, frontmatter=bundle.frontmatter, arms=_arms(args.arms), + budgets=tuple(args.budgets), concurrency=args.concurrency, + allow_self_judge=args.allow_self_judge, placebo_text=placebo_text, + context_framing=args.context_framing, cache_dir=cache_dir, seed=args.seed, + include_resources=args.include_resources, + resource_roles=tuple(args.resource_roles) if args.resource_roles else None, + **settings.config_fields(), + ) + result = analyze(bundle.text, config, extractor=extractor, answerer=answerer, judge=judge, + artifact_bundle=bundle) + if not result.partial: + stamp(result.artifact, args, result.measure_result, answerer=answerer, judge=judge) + return result + + +class _Sink: + """The JSONL file: one flushed line per row. A resumed run rewrites it with the kept rows first.""" + + def __init__(self, path: Path, kept: list[dict], resume: bool) -> None: + self.path = path + try: + if resume: + tmp = path.with_name(path.name + ".part") + with tmp.open("w", encoding="utf-8") as fh: + for row in kept: + fh.write(json.dumps(row) + "\n") + tmp.replace(path) + self._fh = path.open("a" if resume else "w", encoding="utf-8") + except OSError as exc: + raise InputError(f"cannot write --jsonl {path}: {exc}") from exc + + def write(self, row: dict) -> None: + self._fh.write(json.dumps(row) + "\n") + self._fh.flush() + + def close(self) -> None: + self._fh.close() + + +#: How often the sweep's main thread looks at the stop request while it waits for a skill. +POLL_SECONDS = 0.2 + + +def _run_sweep(items: list[_Item], *, jobs: int, review: Callable[[_Item], Optional[dict]], + write: Callable[[dict], None], out_of_budget: Callable[[], bool], + stop_requested: Callable[[], bool]) -> None: + """Review the items with up to *jobs* threads, handing each row to *write* as it finishes. + + ``review`` returns None for a skill it did not run (the budget was spent before it started); + no new skill is submitted once ``out_of_budget()`` holds. A stop request (``stop_requested()``) + does the same, and cancels the skills queued but not started, so they make no call. The skills + in flight finish and their rows are written. A worker's unexpected failure cancels the queue the + same way and propagates after the rows finished so far are written. A KeyboardInterrupt (the second + signal) waits for nothing: it writes the rows already done and propagates. + """ + pending = iter(items) + running: set = set() + pool = ThreadPoolExecutor(max_workers=jobs) + + def guarded(item: _Item) -> Optional[dict]: + # a stop that landed while this skill was queued: it makes no call, even if a worker freed up first + return None if stop_requested() else review(item) + + def fill() -> None: + while len(running) < jobs * 2 and not out_of_budget() and not stop_requested(): + item = next(pending, None) + if item is None: + return + running.add(pool.submit(guarded, item)) + + try: + fill() + while running and not stop_requested(): + finished, _ = wait(running, timeout=POLL_SECONDS, return_when=FIRST_COMPLETED) + for future in finished: + running.discard(future) # before the write: a write that raises is not written again + row = future.result() + if row is not None: + write(row) + fill() + except BaseException as exc: + _unwind(exc, pool, running, write) + raise + _settle(pool, running, write) + + +def _unwind(exc: BaseException, pool: ThreadPoolExecutor, running: set, write: Callable[[dict], None]) -> None: + """What the sweep does on its way out of *exc*. A KeyboardInterrupt (the second signal) waits for nothing: + the queue is cancelled and only the rows already done are written. Any other failure settles.""" + if not isinstance(exc, KeyboardInterrupt): + _settle(pool, running, write) + return + pool.shutdown(wait=False, cancel_futures=True) + for future in list(running): + if future.done(): + _keep(future, write) + running.clear() + + +def _settle(pool: ThreadPoolExecutor, running: set, write: Callable[[dict], None]) -> None: + """Cancel the queued skills, keep the rows already done, then wait for the ones in flight and keep + theirs. The done rows are written before the wait, so an interrupt during the wait keeps them.""" + pool.shutdown(wait=False, cancel_futures=True) + pending = set() + for future in running: + if future.done(): + _keep(future, write) + else: + pending.add(future) + while pending: # each row is written as its skill finishes, so an interrupt keeps the ones done + finished, pending = wait(pending, return_when=FIRST_COMPLETED) + for future in finished: + _keep(future, write) + running.clear() + + +def _keep(future, write: Callable[[dict], None]) -> None: + if not future.cancelled() and future.exception() is None and future.result() is not None: + write(future.result()) + + +def _refuse_to_run(item: _Item, novelty_json: Path) -> Optional[tuple[str, str]]: + """The (reason, message) of an error row when the skill is not analysed at all, else None. Runs first: + the files an earlier run left for this skill describe content this attempt may not measure, so they go.""" + try: + novelty_json.unlink(missing_ok=True) + novelty_json.with_suffix(".md").unlink(missing_ok=True) + except OSError as exc: + return "error", f"could not remove its earlier novelty file: {exc}" + return None if item.error is None else ("not_loaded", item.error) + + +def _save_novelty(artifact: dict, novelty_json: Path) -> Optional[str]: + """Write the novelty file and its report; the reason it could not, else None. The paid analysis is + done by now, so a failed write is this skill's error, not the sweep's.""" + from dokimi.artifacts import save_artifact + from dokimi.novelty.report import render_novelty_markdown + + try: + save_artifact("novelty", artifact, novelty_json) + novelty_json.with_suffix(".md").write_text(render_novelty_markdown(artifact), encoding="utf-8") + except (OSError, ValueError) as exc: + return f"could not save its novelty file: {exc}" + return None + + +def _review_one(item: _Item, *, args: argparse.Namespace, run_cap: "SpendCap", per_skill: float, + hard_stop: Optional[threading.Event] = None, + placebo_text: Optional[str], cache_dir: Optional[Path], out_dir: Path, + fingerprint: dict) -> Optional[dict]: + """The row of one skill: analyse it under its own cap and save its novelty file, or say why it has none.""" + from dokimi.novelty.atoms import ExtractionError + from dokimi.novelty.client import NestedSpendCap, SpendCapReached + from dokimi.novelty.measure import SelfJudgeError + from dokimi.novelty.sweep import has_measure, make_row, measured_fields + + if run_cap.spent_usd >= run_cap.limit_usd: + return None + cap = NestedSpendCap(limit_usd=per_skill, parent=run_cap, hard_stop=hard_stop) + + def row(status: str, **fields: Any) -> dict: + return make_row(key=item.key, name=item.name, path=str(item.skill_md), + skill_sha256=item.sha256, status=status, settings=fingerprint, + version=__version__, usd=cap.spent_usd, calls=cap.calls, **fields) + + novelty_json = out_dir / _dirname(item.key) / "novelty.json" + if (refusal := _refuse_to_run(item, novelty_json)) is not None: + return row("error", reason=refusal[0], message=refusal[1]) + skill_args = copy.copy(args) + skill_args.artifact = f"skill:{item.directory}" + skill_args._spend_cap = cap # cost.spend_cap() hands this cap to every role client of the skill + try: + result = _analyze_skill(skill_args, placebo_text, cache_dir) + except SpendCapReached as stop: # a stop outside analyze's own partial record + return row("partial", reason=stop.reason, message=str(stop)) + except (ExtractionError, SelfJudgeError, ValueError, InputError, OSError) as exc: + return row("error", reason="error", message=str(exc)) + except Exception as exc: # noqa: BLE001 - one skill's failure is its row's error; the sweep goes on + return row("error", reason="unexpected", message=f"{type(exc).__name__}: {exc}") + if result.partial: + return row("partial", reason=result.partial["reason"], message=result.partial["message"]) + measured = measured_fields(result.artifact) + # k0 reads 0.0 when no baseline probe answered, so a zero pair count is the signal, not the share. + if not has_measure(measured): + return row("error", reason="no_pairs", message="no probe was paired, so there is no share") + if (failure := _save_novelty(result.artifact, novelty_json)) is not None: + return row("error", reason="error", message=failure) + return row("ok", measured=measured, novelty_json=str(novelty_json)) + + +def _emit_summary(summary: dict, args: argparse.Namespace) -> None: + for line in summary_lines(summary): + print(line) + if args.summary_out: + write_text_atomic(args.summary_out, json.dumps(summary, indent=2) + "\n") + print(f"summary written to {args.summary_out}") + + +def cmd_catalog_novelty(args: argparse.Namespace) -> None: + from dokimi.commands.inputs import placebo_text_for_arms_or_exit + from dokimi.commands.novelty import _arms + from dokimi.commands.novelty_run import usable_cache_dir + from dokimi.novelty.client import SpendCap + from dokimi.novelty.sweep import load_strata + + root = Path(args.root) + strata = load_strata(Path(args.strata), args.stratum_column) if args.strata else None + out_dir = Path(args.out_dir) + jsonl = Path(args.jsonl) if args.jsonl else out_dir / DEFAULT_JSONL + _refuse_unsafe_summary_out(args, jsonl) + if args.summary_only: + _summary_only(jsonl, strata, args) + return + if not root.is_dir(): + raise InputError(f"ROOT is not a directory: {root}") + _refuse_unsafe_run(args, jsonl) + placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) + cache_dir = usable_cache_dir(args.cache_dir) + items = _items(root, args.limit) + if not items: + raise InputError(f"no SKILL.md under {root}") + fingerprint = _fingerprint(args, placebo_text) + kept, todo, others = _plan(items, jsonl, args, fingerprint) + print(f"# catalog novelty: {len(items)} skills under {root}; {len(kept)} measured and kept, " + f"{len(todo)} to run", file=sys.stderr) + run_cap = SpendCap(limit_usd=args.max_usd) + per = f", ${args.max_usd_per_skill:g} per skill" if args.max_usd_per_skill else "" + print(f"# budget ${args.max_usd:g} for the whole run{per}", file=sys.stderr) + out_dir.mkdir(parents=True, exist_ok=True) + new_rows = _sweep_rows(todo, args=args, run_cap=run_cap, kept=kept, others=others, total=len(items), + jsonl=jsonl, + out_dir=out_dir, placebo_text=placebo_text, cache_dir=cache_dir, + fingerprint=fingerprint) + _emit_summary(summarize(kept + new_rows, strata, resamples=args.bootstrap_resamples), args) + + +def _refuse_unsafe_summary_out(args: argparse.Namespace, jsonl: Path) -> None: + """Refuse a --summary-out that is the --jsonl file (the summary would replace the paid rows) or + whose directory cannot be made, before any paid work.""" + if not args.summary_out: + return + summary_out = Path(args.summary_out) + if summary_out.resolve() == jsonl.resolve(): + raise InputError(f"--summary-out {summary_out} is the --jsonl file: the summary would replace the rows") + try: + summary_out.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise InputError(f"cannot write --summary-out {summary_out}: {exc}") from exc + + +def _summary_only(jsonl: Path, strata, args: argparse.Namespace) -> None: + rows = read_rows(jsonl) + if not rows: + raise InputError(f"--jsonl {jsonl} has no rows to summarise") + _emit_summary(summarize(rows, strata, resamples=args.bootstrap_resamples), args) + + +def _refuse_unsafe_run(args: argparse.Namespace, jsonl: Path) -> None: + """Refuse a paid run that would not be a clean one: a judge2 it cannot run, no budget, + an overwrite of a sweep in progress, or a model flag that does not resolve.""" + from dokimi.novelty import cost + + if args.judge2: + raise InputError("--judge2 is not part of catalog novelty: run it on one skill with skill novelty analyze") + if args.max_usd is None: + raise InputError("catalog novelty makes paid calls: pass --max-usd, the whole-run budget " + "(--max-usd-per-skill caps one skill inside it)") + if jsonl.exists() and not args.resume: + raise InputError(f"--jsonl {jsonl} exists: pass --resume to continue it, or remove it to start over") + try: + cost.role_models(args) + except ValueError as exc: + raise InputError(str(exc)) from exc + + +def _plan(items: list[_Item], jsonl: Path, args: argparse.Namespace, + fingerprint: dict[str, Any]) -> tuple[list[dict], list[_Item], list[dict]]: + """The measured rows a resumed run keeps, the skills it still has to run, and the prior rows of + skills this run does not select (a narrower --limit, a moved skill): those stay in the file as they are.""" + selected = {item.key: item.sha256 for item in items} + prior = read_rows(jsonl) if args.resume else [] + kept = resumable(prior, selected, fingerprint, __version__) + done = {row["key"] for row in kept} + others = [row for row in prior if row["key"] not in selected] + return kept, [item for item in items if item.key not in done], others + + +def _sweep_rows(todo: list[_Item], *, args: argparse.Namespace, run_cap, kept: list[dict], + others: list[dict], total: int, jsonl: Path, out_dir: Path, placebo_text: Optional[str], + cache_dir: Optional[Path], fingerprint: dict[str, Any]) -> list[dict]: + """Run the skills still to do, writing each row as it finishes; returns the rows this run made.""" + sink = _Sink(jsonl, others + kept, resume=args.resume) + stop = _Stop() + review = _bound(args, run_cap=run_cap, hard_stop=stop.hard, placebo_text=placebo_text, cache_dir=cache_dir, + out_dir=out_dir, fingerprint=fingerprint) + new_rows: list[dict] = [] + + def write(row: dict) -> None: + sink.write(row) + new_rows.append(row) + detail = f" new {row['new_lenient']:.2f}" if row["status"] == "ok" else f" {row['reason']}" + print(f"[{len(kept) + len(new_rows)}/{total}] {row['key']}: {row['status']}{detail} " + f"${row['usd']:.3f}", file=sys.stderr) + + def out_of_budget() -> bool: + return run_cap.spent_usd >= run_cap.limit_usd + + try: + with _interruptible(stop): + _run_sweep(todo, jobs=args.jobs, review=review, write=write, out_of_budget=out_of_budget, + stop_requested=lambda: stop.requested) + except KeyboardInterrupt: # a second signal: stop at once, the rows written so far are kept + stop.signum = stop.signum or signal.SIGINT + finally: + sink.close() + not_run = len(todo) - len(new_rows) + if not_run: + print(f"# {not_run} skill(s) not run: the budget was spent or the run was stopped. Rerun with " + "--resume (and a higher --max-usd if the budget was spent); set DOKIMI_RESPONSE_CACHE so the " + "calls already paid replay at $0", file=sys.stderr) + if stop.requested: + print(f"# stopped by signal {stop.signum}: queued skills were cancelled, {len(new_rows)} row(s) " + f"written to {jsonl}; rerun with --resume to continue", file=sys.stderr) + raise SystemExit(stop.exit_code) + return new_rows + + +def _bound(args: argparse.Namespace, **shared: Any) -> Callable[[_Item], Optional[dict]]: + """The per-skill review with this run's shared pieces bound: the arguments, the budget and the output dir.""" + per_skill = args.max_usd_per_skill or math.inf # unset: no cap of its own, only the run's + + def review(item: _Item) -> Optional[dict]: + return _review_one(item, args=args, per_skill=per_skill, **shared) + + return review + + +def register(catalog_sub, positive_int) -> None: + """Add ``catalog novelty``: the corpus sweep, a catalog verb like ``catalog routing``.""" + from dokimi.commands.novelty import _add_bundle_args, _add_model_args + from dokimi.cli_types import positive_usd + from dokimi.novelty.sweep import RESAMPLES + from dokimi.novelty.tiers import TIERS + + p = catalog_sub.add_parser( + "novelty", + help="Corpus novelty sweep: analyze every skill under a catalog root under a budget, resumable, " + "with bootstrap CIs per stratum [paid]", + ) + p.add_argument("root", help="Catalog root: every SKILL.md under it is one skill. Directories with identical " + "contents (SKILL.md and bundled files) that name the same skill count once: the " + "shallowest path is kept, the other copies are named on stderr (not an error, no row)") + p.add_argument("--out-dir", required=True, dest="out_dir", + help="Where each skill's novelty.json and report go (OUT_DIR//)") + p.add_argument("--jsonl", default=None, + help=f"One row per skill, written as each finishes (default: OUT_DIR/{DEFAULT_JSONL}). " + "Ctrl-C or SIGTERM cancels the skills not yet started (no new call), lets the " + "running ones finish, keeps the rows done and exits nonzero (130, 143); " + "a second signal kills the subprocess calls in flight too") + p.add_argument("--jobs", type=positive_int, default=1, + help="Skills analysed in parallel threads, sharing --max-usd (default 1)") + p.add_argument("--resume", action="store_true", + help="Keep the measured rows in --jsonl and run only the other skills") + p.add_argument("--max-usd-per-skill", type=positive_usd, default=None, dest="max_usd_per_skill", + help="Stop one skill once its calls cost this much (USD); the rest of the sweep goes on") + p.add_argument("--limit", type=positive_int, default=None, + help="Only the first N skills in path order (a pilot)") + p.add_argument("--strata", default=None, + help="CSV with a `skill` column (the skill's directory under ROOT, or its name) and a " + "stratum column; the summary gets bootstrap CIs per stratum") + p.add_argument("--stratum-column", default="stratum", dest="stratum_column", + help="The stratum column of --strata (default: stratum)") + p.add_argument("--summary-only", action="store_true", dest="summary_only", + help="Print the summary of --jsonl without running anything") + p.add_argument("--summary-out", default=None, dest="summary_out", + help="Also write the summary as JSON here") + p.add_argument("--bootstrap-resamples", type=positive_int, default=RESAMPLES, + dest="bootstrap_resamples", help=f"Bootstrap resamples per interval (default {RESAMPLES})") + p.add_argument("--tier", default=None, choices=sorted(TIERS), + help="Named cost tier for every skill (see skill novelty analyze --tier)") + p.add_argument("--budgets", type=float, nargs="+", default=[0.75, 0.5], + help="Target token fractions for shortening candidates") + p.add_argument("--max-atoms", type=positive_int, default=None, dest="max_atoms", + help="Atom cap per skill (see skill novelty analyze)") + p.add_argument("--seed", type=int, default=0, help="Seed for the controls and the selection, per skill") + p.add_argument("--cache-dir", default=None, dest="cache_dir", + help="Disk cache for retention adjudications") + _add_model_args(p, positive_int) # --model, --judge-model, ... and --max-usd: the whole-run budget + _add_bundle_args(p, positive_int) + p.set_defaults(func=stops_at_cap(cmd_catalog_novelty)) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 335429d75..468d95cbe 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -129,6 +129,32 @@ def record(self, usage: UsageRecord | None, model: str) -> None: self.spent_usd += usage.cost_usd +@dataclass +class NestedSpendCap(SpendCap): + """A cap for one unit of a larger run (one skill of a sweep), that also charges ``parent``. + + Every call is checked against both limits and recorded on both, so a sweep's whole-run + budget holds across the skills running at once. The parent is charged first: a call the + parent refuses or cannot price never reaches this cap's own count. + """ + + parent: SpendCap | None = None + # Set to stop this unit at once: the call in flight ends on its own, and no later call starts. + hard_stop: threading.Event | None = field(default=None, repr=False, compare=False) + + def before_call(self, model: str) -> None: + if self.hard_stop is not None and self.hard_stop.is_set(): + raise SpendCapReached(f"stopped by a second signal; not calling {model}", reason="stopped") + super().before_call(model) + if self.parent is not None: + self.parent.before_call(model) + + def record(self, usage: UsageRecord | None, model: str) -> None: + if self.parent is not None: + self.parent.record(usage, model) + super().record(usage, model) + + @dataclass class LLMClient: """One model in one role, with its usage recorded. diff --git a/dokimi/dokimi/novelty/sweep.py b/dokimi/dokimi/novelty/sweep.py new file mode 100644 index 000000000..2000f08d2 --- /dev/null +++ b/dokimi/dokimi/novelty/sweep.py @@ -0,0 +1,246 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The corpus novelty sweep's bookkeeping: one JSONL row per skill, resume, strata and the summary. + +No model call and no CLI here: :mod:`dokimi.commands.catalog_novelty` runs the paid +``analyze`` per skill and writes the rows this module builds. The per-skill measure is the +share the model did not already know, ``1 - k0.lenient`` (the lenient K0 of the novelty +artifact, the ``new`` column of the S2/S3 summaries); a skill's CI is the bootstrap of that +share over the skills of a stratum (:func:`dokimi.eval.stats.bootstrap_ci`, the one resampler). +""" + +from __future__ import annotations + +import csv +import json +import statistics +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any, Optional + +from dokimi.utils import InputError + +#: The statuses a row can have. Only ``ok`` rows are measured and kept by a resumed run. +STATUSES = ("ok", "partial", "error") +#: The stratum of a skill the strata file does not name. +UNASSIGNED = "(unassigned)" +#: The stratum name when no strata file is given. +ALL = "all" +#: Bootstrap resamples of a stratum interval; fixed so two summaries of one JSONL agree. +RESAMPLES = 2000 + + +def new_share(known: Optional[float]) -> Optional[float]: + """The share of the skill's content the model did not already know: ``1 - known``.""" + return None if known is None else round(1.0 - known, 6) + + +def measured_fields(artifact: dict) -> dict[str, Any]: + """The per-skill numbers of a finished novelty artifact, read from its ``k0`` and ``paired`` blocks.""" + k0 = artifact.get("k0") or {} + lenient = (artifact.get("paired") or {}).get("lenient") or {} + known = k0.get("lenient") + return { + "tokens": (artifact.get("artifact") or {}).get("tokens"), + "known_lenient": known, + "new_lenient": new_share(known), + "new_strict": new_share(k0.get("strict")), + "n_paired": lenient.get("n_paired"), + } + + +def has_measure(measured: Mapping[str, Any]) -> bool: + """Whether *measured* has a share and a paired probe behind it (k0 reads 0.0 when no baseline probe answered).""" + return measured["new_lenient"] is not None and bool(measured["n_paired"]) + + +def make_row(*, key: str, name: str, path: str, skill_sha256: Optional[str], status: str, + settings: dict, version: str, usd: float = 0.0, calls: int = 0, + reason: Optional[str] = None, message: Optional[str] = None, + measured: Optional[dict] = None, novelty_json: Optional[str] = None) -> dict: + """One skill's row: its identity and digest, what the run paid, and its measure when it has one.""" + if status not in STATUSES: + raise ValueError(f"unknown sweep status {status!r}") + row: dict[str, Any] = { + "key": key, "name": name, "path": path, "skill_sha256": skill_sha256, "status": status, + "reason": reason, "message": message, "usd": round(usd, 6), "llm_calls": calls, + "novelty_json": novelty_json, "settings": settings, "dokimi_version": version, + "tokens": None, "known_lenient": None, "new_lenient": None, "new_strict": None, + "n_paired": None, + } + if measured is not None: + row.update(measured) + return row + + +def load_strata(path: Path, column: str) -> dict[str, str]: + """The skill -> stratum map of a CSV with a ``skill`` column and a *column* column. + + ``skill`` is the skill's key under the catalog root (its directory, relative, with ``/``) or + its name; a skill is matched by key first, then by name. A skill named twice with two strata is an error. + """ + if column == "skill": + raise InputError("--stratum-column must not be `skill`: that column names the skill") + try: + with path.open(newline="", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + fields = reader.fieldnames or [] + if "skill" not in fields or column not in fields: + raise InputError(f"--strata {path} needs a `skill` column and a `{column}` column; " + f"found {', '.join(fields) or 'none'}") + out: dict[str, str] = {} + for lineno, rec in enumerate(reader, start=2): + skill = (rec.get("skill") or "").strip() + stratum = (rec.get(column) or "").strip() + if not skill or not stratum: + raise InputError(f"--strata {path} line {lineno}: `skill` and `{column}` must both be set") + if out.get(skill, stratum) != stratum: + raise InputError(f"--strata {path} line {lineno}: {skill!r} is in two strata " + f"({out[skill]!r} and {stratum!r})") + out[skill] = stratum + except (OSError, UnicodeDecodeError, csv.Error) as exc: + raise InputError(f"cannot read --strata {path}: {exc}") from exc + return out + + +def stratum_of(row: Mapping[str, Any], strata: Optional[Mapping[str, str]]) -> str: + """The stratum of a row: ``all`` without a strata map, else the mapped stratum (by key, then name).""" + if strata is None: + return ALL + return strata.get(row["key"]) or strata.get(row["name"]) or UNASSIGNED + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _row_problem(row: Any) -> Optional[str]: + """What a row lacks of the fields the summary and a resume read, or None when it has them.""" + if not isinstance(row, dict): + return "not an object" + if row.get("status") not in STATUSES: + return "unknown status" + if not isinstance(row.get("key"), str) or not isinstance(row.get("name"), str): + return "`key` and `name` must be strings" + if not _is_number(row.get("usd")): + return "`usd` must be a number" + if row["status"] == "ok" and not (row.get("new_lenient") is None or _is_number(row["new_lenient"])): + return "`new_lenient` must be a number or null" + for field in ("new_strict", "tokens"): + if row.get(field) is not None and not _is_number(row[field]): + return f"`{field}` must be a number or null" + return None + + +def read_rows(path: Path) -> list[dict]: + """The rows of a sweep JSONL file. A last line a killed run left half-written is dropped.""" + rows: list[dict] = [] + try: + with path.open("rb") as fh: + lines = fh.readlines() + except FileNotFoundError: + return rows + except OSError as exc: + raise InputError(f"cannot read --jsonl {path}: {exc}") from exc + for lineno, raw in enumerate(lines, 1): + if not raw.strip(): + continue + try: + row = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + if lineno == len(lines) and not raw.endswith(b"\n"): + break + raise InputError(f"--jsonl {path} line {lineno} is not a sweep row: {exc}") from exc + problem = _row_problem(row) + if problem: + raise InputError(f"--jsonl {path} line {lineno} is not a sweep row ({problem}); " + "fix or remove it before --resume") + rows.append(row) + return rows + + +def resumable(prior: Iterable[dict], selected: Mapping[str, Optional[str]], settings: dict, + version: str) -> list[dict]: + """The prior rows a resumed run keeps: measured, for a skill still selected with the same + directory digest, under the same settings and dokimi version. Every other row is run again.""" + kept = [] + for row in prior: + if (row["status"] == "ok" and row["key"] in selected and selected[row["key"]] is not None + and row.get("skill_sha256") == selected[row["key"]] + and row.get("settings") == settings and row.get("dokimi_version") == version): + kept.append(row) + return kept + + +def group_stats(rows: list[dict], *, resamples: int = RESAMPLES) -> dict[str, Any]: + """The n, the new-share and already-known-share intervals (bootstrap), the spend and the median size of *rows*.""" + from dokimi.eval.stats import bootstrap_ci + + new = bootstrap_ci([r["new_lenient"] for r in rows], n_resamples=resamples) + strict_values = [r["new_strict"] for r in rows if r.get("new_strict") is not None] + strict = bootstrap_ci(strict_values, n_resamples=resamples) if strict_values else None + known = {"mean": round(1 - new["mean"], 4), "lo": round(1 - new["hi"], 4), + "hi": round(1 - new["lo"], 4)} + tokens = [r["tokens"] for r in rows if r.get("tokens")] + return { + "n": len(rows), + "new_lenient": new, + "known_lenient": known, + "new_strict": strict, + "usd": round(sum(r["usd"] for r in rows), 6), + "median_tokens": statistics.median(tokens) if tokens else None, + } + + +def summarize(rows: list[dict], strata: Optional[Mapping[str, str]] = None, *, + resamples: int = RESAMPLES) -> dict[str, Any]: + """Counts by status, the spend, and the measure with bootstrap CIs overall and per stratum. + + Only ``ok`` rows with a measure are summarised; ``partial`` and ``error`` rows are counted and + their spend is in ``usd``. With *strata* the per-stratum table is given, else only ``overall``. + """ + counts = {status: 0 for status in STATUSES} + for row in rows: + counts[row["status"]] += 1 + measured = [r for r in rows if r["status"] == "ok" and r["new_lenient"] is not None] + summary: dict[str, Any] = { + "counts": counts, + "usd": round(sum(r["usd"] for r in rows), 6), + "overall": group_stats(measured, resamples=resamples) if measured else None, + "strata": None, + } + if strata is not None: + groups: dict[str, list[dict]] = {} + for row in measured: + groups.setdefault(stratum_of(row, strata), []).append(row) + summary["strata"] = {name: group_stats(groups[name], resamples=resamples) + for name in sorted(groups)} + return summary + + +def _interval(block: Optional[dict]) -> str: + if not block or block["mean"] is None: + return "-" + return f"{block['mean']:.2f} [{block['lo']:.2f}, {block['hi']:.2f}]" + + +def summary_lines(summary: dict[str, Any]) -> list[str]: + """The summary as text: the already-known share first, then one line per stratum.""" + counts = summary["counts"] + head = (f"# novelty sweep: {counts['ok']} measured, {counts['partial']} partial, " + f"{counts['error']} error; ${summary['usd']:.2f} spent") + lines = [head] + overall = summary["overall"] + if overall is None: + lines.append("no skill was measured: no share to report") + return lines + lines.append(f"already known by the model: {_interval(overall['known_lenient'])} " + f"(new to it: {_interval(overall['new_lenient'])}; 95% bootstrap CI, n={overall['n']})") + if summary["strata"] is not None: + lines.append(f"{'stratum':<16} {'n':>4} {'new (lenient)':<22} {'already known':<22} {'strict new':<22} {'$':>7}") + for name, block in summary["strata"].items(): + lines.append(f"{name[:16]:<16} {block['n']:>4} {_interval(block['new_lenient']):<22} " + f"{_interval(block['known_lenient']):<22} {_interval(block['new_strict']):<22} " + f"{block['usd']:>7.2f}") + return lines diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index fbf2192fa..1b2e959e7 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -263,5 +263,11 @@ --alpha # repo security: the one-sided Fisher level per violation class, no registry concept yet (#360) --canary # repo security: a planted fake secret to look for, repeatable (#360) --canaries-file # repo security: the same canaries one per line (#360) +--max-usd-per-skill # catalog novelty: the cap of one skill of a sweep; --max-usd-per-cell caps a repo-run cell, another unit --tool # skill novelty ground (#472): the command whose --help is the source; no registry concept yet --tool-version # skill novelty ground (#472): recorded with the help snapshot; no registry concept yet +--strata # catalog novelty: the CSV that maps skills to strata; no registry concept yet +--stratum-column # catalog novelty: the column of --strata that names the stratum; no registry concept yet +--summary-only # catalog novelty: print a JSONL's summary with no model call; no registry concept yet +--summary-out # catalog novelty: write the summary as JSON; no registry concept yet +--bootstrap-resamples # catalog novelty: resamples per bootstrap interval; no registry concept yet diff --git a/dokimi/tests/test_caller_provider_choices.py b/dokimi/tests/test_caller_provider_choices.py index 9580cd074..dee7f7676 100644 --- a/dokimi/tests/test_caller_provider_choices.py +++ b/dokimi/tests/test_caller_provider_choices.py @@ -35,7 +35,8 @@ LLM_FLAGS = [ "eval questions analyze", "eval questions review", "eval score panel", "arms run", "arms matrix-run", "subjects run", "skill novelty analyze", "skill novelty atoms", - "skill novelty probe", "skill shorten", "skill routing", "catalog routing", "tools preference", + "skill novelty probe", "skill shorten", "skill routing", "catalog routing", "catalog novelty", + "tools preference", ] diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py new file mode 100644 index 000000000..d4f4d8d1e --- /dev/null +++ b/dokimi/tests/test_catalog_novelty.py @@ -0,0 +1,638 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`catalog novelty ROOT`: the paid sweep, run end to end on a fake LLM, offline.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import signal +import threading + +import pytest + +from dokimi.cli import build_parser +from dokimi.metrics.usage import UsageRecord +from dokimi.novelty.client import SpendCapReached +from dokimi.novelty import cost as novelty_cost +from dokimi.utils import InputError +from tests.test_novelty_analyze import SKILL, _scripted + +COST = 0.01 # every fake call is priced at this, so a budget is a number of calls + + +@pytest.fixture(autouse=True) +def _offline(monkeypatch): + monkeypatch.delenv("DOKIMI_RESPONSE_CACHE", raising=False) + base = _scripted("informed") + + def transport(prompt, system): + text, _ = base(prompt, system) + return text, UsageRecord(cost_usd=COST) + + real_client = novelty_cost.client + + def fake_client(*args, **kwargs): + client = real_client(*args, **kwargs) + client.transport = transport + return client + + monkeypatch.setattr(novelty_cost, "client", fake_client) + + +def _root(tmp_path, names=("accel-tuning", "other-skill")): + root = tmp_path / "catalog" + for name in names: + (root / name).mkdir(parents=True) + (root / name / "SKILL.md").write_text(SKILL.replace("accel-tuning", name), encoding="utf-8") + return root + + +def _sweep(root, out, *extra): + argv = ["catalog", "novelty", str(root), "--out-dir", str(out), + "--model", "answerer-m", "--judge-model", "judge-m", *extra] + args = build_parser().parse_args(argv) + args.func(args) + + +def _rows(path): + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def test_each_skill_gets_one_measured_row_and_a_novelty_file(tmp_path, capsys): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + rows = _rows(out / "sweep.jsonl") + assert sorted(r["key"] for r in rows) == ["accel-tuning", "other-skill"] + assert all(r["status"] == "ok" and r["new_lenient"] is not None for r in rows) + assert all(r["usd"] == pytest.approx(r["llm_calls"] * COST) for r in rows) + assert len(list(out.glob("*/novelty.json"))) == 2 + assert "# novelty sweep: 2 measured, 0 partial, 0 error" in capsys.readouterr().out + + +def test_the_summary_leads_with_the_already_known_share(tmp_path, capsys): + _sweep(_root(tmp_path), tmp_path / "out", "--max-usd", "5") + out = capsys.readouterr().out + assert "already known by the model:" in out + assert "new to it:" in out + + +def test_a_per_skill_cap_leaves_a_partial_row_and_the_sweep_goes_on(tmp_path): + out = tmp_path / "out" + _sweep(_root(tmp_path), out, "--max-usd", "5", "--max-usd-per-skill", "0.02") + rows = _rows(out / "sweep.jsonl") + assert [r["status"] for r in rows] == ["partial", "partial"] + assert {r["reason"] for r in rows} == {"spend_cap"} + assert not list(out.glob("*/novelty.json")) + + +def test_the_run_budget_stops_the_skills_it_cannot_pay_for(tmp_path, capsys): + out = tmp_path / "out" + _sweep(_root(tmp_path), out, "--max-usd", "0.05", "--jobs", "1") + rows = _rows(out / "sweep.jsonl") + assert len(rows) == 1 # the second skill was never started + assert rows[0]["status"] == "partial" + assert "1 skill(s) not run: the budget was spent" in capsys.readouterr().err + + +def test_resume_keeps_the_measured_rows_and_runs_only_the_rest(tmp_path, capsys): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5", "--limit", "1") + first = _rows(out / "sweep.jsonl") + assert len(first) == 1 + _sweep(root, out, "--max-usd", "5", "--resume") + rows = _rows(out / "sweep.jsonl") + assert sorted(r["key"] for r in rows) == ["accel-tuning", "other-skill"] + assert rows[0] == first[0] + assert "1 measured and kept, 1 to run" in capsys.readouterr().err + + +def test_resume_runs_again_a_skill_whose_directory_changed(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5", "--limit", "1") + before = _rows(out / "sweep.jsonl")[0]["skill_sha256"] + (root / "accel-tuning" / "SKILL.md").write_text(SKILL + "\nMore text.\n", encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--resume") + rows = _rows(out / "sweep.jsonl") + assert sorted(r["key"] for r in rows) == ["accel-tuning", "other-skill"] # the stale row is gone + changed = [r for r in rows if r["key"] == "accel-tuning"] + assert changed[0]["skill_sha256"] != before and changed[0]["status"] == "ok" + + +def test_an_existing_jsonl_is_not_overwritten_without_resume(tmp_path): + out = tmp_path / "out" + out.mkdir() + (out / "sweep.jsonl").write_text("", encoding="utf-8") + with pytest.raises(InputError, match="pass --resume"): + _sweep(_root(tmp_path), out, "--max-usd", "5") + + +def test_a_paid_sweep_needs_a_whole_run_budget(tmp_path): + with pytest.raises(InputError, match="pass --max-usd"): + _sweep(_root(tmp_path), tmp_path / "out") + + +def test_summary_only_reads_the_jsonl_and_makes_no_call(tmp_path, monkeypatch, capsys): + out, root = tmp_path / "out", _root(tmp_path) + _sweep(root, out, "--max-usd", "5") + capsys.readouterr() + + def no_client(*args, **kwargs): + raise AssertionError("summary-only must not build a client") + + monkeypatch.setattr(novelty_cost, "client", no_client) + _sweep(root, out, "--summary-only") + assert "# novelty sweep: 2 measured" in capsys.readouterr().out + + +def test_summary_only_needs_no_root(tmp_path, capsys): + out, root = tmp_path / "out", _root(tmp_path) + _sweep(root, out, "--max-usd", "5") + capsys.readouterr() + shutil.rmtree(root) + _sweep(root, out, "--summary-only") + assert "# novelty sweep: 2 measured" in capsys.readouterr().out + + +def test_strata_add_a_line_per_stratum(tmp_path, capsys): + out = tmp_path / "out" + strata = tmp_path / "strata.csv" + strata.write_text("skill,stratum\naccel-tuning,ops\nother-skill,dev\n", encoding="utf-8") + _sweep(_root(tmp_path), out, "--max-usd", "5", "--strata", str(strata)) + text = capsys.readouterr().out + assert "stratum" in text and "\nops " in text and "\ndev " in text + + +def test_an_unloadable_skill_is_an_error_row_and_costs_nothing(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + (root / "broken").mkdir() + (root / "broken" / "SKILL.md").write_text("no frontmatter here\n", encoding="utf-8") + _sweep(root, out, "--max-usd", "5") + broken = [r for r in _rows(out / "sweep.jsonl") if r["key"] == "broken"] + assert broken and broken[0]["status"] == "error" + assert broken[0]["usd"] == 0 + + +def test_a_narrower_resume_keeps_the_paid_rows_of_the_skills_it_does_not_select(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + before = {r["key"]: r for r in _rows(out / "sweep.jsonl")} + _sweep(root, out, "--max-usd", "5", "--resume", "--limit", "1") + after = {r["key"]: r for r in _rows(out / "sweep.jsonl")} + assert after == before # other-skill is outside this run's --limit: its paid row stays + + +def test_a_skill_with_no_paired_probe_is_an_error_not_a_share_of_one(tmp_path, monkeypatch): + """The baseline arm answered nothing: k0 is 0.0 by default, so the new share reads 1.0 unless no pair is checked.""" + from types import SimpleNamespace + + import dokimi.artifacts as artifacts + import dokimi.commands.catalog_novelty as catalog_novelty + import dokimi.novelty.report as report + + artifact = {"k0": {"lenient": 0.0, "strict": 0.0}, "paired": {"lenient": {"n_paired": 0}}, + "artifact": {"tokens": 100}} + monkeypatch.setattr(catalog_novelty, "_analyze_skill", + lambda *a, **k: SimpleNamespace(partial=None, artifact=artifact)) + monkeypatch.setattr(artifacts, "save_artifact", lambda kind, art, path: path.parent.mkdir(parents=True)) + monkeypatch.setattr(report, "render_novelty_markdown", lambda artifact: "") + root, out = _root(tmp_path, names=("accel-tuning",)), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + [row] = _rows(out / "sweep.jsonl") + assert row["status"] == "error" and row["reason"] == "no_pairs" + assert row["new_lenient"] is None + + +def test_a_save_that_fails_is_that_skills_error_row_and_the_sweep_goes_on(tmp_path, monkeypatch): + import dokimi.artifacts as artifacts + + real_save = artifacts.save_artifact + + def save(kind, artifact, path, *a, **k): + if "accel-tuning" in str(path): + raise OSError("disk full") + return real_save(kind, artifact, path, *a, **k) + + monkeypatch.setattr(artifacts, "save_artifact", save) + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") # must not raise: one skill's save is its row's error + rows = {r["key"]: r for r in _rows(out / "sweep.jsonl")} + assert rows["accel-tuning"]["status"] == "error" + assert "disk full" in rows["accel-tuning"]["message"] + assert rows["other-skill"]["status"] == "ok" + + +@pytest.mark.parametrize("flag", [ + ["--max-file-words", "100"], ["--max-resource-words", "100"], ["--model", "other-m"], + ["--judge-model", "other-judge"], ["--arms", "no_artifact,full,shuffled"], ["--budgets", "0.5"], ["--seed", "1"], + ["--include-resources"], ["--context-framing", "neutral"], ["--no-paraphrase"], ["--max-atoms", "3"], + ["--no-reversal"], +]) +def test_resume_runs_again_the_rows_measured_under_other_input_limits(tmp_path, capsys, flag): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + capsys.readouterr() + _sweep(root, out, "--max-usd", "5", "--resume", *flag) + assert "0 measured and kept, 2 to run" in capsys.readouterr().err + + +@pytest.mark.skipif(os.name == "nt", reason="os.kill with SIGINT terminates the process on Windows") +@pytest.mark.parametrize("signum, code", [(signal.SIGINT, 130), (signal.SIGTERM, 143)]) +def test_an_interrupt_cancels_the_queued_skills_keeps_the_rows_done_and_exits_nonzero( + tmp_path, monkeypatch, signum, code): + """Ctrl-C or SIGTERM mid-sweep: the skill in flight finishes and keeps its row, the queued skills + never start (no paid call), and the process exits nonzero. The JSONL stays resumable.""" + import time + from types import SimpleNamespace + + import dokimi.artifacts as artifacts + import dokimi.commands.catalog_novelty as catalog_novelty + import dokimi.novelty.report as report + import dokimi.process as process + + kills: list[bool] = [] + monkeypatch.setattr(process, "kill_children", lambda: kills.append(True)) + artifact = {"k0": {"lenient": 0.2, "strict": 0.1}, "paired": {"lenient": {"n_paired": 3}}, + "artifact": {"tokens": 100}} + started = [] + + def analyze(args, *a, **k): + name = args.artifact.rsplit("/", 1)[-1] + started.append(name) + if len(started) == 1: # the signal arrives while the first skill is in flight; it finishes + os.kill(os.getpid(), signum) + time.sleep(0.5) # the main thread cancels the queued skills well inside this + return SimpleNamespace(partial=None, artifact=artifact) + + monkeypatch.setattr(catalog_novelty, "_analyze_skill", analyze) + monkeypatch.setattr(artifacts, "save_artifact", lambda kind, art, path: path.parent.mkdir(parents=True)) + monkeypatch.setattr(report, "render_novelty_markdown", lambda artifact: "") + root = _root(tmp_path, names=("accel-tuning", "beta", "gamma", "other-skill")) + out = tmp_path / "out" + # a SIGTERM the sweep does not handle would end pytest itself: the stray handler keeps that a failure + stray: list[int] = [] + previous = signal.signal(signal.SIGTERM, lambda s, f: stray.append(s)) + try: + try: + _sweep(root, out, "--max-usd", "5", "--jobs", "1") + except SystemExit as exc: + code_seen = exc.code + except KeyboardInterrupt: # not turned into an exit: fail here, not as an aborted pytest run + pytest.fail("the interrupt escaped the sweep as KeyboardInterrupt, not an exit") + else: + pytest.fail("the sweep returned after an interrupt") + finally: + signal.signal(signal.SIGTERM, previous) + assert code_seen == code + assert started == ["accel-tuning"] # the queued skills were cancelled before they started + assert not kills # the first signal kills nothing: the call in flight finishes + assert not stray + rows = _rows(out / "sweep.jsonl") + assert [(r["key"], r["status"]) for r in rows] == [("accel-tuning", "ok")] + + +@pytest.mark.skipif(os.name == "nt", reason="os.kill with SIGINT terminates the process on Windows") +def test_a_second_signal_kills_the_calls_in_flight_and_stops_at_once(tmp_path, monkeypatch): + """The first Ctrl-C lets the call in flight finish; a second one kills the calls in flight + (process.kill_children) and stops the sweep at once, with the same exit code as the first.""" + import time + from types import SimpleNamespace + + import dokimi.artifacts as artifacts + import dokimi.commands.catalog_novelty as catalog_novelty + import dokimi.novelty.report as report + import dokimi.process as process + + kills: list[bool] = [] + monkeypatch.setattr(process, "kill_children", lambda: kills.append(True)) + artifact = {"k0": {"lenient": 0.2, "strict": 0.1}, "paired": {"lenient": {"n_paired": 3}}, + "artifact": {"tokens": 100}} + started = [] + next_call: list[str] = [] + next_done = threading.Event() + + def analyze(args, *a, **k): + started.append(args.artifact.rsplit("/", 1)[-1]) + if len(started) == 1: # two Ctrl-Cs while the first skill is in flight + os.kill(os.getpid(), signal.SIGINT) + time.sleep(0.1) + os.kill(os.getpid(), signal.SIGINT) + time.sleep(0.5) + try: + args._spend_cap.before_call("m") # the skill's next call, after the sweep stopped + next_call.append("started") + except SpendCapReached: + next_call.append("refused") + finally: + next_done.set() + return SimpleNamespace(partial=None, artifact=artifact) + + monkeypatch.setattr(catalog_novelty, "_analyze_skill", analyze) + monkeypatch.setattr(artifacts, "save_artifact", lambda kind, art, path: path.parent.mkdir(parents=True)) + monkeypatch.setattr(report, "render_novelty_markdown", lambda artifact: "") + root = _root(tmp_path, names=("accel-tuning", "beta", "gamma", "other-skill")) + out = tmp_path / "out" + with pytest.raises(SystemExit) as exc: + _sweep(root, out, "--max-usd", "5", "--jobs", "1") + assert exc.value.code == 130 + assert kills == [True] + assert started == ["accel-tuning"] + assert next_done.wait(5) and next_call == ["refused"] + + +def test_a_row_is_written_once_even_when_its_write_is_interrupted(): + """A second Ctrl-C landing in write(): the row the sink already took is not written again by the drain.""" + from dokimi.commands.catalog_novelty import _Item, _run_sweep + + written: list[str] = [] + + def write(row): + written.append(row["key"]) + raise KeyboardInterrupt # after the append, as a second signal lands in the sink + + items = [_Item(key="a", name="a", skill_md=Path("a/SKILL.md"), sha256=None), + _Item(key="b", name="b", skill_md=Path("b/SKILL.md"), sha256=None)] + with pytest.raises(KeyboardInterrupt): + _run_sweep(items, jobs=1, review=lambda item: {"key": item.key}, write=write, + out_of_budget=lambda: False, stop_requested=lambda: False) + assert written.count("a") == 1 # written once; b may or may not have started before the cancel + + +def test_settle_keeps_the_rows_done_before_it_waits_for_the_skills_in_flight(monkeypatch): + """An interrupt while the drain waits for the skills in flight keeps the rows already done.""" + import threading + from concurrent.futures import ThreadPoolExecutor + + import dokimi.commands.catalog_novelty as catalog_novelty + + release = threading.Event() + slow_started = threading.Event() + pool = ThreadPoolExecutor(max_workers=2) + done = pool.submit(lambda: {"key": "a"}) + done.result() + + def slow_skill(): + slow_started.set() # running, not queued: the cancel of the queue leaves it alone + release.wait(5) + return {"key": "b"} + + slow = pool.submit(slow_skill) + assert slow_started.wait(5) + running = {done, slow} + written: list[str] = [] + + def interrupted_wait(*args, **kwargs): + release.set() + raise KeyboardInterrupt + + monkeypatch.setattr(catalog_novelty, "wait", interrupted_wait) + with pytest.raises(KeyboardInterrupt): + catalog_novelty._settle(pool, running, lambda row: written.append(row["key"])) + assert written == ["a"] + + +def test_a_skill_queued_when_the_stop_lands_makes_no_call(monkeypatch): + """The stop lands while a skill is in flight and that skill finishes at once: the skill the freed + worker takes next must not start, since the stop came first (no new paid call).""" + import time + + import dokimi.commands.catalog_novelty as catalog_novelty + from dokimi.commands.catalog_novelty import _Item, _run_sweep + + real_wait = catalog_novelty.wait + + def late_wait(futures, **kwargs): # the main thread wakes late: the freed worker takes the next skill first + done = real_wait(futures, **kwargs) + time.sleep(0.3) + return done + + monkeypatch.setattr(catalog_novelty, "wait", late_wait) + stop = [False] + started: list[str] = [] + + def review(item): + started.append(item.key) + if item.key == "a": # b is queued behind a by now; the stop lands while a is in flight + time.sleep(0.05) + stop[0] = True + return {"key": item.key} + + items = [_Item(key=k, name=k, skill_md=Path(f"{k}/SKILL.md"), sha256=None) for k in "abc"] + _run_sweep(items, jobs=1, review=review, write=lambda row: None, out_of_budget=lambda: False, + stop_requested=lambda: stop[0]) + assert started == ["a"] + + +def test_a_row_finished_during_the_drain_is_written_before_the_next_wait(monkeypatch): + """A skill that finishes while the drain waits for another is written then, not after all of them: + an interrupt during the wait for the rest keeps it.""" + import threading + from concurrent.futures import ThreadPoolExecutor + + import dokimi.commands.catalog_novelty as catalog_novelty + + import time + + release = threading.Event() + started = threading.Barrier(3) # the two skills and this test: both are running before the drain + pool = ThreadPoolExecutor(max_workers=2) + + def fast_skill(): + started.wait(5) + time.sleep(0.1) # still in flight when the drain starts + return {"key": "fast"} + + def slow_skill(): + started.wait(5) + release.wait(5) + return {"key": "slow"} + + fast = pool.submit(fast_skill) + slow = pool.submit(slow_skill) + started.wait(5) + real_wait = catalog_novelty.wait + waits: list[int] = [] + + def wait_then_interrupt(futures, **kwargs): + waits.append(1) + if len(waits) > 1: + raise KeyboardInterrupt # a second signal while the drain waits for the slow skill + return real_wait(futures, **kwargs) + + monkeypatch.setattr(catalog_novelty, "wait", wait_then_interrupt) + written: list[str] = [] + try: + with pytest.raises(KeyboardInterrupt): + catalog_novelty._settle(pool, {fast, slow}, lambda row: written.append(row["key"])) + assert written == ["fast"] + finally: + release.set() + pool.shutdown(wait=True) + + +def test_identical_skill_copies_are_one_skill_and_the_help_says_so(tmp_path, capsys): + """Two directories with the same SKILL.md are one skill: the shallowest path keeps the row.""" + root = _root(tmp_path, names=("accel-tuning",)) + copy = root / "mirror" / "accel-tuning" + copy.mkdir(parents=True) + (copy / "SKILL.md").write_text((root / "accel-tuning" / "SKILL.md").read_text(encoding="utf-8"), + encoding="utf-8") + out = tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + assert [r["key"] for r in _rows(out / "sweep.jsonl")] == ["accel-tuning"] + capsys.readouterr() + with pytest.raises(SystemExit): + build_parser().parse_args(["catalog", "novelty", "--help"]) + help_text = " ".join(capsys.readouterr().out.split()) + assert "identical" in help_text + assert "Ctrl-C or SIGTERM" in help_text + + +def test_resume_runs_again_the_rows_measured_under_another_placebo_text(tmp_path, capsys): + root, out = _root(tmp_path), tmp_path / "out" + placebo = tmp_path / "placebo.txt" + placebo.write_text("A neutral paragraph about the weather. " * 40, encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--arms", "no_artifact,full,placebo", + "--placebo-text", str(placebo)) + capsys.readouterr() + placebo.write_text("A different neutral paragraph about tea. " * 40, encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--resume", "--arms", "no_artifact,full,placebo", + "--placebo-text", str(placebo)) + assert "0 measured and kept, 2 to run" in capsys.readouterr().err + + +def test_two_bundles_with_one_skill_md_and_other_resources_are_two_skills(tmp_path, capsys): + root = _root(tmp_path, names=("accel-tuning",)) + for side, text in (("a", "one"), ("b", "two")): + bundle = root / side / "accel-tuning" + (bundle / "references").mkdir(parents=True) + (bundle / "SKILL.md").write_text((root / "accel-tuning" / "SKILL.md").read_text(encoding="utf-8"), + encoding="utf-8") + (bundle / "references" / "ref.md").write_text(f"Reference {text}.\n", encoding="utf-8") + twin = root / "c" / "accel-tuning" + shutil.copytree(root / "a" / "accel-tuning", twin) + out = tmp_path / "out" + _sweep(root, out, "--max-usd", "5", "--include-resources") + keys = [r["key"] for r in _rows(out / "sweep.jsonl")] + assert keys == ["a/accel-tuning", "accel-tuning", "b/accel-tuning"] # c/ is a copy of a/: left out + assert "left out" in capsys.readouterr().err + + +def test_resume_runs_again_a_symlinked_skill_md_whose_target_changed(tmp_path): + root, out = _root(tmp_path, names=("other-skill",)), tmp_path / "out" + shared = tmp_path / "shared" / "SKILL.md" + shared.parent.mkdir() + shared.write_text(SKILL, encoding="utf-8") + (root / "linked").mkdir() + try: + (root / "linked" / "SKILL.md").symlink_to(shared) + except OSError: + pytest.skip("symlinks are not available") + _sweep(root, out, "--max-usd", "5") + before = {r["key"]: r["skill_sha256"] for r in _rows(out / "sweep.jsonl")} + shared.write_text(SKILL + "\nMore text.\n", encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--resume") + after = {r["key"]: r["skill_sha256"] for r in _rows(out / "sweep.jsonl")} + assert after["linked"] != before["linked"] and after["other-skill"] == before["other-skill"] + + +def test_a_skill_measured_again_that_ends_partial_or_error_leaves_no_old_novelty_file(tmp_path): + root, out = _root(tmp_path, names=("accel-tuning",)), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + old = Path(_rows(out / "sweep.jsonl")[0]["novelty_json"]) + assert old.is_file() and old.with_suffix(".md").is_file() + (root / "accel-tuning" / "SKILL.md").write_text(SKILL + "\nMore text.\n", encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--resume", "--max-usd-per-skill", "0.02") + assert _rows(out / "sweep.jsonl")[0]["status"] == "partial" + assert not old.exists() and not old.with_suffix(".md").exists() + (root / "accel-tuning" / "SKILL.md").write_text("no frontmatter at all", encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--resume") + assert _rows(out / "sweep.jsonl")[0]["status"] == "error" + assert not old.exists() + + +def test_the_output_directory_of_a_long_nested_key_stays_a_valid_name(tmp_path): + from dokimi.commands.catalog_novelty import _dirname + + key = "/".join(["a" * 100] * 3) + name = _dirname(key) + assert len(name) <= 100 + assert name != _dirname(key[:-1] + "b") # the hash still tells long keys apart + (tmp_path / name).mkdir() + + +@pytest.mark.parametrize("summary_only", [False, True]) +def test_a_summary_out_that_is_the_jsonl_is_refused_before_any_work(tmp_path, summary_only): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + before = (out / "sweep.jsonl").read_text(encoding="utf-8") + alias = out / ".." / "out" / "sweep.jsonl" # the same file by another path + extra = ["--summary-only"] if summary_only else ["--max-usd", "5", "--resume"] + with pytest.raises(InputError, match="--summary-out"): + _sweep(root, out, *extra, "--summary-out", str(alias)) + assert (out / "sweep.jsonl").read_text(encoding="utf-8") == before + + +def test_a_summary_out_in_a_missing_directory_is_made_and_written(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + target = tmp_path / "new" / "dir" / "summary.json" + _sweep(root, out, "--max-usd", "5", "--summary-out", str(target)) + assert json.loads(target.read_text(encoding="utf-8"))["counts"]["ok"] == 2 + + +def test_a_summary_out_whose_directory_cannot_be_made_is_refused_before_a_paid_call(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + blocker = tmp_path / "file" + blocker.write_text("x", encoding="utf-8") + with pytest.raises(InputError, match="--summary-out"): + _sweep(root, out, "--max-usd", "5", "--summary-out", str(blocker / "summary.json")) + assert not (out / "sweep.jsonl").exists() + + +def test_a_second_signal_inside_the_poll_wait_waits_for_no_skill_and_writes_no_row(): + import threading + import time + + from dokimi.commands.catalog_novelty import _Item, _run_sweep + + started, release = threading.Event(), threading.Event() + written: list[str] = [] + + def review(item): + started.set() + release.wait(5) + return {"key": item.key} + + def stop_requested(): # the main thread's poll sees the second signal while the skill is in flight + if threading.current_thread() is threading.main_thread() and started.is_set(): + raise KeyboardInterrupt + return False + + items = [_Item(key="a", name="a", skill_md=Path("a/SKILL.md"), sha256=None)] + begun = time.monotonic() + try: + with pytest.raises(KeyboardInterrupt): + _run_sweep(items, jobs=1, review=review, write=lambda row: written.append(row["key"]), + out_of_budget=lambda: False, stop_requested=stop_requested) + assert time.monotonic() - begun < 2 # the skill in flight is not waited for + assert written == [] + finally: + release.set() + + +def test_the_hard_stop_ends_the_next_call_of_a_skill_in_flight(): + import threading + + from dokimi.novelty.client import NestedSpendCap, SpendCap, SpendCapReached + + hard = threading.Event() + cap = NestedSpendCap(limit_usd=1.0, parent=SpendCap(limit_usd=5.0), hard_stop=hard) + cap.before_call("m") # a call in flight started before the signal + hard.set() + with pytest.raises(SpendCapReached) as stop: + cap.before_call("m") + assert stop.value.reason == "stopped" + assert cap.parent.calls == 0 diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 1588fdbfb..a9276dc4f 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -113,6 +113,7 @@ def test_help_lists_each_root_under_its_section(): ("explore", "run"), ("catalog", "trigger-probe"), ("catalog", "routing"), + ("catalog", "novelty"), ("skill", "novelty", "analyze"), ("skill", "novelty", "atoms"), ("skill", "novelty", "probe"), diff --git a/dokimi/tests/test_judge_provider_choices.py b/dokimi/tests/test_judge_provider_choices.py index 3f8afc15f..a7432ee25 100644 --- a/dokimi/tests/test_judge_provider_choices.py +++ b/dokimi/tests/test_judge_provider_choices.py @@ -30,7 +30,8 @@ def _judge_provider_actions(parser, path=()): #: Judge-role flags whose value never reaches ``Judge``: the novelty commands call #: the model through ``novelty.client.LLMClient`` -> ``llm_call_with_usage``, which #: does route ``openai-codex``. -NOT_THE_JUDGE_CLASS = {"skill novelty analyze --judge-provider", "skill novelty probe --judge-provider"} +NOT_THE_JUDGE_CLASS = {"skill novelty analyze --judge-provider", "skill novelty probe --judge-provider", + "catalog novelty --judge-provider"} def test_every_judge_provider_flag_offers_exactly_judge_providers(): diff --git a/dokimi/tests/test_judge_provider_default.py b/dokimi/tests/test_judge_provider_default.py index 78787ed03..efc834309 100644 --- a/dokimi/tests/test_judge_provider_default.py +++ b/dokimi/tests/test_judge_provider_default.py @@ -27,7 +27,8 @@ def _judge_commands() -> dict[str, list[str]]: def walk(parser: argparse.ArgumentParser, path: list[str]) -> None: dests = {a.dest for a in parser._actions} - if "judge_provider" in dests and path[0] != "novelty" and path[:2] != ["skill", "novelty"]: + if ("judge_provider" in dests and path[0] != "novelty" and path[:2] != ["skill", "novelty"] + and path[:2] != ["catalog", "novelty"]): argv = list(path) for action in parser._actions: if action.required: diff --git a/dokimi/tests/test_novelty_sweep.py b/dokimi/tests/test_novelty_sweep.py new file mode 100644 index 000000000..ef8ffd9f7 --- /dev/null +++ b/dokimi/tests/test_novelty_sweep.py @@ -0,0 +1,227 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The corpus sweep's bookkeeping: rows, resume, strata and the summary. No model call.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.novelty.client import NestedSpendCap, SpendCap, SpendCapReached +from dokimi.metrics.usage import UsageRecord +from dokimi.novelty.sweep import ( + ALL, UNASSIGNED, group_stats, load_strata, make_row, measured_fields, new_share, + read_rows, resumable, stratum_of, summarize, summary_lines, +) +from dokimi.utils import InputError + +SETTINGS = {"max_atoms": 120} + + +def _ok(key, new, usd=0.1, tokens=800, name=None, digest="d1"): + measured = {"tokens": tokens, "known_lenient": round(1 - new, 6), "new_lenient": new, + "new_strict": round(min(1.0, new + 0.1), 6), "n_paired": 4} + return make_row(key=key, name=name or key, path=f"/c/{key}/SKILL.md", skill_sha256=digest, + status="ok", settings=SETTINGS, version="1.0", usd=usd, calls=3, + measured=measured, novelty_json=f"/out/{key}/novelty.json") + + +def _row(key, status, **kw): + return make_row(key=key, name=key, path=f"/c/{key}/SKILL.md", skill_sha256="d1", + status=status, settings=SETTINGS, version="1.0", usd=kw.pop("usd", 0.05), **kw) + + +def test_new_share_is_one_minus_the_already_known_share(): + assert new_share(0.66) == pytest.approx(0.34) + assert new_share(None) is None + + +def test_measured_fields_read_the_lenient_and_strict_k0_and_the_pairs(): + artifact = {"k0": {"lenient": 0.25, "strict": 0.1}, "paired": {"lenient": {"n_paired": 4}}, + "artifact": {"tokens": 900}} + fields = measured_fields(artifact) + assert fields["known_lenient"] == 0.25 + assert fields["new_lenient"] == pytest.approx(0.75) + assert fields["new_strict"] == pytest.approx(0.9) + assert (fields["n_paired"], fields["tokens"]) == (4, 900) + + +def test_measured_fields_of_an_artifact_without_k0_has_no_share(): + assert measured_fields({})["new_lenient"] is None + + +def test_an_unpriced_call_charged_through_a_child_stops_the_parent(): + parent = SpendCap(limit_usd=10.0) + child = NestedSpendCap(limit_usd=1.0, parent=parent) + child.record(UsageRecord(cost_usd=0.4), "m") + assert (child.spent_usd, parent.spent_usd) == (0.4, 0.4) + with pytest.raises(SpendCapReached, match="no price"): + child.record(None, "m") + with pytest.raises(SpendCapReached, match="no known price"): + parent.before_call("m") + + +def test_the_child_cap_checks_its_own_limit_as_well_as_the_parent(): + parent = SpendCap(limit_usd=10.0) + child = NestedSpendCap(limit_usd=0.5, parent=parent) + child.record(UsageRecord(cost_usd=0.5), "m") + with pytest.raises(SpendCapReached, match="spend cap reached"): + child.before_call("m") + assert parent.spent_usd == 0.5 + + +def test_the_parent_refusal_stops_a_child_that_still_has_room(): + parent = SpendCap(limit_usd=0.5) + child = NestedSpendCap(limit_usd=5.0, parent=parent) + parent.record(UsageRecord(cost_usd=0.5), "m") + with pytest.raises(SpendCapReached): + child.before_call("m") + + +def test_load_strata_maps_each_skill_to_its_stratum(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,stratum\nops/deploy,ops\nreview,dev\n", encoding="utf-8") + assert load_strata(path, "stratum") == {"ops/deploy": "ops", "review": "dev"} + + +def test_load_strata_refuses_a_missing_column(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,tier\nreview,dev\n", encoding="utf-8") + with pytest.raises(InputError, match="needs a `skill` column and a `stratum` column"): + load_strata(path, "stratum") + + +def test_load_strata_refuses_skill_as_the_stratum_column(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,skill\nreview,dev\n", encoding="utf-8") + with pytest.raises(InputError, match="must not be `skill`"): + load_strata(path, "skill") + + +def test_load_strata_refuses_a_skill_in_two_strata(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,stratum\nreview,dev\nreview,ops\n", encoding="utf-8") + with pytest.raises(InputError, match="is in two strata"): + load_strata(path, "stratum") + + +def test_load_strata_refuses_an_empty_cell(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,stratum\nreview,\n", encoding="utf-8") + with pytest.raises(InputError, match="must both be set"): + load_strata(path, "stratum") + + +def test_stratum_of_matches_by_key_then_name_and_falls_back_to_unassigned(): + strata = {"ops/deploy": "ops", "review": "dev"} + assert stratum_of({"key": "ops/deploy", "name": "deploy"}, strata) == "ops" + assert stratum_of({"key": "x/review", "name": "review"}, strata) == "dev" + assert stratum_of({"key": "other", "name": "other"}, strata) == UNASSIGNED + assert stratum_of({"key": "other", "name": "other"}, None) == ALL + + +def test_read_rows_drops_a_last_line_a_killed_run_left_half_written(tmp_path): + path = tmp_path / "sweep.jsonl" + path.write_text(json.dumps(_ok("a", 0.3)) + "\n" + '{"key": "b", "sta', encoding="utf-8") + rows = read_rows(path) + assert [r["key"] for r in rows] == ["a"] + + +def test_read_rows_refuses_a_bad_line_that_is_not_the_last(tmp_path): + path = tmp_path / "sweep.jsonl" + path.write_text('{"key": "b", "sta\n' + json.dumps(_ok("a", 0.3)) + "\n", encoding="utf-8") + with pytest.raises(InputError, match="line 1 is not a sweep row"): + read_rows(path) + + +def test_read_rows_refuses_a_line_that_is_not_a_row(tmp_path): + path = tmp_path / "sweep.jsonl" + path.write_text(json.dumps({"hello": 1}) + "\n", encoding="utf-8") + with pytest.raises(InputError, match="is not a sweep row"): + read_rows(path) + + +def test_read_rows_of_a_missing_file_is_empty(tmp_path): + assert read_rows(tmp_path / "absent.jsonl") == [] + + +def test_resumable_keeps_only_measured_rows_for_the_same_skill_digest_settings_and_version(): + good = _ok("keep", 0.3) + changed_skill = _ok("changed", 0.3, digest="old") + errored = _row("errored", "error") + partial = _row("partial", "partial") + gone = _ok("gone", 0.3) + old_version = dict(_ok("version", 0.3), dokimi_version="0.9") + prior = [good, changed_skill, errored, partial, gone, old_version] + selected = {"keep": "d1", "changed": "d2", "errored": "d1", "partial": "d1", "version": "d1"} + kept = resumable(prior, selected, SETTINGS, "1.0") + assert [r["key"] for r in kept] == ["keep"] + assert resumable(prior, selected, {"max_atoms": 90}, "1.0") == [] + + +def test_group_stats_bootstraps_the_new_share_and_flips_the_interval_for_the_known_share(): + rows = [_ok("a", 0.2), _ok("b", 0.4), _ok("c", 0.6)] + stats = group_stats(rows, resamples=300) + assert stats["n"] == 3 + assert stats["new_lenient"]["mean"] == pytest.approx(0.4, abs=1e-3) + known, new = stats["known_lenient"], stats["new_lenient"] + assert known["mean"] == pytest.approx(1 - new["mean"], abs=1e-3) + assert known["lo"] == pytest.approx(1 - new["hi"], abs=1e-3) + assert known["hi"] == pytest.approx(1 - new["lo"], abs=1e-3) + assert stats["usd"] == pytest.approx(0.3) + assert stats["median_tokens"] == 800 + + +def test_summarize_counts_every_status_but_measures_only_ok_rows(): + rows = [_ok("a", 0.2, usd=0.3), _row("b", "partial", usd=0.1), _row("c", "error", usd=0.0)] + summary = summarize(rows, resamples=200) + assert summary["counts"] == {"ok": 1, "partial": 1, "error": 1} + assert summary["usd"] == pytest.approx(0.4) + assert summary["overall"]["n"] == 1 + assert summary["strata"] is None + + +def test_summarize_splits_the_measure_by_stratum_and_names_the_unassigned(): + rows = [_ok("ops/deploy", 0.2), _ok("review", 0.6), _ok("misc", 0.5)] + summary = summarize(rows, {"ops/deploy": "ops", "review": "dev"}, resamples=200) + assert set(summary["strata"]) == {"dev", "ops", UNASSIGNED} + assert summary["strata"]["dev"]["n"] == 1 + assert summary["strata"]["dev"]["new_lenient"]["mean"] == pytest.approx(0.6) + + +def test_summary_lines_say_when_nothing_was_measured(): + lines = summary_lines(summarize([_row("a", "error")], resamples=50)) + assert lines[0].startswith("# novelty sweep: 0 measured, 0 partial, 1 error") + assert lines[1] == "no skill was measured: no share to report" + + +def test_summary_lines_lead_with_the_already_known_share(): + lines = summary_lines(summarize([_ok("a", 0.34)], resamples=50)) + assert lines[1].startswith("already known by the model: 0.66 [") + assert "new to it: 0.34" in lines[1] + + +def test_make_row_refuses_an_unknown_status(): + with pytest.raises(ValueError, match="unknown sweep status"): + _row("a", "done") + + +def test_resumable_never_keeps_a_row_whose_directory_could_not_be_digested(): + prior = [dict(_ok("gone-quiet", 0.3), skill_sha256=None)] + assert resumable(prior, {"gone-quiet": None}, SETTINGS, "1.0") == [] + + +@pytest.mark.parametrize("bad", [ + '{"key": "a", "name": "a", "status": "ok", "usd": "x"}\n', + '{"key": [], "name": "a", "status": "partial", "usd": 0.0}\n', + '{"key": "a", "status": "ok"}\n', + '{"key": "a", "name": "a", "status": "ok", "usd": 0.1, "new_lenient": "high"}\n', + '[1]\n', +]) +def test_a_row_of_the_wrong_shape_is_a_line_numbered_input_error(tmp_path, bad): + path = tmp_path / "s.jsonl" + path.write_text(json.dumps(_ok("a", 0.3)) + "\n" + bad, encoding="utf-8") + with pytest.raises(InputError, match="line 2 is not a sweep row"): + read_rows(path)