From 96f52a47ae09ba57057c5bf1a9279b67ec5c0bf1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 13:37:18 -0800 Subject: [PATCH 01/11] feat(catalog): novelty sweep over a skill catalog, under one budget `dokimi catalog novelty ROOT --out-dir DIR --max-usd USD` runs `skill novelty analyze` on every SKILL.md under ROOT, one row per skill written to --jsonl as each finishes. - --max-usd is the whole-run budget (required; a paid sweep without one is refused). --max-usd-per-skill caps one skill inside it: a skill that hits its cap is a `partial` row and leaves no novelty file, and the sweep goes on. - --jobs runs skills in threads that share the budget through NestedSpendCap, which charges the run's SpendCap first. - --resume keeps the ok rows whose directory digest, settings and dokimi version still match, runs the rest, and keeps the rows of skills this run does not select (a narrower --limit). - Refuses an existing --jsonl without --resume and --judge2. - Summary: already-known share (k0.lenient) headline, new share = 1 - known, bootstrap 95% CIs overall and per stratum (--strata CSV). --summary-only reads the JSONL with no model call. Tests use a fake priced transport; no paid call is made. Signed-off-by: Nikolay Petrov --- ...6_120000_napetrov_catalog-novelty-sweep.md | 14 + dokimi/docs/reference/cli-interface.json | 48 +++ dokimi/docs/skill-novelty.md | 34 ++ dokimi/dokimi/cli_routes.py | 1 + dokimi/dokimi/commands/catalog.py | 4 + dokimi/dokimi/commands/catalog_novelty.py | 406 ++++++++++++++++++ dokimi/dokimi/novelty/client.py | 22 + dokimi/dokimi/novelty/sweep.py | 217 ++++++++++ dokimi/tests/data/cli_flags_allowlist.txt | 6 + dokimi/tests/test_caller_provider_choices.py | 3 +- dokimi/tests/test_catalog_novelty.py | 170 ++++++++ dokimi/tests/test_cli_routes.py | 1 + dokimi/tests/test_judge_provider_choices.py | 3 +- dokimi/tests/test_judge_provider_default.py | 3 +- dokimi/tests/test_novelty_sweep.py | 213 +++++++++ 15 files changed, 1142 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md create mode 100644 dokimi/dokimi/commands/catalog_novelty.py create mode 100644 dokimi/dokimi/novelty/sweep.py create mode 100644 dokimi/tests/test_catalog_novelty.py create mode 100644 dokimi/tests/test_novelty_sweep.py diff --git a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md new file mode 100644 index 000000000..e3637b01c --- /dev/null +++ b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md @@ -0,0 +1,14 @@ + + +### Added + +- **`catalog novelty ROOT`: the corpus novelty sweep under one budget.** Runs + `skill novelty analyze` on every skill under ROOT, with `--max-usd` as the whole-run budget + (required), `--max-usd-per-skill`, `--jobs`, and JSONL output with one row per skill as it + finishes. `--resume` keeps the measured rows whose skill digest and settings are unchanged, and + an existing `--jsonl` is not overwritten without it. `--strata CSV` adds bootstrap 95% + intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. See + `docs/skill-novelty.md`, Corpus sweep. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index fe220c6ea..39c16f3d1 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -352,6 +352,54 @@ "subcommand_required": null, "subcommands": {} }, + "novelty": { + "allow_abbrev": true, + "params": [ + {"key": "root", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": null}, + {"key": "--allow-self-judge", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--arms", "action": "_StoreAction", "choices": null, "const": null, "default": "no_artifact,full", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--bootstrap-resamples", "action": "_StoreAction", "choices": null, "const": null, "default": 2000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--budgets", "action": "_StoreAction", "choices": null, "const": null, "default": [0.75, 0.5], "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": "+", "position": null, "required": false, "type": "float"}, + {"key": "--cache-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--concurrency", "action": "_StoreAction", "choices": null, "const": null, "default": 5, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--context-framing", "action": "_StoreAction", "choices": ["'neutral'", "'reference'"], "const": null, "default": "reference", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--extract-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--extract-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--include-resources", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--jobs", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--jsonl", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge2-frac", "action": "_StoreAction", "choices": null, "const": null, "default": 0.15, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "float"}, + {"key": "--judge2-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--limit", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-atoms", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-file-words", "action": "_StoreAction", "choices": null, "const": null, "default": 4000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-resource-words", "action": "_StoreAction", "choices": null, "const": null, "default": 20000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-tokens", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--max-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.novelty_run.usd_amount#af10a4ceaa08"}, + {"key": "--max-usd-per-skill", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.novelty_run.usd_amount#af10a4ceaa08"}, + {"key": "--model", "action": "_StoreAction", "choices": null, "const": null, "default": "gpt-6.1-sol", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--no-paraphrase", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--no-reversal", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, + {"key": "--placebo-text", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": "openai", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--resource-roles", "action": "_StoreAction", "choices": ["'data'", "'eval'", "'other'", "'pattern'", "'reference'", "'script'", "'template'", "'top'", "'trigger'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": "+", "position": null, "required": false, "type": null}, + {"key": "--resume", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--seed", "action": "_StoreAction", "choices": null, "const": null, "default": 0, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "int"}, + {"key": "--strata", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--stratum-column", "action": "_StoreAction", "choices": null, "const": null, "default": "stratum", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--summary-only", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--summary-out", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--tier", "action": "_StoreAction", "choices": ["'standard'", "'study'", "'triage'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} + ], + "path": "dokimi catalog novelty", + "subcommand_required": null, + "subcommands": {} + }, "routing": { "allow_abbrev": true, "params": [ diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 1f3bfbaf2..56e18291b 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1110,6 +1110,40 @@ A cell with no check measured at all is `NOT MEASURED`, not `PASS`. Failing `warn`-severity checks and the artifact's own `warnings` also surface in the scorecard's top-level `warnings`. +## Corpus sweep: `catalog novelty` + +`catalog novelty ROOT --out-dir DIR --max-usd USD` runs `skill novelty analyze` on every +`SKILL.md` under ROOT and writes one row per skill to `--jsonl` (default `DIR/sweep.jsonl`) as +each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets +`DIR//novelty.json` and `novelty.md`. The calls are paid: `--max-usd` is required. + +- **Budget.** `--max-usd` is the whole-run budget, shared by the skills analysed at once + (`--jobs`, threads). `--max-usd-per-skill` caps one skill inside it; a skill that hits either + cap is a `partial` row with no novelty file, and the sweep goes on. Once the whole-run budget + is spent, no further skill starts: it is not in the JSONL, and the run says how many were not run. +- **Resume.** `--resume` keeps the `ok` rows whose skill directory digest, measurement settings + and dokimi version are unchanged, rewrites the JSONL with them and with the rows of skills this + run does not select (a narrower `--limit`), and runs the other skills. Set + `DOKIMI_RESPONSE_CACHE` so the calls already paid replay at $0. Without `--resume`, an existing + `--jsonl` is refused rather than overwritten. +- **Settings.** The sweep takes the measurement flags it needs (`--model`, `--judge-model`, + `--tier`, `--max-atoms`, `--budgets`, `--arms`, `--include-resources`, ...); the rest keep their + `skill novelty analyze` defaults. `--judge2` is not part of the sweep: run it on one skill. +- **Summary.** The first line of the summary is the share the model already knew without the + skill, `k0.lenient`, and the share it did not know (`1 - known`): the new-to-the-model share. + Each interval is a 95% percentile bootstrap over the skills (2000 resamples, + `--bootstrap-resamples`). `--strata CSV` adds one line per stratum: the CSV has a `skill` + column (the skill's directory under ROOT, or its name) and a stratum column (`--stratum-column`, + default `stratum`); a skill the CSV does not name is `(unassigned)`. +- **Summary only.** `--summary-only --jsonl FILE` prints the summary of a finished or partial + sweep with no model call. `--summary-out FILE` also writes it as JSON. + +```bash +dokimi catalog novelty skills/ --out-dir out/sweep --max-usd 20 --max-usd-per-skill 1 \ + --jobs 4 --model --judge-model --strata strata.csv +dokimi catalog novelty skills/ --out-dir out/sweep --summary-only +``` + ## Cost, and how it is kept down The measurement is LLM-call bound, and the probe term is predictable: **every probe costs an diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index d44d6ac72..6e010ee6d 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -130,6 +130,7 @@ class Tombstone: Leaf(("skill", "check"), FREE), Leaf(("skill", "routing"), PAID), Leaf(("catalog", "routing"), PAID), + Leaf(("catalog", "novelty"), PAID), Leaf(("repo", "validate"), FREE), Leaf(("repo", "scan"), FREE), Leaf(("repo", "inspect"), FREE), diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index 30df4f882..ed0b6a54e 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -458,6 +458,10 @@ def register(sub, positive_int, eval_sub) -> None: add_catalog_routing(catalog_sub, positive_int) # catalog routing: all-vs-all (#94 M3) + from dokimi.commands import catalog_novelty + + catalog_novelty.register(catalog_sub, positive_int) # catalog novelty: the corpus sweep + kinds = ["product", "subject"] list_p = catalog_sub.add_parser( "list", help="List products (--kind product) or subject descriptors (--kind subject)" diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py new file mode 100644 index 000000000..1d9b5f4c2 --- /dev/null +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -0,0 +1,406 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``catalog novelty ROOT``: the corpus novelty sweep, one ``skill novelty analyze`` per skill. + +Every ``SKILL.md`` under ROOT is analysed with the flags of ``skill novelty analyze``, and the +model calls are paid. ``--max-usd`` is one budget for the whole sweep and ``--max-usd-per-skill`` +a cap on each skill; ``--jobs`` runs skills in parallel threads that share the budget. Each skill +writes one row to ``--jsonl`` as it finishes, so a killed sweep keeps its rows, and ``--resume`` +runs only the skills without a measured row (same directory digest, same settings). +``--summary-only`` prints the summary of a JSONL file without any model call. + +A skill the budget stops mid-analysis is a ``partial`` row and leaves no novelty file. Set +``DOKIMI_RESPONSE_CACHE`` so a resumed run replays the calls already paid for at $0. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import math +import re +import sys +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Optional + +from dokimi import __version__ +from dokimi.commands.novelty_run import stops_at_cap +from dokimi.novelty.sweep import read_rows, resumable, summarize, summary_lines +from dokimi.utils import InputError + +if TYPE_CHECKING: + from dokimi.novelty.client import SpendCap + +#: The JSONL file a sweep writes unless ``--jsonl`` names another. +DEFAULT_JSONL = "sweep.jsonl" + + +@dataclass(frozen=True) +class _Item: + """One skill of the root: its key (its directory relative to ROOT), SKILL.md, digest and load error.""" + + key: str + name: str + skill_md: Path + sha256: Optional[str] + error: Optional[str] = None # a SKILL.md that does not load: its row is an error, and no call is made + + @property + def directory(self) -> Path: + return self.skill_md.parent + + +def _items(root: Path, limit: Optional[int]) -> list[_Item]: + """The skills under *root* in path order, the first *limit* of them.""" + from dokimi.skills.loader import load_skill, skill_files + from dokimi.utils import tree_digest + + items: list[_Item] = [] + for skill_md in skill_files(root): + directory = skill_md.parent + error: Optional[str] = None + try: + name = load_skill(skill_md).name + except (OSError, ValueError, RecursionError) as exc: + name, error = directory.name, str(exc) + try: + sha: Optional[str] = tree_digest(directory, tracked={}) + except OSError: + sha = None + items.append(_Item(key=directory.relative_to(root).as_posix(), name=name, skill_md=skill_md, + sha256=sha, error=error)) + if limit is not None and len(items) == limit: + break + return items + + +def _dirname(key: str) -> str: + """A directory name for one skill's outputs: readable, and unique even when two keys read alike.""" + readable = re.sub(r"[^A-Za-z0-9._-]+", "_", key).strip("_") or "root" + return f"{readable}-{hashlib.sha256(key.encode('utf-8')).hexdigest()[:8]}" + + +def _fingerprint(args: argparse.Namespace) -> dict[str, Any]: + """The settings a measured row was made under; a resumed run keeps a row only when these are unchanged.""" + from dokimi.commands.novelty import _settings + from dokimi.novelty import cost + + fingerprint = { + "roles": {role: list(pair) for role, pair in cost.role_models(args).items()}, + "settings": dict(_settings(args).values), + "arms": args.arms, + "budgets": list(args.budgets), + "max_atoms": args.max_atoms, + "seed": args.seed, + "max_tokens": args.max_tokens, + "context_framing": args.context_framing, + "include_resources": args.include_resources, + "resource_roles": args.resource_roles, + "no_paraphrase": args.no_paraphrase, + "no_reversal": args.no_reversal, + } + return json.loads(json.dumps(fingerprint, default=str)) + + +def _analyze_skill(args: argparse.Namespace, placebo_text: Optional[str], cache_dir: Optional[Path]): + """One skill through the analyze pipeline, as ``cmd_novelty_analyze`` runs it, without the writes. + + Returns the :class:`~dokimi.novelty.analyze.AnalyzeResult`; the caller decides what to save. + """ + from dokimi.commands.novelty import _arms, _bundle_from_args, _settings, stamp + from dokimi.novelty import cost + from dokimi.novelty.analyze import AnalyzeConfig, analyze + + bundle = _bundle_from_args(args) + settings = _settings(args) + extractor, answerer, judge = cost.clients(args) + config = AnalyzeConfig( + artifact_ref=args.artifact, artifact_kind=bundle.kind, name=bundle.name, + description=bundle.description, frontmatter=bundle.frontmatter, arms=_arms(args.arms), + budgets=tuple(args.budgets), concurrency=args.concurrency, + allow_self_judge=args.allow_self_judge, placebo_text=placebo_text, + context_framing=args.context_framing, cache_dir=cache_dir, seed=args.seed, + include_resources=args.include_resources, + resource_roles=tuple(args.resource_roles) if args.resource_roles else None, + **settings.config_fields(), + ) + result = analyze(bundle.text, config, extractor=extractor, answerer=answerer, judge=judge, + artifact_bundle=bundle) + if not result.partial: + stamp(result.artifact, args, result.measure_result, answerer=answerer, judge=judge) + return result + + +class _Sink: + """The JSONL file: one flushed line per row. A resumed run rewrites it with the kept rows first.""" + + def __init__(self, path: Path, kept: list[dict], resume: bool) -> None: + self.path = path + try: + if resume: + tmp = path.with_name(path.name + ".part") + with tmp.open("w", encoding="utf-8") as fh: + for row in kept: + fh.write(json.dumps(row) + "\n") + tmp.replace(path) + self._fh = path.open("a" if resume else "w", encoding="utf-8") + except OSError as exc: + raise InputError(f"cannot write --jsonl {path}: {exc}") from exc + + def write(self, row: dict) -> None: + self._fh.write(json.dumps(row) + "\n") + self._fh.flush() + + def close(self) -> None: + self._fh.close() + + +def _run_sweep(items: list[_Item], *, jobs: int, review: Callable[[_Item], Optional[dict]], + write: Callable[[dict], None], out_of_budget: Callable[[], bool]) -> None: + """Review the items with up to *jobs* threads, handing each row to *write* as it finishes. + + ``review`` returns None for a skill it did not run (the budget was spent before it started); + no new skill is submitted once ``out_of_budget()`` holds. A worker's unexpected failure + propagates after the rows finished so far are written. + """ + pending = iter(items) + running: set = set() + with ThreadPoolExecutor(max_workers=jobs) as pool: + + def fill() -> None: + while len(running) < jobs * 2 and not out_of_budget(): + item = next(pending, None) + if item is None: + return + running.add(pool.submit(review, item)) + + fill() + while running: + finished, _ = wait(running, return_when=FIRST_COMPLETED) + for future in finished: + running.discard(future) + row = future.result() + if row is not None: + write(row) + fill() + + +def _review_one(item: _Item, *, args: argparse.Namespace, run_cap: "SpendCap", per_skill: Optional[float], + placebo_text: Optional[str], cache_dir: Optional[Path], out_dir: Path, + fingerprint: dict) -> Optional[dict]: + """The row of one skill: analyse it under its own cap and save its novelty file, or say why it has none.""" + from dokimi.artifacts import save_artifact + from dokimi.novelty.atoms import ExtractionError + from dokimi.novelty.client import NestedSpendCap, SpendCapReached + from dokimi.novelty.measure import SelfJudgeError + from dokimi.novelty.report import render_novelty_markdown + from dokimi.novelty.sweep import make_row, measured_fields + + if run_cap.spent_usd >= run_cap.limit_usd: + return None + cap = NestedSpendCap(limit_usd=per_skill if per_skill is not None else math.inf, parent=run_cap) + + def row(status: str, **fields: Any) -> dict: + return make_row(key=item.key, name=item.name, path=str(item.skill_md), + skill_sha256=item.sha256, status=status, settings=fingerprint, + version=__version__, usd=cap.spent_usd, calls=cap.calls, **fields) + + if item.error is not None: + return row("error", reason="not_loaded", message=item.error) + skill_args = copy.copy(args) + skill_args.artifact = f"skill:{item.directory}" + skill_args._spend_cap = cap # cost.spend_cap() hands this cap to every role client of the skill + try: + result = _analyze_skill(skill_args, placebo_text, cache_dir) + except SpendCapReached as stop: # a stop outside analyze's own partial record + return row("partial", reason=stop.reason, message=str(stop)) + except (ExtractionError, SelfJudgeError, ValueError, InputError, OSError) as exc: + return row("error", reason="error", message=str(exc)) + except Exception as exc: # noqa: BLE001 - one skill's failure is its row's error; the sweep goes on + return row("error", reason="unexpected", message=f"{type(exc).__name__}: {exc}") + if result.partial: + return row("partial", reason=result.partial["reason"], message=result.partial["message"]) + measured = measured_fields(result.artifact) + if measured["new_lenient"] is None: + return row("error", reason="no_pairs", message="no probe was paired, so there is no share") + novelty_json = out_dir / _dirname(item.key) / "novelty.json" + save_artifact("novelty", result.artifact, novelty_json) + novelty_json.with_suffix(".md").write_text(render_novelty_markdown(result.artifact), encoding="utf-8") + return row("ok", measured=measured, novelty_json=str(novelty_json)) + + +def _emit_summary(summary: dict, args: argparse.Namespace) -> None: + for line in summary_lines(summary): + print(line) + if args.summary_out: + Path(args.summary_out).write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(f"summary written to {args.summary_out}") + + +def cmd_catalog_novelty(args: argparse.Namespace) -> None: + from dokimi.commands.inputs import placebo_text_for_arms_or_exit + from dokimi.commands.novelty import _arms + from dokimi.commands.novelty_run import usable_cache_dir + from dokimi.novelty.client import SpendCap + from dokimi.novelty.sweep import load_strata + + root = Path(args.root) + if not root.is_dir(): + raise InputError(f"ROOT is not a directory: {root}") + strata = load_strata(Path(args.strata), args.stratum_column) if args.strata else None + out_dir = Path(args.out_dir) + jsonl = Path(args.jsonl) if args.jsonl else out_dir / DEFAULT_JSONL + if args.summary_only: + _summary_only(jsonl, strata, args) + return + _refuse_unsafe_run(args, jsonl) + placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) + cache_dir = usable_cache_dir(args.cache_dir) + items = _items(root, args.limit) + if not items: + raise InputError(f"no SKILL.md under {root}") + fingerprint = _fingerprint(args) + kept, todo, others = _plan(items, jsonl, args, fingerprint) + print(f"# catalog novelty: {len(items)} skills under {root}; {len(kept)} measured and kept, " + f"{len(todo)} to run", file=sys.stderr) + run_cap = SpendCap(limit_usd=args.max_usd) + per = f", ${args.max_usd_per_skill:g} per skill" if args.max_usd_per_skill else "" + print(f"# budget ${args.max_usd:g} for the whole run{per}", file=sys.stderr) + out_dir.mkdir(parents=True, exist_ok=True) + new_rows = _sweep_rows(todo, args=args, run_cap=run_cap, kept=kept, others=others, total=len(items), + jsonl=jsonl, + out_dir=out_dir, placebo_text=placebo_text, cache_dir=cache_dir, + fingerprint=fingerprint) + _emit_summary(summarize(kept + new_rows, strata, resamples=args.bootstrap_resamples), args) + + +def _summary_only(jsonl: Path, strata, args: argparse.Namespace) -> None: + rows = read_rows(jsonl) + if not rows: + raise InputError(f"--jsonl {jsonl} has no rows to summarise") + _emit_summary(summarize(rows, strata, resamples=args.bootstrap_resamples), args) + + +def _refuse_unsafe_run(args: argparse.Namespace, jsonl: Path) -> None: + """Refuse a paid run that would not be a clean one: a judge2 it cannot run, no budget, + an overwrite of a sweep in progress, or a model flag that does not resolve.""" + from dokimi.novelty import cost + + if args.judge2: + raise InputError("--judge2 is not part of catalog novelty: run it on one skill with skill novelty analyze") + if args.max_usd is None: + raise InputError("catalog novelty makes paid calls: pass --max-usd, the whole-run budget " + "(--max-usd-per-skill caps one skill inside it)") + if jsonl.exists() and not args.resume: + raise InputError(f"--jsonl {jsonl} exists: pass --resume to continue it, or remove it to start over") + try: + cost.role_models(args) + except ValueError as exc: + raise InputError(str(exc)) from exc + + +def _plan(items: list[_Item], jsonl: Path, args: argparse.Namespace, + fingerprint: dict[str, Any]) -> tuple[list[dict], list[_Item], list[dict]]: + """The measured rows a resumed run keeps, the skills it still has to run, and the prior rows of + skills this run does not select (a narrower --limit, a moved skill): those stay in the file as they are.""" + selected = {item.key: item.sha256 for item in items} + prior = read_rows(jsonl) if args.resume else [] + kept = resumable(prior, selected, fingerprint, __version__) + done = {row["key"] for row in kept} + others = [row for row in prior if row["key"] not in selected] + return kept, [item for item in items if item.key not in done], others + + +def _sweep_rows(todo: list[_Item], *, args: argparse.Namespace, run_cap, kept: list[dict], + others: list[dict], total: int, jsonl: Path, out_dir: Path, placebo_text: Optional[str], cache_dir: Optional[Path], + fingerprint: dict[str, Any]) -> list[dict]: + """Run the skills still to do, writing each row as it finishes; returns the rows this run made.""" + sink = _Sink(jsonl, others + kept, resume=args.resume) + review = _bound(args, run_cap=run_cap, placebo_text=placebo_text, cache_dir=cache_dir, + out_dir=out_dir, fingerprint=fingerprint) + new_rows: list[dict] = [] + + def write(row: dict) -> None: + sink.write(row) + new_rows.append(row) + detail = f" new {row['new_lenient']:.2f}" if row["status"] == "ok" else f" {row['reason']}" + print(f"[{len(kept) + len(new_rows)}/{total}] {row['key']}: {row['status']}{detail} " + f"${row['usd']:.3f}", file=sys.stderr) + + def out_of_budget() -> bool: + return run_cap.spent_usd >= run_cap.limit_usd + + try: + _run_sweep(todo, jobs=args.jobs, review=review, write=write, out_of_budget=out_of_budget) + finally: + sink.close() + not_run = len(todo) - len(new_rows) + if not_run: + print(f"# {not_run} skill(s) not run: the budget was spent. Rerun with --resume and a higher " + "--max-usd; set DOKIMI_RESPONSE_CACHE so the calls already paid replay at $0", file=sys.stderr) + return new_rows + + +def _bound(args: argparse.Namespace, **shared: Any) -> Callable[[_Item], Optional[dict]]: + """The per-skill review with this run's shared pieces bound: the arguments, the budget and the output dir.""" + per_skill = args.max_usd_per_skill + + def review(item: _Item) -> Optional[dict]: + return _review_one(item, args=args, per_skill=per_skill, **shared) + + return review + + +def register(catalog_sub, positive_int) -> None: + """Add ``catalog novelty``: the corpus sweep, a catalog verb like ``catalog routing``.""" + from dokimi.commands.novelty import _add_bundle_args, _add_model_args + from dokimi.commands.novelty_run import usd_amount + from dokimi.novelty.sweep import RESAMPLES + from dokimi.novelty.tiers import TIERS + + p = catalog_sub.add_parser( + "novelty", + help="Corpus novelty sweep: analyze every skill under a catalog root under a budget, resumable, " + "with bootstrap CIs per stratum [paid]", + ) + p.add_argument("root", help="Catalog root: every SKILL.md under it is one skill") + p.add_argument("--out-dir", required=True, dest="out_dir", + help="Where each skill's novelty.json and report go (OUT_DIR//)") + p.add_argument("--jsonl", default=None, + help=f"One row per skill, written as each finishes (default: OUT_DIR/{DEFAULT_JSONL})") + p.add_argument("--jobs", type=positive_int, default=1, + help="Skills analysed in parallel threads, sharing --max-usd (default 1)") + p.add_argument("--resume", action="store_true", + help="Keep the measured rows in --jsonl and run only the other skills") + p.add_argument("--max-usd-per-skill", type=usd_amount, default=None, dest="max_usd_per_skill", + help="Stop one skill once its calls cost this much (USD); the rest of the sweep goes on") + p.add_argument("--limit", type=positive_int, default=None, + help="Only the first N skills in path order (a pilot)") + p.add_argument("--strata", default=None, + help="CSV with a `skill` column (the skill's directory under ROOT, or its name) and a " + "stratum column; the summary gets bootstrap CIs per stratum") + p.add_argument("--stratum-column", default="stratum", dest="stratum_column", + help="The stratum column of --strata (default: stratum)") + p.add_argument("--summary-only", action="store_true", dest="summary_only", + help="Print the summary of --jsonl without running anything") + p.add_argument("--summary-out", default=None, dest="summary_out", + help="Also write the summary as JSON here") + p.add_argument("--bootstrap-resamples", type=positive_int, default=RESAMPLES, + dest="bootstrap_resamples", help=f"Bootstrap resamples per interval (default {RESAMPLES})") + p.add_argument("--tier", default=None, choices=sorted(TIERS), + help="Named cost tier for every skill (see skill novelty analyze --tier)") + p.add_argument("--budgets", type=float, nargs="+", default=[0.75, 0.5], + help="Target token fractions for shortening candidates") + p.add_argument("--max-atoms", type=positive_int, default=None, dest="max_atoms", + help="Atom cap per skill (see skill novelty analyze)") + p.add_argument("--seed", type=int, default=0, help="Seed for the controls and the selection, per skill") + p.add_argument("--cache-dir", default=None, dest="cache_dir", + help="Disk cache for retention adjudications") + _add_model_args(p, positive_int) # --model, --judge-model, ... and --max-usd: the whole-run budget + _add_bundle_args(p, positive_int) + p.set_defaults(func=stops_at_cap(cmd_catalog_novelty)) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index e625d0b5a..c63d1df04 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -127,6 +127,28 @@ def record(self, usage: UsageRecord | None, model: str) -> None: self.spent_usd += usage.cost_usd +@dataclass +class NestedSpendCap(SpendCap): + """A cap for one unit of a larger run (one skill of a sweep), that also charges ``parent``. + + Every call is checked against both limits and recorded on both, so a sweep's whole-run + budget holds across the skills running at once. The parent is charged first: a call the + parent refuses or cannot price never reaches this cap's own count. + """ + + parent: SpendCap | None = None + + def before_call(self, model: str) -> None: + super().before_call(model) + if self.parent is not None: + self.parent.before_call(model) + + def record(self, usage: UsageRecord | None, model: str) -> None: + if self.parent is not None: + self.parent.record(usage, model) + super().record(usage, model) + + @dataclass class LLMClient: """One model in one role, with its usage recorded. diff --git a/dokimi/dokimi/novelty/sweep.py b/dokimi/dokimi/novelty/sweep.py new file mode 100644 index 000000000..29f61f219 --- /dev/null +++ b/dokimi/dokimi/novelty/sweep.py @@ -0,0 +1,217 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The corpus novelty sweep's bookkeeping: one JSONL row per skill, resume, strata and the summary. + +No model call and no CLI here: :mod:`dokimi.commands.catalog_novelty` runs the paid +``analyze`` per skill and writes the rows this module builds. The per-skill measure is the +share the model did not already know, ``1 - k0.lenient`` (the lenient K0 of the novelty +artifact, the ``new`` column of the S2/S3 summaries); a skill's CI is the bootstrap of that +share over the skills of a stratum (:func:`dokimi.eval.stats.bootstrap_ci`, the one resampler). +""" + +from __future__ import annotations + +import csv +import json +import statistics +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any, Optional + +from dokimi.utils import InputError + +#: The statuses a row can have. Only ``ok`` rows are measured and kept by a resumed run. +STATUSES = ("ok", "partial", "error") +#: The stratum of a skill the strata file does not name. +UNASSIGNED = "(unassigned)" +#: The stratum name when no strata file is given. +ALL = "all" +#: Bootstrap resamples of a stratum interval; fixed so two summaries of one JSONL agree. +RESAMPLES = 2000 + + +def new_share(known: Optional[float]) -> Optional[float]: + """The share of the skill's content the model did not already know: ``1 - known``.""" + return None if known is None else round(1.0 - known, 6) + + +def measured_fields(artifact: dict) -> dict[str, Any]: + """The per-skill numbers of a finished novelty artifact, read from its ``k0`` and ``paired`` blocks.""" + k0 = artifact.get("k0") or {} + lenient = (artifact.get("paired") or {}).get("lenient") or {} + known = k0.get("lenient") + return { + "tokens": (artifact.get("artifact") or {}).get("tokens"), + "known_lenient": known, + "new_lenient": new_share(known), + "new_strict": new_share(k0.get("strict")), + "n_paired": lenient.get("n_paired"), + } + + +def make_row(*, key: str, name: str, path: str, skill_sha256: Optional[str], status: str, + settings: dict, version: str, usd: float = 0.0, calls: int = 0, + reason: Optional[str] = None, message: Optional[str] = None, + measured: Optional[dict] = None, novelty_json: Optional[str] = None) -> dict: + """One skill's row: its identity and digest, what the run paid, and its measure when it has one.""" + if status not in STATUSES: + raise ValueError(f"unknown sweep status {status!r}") + row: dict[str, Any] = { + "key": key, "name": name, "path": path, "skill_sha256": skill_sha256, "status": status, + "reason": reason, "message": message, "usd": round(usd, 6), "llm_calls": calls, + "novelty_json": novelty_json, "settings": settings, "dokimi_version": version, + "tokens": None, "known_lenient": None, "new_lenient": None, "new_strict": None, + "n_paired": None, + } + if measured is not None: + row.update(measured) + return row + + +def load_strata(path: Path, column: str) -> dict[str, str]: + """The skill -> stratum map of a CSV with a ``skill`` column and a *column* column. + + ``skill`` is the skill's key under the catalog root (its directory, relative, with ``/``) or + its name; a skill is matched by key first, then by name. A skill named twice with two strata is an error. + """ + if column == "skill": + raise InputError("--stratum-column must not be `skill`: that column names the skill") + try: + with path.open(newline="", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + fields = reader.fieldnames or [] + if "skill" not in fields or column not in fields: + raise InputError(f"--strata {path} needs a `skill` column and a `{column}` column; " + f"found {', '.join(fields) or 'none'}") + out: dict[str, str] = {} + for lineno, rec in enumerate(reader, start=2): + skill = (rec.get("skill") or "").strip() + stratum = (rec.get(column) or "").strip() + if not skill or not stratum: + raise InputError(f"--strata {path} line {lineno}: `skill` and `{column}` must both be set") + if out.get(skill, stratum) != stratum: + raise InputError(f"--strata {path} line {lineno}: {skill!r} is in two strata " + f"({out[skill]!r} and {stratum!r})") + out[skill] = stratum + except (OSError, UnicodeDecodeError, csv.Error) as exc: + raise InputError(f"cannot read --strata {path}: {exc}") from exc + return out + + +def stratum_of(row: Mapping[str, Any], strata: Optional[Mapping[str, str]]) -> str: + """The stratum of a row: ``all`` without a strata map, else the mapped stratum (by key, then name).""" + if strata is None: + return ALL + return strata.get(row["key"]) or strata.get(row["name"]) or UNASSIGNED + + +def read_rows(path: Path) -> list[dict]: + """The rows of a sweep JSONL file. A last line a killed run left half-written is dropped.""" + rows: list[dict] = [] + try: + with path.open("rb") as fh: + lines = fh.readlines() + except FileNotFoundError: + return rows + except OSError as exc: + raise InputError(f"cannot read --jsonl {path}: {exc}") from exc + for lineno, raw in enumerate(lines, 1): + if not raw.strip(): + continue + try: + row = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + if lineno == len(lines) and not raw.endswith(b"\n"): + break + raise InputError(f"--jsonl {path} line {lineno} is not a sweep row: {exc}") from exc + if not isinstance(row, dict) or row.get("status") not in STATUSES or "key" not in row: + raise InputError(f"--jsonl {path} line {lineno} is not a sweep row; fix or remove it before --resume") + rows.append(row) + return rows + + +def resumable(prior: Iterable[dict], selected: Mapping[str, Optional[str]], settings: dict, + version: str) -> list[dict]: + """The prior rows a resumed run keeps: measured, for a skill still selected with the same + directory digest, under the same settings and dokimi version. Every other row is run again.""" + kept = [] + for row in prior: + if (row["status"] == "ok" and row["key"] in selected and selected[row["key"]] is not None + and row.get("skill_sha256") == selected[row["key"]] + and row.get("settings") == settings and row.get("dokimi_version") == version): + kept.append(row) + return kept + + +def group_stats(rows: list[dict], *, resamples: int = RESAMPLES) -> dict[str, Any]: + """The n, the new-share and already-known-share intervals (bootstrap), the spend and the median size of *rows*.""" + from dokimi.eval.stats import bootstrap_ci + + new = bootstrap_ci([r["new_lenient"] for r in rows], n_resamples=resamples) + strict_values = [r["new_strict"] for r in rows if r.get("new_strict") is not None] + strict = bootstrap_ci(strict_values, n_resamples=resamples) if strict_values else None + known = {"mean": round(1 - new["mean"], 4), "lo": round(1 - new["hi"], 4), + "hi": round(1 - new["lo"], 4)} + tokens = [r["tokens"] for r in rows if r.get("tokens")] + return { + "n": len(rows), + "new_lenient": new, + "known_lenient": known, + "new_strict": strict, + "usd": round(sum(r["usd"] for r in rows), 6), + "median_tokens": statistics.median(tokens) if tokens else None, + } + + +def summarize(rows: list[dict], strata: Optional[Mapping[str, str]] = None, *, + resamples: int = RESAMPLES) -> dict[str, Any]: + """Counts by status, the spend, and the measure with bootstrap CIs overall and per stratum. + + Only ``ok`` rows with a measure are summarised; ``partial`` and ``error`` rows are counted and + their spend is in ``usd``. With *strata* the per-stratum table is given, else only ``overall``. + """ + counts = {status: 0 for status in STATUSES} + for row in rows: + counts[row["status"]] += 1 + measured = [r for r in rows if r["status"] == "ok" and r["new_lenient"] is not None] + summary: dict[str, Any] = { + "counts": counts, + "usd": round(sum(r["usd"] for r in rows), 6), + "overall": group_stats(measured, resamples=resamples) if measured else None, + "strata": None, + } + if strata is not None: + groups: dict[str, list[dict]] = {} + for row in measured: + groups.setdefault(stratum_of(row, strata), []).append(row) + summary["strata"] = {name: group_stats(groups[name], resamples=resamples) + for name in sorted(groups)} + return summary + + +def _interval(block: Optional[dict]) -> str: + if not block or block["mean"] is None: + return "-" + return f"{block['mean']:.2f} [{block['lo']:.2f}, {block['hi']:.2f}]" + + +def summary_lines(summary: dict[str, Any]) -> list[str]: + """The summary as text: the already-known share first, then one line per stratum.""" + counts = summary["counts"] + head = (f"# novelty sweep: {counts['ok']} measured, {counts['partial']} partial, " + f"{counts['error']} error; ${summary['usd']:.2f} spent") + lines = [head] + overall = summary["overall"] + if overall is None: + lines.append("no skill was measured: no share to report") + return lines + lines.append(f"already known by the model: {_interval(overall['known_lenient'])} " + f"(new to it: {_interval(overall['new_lenient'])}; 95% bootstrap CI, n={overall['n']})") + if summary["strata"] is not None: + lines.append(f"{'stratum':<16} {'n':>4} {'new (lenient)':<22} {'already known':<22} {'strict new':<22} {'$':>7}") + for name, block in summary["strata"].items(): + lines.append(f"{name[:16]:<16} {block['n']:>4} {_interval(block['new_lenient']):<22} " + f"{_interval(block['known_lenient']):<22} {_interval(block['new_strict']):<22} " + f"{block['usd']:>7.2f}") + return lines diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index fbf2192fa..1b2e959e7 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -263,5 +263,11 @@ --alpha # repo security: the one-sided Fisher level per violation class, no registry concept yet (#360) --canary # repo security: a planted fake secret to look for, repeatable (#360) --canaries-file # repo security: the same canaries one per line (#360) +--max-usd-per-skill # catalog novelty: the cap of one skill of a sweep; --max-usd-per-cell caps a repo-run cell, another unit --tool # skill novelty ground (#472): the command whose --help is the source; no registry concept yet --tool-version # skill novelty ground (#472): recorded with the help snapshot; no registry concept yet +--strata # catalog novelty: the CSV that maps skills to strata; no registry concept yet +--stratum-column # catalog novelty: the column of --strata that names the stratum; no registry concept yet +--summary-only # catalog novelty: print a JSONL's summary with no model call; no registry concept yet +--summary-out # catalog novelty: write the summary as JSON; no registry concept yet +--bootstrap-resamples # catalog novelty: resamples per bootstrap interval; no registry concept yet diff --git a/dokimi/tests/test_caller_provider_choices.py b/dokimi/tests/test_caller_provider_choices.py index 9580cd074..dee7f7676 100644 --- a/dokimi/tests/test_caller_provider_choices.py +++ b/dokimi/tests/test_caller_provider_choices.py @@ -35,7 +35,8 @@ LLM_FLAGS = [ "eval questions analyze", "eval questions review", "eval score panel", "arms run", "arms matrix-run", "subjects run", "skill novelty analyze", "skill novelty atoms", - "skill novelty probe", "skill shorten", "skill routing", "catalog routing", "tools preference", + "skill novelty probe", "skill shorten", "skill routing", "catalog routing", "catalog novelty", + "tools preference", ] diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py new file mode 100644 index 000000000..890f2d9ea --- /dev/null +++ b/dokimi/tests/test_catalog_novelty.py @@ -0,0 +1,170 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`catalog novelty ROOT`: the paid sweep, run end to end on a fake LLM, offline.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.metrics.usage import UsageRecord +from dokimi.novelty import cost as novelty_cost +from dokimi.utils import InputError +from tests.test_novelty_analyze import SKILL, _scripted + +COST = 0.01 # every fake call is priced at this, so a budget is a number of calls + + +@pytest.fixture(autouse=True) +def _offline(monkeypatch): + monkeypatch.delenv("DOKIMI_RESPONSE_CACHE", raising=False) + base = _scripted("informed") + + def transport(prompt, system): + text, _ = base(prompt, system) + return text, UsageRecord(cost_usd=COST) + + real_client = novelty_cost.client + + def fake_client(*args, **kwargs): + client = real_client(*args, **kwargs) + client.transport = transport + return client + + monkeypatch.setattr(novelty_cost, "client", fake_client) + + +def _root(tmp_path, names=("accel-tuning", "other-skill")): + root = tmp_path / "catalog" + for name in names: + (root / name).mkdir(parents=True) + (root / name / "SKILL.md").write_text(SKILL.replace("accel-tuning", name), encoding="utf-8") + return root + + +def _sweep(root, out, *extra): + argv = ["catalog", "novelty", str(root), "--out-dir", str(out), + "--model", "answerer-m", "--judge-model", "judge-m", *extra] + args = build_parser().parse_args(argv) + args.func(args) + + +def _rows(path): + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def test_each_skill_gets_one_measured_row_and_a_novelty_file(tmp_path, capsys): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + rows = _rows(out / "sweep.jsonl") + assert sorted(r["key"] for r in rows) == ["accel-tuning", "other-skill"] + assert all(r["status"] == "ok" and r["new_lenient"] is not None for r in rows) + assert all(r["usd"] == pytest.approx(r["llm_calls"] * COST) for r in rows) + assert len(list(out.glob("*/novelty.json"))) == 2 + assert "# novelty sweep: 2 measured, 0 partial, 0 error" in capsys.readouterr().out + + +def test_the_summary_leads_with_the_already_known_share(tmp_path, capsys): + _sweep(_root(tmp_path), tmp_path / "out", "--max-usd", "5") + out = capsys.readouterr().out + assert "already known by the model:" in out + assert "new to it:" in out + + +def test_a_per_skill_cap_leaves_a_partial_row_and_the_sweep_goes_on(tmp_path): + out = tmp_path / "out" + _sweep(_root(tmp_path), out, "--max-usd", "5", "--max-usd-per-skill", "0.02") + rows = _rows(out / "sweep.jsonl") + assert [r["status"] for r in rows] == ["partial", "partial"] + assert {r["reason"] for r in rows} == {"spend_cap"} + assert not list(out.glob("*/novelty.json")) + + +def test_the_run_budget_stops_the_skills_it_cannot_pay_for(tmp_path, capsys): + out = tmp_path / "out" + _sweep(_root(tmp_path), out, "--max-usd", "0.05", "--jobs", "1") + rows = _rows(out / "sweep.jsonl") + assert len(rows) == 1 # the second skill was never started + assert rows[0]["status"] == "partial" + assert "1 skill(s) not run: the budget was spent" in capsys.readouterr().err + + +def test_resume_keeps_the_measured_rows_and_runs_only_the_rest(tmp_path, capsys): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5", "--limit", "1") + first = _rows(out / "sweep.jsonl") + assert len(first) == 1 + _sweep(root, out, "--max-usd", "5", "--resume") + rows = _rows(out / "sweep.jsonl") + assert sorted(r["key"] for r in rows) == ["accel-tuning", "other-skill"] + assert rows[0] == first[0] + assert "1 measured and kept, 1 to run" in capsys.readouterr().err + + +def test_resume_runs_again_a_skill_whose_directory_changed(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5", "--limit", "1") + before = _rows(out / "sweep.jsonl")[0]["skill_sha256"] + (root / "accel-tuning" / "SKILL.md").write_text(SKILL + "\nMore text.\n", encoding="utf-8") + _sweep(root, out, "--max-usd", "5", "--resume") + rows = _rows(out / "sweep.jsonl") + assert sorted(r["key"] for r in rows) == ["accel-tuning", "other-skill"] # the stale row is gone + changed = [r for r in rows if r["key"] == "accel-tuning"] + assert changed[0]["skill_sha256"] != before and changed[0]["status"] == "ok" + + +def test_an_existing_jsonl_is_not_overwritten_without_resume(tmp_path): + out = tmp_path / "out" + out.mkdir() + (out / "sweep.jsonl").write_text("", encoding="utf-8") + with pytest.raises(InputError, match="pass --resume"): + _sweep(_root(tmp_path), out, "--max-usd", "5") + + +def test_a_paid_sweep_needs_a_whole_run_budget(tmp_path): + with pytest.raises(InputError, match="pass --max-usd"): + _sweep(_root(tmp_path), tmp_path / "out") + + +def test_summary_only_reads_the_jsonl_and_makes_no_call(tmp_path, monkeypatch, capsys): + out, root = tmp_path / "out", _root(tmp_path) + _sweep(root, out, "--max-usd", "5") + capsys.readouterr() + + def no_client(*args, **kwargs): + raise AssertionError("summary-only must not build a client") + + monkeypatch.setattr(novelty_cost, "client", no_client) + _sweep(root, out, "--summary-only") + assert "# novelty sweep: 2 measured" in capsys.readouterr().out + + +def test_strata_add_a_line_per_stratum(tmp_path, capsys): + out = tmp_path / "out" + strata = tmp_path / "strata.csv" + strata.write_text("skill,stratum\naccel-tuning,ops\nother-skill,dev\n", encoding="utf-8") + _sweep(_root(tmp_path), out, "--max-usd", "5", "--strata", str(strata)) + text = capsys.readouterr().out + assert "stratum" in text and "\nops " in text and "\ndev " in text + + +def test_an_unloadable_skill_is_an_error_row_and_costs_nothing(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + (root / "broken").mkdir() + (root / "broken" / "SKILL.md").write_text("no frontmatter here\n", encoding="utf-8") + _sweep(root, out, "--max-usd", "5") + broken = [r for r in _rows(out / "sweep.jsonl") if r["key"] == "broken"] + assert broken and broken[0]["status"] == "error" + assert broken[0]["usd"] == 0 + + +def test_a_narrower_resume_keeps_the_paid_rows_of_the_skills_it_does_not_select(tmp_path): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + before = {r["key"]: r for r in _rows(out / "sweep.jsonl")} + _sweep(root, out, "--max-usd", "5", "--resume", "--limit", "1") + after = {r["key"]: r for r in _rows(out / "sweep.jsonl")} + assert after == before # other-skill is outside this run's --limit: its paid row stays diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 1588fdbfb..a9276dc4f 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -113,6 +113,7 @@ def test_help_lists_each_root_under_its_section(): ("explore", "run"), ("catalog", "trigger-probe"), ("catalog", "routing"), + ("catalog", "novelty"), ("skill", "novelty", "analyze"), ("skill", "novelty", "atoms"), ("skill", "novelty", "probe"), diff --git a/dokimi/tests/test_judge_provider_choices.py b/dokimi/tests/test_judge_provider_choices.py index 3f8afc15f..a7432ee25 100644 --- a/dokimi/tests/test_judge_provider_choices.py +++ b/dokimi/tests/test_judge_provider_choices.py @@ -30,7 +30,8 @@ def _judge_provider_actions(parser, path=()): #: Judge-role flags whose value never reaches ``Judge``: the novelty commands call #: the model through ``novelty.client.LLMClient`` -> ``llm_call_with_usage``, which #: does route ``openai-codex``. -NOT_THE_JUDGE_CLASS = {"skill novelty analyze --judge-provider", "skill novelty probe --judge-provider"} +NOT_THE_JUDGE_CLASS = {"skill novelty analyze --judge-provider", "skill novelty probe --judge-provider", + "catalog novelty --judge-provider"} def test_every_judge_provider_flag_offers_exactly_judge_providers(): diff --git a/dokimi/tests/test_judge_provider_default.py b/dokimi/tests/test_judge_provider_default.py index 78787ed03..efc834309 100644 --- a/dokimi/tests/test_judge_provider_default.py +++ b/dokimi/tests/test_judge_provider_default.py @@ -27,7 +27,8 @@ def _judge_commands() -> dict[str, list[str]]: def walk(parser: argparse.ArgumentParser, path: list[str]) -> None: dests = {a.dest for a in parser._actions} - if "judge_provider" in dests and path[0] != "novelty" and path[:2] != ["skill", "novelty"]: + if ("judge_provider" in dests and path[0] != "novelty" and path[:2] != ["skill", "novelty"] + and path[:2] != ["catalog", "novelty"]): argv = list(path) for action in parser._actions: if action.required: diff --git a/dokimi/tests/test_novelty_sweep.py b/dokimi/tests/test_novelty_sweep.py new file mode 100644 index 000000000..7c1007901 --- /dev/null +++ b/dokimi/tests/test_novelty_sweep.py @@ -0,0 +1,213 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The corpus sweep's bookkeeping: rows, resume, strata and the summary. No model call.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.novelty.client import NestedSpendCap, SpendCap, SpendCapReached +from dokimi.metrics.usage import UsageRecord +from dokimi.novelty.sweep import ( + ALL, UNASSIGNED, group_stats, load_strata, make_row, measured_fields, new_share, + read_rows, resumable, stratum_of, summarize, summary_lines, +) +from dokimi.utils import InputError + +SETTINGS = {"max_atoms": 120} + + +def _ok(key, new, usd=0.1, tokens=800, name=None, digest="d1"): + measured = {"tokens": tokens, "known_lenient": round(1 - new, 6), "new_lenient": new, + "new_strict": round(min(1.0, new + 0.1), 6), "n_paired": 4} + return make_row(key=key, name=name or key, path=f"/c/{key}/SKILL.md", skill_sha256=digest, + status="ok", settings=SETTINGS, version="1.0", usd=usd, calls=3, + measured=measured, novelty_json=f"/out/{key}/novelty.json") + + +def _row(key, status, **kw): + return make_row(key=key, name=key, path=f"/c/{key}/SKILL.md", skill_sha256="d1", + status=status, settings=SETTINGS, version="1.0", usd=kw.pop("usd", 0.05), **kw) + + +def test_new_share_is_one_minus_the_already_known_share(): + assert new_share(0.66) == pytest.approx(0.34) + assert new_share(None) is None + + +def test_measured_fields_read_the_lenient_and_strict_k0_and_the_pairs(): + artifact = {"k0": {"lenient": 0.25, "strict": 0.1}, "paired": {"lenient": {"n_paired": 4}}, + "artifact": {"tokens": 900}} + fields = measured_fields(artifact) + assert fields["known_lenient"] == 0.25 + assert fields["new_lenient"] == pytest.approx(0.75) + assert fields["new_strict"] == pytest.approx(0.9) + assert (fields["n_paired"], fields["tokens"]) == (4, 900) + + +def test_measured_fields_of_an_artifact_without_k0_has_no_share(): + assert measured_fields({})["new_lenient"] is None + + +def test_an_unpriced_call_charged_through_a_child_stops_the_parent(): + parent = SpendCap(limit_usd=10.0) + child = NestedSpendCap(limit_usd=1.0, parent=parent) + child.record(UsageRecord(cost_usd=0.4), "m") + assert (child.spent_usd, parent.spent_usd) == (0.4, 0.4) + with pytest.raises(SpendCapReached, match="no price"): + child.record(None, "m") + with pytest.raises(SpendCapReached, match="no known price"): + parent.before_call("m") + + +def test_the_child_cap_checks_its_own_limit_as_well_as_the_parent(): + parent = SpendCap(limit_usd=10.0) + child = NestedSpendCap(limit_usd=0.5, parent=parent) + child.record(UsageRecord(cost_usd=0.5), "m") + with pytest.raises(SpendCapReached, match="spend cap reached"): + child.before_call("m") + assert parent.spent_usd == 0.5 + + +def test_the_parent_refusal_stops_a_child_that_still_has_room(): + parent = SpendCap(limit_usd=0.5) + child = NestedSpendCap(limit_usd=5.0, parent=parent) + parent.record(UsageRecord(cost_usd=0.5), "m") + with pytest.raises(SpendCapReached): + child.before_call("m") + + +def test_load_strata_maps_each_skill_to_its_stratum(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,stratum\nops/deploy,ops\nreview,dev\n", encoding="utf-8") + assert load_strata(path, "stratum") == {"ops/deploy": "ops", "review": "dev"} + + +def test_load_strata_refuses_a_missing_column(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,tier\nreview,dev\n", encoding="utf-8") + with pytest.raises(InputError, match="needs a `skill` column and a `stratum` column"): + load_strata(path, "stratum") + + +def test_load_strata_refuses_skill_as_the_stratum_column(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,skill\nreview,dev\n", encoding="utf-8") + with pytest.raises(InputError, match="must not be `skill`"): + load_strata(path, "skill") + + +def test_load_strata_refuses_a_skill_in_two_strata(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,stratum\nreview,dev\nreview,ops\n", encoding="utf-8") + with pytest.raises(InputError, match="is in two strata"): + load_strata(path, "stratum") + + +def test_load_strata_refuses_an_empty_cell(tmp_path): + path = tmp_path / "strata.csv" + path.write_text("skill,stratum\nreview,\n", encoding="utf-8") + with pytest.raises(InputError, match="must both be set"): + load_strata(path, "stratum") + + +def test_stratum_of_matches_by_key_then_name_and_falls_back_to_unassigned(): + strata = {"ops/deploy": "ops", "review": "dev"} + assert stratum_of({"key": "ops/deploy", "name": "deploy"}, strata) == "ops" + assert stratum_of({"key": "x/review", "name": "review"}, strata) == "dev" + assert stratum_of({"key": "other", "name": "other"}, strata) == UNASSIGNED + assert stratum_of({"key": "other", "name": "other"}, None) == ALL + + +def test_read_rows_drops_a_last_line_a_killed_run_left_half_written(tmp_path): + path = tmp_path / "sweep.jsonl" + path.write_text(json.dumps(_ok("a", 0.3)) + "\n" + '{"key": "b", "sta', encoding="utf-8") + rows = read_rows(path) + assert [r["key"] for r in rows] == ["a"] + + +def test_read_rows_refuses_a_bad_line_that_is_not_the_last(tmp_path): + path = tmp_path / "sweep.jsonl" + path.write_text('{"key": "b", "sta\n' + json.dumps(_ok("a", 0.3)) + "\n", encoding="utf-8") + with pytest.raises(InputError, match="line 1 is not a sweep row"): + read_rows(path) + + +def test_read_rows_refuses_a_line_that_is_not_a_row(tmp_path): + path = tmp_path / "sweep.jsonl" + path.write_text(json.dumps({"hello": 1}) + "\n", encoding="utf-8") + with pytest.raises(InputError, match="is not a sweep row"): + read_rows(path) + + +def test_read_rows_of_a_missing_file_is_empty(tmp_path): + assert read_rows(tmp_path / "absent.jsonl") == [] + + +def test_resumable_keeps_only_measured_rows_for_the_same_skill_digest_settings_and_version(): + good = _ok("keep", 0.3) + changed_skill = _ok("changed", 0.3, digest="old") + errored = _row("errored", "error") + partial = _row("partial", "partial") + gone = _ok("gone", 0.3) + old_version = dict(_ok("version", 0.3), dokimi_version="0.9") + prior = [good, changed_skill, errored, partial, gone, old_version] + selected = {"keep": "d1", "changed": "d2", "errored": "d1", "partial": "d1", "version": "d1"} + kept = resumable(prior, selected, SETTINGS, "1.0") + assert [r["key"] for r in kept] == ["keep"] + assert resumable(prior, selected, {"max_atoms": 90}, "1.0") == [] + + +def test_group_stats_bootstraps_the_new_share_and_flips_the_interval_for_the_known_share(): + rows = [_ok("a", 0.2), _ok("b", 0.4), _ok("c", 0.6)] + stats = group_stats(rows, resamples=300) + assert stats["n"] == 3 + assert stats["new_lenient"]["mean"] == pytest.approx(0.4, abs=1e-3) + known, new = stats["known_lenient"], stats["new_lenient"] + assert known["mean"] == pytest.approx(1 - new["mean"], abs=1e-3) + assert known["lo"] == pytest.approx(1 - new["hi"], abs=1e-3) + assert known["hi"] == pytest.approx(1 - new["lo"], abs=1e-3) + assert stats["usd"] == pytest.approx(0.3) + assert stats["median_tokens"] == 800 + + +def test_summarize_counts_every_status_but_measures_only_ok_rows(): + rows = [_ok("a", 0.2, usd=0.3), _row("b", "partial", usd=0.1), _row("c", "error", usd=0.0)] + summary = summarize(rows, resamples=200) + assert summary["counts"] == {"ok": 1, "partial": 1, "error": 1} + assert summary["usd"] == pytest.approx(0.4) + assert summary["overall"]["n"] == 1 + assert summary["strata"] is None + + +def test_summarize_splits_the_measure_by_stratum_and_names_the_unassigned(): + rows = [_ok("ops/deploy", 0.2), _ok("review", 0.6), _ok("misc", 0.5)] + summary = summarize(rows, {"ops/deploy": "ops", "review": "dev"}, resamples=200) + assert set(summary["strata"]) == {"dev", "ops", UNASSIGNED} + assert summary["strata"]["dev"]["n"] == 1 + assert summary["strata"]["dev"]["new_lenient"]["mean"] == pytest.approx(0.6) + + +def test_summary_lines_say_when_nothing_was_measured(): + lines = summary_lines(summarize([_row("a", "error")], resamples=50)) + assert lines[0].startswith("# novelty sweep: 0 measured, 0 partial, 1 error") + assert lines[1] == "no skill was measured: no share to report" + + +def test_summary_lines_lead_with_the_already_known_share(): + lines = summary_lines(summarize([_ok("a", 0.34)], resamples=50)) + assert lines[1].startswith("already known by the model: 0.66 [") + assert "new to it: 0.34" in lines[1] + + +def test_make_row_refuses_an_unknown_status(): + with pytest.raises(ValueError, match="unknown sweep status"): + _row("a", "done") + + +def test_resumable_never_keeps_a_row_whose_directory_could_not_be_digested(): + prior = [dict(_ok("gone-quiet", 0.3), skill_sha256=None)] + assert resumable(prior, {"gone-quiet": None}, SETTINGS, "1.0") == [] From d398bd05d88fe835161121104531ac695472f67c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 15:29:43 -0800 Subject: [PATCH 02/11] fix(catalog): no-pair skills are errors, a failed save is that skill's row - A skill with no paired probe read k0 0.0 and so a new share of 1.0 and status ok. A zero pair count is now the no_pairs error row. - A failed save of novelty.json escaped the per-skill try, ended the sweep and dropped the rows of the other in-flight skills. It is that skill's error row now; the paid analysis is already spent. - The resume fingerprint omits --max-file-words, --max-resource-words, --allow-self-judge and the placebo text, so a resume kept rows measured under other inputs. They are in it now. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/catalog_novelty.py | 18 ++++++--- dokimi/tests/test_catalog_novelty.py | 49 +++++++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index 1d9b5f4c2..800da26fe 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -85,7 +85,7 @@ def _dirname(key: str) -> str: return f"{readable}-{hashlib.sha256(key.encode('utf-8')).hexdigest()[:8]}" -def _fingerprint(args: argparse.Namespace) -> dict[str, Any]: +def _fingerprint(args: argparse.Namespace, placebo_text: Optional[str]) -> dict[str, Any]: """The settings a measured row was made under; a resumed run keeps a row only when these are unchanged.""" from dokimi.commands.novelty import _settings from dokimi.novelty import cost @@ -103,6 +103,10 @@ def _fingerprint(args: argparse.Namespace) -> dict[str, Any]: "resource_roles": args.resource_roles, "no_paraphrase": args.no_paraphrase, "no_reversal": args.no_reversal, + "allow_self_judge": args.allow_self_judge, + "max_file_words": args.max_file_words, + "max_resource_words": args.max_resource_words, + "placebo_text": hashlib.sha256(placebo_text.encode("utf-8")).hexdigest() if placebo_text else None, } return json.loads(json.dumps(fingerprint, default=str)) @@ -226,11 +230,15 @@ def row(status: str, **fields: Any) -> dict: if result.partial: return row("partial", reason=result.partial["reason"], message=result.partial["message"]) measured = measured_fields(result.artifact) - if measured["new_lenient"] is None: + # k0 reads 0.0 when no baseline probe answered, so a zero pair count is the signal, not the share. + if measured["new_lenient"] is None or not measured["n_paired"]: return row("error", reason="no_pairs", message="no probe was paired, so there is no share") novelty_json = out_dir / _dirname(item.key) / "novelty.json" - save_artifact("novelty", result.artifact, novelty_json) - novelty_json.with_suffix(".md").write_text(render_novelty_markdown(result.artifact), encoding="utf-8") + try: + save_artifact("novelty", result.artifact, novelty_json) + novelty_json.with_suffix(".md").write_text(render_novelty_markdown(result.artifact), encoding="utf-8") + except (OSError, ValueError) as exc: # the paid analysis is done: a failed write is this skill's error + return row("error", reason="error", message=f"could not save its novelty file: {exc}") return row("ok", measured=measured, novelty_json=str(novelty_json)) @@ -264,7 +272,7 @@ def cmd_catalog_novelty(args: argparse.Namespace) -> None: items = _items(root, args.limit) if not items: raise InputError(f"no SKILL.md under {root}") - fingerprint = _fingerprint(args) + fingerprint = _fingerprint(args, placebo_text) kept, todo, others = _plan(items, jsonl, args, fingerprint) print(f"# catalog novelty: {len(items)} skills under {root}; {len(kept)} measured and kept, " f"{len(todo)} to run", file=sys.stderr) diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index 890f2d9ea..af904a2d4 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -168,3 +168,52 @@ def test_a_narrower_resume_keeps_the_paid_rows_of_the_skills_it_does_not_select( _sweep(root, out, "--max-usd", "5", "--resume", "--limit", "1") after = {r["key"]: r for r in _rows(out / "sweep.jsonl")} assert after == before # other-skill is outside this run's --limit: its paid row stays + + +def test_a_skill_with_no_paired_probe_is_an_error_not_a_share_of_one(tmp_path, monkeypatch): + """The baseline arm answered nothing: k0 is 0.0 by default, so the new share reads 1.0 unless no pair is checked.""" + from types import SimpleNamespace + + import dokimi.artifacts as artifacts + import dokimi.commands.catalog_novelty as catalog_novelty + import dokimi.novelty.report as report + + artifact = {"k0": {"lenient": 0.0, "strict": 0.0}, "paired": {"lenient": {"n_paired": 0}}, + "artifact": {"tokens": 100}} + monkeypatch.setattr(catalog_novelty, "_analyze_skill", + lambda *a, **k: SimpleNamespace(partial=None, artifact=artifact)) + monkeypatch.setattr(artifacts, "save_artifact", lambda kind, art, path: path.parent.mkdir(parents=True)) + monkeypatch.setattr(report, "render_novelty_markdown", lambda artifact: "") + root, out = _root(tmp_path, names=("accel-tuning",)), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + [row] = _rows(out / "sweep.jsonl") + assert row["status"] == "error" and row["reason"] == "no_pairs" + assert row["new_lenient"] is None + + +def test_a_save_that_fails_is_that_skills_error_row_and_the_sweep_goes_on(tmp_path, monkeypatch): + import dokimi.artifacts as artifacts + + real_save = artifacts.save_artifact + + def save(kind, artifact, path, *a, **k): + if "accel-tuning" in str(path): + raise OSError("disk full") + return real_save(kind, artifact, path, *a, **k) + + monkeypatch.setattr(artifacts, "save_artifact", save) + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") # must not raise: one skill's save is its row's error + rows = {r["key"]: r for r in _rows(out / "sweep.jsonl")} + assert rows["accel-tuning"]["status"] == "error" + assert "disk full" in rows["accel-tuning"]["message"] + assert rows["other-skill"]["status"] == "ok" + + +@pytest.mark.parametrize("flag", [["--max-file-words", "100"], ["--max-resource-words", "100"]]) +def test_resume_runs_again_the_rows_measured_under_other_input_limits(tmp_path, capsys, flag): + root, out = _root(tmp_path), tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + capsys.readouterr() + _sweep(root, out, "--max-usd", "5", "--resume", *flag) + assert "0 measured and kept, 2 to run" in capsys.readouterr().err From 1ec8946bb215090ced6eed9e15d881f7c707dde8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 20:53:35 -0800 Subject: [PATCH 03/11] fix(catalog): an interrupt cancels the queued skills; identical copies documented Ctrl-C or SIGTERM during `catalog novelty` used to let the queued skills run, each with its paid calls, and their rows were dropped. Now the first signal cancels the skills not started (no new call), the skills in flight finish and keep their rows, and the run exits 130 (SIGINT) or 143 (SIGTERM). The rows written stay in the JSONL, so `--resume` continues. A second signal raises KeyboardInterrupt at once. Two directories whose SKILL.md files are identical stay one skill (the shallowest path is kept, no row for the copy); this is now stated in the command docs and --help, and pinned by a test. Signal handling is polled, not an async raise: a KeyboardInterrupt raised between submit and bookkeeping lost a finished row in a test, so the sweep checks a stop request every 0.2 s instead. Signed-off-by: Nikolay Petrov --- ...6_120000_napetrov_catalog-novelty-sweep.md | 3 +- dokimi/docs/skill-novelty.md | 7 + dokimi/dokimi/commands/catalog_novelty.py | 124 +++++++++++++++--- dokimi/tests/test_catalog_novelty.py | 70 ++++++++++ 4 files changed, 182 insertions(+), 22 deletions(-) diff --git a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md index e3637b01c..6deced0da 100644 --- a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md +++ b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md @@ -11,4 +11,5 @@ SPDX-License-Identifier: Apache-2.0 finishes. `--resume` keeps the measured rows whose skill digest and settings are unchanged, and an existing `--jsonl` is not overwritten without it. `--strata CSV` adds bootstrap 95% intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. See - `docs/skill-novelty.md`, Corpus sweep. + Ctrl-C or SIGTERM cancels the skills not yet started and keeps the rows done; identical + SKILL.md copies count once. See `docs/skill-novelty.md`, Corpus sweep. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 56e18291b..3f0ad687c 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1137,6 +1137,13 @@ each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets default `stratum`); a skill the CSV does not name is `(unassigned)`. - **Summary only.** `--summary-only --jsonl FILE` prints the summary of a finished or partial sweep with no model call. `--summary-out FILE` also writes it as JSON. +- **Identical copies.** Directories under ROOT whose `SKILL.md` files are byte-identical are one + skill: the shallowest path is kept, the other copies get no row and no call (not an error). +- **Interrupt.** Ctrl-C or SIGTERM stops the sweep without new calls: the skills not yet started + are cancelled, the skills in flight finish and keep their rows, and the run exits nonzero (130 + for SIGINT, 143 for SIGTERM). The rows already written stay in the JSONL, so `--resume` continues + from them. A second signal raises KeyboardInterrupt: the queued skills stay cancelled, but the + rows of the skills in flight are not written (their calls are paid, so set `DOKIMI_RESPONSE_CACHE`). ```bash dokimi catalog novelty skills/ --out-dir out/sweep --max-usd 20 --max-usd-per-skill 1 \ diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index 800da26fe..a62585968 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -8,7 +8,10 @@ a cap on each skill; ``--jobs`` runs skills in parallel threads that share the budget. Each skill writes one row to ``--jsonl`` as it finishes, so a killed sweep keeps its rows, and ``--resume`` runs only the skills without a measured row (same directory digest, same settings). -``--summary-only`` prints the summary of a JSONL file without any model call. +``--summary-only`` prints the summary of a JSONL file without any model call. Directories whose +``SKILL.md`` files are identical count as one skill (the shallowest path is kept). Ctrl-C or SIGTERM +cancels the skills not yet started, lets the running ones finish, keeps the rows done and exits nonzero +(a second signal raises KeyboardInterrupt and the rows in flight are not written). A skill the budget stops mid-analysis is a ``partial`` row and leaves no novelty file. Set ``DOKIMI_RESPONSE_CACHE`` so a resumed run replays the calls already paid for at $0. @@ -22,8 +25,11 @@ import json import math import re +import signal import sys +import threading from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional @@ -40,6 +46,47 @@ DEFAULT_JSONL = "sweep.jsonl" +class _Stop: + """A stop request: the signal (SIGINT or SIGTERM) that asked the sweep to stop, once it has.""" + + def __init__(self) -> None: + self.signum: Optional[int] = None + + @property + def requested(self) -> bool: + return self.signum is not None + + @property + def exit_code(self) -> int: + return 128 + (self.signum or signal.SIGINT) # the shell's code for a process a signal stopped + + +@contextmanager +def _interruptible(stop: _Stop): + """While the sweep runs, the first SIGINT or SIGTERM only sets *stop*: the sweep cancels the skills + not started and keeps the rows done. A second signal raises KeyboardInterrupt. The handlers are + restored after. Only the main thread can install them.""" + if threading.current_thread() is not threading.main_thread(): + yield + return + + def on_signal(signum: int, frame: object) -> None: + if stop.requested: + raise KeyboardInterrupt + stop.signum = signum + if signum == signal.SIGTERM: + from dokimi.process import kill_children + + kill_children() # what the CLI's SIGTERM handler does for the processes it started + + previous = {signum: signal.signal(signum, on_signal) for signum in (signal.SIGINT, signal.SIGTERM)} + try: + yield + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler if handler is not None else signal.SIG_DFL) + + @dataclass(frozen=True) class _Item: """One skill of the root: its key (its directory relative to ROOT), SKILL.md, digest and load error.""" @@ -164,34 +211,55 @@ def close(self) -> None: self._fh.close() +#: How often the sweep's main thread looks at the stop request while it waits for a skill. +POLL_SECONDS = 0.2 + + def _run_sweep(items: list[_Item], *, jobs: int, review: Callable[[_Item], Optional[dict]], - write: Callable[[dict], None], out_of_budget: Callable[[], bool]) -> None: + write: Callable[[dict], None], out_of_budget: Callable[[], bool], + stop_requested: Callable[[], bool]) -> None: """Review the items with up to *jobs* threads, handing each row to *write* as it finishes. ``review`` returns None for a skill it did not run (the budget was spent before it started); - no new skill is submitted once ``out_of_budget()`` holds. A worker's unexpected failure - propagates after the rows finished so far are written. + no new skill is submitted once ``out_of_budget()`` holds. A stop request (``stop_requested()``) + does the same, and cancels the skills queued but not started, so they make no call. The skills + in flight finish and their rows are written. A worker's unexpected failure cancels the queue the + same way and propagates after the rows finished so far are written. """ pending = iter(items) running: set = set() - with ThreadPoolExecutor(max_workers=jobs) as pool: + pool = ThreadPoolExecutor(max_workers=jobs) - def fill() -> None: - while len(running) < jobs * 2 and not out_of_budget(): - item = next(pending, None) - if item is None: - return - running.add(pool.submit(review, item)) + def fill() -> None: + while len(running) < jobs * 2 and not out_of_budget() and not stop_requested(): + item = next(pending, None) + if item is None: + return + running.add(pool.submit(review, item)) + try: fill() - while running: - finished, _ = wait(running, return_when=FIRST_COMPLETED) + while running and not stop_requested(): + finished, _ = wait(running, timeout=POLL_SECONDS, return_when=FIRST_COMPLETED) for future in finished: - running.discard(future) row = future.result() if row is not None: write(row) + running.discard(future) fill() + except BaseException: + _settle(pool, running, write) + raise + _settle(pool, running, write) + + +def _settle(pool: ThreadPoolExecutor, running: set, write: Callable[[dict], None]) -> None: + """Cancel the queued skills, wait for the ones in flight and keep the rows they finished.""" + pool.shutdown(wait=True, cancel_futures=True) + for future in running: + if not future.cancelled() and future.exception() is None and future.result() is not None: + write(future.result()) + running.clear() def _review_one(item: _Item, *, args: argparse.Namespace, run_cap: "SpendCap", per_skill: Optional[float], @@ -325,8 +393,8 @@ def _plan(items: list[_Item], jsonl: Path, args: argparse.Namespace, def _sweep_rows(todo: list[_Item], *, args: argparse.Namespace, run_cap, kept: list[dict], - others: list[dict], total: int, jsonl: Path, out_dir: Path, placebo_text: Optional[str], cache_dir: Optional[Path], - fingerprint: dict[str, Any]) -> list[dict]: + others: list[dict], total: int, jsonl: Path, out_dir: Path, placebo_text: Optional[str], + cache_dir: Optional[Path], fingerprint: dict[str, Any]) -> list[dict]: """Run the skills still to do, writing each row as it finishes; returns the rows this run made.""" sink = _Sink(jsonl, others + kept, resume=args.resume) review = _bound(args, run_cap=run_cap, placebo_text=placebo_text, cache_dir=cache_dir, @@ -343,14 +411,24 @@ def write(row: dict) -> None: def out_of_budget() -> bool: return run_cap.spent_usd >= run_cap.limit_usd + stop = _Stop() try: - _run_sweep(todo, jobs=args.jobs, review=review, write=write, out_of_budget=out_of_budget) + with _interruptible(stop): + _run_sweep(todo, jobs=args.jobs, review=review, write=write, out_of_budget=out_of_budget, + stop_requested=lambda: stop.requested) + except KeyboardInterrupt: # a second signal: stop at once, the rows written so far are kept + stop.signum = stop.signum or signal.SIGINT finally: sink.close() not_run = len(todo) - len(new_rows) if not_run: - print(f"# {not_run} skill(s) not run: the budget was spent. Rerun with --resume and a higher " - "--max-usd; set DOKIMI_RESPONSE_CACHE so the calls already paid replay at $0", file=sys.stderr) + print(f"# {not_run} skill(s) not run: the budget was spent or the run was stopped. Rerun with " + "--resume (and a higher --max-usd if the budget was spent); set DOKIMI_RESPONSE_CACHE so the " + "calls already paid replay at $0", file=sys.stderr) + if stop.requested: + print(f"# stopped by signal {stop.signum}: queued skills were cancelled, {len(new_rows)} row(s) " + f"written to {jsonl}; rerun with --resume to continue", file=sys.stderr) + raise SystemExit(stop.exit_code) return new_rows @@ -376,11 +454,15 @@ def register(catalog_sub, positive_int) -> None: help="Corpus novelty sweep: analyze every skill under a catalog root under a budget, resumable, " "with bootstrap CIs per stratum [paid]", ) - p.add_argument("root", help="Catalog root: every SKILL.md under it is one skill") + p.add_argument("root", help="Catalog root: every SKILL.md under it is one skill. Directories whose " + "SKILL.md files are identical count once: the shallowest path is kept, the " + "other copies are left out (not an error, no row)") p.add_argument("--out-dir", required=True, dest="out_dir", help="Where each skill's novelty.json and report go (OUT_DIR//)") p.add_argument("--jsonl", default=None, - help=f"One row per skill, written as each finishes (default: OUT_DIR/{DEFAULT_JSONL})") + help=f"One row per skill, written as each finishes (default: OUT_DIR/{DEFAULT_JSONL}). " + "Ctrl-C or SIGTERM cancels the skills not yet started (no new call), lets the " + "running ones finish, keeps the rows done and exits nonzero (130, 143)") p.add_argument("--jobs", type=positive_int, default=1, help="Skills analysed in parallel threads, sharing --max-usd (default 1)") p.add_argument("--resume", action="store_true", diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index af904a2d4..384850f80 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -6,6 +6,8 @@ from __future__ import annotations import json +import os +import signal import pytest @@ -217,3 +219,71 @@ def test_resume_runs_again_the_rows_measured_under_other_input_limits(tmp_path, capsys.readouterr() _sweep(root, out, "--max-usd", "5", "--resume", *flag) assert "0 measured and kept, 2 to run" in capsys.readouterr().err + + +@pytest.mark.parametrize("signum, code", [(signal.SIGINT, 130), (signal.SIGTERM, 143)]) +def test_an_interrupt_cancels_the_queued_skills_keeps_the_rows_done_and_exits_nonzero( + tmp_path, monkeypatch, signum, code): + """Ctrl-C or SIGTERM mid-sweep: the skill in flight finishes and keeps its row, the queued skills + never start (no paid call), and the process exits nonzero. The JSONL stays resumable.""" + import time + from types import SimpleNamespace + + import dokimi.artifacts as artifacts + import dokimi.commands.catalog_novelty as catalog_novelty + import dokimi.novelty.report as report + + artifact = {"k0": {"lenient": 0.2, "strict": 0.1}, "paired": {"lenient": {"n_paired": 3}}, + "artifact": {"tokens": 100}} + started = [] + + def analyze(args, *a, **k): + name = args.artifact.rsplit("/", 1)[-1] + started.append(name) + if len(started) == 1: # the signal arrives while the first skill is in flight; it finishes + os.kill(os.getpid(), signum) + time.sleep(0.5) # the main thread cancels the queued skills well inside this + return SimpleNamespace(partial=None, artifact=artifact) + + monkeypatch.setattr(catalog_novelty, "_analyze_skill", analyze) + monkeypatch.setattr(artifacts, "save_artifact", lambda kind, art, path: path.parent.mkdir(parents=True)) + monkeypatch.setattr(report, "render_novelty_markdown", lambda artifact: "") + root = _root(tmp_path, names=("accel-tuning", "beta", "gamma", "other-skill")) + out = tmp_path / "out" + # a SIGTERM the sweep does not handle would end pytest itself: the stray handler keeps that a failure + stray: list[int] = [] + previous = signal.signal(signal.SIGTERM, lambda s, f: stray.append(s)) + try: + try: + _sweep(root, out, "--max-usd", "5", "--jobs", "1") + except SystemExit as exc: + code_seen = exc.code + except KeyboardInterrupt: # not turned into an exit: fail here, not as an aborted pytest run + pytest.fail("the interrupt escaped the sweep as KeyboardInterrupt, not an exit") + else: + pytest.fail("the sweep returned after an interrupt") + finally: + signal.signal(signal.SIGTERM, previous) + assert code_seen == code + assert started == ["accel-tuning"] # the queued skills were cancelled before they started + assert not stray + rows = _rows(out / "sweep.jsonl") + assert [(r["key"], r["status"]) for r in rows] == [("accel-tuning", "ok")] + + +def test_identical_skill_copies_are_one_skill_and_the_help_says_so(tmp_path, capsys): + """Two directories with the same SKILL.md are one skill: the shallowest path keeps the row.""" + root = _root(tmp_path, names=("accel-tuning",)) + copy = root / "mirror" / "accel-tuning" + copy.mkdir(parents=True) + (copy / "SKILL.md").write_text((root / "accel-tuning" / "SKILL.md").read_text(encoding="utf-8"), + encoding="utf-8") + out = tmp_path / "out" + _sweep(root, out, "--max-usd", "5") + assert [r["key"] for r in _rows(out / "sweep.jsonl")] == ["accel-tuning"] + capsys.readouterr() + with pytest.raises(SystemExit): + build_parser().parse_args(["catalog", "novelty", "--help"]) + help_text = " ".join(capsys.readouterr().out.split()) + assert "identical" in help_text + assert "Ctrl-C or SIGTERM" in help_text From ee2190c36d4ebe567da0ad029aea4189628ee8d7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 21:04:57 -0800 Subject: [PATCH 04/11] fix(catalog): a second signal kills the calls in flight; the first lets them finish The first Ctrl-C or SIGTERM of a catalog novelty sweep cancels the queued skills and lets the running ones finish, as the docs say. The SIGTERM branch also called process.kill_children, which killed those running calls (error rows) and latched the process as closing. Now only a second signal kills the calls in flight, then stops at once. Signed-off-by: Nikolay Petrov --- ...6_120000_napetrov_catalog-novelty-sweep.md | 5 ++- dokimi/docs/skill-novelty.md | 7 ++-- dokimi/dokimi/commands/catalog_novelty.py | 19 +++++---- dokimi/tests/test_catalog_novelty.py | 42 +++++++++++++++++++ 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md index 6deced0da..d3505033d 100644 --- a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md +++ b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md @@ -11,5 +11,6 @@ SPDX-License-Identifier: Apache-2.0 finishes. `--resume` keeps the measured rows whose skill digest and settings are unchanged, and an existing `--jsonl` is not overwritten without it. `--strata CSV` adds bootstrap 95% intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. See - Ctrl-C or SIGTERM cancels the skills not yet started and keeps the rows done; identical - SKILL.md copies count once. See `docs/skill-novelty.md`, Corpus sweep. + Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish and keeps the + rows done; a second signal kills the calls in flight. Identical SKILL.md copies count once. + See `docs/skill-novelty.md`, Corpus sweep. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 3f0ad687c..a566057ce 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1141,9 +1141,10 @@ each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets skill: the shallowest path is kept, the other copies get no row and no call (not an error). - **Interrupt.** Ctrl-C or SIGTERM stops the sweep without new calls: the skills not yet started are cancelled, the skills in flight finish and keep their rows, and the run exits nonzero (130 - for SIGINT, 143 for SIGTERM). The rows already written stay in the JSONL, so `--resume` continues - from them. A second signal raises KeyboardInterrupt: the queued skills stay cancelled, but the - rows of the skills in flight are not written (their calls are paid, so set `DOKIMI_RESPONSE_CACHE`). + for SIGINT, 143 for SIGTERM). A SIGTERM therefore waits for the calls in flight. The rows already + written stay in the JSONL, so `--resume` continues from them. A second signal kills the calls in + flight (their paid work is lost, so set `DOKIMI_RESPONSE_CACHE` to replay what was done) and stops + at once. ```bash dokimi catalog novelty skills/ --out-dir out/sweep --max-usd 20 --max-usd-per-skill 1 \ diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index a62585968..aa942947f 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -11,7 +11,7 @@ ``--summary-only`` prints the summary of a JSONL file without any model call. Directories whose ``SKILL.md`` files are identical count as one skill (the shallowest path is kept). Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish, keeps the rows done and exits nonzero -(a second signal raises KeyboardInterrupt and the rows in flight are not written). +(a second signal kills the calls in flight and stops at once). A skill the budget stops mid-analysis is a ``partial`` row and leaves no novelty file. Set ``DOKIMI_RESPONSE_CACHE`` so a resumed run replays the calls already paid for at $0. @@ -64,20 +64,20 @@ def exit_code(self) -> int: @contextmanager def _interruptible(stop: _Stop): """While the sweep runs, the first SIGINT or SIGTERM only sets *stop*: the sweep cancels the skills - not started and keeps the rows done. A second signal raises KeyboardInterrupt. The handlers are - restored after. Only the main thread can install them.""" + not started, and the skills in flight finish. A second signal kills the calls in flight + (``process.kill_children``) and raises KeyboardInterrupt. The handlers are restored after. + Only the main thread can install them.""" if threading.current_thread() is not threading.main_thread(): yield return def on_signal(signum: int, frame: object) -> None: - if stop.requested: - raise KeyboardInterrupt - stop.signum = signum - if signum == signal.SIGTERM: + if stop.requested: # a second signal: stop at once, the calls in flight too from dokimi.process import kill_children - kill_children() # what the CLI's SIGTERM handler does for the processes it started + kill_children() + raise KeyboardInterrupt + stop.signum = signum previous = {signum: signal.signal(signum, on_signal) for signum in (signal.SIGINT, signal.SIGTERM)} try: @@ -462,7 +462,8 @@ def register(catalog_sub, positive_int) -> None: p.add_argument("--jsonl", default=None, help=f"One row per skill, written as each finishes (default: OUT_DIR/{DEFAULT_JSONL}). " "Ctrl-C or SIGTERM cancels the skills not yet started (no new call), lets the " - "running ones finish, keeps the rows done and exits nonzero (130, 143)") + "running ones finish, keeps the rows done and exits nonzero (130, 143); " + "a second signal kills the calls in flight too") p.add_argument("--jobs", type=positive_int, default=1, help="Skills analysed in parallel threads, sharing --max-usd (default 1)") p.add_argument("--resume", action="store_true", diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index 384850f80..340b9501b 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -232,7 +232,10 @@ def test_an_interrupt_cancels_the_queued_skills_keeps_the_rows_done_and_exits_no import dokimi.artifacts as artifacts import dokimi.commands.catalog_novelty as catalog_novelty import dokimi.novelty.report as report + import dokimi.process as process + kills: list[bool] = [] + monkeypatch.setattr(process, "kill_children", lambda: kills.append(True)) artifact = {"k0": {"lenient": 0.2, "strict": 0.1}, "paired": {"lenient": {"n_paired": 3}}, "artifact": {"tokens": 100}} started = [] @@ -266,11 +269,50 @@ def analyze(args, *a, **k): signal.signal(signal.SIGTERM, previous) assert code_seen == code assert started == ["accel-tuning"] # the queued skills were cancelled before they started + assert not kills # the first signal kills nothing: the call in flight finishes assert not stray rows = _rows(out / "sweep.jsonl") assert [(r["key"], r["status"]) for r in rows] == [("accel-tuning", "ok")] +def test_a_second_signal_kills_the_calls_in_flight_and_stops_at_once(tmp_path, monkeypatch): + """The first Ctrl-C lets the call in flight finish; a second one kills the calls in flight + (process.kill_children) and stops the sweep at once, with the same exit code as the first.""" + import time + from types import SimpleNamespace + + import dokimi.artifacts as artifacts + import dokimi.commands.catalog_novelty as catalog_novelty + import dokimi.novelty.report as report + import dokimi.process as process + + kills: list[bool] = [] + monkeypatch.setattr(process, "kill_children", lambda: kills.append(True)) + artifact = {"k0": {"lenient": 0.2, "strict": 0.1}, "paired": {"lenient": {"n_paired": 3}}, + "artifact": {"tokens": 100}} + started = [] + + def analyze(args, *a, **k): + started.append(args.artifact.rsplit("/", 1)[-1]) + if len(started) == 1: # two Ctrl-Cs while the first skill is in flight + os.kill(os.getpid(), signal.SIGINT) + time.sleep(0.1) + os.kill(os.getpid(), signal.SIGINT) + time.sleep(0.5) + return SimpleNamespace(partial=None, artifact=artifact) + + monkeypatch.setattr(catalog_novelty, "_analyze_skill", analyze) + monkeypatch.setattr(artifacts, "save_artifact", lambda kind, art, path: path.parent.mkdir(parents=True)) + monkeypatch.setattr(report, "render_novelty_markdown", lambda artifact: "") + root = _root(tmp_path, names=("accel-tuning", "beta", "gamma", "other-skill")) + out = tmp_path / "out" + with pytest.raises(SystemExit) as exc: + _sweep(root, out, "--max-usd", "5", "--jobs", "1") + assert exc.value.code == 130 + assert kills == [True] + assert started == ["accel-tuning"] + + def test_identical_skill_copies_are_one_skill_and_the_help_says_so(tmp_path, capsys): """Two directories with the same SKILL.md are one skill: the shallowest path keeps the row.""" root = _root(tmp_path, names=("accel-tuning",)) From a75a50f29d88669c506f3aa88053ff16a87db192 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 21:21:54 -0800 Subject: [PATCH 05/11] fix(catalog): a row is written once; the rows done survive a second signal in the drain A row whose write raised after the append was written again by the drain, since the future was discarded only after the write. The drain now discards before the write and writes the rows already done before it waits for the skills in flight, so an interrupt during the wait keeps them. The changelog sentence is fixed. Red tests first. Signed-off-by: Nikolay Petrov --- ...6_120000_napetrov_catalog-novelty-sweep.md | 2 +- dokimi/dokimi/commands/catalog_novelty.py | 22 +++++++--- dokimi/tests/test_catalog_novelty.py | 44 +++++++++++++++++++ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md index d3505033d..576538d10 100644 --- a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md +++ b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md @@ -10,7 +10,7 @@ SPDX-License-Identifier: Apache-2.0 (required), `--max-usd-per-skill`, `--jobs`, and JSONL output with one row per skill as it finishes. `--resume` keeps the measured rows whose skill digest and settings are unchanged, and an existing `--jsonl` is not overwritten without it. `--strata CSV` adds bootstrap 95% - intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. See + intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish and keeps the rows done; a second signal kills the calls in flight. Identical SKILL.md copies count once. See `docs/skill-novelty.md`, Corpus sweep. diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index aa942947f..8e5c83a04 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -242,10 +242,10 @@ def fill() -> None: while running and not stop_requested(): finished, _ = wait(running, timeout=POLL_SECONDS, return_when=FIRST_COMPLETED) for future in finished: + running.discard(future) # before the write: a write that raises is not written again row = future.result() if row is not None: write(row) - running.discard(future) fill() except BaseException: _settle(pool, running, write) @@ -254,14 +254,26 @@ def fill() -> None: def _settle(pool: ThreadPoolExecutor, running: set, write: Callable[[dict], None]) -> None: - """Cancel the queued skills, wait for the ones in flight and keep the rows they finished.""" - pool.shutdown(wait=True, cancel_futures=True) + """Cancel the queued skills, keep the rows already done, then wait for the ones in flight and keep + theirs. The done rows are written before the wait, so an interrupt during the wait keeps them.""" + pool.shutdown(wait=False, cancel_futures=True) + pending = set() for future in running: - if not future.cancelled() and future.exception() is None and future.result() is not None: - write(future.result()) + if future.done(): + _keep(future, write) + else: + pending.add(future) + wait(pending) + for future in pending: + _keep(future, write) running.clear() +def _keep(future, write: Callable[[dict], None]) -> None: + if not future.cancelled() and future.exception() is None and future.result() is not None: + write(future.result()) + + def _review_one(item: _Item, *, args: argparse.Namespace, run_cap: "SpendCap", per_skill: Optional[float], placebo_text: Optional[str], cache_dir: Optional[Path], out_dir: Path, fingerprint: dict) -> Optional[dict]: diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index 340b9501b..24a9bca4e 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -7,6 +7,7 @@ import json import os +from pathlib import Path import signal import pytest @@ -313,6 +314,49 @@ def analyze(args, *a, **k): assert started == ["accel-tuning"] +def test_a_row_is_written_once_even_when_its_write_is_interrupted(): + """A second Ctrl-C landing in write(): the row the sink already took is not written again by the drain.""" + from dokimi.commands.catalog_novelty import _Item, _run_sweep + + written: list[str] = [] + + def write(row): + written.append(row["key"]) + raise KeyboardInterrupt # after the append, as a second signal lands in the sink + + items = [_Item(key="a", name="a", skill_md=Path("a/SKILL.md"), sha256=None), + _Item(key="b", name="b", skill_md=Path("b/SKILL.md"), sha256=None)] + with pytest.raises(KeyboardInterrupt): + _run_sweep(items, jobs=1, review=lambda item: {"key": item.key}, write=write, + out_of_budget=lambda: False, stop_requested=lambda: False) + assert written == ["a", "b"] # each row once; b finished and is kept too + + +def test_settle_keeps_the_rows_done_before_it_waits_for_the_skills_in_flight(monkeypatch): + """An interrupt while the drain waits for the skills in flight keeps the rows already done.""" + import threading + from concurrent.futures import ThreadPoolExecutor + + import dokimi.commands.catalog_novelty as catalog_novelty + + release = threading.Event() + pool = ThreadPoolExecutor(max_workers=2) + done = pool.submit(lambda: {"key": "a"}) + done.result() + slow = pool.submit(lambda: (release.wait(5), {"key": "b"})[1]) + running = {done, slow} + written: list[str] = [] + + def interrupted_wait(*args, **kwargs): + release.set() + raise KeyboardInterrupt + + monkeypatch.setattr(catalog_novelty, "wait", interrupted_wait) + with pytest.raises(KeyboardInterrupt): + catalog_novelty._settle(pool, running, lambda row: written.append(row["key"])) + assert written == ["a"] + + def test_identical_skill_copies_are_one_skill_and_the_help_says_so(tmp_path, capsys): """Two directories with the same SKILL.md are one skill: the shallowest path keeps the row.""" root = _root(tmp_path, names=("accel-tuning",)) From 551c1fee471526d9aa7798627759a851d358d3d7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 21:38:35 -0800 Subject: [PATCH 06/11] fix(catalog): a queued skill makes no call after a stop; drain rows as they finish A skill queued behind one in flight could start when a worker freed up before the main thread saw the stop. The queued call now checks the stop first. The drain writes each row as its skill finishes, so an interrupt during the wait keeps the ones done. The docs say a second signal kills only subprocess calls (an API call is waited for), and that identical copies must also name the same skill. Red tests first. Signed-off-by: Nikolay Petrov --- ...6_120000_napetrov_catalog-novelty-sweep.md | 3 +- dokimi/docs/skill-novelty.md | 8 +-- dokimi/dokimi/commands/catalog_novelty.py | 23 ++++--- dokimi/tests/test_catalog_novelty.py | 66 +++++++++++++++++++ 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md index 576538d10..b0434c4e8 100644 --- a/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md +++ b/dokimi/changelog.d/20261116_120000_napetrov_catalog-novelty-sweep.md @@ -12,5 +12,6 @@ SPDX-License-Identifier: Apache-2.0 an existing `--jsonl` is not overwritten without it. `--strata CSV` adds bootstrap 95% intervals per stratum; `--summary-only` prints the summary of a JSONL with no model call. Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish and keeps the - rows done; a second signal kills the calls in flight. Identical SKILL.md copies count once. + rows done; a second signal kills the subprocess calls in flight. Identical SKILL.md copies of one + skill count once. See `docs/skill-novelty.md`, Corpus sweep. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index a566057ce..0fef58f8d 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1137,14 +1137,14 @@ each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets default `stratum`); a skill the CSV does not name is `(unassigned)`. - **Summary only.** `--summary-only --jsonl FILE` prints the summary of a finished or partial sweep with no model call. `--summary-out FILE` also writes it as JSON. -- **Identical copies.** Directories under ROOT whose `SKILL.md` files are byte-identical are one - skill: the shallowest path is kept, the other copies get no row and no call (not an error). +- **Identical copies.** Directories under ROOT whose `SKILL.md` files are byte-identical and name the + same skill (the frontmatter `name`, or the directory name when it has none) are one skill: the shallowest path is kept, the other copies get no row and no call (not an error). - **Interrupt.** Ctrl-C or SIGTERM stops the sweep without new calls: the skills not yet started are cancelled, the skills in flight finish and keep their rows, and the run exits nonzero (130 for SIGINT, 143 for SIGTERM). A SIGTERM therefore waits for the calls in flight. The rows already - written stay in the JSONL, so `--resume` continues from them. A second signal kills the calls in + written stay in the JSONL, so `--resume` continues from them. A second signal kills the subprocess calls in flight (their paid work is lost, so set `DOKIMI_RESPONSE_CACHE` to replay what was done) and stops - at once. + at once. An API call cannot be killed: the process waits for it before it exits. ```bash dokimi catalog novelty skills/ --out-dir out/sweep --max-usd 20 --max-usd-per-skill 1 \ diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index 8e5c83a04..204fc2806 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -9,9 +9,9 @@ writes one row to ``--jsonl`` as it finishes, so a killed sweep keeps its rows, and ``--resume`` runs only the skills without a measured row (same directory digest, same settings). ``--summary-only`` prints the summary of a JSONL file without any model call. Directories whose -``SKILL.md`` files are identical count as one skill (the shallowest path is kept). Ctrl-C or SIGTERM +``SKILL.md`` files are identical and name the same skill count as one (the shallowest path is kept). Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish, keeps the rows done and exits nonzero -(a second signal kills the calls in flight and stops at once). +(a second signal kills the subprocess calls in flight and stops at once; an API call cannot be killed, so the process waits for it). A skill the budget stops mid-analysis is a ``partial`` row and leaves no novelty file. Set ``DOKIMI_RESPONSE_CACHE`` so a resumed run replays the calls already paid for at $0. @@ -64,7 +64,7 @@ def exit_code(self) -> int: @contextmanager def _interruptible(stop: _Stop): """While the sweep runs, the first SIGINT or SIGTERM only sets *stop*: the sweep cancels the skills - not started, and the skills in flight finish. A second signal kills the calls in flight + not started, and the skills in flight finish. A second signal kills the subprocess calls in flight (``process.kill_children``) and raises KeyboardInterrupt. The handlers are restored after. Only the main thread can install them.""" if threading.current_thread() is not threading.main_thread(): @@ -230,12 +230,16 @@ def _run_sweep(items: list[_Item], *, jobs: int, review: Callable[[_Item], Optio running: set = set() pool = ThreadPoolExecutor(max_workers=jobs) + def guarded(item: _Item) -> Optional[dict]: + # a stop that landed while this skill was queued: it makes no call, even if a worker freed up first + return None if stop_requested() else review(item) + def fill() -> None: while len(running) < jobs * 2 and not out_of_budget() and not stop_requested(): item = next(pending, None) if item is None: return - running.add(pool.submit(review, item)) + running.add(pool.submit(guarded, item)) try: fill() @@ -263,9 +267,10 @@ def _settle(pool: ThreadPoolExecutor, running: set, write: Callable[[dict], None _keep(future, write) else: pending.add(future) - wait(pending) - for future in pending: - _keep(future, write) + while pending: # each row is written as its skill finishes, so an interrupt keeps the ones done + finished, pending = wait(pending, return_when=FIRST_COMPLETED) + for future in finished: + _keep(future, write) running.clear() @@ -467,7 +472,7 @@ def register(catalog_sub, positive_int) -> None: "with bootstrap CIs per stratum [paid]", ) p.add_argument("root", help="Catalog root: every SKILL.md under it is one skill. Directories whose " - "SKILL.md files are identical count once: the shallowest path is kept, the " + "SKILL.md files are identical and name the same skill count once: the shallowest path is kept, the " "other copies are left out (not an error, no row)") p.add_argument("--out-dir", required=True, dest="out_dir", help="Where each skill's novelty.json and report go (OUT_DIR//)") @@ -475,7 +480,7 @@ def register(catalog_sub, positive_int) -> None: help=f"One row per skill, written as each finishes (default: OUT_DIR/{DEFAULT_JSONL}). " "Ctrl-C or SIGTERM cancels the skills not yet started (no new call), lets the " "running ones finish, keeps the rows done and exits nonzero (130, 143); " - "a second signal kills the calls in flight too") + "a second signal kills the subprocess calls in flight too") p.add_argument("--jobs", type=positive_int, default=1, help="Skills analysed in parallel threads, sharing --max-usd (default 1)") p.add_argument("--resume", action="store_true", diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index 24a9bca4e..05b33c1f5 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -357,6 +357,72 @@ def interrupted_wait(*args, **kwargs): assert written == ["a"] +def test_a_skill_queued_when_the_stop_lands_makes_no_call(monkeypatch): + """The stop lands while a skill is in flight and that skill finishes at once: the skill the freed + worker takes next must not start, since the stop came first (no new paid call).""" + import time + + import dokimi.commands.catalog_novelty as catalog_novelty + from dokimi.commands.catalog_novelty import _Item, _run_sweep + + real_wait = catalog_novelty.wait + + def late_wait(futures, **kwargs): # the main thread wakes late: the freed worker takes the next skill first + done = real_wait(futures, **kwargs) + time.sleep(0.3) + return done + + monkeypatch.setattr(catalog_novelty, "wait", late_wait) + stop = [False] + started: list[str] = [] + + def review(item): + started.append(item.key) + if item.key == "a": # b is queued behind a by now; the stop lands while a is in flight + time.sleep(0.05) + stop[0] = True + return {"key": item.key} + + items = [_Item(key=k, name=k, skill_md=Path(f"{k}/SKILL.md"), sha256=None) for k in "abc"] + _run_sweep(items, jobs=1, review=review, write=lambda row: None, out_of_budget=lambda: False, + stop_requested=lambda: stop[0]) + assert started == ["a"] + + +def test_a_row_finished_during_the_drain_is_written_before_the_next_wait(monkeypatch): + """A skill that finishes while the drain waits for another is written then, not after all of them: + an interrupt during the wait for the rest keeps it.""" + import threading + from concurrent.futures import ThreadPoolExecutor + + import dokimi.commands.catalog_novelty as catalog_novelty + + import time + + release = threading.Event() + pool = ThreadPoolExecutor(max_workers=2) + fast = pool.submit(lambda: (time.sleep(0.1), {"key": "fast"})[1]) # still in flight when the drain starts + slow = pool.submit(lambda: (release.wait(5), {"key": "slow"})[1]) + real_wait = catalog_novelty.wait + waits: list[int] = [] + + def wait_then_interrupt(futures, **kwargs): + waits.append(1) + if len(waits) > 1: + raise KeyboardInterrupt # a second signal while the drain waits for the slow skill + return real_wait(futures, **kwargs) + + monkeypatch.setattr(catalog_novelty, "wait", wait_then_interrupt) + written: list[str] = [] + try: + with pytest.raises(KeyboardInterrupt): + catalog_novelty._settle(pool, {fast, slow}, lambda row: written.append(row["key"])) + assert written == ["fast"] + finally: + release.set() + pool.shutdown(wait=True) + + def test_identical_skill_copies_are_one_skill_and_the_help_says_so(tmp_path, capsys): """Two directories with the same SKILL.md are one skill: the shallowest path keeps the row.""" root = _root(tmp_path, names=("accel-tuning",)) From 393d293db3721106c0d427b42ec77b9aebd66dde Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 21:53:19 -0800 Subject: [PATCH 07/11] test(catalog): the drain tests wait for their skills to run; a second signal's API row is documented The three drain tests depended on thread timing: a queued skill could be cancelled before it ran, and the test asserted it was kept. They now wait until the skills are running, and the write-once test asserts only that the finished row is written once. The docs and help say a second signal does not kill an API call in flight, and that its row is then not written. Signed-off-by: Nikolay Petrov --- dokimi/docs/skill-novelty.md | 3 ++- dokimi/dokimi/commands/catalog_novelty.py | 2 +- dokimi/tests/test_catalog_novelty.py | 29 +++++++++++++++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 0fef58f8d..58b329ef4 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1144,7 +1144,8 @@ each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets for SIGINT, 143 for SIGTERM). A SIGTERM therefore waits for the calls in flight. The rows already written stay in the JSONL, so `--resume` continues from them. A second signal kills the subprocess calls in flight (their paid work is lost, so set `DOKIMI_RESPONSE_CACHE` to replay what was done) and stops - at once. An API call cannot be killed: the process waits for it before it exits. + at once. An API call in flight cannot be killed: its row is not written, and the process waits for it to end + before it exits, so its paid call is lost unless `DOKIMI_RESPONSE_CACHE` is set. ```bash dokimi catalog novelty skills/ --out-dir out/sweep --max-usd 20 --max-usd-per-skill 1 \ diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index 204fc2806..032948566 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -11,7 +11,7 @@ ``--summary-only`` prints the summary of a JSONL file without any model call. Directories whose ``SKILL.md`` files are identical and name the same skill count as one (the shallowest path is kept). Ctrl-C or SIGTERM cancels the skills not yet started, lets the running ones finish, keeps the rows done and exits nonzero -(a second signal kills the subprocess calls in flight and stops at once; an API call cannot be killed, so the process waits for it). +(a second signal kills the subprocess calls in flight and stops at once; an API call in flight cannot be killed, so its row is not written). A skill the budget stops mid-analysis is a ``partial`` row and leaves no novelty file. Set ``DOKIMI_RESPONSE_CACHE`` so a resumed run replays the calls already paid for at $0. diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index 05b33c1f5..d59db29e1 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -329,7 +329,7 @@ def write(row): with pytest.raises(KeyboardInterrupt): _run_sweep(items, jobs=1, review=lambda item: {"key": item.key}, write=write, out_of_budget=lambda: False, stop_requested=lambda: False) - assert written == ["a", "b"] # each row once; b finished and is kept too + assert written.count("a") == 1 # written once; b may or may not have started before the cancel def test_settle_keeps_the_rows_done_before_it_waits_for_the_skills_in_flight(monkeypatch): @@ -340,10 +340,18 @@ def test_settle_keeps_the_rows_done_before_it_waits_for_the_skills_in_flight(mon import dokimi.commands.catalog_novelty as catalog_novelty release = threading.Event() + slow_started = threading.Event() pool = ThreadPoolExecutor(max_workers=2) done = pool.submit(lambda: {"key": "a"}) done.result() - slow = pool.submit(lambda: (release.wait(5), {"key": "b"})[1]) + + def slow_skill(): + slow_started.set() # running, not queued: the cancel of the queue leaves it alone + release.wait(5) + return {"key": "b"} + + slow = pool.submit(slow_skill) + assert slow_started.wait(5) running = {done, slow} written: list[str] = [] @@ -400,9 +408,22 @@ def test_a_row_finished_during_the_drain_is_written_before_the_next_wait(monkeyp import time release = threading.Event() + started = threading.Barrier(3) # the two skills and this test: both are running before the drain pool = ThreadPoolExecutor(max_workers=2) - fast = pool.submit(lambda: (time.sleep(0.1), {"key": "fast"})[1]) # still in flight when the drain starts - slow = pool.submit(lambda: (release.wait(5), {"key": "slow"})[1]) + + def fast_skill(): + started.wait(5) + time.sleep(0.1) # still in flight when the drain starts + return {"key": "fast"} + + def slow_skill(): + started.wait(5) + release.wait(5) + return {"key": "slow"} + + fast = pool.submit(fast_skill) + slow = pool.submit(slow_skill) + started.wait(5) real_wait = catalog_novelty.wait waits: list[int] = [] From f236c7cb19ad57ba9c23870d04321cf2d69d609f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 14:36:06 -0800 Subject: [PATCH 08/11] feat(catalog): rank the novelty sweep by new share per 1k tokens The catalog novelty summary lists the measured skills by how much of each one the model did not already know, per 1k tokens of the skill: the top ten in text, all of them in the --summary-out JSON as `ranking`. The already-known share stays the headline. No model call and no CLI change. Signed-off-by: Nikolay Petrov --- ...16_130000_napetrov_catalog-novelty-rank.md | 10 +++++++ dokimi/docs/skill-novelty.md | 8 ++++-- dokimi/dokimi/novelty/sweep.py | 28 +++++++++++++++++++ dokimi/tests/test_novelty_sweep.py | 20 +++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md diff --git a/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md b/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md new file mode 100644 index 000000000..3bdea3e66 --- /dev/null +++ b/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md @@ -0,0 +1,10 @@ + + +### Added + +- **`catalog novelty` summary: a ranking by new share per 1k tokens.** The sweep summary lists the + measured skills by how much of each one the model did not already know, per 1k tokens of the + skill (top ten in text; every skill in the `--summary-out` JSON as `ranking`). diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 58b329ef4..790fa6d4b 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1136,9 +1136,13 @@ each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets column (the skill's directory under ROOT, or its name) and a stratum column (`--stratum-column`, default `stratum`); a skill the CSV does not name is `(unassigned)`. - **Summary only.** `--summary-only --jsonl FILE` prints the summary of a finished or partial - sweep with no model call. `--summary-out FILE` also writes it as JSON. + sweep with no model call. `--summary-out FILE` also writes it as JSON. The summary leads with + the already-known share, then lists the skills by **new share per 1k tokens** (the new share + divided by the skill's size in thousands of tokens; the top ten in text, all in the JSON + `ranking`). It sums every row in the JSONL, so a file shared by two sweeps reports both. - **Identical copies.** Directories under ROOT whose `SKILL.md` files are byte-identical and name the - same skill (the frontmatter `name`, or the directory name when it has none) are one skill: the shallowest path is kept, the other copies get no row and no call (not an error). + same skill (the frontmatter `name`, or the directory name when it has none) are one skill: the + shallowest path is kept, the other copies get no row and no call (not an error). - **Interrupt.** Ctrl-C or SIGTERM stops the sweep without new calls: the skills not yet started are cancelled, the skills in flight finish and keep their rows, and the run exits nonzero (130 for SIGINT, 143 for SIGTERM). A SIGTERM therefore waits for the calls in flight. The rows already diff --git a/dokimi/dokimi/novelty/sweep.py b/dokimi/dokimi/novelty/sweep.py index 29f61f219..64b2efc45 100644 --- a/dokimi/dokimi/novelty/sweep.py +++ b/dokimi/dokimi/novelty/sweep.py @@ -164,12 +164,33 @@ def group_stats(rows: list[dict], *, resamples: int = RESAMPLES) -> dict[str, An } +#: How many skills the text summary lists in the ranking; the JSON summary has them all. +RANK_LINES = 10 + + +def ranking(rows: list[dict]) -> list[dict]: + """The measured skills with a token count, most new content per 1k tokens first. + + ``new_per_1k_tokens`` is the skill's new share divided by its size in thousands of tokens: + what a model learns from reading the skill, per token it must read. Ties keep the key order. + """ + scored = [ + {"key": r["key"], "name": r["name"], "new_lenient": r["new_lenient"], "tokens": r["tokens"], + "new_per_1k_tokens": round(r["new_lenient"] * 1000 / r["tokens"], 6)} + for r in rows + if r["status"] == "ok" and r["new_lenient"] is not None and r.get("tokens") + ] + scored.sort(key=lambda s: (-s["new_per_1k_tokens"], s["key"])) + return scored + + def summarize(rows: list[dict], strata: Optional[Mapping[str, str]] = None, *, resamples: int = RESAMPLES) -> dict[str, Any]: """Counts by status, the spend, and the measure with bootstrap CIs overall and per stratum. Only ``ok`` rows with a measure are summarised; ``partial`` and ``error`` rows are counted and their spend is in ``usd``. With *strata* the per-stratum table is given, else only ``overall``. + ``ranking`` lists every measured skill with a token count by new share per 1k tokens. """ counts = {status: 0 for status in STATUSES} for row in rows: @@ -180,6 +201,7 @@ def summarize(rows: list[dict], strata: Optional[Mapping[str, str]] = None, *, "usd": round(sum(r["usd"] for r in rows), 6), "overall": group_stats(measured, resamples=resamples) if measured else None, "strata": None, + "ranking": ranking(measured), } if strata is not None: groups: dict[str, list[dict]] = {} @@ -214,4 +236,10 @@ def summary_lines(summary: dict[str, Any]) -> list[str]: lines.append(f"{name[:16]:<16} {block['n']:>4} {_interval(block['new_lenient']):<22} " f"{_interval(block['known_lenient']):<22} {_interval(block['new_strict']):<22} " f"{block['usd']:>7.2f}") + ranked = summary.get("ranking") or [] + if ranked: + lines.append(f"ranking by new share per 1k tokens (top {min(RANK_LINES, len(ranked))} of {len(ranked)}):") + for pos, item in enumerate(ranked[:RANK_LINES], 1): + lines.append(f" {pos:>2}. {item['key']}: {item['new_per_1k_tokens']:.4f} " + f"(new {item['new_lenient']:.2f}, {item['tokens']} tokens)") return lines diff --git a/dokimi/tests/test_novelty_sweep.py b/dokimi/tests/test_novelty_sweep.py index 7c1007901..2bdc270e1 100644 --- a/dokimi/tests/test_novelty_sweep.py +++ b/dokimi/tests/test_novelty_sweep.py @@ -211,3 +211,23 @@ def test_make_row_refuses_an_unknown_status(): def test_resumable_never_keeps_a_row_whose_directory_could_not_be_digested(): prior = [dict(_ok("gone-quiet", 0.3), skill_sha256=None)] assert resumable(prior, {"gone-quiet": None}, SETTINGS, "1.0") == [] + + +def test_ranking_orders_by_new_share_per_1k_tokens(): + rows = [_ok("a", 0.2, tokens=800), _ok("b", 0.6, tokens=3000), _ok("c", 0.5, tokens=500)] + ranked = summarize(rows, resamples=50)["ranking"] + assert [r["key"] for r in ranked] == ["c", "a", "b"] # 1.0, 0.25, 0.2 per 1k tokens + assert ranked[0]["new_per_1k_tokens"] == pytest.approx(1.0) + + +def test_ranking_leaves_out_rows_that_are_not_measured_or_have_no_size(): + rows = [_ok("a", 0.2, tokens=800), _ok("sizeless", 0.9, tokens=None), _row("b", "partial"), + _row("c", "error")] + assert [r["key"] for r in summarize(rows, resamples=50)["ranking"]] == ["a"] + + +def test_summary_lines_list_the_ranking_after_the_stratum_table(): + lines = summary_lines(summarize([_ok("a", 0.2, tokens=800), _ok("b", 0.6, tokens=3000)], resamples=50)) + head = next(i for i, line in enumerate(lines) if line.startswith("ranking by new share per 1k tokens")) + assert lines[head] == "ranking by new share per 1k tokens (top 2 of 2):" + assert lines[head + 1].lstrip().startswith("1. a:") From fa601894c38cd2dba78c3bfe3956972d7a3c3db9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 20:54:41 -0800 Subject: [PATCH 09/11] fix(catalog): the ranking lists a skill once, from its latest ok row A JSONL that two sweeps wrote lists a skill twice in the ranking. It now has one entry per skill key, taken from the latest ok row; the overall figures still sum every ok row. Signed-off-by: Nikolay Petrov --- ...16_130000_napetrov_catalog-novelty-rank.md | 3 ++- dokimi/docs/skill-novelty.md | 3 ++- dokimi/dokimi/novelty/sweep.py | 20 ++++++++++++------- dokimi/tests/test_novelty_sweep.py | 14 +++++++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md b/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md index 3bdea3e66..52895035b 100644 --- a/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md +++ b/dokimi/changelog.d/20261116_130000_napetrov_catalog-novelty-rank.md @@ -7,4 +7,5 @@ SPDX-License-Identifier: Apache-2.0 - **`catalog novelty` summary: a ranking by new share per 1k tokens.** The sweep summary lists the measured skills by how much of each one the model did not already know, per 1k tokens of the - skill (top ten in text; every skill in the `--summary-out` JSON as `ranking`). + skill (top ten in text; every skill in the `--summary-out` JSON as `ranking`, one entry per skill + from its latest ok row). diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 790fa6d4b..a77c8a355 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -1139,7 +1139,8 @@ each finishes, so a killed sweep keeps what it paid for. An `ok` skill also gets sweep with no model call. `--summary-out FILE` also writes it as JSON. The summary leads with the already-known share, then lists the skills by **new share per 1k tokens** (the new share divided by the skill's size in thousands of tokens; the top ten in text, all in the JSON - `ranking`). It sums every row in the JSONL, so a file shared by two sweeps reports both. + `ranking`). The ranking has one entry per skill, from its latest `ok` row, so a skill measured by + two sweeps into one JSONL is listed once; the overall figures still sum every `ok` row. - **Identical copies.** Directories under ROOT whose `SKILL.md` files are byte-identical and name the same skill (the frontmatter `name`, or the directory name when it has none) are one skill: the shallowest path is kept, the other copies get no row and no call (not an error). diff --git a/dokimi/dokimi/novelty/sweep.py b/dokimi/dokimi/novelty/sweep.py index 64b2efc45..6afd9fd39 100644 --- a/dokimi/dokimi/novelty/sweep.py +++ b/dokimi/dokimi/novelty/sweep.py @@ -169,16 +169,22 @@ def group_stats(rows: list[dict], *, resamples: int = RESAMPLES) -> dict[str, An def ranking(rows: list[dict]) -> list[dict]: - """The measured skills with a token count, most new content per 1k tokens first. + """The measured skills with a token count, most new content per 1k tokens first, one entry per key. - ``new_per_1k_tokens`` is the skill's new share divided by its size in thousands of tokens: - what a model learns from reading the skill, per token it must read. Ties keep the key order. + A skill measured by more than one sweep into the same JSONL is ranked from its latest ``ok`` row + (the last in file order); the overall figures are not affected. ``new_per_1k_tokens`` is the + skill's new share divided by its size in thousands of tokens: what a model learns from reading + the skill, per token it must read. Ties keep the key order. """ + latest: dict[str, dict] = {} + for r in rows: + if r["status"] == "ok": + latest[r["key"]] = r scored = [ {"key": r["key"], "name": r["name"], "new_lenient": r["new_lenient"], "tokens": r["tokens"], "new_per_1k_tokens": round(r["new_lenient"] * 1000 / r["tokens"], 6)} - for r in rows - if r["status"] == "ok" and r["new_lenient"] is not None and r.get("tokens") + for r in latest.values() + if r["new_lenient"] is not None and r.get("tokens") ] scored.sort(key=lambda s: (-s["new_per_1k_tokens"], s["key"])) return scored @@ -190,7 +196,7 @@ def summarize(rows: list[dict], strata: Optional[Mapping[str, str]] = None, *, Only ``ok`` rows with a measure are summarised; ``partial`` and ``error`` rows are counted and their spend is in ``usd``. With *strata* the per-stratum table is given, else only ``overall``. - ``ranking`` lists every measured skill with a token count by new share per 1k tokens. + ``ranking`` lists every measured skill with a token count by new share per 1k tokens, once per skill. """ counts = {status: 0 for status in STATUSES} for row in rows: @@ -201,7 +207,7 @@ def summarize(rows: list[dict], strata: Optional[Mapping[str, str]] = None, *, "usd": round(sum(r["usd"] for r in rows), 6), "overall": group_stats(measured, resamples=resamples) if measured else None, "strata": None, - "ranking": ranking(measured), + "ranking": ranking(rows), } if strata is not None: groups: dict[str, list[dict]] = {} diff --git a/dokimi/tests/test_novelty_sweep.py b/dokimi/tests/test_novelty_sweep.py index 2bdc270e1..264146ee8 100644 --- a/dokimi/tests/test_novelty_sweep.py +++ b/dokimi/tests/test_novelty_sweep.py @@ -231,3 +231,17 @@ def test_summary_lines_list_the_ranking_after_the_stratum_table(): head = next(i for i, line in enumerate(lines) if line.startswith("ranking by new share per 1k tokens")) assert lines[head] == "ranking by new share per 1k tokens (top 2 of 2):" assert lines[head + 1].lstrip().startswith("1. a:") + + +def test_ranking_lists_a_skill_once_from_its_latest_ok_row(): + """Two sweeps into one JSONL measure `a` twice: the ranking has one entry for it, the later one, + while the overall figures still sum every ok row.""" + rows = [_ok("a", 0.2, tokens=800), _ok("b", 0.6, tokens=3000), _ok("a", 0.5, tokens=800)] + summary = summarize(rows, resamples=50) + assert [(r["key"], r["new_lenient"]) for r in summary["ranking"]] == [("a", 0.5), ("b", 0.6)] + assert summary["overall"]["n"] == 3 + + +def test_ranking_keeps_an_earlier_ok_row_past_a_later_error(): + rows = [_ok("a", 0.5, tokens=800), _row("a", "error")] + assert [(r["key"], r["new_lenient"]) for r in summarize(rows, resamples=50)["ranking"]] == [("a", 0.5)] From 919aa6533b68b424312293e507ab49dec4ac6c33 Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Tue, 17 Nov 2026 23:03:05 +0000 Subject: [PATCH 10/11] fix(novelty): refuse a sweep row missing usd, tokens or new_lenient read_rows accepted an ok row with only key and status, and summarize then raised a bare KeyError. It now raises InputError naming the line. Adds a test for a tree_digest OSError leaving the item without a digest. Signed-off-by: vchamarthi --- dokimi/dokimi/novelty/sweep.py | 4 ++++ dokimi/tests/test_catalog_novelty.py | 12 ++++++++++++ dokimi/tests/test_novelty_sweep.py | 12 ++++++++++++ 3 files changed, 28 insertions(+) diff --git a/dokimi/dokimi/novelty/sweep.py b/dokimi/dokimi/novelty/sweep.py index 6afd9fd39..5ddb080f2 100644 --- a/dokimi/dokimi/novelty/sweep.py +++ b/dokimi/dokimi/novelty/sweep.py @@ -127,6 +127,10 @@ def read_rows(path: Path) -> list[dict]: raise InputError(f"--jsonl {path} line {lineno} is not a sweep row: {exc}") from exc if not isinstance(row, dict) or row.get("status") not in STATUSES or "key" not in row: raise InputError(f"--jsonl {path} line {lineno} is not a sweep row; fix or remove it before --resume") + needed = ("usd", "new_lenient", "tokens") if row["status"] == "ok" else ("usd",) + missing = [k for k in needed if k not in row] + if missing: + raise InputError(f"--jsonl {path} line {lineno} has no {', '.join(missing)}; fix or remove it before --resume") rows.append(row) return rows diff --git a/dokimi/tests/test_catalog_novelty.py b/dokimi/tests/test_catalog_novelty.py index d59db29e1..01236b902 100644 --- a/dokimi/tests/test_catalog_novelty.py +++ b/dokimi/tests/test_catalog_novelty.py @@ -460,3 +460,15 @@ def test_identical_skill_copies_are_one_skill_and_the_help_says_so(tmp_path, cap help_text = " ".join(capsys.readouterr().out.split()) assert "identical" in help_text assert "Ctrl-C or SIGTERM" in help_text + + +def test_a_tree_digest_failure_leaves_no_digest_so_resume_runs_the_skill_again(tmp_path, monkeypatch): + import dokimi.utils as utils + from dokimi.commands.catalog_novelty import _items + + def broken(*args, **kwargs): + raise OSError("unreadable") + + monkeypatch.setattr(utils, "tree_digest", broken) + items = _items(_root(tmp_path), None) + assert [i.sha256 for i in items] == [None, None] diff --git a/dokimi/tests/test_novelty_sweep.py b/dokimi/tests/test_novelty_sweep.py index 264146ee8..7509369f5 100644 --- a/dokimi/tests/test_novelty_sweep.py +++ b/dokimi/tests/test_novelty_sweep.py @@ -245,3 +245,15 @@ def test_ranking_lists_a_skill_once_from_its_latest_ok_row(): def test_ranking_keeps_an_earlier_ok_row_past_a_later_error(): rows = [_ok("a", 0.5, tokens=800), _row("a", "error")] assert [(r["key"], r["new_lenient"]) for r in summarize(rows, resamples=50)["ranking"]] == [("a", 0.5)] + + +@pytest.mark.parametrize("row, missing", [ + ({"key": "a", "status": "ok"}, "usd, new_lenient, tokens"), + ({"key": "a", "status": "ok", "usd": 0.1, "tokens": 5}, "new_lenient"), + ({"key": "a", "status": "error"}, "usd"), +]) +def test_read_rows_refuses_a_row_without_the_fields_the_summary_reads(tmp_path, row, missing): + path = tmp_path / "s.jsonl" + path.write_text(json.dumps(row) + "\n", encoding="utf-8") + with pytest.raises(InputError, match=f"line 1 has no {missing}"): + read_rows(path) From 5dbfee08bf3f1ba26266f21f061daf5ffe248c76 Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Tue, 17 Nov 2026 23:25:57 +0000 Subject: [PATCH 11/11] fix(catalog): use cli_types.positive_usd for --max-usd-per-skill main moved usd_amount out of novelty_run into cli_types as positive_usd, so importing it in catalog_novelty failed and broke the CLI. Imports the new name and refreshes the CLI interface snapshot. Signed-off-by: vchamarthi --- dokimi/docs/reference/cli-interface.json | 6 ++++-- dokimi/dokimi/commands/catalog_novelty.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 48328b3f0..a36b7f5a9 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -371,6 +371,8 @@ {"key": "--include-resources", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--jobs", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--jsonl", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-aggregate", "action": "_StoreAction", "choices": ["'majority'", "'median'", "'unanimous'"], "const": null, "default": "median", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -381,8 +383,8 @@ {"key": "--max-file-words", "action": "_StoreAction", "choices": null, "const": null, "default": 4000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--max-resource-words", "action": "_StoreAction", "choices": null, "const": null, "default": 20000, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--max-tokens", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, - {"key": "--max-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.novelty_run.usd_amount#af10a4ceaa08"}, - {"key": "--max-usd-per-skill", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.novelty_run.usd_amount#af10a4ceaa08"}, + {"key": "--max-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli_types.positive_usd#3f7e5c8902a3"}, + {"key": "--max-usd-per-skill", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli_types.positive_usd#3f7e5c8902a3"}, {"key": "--model", "action": "_StoreAction", "choices": null, "const": null, "default": "gpt-6.1-sol", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--no-paraphrase", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--no-reversal", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, diff --git a/dokimi/dokimi/commands/catalog_novelty.py b/dokimi/dokimi/commands/catalog_novelty.py index 032948566..9ea2e1cff 100644 --- a/dokimi/dokimi/commands/catalog_novelty.py +++ b/dokimi/dokimi/commands/catalog_novelty.py @@ -462,7 +462,7 @@ def review(item: _Item) -> Optional[dict]: def register(catalog_sub, positive_int) -> None: """Add ``catalog novelty``: the corpus sweep, a catalog verb like ``catalog routing``.""" from dokimi.commands.novelty import _add_bundle_args, _add_model_args - from dokimi.commands.novelty_run import usd_amount + from dokimi.cli_types import positive_usd from dokimi.novelty.sweep import RESAMPLES from dokimi.novelty.tiers import TIERS @@ -485,7 +485,7 @@ def register(catalog_sub, positive_int) -> None: help="Skills analysed in parallel threads, sharing --max-usd (default 1)") p.add_argument("--resume", action="store_true", help="Keep the measured rows in --jsonl and run only the other skills") - p.add_argument("--max-usd-per-skill", type=usd_amount, default=None, dest="max_usd_per_skill", + p.add_argument("--max-usd-per-skill", type=positive_usd, default=None, dest="max_usd_per_skill", help="Stop one skill once its calls cost this much (USD); the rest of the sweep goes on") p.add_argument("--limit", type=positive_int, default=None, help="Only the first N skills in path order (a pilot)")